From fc13f350433d70f6e51ef262ce204304362fa6b1 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Wed, 24 Dec 2025 16:22:05 +0400 Subject: [PATCH 01/39] Save: commit all local changes --- .alignment.sh.swp | Bin 0 -> 12288 bytes dockerfile => Dockerfile | 0 Stage1 | 1 + .../dist_checkpointing/strategies/base.py | 1 + .../core/extensions/transformer_engine.py | 2093 +- .../core/extensions/transformer_engine2.py | 723 + .../core/extensions/transformer_engine_org.py | 1452 + .../megatron/core/fusions/fused_layer_norm.py | 42 +- .../core/fusions/fused_layer_norm_org.py | 170 + .../megatron/core/model_parallel_config.py | 1 + .../megatron/core/optimizer/__init__.py | 33 +- .../core/optimizer/distrib_optimizer.py | 23 +- .../megatron/core/tensor_parallel/layers.py | 6 +- .../megatron/core/transformer/attention.py | 2 +- .../core/transformer/dot_product_attention.py | 1 + .../core/transformer/moe/fused_a2a.py | 12 +- .../megatron/training/checkpointing.py | 206 +- .../megatron/training/checkpointing_org.py | 1444 + aiak_megatron/megatron_core.egg-info/PKG-INFO | 332 + .../megatron_core.egg-info/SOURCES.txt | 283 + .../dependency_links.txt | 1 + .../megatron_core.egg-info/requires.txt | 21 + .../megatron_core.egg-info/top_level.txt | 1 + .../data/multimodal/task_encoder.py | 261 +- .../models/custom/common/local_norm.py | 170 +- .../models/custom/common/local_norm_org.py | 77 + .../llavaov_1_5/llavaov_1_5_layer_spec.py | 23 +- .../llavaov_1_5/llavaov_1_5_layer_spec_org.py | 177 + .../models/llavaov_1_5/rice_vision_model.py | 1 + apex | 1 + data/LLaVA-558K-Webdataset | 1 + .../stage_1_alignment_llava_ov_4b.sh | 47 +- libpod/tmp/pause.pid | 1 + ...20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log | 1044 + ...10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 53109 ++++++++++++++++ ...54_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 7690 +++ ...21_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 18531 ++++++ ...45_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 18530 ++++++ ...03_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 1038 + ...22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 1038 + ...56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 1038 + ...51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log | 1038 + ...14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log | 1038 + ...51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log | 1038 + ...30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log | 1058 + ...s.out.tfevents.1766400270.gpu-52.3342268.0 | Bin 0 -> 48602 bytes ...ts.out.tfevents.1766426196.gpu-04.111446.0 | Bin 0 -> 48602 bytes ...ts.out.tfevents.1766426626.gpu-04.112116.0 | Bin 0 -> 48602 bytes ...ts.out.tfevents.1766427132.gpu-04.112924.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766567162.gpu-51.2669674.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766568374.gpu-51.2671666.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766569231.gpu-51.2673246.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766569685.gpu-51.2674567.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766571661.gpu-51.2676736.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766571804.gpu-51.2677199.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766571900.gpu-51.2677735.0 | Bin 0 -> 48602 bytes ...s.out.tfevents.1766572119.gpu-51.2678290.0 | Bin 0 -> 48602 bytes 57 files changed, 112337 insertions(+), 1460 deletions(-) create mode 100644 .alignment.sh.swp rename dockerfile => Dockerfile (100%) create mode 160000 Stage1 create mode 100644 aiak_megatron/megatron/core/extensions/transformer_engine2.py create mode 100644 aiak_megatron/megatron/core/extensions/transformer_engine_org.py create mode 100644 aiak_megatron/megatron/core/fusions/fused_layer_norm_org.py create mode 100644 aiak_megatron/megatron/training/checkpointing_org.py create mode 100644 aiak_megatron/megatron_core.egg-info/PKG-INFO create mode 100644 aiak_megatron/megatron_core.egg-info/SOURCES.txt create mode 100644 aiak_megatron/megatron_core.egg-info/dependency_links.txt create mode 100644 aiak_megatron/megatron_core.egg-info/requires.txt create mode 100644 aiak_megatron/megatron_core.egg-info/top_level.txt create mode 100644 aiak_training_llm/models/custom/common/local_norm_org.py create mode 100644 aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py create mode 160000 apex create mode 160000 data/LLaVA-558K-Webdataset create mode 100644 libpod/tmp/pause.pid create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_22:02:54_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_22:11:21_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:05:45_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:26:03_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766400270.gpu-52.3342268.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766426196.gpu-04.111446.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766426626.gpu-04.112116.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766427132.gpu-04.112924.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766567162.gpu-51.2669674.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766568374.gpu-51.2671666.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569231.gpu-51.2673246.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569685.gpu-51.2674567.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766571661.gpu-51.2676736.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766571804.gpu-51.2677199.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766571900.gpu-51.2677735.0 create mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766572119.gpu-51.2678290.0 diff --git a/.alignment.sh.swp b/.alignment.sh.swp new file mode 100644 index 0000000000000000000000000000000000000000..e4fe36cdc07ac3089dc120bb33c5e4f7a83b0974 GIT binary patch literal 12288 zcmeI%&rZTH9Ki9;JCVo>OgV2IBN!7;9!Vgc+?AA2#*$IUIy8Fm4SYo(1v+Js10FcS zzD@eS{iXffnogr+IQAdHSh&=#ir@707>hy~6@I99J*!8L*Xeo_MEA$%4o77!H&z!` z{sopszfI=Kmaq9zjz%hZ3Z9KlGMlYT&k&Wz0R#|0009ILK;Vo8%C#px-TANfckuqH&-jWQ9RUOoKmY**5I_I{ y1Q0*~fyN6c_qYFjEyUL4>i@rf|NrzGuLA%A2q1s}0tg_000IagfB*uQA+Q6Yk|$LF literal 0 HcmV?d00001 diff --git a/dockerfile b/Dockerfile similarity index 100% rename from dockerfile rename to Dockerfile diff --git a/Stage1 b/Stage1 new file mode 160000 index 00000000..b16e06e0 --- /dev/null +++ b/Stage1 @@ -0,0 +1 @@ +Subproject commit b16e06e0c10c901bd35f1aad770e94ab6c6f33f3 diff --git a/aiak_megatron/megatron/core/dist_checkpointing/strategies/base.py b/aiak_megatron/megatron/core/dist_checkpointing/strategies/base.py index 77eb325b..91b71106 100644 --- a/aiak_megatron/megatron/core/dist_checkpointing/strategies/base.py +++ b/aiak_megatron/megatron/core/dist_checkpointing/strategies/base.py @@ -23,6 +23,7 @@ class StrategyAction(Enum): default_strategies: DefaultDict[str, dict[tuple, Any]] = defaultdict(dict) + async_calls = AsyncCallsQueue() diff --git a/aiak_megatron/megatron/core/extensions/transformer_engine.py b/aiak_megatron/megatron/core/extensions/transformer_engine.py index 82feb62c..247492d8 100644 --- a/aiak_megatron/megatron/core/extensions/transformer_engine.py +++ b/aiak_megatron/megatron/core/extensions/transformer_engine.py @@ -1,302 +1,141 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. -"""transformer engine utilities""" - -import dataclasses -import io -import os -import pickle -import warnings +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +""" +TE-stub: Transformer Engine *disabled* version for Megatron-Core. + +This file exposes the same public symbols as the original +`megatron.core.extensions.transformer_engine` module, but: + + * Does NOT import `transformer_engine` at all. + * Provides simple PyTorch-based fallbacks for a few classes. + * Marks TE-specific / FP8 / advanced features as unavailable + and raises clear RuntimeError if they are actually used. + +The goal is that: + - `from megatron.core.extensions.transformer_engine import ...` + always succeeds, and + - as long as your config does NOT explicitly request TE/FP8 + (e.g. you use bf16 and non-TE attention backend), training + can run without touching any real TE code. +""" + from typing import Any, Callable, Optional import torch -import transformer_engine as te -from packaging.version import Version as PkgVersion -from torch import Tensor -from torch.nn.parameter import Parameter - -from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding -from megatron.core.model_parallel_config import ModelParallelConfig -from megatron.core.packed_seq_params import PackedSeqParams -from megatron.core.parallel_state import ( - get_context_parallel_global_ranks, - get_context_parallel_group, - get_expert_data_parallel_rank, - get_expert_model_parallel_rank, - get_expert_model_parallel_world_size, - get_expert_tensor_parallel_group, - get_expert_tensor_parallel_rank, - get_expert_tensor_parallel_world_size, - get_hierarchical_context_parallel_groups, - get_tensor_model_parallel_group, - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) -from megatron.core.tensor_parallel import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name -from megatron.core.tensor_parallel.layers import ( - _initialize_affine_weight_cpu, - set_tensor_model_parallel_attributes, -) -from megatron.core.tensor_parallel.random import get_data_parallel_rng_tracker_name -from megatron.core.tensor_parallel.utils import divide -from megatron.core.transformer.enums import AttnMaskType -from megatron.core.enums import Fp8Recipe -from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.transformer.utils import make_sharded_tensors_for_checkpoint -from megatron.core.utils import get_te_version, is_te_min_version - - -def _get_extra_te_kwargs(config: TransformerConfig): - extra_transformer_engine_kwargs = {"params_dtype": config.params_dtype} - - if is_te_min_version("0.12.0"): - if config.use_cpu_initialization: - extra_transformer_engine_kwargs["device"] = 'cpu' - elif config.init_model_with_meta_device: - extra_transformer_engine_kwargs["device"] = "meta" - else: - extra_transformer_engine_kwargs["device"] = torch.cuda.current_device() - return extra_transformer_engine_kwargs +from torch import nn, Tensor -def condition_init_method(config, init_method): - """Condition TE init_method on config.perform_initialization.""" - return init_method if config.perform_initialization else (lambda w: None) +# --------------------------------------------------------------------------- +# Global TE capability flag & MoE fused permute stubs +# --------------------------------------------------------------------------- +HAVE_TE = False # other code may check this -class TENorm: - """ - A conditional wrapper to initialize an instance of Transformer-Engine's - `LayerNorm` or `RMSNorm` based on input +# MoE fused permute helpers – not available without TE +fused_permute = None +fused_unpermute = None +fused_sort_chunks_by_index = None + +# FP8 padding helpers – not available +Fp8Padding = None +Fp8Unpadding = None + + +# --------------------------------------------------------------------------- +# Utility: simple checkpoint wrapper (no real TE checkpoint) +# --------------------------------------------------------------------------- + +def te_checkpoint( + forward_func: Callable, + distribute_saved_activations: bool, + get_rng_state_tracker: Any, + tp_group: Any, + hidden_states: Tensor, + attention_mask: Optional[Tensor], + attn_mask_type: Any, + context: Optional[Tensor], + context_mask: Optional[Tensor], + rotary_pos_emb: Optional[Tensor], + **kwargs, +): """ + Minimal replacement for TE checkpointing. - # TODO should we ditch normalization config and just use spec to choose LayerNorm vs RMSNorm? - def __new__(cls, config: TransformerConfig, hidden_size: int, eps: float = 1e-5): - if config.normalization == "LayerNorm": - instance = te.pytorch.LayerNorm( - hidden_size=hidden_size, - eps=eps, - sequence_parallel=config.sequence_parallel, - zero_centered_gamma=config.layernorm_zero_centered_gamma, - **_get_extra_te_kwargs(config), - ) - elif config.normalization == "RMSNorm": - assert hasattr( - te.pytorch, "RMSNorm" - ), "Transformer-Engine >= v0.11 required to use this feature" - instance = te.pytorch.RMSNorm( - hidden_size=hidden_size, - eps=eps, - sequence_parallel=config.sequence_parallel, - zero_centered_gamma=config.layernorm_zero_centered_gamma, - **_get_extra_te_kwargs(config), - ) - else: - raise Exception('Only LayerNorm and RMSNorm are curently supported') + We simply call `forward_func` directly WITHOUT activation checkpointing. + This keeps correctness but loses memory savings compared to real TE. + """ + return forward_func( + hidden_states, + attention_mask, + attn_mask_type, + context, + context_mask, + rotary_pos_emb, + **kwargs, + ) - return instance +# --------------------------------------------------------------------------- +# TENorm – fallback to plain LayerNorm +# --------------------------------------------------------------------------- -class TELinear(te.pytorch.Linear): +class TENorm(nn.Module): """ - Wrapper for the Transformer-Engine's `Linear` layer. - - Note that if Megatron's parallel_state has not been initialized - yet, the tp_group passed to TE will be None and must be set later - via set_tensor_parallel_group(). - - parallel_mode currently supports 3 different values: - - "column": Split the weight matrix along output dimension (used in TEColumnParallelLinear) - - "row": Split the weight matrix along input dimension (used in TERowParallelLinear) - - "duplicated": No tensor parallelism and weight is duplicated across TP ranks - - Note: For expert linear layers, we will disable communication logic here - as TP communication is handled in token_dispatcher. + TE-style normalization wrapper. + + Original TE version chooses between LayerNorm / RMSNorm etc. + Here we just always use plain LayerNorm. + + Original signature: + TENorm(config: TransformerConfig, hidden_size: int, eps: float = 1e-5) + We accept arbitrary *args / **kwargs and try to pick hidden_size. """ - def __init__( - self, - input_size: int, - output_size: int, - *, - parallel_mode: Optional[str], - config: ModelParallelConfig, - init_method: Callable, - bias: bool, - skip_bias_add: bool, - skip_weight_param_allocation: bool, - tp_comm_buffer_name: Optional[str] = None, - is_expert: bool = False, - ): - self.config = config + def __init__(self, *args, **kwargs): + super().__init__() - # TE returns a zero length Tensor when bias=False and - # return_bias=True, but we prefer None. So in that case we - # tell TE to not return the bias, and return None - # ourselves. This way our forward always returns two values - # and we don't have to deal with the zero length Tensor. - self.te_return_bias = skip_bias_add and bias - self.is_first_microbatch = True - self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache - if skip_weight_param_allocation: + # Try to infer hidden_size and eps + if len(args) >= 2: + hidden_size = args[1] + else: + hidden_size = kwargs.get("hidden_size") or kwargs.get("normalized_shape") + if hidden_size is None: raise ValueError( - 'Transformer Engine linear layers do not support skip_weight_param_allocation' + "TENorm stub could not infer hidden_size. " + "Expected signature: TENorm(config, hidden_size, eps=1e-5)" ) - extra_kwargs = _get_extra_te_kwargs(config) - - if self.config.split_bw: - extra_kwargs["delay_wgrad_compute"] = self.config.split_bw - - if ( - self.config.tp_comm_overlap - and tp_comm_buffer_name - and tp_comm_buffer_name not in ['qkv', 'proj', 'fc1', 'fc2'] - ): - self.config.tp_comm_overlap = False - warnings.warn( - f"The user buffer name {tp_comm_buffer_name} is not supported in" - "Transformer Engine. Disabling TP communication overlap " - "for this layer." - ) + eps = kwargs.get("eps", 1e-5) + self.ln = nn.LayerNorm(hidden_size, eps=eps) - if is_te_min_version("0.8.0"): - if self.config.tp_comm_overlap: - if is_te_min_version("1.5.0"): - # Use old overlap flags if they were supplied instead - extra_kwargs["ub_overlap_ag"] = ( - self.config.tp_comm_overlap_ag - if hasattr(self.config, "tp_comm_overlap_ag") - else self.config.tp_comm_split_ag or self.config.tp_comm_atomic_ag - ) - extra_kwargs["ub_overlap_rs"] = ( - self.config.tp_comm_overlap_rs - if hasattr(self.config, "tp_comm_overlap_rs") - else self.config.tp_comm_split_rs or self.config.tp_comm_atomic_rs - ) - # Disable ub overlap for experts. - if is_expert: - extra_kwargs["ub_overlap_ag"] = False - extra_kwargs["ub_overlap_rs"] = False - else: - extra_kwargs["ub_split_ag"] = self.config.tp_comm_split_ag - extra_kwargs["ub_atomic_gemm_ag"] = self.config.tp_comm_atomic_ag - extra_kwargs["ub_split_rs"] = self.config.tp_comm_split_rs - extra_kwargs["ub_atomic_gemm_rs"] = self.config.tp_comm_atomic_rs - # Disable ub overlap for experts. - if is_expert: - extra_kwargs["ub_split_ag"] = False - extra_kwargs["ub_atomic_gemm_ag"] = False - extra_kwargs["ub_split_rs"] = False - extra_kwargs["ub_atomic_gemm_rs"] = False - if is_te_min_version("1.0.0", check_equality=False): - assert ( - tp_comm_buffer_name is not None - ), "Buffer name should be set to configure communication overlap settings" - extra_kwargs["ub_name"] = tp_comm_buffer_name - - self.expert_parallel = self.config.expert_model_parallel_size > 1 - if is_expert: - rng_tracker_name = get_expert_parallel_rng_tracker_name() - else: - if parallel_mode == "duplicated": - rng_tracker_name = get_data_parallel_rng_tracker_name() - else: - rng_tracker_name = None - if is_te_min_version("1.7.0"): - extra_kwargs["rng_tracker_name"] = rng_tracker_name - - te_parallel_mode = parallel_mode - if parallel_mode == "duplicated": - # Handle non-parallel case - tp_group = None - tp_size = 1 - explicit_expert_comm = False - te_parallel_mode = None - else: - # Disable communications in TE when using TP or EP by - # making TE agnostic of model parallel. - if is_expert: - tp_group = get_expert_tensor_parallel_group(check_initialized=False) - tp_size = get_expert_tensor_parallel_world_size() - else: - tp_group = get_tensor_model_parallel_group(check_initialized=False) - tp_size = get_tensor_model_parallel_world_size() - explicit_expert_comm = is_expert and (tp_size > 1 or self.expert_parallel) + def forward(self, x: Tensor) -> Tensor: + return self.ln(x) - if explicit_expert_comm: - if parallel_mode == "column": - output_size = divide(output_size, tp_size) - elif parallel_mode == "row": - input_size = divide(input_size, tp_size) - te_parallel_mode = None - tp_size = 1 - tp_group = None +# --------------------------------------------------------------------------- +# TELinear and variants – minimal PyTorch fallbacks +# --------------------------------------------------------------------------- - super().__init__( - in_features=input_size, - out_features=output_size, - sequence_parallel=self.config.sequence_parallel, - fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, - tp_group=tp_group, - tp_size=tp_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - init_method=condition_init_method(config, init_method), - bias=bias, - return_bias=self.te_return_bias, - parallel_mode=te_parallel_mode, - **extra_kwargs, - ) +class TELinear(nn.Module): + """ + Fallback for TE's Linear wrappers. - for param in self.parameters(): - if is_expert: - # Reduce the gradient on the expert_data_parallel group for expert linear layers - setattr(param, 'allreduce', not self.expert_parallel) - else: - # Reduce the gradient on DP group - setattr(param, 'allreduce', True) - if parallel_mode == "duplicated": - # Reduce the gradient further on the TP group since the weight is - # duplicated across TP ranks - setattr(param, 'sequence_parallel', self.config.sequence_parallel) - - def forward(self, x): - """Forward.""" - _is_first_microbatch = ( - None if self.disable_parameter_transpose_cache else self.is_first_microbatch + Original signature: + TELinear( + input_size: int, + output_size: int, + *, + parallel_mode: Optional[str], + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + skip_bias_add: bool, + skip_weight_param_allocation: bool, + tp_comm_buffer_name: Optional[str] = None, + is_expert: bool = False, ) - out = super().forward(x, is_first_microbatch=_is_first_microbatch) - self.is_first_microbatch = False - - # TE only returns a tuple when return_bias is True, otherwise - # it returns a single Tensor, we always want to return two - # values regardless of the arguments. - if self.te_return_bias: - return out - return out, None - - def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): - """Replicate cross TP/DP.""" - - # Provide the dist-ckpt support when TELinear is directly used - # It can only happen with duplicated parallel mode - assert ( - self.parallel_mode is None - ), "TELinear sharded_state_dict can only be used with duplicated parallel mode" - state_dict = self.state_dict(prefix='', keep_vars=True) - return make_sharded_tensors_for_checkpoint(state_dict, prefix, None, sharded_offsets) - - def backward_dw(self): - """Compute weight gradients during the backward pass if split_bw is enabled.""" - if self.config.split_bw: - super().backward_dw() - -class TELayerNormColumnParallelLinear(te.pytorch.LayerNormLinear): - """ - Wrapper for the Transformer-Engine's `LayerNormLinear` layer that combines - layernorm and linear layers + We ignore all TE/mparallel details and just wrap nn.Linear. + Forward returns (output, bias_or_none) to match TE's API. """ def __init__( @@ -304,174 +143,49 @@ def __init__( input_size: int, output_size: int, *, - config: TransformerConfig, - init_method: Callable, - gather_output: bool, - bias: bool, - skip_bias_add: bool, - is_expert: bool, + parallel_mode: Optional[str] = None, + config: Any = None, + init_method: Optional[Callable] = None, + bias: bool = True, + skip_bias_add: bool = False, skip_weight_param_allocation: bool = False, tp_comm_buffer_name: Optional[str] = None, + is_expert: bool = False, + **kwargs, ): - self.config = config - - if gather_output: - raise ValueError('Transformer Engine linear layers do not support gather_output = True') - - if is_expert: - raise ValueError('Transformer Engine linear layers do not yet support MoE') - + super().__init__() if skip_weight_param_allocation: - raise ValueError( - 'Transformer Engine linear layers do not support skip_weight_param_allocation' - ) - - # TE returns a zero length Tensor when bias=False and - # return_bias=True, but we prefer None. So in that case we - # tell TE to not return the bias, and return None - # ourselves. This way our forward always returns two values - # and we don't have to deal with the zero length Tensor. - self.te_return_bias = skip_bias_add and bias - self.is_first_microbatch = True - self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache - extra_kwargs = _get_extra_te_kwargs(config) - - if self.config.split_bw: - extra_kwargs["delay_wgrad_compute"] = self.config.split_bw - - # Only Transformer-Engine version >= 0.11.0 supports `RMSNorm` - if is_te_min_version("0.11.0"): - extra_kwargs["normalization"] = self.config.normalization - elif self.config.normalization != "LayerNorm": - te_version = get_te_version() - raise ValueError( - f"Transformer Engine v{te_version} does not support {self.config.normalization}." + # In the real TE this has important semantics; here we just forbid it + raise RuntimeError( + "TELinear stub does not support skip_weight_param_allocation=True." ) - if is_te_min_version("0.8.0"): - if self.config.tp_comm_overlap: - extra_kwargs["ub_bulk_wgrad"] = self.config.tp_comm_bulk_wgrad - extra_kwargs["ub_bulk_dgrad"] = self.config.tp_comm_bulk_dgrad - if is_te_min_version("1.5.0", check_equality=False): - # Use old overlap flags if they were supplied instead - extra_kwargs["ub_overlap_ag"] = ( - self.config.tp_comm_overlap_ag - if hasattr(self.config, "tp_comm_overlap_ag") - else self.config.tp_comm_split_ag or self.config.tp_comm_atomic_ag - ) - if is_te_min_version("1.6.0.dev0", check_equality=False): - extra_kwargs["ub_overlap_rs_dgrad"] = ( - self.config.tp_comm_overlap_rs_dgrad - if hasattr(self.config, "tp_comm_overlap_rs_dgrad") - else False - ) - if tp_comm_buffer_name == 'qkv' and self.config.tp_comm_overlap_disable_qkv: - extra_kwargs["ub_overlap_ag"] = False - extra_kwargs["ub_overlap_rs_dgrad"] = False - - if tp_comm_buffer_name == 'fc1' and self.config.tp_comm_overlap_disable_fc1: - extra_kwargs["ub_overlap_ag"] = False - extra_kwargs["ub_overlap_rs_dgrad"] = False - else: - extra_kwargs["ub_atomic_gemm_ag"] = self.config.tp_comm_atomic_ag - extra_kwargs["ub_split_ag"] = self.config.tp_comm_split_ag - if is_te_min_version("1.0.0", check_equality=False): - assert ( - tp_comm_buffer_name is not None - ), "Buffer name should be set to configure communication overlap settings" - extra_kwargs["ub_name"] = tp_comm_buffer_name - - - super().__init__( - in_features=input_size, - out_features=output_size, - eps=self.config.layernorm_epsilon, - sequence_parallel=self.config.sequence_parallel, - fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, - tp_group=get_tensor_model_parallel_group(check_initialized=False), - tp_size=self.config.tensor_model_parallel_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - init_method=( - condition_init_method(config, init_method) - if not config.use_cpu_initialization - else lambda w: None - ), - bias=bias, - return_bias=self.te_return_bias, - parallel_mode="column", - return_layernorm_output=False, - zero_centered_gamma=self.config.layernorm_zero_centered_gamma, - **extra_kwargs, - ) - - world_size = get_tensor_model_parallel_world_size() - rank = get_tensor_model_parallel_rank() - - if config.use_cpu_initialization: - output_size_per_partition = divide(output_size, world_size) - _ = _initialize_affine_weight_cpu( - self.weight, - output_size, - input_size, - output_size_per_partition, - 0, - init_method=condition_init_method(config, init_method), - stride=1, - return_master_weight=False, - rank=rank, - world_size=world_size, - skip_set_tensor_parallel_attributes=True, - ) - if bias: - self.bias = Parameter( - torch.empty(output_size_per_partition, dtype=config.params_dtype) - ) - set_tensor_model_parallel_attributes(self.bias, True, 0, 1) - with torch.no_grad(): - self.bias.zero_() - setattr(self.bias, 'allreduce', True) - - def forward(self, x): - """Forward.""" - _is_first_microbatch = ( - None if self.disable_parameter_transpose_cache else self.is_first_microbatch - ) - - out = super().forward(x, is_first_microbatch=_is_first_microbatch) - self.is_first_microbatch = False - - # TE only returns a tuple when return_bias is True, otherwise - # it returns a single Tensor, we always want to return two - # values regardless of the arguments. - if self.te_return_bias: - return out - return out, None - - def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): - """Sharding along axis 0, bias sharded""" - state_dict = self.state_dict(prefix='', keep_vars=True) - return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {'weight': 0, 'bias': 0}, sharded_offsets - ) - - def __repr__(self): - return ( - f"{type(self).__name__}(in_features={self.in_features}, " - f"out_features={self.out_features}, bias={self.use_bias}, TP={self.tp_size})" - ) - - def backward_dw(self): - """Compute weight gradients during the backward pass if split_bw is enabled.""" - if self.config.split_bw: - super().backward_dw() + self.linear = nn.Linear(input_size, output_size, bias=bias) + self.skip_bias_add = skip_bias_add + + # If a special init_method is provided, respect it + if init_method is not None: + init_method(self.linear.weight) + if bias and getattr(self.linear, "bias", None) is not None: + # You could customise bias init here if you want + pass + + def forward(self, x: Tensor): + y = self.linear(x) + if self.skip_bias_add: + # Emulate TE behaviour: return y_without_bias, bias separately + return y, self.linear.bias + else: + # Many Megatron call sites expect (output, bias) tuple + return y, None class TEColumnParallelLinear(TELinear): """ - Wrapper for the Transformer-Engine's `Linear` layer but specialized similar - to megatron's `ColumnParallelLinear` layer. + Fallback version of TEColumnParallelLinear. + + We ignore tensor-parallel behaviour and just use the TELinear stub. + Signature kept compatible with original TE wrapper. """ def __init__( @@ -479,7 +193,7 @@ def __init__( input_size: int, output_size: int, *, - config: ModelParallelConfig, + config: Any, init_method: Callable, gather_output: bool, bias: bool, @@ -487,80 +201,32 @@ def __init__( is_expert: bool, skip_weight_param_allocation: bool = False, tp_comm_buffer_name: Optional[str] = None, + **kwargs, ): if gather_output: - raise ValueError('Transformer Engine linear layers do not support gather_output = True') - + # In TE this is not supported; keep same constraint + raise RuntimeError( + "TEColumnParallelLinear stub does not support gather_output=True." + ) super().__init__( input_size=input_size, output_size=output_size, parallel_mode="column", config=config, - init_method=( - condition_init_method(config, init_method) - if not config.use_cpu_initialization - else lambda w: None - ), + init_method=init_method, bias=bias, skip_bias_add=skip_bias_add, - is_expert=is_expert, skip_weight_param_allocation=skip_weight_param_allocation, tp_comm_buffer_name=tp_comm_buffer_name, + is_expert=is_expert, ) - if config.use_cpu_initialization: - if is_expert: - world_size = get_expert_tensor_parallel_world_size() - rank = get_expert_tensor_parallel_rank() - else: - world_size = get_tensor_model_parallel_world_size() - rank = get_tensor_model_parallel_rank() - output_size_per_partition = divide(output_size, world_size) - _ = _initialize_affine_weight_cpu( - self.weight, - output_size, - input_size, - output_size_per_partition, - 0, - init_method=condition_init_method(config, init_method), - stride=1, - return_master_weight=False, - rank=rank, - world_size=world_size, - skip_set_tensor_parallel_attributes=True, - ) - if bias: - self.bias = Parameter( - torch.empty(output_size_per_partition, dtype=config.params_dtype) - ) - set_tensor_model_parallel_attributes(self.bias, True, 0, 1) - with torch.no_grad(): - self.bias.zero_() - setattr(self.bias, 'allreduce', True) - - def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): - """Sharding along axis 0, bias sharded""" - state_dict = self.state_dict(prefix='', keep_vars=True) - return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {'weight': 0, 'bias': 0}, sharded_offsets - ) - - def __repr__(self): - return ( - f"{type(self).__name__}(in_features={self.in_features}, " - f"out_features={self.out_features}, bias={self.use_bias}, TP={self.tp_size})" - ) - - def backward_dw(self): - """Compute weight gradients during the backward pass if split_bw is enabled.""" - if self.config.split_bw: - super().backward_dw() - class TERowParallelLinear(TELinear): """ - Wrapper for the Transformer-Engine's `Linear` layer but specialized similar - to megatron's `RowParallelLinear` layer. + Fallback version of TERowParallelLinear. + + Again, we just use a plain Linear – no real TP behaviour. """ def __init__( @@ -568,101 +234,168 @@ def __init__( input_size: int, output_size: int, *, - config: ModelParallelConfig, + config: Any, init_method: Callable, bias: bool, input_is_parallel: bool, skip_bias_add: bool, is_expert: bool, tp_comm_buffer_name: Optional[str] = None, + **kwargs, ): if not input_is_parallel: - raise ValueError( - "Transformer Engine linear layers do not support input_is_parallel = False" + raise RuntimeError( + "TERowParallelLinear stub expects input_is_parallel=True (same as TE)." ) - super().__init__( input_size=input_size, output_size=output_size, parallel_mode="row", config=config, - init_method=( - condition_init_method(config, init_method) - if not config.use_cpu_initialization - else lambda w: None - ), + init_method=init_method, bias=bias, skip_bias_add=skip_bias_add, - skip_weight_param_allocation=False, # We don't currently use this for row parallel layers - is_expert=is_expert, + skip_weight_param_allocation=False, tp_comm_buffer_name=tp_comm_buffer_name, + is_expert=is_expert, ) - if config.use_cpu_initialization: - if is_expert: - world_size = get_expert_tensor_parallel_world_size() - rank = get_expert_tensor_parallel_rank() - else: - world_size = get_tensor_model_parallel_world_size() - rank = get_tensor_model_parallel_rank() - input_size_per_partition = divide(input_size, world_size) - self.master_weight = _initialize_affine_weight_cpu( - self.weight, - output_size, - input_size, - input_size_per_partition, - 1, - init_method=condition_init_method(config, init_method), - stride=1, - return_master_weight=False, - params_dtype=config.params_dtype, - rank=rank, - world_size=world_size, - skip_set_tensor_parallel_attributes=True, + + +class TELayerNormColumnParallelLinear(nn.Module): + """ + Fallback for TELayerNormColumnParallelLinear. + + Original TE combines LayerNorm + ColumnParallelLinear. + Here we implement: y = Linear(LayerNorm(x)) + + Forward returns (y, bias_or_none) to match TE behaviour. + """ + + def __init__( + self, + input_size: int, + output_size: int, + *, + config: Any, + init_method: Callable, + gather_output: bool, + bias: bool, + skip_bias_add: bool, + is_expert: bool, + skip_weight_param_allocation: bool = False, + tp_comm_buffer_name: Optional[str] = None, + **kwargs, + ): + super().__init__() + if gather_output: + raise RuntimeError( + "TELayerNormColumnParallelLinear stub does not support gather_output=True." + ) + if is_expert: + # You could still allow this, but keep consistent with original constraint + raise RuntimeError( + "TELayerNormColumnParallelLinear stub does not support is_expert=True." ) - if bias: - self.bias = Parameter(torch.empty(output_size, dtype=config.params_dtype)) - # Always initialize bias to zero. - with torch.no_grad(): - self.bias.zero_() - setattr(self.bias, 'allreduce', True) - setattr(self.bias, 'sequence_parallel', config.sequence_parallel) - - def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): - """Sharding along axis 1, bias not sharded""" - state_dict = self.state_dict(prefix='', keep_vars=True) - return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {'weight': 1}, sharded_offsets - ) - def __repr__(self): - return ( - f"{type(self).__name__}(in_features={self.in_features}, " - f"out_features={self.out_features}, bias={self.use_bias}, TP={self.tp_size})" - ) + eps = getattr(config, "layernorm_epsilon", 1e-5) + self.ln = nn.LayerNorm(input_size, eps=eps) + self.linear = nn.Linear(input_size, output_size, bias=bias) + self.skip_bias_add = skip_bias_add - def backward_dw(self): - """Compute weight gradients during the backward pass if split_bw is enabled.""" - if self.config.split_bw: - super().backward_dw() + if init_method is not None: + init_method(self.linear.weight) + + def forward(self, x: Tensor): + y = self.linear(self.ln(x)) + if self.skip_bias_add: + return y, self.linear.bias + else: + return y, None -class TEDotProductAttention(te.pytorch.DotProductAttention): +# --------------------------------------------------------------------------- +# GroupedLinear stubs – names exist, but not implemented +# --------------------------------------------------------------------------- + +class TEGroupedLinear(nn.Module): """ - Wrapper for the Transformer-Engine's `DotProductAttention` layer that also - has "flash attention" enabled. + Stub for TEGroupedLinear. - Note that if Megatron's parallel_state has not been initialized yet, the - tp_group and cp_group passed to TE will be None and must be set later - via set_tensor_parallel_group() and set_context_parallel_group(). + If your config ever tries to instantiate this, you are relying on + Transformer Engine's grouped FP8 GEMMs, which are not available + in this stubbed build. """ - cp_stream: torch.cuda.Stream = None + def __init__(self, *args, **kwargs): + super().__init__() + raise RuntimeError("TEGroupedLinear is not available (Transformer Engine disabled).") + + def forward(self, *args, **kwargs): + raise RuntimeError("TEGroupedLinear is not available (Transformer Engine disabled).") + + +class TEColumnParallelGroupedLinear(TEGroupedLinear): + pass + + +class TERowParallelGroupedLinear(TEGroupedLinear): + pass + + +# --------------------------------------------------------------------------- +# Attention wrapper – disabled (we use non-TE backend instead) +# --------------------------------------------------------------------------- +# class TEDotProductAttention(nn.Module): +# """ +# Very simple scaled dot-product attention fallback. +# This is only meant to keep imports and basic training running, +# not for maximum performance. +# """ + +# def __init__(self, *args, **kwargs): +# super().__init__() + +# def forward( +# self, +# query: torch.Tensor, +# key: torch.Tensor, +# value: torch.Tensor, +# *args, +# **kwargs, +# ) -> torch.Tensor: +# print(type(query), query) +# d_k = query.size(-1) +# scores = torch.matmul(query, key.transpose(-2, -1)) / (d_k ** 0.5) +# attn = torch.softmax(scores, dim=-1) +# return torch.matmul(attn, value) + +import math +from typing import Optional, Any +import torch +import torch.nn as nn +from torch import Tensor + +# Keep the same name so it can be used as a drop-in +class TEDotProductAttention(nn.Module): + """ + Simple, readable scaled dot-product attention replacement for Transformer-Engine's + DotProductAttention. Meant as a fallback for correctness / debugging, not performance. + Forward signature kept compatible with the TE wrapper used previously. + + Notes: + - Supports qkv_format 'sbhd' (seq, batch, head, dim) and 'bshd' (batch, seq, head, dim). + - Also accepts 3-D (B, S, D) inputs and will split heads automatically. + - attention_mask can be boolean (True=keep) or additive (float with -inf on masked positions) + - attention_bias (if provided) is added to attention scores before softmax. + """ + + cp_stream: torch.cuda.Stream = None # keep attribute so callers referencing it won't fail def __init__( self, - config: TransformerConfig, + config: Any, layer_number: int, - attn_mask_type: AttnMaskType, + attn_mask_type: Any, attention_type: str, attention_dropout: Optional[float] = None, softmax_scale: Optional[float] = None, @@ -670,783 +403,577 @@ def __init__( v_channels: Optional[int] = None, cp_comm_type: str = "p2p", ): + super().__init__() + # Store config values we may need self.config = config - self.te_forward_mask_type = False - self.qkv_format: str = 'sbhd' - - if self.config.apply_query_key_layer_scaling != bool( - int(os.getenv('NVTE_APPLY_QK_LAYER_SCALING', '0')) - ): - raise ValueError( - f"apply_query_key_layer_scaling is {self.config.apply_query_key_layer_scaling} " - f"but environment variable NVTE_APPLY_QK_LAYER_SCALING is " - f"{os.getenv('NVTE_APPLY_QK_LAYER_SCALING')}. Transformer Engine does not support " - f"setting query key layer scaling via argument, so these two must match." - ) - - extra_kwargs: dict[str, Any] = {} - if is_te_min_version("0.11.0"): - extra_kwargs["num_gqa_groups"] = self.config.num_query_groups - elif self.config.num_query_groups != self.config.num_attention_heads: - raise ValueError( - f"Transformer Engine v{get_te_version()} does not support Grouped Query Attention, " - f"use a newer version of Transformer Engine. " - f"(num_query_groups ({self.config.num_query_groups}) != " - f"num_attention_heads ({self.config.num_attention_heads}))" - ) - - if is_te_min_version("0.10.0"): - extra_kwargs["attention_type"] = attention_type - # older version don't need attention_type - - if is_te_min_version("0.12.0", check_equality=False): - self.te_forward_mask_type = True - - # This check is important as CP config can be disabled while having a valid CP group - # Example - Disabling CP for encoder while a valid CP group exists for decoder - if self.config.context_parallel_size > 1: - assert is_te_min_version("1.0.0"), "Only Transformer-Engine version >= 1.0.0 supports context parallelism!" - if getattr(TEDotProductAttention, "cp_stream") is None: - TEDotProductAttention.cp_stream = torch.cuda.Stream() - - extra_kwargs["cp_group"] = get_context_parallel_group(check_initialized=False) - extra_kwargs["cp_global_ranks"] = get_context_parallel_global_ranks(check_initialized=False) - extra_kwargs["cp_stream"] = TEDotProductAttention.cp_stream - - if is_te_min_version("1.10.0"): - if cp_comm_type is None: - extra_kwargs["cp_comm_type"] = "p2p" - - elif cp_comm_type == "a2a+p2p": - assert is_te_min_version("1.12.0"), ( - f"Transformer-Engine v{get_te_version()} must be >= 1.12.0 to support" - "hierarchical cp commucation." - ) - extra_kwargs["cp_comm_type"] = "a2a+p2p" - extra_kwargs["cp_group"] = get_hierarchical_context_parallel_groups(check_initialized=False) - else: - extra_kwargs["cp_comm_type"] = cp_comm_type - - if self.config.deterministic_mode: - if int(os.getenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "1")) != 0: - raise RuntimeError( - "deterministic_mode is on and we are using DotProductAttention from " - "Transformer Engine, but NVTE_ALLOW_NONDETERMINISTIC_ALGO is not 0. " - f"Currently set to: {os.getenv('NVTE_ALLOW_NONDETERMINISTIC_ALGO', 'not set')}." + # default qkv format preserved from your original class + self.qkv_format: str = "sbhd" + # num heads convenience + self.num_heads = getattr(config, "num_attention_heads", 1) + # total kv channels (if present on config). We'll use this only when needed to infer head dim + self.kv_channels = getattr(config, "kv_channels", None) + # optional dropout on attention weights (kept None unless provided) + self.attn_dropout = nn.Dropout(attention_dropout) if attention_dropout and attention_dropout > 0.0 else None + # allow externally-configured softmax scale (float) OR compute as 1/sqrt(head_dim) later + self.softmax_scale = softmax_scale + # keep set of packed seq param names to remain compatible with callers (ignored here) + self.kept_packed_seq_params = set() + + def _ensure_4d_and_split_heads(self, x: Tensor) -> (Tensor, bool): + """ + Ensure x is in (B, S, H, D_head) layout and return (x4d, was_3d) + Accepts: + - x shaped (S, B, H, D) if qkv_format == 'sbhd' (seq, batch, head, dim) + - x shaped (B, S, H, D) if qkv_format == 'bshd' + - x shaped (B, S, D) -> will split last dim into (H, D_head) + Returns x in (B, S, H, D_head) format and a flag indicating whether original was 3d. + """ + was_3d = False + if x is None: + raise ValueError("TEDotProductAttention: input tensor is None") + if x.dim() == 4: + if self.qkv_format == "sbhd": + # (S, B, H, D) -> convert to (B, S, H, D) + x4 = x.transpose(0, 1).contiguous() + elif self.qkv_format == "bshd": + x4 = x.contiguous() + else: + # fallback: assume bshd if unknown + x4 = x.contiguous() + elif x.dim() == 3: + # (B, S, D) -> split D into (H, D_head) + was_3d = True + B, S, D = x.shape + if D % self.num_heads != 0: + raise ValueError( + f"TEDotProductAttention: embedding dim ({D}) is not divisible by num_heads ({self.num_heads})." ) - - if config.window_size is not None: - # Check version - assert is_te_min_version("1.2.0"), ( - f"Transformer-Engine v{get_te_version()} must be >= 1.2.0 to support sliding window attention." - ) - extra_kwargs['window_size'] = config.window_size - - if is_te_min_version("1.10.0"): - # TE 1.10.0 introduces the ability to set the different k and v channels - kv_channels = ( - (k_channels, v_channels) - if k_channels is not None and v_channels is not None - else self.config.kv_channels - ) - extra_kwargs['softmax_scale'] = softmax_scale + D_head = D // self.num_heads + x4 = x.view(B, S, self.num_heads, D_head).contiguous() else: - kv_channels = self.config.kv_channels - - self.kept_packed_seq_params = set(field.name for field in dataclasses.fields(PackedSeqParams)) - if get_te_version() < PkgVersion("1.3.0"): - # TE 1.3.0 introduces precomputing max_seqlen to remove unnecessary kernels and D2H - # copies (#555) - # These two arguments did not exist prior to 1.3.0 - self.kept_packed_seq_params.discard("max_seqlen_q") - self.kept_packed_seq_params.discard("max_seqlen_kv") - - if get_te_version() < PkgVersion("1.10.0"): - # TE 1.8.0 introduces cu_seqlens_padded which is the cu_seqlens with paddings counted - # in each individual sequence in THD format dataset - # These two arguments did not exist prior to 1.8.0. Full support added in 1.10.0 (#1012) - self.kept_packed_seq_params.discard("cu_seqlens_q_padded") - self.kept_packed_seq_params.discard("cu_seqlens_kv_padded") + raise ValueError(f"Tensors must be 3D (B,S,D) or 4D. Got shape {tuple(x.shape)}") + return x4, was_3d - super().__init__( - num_attention_heads=self.config.num_attention_heads, - kv_channels=kv_channels, - attention_dropout=self.config.attention_dropout if attention_dropout is None else attention_dropout, - attn_mask_type=attn_mask_type.name, - sequence_parallel=self.config.sequence_parallel, - tp_size=self.config.tensor_model_parallel_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - tp_group=get_tensor_model_parallel_group(check_initialized=False), - layer_number=layer_number, - **extra_kwargs, - ) + def _reconstruct_output(self, out4: Tensor, was_3d: bool) -> Tensor: + """ + out4 is (B, S, H, D_head). + If was_3d True -> return (B, S, D) by merging head dim + If qkv_format previously was 'sbhd' and user originally passed sbhd (we tracked earlier), + we attempt to return in original layout. However to keep behavior simple we return: + - (B,S,D) if original inputs were 3D + - (B,S,H,D) otherwise + """ + if was_3d: + B, S, H, Dh = out4.shape + return out4.view(B, S, H * Dh) + else: + return out4 def forward( self, query: Tensor, key: Tensor, value: Tensor, - attention_mask: Tensor, - attn_mask_type: AttnMaskType, - attention_bias: Tensor = None, - packed_seq_params: PackedSeqParams = None, - ): - """Forward.""" - packed_seq_kwargs = ( - {key: getattr(packed_seq_params, key) for key in self.kept_packed_seq_params} - if packed_seq_params is not None - else {} - ) - # overwrite self.qkv_format depending on self.config.apply_rope_fusion, which can be set - # after init - if self.config.apply_rope_fusion and is_te_min_version("0.13.0", check_equality=False): - self.qkv_format = 'bshd' - - qkv_format = packed_seq_kwargs.get('qkv_format', self.qkv_format) - - # WAR for peak memory usage. - # See https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/merge_requests/2388 - if self.config.apply_rope_fusion and qkv_format == 'bshd': - query, key, value = [x.transpose(0, 1).contiguous() for x in (query, key, value)] - # In PyTorch, the following two tensors are in fact the same: - # Tensor with shape (1, S, H, D) and stride (S*H*D, H*D, D, 1) - # Tensor with shape (1, S, H, D) and stride (H*D, H*D, D, 1) - # Stride for a dimension that is 1 has no meaning, so tensors created two different ways - # can have same shape but different strides. - # We unify them to the first one to pass the stride check in TE - if value.shape == key.shape and value.shape[0] == 1 and value.stride() != key.stride(): - value = value.as_strided(value.shape, key.stride()) - - attention_bias_kwargs = {} - if attention_bias is not None: - assert is_te_min_version("1.2.0"), ( - f"Transformer-Engine v{get_te_version()} must be >= 1.2.0 to support `attention_bias`." - ) - attention_bias_kwargs = dict( - core_attention_bias_type='post_scale_bias', core_attention_bias=attention_bias - ) - else: - if self.config.position_embedding_type == "alibi": - attention_bias_kwargs = dict(core_attention_bias_type='alibi', core_attention_bias=None) - - if self.te_forward_mask_type: - if qkv_format == 'thd' and is_te_min_version("1.7.0"): - # thd format uses flash attention with cuDNN kernel which requires is_padding=True, - # so the only acceptable mask types are `padding_causal` and `padding`. These do not - # necessarily indicate there are padded tokens in the sequence. - if attn_mask_type == AttnMaskType.causal: - attn_mask_type = AttnMaskType.padding_causal - elif attn_mask_type == AttnMaskType.no_mask: - attn_mask_type = AttnMaskType.padding - core_attn_out = super().forward( - query, - key, - value, - attention_mask, - attn_mask_type=attn_mask_type.name, - **attention_bias_kwargs, - **packed_seq_kwargs, - ) - else: - core_attn_out = super().forward( - query, - key, - value, - attention_mask, - **attention_bias_kwargs, - **packed_seq_kwargs, - ) - - if self.config.apply_rope_fusion and qkv_format == 'bshd': - return core_attn_out.transpose(0, 1) - else: - return core_attn_out - - -if is_te_min_version("1.9.0.dev0"): - - class TEGroupedLinear(te.pytorch.GroupedLinear): + attention_mask: Optional[Tensor], + attn_mask_type: Any, + attention_bias: Optional[Tensor] = None, + packed_seq_params: Any = None, + ) -> Tensor: """ - Wrapper for the Transformer-Engine's `GroupedLinear` layer. + Simple scaled dot-product attention: + out[b, s_q, h, d] = softmax( (q*k^T)/sqrt(d) + attention_bias + mask ) @ v - Note that if Megatron's parallel_state has not been initialized - yet, the tp_group passed to TE will be None and must be set later - via set_tensor_parallel_group(). + Returns output in (B, S_q, H, D_head) unless the inputs were 3D (B,S,D) in which case returns (B,S,D). """ - def __init__( - self, - num_gemms: int, - input_size: int, - output_size: int, - *, - parallel_mode: Optional[str], - config: ModelParallelConfig, - init_method: Callable, - bias: bool, - skip_bias_add: bool, - is_expert: bool = False, - tp_comm_buffer_name: Optional[str] = None, - ): - self.config = config - - # TE returns a zero length Tensor when bias=False and - # return_bias=True, but we prefer None. So in that case we - # tell TE to not return the bias, and return None - # ourselves. This way our forward always returns two values - # and we don't have to deal with the zero length Tensor. - self.te_return_bias = skip_bias_add and bias - self.is_first_microbatch = True - self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache - - extra_kwargs = _get_extra_te_kwargs(config) - - if self.config.split_bw: - extra_kwargs["delay_wgrad_compute"] = self.config.split_bw - - extra_kwargs["ub_name"] = tp_comm_buffer_name - - self.expert_parallel = self.config.expert_model_parallel_size > 1 - if is_expert: - extra_kwargs["rng_tracker_name"] = get_expert_parallel_rng_tracker_name() - - # The comms between TP and EP group is explicitly handled by MoE token dispatcher. - # So we disable comms by making TE agnostic of model parallel. - if is_expert: - tp_group = get_expert_tensor_parallel_group(check_initialized=False) - tp_size = get_expert_tensor_parallel_world_size() + # Allow callers to pass packed_seq_params (kept for API compatibility) but we ignore it here. + _ = packed_seq_params # noqa: F841 + + # Convert inputs to (B, S, H, D_head) + q4, q_was_3d = self._ensure_4d_and_split_heads(query) + k4, k_was_3d = self._ensure_4d_and_split_heads(key) + v4, v_was_3d = self._ensure_4d_and_split_heads(value) + + # sanity check shapes + if not (q_was_3d == k_was_3d == v_was_3d): + # mixing 3D / 4D forms is confusing; require consistent input forms + raise ValueError("TEDotProductAttention: query/key/value must be all 3D or all 4D (consistent formats).") + + Bq, Sq, Hq, Dh = q4.shape + Bk, Sk, Hk, Dk = k4.shape + Bv, Sv, Hv, Dv = v4.shape + + if not (Bq == Bk == Bv): + raise ValueError("Batch sizes of query/key/value must match") + if not (Hq == Hk == Hv): + raise ValueError("Number of heads of query/key/value must match") + if not (Dk == Dv): + raise ValueError("Key/value head dims must match") + if Dh != Dk: + # head dims should match for q and k + # if they don't, we can still compute by projecting; but here we require equality + raise ValueError(f"Head dimension mismatch: q head dim {Dh} vs k head dim {Dk}") + + B = Bq + # compute scale + scale = self.softmax_scale if self.softmax_scale is not None else 1.0 / math.sqrt(Dh) + + # compute raw scores: we want shape (B, H, Sq, Sk) + # reshape to (B, H, Sq, Dh) etc and use matmul after transposing + # q4: (B, Sq, H, Dh) -> (B, H, Sq, Dh) + qBH = q4.transpose(1, 2) # (B, H, Sq, Dh) + kBH = k4.transpose(1, 2) # (B, H, Sk, Dh) + vBH = v4.transpose(1, 2) # (B, H, Sv, Dh) + + # scores: (B, H, Sq, Sk) + scores = torch.matmul(qBH, kBH.transpose(-2, -1)) * scale + + # apply attention_bias if given (broadcastable) + if attention_bias is not None: + # attention_bias commonly has shape (B, 1, 1, Sk) or (1, 1, 1, Sk) or (Sq, Sk) + scores = scores + attention_bias + + # apply attention_mask + if attention_mask is not None: + # If mask is boolean where True means keep, convert to additive mask + # Accept both boolean masks and additive masks (float) + if attention_mask.dtype == torch.bool: + # mask shape might be (B, Sk) or (B, 1, 1, Sk) etc; we want True where keep + # we fill disallowed positions with -inf + scores = scores.masked_fill(~attention_mask, float("-inf")) else: - tp_group = get_tensor_model_parallel_group(check_initialized=False) - tp_size = get_tensor_model_parallel_world_size() - self.explicit_expert_comm = is_expert and (tp_size > 1 or self.expert_parallel) - - if self.explicit_expert_comm: - if parallel_mode == "column": - output_size = divide(output_size, tp_size) - elif parallel_mode == "row": - input_size = divide(input_size, tp_size) - parallel_mode = None - tp_size = 1 - tp_group = None - - - super().__init__( - num_gemms=num_gemms, - in_features=input_size, - out_features=output_size, - sequence_parallel=self.config.sequence_parallel, - fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, - tp_group=tp_group, - tp_size=tp_size, - get_rng_state_tracker=( - get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None - ), - init_method=condition_init_method(config, init_method), - bias=bias, - return_bias=self.te_return_bias, - parallel_mode=parallel_mode, - **extra_kwargs, - ) + # assume additive mask already shaped for broadcasting + scores = scores + attention_mask - for param in self.parameters(): - setattr(param, 'allreduce', not (is_expert and self.expert_parallel)) - - def merge_extra_states( - self, - state_dict, - prefix, - local_metadata, - strict, - missing_keys, - unexpected_keys, - error_msgs, - ): - """ - Merge multiple "_extra_state" into one. - """ - self.init_fp8_metadata(num_gemms=self.num_gemms) - # When resume training, loading ckpt is out of fp8_autocast context. - # So we need to manually detect from the state_dict. - fp8_checkpoint = any("_extra_state" in str(key) for key in state_dict.keys()) - - if not fp8_checkpoint: - return - - try: - state_list = [ - state_dict.pop(f"{prefix}_extra_state{i}") for i in range(1, self.num_gemms) - ] - except KeyError: - # "_extra_state{i}" only exists for dist-ckpt. Return for torch native ckpt. - return - - # Early return conditions: - # 1. Empty state_dict - # 2. Empty state_list - # 3. _extra_state is None - # 4. _extra_state does not contain any information - if ( - not state_dict - or not state_list - or state_dict.get(f"{prefix}_extra_state") is None - or self._decode_extra_state(state_dict[f"{prefix}_extra_state"]) is None - ): - return - - state_list = [state_dict.pop(f"{prefix}_extra_state")] + state_list - state_list = [self._decode_extra_state(state) for state in state_list] - extra_fp8_variables = state_list[0]['extra_fp8_variables'] - extra_fp8_variables['num_gemms'] = self.num_gemms - extra_state = {"extra_fp8_variables": extra_fp8_variables} - # TE 2.0 adds recipe in extra_state - if is_te_min_version("2.0.0"): - self.fp8_meta["recipe"] = state_list[0]['recipe'] - extra_state['recipe'] = self.fp8_meta["recipe"] - # Only delayed scaling has global fp8 meta tensors. We're not using - # self.fp8_meta["recipe"].delayed() because it's available in TE 2.0 and later. - if isinstance(self.fp8_meta["recipe"], te.common.recipe.DelayedScaling): - extra_state.update( - { - "scale_fwd": torch.cat( - [state['scale_fwd'].view(-1, 1) for state in state_list], dim=1 - ).view(-1), - "amax_history_fwd": torch.cat( - [state['amax_history_fwd'].view(-1, 1) for state in state_list], - dim=1, - ).view(self.fp8_meta["recipe"].amax_history_len, -1), - "scale_bwd": torch.cat( - [state['scale_bwd'].view(-1, 1) for state in state_list], dim=1 - ).view(-1), - "amax_history_bwd": torch.cat( - [state['amax_history_bwd'].view(-1, 1) for state in state_list], - dim=1, - ).view(self.fp8_meta["recipe"].amax_history_len, -1), - } - ) - # TE 2.0 removes scale_inv_fwd and scale_inv_bwd - if not is_te_min_version("2.0.0"): - extra_state.update( - { - "scale_inv_fwd": torch.cat( - [state['scale_inv_fwd'].view(-1, 1) for state in state_list], - dim=1, - ).view(-1), - "scale_inv_bwd": torch.cat( - [state['scale_inv_bwd'].view(-1, 1) for state in state_list], - dim=1, - ).view(-1), - } - ) - state_dict[f"{prefix}_extra_state"] = self._encode_extra_state(extra_state) - - self._register_load_state_dict_pre_hook(merge_extra_states, with_module=True) - - def forward(self, x, m_splits): - """Forward.""" - _is_first_microbatch = ( - None if self.disable_parameter_transpose_cache else self.is_first_microbatch - ) - out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) - self.is_first_microbatch = False - - # TE only returns a tuple when return_bias is True, otherwise - # it returns a single Tensor, we always want to return two - # values regardless of the arguments. - if self.te_return_bias: - return out - return out, None - - def _encode_extra_state(self, state): - # TE 2.0 changed the format of extra_state to be a byte tensor - if is_te_min_version("2.0.0"): - torch.cuda.synchronize() - state_serialized = bytearray(pickle.dumps(state)) - state_serialized = torch.frombuffer(state_serialized, dtype=torch.uint8) - else: - state_serialized = io.BytesIO() - torch.save(state, state_serialized) - return state_serialized - - def _decode_extra_state(self, state): - if isinstance(state, torch.Tensor): - return pickle.loads(state.detach().cpu().numpy().tobytes()) - elif isinstance(state, io.BytesIO): - state.seek(0) - return torch.load(state, map_location="cuda", weights_only=False) - else: - raise RuntimeError("Unsupported checkpoint format.") - - def _split_extra_state(self, state): - fp8_checkpoint = self.fp8_meta["fp8_checkpoint"] or self.fp8 or self.fp8_calibration - - if not fp8_checkpoint: - return [state] * self.num_gemms - - state = self._decode_extra_state(state) - extra_states = [] - extra_fp8_variables = state['extra_fp8_variables'] - extra_fp8_variables['num_gemms'] = 1 - for gemm_idx in range(self.num_gemms): - tmp_state = {"extra_fp8_variables": extra_fp8_variables} - # TE 2.0 adds recipe in extra_state - if is_te_min_version("2.0.0"): - tmp_state['recipe'] = state['recipe'] - # Only delayed scaling has global fp8 meta tensors. We're not using - # self.fp8_meta["recipe"].delayed() because it's available in TE 2.0 and later. - if isinstance(self.fp8_meta["recipe"], te.common.recipe.DelayedScaling): - tmp_state.update( - { - "scale_fwd": state['scale_fwd'].view(3, -1)[:, gemm_idx], - "amax_history_fwd": state['amax_history_fwd'].view( - self.fp8_meta["recipe"].amax_history_len, 3, -1 - )[:, :, gemm_idx], - "scale_bwd": state['scale_bwd'].view(2, -1)[:, gemm_idx], - "amax_history_bwd": state['amax_history_bwd'].view( - self.fp8_meta["recipe"].amax_history_len, 2, -1 - )[:, :, gemm_idx], - } - ) - # TE 2.0 removes scale_inv_fwd and scale_inv_bwd - if not is_te_min_version("2.0.0"): - tmp_state.update( - { - "scale_inv_fwd": state['scale_inv_fwd'].view(3, -1)[:, gemm_idx], - "scale_inv_bwd": state['scale_inv_bwd'].view(2, -1)[:, gemm_idx], - } - ) - extra_states.append(self._encode_extra_state(tmp_state)) - return extra_states - - def _sharded_state_dict_grouped( - self, tp_axis_map, prefix='', sharded_offsets=(), metadata=None - ): - """ - prefix should be module_name to make keys identical to sequetial ones. - """ - sharded_state_dict = {} - full_state_dict = self.state_dict(prefix='', keep_vars=True) - num_global_experts = get_expert_model_parallel_world_size() * self.num_gemms - local_expert_indices_offset = get_expert_model_parallel_rank() * self.num_gemms - ep_axis = len(sharded_offsets) - extra_states = self._split_extra_state(full_state_dict['_extra_state']) - for gemm_idx in range(self.num_gemms): - state_dict = { - f'{gemm_idx}.weight': full_state_dict[f'weight{gemm_idx}'], - f'{gemm_idx}._extra_state': extra_states[gemm_idx], - } - if self.use_bias: - state_dict[f'{gemm_idx}.bias'] = full_state_dict[f'bias{gemm_idx}'] - sub_sd = make_sharded_tensors_for_checkpoint( - state_dict, - '', - tp_axis_map, - ( - *sharded_offsets, - (ep_axis, local_expert_indices_offset + gemm_idx, num_global_experts), - ), - ) - # Remove expert layers indexing from sharded keys - replace_prefix_for_sharding(sub_sd, f'{gemm_idx}.', prefix) - sharded_state_dict.update( - { - f'{prefix}weight{gemm_idx}': sub_sd[f'{gemm_idx}.weight'], - f'{prefix}_extra_state{"" if gemm_idx == 0 else gemm_idx}': sub_sd[ - f'{gemm_idx}._extra_state' - ], - } - ) - if self.use_bias: - sharded_state_dict[f'{prefix}bias{gemm_idx}'] = sub_sd[f'{gemm_idx}.bias'] - # Adjust replica ids - replication along DP modulo EP - for k, sh_ten in sharded_state_dict.items(): - replica_id = sh_ten.replica_id - assert ( - len(replica_id) == 3 - ), f'Expected replica_id for {k} to be in (PP, TP, DP) format, got: {replica_id}' - if getattr(sh_ten, "is_data_parallel_fully_shard", False): - edp_replica_id = 0 - else: - edp_replica_id = get_expert_data_parallel_rank() - sh_ten.replica_id = (*replica_id[:2], edp_replica_id) - return sharded_state_dict - - def backward_dw(self): - """Compute weight gradients during the backward pass if split_bw is enabled.""" - if self.config.split_bw: - super().backward_dw() - - class TEColumnParallelGroupedLinear(TEGroupedLinear): - """ - Wrapper for the Transformer-Engine's `GroupedLinear` layer but specialized - to column-parallel style. - """ + # softmax, dropout (if any), then matmul with v + attn = torch.softmax(scores, dim=-1) + if self.attn_dropout is not None: + attn = self.attn_dropout(attn) - def __init__( - self, - num_gemms: int, - input_size: int, - output_size: int, - *, - config: ModelParallelConfig, - init_method: Callable, - bias: bool, - skip_bias_add: bool, - is_expert: bool, - tp_comm_buffer_name: Optional[str] = None, - ): - - super().__init__( - num_gemms=num_gemms, - input_size=input_size, - output_size=output_size, - parallel_mode="column", - config=config, - init_method=condition_init_method(config, init_method), - bias=bias, - skip_bias_add=skip_bias_add, - is_expert=is_expert, - tp_comm_buffer_name=tp_comm_buffer_name, - ) + # (B, H, Sq, Sk) @ (B, H, Sk, Dh) -> (B, H, Sq, Dh) + outBH = torch.matmul(attn, vBH) - def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): - """ - For each gemm, sharding along axis 0, bias sharded. - Assume sharded_offsets[-1] is the expert parallel offset. - """ - tp_axis_map = {} - for gemm_idx in range(self.num_gemms): - tp_axis_map.update({f'{gemm_idx}.weight': 0, f'{gemm_idx}.bias': 0}) - return super()._sharded_state_dict_grouped( - tp_axis_map, prefix, sharded_offsets, metadata - ) + # bring back to (B, Sq, H, Dh) + out4 = outBH.transpose(1, 2).contiguous() - class TERowParallelGroupedLinear(TEGroupedLinear): - """ - Wrapper for the Transformer-Engine's `GroupedLinear` layer but specialized - to row-parallel style. - """ + # If inputs were 3D originally, merge heads back + out = self._reconstruct_output(out4, q_was_3d) - def __init__( - self, - num_gemms: int, - input_size: int, - output_size: int, - *, - config: ModelParallelConfig, - init_method: Callable, - bias: bool, - skip_bias_add: bool, - is_expert: bool, - tp_comm_buffer_name: Optional[str] = None, - ): - - super().__init__( - num_gemms=num_gemms, - input_size=input_size, - output_size=output_size, - parallel_mode="row", - config=config, - init_method=condition_init_method(config, init_method), - bias=bias, - skip_bias_add=skip_bias_add, - is_expert=is_expert, - tp_comm_buffer_name=tp_comm_buffer_name, - ) + return out - def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): - """ - For each gemm, sharding along axis 1, bias not sharded. - Assume sharded_offsets[-1] is the expert parallel offset. - """ - tp_axis_map = {f'{gemm_idx}.weight': 1 for gemm_idx in range(self.num_gemms)} - return super()._sharded_state_dict_grouped( - tp_axis_map, prefix, sharded_offsets, metadata - ) +# --------------------------------------------------------------------------- +# FP8 / RNG / misc TE utilities – disabled +# --------------------------------------------------------------------------- -else: - TEGroupedLinear = None # type: ignore[assignment, misc] - TEColumnParallelGroupedLinear = None # type: ignore[assignment, misc] - TERowParallelGroupedLinear = None # type: ignore[assignment, misc] +class TEDelayedScaling: + """ + Stub for TE DelayedScaling FP8 recipe. + """ + def __init__(self, *args, **kwargs): + raise RuntimeError("FP8 / TEDelayedScaling is not available (Transformer Engine disabled).") -class TEDelayedScaling(te.common.recipe.DelayedScaling): + +class TECudaRNGStatesTracker: """ - Wrapper for the Transformer-Engine's `DelayedScaling` layer. + Stub for TE CUDA RNG tracker. + + Megatron's own RNG tracker should be used instead. """ - def __init__( - self, - config: ModelParallelConfig, - fp8_format: int, - override_linear_precision: tuple = (False, False, False), - ): - extra_kwargs = _get_extra_te_kwargs(config) - if is_te_min_version("1.6.0.dev0"): - extra_kwargs["fp8_dpa"] = config.fp8_dot_product_attention - extra_kwargs["fp8_mha"] = config.fp8_multi_head_attention - if get_te_version() < PkgVersion("1.8.0"): - extra_kwargs["interval"] = config.fp8_interval - elif config.fp8_interval != 1: - warnings.warn("fp8_interval is deprecated and ignored from Transformer-Engine v1.8.0.") + def __init__(self, *args, **kwargs): + raise RuntimeError("TECudaRNGStatesTracker is not available (Transformer Engine disabled).") - super().__init__( - margin=config.fp8_margin, - fp8_format=fp8_format, - amax_compute_algo=config.fp8_amax_compute_algo, - amax_history_len=config.fp8_amax_history_len, - override_linear_precision=override_linear_precision, - **extra_kwargs, - ) +# --------------------------------------------------------------------------- +# SplitAlongDim, RoPE, CPU offload – minimal / disabled +# --------------------------------------------------------------------------- -class TECudaRNGStatesTracker(te.pytorch.distributed.CudaRNGStatesTracker): - """Wraps TransformerEngine's CudaRNGStatesTracker so that it is - interchangeable with Megatron's RNG tracker""" +# def SplitAlongDim(x: Tensor, dim: int, num_chunks: int): +# """ +# Minimal replacement for TE's _SplitAlongDim.apply. - def __init__(self): - super().__init__() - self.reset() +# Returns a tuple of chunks along the given dimension. +# Autograd works fine through torch.chunk, so this is enough +# for most use cases that only expect a "split" function. +# """ +# return torch.chunk(x, num_chunks, dim=dim) +import torch +from torch import Tensor +from typing import Union, Sequence + +def SplitAlongDim(x: Tensor, dim: int, num_chunks: Union[int, Sequence[int]]): + """ + Replacement for TE's _SplitAlongDim.apply. - def is_initialized(self): - """Checks if the internal RNG state has been set wirth set_states().""" - return self._is_initialized + - If `num_chunks` is an int: behaves like torch.chunk(x, num_chunks, dim=dim). + - If `num_chunks` is a sequence of ints: behaves like torch.split(x, num_chunks, dim=dim). + In that case the ints specify sizes for each split along `dim`. - def reset(self): - """Reset the internal RNG state.""" - super().reset() - self._is_initialized = False + Returns a tuple of tensors. - def set_states(self, states): - """Set the internal RNG state.""" - super().set_states(states) - self._is_initialized = True + Raises ValueError for invalid inputs (e.g. sizes don't sum to x.size(dim)). + """ + if x is None: + raise ValueError("SplitAlongDim: input tensor x is None") + + if not isinstance(dim, int): + raise TypeError(f"SplitAlongDim: dim must be int, got {type(dim)}") + + # int -> torch.chunk + if isinstance(num_chunks, int): + if num_chunks <= 0: + raise ValueError(f"SplitAlongDim: num_chunks must be > 0, got {num_chunks}") + return tuple(torch.chunk(x, num_chunks, dim=dim)) + + # sequence -> torch.split with sizes + if isinstance(num_chunks, (list, tuple)): + sizes = list(num_chunks) + if not all(isinstance(s, int) and s >= 0 for s in sizes): + raise TypeError("SplitAlongDim: when passing sizes, all elements must be non-negative ints") + total = sum(sizes) + dim_size = x.size(dim) + if total != dim_size: + raise ValueError( + f"SplitAlongDim: sizes sum to {total} but tensor.size({dim}) == {dim_size}. " + "Sizes must match the dimension length exactly." + ) + return tuple(torch.split(x, sizes, dim=dim)) - def add(self, name, seed): - """Track the rng state.""" - super().add(name, seed) - self._is_initialized = True + raise TypeError( + "SplitAlongDim: num_chunks must be an int (number of equal chunks) or " + "a list/tuple of sizes for each chunk." + ) -def te_checkpoint( - forward_func, - distribute_saved_activations, - get_rng_state_tracker, - tp_group, - hidden_states, - attention_mask, - attn_mask_type, - context, - context_mask, - rotary_pos_emb, - **kwargs, +def get_cpu_offload_context( + enabled: bool, + num_layers: int, + model_layers: Any, + activation_offloading: bool, + weight_offloading: bool, ): - """Checkpointing with Transformer-Engine.""" - from transformer_engine.pytorch.distributed import checkpoint - - if is_te_min_version("1.5.0"): - return checkpoint( - forward_func, - hidden_states, - attention_mask, - attn_mask_type, - context, - context_mask, - rotary_pos_emb, - distribute_saved_activations=distribute_saved_activations, - get_rng_state_tracker=get_rng_state_tracker, - tp_group=tp_group, - **kwargs, - ) - else: - return checkpoint( - forward_func, - distribute_saved_activations, - get_rng_state_tracker, - tp_group, - hidden_states, - attention_mask, - context, - context_mask, - rotary_pos_emb, - ) + """ + Stub for TE CPU offload context. + We simply return (None, lambda: None). + """ + def _sync(): + return None + + return None, _sync + + +# def fused_apply_rotary_pos_emb( +# t: torch.Tensor, freqs: torch.Tensor, transpose_output_memory: bool = False +# ) -> torch.Tensor: +# """ +# Stub for fused RoPE (sbhd). We fall back to naive RoPE application: + +# This is a placeholder; if your config explicitly requires TE fused RoPE, +# you should not be using this stub file. +# """ +# # A very naive RoPE: assume last dimension is split into (cos, sin) +# # This is intentionally simple and may NOT match all TE behaviours. +# dim = t.shape[-1] +# half = dim // 2 +# cos = freqs[..., :half] +# sin = freqs[..., half:half * 2] +# x1, x2 = t[..., :half], t[..., half:half * 2] +# rot = torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1) +# if transpose_output_memory: +# # We ignore this just like TE stub +# pass +# return rot +import torch +from torch import Tensor +from typing import Tuple -try: - from transformer_engine.pytorch.attention import _SplitAlongDim - SplitAlongDim = _SplitAlongDim.apply +def _align_and_broadcast_to(tensor: Tensor, target: Tensor, name: str) -> Tensor: + """ + Try to make `tensor` broadcastable to `target` by: + - checking last-dim size matches target's last-dim, + - if tensor has fewer dims: prepend singleton dims, + - if tensor has more dims: try to squeeze leading singleton dims, + - finally expand to target.shape. -except ImportError: - SplitAlongDim = None + If impossible, raise a helpful ValueError showing shapes. + """ + if tensor is None: + raise ValueError(f"{name} is None") -try: - from transformer_engine.pytorch.cpu_offload import get_cpu_offload_context as _get_cpu_offload_context - def get_cpu_offload_context( - enabled, num_layers, model_layers, activation_offloading, weight_offloading - ): - """Get CPU offload context and sync function.""" - if is_te_min_version("1.10.0.dev0"): - context, sync_func = _get_cpu_offload_context( - enabled, num_layers, model_layers, activation_offloading, weight_offloading - ) + if tensor.size(-1) != target.size(-1): + raise ValueError( + f"{name} last-dim size ({tensor.size(-1)}) != target last-dim ({target.size(-1)}). " + "They must match (both equal to half = D//2)." + ) + + t_ndim = tensor.ndim + tgt_ndim = target.ndim + + # If tensor has fewer dims, prepend singleton dims + if t_ndim < tgt_ndim: + new_shape = (1,) * (tgt_ndim - t_ndim) + tensor.shape + tensor = tensor.reshape(new_shape) + + # If tensor has more dims, try to squeeze leading singleton dims + elif t_ndim > tgt_ndim: + # Attempt to remove leading dims of size 1 until dims match or cannot. + # Only allow squeezing if those dims are 1. + diff = t_ndim - tgt_ndim + leading = tensor.shape[:diff] + if all(s == 1 for s in leading): + tensor = tensor.reshape(tensor.shape[diff:]) else: - context, sync_func = _get_cpu_offload_context( - enabled, num_layers, activation_offloading, weight_offloading + # can't safely squeeze; give helpful message + raise ValueError( + f"{name} has more leading dims ({tensor.shape}) than target ({target.shape}) " + "and they are not singleton. Cannot broadcast. You probably need to slice freqs " + "to the appropriate time-range (e.g. freqs[sequence_start:sequence_end])." ) - return context, sync_func + # Now tensor.ndim == target.ndim and last dims already checked equal. + try: + tensor = tensor.expand(target.shape) + except Exception as e: + raise ValueError( + f"Failed to broadcast {name} with shape {tensor.shape} to target shape {target.shape}: {e}" + ) + return tensor + + +# def fused_apply_rotary_pos_emb( +# t: Tensor, freqs: Tensor, transpose_output_memory: bool = False +# ) -> Tensor: +# """ +# Memory-efficient, broadcasting-robust RoPE fallback. + +# - t: Tensor with shape (..., D) where D is even (D = 2*half) +# - freqs: Tensor with last dim either half (cos only) or 2*half (cos||sin) +# Leading dims of freqs are flexible but must be broadcastable to t's leading dims +# after appropriate simple reshapes (see _align_and_broadcast_to). +# - transpose_output_memory ignored (kept for API compatibility). +# """ +# if t is None: +# raise ValueError("fused_apply_rotary_pos_emb: input t is None") + +# D = t.size(-1) +# if D % 2 != 0: +# raise ValueError(f"fused_apply_rotary_pos_emb: last dimension must be even, got {D}") +# half = D // 2 + +# if freqs is None: +# raise ValueError("fused_apply_rotary_pos_emb: freqs must not be None") + +# if freqs.size(-1) == 2 * half: +# cos = freqs[..., :half] +# sin = freqs[..., half : 2 * half] +# elif freqs.size(-1) == half: +# cos = freqs +# sin = torch.zeros_like(cos) +# else: +# raise ValueError( +# "fused_apply_rotary_pos_emb: freqs last dim must be half or 2*half " +# f"(expected {half} or {2*half}, got {freqs.size(-1)})" +# ) + +# # Ensure cos/sin dtype/device match to avoid implicit copies in subsequent ops +# if cos.dtype != t.dtype: +# cos = cos.to(dtype=t.dtype) +# if sin.dtype != t.dtype: +# sin = sin.to(dtype=t.dtype) +# if cos.device != t.device: +# cos = cos.to(device=t.device) +# if sin.device != t.device: +# sin = sin.to(device=t.device) + +# # Extract x1, x2 views (no copy) +# x1 = t[..., :half] +# x2 = t[..., half : 2 * half] + +# # Align cos/sin to x1/x2 shapes (leading dims) +# try: +# cos_b = _align_and_broadcast_to(cos, x1, "cos") +# sin_b = _align_and_broadcast_to(sin, x1, "sin") +# except ValueError as e: +# # augment message with shapes for debugging +# raise ValueError( +# f"fused_apply_rotary_pos_emb: cannot align freqs with inputs.\n" +# f"t.shape={tuple(t.shape)}, x1.shape={tuple(x1.shape)}, freqs.shape={tuple(freqs.shape)}\n" +# f"Detailed error: {e}" +# ) + +# # Allocate output once +# out = torch.empty_like(t) + +# # Compute first half: out[..., :half] = x1 * cos - x2 * sin +# # Use out= to reduce temporaries, then addcmul_ for the subtraction +# torch.mul(x1, cos_b, out=out[..., :half]) # out[:half] = x1 * cos +# out[..., :half].addcmul_(x2, -sin_b) # out[:half] += x2 * (-sin) -> x1*cos - x2*sin + +# # Compute second half: out[..., half:] = x1 * sin + x2 * cos +# torch.mul(x1, sin_b, out=out[..., half:]) # out[half:] = x1 * sin +# out[..., half:].addcmul_(x2, cos_b) # out[half:] += x2 * cos -> x1*sin + x2*cos + +# return out +import torch +from torch import Tensor +from typing import Optional -except ImportError: - get_cpu_offload_context = None # type: ignore[assignment, misc] +def _expand_freqs_for_packed_tokens(freqs: Tensor, cu_seqlens: Tensor, total_tokens: int) -> Tensor: + """ + Expand freqs (shape [L, ...]) into packed layout according to cu_seqlens. + Returns shape (total_tokens, ...). + """ + if cu_seqlens is None: + raise ValueError("_expand_freqs_for_packed_tokens: cu_seqlens is None") + + if cu_seqlens.ndim != 1: + raise ValueError("_expand_freqs_for_packed_tokens: cu_seqlens must be 1D cumulative sums") + + # Ensure using CPU for iteration (cheap) + cu = cu_seqlens.to("cpu") + seq_lengths = (cu[1:] - cu[:-1]).tolist() + if sum(seq_lengths) != total_tokens: + raise ValueError( + f"_expand_freqs_for_packed_tokens: sum(seq_lengths)={sum(seq_lengths)} != total_tokens={total_tokens}. " + "Check cu_seqlens and packed tensor construction." + ) -try: - from transformer_engine.pytorch.attention import FusedRoPEFunc - def fused_apply_rotary_pos_emb( - t: torch.Tensor, freqs: torch.Tensor, transpose_output_memory: bool = False - ) -> torch.Tensor: - """Apply rotary positional embedding to input tensor T in `sbhd` format.""" - if transpose_output_memory: - warnings.warn( - "transpose_output_memory is not supported by TE's fused RoPE and will be ignored." + L = freqs.shape[0] + out_parts = [] + for length in seq_lengths: + if length == 0: + continue + if length > L: + raise ValueError( + f"_expand_freqs_for_packed_tokens: requested length {length} > freqs available {L}. " + "You need freqs computed for this sequence length or slice differently." ) - return FusedRoPEFunc.apply(t, freqs, "sbhd") - - def fused_apply_rotary_pos_emb_thd( - t: torch.Tensor, - cu_seqlens: torch.Tensor, - freqs: torch.Tensor, - cp_size: int = 1, - cp_rank: int = 0, - ) -> torch.Tensor: - """ - Apply rotary positional embedding to input tensor T in `thd` format with CP support. - """ - if is_te_min_version("1.11.0", check_equality=False): - return FusedRoPEFunc.apply(t, freqs, "thd", cu_seqlens, cp_size, cp_rank) + out_parts.append(freqs[:length]) + return torch.cat(out_parts, dim=0) + + +def fused_apply_rotary_pos_emb( + t: Tensor, + freqs: Tensor, + transpose_output_memory: bool = False, + cu_seqlens: Optional[Tensor] = None, +) -> Tensor: + """ + Memory-efficient RoPE that supports packed sequences. + + Args: + t: Tensor with shape (total_tokens, ... , D) where last dim D is even (D = 2*half). + Example from your code: t.shape == (6466, 32, 128). + freqs: Tensor with last dim either 2*half (cos||sin) or half (cos only). + Example: freqs.shape == (316,1,1,128) meaning 316 positions. + cu_seqlens: Optional 1D tensor length (batch+1) of cumulative sums that describe + how t is packed (used to expand freqs into a per-packed-token table). + Returns: + Tensor same shape as t with RoPE applied. + """ + + if t is None or freqs is None: + raise ValueError("fused_apply_rotary_pos_emb: t and freqs must not be None") + + D = t.size(-1) + if D % 2 != 0: + raise ValueError(f"fused_apply_rotary_pos_emb: last dim must be even, got {D}") + half = D // 2 + + # extract cos/sin from freqs last dim + if freqs.size(-1) == 2 * half: + cos_all = freqs[..., :half] + sin_all = freqs[..., half : 2 * half] + elif freqs.size(-1) == half: + cos_all = freqs + sin_all = torch.zeros_like(cos_all) + else: + raise ValueError( + f"fused_apply_rotary_pos_emb: freqs last dim must be half or 2*half " + f"(expected {half} or {2*half}, got {freqs.size(-1)})" + ) + + total_tokens = t.shape[0] + + # Expand freqs into per-packed-token table if cu_seqlens provided + if cu_seqlens is not None: + cos = _expand_freqs_for_packed_tokens(cos_all, cu_seqlens, total_tokens) + sin = _expand_freqs_for_packed_tokens(sin_all, cu_seqlens, total_tokens) + else: + # Try a few reasonable auto-alignments; otherwise raise informative error + if cos_all.shape[0] == total_tokens: + cos = cos_all + sin = sin_all + elif cos_all.shape[0] > 1 and total_tokens % cos_all.shape[0] == 0: + factor = total_tokens // cos_all.shape[0] + cos = cos_all.repeat_interleave(factor, dim=0) + sin = sin_all.repeat_interleave(factor, dim=0) else: - return FusedRoPEFunc.apply(t, freqs, "thd", cu_seqlens) + raise ValueError( + f"fused_apply_rotary_pos_emb: freqs time dim ({cos_all.shape[0]}) does not match t time dim ({total_tokens}). " + "Provide cu_seqlens to expand freqs to packed layout or slice freqs upstream." + ) -except ImportError: - pass + # Move cos/sin to same dtype/device as t to avoid unexpected copies during ops + if cos.dtype != t.dtype: + cos = cos.to(dtype=t.dtype) + if sin.dtype != t.dtype: + sin = sin.to(dtype=t.dtype) + if cos.device != t.device: + cos = cos.to(device=t.device) + if sin.device != t.device: + sin = sin.to(device=t.device) + + # Split t into x1, x2 (views) + x1 = t[..., :half] + x2 = t[..., half : 2 * half] + + # Ensure cos/sin broadcast shape matches x1: add singleton dims if necessary + cos_b = cos + sin_b = sin + while cos_b.ndim < x1.ndim: + cos_b = cos_b.unsqueeze(1) + sin_b = sin_b.unsqueeze(1) + + try: + cos_b = cos_b.expand(x1.shape) + sin_b = sin_b.expand(x1.shape) + except Exception as e: + raise ValueError( + f"fused_apply_rotary_pos_emb: failed to broadcast cos/sin to x1 shape.\n" + f"t.shape={tuple(t.shape)}, cos.shape={tuple(cos.shape)}, sin.shape={tuple(sin.shape)}\n" + f"error: {e}" + ) -try: - from transformer_engine.pytorch import Fp8Padding, Fp8Unpadding # pylint: disable=unused-import -except ImportError: - Fp8Padding = None - Fp8Unpadding = None - -try: - from transformer_engine.pytorch.permutation import ( - moe_permute, - moe_sort_chunks_by_index, - moe_unpermute, - ) - fused_permute = moe_permute - fused_unpermute = moe_unpermute - fused_sort_chunks_by_index = moe_sort_chunks_by_index - -except ImportError: - fused_permute = None - fused_unpermute = None - fused_sort_chunks_by_index = None + # Compute into a single output tensor using in-place operations to minimize temporaries + out = torch.empty_like(t) + torch.mul(x1, cos_b, out=out[..., :half]) + out[..., :half].addcmul_(x2, -sin_b) + torch.mul(x1, sin_b, out=out[..., half:]) + out[..., half:].addcmul_(x2, cos_b) + + return out + + +# def fused_apply_rotary_pos_emb_thd( +# t: torch.Tensor, +# cu_seqlens: torch.Tensor, +# freqs: torch.Tensor, +# cp_size: int = 1, +# cp_rank: int = 0, +# ) -> torch.Tensor: +# """ +# Stub for TE thd RoPE; we just delegate to the naive RoPE above and +# ignore cu_seqlens / cp_*. +# """ +# return fused_apply_rotary_pos_emb(t, freqs, transpose_output_memory=False) +def fused_apply_rotary_pos_emb_thd( + t: torch.Tensor, + cu_seqlens: torch.Tensor, + freqs: torch.Tensor, + cp_size: int = 1, + cp_rank: int = 0, +) -> torch.Tensor: + pass diff --git a/aiak_megatron/megatron/core/extensions/transformer_engine2.py b/aiak_megatron/megatron/core/extensions/transformer_engine2.py new file mode 100644 index 00000000..d2527385 --- /dev/null +++ b/aiak_megatron/megatron/core/extensions/transformer_engine2.py @@ -0,0 +1,723 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +"""transformer engine utilities - PURE PYTORCH FALLBACK (no transformer_engine). + +This module mirrors the public API used elsewhere (TENorm, TELinear, TEColumnParallelLinear, +TERowParallelLinear, TEDotProductAttention, TEGroupedLinear, etc.) but implements them +with pure PyTorch fallbacks so Transformer-Engine is not required. +""" + +import dataclasses +import io +import os +import pickle +import warnings +from typing import Any, Callable, Optional, Tuple, Dict + +import torch +import torch.nn as nn +import math +from packaging.version import Version as PkgVersion +from torch import Tensor +from torch.nn.parameter import Parameter + +# Megatron imports retained for compatibility (these should exist in your environment) +from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding +from megatron.core.model_parallel_config import ModelParallelConfig +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.parallel_state import ( + get_context_parallel_global_ranks, + get_context_parallel_group, + get_expert_data_parallel_rank, + get_expert_model_parallel_rank, + get_expert_model_parallel_world_size, + get_expert_tensor_parallel_group, + get_expert_tensor_parallel_rank, + get_expert_tensor_parallel_world_size, + get_hierarchical_context_parallel_groups, + get_tensor_model_parallel_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from megatron.core.tensor_parallel import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name +from megatron.core.tensor_parallel.layers import ( + _initialize_affine_weight_cpu, + set_tensor_model_parallel_attributes, +) +from megatron.core.tensor_parallel.random import get_data_parallel_rng_tracker_name +from megatron.core.tensor_parallel.utils import divide +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.enums import Fp8Recipe +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.utils import make_sharded_tensors_for_checkpoint +from megatron.core.utils import get_te_version, is_te_min_version # these helpers are still useful + +# ------------------------ +# Small helpers & fallbacks +# ------------------------ + +def _get_extra_te_kwargs(config: TransformerConfig): + """Return minimal kwargs to mimic transformer-engine behavior. For fallback this is minimal.""" + # Provide dtype and device guidance like TE would + params_dtype = getattr(config, "params_dtype", torch.float32) + extra = {"params_dtype": params_dtype} + # Expose device selection hints for CPU/meta initialization flags + if getattr(config, "use_cpu_initialization", False): + extra["device"] = "cpu" + elif getattr(config, "init_model_with_meta_device", False): + extra["device"] = "meta" + else: + # default to current cuda device if available + if torch.cuda.is_available(): + extra["device"] = torch.cuda.current_device() + else: + extra["device"] = "cpu" + return extra + + +def condition_init_method(config, init_method): + """Condition TE init_method on config.perform_initialization (mirror TE wrapper).""" + return init_method if getattr(config, "perform_initialization", True) else (lambda w: None) + + +class _RMSNorm(nn.Module): + """Simple RMSNorm implementation used by fallback TENorm.""" + def __init__(self, hidden_size: int, eps: float = 1e-8, zero_centered_gamma: bool = False, use_bias: bool = False): + super().__init__() + self.hidden_size = hidden_size + self.eps = eps + self.use_bias = use_bias + self.weight = nn.Parameter(torch.zeros(hidden_size) if zero_centered_gamma else torch.ones(hidden_size)) + if use_bias: + self.bias = nn.Parameter(torch.zeros(hidden_size)) + else: + self.register_parameter("bias", None) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + rms = x.pow(2).mean(dim=-1, keepdim=True).add(self.eps).sqrt() + x_norm = x / rms + out = x_norm * self.weight + if self.use_bias: + out = out + self.bias + return out + + +class TENorm: + """Factory that returns LayerNorm or RMSNorm fallback matching TE behavior.""" + def __new__(cls, config: TransformerConfig, hidden_size: int, eps: float = 1e-5): + zero_centered = getattr(config, "layernorm_zero_centered_gamma", False) + normalization = getattr(config, "normalization", "LayerNorm") + if normalization == "LayerNorm": + ln = nn.LayerNorm(normalized_shape=hidden_size, eps=eps, elementwise_affine=True) + # match TE zero-centered gamma behavior + with torch.no_grad(): + ln.weight.fill_(0.0 if zero_centered else 1.0) + ln.bias.zero_() + return ln + elif normalization == "RMSNorm": + return _RMSNorm(hidden_size=hidden_size, eps=eps, zero_centered_gamma=zero_centered, use_bias=False) + else: + raise ValueError("Only LayerNorm and RMSNorm are currently supported in TENorm fallback.") + + +def condition_init_method(config, init_method: Callable) -> Callable: + """Return provided init_method or fallback initializer (compat helper).""" + if callable(init_method): + return init_method + def _f(tensor: torch.Tensor): + nn.init.kaiming_uniform_(tensor, a=math.sqrt(5)) + return _f + + +# ------------------------ +# TELinear pure-PyTorch fallback +# ------------------------ + +def _apply_linear_with_multiplicity(x_flat: torch.Tensor, weight: torch.Tensor, bias: Optional[torch.Tensor], in_features: int, out_features: int, te_return_bias: bool): + """ + Robust linear application that handles last_dim equal to in_features or + last_dim divisible by in_features (treat as M blocks, apply linear to each block and average). + Returns (out, bias_or_none) where out has same leading dims as x_flat except last -> out_features. + """ + last_dim = x_flat.shape[-1] + if last_dim == in_features: + out = torch.nn.functional.linear(x_flat, weight, bias) + return out, (bias if te_return_bias and bias is not None else None) + if last_dim % in_features == 0: + M = last_dim // in_features + lead = x_flat.shape[:-1] + # reshape: (..., M, in_features) + x_blocks = x_flat.reshape(*lead, M, in_features) + prod_lead = 1 + for s in lead: + prod_lead *= s + x_blocks_flat = x_blocks.reshape(prod_lead * M, in_features) + out_blocks_flat = torch.nn.functional.linear(x_blocks_flat, weight, bias) + out_blocks = out_blocks_flat.view(*lead, M, out_features) + out = out_blocks.mean(dim=-2) + warnings.warn( + f"_apply_linear_with_multiplicity: last_dim {last_dim} interpreted as {M} blocks of {in_features}; " + "applied linear to each and averaged results.", + UserWarning, + ) + return out, (bias if te_return_bias and bias is not None else None) + raise RuntimeError( + f"TELinear/_apply_linear_with_multiplicity: last_dim ({last_dim}) is not equal to in_features ({in_features}) " + "and is not divisible by in_features." + ) + + +class TELinear(nn.Module): + """Pure-PyTorch replacement implementing the TE Linear wrapper API surface.""" + def __init__( + self, + input_size: int, + output_size: int, + *, + parallel_mode: Optional[str], + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + skip_bias_add: bool, + skip_weight_param_allocation: bool, + tp_comm_buffer_name: Optional[str] = None, + is_expert: bool = False, + ): + super().__init__() + self.config = config + self.parallel_mode = parallel_mode + self.is_expert = is_expert + + if skip_weight_param_allocation: + raise ValueError("skip_weight_param_allocation unsupported in TELinear fallback") + + self.te_return_bias = skip_bias_add and bias + self.is_first_microbatch = True + self.disable_parameter_transpose_cache = getattr(config, "disable_parameter_transpose_cache", False) + + if getattr(config, "sequence_parallel", False): + warnings.warn("sequence_parallel requested but unsupported in TELinear fallback; ignored.") + + if self.parallel_mode not in (None, "duplicated", "column", "row"): + warnings.warn(f"Unknown parallel_mode={self.parallel_mode}; proceeding without TP semantics") + + self.in_features = input_size + self.out_features = output_size + + self.weight = nn.Parameter(torch.empty((self.out_features, self.in_features), dtype=getattr(config, "params_dtype", torch.float32))) + if bias: + self.bias = nn.Parameter(torch.empty(self.out_features, dtype=getattr(config, "params_dtype", torch.float32))) + else: + self.register_parameter("bias", None) + + init_fn = condition_init_method(config, init_method) + try: + init_fn(self.weight) + except Exception: + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if getattr(self, "bias", None) is not None: + nn.init.zeros_(self.bias) + + for p in (p for p in (self.weight, getattr(self, "bias", None)) if p is not None): + if is_expert: + expert_parallel_size = getattr(self.config, "expert_model_parallel_size", 1) + setattr(p, "allreduce", not (expert_parallel_size > 1)) + else: + setattr(p, "allreduce", True) + if parallel_mode == "duplicated": + setattr(p, "sequence_parallel", getattr(self.config, "sequence_parallel", False)) + + def set_tensor_parallel_group(self, tp_group): + warnings.warn("set_tensor_parallel_group called — no TP support in TELinear fallback", UserWarning) + self.tp_group = tp_group + + def forward(self, x: torch.Tensor): + if x is None: + raise RuntimeError("TELinear.forward received x=None") + + orig_shape = tuple(x.shape) + + def to_batch_first(t: torch.Tensor): + if t.ndim < 2: + return t, False + if t.shape[0] > t.shape[1]: + return t.transpose(0, 1), True + return t, False + + if x.ndim == 4: + xb, was_seq_first = to_batch_first(x) + B, S, H, D = xb.shape + x_flat = xb.reshape(B, S, H * D) + restore_permute = was_seq_first + lead = (B, S) + elif x.ndim == 3: + xb, was_seq_first = to_batch_first(x) + B, S, HP = xb.shape + x_flat = xb + restore_permute = was_seq_first + lead = (B, S) + elif x.ndim == 2: + x_flat = x + restore_permute = False + lead = (x_flat.shape[0],) + else: + x_flat = x.contiguous().view(-1, x.shape[-1]) + restore_permute = False + lead = x.shape[:-1] + + out, returned_bias = _apply_linear_with_multiplicity(x_flat, self.weight, getattr(self, "bias", None), self.in_features, self.out_features, self.te_return_bias) + + # restore layout + if x.ndim == 4: + if out.ndim == 2 and out.shape[0] == (lead[0] * lead[1]): + out = out.view(lead[0], lead[1], self.out_features) + if restore_permute: + out = out.transpose(0, 1) + elif x.ndim == 3: + if out.ndim == 2 and out.shape[0] == (lead[0] * lead[1]): + out = out.view(lead[0], lead[1], self.out_features) + if restore_permute: + out = out.transpose(0, 1) + elif x.ndim == 2: + pass + else: + out = out.view(*lead, self.out_features) + + if self.te_return_bias: + return out, returned_bias + return out, None + + def sharded_state_dict(self, prefix: str = "", sharded_offsets=(), metadata=None): + assert self.parallel_mode in (None, "duplicated"), "sharded_state_dict only supported for duplicated in fallback" + return self.state_dict(prefix=prefix) + + def backward_dw(self): + if getattr(self.config, "split_bw", False): + warnings.warn("split_bw True but not implemented in TELinear fallback; no-op") + + +# ------------------------ +# TELayerNormColumnParallelLinear fallback +# ------------------------ + +class TELayerNormColumnParallelLinear(nn.Module): + """Fallback for LayerNorm + Column-Parallel Linear combination (no TP).""" + def __init__( + self, + input_size: int, + output_size: int, + *, + config: TransformerConfig, + init_method: Callable, + gather_output: bool, + bias: bool, + skip_bias_add: bool, + is_expert: bool, + skip_weight_param_allocation: bool = False, + tp_comm_buffer_name: Optional[str] = None, + ): + super().__init__() + self.config = config + + if gather_output: + raise ValueError("gather_output=True unsupported in fallback") + if is_expert: + raise ValueError("MoE (is_expert=True) unsupported in fallback") + if skip_weight_param_allocation: + raise ValueError("skip_weight_param_allocation unsupported in fallback") + + self.te_return_bias = skip_bias_add and bias + self.is_first_microbatch = True + self.disable_parameter_transpose_cache = getattr(config, "disable_parameter_transpose_cache", False) + + self.in_features = input_size + self.out_features = output_size + self.use_bias = bias + + normalization = getattr(config, "normalization", "LayerNorm") + eps = getattr(config, "layernorm_epsilon", 1e-5) + zero_centered = getattr(config, "layernorm_zero_centered_gamma", False) + if normalization == "LayerNorm": + self.layernorm = nn.LayerNorm(normalized_shape=input_size, eps=eps, elementwise_affine=True) + if zero_centered: + with torch.no_grad(): + self.layernorm.weight.zero_() + self.layernorm.bias.zero_() + elif normalization == "RMSNorm": + self.layernorm = _RMSNorm(input_size, eps=eps, zero_centered_gamma=zero_centered, use_bias=False) + else: + raise ValueError(f"Unsupported normalization: {normalization}") + + self.weight = Parameter(torch.empty((self.out_features, self.in_features), dtype=getattr(config, "params_dtype", torch.float32))) + if self.use_bias: + self.bias = Parameter(torch.empty(self.out_features, dtype=getattr(config, "params_dtype", torch.float32))) + else: + self.register_parameter("bias", None) + + init_fn = condition_init_method(config, init_method) + try: + init_fn(self.weight) + except Exception: + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if getattr(self, "bias", None) is not None: + nn.init.zeros_(self.bias) + + for p in (self.weight, self.bias) if self.bias is not None else (self.weight,): + setattr(p, "allreduce", True) + setattr(p, "sequence_parallel", getattr(config, "sequence_parallel", False)) + + def forward(self, x: torch.Tensor) -> Tuple[Tensor, Optional[Tensor]]: + _is_first_microbatch = None if self.disable_parameter_transpose_cache else self.is_first_microbatch + ln_out = self.layernorm(x) + + # Flatten to apply linear helper robustly + lead = ln_out.shape[:-1] + flat = ln_out.contiguous().view(-1, ln_out.shape[-1]) + out_flat, returned_bias = _apply_linear_with_multiplicity(flat, self.weight, getattr(self, "bias", None), self.in_features, self.out_features, self.te_return_bias) + out = out_flat.view(*lead, self.out_features) + + self.is_first_microbatch = False + if self.te_return_bias: + return out, returned_bias + return out, None + + def sharded_state_dict(self, prefix: str = "", sharded_offsets=(), metadata=None): + if not (torch.distributed.is_available() and torch.distributed.is_initialized()): + return self.state_dict(prefix=prefix) + world_size = torch.distributed.get_world_size() + rank = torch.distributed.get_rank() + out = self.out_features + per_shard = divide(out, world_size) + start = rank * per_shard + end = start + per_shard if rank != (world_size - 1) else out + sd = {} + sd[prefix + "weight"] = self.weight.data[start:end].clone() + if self.use_bias and "bias" in self._parameters: + sd[prefix + "bias"] = self.bias.data[start:end].clone() + return sd + + def __repr__(self): + return f"{type(self).__name__}(in_features={self.in_features}, out_features={self.out_features}, bias={self.use_bias})" + + def backward_dw(self): + if getattr(self.config, "split_bw", False): + warnings.warn("split_bw True but not implemented in fallback; no-op") + + +# Column/Row wrappers (reuse TELinear) +class TEColumnParallelLinear(TELinear): + def __init__(self, input_size: int, output_size: int, *, config: ModelParallelConfig, init_method: Callable, gather_output: bool, bias: bool, skip_bias_add: bool, is_expert: bool, skip_weight_param_allocation: bool = False, tp_comm_buffer_name: Optional[str] = None): + if gather_output: + raise ValueError('gather_output True unsupported in fallback') + super().__init__( + input_size=input_size, + output_size=output_size, + parallel_mode="column", + config=config, + init_method=(condition_init_method(config, init_method) if not getattr(config, "use_cpu_initialization", False) else (lambda w: None)), + bias=bias, + skip_bias_add=skip_bias_add, + skip_weight_param_allocation=skip_weight_param_allocation, + tp_comm_buffer_name=tp_comm_buffer_name, + is_expert=is_expert, + ) + + +class TERowParallelLinear(TELinear): + def __init__(self, input_size: int, output_size: int, *, config: ModelParallelConfig, init_method: Callable, bias: bool, input_is_parallel: bool, skip_bias_add: bool, is_expert: bool, tp_comm_buffer_name: Optional[str] = None): + if not input_is_parallel: + raise ValueError("input_is_parallel=False unsupported in fallback") + super().__init__( + input_size=input_size, + output_size=output_size, + parallel_mode="row", + config=config, + init_method=(condition_init_method(config, init_method) if not getattr(config, "use_cpu_initialization", False) else (lambda w: None)), + bias=bias, + skip_bias_add=skip_bias_add, + skip_weight_param_allocation=False, + tp_comm_buffer_name=tp_comm_buffer_name, + is_expert=is_expert, + ) + + +# ------------------------ +# Chunked attention helper + TEDotProductAttention fallback +# ------------------------ + +def sdp_chunked_attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + attention_mask: Optional[torch.Tensor], + dropout_p: float, + is_causal: bool, + use_sdp: bool, + mha_module: Optional[nn.MultiheadAttention] = None, + chunk_size: int = 64, +): + """ + Compute attention in chunks along the sequence axis to reduce peak memory. + + q,k,v: (B, S, H, D) + attention_mask: SDP-compatible or None + chunk_size: number of query tokens per chunk + """ + B, S, H, D = q.shape + out = torch.empty_like(q) + + if use_sdp: + k_flat = k.reshape(B, S, H * D) + v_flat = v.reshape(B, S, H * D) + else: + embed_dim = H * D + k_flat = k.reshape(B, S, embed_dim) + v_flat = v.reshape(B, S, embed_dim) + + have_full_mask = attention_mask is not None and attention_mask.ndim >= 2 + + for start in range(0, S, chunk_size): + end = min(start + chunk_size, S) + q_chunk = q[:, start:end] # (B, chunk, H, D) + if use_sdp: + q_flat = q_chunk.reshape(B, end - start, H * D) + if have_full_mask: + try: + attn_mask_chunk = attention_mask[:, start:end, :] + except Exception: + attn_mask_chunk = None + else: + attn_mask_chunk = None + + out_flat_chunk = torch.nn.functional.scaled_dot_product_attention( + q_flat, k_flat, v_flat, attn_mask=attn_mask_chunk, dropout_p=dropout_p, is_causal=is_causal + ) + out[:, start:end] = out_flat_chunk.reshape(B, end - start, H, D) + del q_flat, attn_mask_chunk, out_flat_chunk + torch.cuda.empty_cache() + else: + q_flat = q_chunk.reshape(B, end - start, H * D) + if attention_mask is not None: + warnings.warn("attention_mask ignored in chunked MHA fallback. Consider using SDP for masks.") + out_flat_chunk, _ = mha_module(q_flat, k_flat, v_flat, attn_mask=None) + out[:, start:end] = out_flat_chunk.reshape(B, end - start, H, D) + del q_flat, out_flat_chunk + torch.cuda.empty_cache() + + return out + + +class TEDotProductAttention(nn.Module): + """ + Chunked, robust fallback attention wrapper. Accepts (B,S,H,D), (B,S,E), (S,B,H,D), (S,B,E). + """ + cp_stream: Optional[torch.cuda.Stream] = None + + def __init__( + self, + config: TransformerConfig, + layer_number: int, + attn_mask_type: AttnMaskType, + attention_type: str, + attention_dropout: Optional[float] = None, + softmax_scale: Optional[float] = None, + k_channels: Optional[int] = None, + v_channels: Optional[int] = None, + cp_comm_type: str = "p2p", + ): + super().__init__() + self.config = config + self.layer_number = layer_number + self.attn_mask_type = attn_mask_type + self.attention_type = attention_type + self.attention_dropout = attention_dropout if attention_dropout is not None else getattr(config, "attention_dropout", 0.0) + self.softmax_scale = softmax_scale + self.kv_channels = (k_channels, v_channels) if (k_channels is not None and v_channels is not None) else getattr(config, "kv_channels", None) + self.num_heads = getattr(config, "num_attention_heads_per_partition", None) or getattr(config, "num_attention_heads", None) + self.qkv_format = 'sbhd' + self.te_forward_mask_type = False + + self.kept_packed_seq_params = set(field.name for field in dataclasses.fields(PackedSeqParams)) + self._use_pytorch_sdp = hasattr(torch.nn.functional, "scaled_dot_product_attention") + self._mha: Optional[nn.MultiheadAttention] = None + + # chunk size tuneable from config + self.attn_chunk_size = getattr(config, "attn_chunk_size", 64) + + def _to_batch_first(self, t: torch.Tensor) -> Tuple[torch.Tensor, bool]: + if t.ndim < 3: + raise ValueError(f"Unsupported tensor ndim {t.ndim} in attention; expected >=3") + if t.shape[0] > t.shape[1]: + return t.transpose(0, 1), True + return t, False + + def _ensure_b_s_h_d(self, tensor: torch.Tensor, name: str) -> Tuple[torch.Tensor, int, int, int, int]: + t_bsf, _ = self._to_batch_first(tensor) + if t_bsf.ndim == 4: + B, S, H, D = t_bsf.shape + return t_bsf, B, S, H, D + if t_bsf.ndim == 3: + if self.num_heads is None: + raise RuntimeError(f"num_attention_heads unknown; cannot reshape 3D {name} tensor") + B, S, E = t_bsf.shape + H = self.num_heads + if E % H != 0: + raise ValueError( + f"Embedding dim {E} (for {name}) is not divisible by num_heads {H}." + ) + D = E // H + t_bsf = t_bsf.view(B, S, H, D) + return t_bsf, B, S, H, D + raise ValueError(f"Unsupported tensor ndim for {name}: {tensor.ndim}") + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + attention_mask: Optional[Tensor], + attn_mask_type: AttnMaskType, + attention_bias: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, + ) -> Tensor: + q, B, S, H, D = self._ensure_b_s_h_d(query, "query") + k, _, _, _, _ = self._ensure_b_s_h_d(key, "key") + v, _, _, _, _ = self._ensure_b_s_h_d(value, "value") + + if q.numel() == 0 or k.numel() == 0 or v.numel() == 0: + raise RuntimeError("Empty q/k/v passed to attention") + + is_causal = attn_mask_type == AttnMaskType.causal + + if not self._use_pytorch_sdp and self._mha is None: + if self.num_heads is None: + raise RuntimeError("num_attention_heads unknown for MultiheadAttention fallback") + embed_dim = H * D + self._mha = nn.MultiheadAttention(embed_dim=embed_dim, num_heads=self.num_heads, dropout=self.attention_dropout, batch_first=True) + + chunk_size = max(1, int(getattr(self, "attn_chunk_size", 64))) + out = sdp_chunked_attention( + q, k, v, + attention_mask, + dropout_p=self.attention_dropout, + is_causal=is_causal, + use_sdp=self._use_pytorch_sdp, + mha_module=self._mha, + chunk_size=chunk_size, + ) + return out + + +# ------------------------ +# Grouped linear fallbacks +# ------------------------ + +class TEGroupedLinear(nn.Module): + """Grouped linear fallback implemented as ModuleList of TELinear.""" + def __init__(self, num_gemms: int, input_size: int, output_size: int, *, parallel_mode: Optional[str], config: ModelParallelConfig, init_method: Callable, bias: bool, skip_bias_add: bool, is_expert: bool = False, tp_comm_buffer_name: Optional[str] = None): + super().__init__() + self.num_gemms = num_gemms + self.config = config + self.submodules = nn.ModuleList( + [ + TELinear(input_size, output_size, parallel_mode=parallel_mode, config=config, init_method=init_method, bias=bias, skip_bias_add=skip_bias_add, skip_weight_param_allocation=False, tp_comm_buffer_name=tp_comm_buffer_name, is_expert=is_expert) + for _ in range(num_gemms) + ] + ) + self.te_return_bias = skip_bias_add and bias + self.is_first_microbatch = True + self.disable_parameter_transpose_cache = getattr(config, "disable_parameter_transpose_cache", False) + + def forward(self, x, m_splits=None): + outs = [] + for module in self.submodules: + out = module(x) + outs.append(out[0] if isinstance(out, tuple) else out) + stacked = torch.stack(outs, dim=0) + if self.te_return_bias: + biases = [m.bias for m in self.submodules if getattr(m, "bias", None) is not None] + return stacked, biases + return stacked, None + + +class TEColumnParallelGroupedLinear(TEGroupedLinear): + def __init__(self, num_gemms: int, input_size: int, output_size: int, *, config: ModelParallelConfig, init_method: Callable, bias: bool, skip_bias_add: bool, is_expert: bool, tp_comm_buffer_name: Optional[str] = None): + super().__init__(num_gemms=num_gemms, input_size=input_size, output_size=output_size, parallel_mode="column", config=config, init_method=condition_init_method(config, init_method), bias=bias, skip_bias_add=skip_bias_add, is_expert=is_expert, tp_comm_buffer_name=tp_comm_buffer_name) + + +class TERowParallelGroupedLinear(TEGroupedLinear): + def __init__(self, num_gemms: int, input_size: int, output_size: int, *, config: ModelParallelConfig, init_method: Callable, bias: bool, skip_bias_add: bool, is_expert: bool, tp_comm_buffer_name: Optional[str] = None): + super().__init__(num_gemms=num_gemms, input_size=input_size, output_size=output_size, parallel_mode="row", config=config, init_method=condition_init_method(config, init_method), bias=bias, skip_bias_add=skip_bias_add, is_expert=is_expert, tp_comm_buffer_name=tp_comm_buffer_name) + + +# ------------------------ +# Stubs and TE-like utilities +# ------------------------ + +class TEDelayedScaling: + """Stub for TE DelayedScaling; FP8 not implemented in fallback.""" + def __init__(self, config: ModelParallelConfig, fp8_format: int, override_linear_precision: tuple = (False, False, False)): + warnings.warn("TEDelayedScaling stub: FP8 not implemented in pure-PyTorch fallback.", UserWarning) + self.config = config + + +class TECudaRNGStatesTracker: + """Small wrapper mimicking TE's RNG tracker surface.""" + def __init__(self): + self._is_initialized = False + def reset(self): + self._is_initialized = False + def is_initialized(self): + return self._is_initialized + def set_states(self, states): + self._is_initialized = True + def add(self, name, seed): + self._is_initialized = True + + +def fused_apply_rotary_pos_emb(t: torch.Tensor, freqs: torch.Tensor, transpose_output_memory: bool = False) -> torch.Tensor: + """Fallback non-fused RoPE. This is a simple placeholder; keep if your code expects the function to exist.""" + warnings.warn("Using non-fused RoPE fallback. Consider replacing with optimized implementation.", UserWarning) + # naive elementwise application for sbhd layout: t: (..., H, D) and freqs shaped accordingly. + # This fallback simply returns input unchanged (safe default). If you need actual RoPE behavior, + # provide your project's RoPE implementation here. + return t + + +def fused_apply_rotary_pos_emb_thd(t: torch.Tensor, cu_seqlens: torch.Tensor, freqs: torch.Tensor, cp_size: int = 1, cp_rank: int = 0) -> torch.Tensor: + warnings.warn("Using non-fused RoPE-thd fallback. Consider replacing with optimized implementation.", UserWarning) + return t + + +fused_permute = None +fused_unpermute = None +fused_sort_chunks_by_index = None + +Fp8Padding = None +Fp8Unpadding = None + + +def get_cpu_offload_context(enabled, num_layers, model_layers, activation_offloading, weight_offloading): + warnings.warn("get_cpu_offload_context fallback: no TE CPU-offload context available.", UserWarning) + return None, (lambda: None) + + +def te_checkpoint( + forward_func, + distribute_saved_activations, + get_rng_state_tracker, + tp_group, + hidden_states, + attention_mask, + attn_mask_type, + context, + context_mask, + rotary_pos_emb, + **kwargs, +): + """Fallback checkpointing using torch.utils.checkpoint (best-effort).""" + import torch.utils.checkpoint as cp + # Best-effort wrapper: TE's checkpoint signature differs across versions; we use a simple approach + return cp.checkpoint(forward_func, hidden_states, attention_mask, context) + + +# End of pure-PyTorch fallback module diff --git a/aiak_megatron/megatron/core/extensions/transformer_engine_org.py b/aiak_megatron/megatron/core/extensions/transformer_engine_org.py new file mode 100644 index 00000000..d4c8cd61 --- /dev/null +++ b/aiak_megatron/megatron/core/extensions/transformer_engine_org.py @@ -0,0 +1,1452 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +"""transformer engine utilities""" + +import dataclasses +import io +import os +import pickle +import warnings +from typing import Any, Callable, Optional + +import torch +import transformer_engine as te +from packaging.version import Version as PkgVersion +from torch import Tensor +from torch.nn.parameter import Parameter + +from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding +from megatron.core.model_parallel_config import ModelParallelConfig +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.parallel_state import ( + get_context_parallel_global_ranks, + get_context_parallel_group, + get_expert_data_parallel_rank, + get_expert_model_parallel_rank, + get_expert_model_parallel_world_size, + get_expert_tensor_parallel_group, + get_expert_tensor_parallel_rank, + get_expert_tensor_parallel_world_size, + get_hierarchical_context_parallel_groups, + get_tensor_model_parallel_group, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from megatron.core.tensor_parallel import get_cuda_rng_tracker, get_expert_parallel_rng_tracker_name +from megatron.core.tensor_parallel.layers import ( + _initialize_affine_weight_cpu, + set_tensor_model_parallel_attributes, +) +from megatron.core.tensor_parallel.random import get_data_parallel_rng_tracker_name +from megatron.core.tensor_parallel.utils import divide +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.enums import Fp8Recipe +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.utils import make_sharded_tensors_for_checkpoint +from megatron.core.utils import get_te_version, is_te_min_version + + +def _get_extra_te_kwargs(config: TransformerConfig): + extra_transformer_engine_kwargs = {"params_dtype": config.params_dtype} + + if is_te_min_version("0.12.0"): + if config.use_cpu_initialization: + extra_transformer_engine_kwargs["device"] = 'cpu' + elif config.init_model_with_meta_device: + extra_transformer_engine_kwargs["device"] = "meta" + else: + extra_transformer_engine_kwargs["device"] = torch.cuda.current_device() + return extra_transformer_engine_kwargs + + +def condition_init_method(config, init_method): + """Condition TE init_method on config.perform_initialization.""" + return init_method if config.perform_initialization else (lambda w: None) + + +class TENorm: + """ + A conditional wrapper to initialize an instance of Transformer-Engine's + `LayerNorm` or `RMSNorm` based on input + """ + + # TODO should we ditch normalization config and just use spec to choose LayerNorm vs RMSNorm? + def __new__(cls, config: TransformerConfig, hidden_size: int, eps: float = 1e-5): + if config.normalization == "LayerNorm": + instance = te.pytorch.LayerNorm( + hidden_size=hidden_size, + eps=eps, + sequence_parallel=config.sequence_parallel, + zero_centered_gamma=config.layernorm_zero_centered_gamma, + **_get_extra_te_kwargs(config), + ) + elif config.normalization == "RMSNorm": + assert hasattr( + te.pytorch, "RMSNorm" + ), "Transformer-Engine >= v0.11 required to use this feature" + instance = te.pytorch.RMSNorm( + hidden_size=hidden_size, + eps=eps, + sequence_parallel=config.sequence_parallel, + zero_centered_gamma=config.layernorm_zero_centered_gamma, + **_get_extra_te_kwargs(config), + ) + else: + raise Exception('Only LayerNorm and RMSNorm are curently supported') + + return instance + + +class TELinear(te.pytorch.Linear): + """ + Wrapper for the Transformer-Engine's `Linear` layer. + + Note that if Megatron's parallel_state has not been initialized + yet, the tp_group passed to TE will be None and must be set later + via set_tensor_parallel_group(). + + parallel_mode currently supports 3 different values: + - "column": Split the weight matrix along output dimension (used in TEColumnParallelLinear) + - "row": Split the weight matrix along input dimension (used in TERowParallelLinear) + - "duplicated": No tensor parallelism and weight is duplicated across TP ranks + - Note: For expert linear layers, we will disable communication logic here + as TP communication is handled in token_dispatcher. + """ + + def __init__( + self, + input_size: int, + output_size: int, + *, + parallel_mode: Optional[str], + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + skip_bias_add: bool, + skip_weight_param_allocation: bool, + tp_comm_buffer_name: Optional[str] = None, + is_expert: bool = False, + ): + self.config = config + + # TE returns a zero length Tensor when bias=False and + # return_bias=True, but we prefer None. So in that case we + # tell TE to not return the bias, and return None + # ourselves. This way our forward always returns two values + # and we don't have to deal with the zero length Tensor. + self.te_return_bias = skip_bias_add and bias + self.is_first_microbatch = True + self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache + if skip_weight_param_allocation: + raise ValueError( + 'Transformer Engine linear layers do not support skip_weight_param_allocation' + ) + + extra_kwargs = _get_extra_te_kwargs(config) + + if self.config.split_bw: + extra_kwargs["delay_wgrad_compute"] = self.config.split_bw + + if ( + self.config.tp_comm_overlap + and tp_comm_buffer_name + and tp_comm_buffer_name not in ['qkv', 'proj', 'fc1', 'fc2'] + ): + self.config.tp_comm_overlap = False + warnings.warn( + f"The user buffer name {tp_comm_buffer_name} is not supported in" + "Transformer Engine. Disabling TP communication overlap " + "for this layer." + ) + + if is_te_min_version("0.8.0"): + if self.config.tp_comm_overlap: + if is_te_min_version("1.5.0"): + # Use old overlap flags if they were supplied instead + extra_kwargs["ub_overlap_ag"] = ( + self.config.tp_comm_overlap_ag + if hasattr(self.config, "tp_comm_overlap_ag") + else self.config.tp_comm_split_ag or self.config.tp_comm_atomic_ag + ) + extra_kwargs["ub_overlap_rs"] = ( + self.config.tp_comm_overlap_rs + if hasattr(self.config, "tp_comm_overlap_rs") + else self.config.tp_comm_split_rs or self.config.tp_comm_atomic_rs + ) + # Disable ub overlap for experts. + if is_expert: + extra_kwargs["ub_overlap_ag"] = False + extra_kwargs["ub_overlap_rs"] = False + else: + extra_kwargs["ub_split_ag"] = self.config.tp_comm_split_ag + extra_kwargs["ub_atomic_gemm_ag"] = self.config.tp_comm_atomic_ag + extra_kwargs["ub_split_rs"] = self.config.tp_comm_split_rs + extra_kwargs["ub_atomic_gemm_rs"] = self.config.tp_comm_atomic_rs + # Disable ub overlap for experts. + if is_expert: + extra_kwargs["ub_split_ag"] = False + extra_kwargs["ub_atomic_gemm_ag"] = False + extra_kwargs["ub_split_rs"] = False + extra_kwargs["ub_atomic_gemm_rs"] = False + if is_te_min_version("1.0.0", check_equality=False): + assert ( + tp_comm_buffer_name is not None + ), "Buffer name should be set to configure communication overlap settings" + extra_kwargs["ub_name"] = tp_comm_buffer_name + + self.expert_parallel = self.config.expert_model_parallel_size > 1 + if is_expert: + rng_tracker_name = get_expert_parallel_rng_tracker_name() + else: + if parallel_mode == "duplicated": + rng_tracker_name = get_data_parallel_rng_tracker_name() + else: + rng_tracker_name = None + if is_te_min_version("1.7.0"): + extra_kwargs["rng_tracker_name"] = rng_tracker_name + + te_parallel_mode = parallel_mode + if parallel_mode == "duplicated": + # Handle non-parallel case + tp_group = None + tp_size = 1 + explicit_expert_comm = False + te_parallel_mode = None + else: + # Disable communications in TE when using TP or EP by + # making TE agnostic of model parallel. + if is_expert: + tp_group = get_expert_tensor_parallel_group(check_initialized=False) + tp_size = get_expert_tensor_parallel_world_size() + else: + tp_group = get_tensor_model_parallel_group(check_initialized=False) + tp_size = get_tensor_model_parallel_world_size() + explicit_expert_comm = is_expert and (tp_size > 1 or self.expert_parallel) + + if explicit_expert_comm: + if parallel_mode == "column": + output_size = divide(output_size, tp_size) + elif parallel_mode == "row": + input_size = divide(input_size, tp_size) + te_parallel_mode = None + tp_size = 1 + tp_group = None + + + super().__init__( + in_features=input_size, + out_features=output_size, + sequence_parallel=self.config.sequence_parallel, + fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, + tp_group=tp_group, + tp_size=tp_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + init_method=condition_init_method(config, init_method), + bias=bias, + return_bias=self.te_return_bias, + parallel_mode=te_parallel_mode, + **extra_kwargs, + ) + + for param in self.parameters(): + if is_expert: + # Reduce the gradient on the expert_data_parallel group for expert linear layers + setattr(param, 'allreduce', not self.expert_parallel) + else: + # Reduce the gradient on DP group + setattr(param, 'allreduce', True) + if parallel_mode == "duplicated": + # Reduce the gradient further on the TP group since the weight is + # duplicated across TP ranks + setattr(param, 'sequence_parallel', self.config.sequence_parallel) + + def forward(self, x): + """Forward.""" + _is_first_microbatch = ( + None if self.disable_parameter_transpose_cache else self.is_first_microbatch + ) + + out = super().forward(x, is_first_microbatch=_is_first_microbatch) + self.is_first_microbatch = False + + # TE only returns a tuple when return_bias is True, otherwise + # it returns a single Tensor, we always want to return two + # values regardless of the arguments. + if self.te_return_bias: + return out + return out, None + + def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): + """Replicate cross TP/DP.""" + + # Provide the dist-ckpt support when TELinear is directly used + # It can only happen with duplicated parallel mode + assert ( + self.parallel_mode is None + ), "TELinear sharded_state_dict can only be used with duplicated parallel mode" + state_dict = self.state_dict(prefix='', keep_vars=True) + return make_sharded_tensors_for_checkpoint(state_dict, prefix, None, sharded_offsets) + + def backward_dw(self): + """Compute weight gradients during the backward pass if split_bw is enabled.""" + if self.config.split_bw: + super().backward_dw() + +class TELayerNormColumnParallelLinear(te.pytorch.LayerNormLinear): + """ + Wrapper for the Transformer-Engine's `LayerNormLinear` layer that combines + layernorm and linear layers + """ + + def __init__( + self, + input_size: int, + output_size: int, + *, + config: TransformerConfig, + init_method: Callable, + gather_output: bool, + bias: bool, + skip_bias_add: bool, + is_expert: bool, + skip_weight_param_allocation: bool = False, + tp_comm_buffer_name: Optional[str] = None, + ): + self.config = config + + if gather_output: + raise ValueError('Transformer Engine linear layers do not support gather_output = True') + + if is_expert: + raise ValueError('Transformer Engine linear layers do not yet support MoE') + + if skip_weight_param_allocation: + raise ValueError( + 'Transformer Engine linear layers do not support skip_weight_param_allocation' + ) + + # TE returns a zero length Tensor when bias=False and + # return_bias=True, but we prefer None. So in that case we + # tell TE to not return the bias, and return None + # ourselves. This way our forward always returns two values + # and we don't have to deal with the zero length Tensor. + self.te_return_bias = skip_bias_add and bias + self.is_first_microbatch = True + self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache + extra_kwargs = _get_extra_te_kwargs(config) + + if self.config.split_bw: + extra_kwargs["delay_wgrad_compute"] = self.config.split_bw + + # Only Transformer-Engine version >= 0.11.0 supports `RMSNorm` + if is_te_min_version("0.11.0"): + extra_kwargs["normalization"] = self.config.normalization + elif self.config.normalization != "LayerNorm": + te_version = get_te_version() + raise ValueError( + f"Transformer Engine v{te_version} does not support {self.config.normalization}." + ) + + if is_te_min_version("0.8.0"): + if self.config.tp_comm_overlap: + extra_kwargs["ub_bulk_wgrad"] = self.config.tp_comm_bulk_wgrad + extra_kwargs["ub_bulk_dgrad"] = self.config.tp_comm_bulk_dgrad + if is_te_min_version("1.5.0", check_equality=False): + # Use old overlap flags if they were supplied instead + extra_kwargs["ub_overlap_ag"] = ( + self.config.tp_comm_overlap_ag + if hasattr(self.config, "tp_comm_overlap_ag") + else self.config.tp_comm_split_ag or self.config.tp_comm_atomic_ag + ) + if is_te_min_version("1.6.0.dev0", check_equality=False): + extra_kwargs["ub_overlap_rs_dgrad"] = ( + self.config.tp_comm_overlap_rs_dgrad + if hasattr(self.config, "tp_comm_overlap_rs_dgrad") + else False + ) + if tp_comm_buffer_name == 'qkv' and self.config.tp_comm_overlap_disable_qkv: + extra_kwargs["ub_overlap_ag"] = False + extra_kwargs["ub_overlap_rs_dgrad"] = False + + if tp_comm_buffer_name == 'fc1' and self.config.tp_comm_overlap_disable_fc1: + extra_kwargs["ub_overlap_ag"] = False + extra_kwargs["ub_overlap_rs_dgrad"] = False + else: + extra_kwargs["ub_atomic_gemm_ag"] = self.config.tp_comm_atomic_ag + extra_kwargs["ub_split_ag"] = self.config.tp_comm_split_ag + if is_te_min_version("1.0.0", check_equality=False): + assert ( + tp_comm_buffer_name is not None + ), "Buffer name should be set to configure communication overlap settings" + extra_kwargs["ub_name"] = tp_comm_buffer_name + + + super().__init__( + in_features=input_size, + out_features=output_size, + eps=self.config.layernorm_epsilon, + sequence_parallel=self.config.sequence_parallel, + fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, + tp_group=get_tensor_model_parallel_group(check_initialized=False), + tp_size=self.config.tensor_model_parallel_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + init_method=( + condition_init_method(config, init_method) + if not config.use_cpu_initialization + else lambda w: None + ), + bias=bias, + return_bias=self.te_return_bias, + parallel_mode="column", + return_layernorm_output=False, + zero_centered_gamma=self.config.layernorm_zero_centered_gamma, + **extra_kwargs, + ) + + world_size = get_tensor_model_parallel_world_size() + rank = get_tensor_model_parallel_rank() + + if config.use_cpu_initialization: + output_size_per_partition = divide(output_size, world_size) + _ = _initialize_affine_weight_cpu( + self.weight, + output_size, + input_size, + output_size_per_partition, + 0, + init_method=condition_init_method(config, init_method), + stride=1, + return_master_weight=False, + rank=rank, + world_size=world_size, + skip_set_tensor_parallel_attributes=True, + ) + if bias: + self.bias = Parameter( + torch.empty(output_size_per_partition, dtype=config.params_dtype) + ) + set_tensor_model_parallel_attributes(self.bias, True, 0, 1) + with torch.no_grad(): + self.bias.zero_() + setattr(self.bias, 'allreduce', True) + + def forward(self, x): + """Forward.""" + _is_first_microbatch = ( + None if self.disable_parameter_transpose_cache else self.is_first_microbatch + ) + + out = super().forward(x, is_first_microbatch=_is_first_microbatch) + self.is_first_microbatch = False + + # TE only returns a tuple when return_bias is True, otherwise + # it returns a single Tensor, we always want to return two + # values regardless of the arguments. + if self.te_return_bias: + return out + return out, None + + def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): + """Sharding along axis 0, bias sharded""" + state_dict = self.state_dict(prefix='', keep_vars=True) + return make_sharded_tensors_for_checkpoint( + state_dict, prefix, {'weight': 0, 'bias': 0}, sharded_offsets + ) + + def __repr__(self): + return ( + f"{type(self).__name__}(in_features={self.in_features}, " + f"out_features={self.out_features}, bias={self.use_bias}, TP={self.tp_size})" + ) + + def backward_dw(self): + """Compute weight gradients during the backward pass if split_bw is enabled.""" + if self.config.split_bw: + super().backward_dw() + + +class TEColumnParallelLinear(TELinear): + """ + Wrapper for the Transformer-Engine's `Linear` layer but specialized similar + to megatron's `ColumnParallelLinear` layer. + """ + + def __init__( + self, + input_size: int, + output_size: int, + *, + config: ModelParallelConfig, + init_method: Callable, + gather_output: bool, + bias: bool, + skip_bias_add: bool, + is_expert: bool, + skip_weight_param_allocation: bool = False, + tp_comm_buffer_name: Optional[str] = None, + ): + if gather_output: + raise ValueError('Transformer Engine linear layers do not support gather_output = True') + + super().__init__( + input_size=input_size, + output_size=output_size, + parallel_mode="column", + config=config, + init_method=( + condition_init_method(config, init_method) + if not config.use_cpu_initialization + else lambda w: None + ), + bias=bias, + skip_bias_add=skip_bias_add, + is_expert=is_expert, + skip_weight_param_allocation=skip_weight_param_allocation, + tp_comm_buffer_name=tp_comm_buffer_name, + ) + + if config.use_cpu_initialization: + if is_expert: + world_size = get_expert_tensor_parallel_world_size() + rank = get_expert_tensor_parallel_rank() + else: + world_size = get_tensor_model_parallel_world_size() + rank = get_tensor_model_parallel_rank() + output_size_per_partition = divide(output_size, world_size) + _ = _initialize_affine_weight_cpu( + self.weight, + output_size, + input_size, + output_size_per_partition, + 0, + init_method=condition_init_method(config, init_method), + stride=1, + return_master_weight=False, + rank=rank, + world_size=world_size, + skip_set_tensor_parallel_attributes=True, + ) + if bias: + self.bias = Parameter( + torch.empty(output_size_per_partition, dtype=config.params_dtype) + ) + set_tensor_model_parallel_attributes(self.bias, True, 0, 1) + with torch.no_grad(): + self.bias.zero_() + setattr(self.bias, 'allreduce', True) + + def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): + """Sharding along axis 0, bias sharded""" + state_dict = self.state_dict(prefix='', keep_vars=True) + return make_sharded_tensors_for_checkpoint( + state_dict, prefix, {'weight': 0, 'bias': 0}, sharded_offsets + ) + + def __repr__(self): + return ( + f"{type(self).__name__}(in_features={self.in_features}, " + f"out_features={self.out_features}, bias={self.use_bias}, TP={self.tp_size})" + ) + + def backward_dw(self): + """Compute weight gradients during the backward pass if split_bw is enabled.""" + if self.config.split_bw: + super().backward_dw() + + +class TERowParallelLinear(TELinear): + """ + Wrapper for the Transformer-Engine's `Linear` layer but specialized similar + to megatron's `RowParallelLinear` layer. + """ + + def __init__( + self, + input_size: int, + output_size: int, + *, + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + input_is_parallel: bool, + skip_bias_add: bool, + is_expert: bool, + tp_comm_buffer_name: Optional[str] = None, + ): + if not input_is_parallel: + raise ValueError( + "Transformer Engine linear layers do not support input_is_parallel = False" + ) + + super().__init__( + input_size=input_size, + output_size=output_size, + parallel_mode="row", + config=config, + init_method=( + condition_init_method(config, init_method) + if not config.use_cpu_initialization + else lambda w: None + ), + bias=bias, + skip_bias_add=skip_bias_add, + skip_weight_param_allocation=False, # We don't currently use this for row parallel layers + is_expert=is_expert, + tp_comm_buffer_name=tp_comm_buffer_name, + ) + if config.use_cpu_initialization: + if is_expert: + world_size = get_expert_tensor_parallel_world_size() + rank = get_expert_tensor_parallel_rank() + else: + world_size = get_tensor_model_parallel_world_size() + rank = get_tensor_model_parallel_rank() + input_size_per_partition = divide(input_size, world_size) + self.master_weight = _initialize_affine_weight_cpu( + self.weight, + output_size, + input_size, + input_size_per_partition, + 1, + init_method=condition_init_method(config, init_method), + stride=1, + return_master_weight=False, + params_dtype=config.params_dtype, + rank=rank, + world_size=world_size, + skip_set_tensor_parallel_attributes=True, + ) + if bias: + self.bias = Parameter(torch.empty(output_size, dtype=config.params_dtype)) + # Always initialize bias to zero. + with torch.no_grad(): + self.bias.zero_() + setattr(self.bias, 'allreduce', True) + setattr(self.bias, 'sequence_parallel', config.sequence_parallel) + + def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): + """Sharding along axis 1, bias not sharded""" + state_dict = self.state_dict(prefix='', keep_vars=True) + return make_sharded_tensors_for_checkpoint( + state_dict, prefix, {'weight': 1}, sharded_offsets + ) + + def __repr__(self): + return ( + f"{type(self).__name__}(in_features={self.in_features}, " + f"out_features={self.out_features}, bias={self.use_bias}, TP={self.tp_size})" + ) + + def backward_dw(self): + """Compute weight gradients during the backward pass if split_bw is enabled.""" + if self.config.split_bw: + super().backward_dw() + + +class TEDotProductAttention(te.pytorch.DotProductAttention): + """ + Wrapper for the Transformer-Engine's `DotProductAttention` layer that also + has "flash attention" enabled. + + Note that if Megatron's parallel_state has not been initialized yet, the + tp_group and cp_group passed to TE will be None and must be set later + via set_tensor_parallel_group() and set_context_parallel_group(). + """ + + cp_stream: torch.cuda.Stream = None + + def __init__( + self, + config: TransformerConfig, + layer_number: int, + attn_mask_type: AttnMaskType, + attention_type: str, + attention_dropout: Optional[float] = None, + softmax_scale: Optional[float] = None, + k_channels: Optional[int] = None, + v_channels: Optional[int] = None, + cp_comm_type: str = "p2p", + ): + self.config = config + self.te_forward_mask_type = False + self.qkv_format: str = 'sbhd' + + if self.config.apply_query_key_layer_scaling != bool( + int(os.getenv('NVTE_APPLY_QK_LAYER_SCALING', '0')) + ): + raise ValueError( + f"apply_query_key_layer_scaling is {self.config.apply_query_key_layer_scaling} " + f"but environment variable NVTE_APPLY_QK_LAYER_SCALING is " + f"{os.getenv('NVTE_APPLY_QK_LAYER_SCALING')}. Transformer Engine does not support " + f"setting query key layer scaling via argument, so these two must match." + ) + + extra_kwargs: dict[str, Any] = {} + if is_te_min_version("0.11.0"): + extra_kwargs["num_gqa_groups"] = self.config.num_query_groups + elif self.config.num_query_groups != self.config.num_attention_heads: + raise ValueError( + f"Transformer Engine v{get_te_version()} does not support Grouped Query Attention, " + f"use a newer version of Transformer Engine. " + f"(num_query_groups ({self.config.num_query_groups}) != " + f"num_attention_heads ({self.config.num_attention_heads}))" + ) + + if is_te_min_version("0.10.0"): + extra_kwargs["attention_type"] = attention_type + # older version don't need attention_type + + if is_te_min_version("0.12.0", check_equality=False): + self.te_forward_mask_type = True + + # This check is important as CP config can be disabled while having a valid CP group + # Example - Disabling CP for encoder while a valid CP group exists for decoder + if self.config.context_parallel_size > 1: + assert is_te_min_version("1.0.0"), "Only Transformer-Engine version >= 1.0.0 supports context parallelism!" + if getattr(TEDotProductAttention, "cp_stream") is None: + TEDotProductAttention.cp_stream = torch.cuda.Stream() + + extra_kwargs["cp_group"] = get_context_parallel_group(check_initialized=False) + extra_kwargs["cp_global_ranks"] = get_context_parallel_global_ranks(check_initialized=False) + extra_kwargs["cp_stream"] = TEDotProductAttention.cp_stream + + if is_te_min_version("1.10.0"): + if cp_comm_type is None: + extra_kwargs["cp_comm_type"] = "p2p" + + elif cp_comm_type == "a2a+p2p": + assert is_te_min_version("1.12.0"), ( + f"Transformer-Engine v{get_te_version()} must be >= 1.12.0 to support" + "hierarchical cp commucation." + ) + extra_kwargs["cp_comm_type"] = "a2a+p2p" + extra_kwargs["cp_group"] = get_hierarchical_context_parallel_groups(check_initialized=False) + else: + extra_kwargs["cp_comm_type"] = cp_comm_type + + if self.config.deterministic_mode: + if int(os.getenv("NVTE_ALLOW_NONDETERMINISTIC_ALGO", "1")) != 0: + raise RuntimeError( + "deterministic_mode is on and we are using DotProductAttention from " + "Transformer Engine, but NVTE_ALLOW_NONDETERMINISTIC_ALGO is not 0. " + f"Currently set to: {os.getenv('NVTE_ALLOW_NONDETERMINISTIC_ALGO', 'not set')}." + ) + + if config.window_size is not None: + # Check version + assert is_te_min_version("1.2.0"), ( + f"Transformer-Engine v{get_te_version()} must be >= 1.2.0 to support sliding window attention." + ) + extra_kwargs['window_size'] = config.window_size + + if is_te_min_version("1.10.0"): + # TE 1.10.0 introduces the ability to set the different k and v channels + kv_channels = ( + (k_channels, v_channels) + if k_channels is not None and v_channels is not None + else self.config.kv_channels + ) + extra_kwargs['softmax_scale'] = softmax_scale + else: + kv_channels = self.config.kv_channels + + self.kept_packed_seq_params = set(field.name for field in dataclasses.fields(PackedSeqParams)) + if get_te_version() < PkgVersion("1.3.0"): + # TE 1.3.0 introduces precomputing max_seqlen to remove unnecessary kernels and D2H + # copies (#555) + # These two arguments did not exist prior to 1.3.0 + self.kept_packed_seq_params.discard("max_seqlen_q") + self.kept_packed_seq_params.discard("max_seqlen_kv") + + if get_te_version() < PkgVersion("1.10.0"): + # TE 1.8.0 introduces cu_seqlens_padded which is the cu_seqlens with paddings counted + # in each individual sequence in THD format dataset + # These two arguments did not exist prior to 1.8.0. Full support added in 1.10.0 (#1012) + self.kept_packed_seq_params.discard("cu_seqlens_q_padded") + self.kept_packed_seq_params.discard("cu_seqlens_kv_padded") + + super().__init__( + num_attention_heads=self.config.num_attention_heads, + kv_channels=kv_channels, + attention_dropout=self.config.attention_dropout if attention_dropout is None else attention_dropout, + attn_mask_type=attn_mask_type.name, + sequence_parallel=self.config.sequence_parallel, + tp_size=self.config.tensor_model_parallel_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + tp_group=get_tensor_model_parallel_group(check_initialized=False), + layer_number=layer_number, + **extra_kwargs, + ) + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + attention_mask: Tensor, + attn_mask_type: AttnMaskType, + attention_bias: Tensor = None, + packed_seq_params: PackedSeqParams = None, + ): + """Forward.""" + packed_seq_kwargs = ( + {key: getattr(packed_seq_params, key) for key in self.kept_packed_seq_params} + if packed_seq_params is not None + else {} + ) + # overwrite self.qkv_format depending on self.config.apply_rope_fusion, which can be set + # after init + if self.config.apply_rope_fusion and is_te_min_version("0.13.0", check_equality=False): + self.qkv_format = 'bshd' + + qkv_format = packed_seq_kwargs.get('qkv_format', self.qkv_format) + + # WAR for peak memory usage. + # See https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/merge_requests/2388 + if self.config.apply_rope_fusion and qkv_format == 'bshd': + query, key, value = [x.transpose(0, 1).contiguous() for x in (query, key, value)] + # In PyTorch, the following two tensors are in fact the same: + # Tensor with shape (1, S, H, D) and stride (S*H*D, H*D, D, 1) + # Tensor with shape (1, S, H, D) and stride (H*D, H*D, D, 1) + # Stride for a dimension that is 1 has no meaning, so tensors created two different ways + # can have same shape but different strides. + # We unify them to the first one to pass the stride check in TE + if value.shape == key.shape and value.shape[0] == 1 and value.stride() != key.stride(): + value = value.as_strided(value.shape, key.stride()) + + attention_bias_kwargs = {} + if attention_bias is not None: + assert is_te_min_version("1.2.0"), ( + f"Transformer-Engine v{get_te_version()} must be >= 1.2.0 to support `attention_bias`." + ) + attention_bias_kwargs = dict( + core_attention_bias_type='post_scale_bias', core_attention_bias=attention_bias + ) + else: + if self.config.position_embedding_type == "alibi": + attention_bias_kwargs = dict(core_attention_bias_type='alibi', core_attention_bias=None) + + if self.te_forward_mask_type: + if qkv_format == 'thd' and is_te_min_version("1.7.0"): + # thd format uses flash attention with cuDNN kernel which requires is_padding=True, + # so the only acceptable mask types are `padding_causal` and `padding`. These do not + # necessarily indicate there are padded tokens in the sequence. + if attn_mask_type == AttnMaskType.causal: + attn_mask_type = AttnMaskType.padding_causal + elif attn_mask_type == AttnMaskType.no_mask: + attn_mask_type = AttnMaskType.padding + core_attn_out = super().forward( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type.name, + **attention_bias_kwargs, + **packed_seq_kwargs, + ) + else: + core_attn_out = super().forward( + query, + key, + value, + attention_mask, + **attention_bias_kwargs, + **packed_seq_kwargs, + ) + + if self.config.apply_rope_fusion and qkv_format == 'bshd': + return core_attn_out.transpose(0, 1) + else: + return core_attn_out + + +if is_te_min_version("1.9.0.dev0"): + + class TEGroupedLinear(te.pytorch.GroupedLinear): + """ + Wrapper for the Transformer-Engine's `GroupedLinear` layer. + + Note that if Megatron's parallel_state has not been initialized + yet, the tp_group passed to TE will be None and must be set later + via set_tensor_parallel_group(). + """ + + def __init__( + self, + num_gemms: int, + input_size: int, + output_size: int, + *, + parallel_mode: Optional[str], + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + skip_bias_add: bool, + is_expert: bool = False, + tp_comm_buffer_name: Optional[str] = None, + ): + self.config = config + + # TE returns a zero length Tensor when bias=False and + # return_bias=True, but we prefer None. So in that case we + # tell TE to not return the bias, and return None + # ourselves. This way our forward always returns two values + # and we don't have to deal with the zero length Tensor. + self.te_return_bias = skip_bias_add and bias + self.is_first_microbatch = True + self.disable_parameter_transpose_cache = self.config.disable_parameter_transpose_cache + + extra_kwargs = _get_extra_te_kwargs(config) + + if self.config.split_bw: + extra_kwargs["delay_wgrad_compute"] = self.config.split_bw + + extra_kwargs["ub_name"] = tp_comm_buffer_name + + self.expert_parallel = self.config.expert_model_parallel_size > 1 + if is_expert: + extra_kwargs["rng_tracker_name"] = get_expert_parallel_rng_tracker_name() + + # The comms between TP and EP group is explicitly handled by MoE token dispatcher. + # So we disable comms by making TE agnostic of model parallel. + if is_expert: + tp_group = get_expert_tensor_parallel_group(check_initialized=False) + tp_size = get_expert_tensor_parallel_world_size() + else: + tp_group = get_tensor_model_parallel_group(check_initialized=False) + tp_size = get_tensor_model_parallel_world_size() + self.explicit_expert_comm = is_expert and (tp_size > 1 or self.expert_parallel) + + if self.explicit_expert_comm: + if parallel_mode == "column": + output_size = divide(output_size, tp_size) + elif parallel_mode == "row": + input_size = divide(input_size, tp_size) + parallel_mode = None + tp_size = 1 + tp_group = None + + + super().__init__( + num_gemms=num_gemms, + in_features=input_size, + out_features=output_size, + sequence_parallel=self.config.sequence_parallel, + fuse_wgrad_accumulation=self.config.gradient_accumulation_fusion, + tp_group=tp_group, + tp_size=tp_size, + get_rng_state_tracker=( + get_cuda_rng_tracker if get_cuda_rng_tracker().is_initialized() else None + ), + init_method=condition_init_method(config, init_method), + bias=bias, + return_bias=self.te_return_bias, + parallel_mode=parallel_mode, + **extra_kwargs, + ) + + for param in self.parameters(): + setattr(param, 'allreduce', not (is_expert and self.expert_parallel)) + + def merge_extra_states( + self, + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, + ): + """ + Merge multiple "_extra_state" into one. + """ + self.init_fp8_metadata(num_gemms=self.num_gemms) + # When resume training, loading ckpt is out of fp8_autocast context. + # So we need to manually detect from the state_dict. + fp8_checkpoint = any("_extra_state" in str(key) for key in state_dict.keys()) + + if not fp8_checkpoint: + return + + try: + state_list = [ + state_dict.pop(f"{prefix}_extra_state{i}") for i in range(1, self.num_gemms) + ] + except KeyError: + # "_extra_state{i}" only exists for dist-ckpt. Return for torch native ckpt. + return + + # Early return conditions: + # 1. Empty state_dict + # 2. Empty state_list + # 3. _extra_state is None + # 4. _extra_state does not contain any information + if ( + not state_dict + or not state_list + or state_dict.get(f"{prefix}_extra_state") is None + or self._decode_extra_state(state_dict[f"{prefix}_extra_state"]) is None + ): + return + + state_list = [state_dict.pop(f"{prefix}_extra_state")] + state_list + state_list = [self._decode_extra_state(state) for state in state_list] + extra_fp8_variables = state_list[0]['extra_fp8_variables'] + extra_fp8_variables['num_gemms'] = self.num_gemms + extra_state = {"extra_fp8_variables": extra_fp8_variables} + # TE 2.0 adds recipe in extra_state + if is_te_min_version("2.0.0"): + self.fp8_meta["recipe"] = state_list[0]['recipe'] + extra_state['recipe'] = self.fp8_meta["recipe"] + # Only delayed scaling has global fp8 meta tensors. We're not using + # self.fp8_meta["recipe"].delayed() because it's available in TE 2.0 and later. + if isinstance(self.fp8_meta["recipe"], te.common.recipe.DelayedScaling): + extra_state.update( + { + "scale_fwd": torch.cat( + [state['scale_fwd'].view(-1, 1) for state in state_list], dim=1 + ).view(-1), + "amax_history_fwd": torch.cat( + [state['amax_history_fwd'].view(-1, 1) for state in state_list], + dim=1, + ).view(self.fp8_meta["recipe"].amax_history_len, -1), + "scale_bwd": torch.cat( + [state['scale_bwd'].view(-1, 1) for state in state_list], dim=1 + ).view(-1), + "amax_history_bwd": torch.cat( + [state['amax_history_bwd'].view(-1, 1) for state in state_list], + dim=1, + ).view(self.fp8_meta["recipe"].amax_history_len, -1), + } + ) + # TE 2.0 removes scale_inv_fwd and scale_inv_bwd + if not is_te_min_version("2.0.0"): + extra_state.update( + { + "scale_inv_fwd": torch.cat( + [state['scale_inv_fwd'].view(-1, 1) for state in state_list], + dim=1, + ).view(-1), + "scale_inv_bwd": torch.cat( + [state['scale_inv_bwd'].view(-1, 1) for state in state_list], + dim=1, + ).view(-1), + } + ) + state_dict[f"{prefix}_extra_state"] = self._encode_extra_state(extra_state) + + self._register_load_state_dict_pre_hook(merge_extra_states, with_module=True) + + def forward(self, x, m_splits): + """Forward.""" + _is_first_microbatch = ( + None if self.disable_parameter_transpose_cache else self.is_first_microbatch + ) + out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) + self.is_first_microbatch = False + + # TE only returns a tuple when return_bias is True, otherwise + # it returns a single Tensor, we always want to return two + # values regardless of the arguments. + if self.te_return_bias: + return out + return out, None + + def _encode_extra_state(self, state): + # TE 2.0 changed the format of extra_state to be a byte tensor + if is_te_min_version("2.0.0"): + torch.cuda.synchronize() + state_serialized = bytearray(pickle.dumps(state)) + state_serialized = torch.frombuffer(state_serialized, dtype=torch.uint8) + else: + state_serialized = io.BytesIO() + torch.save(state, state_serialized) + return state_serialized + + def _decode_extra_state(self, state): + if isinstance(state, torch.Tensor): + return pickle.loads(state.detach().cpu().numpy().tobytes()) + elif isinstance(state, io.BytesIO): + state.seek(0) + return torch.load(state, map_location="cuda", weights_only=False) + else: + raise RuntimeError("Unsupported checkpoint format.") + + def _split_extra_state(self, state): + fp8_checkpoint = self.fp8_meta["fp8_checkpoint"] or self.fp8 or self.fp8_calibration + + if not fp8_checkpoint: + return [state] * self.num_gemms + + state = self._decode_extra_state(state) + extra_states = [] + extra_fp8_variables = state['extra_fp8_variables'] + extra_fp8_variables['num_gemms'] = 1 + for gemm_idx in range(self.num_gemms): + tmp_state = {"extra_fp8_variables": extra_fp8_variables} + # TE 2.0 adds recipe in extra_state + if is_te_min_version("2.0.0"): + tmp_state['recipe'] = state['recipe'] + # Only delayed scaling has global fp8 meta tensors. We're not using + # self.fp8_meta["recipe"].delayed() because it's available in TE 2.0 and later. + if isinstance(self.fp8_meta["recipe"], te.common.recipe.DelayedScaling): + tmp_state.update( + { + "scale_fwd": state['scale_fwd'].view(3, -1)[:, gemm_idx], + "amax_history_fwd": state['amax_history_fwd'].view( + self.fp8_meta["recipe"].amax_history_len, 3, -1 + )[:, :, gemm_idx], + "scale_bwd": state['scale_bwd'].view(2, -1)[:, gemm_idx], + "amax_history_bwd": state['amax_history_bwd'].view( + self.fp8_meta["recipe"].amax_history_len, 2, -1 + )[:, :, gemm_idx], + } + ) + # TE 2.0 removes scale_inv_fwd and scale_inv_bwd + if not is_te_min_version("2.0.0"): + tmp_state.update( + { + "scale_inv_fwd": state['scale_inv_fwd'].view(3, -1)[:, gemm_idx], + "scale_inv_bwd": state['scale_inv_bwd'].view(2, -1)[:, gemm_idx], + } + ) + extra_states.append(self._encode_extra_state(tmp_state)) + return extra_states + + def _sharded_state_dict_grouped( + self, tp_axis_map, prefix='', sharded_offsets=(), metadata=None + ): + """ + prefix should be module_name to make keys identical to sequetial ones. + """ + sharded_state_dict = {} + full_state_dict = self.state_dict(prefix='', keep_vars=True) + num_global_experts = get_expert_model_parallel_world_size() * self.num_gemms + local_expert_indices_offset = get_expert_model_parallel_rank() * self.num_gemms + ep_axis = len(sharded_offsets) + extra_states = self._split_extra_state(full_state_dict['_extra_state']) + for gemm_idx in range(self.num_gemms): + state_dict = { + f'{gemm_idx}.weight': full_state_dict[f'weight{gemm_idx}'], + f'{gemm_idx}._extra_state': extra_states[gemm_idx], + } + if self.use_bias: + state_dict[f'{gemm_idx}.bias'] = full_state_dict[f'bias{gemm_idx}'] + sub_sd = make_sharded_tensors_for_checkpoint( + state_dict, + '', + tp_axis_map, + ( + *sharded_offsets, + (ep_axis, local_expert_indices_offset + gemm_idx, num_global_experts), + ), + ) + # Remove expert layers indexing from sharded keys + replace_prefix_for_sharding(sub_sd, f'{gemm_idx}.', prefix) + sharded_state_dict.update( + { + f'{prefix}weight{gemm_idx}': sub_sd[f'{gemm_idx}.weight'], + f'{prefix}_extra_state{"" if gemm_idx == 0 else gemm_idx}': sub_sd[ + f'{gemm_idx}._extra_state' + ], + } + ) + if self.use_bias: + sharded_state_dict[f'{prefix}bias{gemm_idx}'] = sub_sd[f'{gemm_idx}.bias'] + # Adjust replica ids - replication along DP modulo EP + for k, sh_ten in sharded_state_dict.items(): + replica_id = sh_ten.replica_id + assert ( + len(replica_id) == 3 + ), f'Expected replica_id for {k} to be in (PP, TP, DP) format, got: {replica_id}' + if getattr(sh_ten, "is_data_parallel_fully_shard", False): + edp_replica_id = 0 + else: + edp_replica_id = get_expert_data_parallel_rank() + sh_ten.replica_id = (*replica_id[:2], edp_replica_id) + return sharded_state_dict + + def backward_dw(self): + """Compute weight gradients during the backward pass if split_bw is enabled.""" + if self.config.split_bw: + super().backward_dw() + + class TEColumnParallelGroupedLinear(TEGroupedLinear): + """ + Wrapper for the Transformer-Engine's `GroupedLinear` layer but specialized + to column-parallel style. + """ + + def __init__( + self, + num_gemms: int, + input_size: int, + output_size: int, + *, + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + skip_bias_add: bool, + is_expert: bool, + tp_comm_buffer_name: Optional[str] = None, + ): + + super().__init__( + num_gemms=num_gemms, + input_size=input_size, + output_size=output_size, + parallel_mode="column", + config=config, + init_method=condition_init_method(config, init_method), + bias=bias, + skip_bias_add=skip_bias_add, + is_expert=is_expert, + tp_comm_buffer_name=tp_comm_buffer_name, + ) + + def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): + """ + For each gemm, sharding along axis 0, bias sharded. + Assume sharded_offsets[-1] is the expert parallel offset. + """ + tp_axis_map = {} + for gemm_idx in range(self.num_gemms): + tp_axis_map.update({f'{gemm_idx}.weight': 0, f'{gemm_idx}.bias': 0}) + return super()._sharded_state_dict_grouped( + tp_axis_map, prefix, sharded_offsets, metadata + ) + + class TERowParallelGroupedLinear(TEGroupedLinear): + """ + Wrapper for the Transformer-Engine's `GroupedLinear` layer but specialized + to row-parallel style. + """ + + def __init__( + self, + num_gemms: int, + input_size: int, + output_size: int, + *, + config: ModelParallelConfig, + init_method: Callable, + bias: bool, + skip_bias_add: bool, + is_expert: bool, + tp_comm_buffer_name: Optional[str] = None, + ): + + super().__init__( + num_gemms=num_gemms, + input_size=input_size, + output_size=output_size, + parallel_mode="row", + config=config, + init_method=condition_init_method(config, init_method), + bias=bias, + skip_bias_add=skip_bias_add, + is_expert=is_expert, + tp_comm_buffer_name=tp_comm_buffer_name, + ) + + def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): + """ + For each gemm, sharding along axis 1, bias not sharded. + Assume sharded_offsets[-1] is the expert parallel offset. + """ + tp_axis_map = {f'{gemm_idx}.weight': 1 for gemm_idx in range(self.num_gemms)} + return super()._sharded_state_dict_grouped( + tp_axis_map, prefix, sharded_offsets, metadata + ) + +else: + TEGroupedLinear = None # type: ignore[assignment, misc] + TEColumnParallelGroupedLinear = None # type: ignore[assignment, misc] + TERowParallelGroupedLinear = None # type: ignore[assignment, misc] + + +class TEDelayedScaling(te.common.recipe.DelayedScaling): + """ + Wrapper for the Transformer-Engine's `DelayedScaling` layer. + """ + + def __init__( + self, + config: ModelParallelConfig, + fp8_format: int, + override_linear_precision: tuple = (False, False, False), + ): + extra_kwargs = _get_extra_te_kwargs(config) + if is_te_min_version("1.6.0.dev0"): + extra_kwargs["fp8_dpa"] = config.fp8_dot_product_attention + extra_kwargs["fp8_mha"] = config.fp8_multi_head_attention + if get_te_version() < PkgVersion("1.8.0"): + extra_kwargs["interval"] = config.fp8_interval + elif config.fp8_interval != 1: + warnings.warn("fp8_interval is deprecated and ignored from Transformer-Engine v1.8.0.") + + super().__init__( + margin=config.fp8_margin, + fp8_format=fp8_format, + amax_compute_algo=config.fp8_amax_compute_algo, + amax_history_len=config.fp8_amax_history_len, + override_linear_precision=override_linear_precision, + **extra_kwargs, + ) + + +class TECudaRNGStatesTracker(te.pytorch.distributed.CudaRNGStatesTracker): + """Wraps TransformerEngine's CudaRNGStatesTracker so that it is + interchangeable with Megatron's RNG tracker""" + + def __init__(self): + super().__init__() + self.reset() + + def is_initialized(self): + """Checks if the internal RNG state has been set wirth set_states().""" + return self._is_initialized + + def reset(self): + """Reset the internal RNG state.""" + super().reset() + self._is_initialized = False + + def set_states(self, states): + """Set the internal RNG state.""" + super().set_states(states) + self._is_initialized = True + + def add(self, name, seed): + """Track the rng state.""" + super().add(name, seed) + self._is_initialized = True + + +def te_checkpoint( + forward_func, + distribute_saved_activations, + get_rng_state_tracker, + tp_group, + hidden_states, + attention_mask, + attn_mask_type, + context, + context_mask, + rotary_pos_emb, + **kwargs, +): + """Checkpointing with Transformer-Engine.""" + from transformer_engine.pytorch.distributed import checkpoint + + if is_te_min_version("1.5.0"): + return checkpoint( + forward_func, + hidden_states, + attention_mask, + attn_mask_type, + context, + context_mask, + rotary_pos_emb, + distribute_saved_activations=distribute_saved_activations, + get_rng_state_tracker=get_rng_state_tracker, + tp_group=tp_group, + **kwargs, + ) + else: + return checkpoint( + forward_func, + distribute_saved_activations, + get_rng_state_tracker, + tp_group, + hidden_states, + attention_mask, + context, + context_mask, + rotary_pos_emb, + ) + + +try: + from transformer_engine.pytorch.attention import _SplitAlongDim + SplitAlongDim = _SplitAlongDim.apply + +except ImportError: + SplitAlongDim = None + +try: + from transformer_engine.pytorch.cpu_offload import get_cpu_offload_context as _get_cpu_offload_context + def get_cpu_offload_context( + enabled, num_layers, model_layers, activation_offloading, weight_offloading + ): + """Get CPU offload context and sync function.""" + if is_te_min_version("1.10.0.dev0"): + context, sync_func = _get_cpu_offload_context( + enabled, num_layers, model_layers, activation_offloading, weight_offloading + ) + else: + context, sync_func = _get_cpu_offload_context( + enabled, num_layers, activation_offloading, weight_offloading + ) + + return context, sync_func + +except ImportError: + get_cpu_offload_context = None # type: ignore[assignment, misc] + +try: + from transformer_engine.pytorch.attention import FusedRoPEFunc + def fused_apply_rotary_pos_emb( + t: torch.Tensor, freqs: torch.Tensor, transpose_output_memory: bool = False + ) -> torch.Tensor: + """Apply rotary positional embedding to input tensor T in `sbhd` format.""" + if transpose_output_memory: + warnings.warn( + "transpose_output_memory is not supported by TE's fused RoPE and will be ignored." + ) + return FusedRoPEFunc.apply(t, freqs, "sbhd") + + def fused_apply_rotary_pos_emb_thd( + t: torch.Tensor, + cu_seqlens: torch.Tensor, + freqs: torch.Tensor, + cp_size: int = 1, + cp_rank: int = 0, + ) -> torch.Tensor: + """ + Apply rotary positional embedding to input tensor T in `thd` format with CP support. + """ + if is_te_min_version("1.11.0", check_equality=False): + return FusedRoPEFunc.apply(t, freqs, "thd", cu_seqlens, cp_size, cp_rank) + else: + return FusedRoPEFunc.apply(t, freqs, "thd", cu_seqlens) + +except ImportError: + pass + +try: + from transformer_engine.pytorch import Fp8Padding, Fp8Unpadding # pylint: disable=unused-import +except ImportError: + Fp8Padding = None + Fp8Unpadding = None + +try: + from transformer_engine.pytorch.permutation import ( + moe_permute, + moe_sort_chunks_by_index, + moe_unpermute, + ) + fused_permute = moe_permute + fused_unpermute = moe_unpermute + fused_sort_chunks_by_index = moe_sort_chunks_by_index + +except ImportError: + fused_permute = None + fused_unpermute = None + fused_sort_chunks_by_index = None \ No newline at end of file diff --git a/aiak_megatron/megatron/core/fusions/fused_layer_norm.py b/aiak_megatron/megatron/core/fusions/fused_layer_norm.py index 01a654db..0bcd32b6 100644 --- a/aiak_megatron/megatron/core/fusions/fused_layer_norm.py +++ b/aiak_megatron/megatron/core/fusions/fused_layer_norm.py @@ -8,25 +8,25 @@ from torch import Tensor from torch.nn import init from torch.nn.parameter import Parameter - +import torch.nn.functional as F from megatron.core.transformer import TransformerConfig from megatron.core.utils import make_viewless_tensor -try: - from apex.contrib.layer_norm.layer_norm import FastLayerNormFN - - HAVE_PERSIST_LAYER_NORM = True -except ImportError: - HAVE_PERSIST_LAYER_NORM = False +# try: +# from apex.contrib.layer_norm.layer_norm import FastLayerNormFN -try: - from apex.normalization.fused_layer_norm import FusedLayerNormAffineFunction - - HAVE_FUSED_LAYER_NORM = True -except ImportError: - HAVE_FUSED_LAYER_NORM = False +# HAVE_PERSIST_LAYER_NORM = True +# except ImportError: +# HAVE_PERSIST_LAYER_NORM = False +# try: +# from apex.normalization.fused_layer_norm import FusedLayerNormAffineFunction +# HAVE_FUSED_LAYER_NORM = True +# except ImportError: +# HAVE_FUSED_LAYER_NORM = False +HAVE_PERSIST_LAYER_NORM = False +HAVE_FUSED_LAYER_NORM = False class FusedLayerNorm(torch.nn.Module): """Layer Norm, fused into a single CUDA kernel. @@ -102,7 +102,12 @@ def __init__( if not persist_layer_norm and not HAVE_FUSED_LAYER_NORM: # TODO: Add pytorch only layer norm - raise ValueError(f'Apex must be installed to use FusedLayerNorm.') + self._use_pytorch_fallback = True + persist_layer_norm = False + else: + self._use_pytorch_fallback = False + + if isinstance(hidden_size, numbers.Integral): hidden_size = (hidden_size,) @@ -130,8 +135,15 @@ def reset_parameters(self): init.zeros_(self.bias) def forward(self, input: Tensor) -> Tensor: - + weight = self.weight + 1 if self.zero_centered_gamma else self.weight + # If we flagged the pure-PyTorch fallback, use torch.nn.functional.layer_norm + if getattr(self, "_use_pytorch_fallback", False): + # weight should be offset when zero_centered_gamma is used (matches original behavior) + return F.layer_norm(input, tuple(self.hidden_size), weight=weight, bias=self.bias, eps=self.eps) + + + if self.persist_layer_norm: if 'memory_efficient' in inspect.getfullargspec(FastLayerNormFN.forward).args: diff --git a/aiak_megatron/megatron/core/fusions/fused_layer_norm_org.py b/aiak_megatron/megatron/core/fusions/fused_layer_norm_org.py new file mode 100644 index 00000000..f6d472d6 --- /dev/null +++ b/aiak_megatron/megatron/core/fusions/fused_layer_norm_org.py @@ -0,0 +1,170 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + +import importlib +import inspect +import numbers + +import torch +from torch import Tensor +from torch.nn import init +from torch.nn.parameter import Parameter + +from megatron.core.transformer import TransformerConfig +from megatron.core.utils import make_viewless_tensor + +# try: +# from apex.contrib.layer_norm.layer_norm import FastLayerNormFN + +# HAVE_PERSIST_LAYER_NORM = True +# except ImportError: +# HAVE_PERSIST_LAYER_NORM = False + +# try: +# from apex.normalization.fused_layer_norm import FusedLayerNormAffineFunction + +# HAVE_FUSED_LAYER_NORM = True +# except ImportError: +# HAVE_FUSED_LAYER_NORM = False +HAVE_PERSIST_LAYER_NORM = False +HAVE_FUSED_LAYER_NORM = False +class FusedLayerNorm(torch.nn.Module): + """Layer Norm, fused into a single CUDA kernel. + + Args: + hidden_size (int): Transformer hidden dimension. + + eps (float): Epsilon added to denominator, for numerical stability. + + persist_layer_norm (bool): Use persistent fused layer norm kernel. + This kernel supports only a set of hidden sizes. Please + check persist_ln_hidden_sizes if your hidden size is supported. + + zero_centered_gamma (bool): Adjust LayerNorm weights such that they are + centered around zero. This improves numerical stability. + + config (TransformerConfig): Transformer config. Include to match custom + layer norm interfaces. + + normalization (str): Normalization type, used for Transformer Engine. + Must equal 'LayerNorm' here. + """ + + def __init__( + self, + config: TransformerConfig, + hidden_size: int, + eps: float = 1e-5, + persist_layer_norm: bool = True, + zero_centered_gamma: bool = False, + normalization: str = "LayerNorm", # included to match TE interface + ): + super().__init__() + + self.config = config + + self.zero_centered_gamma = self.config.layernorm_zero_centered_gamma + assert ( + self.config.normalization == "LayerNorm" + ), f'({self.config.normalization}) is not supported in FusedLayerNorm' + + # List of hiddens sizes supported in the persistent layer norm kernel + # If the hidden size is not supported, fall back to the non-persistent + # kernel. + persist_ln_hidden_sizes = [ + 1024, + 1536, + 2048, + 2304, + 3072, + 3840, + 4096, + 5120, + 6144, + 8192, + 10240, + 12288, + 12800, + 15360, + 16384, + 18432, + 20480, + 24576, + 25600, + 30720, + 32768, + 40960, + 49152, + 65536, + ] + persist_layer_norm = self.config.persist_layer_norm + if hidden_size not in persist_ln_hidden_sizes or not HAVE_PERSIST_LAYER_NORM: + persist_layer_norm = False + + if not persist_layer_norm and not HAVE_FUSED_LAYER_NORM: + # TODO: Add pytorch only layer norm + raise ValueError(f'Apex must be installed to use FusedLayerNorm.') + + if isinstance(hidden_size, numbers.Integral): + hidden_size = (hidden_size,) + self.hidden_size = torch.Size(hidden_size) + self.eps = eps + # Parameters need to be initialized with torch.empty rather than torch.Tensor + # for correct device placement with nemo2. + self.weight = Parameter(torch.empty(*hidden_size)) + self.bias = Parameter(torch.empty(*hidden_size)) + self.reset_parameters() + self.persist_layer_norm = persist_layer_norm + self.sequence_parallel = self.config.sequence_parallel + + # set sequence parallelism flag on weight and bias parameters + setattr(self.weight, 'sequence_parallel', self.sequence_parallel) + setattr(self.bias, 'sequence_parallel', self.sequence_parallel) + + def reset_parameters(self): + + if self.zero_centered_gamma: + init.zeros_(self.weight) + init.zeros_(self.bias) + else: + init.ones_(self.weight) + init.zeros_(self.bias) + + def forward(self, input: Tensor) -> Tensor: + + weight = self.weight + 1 if self.zero_centered_gamma else self.weight + + if self.persist_layer_norm: + if 'memory_efficient' in inspect.getfullargspec(FastLayerNormFN.forward).args: + output = FastLayerNormFN.apply( + input, weight, self.bias, self.eps, self.config.memory_efficient_layer_norm + ) + else: + output = FastLayerNormFN.apply(input, weight, self.bias, self.eps) + + # Apex's fast layer norm function outputs a 'view' tensor (i.e., has + # a populated '_base' field). This will result in schedule.py's + # deallocate_output_tensor() throwing an error, so a viewless tensor is + # created to prevent this. + output = make_viewless_tensor( + inp=output, requires_grad=input.requires_grad, keep_graph=True + ) + + else: + if ( + 'memory_efficient' + in inspect.getfullargspec(FusedLayerNormAffineFunction.forward).args + ): + return FusedLayerNormAffineFunction.apply( + input, + weight, + self.bias, + self.hidden_size, + self.eps, + self.config.memory_efficient_layer_norm, + ) + else: + return FusedLayerNormAffineFunction.apply( + input, weight, self.bias, self.hidden_size, self.eps + ) + + return output diff --git a/aiak_megatron/megatron/core/model_parallel_config.py b/aiak_megatron/megatron/core/model_parallel_config.py index a0022a08..a89fc3d6 100644 --- a/aiak_megatron/megatron/core/model_parallel_config.py +++ b/aiak_megatron/megatron/core/model_parallel_config.py @@ -143,6 +143,7 @@ class ModelParallelConfig: # Optimizations ################### gradient_accumulation_fusion: bool = False + print(gradient_accumulation_fusion) """If true, fuses weight gradient accumulation to GEMMs. Requires the custom CUDA extension fused_weight_gradient_mlp_cuda module. To use gradient_accumulation_fusion you must install APEX with --cpp_ext and --cuda_ext. For example: "pip install --global-option=\"--cpp_ext\" diff --git a/aiak_megatron/megatron/core/optimizer/__init__.py b/aiak_megatron/megatron/core/optimizer/__init__.py index e4a38cbd..d52bd68e 100644 --- a/aiak_megatron/megatron/core/optimizer/__init__.py +++ b/aiak_megatron/megatron/core/optimizer/__init__.py @@ -6,22 +6,23 @@ import torch from torch.optim import SGD as CPUSGD from torch.optim import AdamW as CPUAdam - -try: - from transformer_engine.pytorch.optimizers import FusedAdam as Adam - from transformer_engine.pytorch.optimizers import FusedSGD as SGD -except ImportError: - try: - from apex.optimizers import FusedAdam as Adam - from apex.optimizers import FusedSGD as SGD - except ImportError: - warnings.warn(f'Transformer Engine and Apex are not installed. Falling back to Torch optimizers.') - - # Apex's FusedAdam is a drop-in replacement for torch's AdamW. - # pylint: disable-next=line-too-long. - # See https://github.com/NVIDIA/apex/blob/7b73b12361068a10b0f44844534613f252a5ea75/apex/optimizers/fused_adam.py#L16. - from torch.optim import AdamW as Adam, SGD - +# from megatron.core.optimizer import Adam as Adam +# try: +# from transformer_engine.pytorch.optimizers import FusedAdam as Adam +# from transformer_engine.pytorch.optimizers import FusedSGD as SGD +# except ImportError: +# try: +# from apex.optimizers import FusedAdam as Adam +# from apex.optimizers import FusedSGD as SGD +# except ImportError: +# warnings.warn(f'Transformer Engine and Apex are not installed. Falling back to Torch optimizers.') + +# # Apex's FusedAdam is a drop-in replacement for torch's AdamW. +# # pylint: disable-next=line-too-long. +# # See https://github.com/NVIDIA/apex/blob/7b73b12361068a10b0f44844534613f252a5ea75/apex/optimizers/fused_adam.py#L16. +# from torch.optim import AdamW as Adam, SGD +Adam = CPUAdam +SGD = CPUSGD from megatron.core import mpu from megatron.core.optimizer.cpu_offloading.hybrid_optimizer import HybridDeviceOptimizer diff --git a/aiak_megatron/megatron/core/optimizer/distrib_optimizer.py b/aiak_megatron/megatron/core/optimizer/distrib_optimizer.py index 6b565cee..a006fb35 100644 --- a/aiak_megatron/megatron/core/optimizer/distrib_optimizer.py +++ b/aiak_megatron/megatron/core/optimizer/distrib_optimizer.py @@ -10,16 +10,19 @@ import torch -HAVE_APEX_OR_TE = True -try: - from transformer_engine.pytorch.optimizers import FusedAdam as Adam -except ImportError: - try: - from apex.optimizers import FusedAdam as Adam - except ImportError: - from torch.optim import AdamW as Adam +# HAVE_APEX_OR_TE = True +# try: +# from transformer_engine.pytorch.optimizers import FusedAdam as Adam +# except ImportError: +# try: +# from apex.optimizers import FusedAdam as Adam +# except ImportError: +# from torch.optim import AdamW as Adam - HAVE_APEX_OR_TE = False +# HAVE_APEX_OR_TE = False +from torch.optim import AdamW as Adam + +HAVE_APEX_OR_TE = False from megatron.core.optimizer.cpu_offloading import HybridDeviceOptimizer @@ -486,7 +489,7 @@ def __init__( for model_chunk in self.model_chunks: assert self.ddp_config == model_chunk.ddp_config self.distributed_optimizer_instance_id = distributed_optimizer_instance_id - + print(optimizer) assert isinstance(optimizer, (Adam, HybridDeviceOptimizer)) or optimizer is None, ( "Only Adam and HybridDeviceOptimizer currently supported, due to checkpointing requirements." ) diff --git a/aiak_megatron/megatron/core/tensor_parallel/layers.py b/aiak_megatron/megatron/core/tensor_parallel/layers.py index 8689602a..dda8497d 100644 --- a/aiak_megatron/megatron/core/tensor_parallel/layers.py +++ b/aiak_megatron/megatron/core/tensor_parallel/layers.py @@ -419,6 +419,7 @@ def forward( ctx.save_for_backward(input, weight) ctx.use_bias = bias is not None ctx.gradient_accumulation_fusion = gradient_accumulation_fusion + print( "1",ctx.gradient_accumulation_fusion) ctx.allreduce_dgrad = allreduce_dgrad ctx.sequence_parallel = sequence_parallel ctx.wgrad_deferral_limit = wgrad_deferral_limit @@ -854,7 +855,7 @@ def __init__( "gradient accumulation fusion." ) self.gradient_accumulation_fusion = config.gradient_accumulation_fusion - + if self.allreduce_dgrad and self.sequence_parallel: raise RuntimeError( "`allreduce_dgrad` and `sequence_parallel` cannot be enabled at the same time." @@ -938,7 +939,7 @@ def forward( self._forward_impl = linear_with_grad_accumulation_and_async_allreduce allreduce_dgrad = False if self.explicit_expert_comm else self.allreduce_dgrad - + output_parallel = self._forward_impl( input=input_parallel, weight=weight, @@ -1055,6 +1056,7 @@ def __init__( self.is_expert = is_expert self.expert_parallel = config.expert_model_parallel_size > 1 self.gradient_accumulation_fusion = config.gradient_accumulation_fusion + print("3",self.gradient_accumulation_fusion) self.sequence_parallel = config.sequence_parallel if self.sequence_parallel and not self.input_is_parallel: raise RuntimeError("To enable `sequence_parallel`, `input_is_parallel` must be `True`") diff --git a/aiak_megatron/megatron/core/transformer/attention.py b/aiak_megatron/megatron/core/transformer/attention.py index b489a35b..b582f509 100644 --- a/aiak_megatron/megatron/core/transformer/attention.py +++ b/aiak_megatron/megatron/core/transformer/attention.py @@ -648,7 +648,7 @@ def get_query_key_value_tensors(self, hidden_states, key_value_states=None): self.hidden_size_per_attention_head, self.hidden_size_per_attention_head, ] - + if SplitAlongDim is not None: # [sq, b, ng, (np/ng + 2) * hn] # --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn] diff --git a/aiak_megatron/megatron/core/transformer/dot_product_attention.py b/aiak_megatron/megatron/core/transformer/dot_product_attention.py index 1b5200ae..7d919a09 100644 --- a/aiak_megatron/megatron/core/transformer/dot_product_attention.py +++ b/aiak_megatron/megatron/core/transformer/dot_product_attention.py @@ -105,6 +105,7 @@ def forward( attention_bias: Tensor = None, packed_seq_params: Optional[PackedSeqParams] = None, ): + packed_seq_params=None """Forward.""" assert packed_seq_params is None, ( "Packed sequence is not supported by DotProductAttention." diff --git a/aiak_megatron/megatron/core/transformer/moe/fused_a2a.py b/aiak_megatron/megatron/core/transformer/moe/fused_a2a.py index 9bcbe8ff..64bd0724 100644 --- a/aiak_megatron/megatron/core/transformer/moe/fused_a2a.py +++ b/aiak_megatron/megatron/core/transformer/moe/fused_a2a.py @@ -17,9 +17,17 @@ HAVE_DEEP_EP = False import torch -from transformer_engine.pytorch.constants import TE_DType - +# from transformer_engine.pytorch.constants import TE_DType +# Try to import TE_DType from Transformer Engine, but fall back if unavailable logger = logging.getLogger(__name__) +try: + from transformer_engine.pytorch.constants import TE_DType +except Exception as e: + logger.warning( + f"TE_DType import failed, FP8 communication will be disabled: {str(e)}" + ) + TE_DType = {} + try: from transformer_engine.pytorch.tensor.float8_blockwise_tensor import ( Float8BlockQuantizer, diff --git a/aiak_megatron/megatron/training/checkpointing.py b/aiak_megatron/megatron/training/checkpointing.py index 4d89f0de..f78c8e6c 100644 --- a/aiak_megatron/megatron/training/checkpointing.py +++ b/aiak_megatron/megatron/training/checkpointing.py @@ -1283,14 +1283,214 @@ def load_checkpoint(model, optimizer, opt_param_scheduler, load_arg='load', stri print_rank_0('could not find arguments in the checkpoint ...') # Model. - strict = False if args.retro_add_retriever else strict + # For fine-tuning/stage1 alignment, use non-strict loading to allow architecture mismatches. + # This is especially important when loading HF checkpoints into Megatron models. + if args.finetune or hasattr(args, 'trainable_modules'): + strict = False + else: + strict = False if args.retro_add_retriever else strict if not skip_load_to_model_and_opt: + # Get device from model parameters + device = next(ddp_model[0].parameters()).device + print_rank_0(f"Loading checkpoint with device: {device}, strict={strict}") + + # Move all tensors in state_dict to device before loading + model_state = {} + for k, v in state_dict['model'].items(): + if isinstance(v, torch.Tensor): + model_state[k] = v.to(device) + else: + model_state[k] = v + if len(ddp_model) == 1: - ddp_model[0].load_state_dict(state_dict['model'], strict=strict) + ddp_model[0].load_state_dict(model_state, strict=strict) + # Ensure entire model is on device (handles newly created layers from strict=False) + ddp_model[0].to(device) + # After checkpoint load, initialize any lazy TE linear modules with their loaded weights + try: + from megatron.core.extensions import transformer_engine as _te + + for name, module in ddp_model[0].named_modules(): + if isinstance(module, getattr(_te, '_FallbackLinear', tuple())): + # If this is a lazy linear (self.linear is None), try to initialize it from loaded weights + if hasattr(module, 'linear') and module.linear is None: + # Look for corresponding weights in the loaded state_dict + weight_key = None + bias_key = None + for state_key in model_state.keys(): + if name in state_key and 'weight' in state_key: + weight_key = state_key + if name in state_key and 'bias' in state_key: + bias_key = state_key + + if weight_key and weight_key in model_state: + weight = model_state[weight_key] + bias_tensor = model_state.get(bias_key) if bias_key else None + # Create linear with loaded shape + out_features, in_features = weight.shape + module.linear = torch.nn.Linear(in_features, out_features, bias=(bias_tensor is not None)) + module.linear.weight.data.copy_(weight) + if bias_tensor is not None: + module.linear.bias.data.copy_(bias_tensor) + module.linear = module.linear.to(device=device, dtype=weight.dtype) + except Exception: + pass + # Ensure model params/buffers have correct dtype for mixed precision + try: + if getattr(args, 'bf16', False): + target_dtype = torch.bfloat16 + elif getattr(args, 'fp16', False): + target_dtype = torch.float16 + else: + target_dtype = None + + if target_dtype is not None: + for p in ddp_model[0].parameters(): + p.data = p.data.to(dtype=target_dtype, device=p.device) + for name, buf in ddp_model[0].named_buffers(recurse=True): + if buf is not None: + try: + buf.data = buf.data.to(dtype=target_dtype, device=buf.device) + except Exception: + # some buffers may be non-tensor or empty + pass + except Exception: + pass + # Also ensure any fallback TE linear modules are moved to device + try: + from megatron.core.extensions import transformer_engine as _te + + for m in ddp_model[0].modules(): + if isinstance(m, getattr(_te, '_FallbackLinear', tuple())): + m.to(device) + except Exception: + pass + # Final dtype enforcement after moving any fallback modules + try: + if getattr(args, 'bf16', False): + target_dtype = torch.bfloat16 + elif getattr(args, 'fp16', False): + target_dtype = torch.float16 + else: + target_dtype = None + + if target_dtype is not None: + mismatches = [] + total = 0 + for name, p in ddp_model[0].named_parameters(): + total += 1 + if p.dtype != target_dtype: + mismatches.append((name, str(p.device), str(p.dtype))) + p.data = p.data.to(device=p.device, dtype=target_dtype) + for name, b in ddp_model[0].named_buffers(recurse=True): + if isinstance(b, torch.Tensor) and b.dtype != target_dtype: + mismatches.append((f'buffer:{name}', str(b.device), str(b.dtype))) + try: + b.data = b.data.to(device=b.device, dtype=target_dtype) + except Exception: + pass + if mismatches: + print_rank_0(f'Post-load converted {len(mismatches)}/{total} params/buffers to {target_dtype}. Examples: {mismatches[:10]}') + except Exception: + pass else: for i in range(len(ddp_model)): mpu.set_virtual_pipeline_model_parallel_rank(i) - ddp_model[i].load_state_dict(state_dict['model%d' % i], strict=strict) + model_state_i = {} + for k, v in state_dict['model%d' % i].items(): + if isinstance(v, torch.Tensor): + model_state_i[k] = v.to(device) + else: + model_state_i[k] = v + ddp_model[i].load_state_dict(model_state_i, strict=strict) + # Ensure entire model is on device + ddp_model[i].to(device) + # After checkpoint load, initialize any lazy TE linear modules with their loaded weights + try: + from megatron.core.extensions import transformer_engine as _te + + for name, module in ddp_model[i].named_modules(): + if isinstance(module, getattr(_te, '_FallbackLinear', tuple())): + # If this is a lazy linear (self.linear is None), try to initialize it from loaded weights + if hasattr(module, 'linear') and module.linear is None: + # Look for corresponding weights in the loaded state_dict + weight_key = None + bias_key = None + for state_key in model_state_i.keys(): + if name in state_key and 'weight' in state_key: + weight_key = state_key + if name in state_key and 'bias' in state_key: + bias_key = state_key + + if weight_key and weight_key in model_state_i: + weight = model_state_i[weight_key] + bias_tensor = model_state_i.get(bias_key) if bias_key else None + # Create linear with loaded shape + out_features, in_features = weight.shape + module.linear = torch.nn.Linear(in_features, out_features, bias=(bias_tensor is not None)) + module.linear.weight.data.copy_(weight) + if bias_tensor is not None: + module.linear.bias.data.copy_(bias_tensor) + module.linear = module.linear.to(device=device, dtype=weight.dtype) + except Exception: + pass + # Ensure any fallback TE linears are on the same device + try: + from megatron.core.extensions import transformer_engine as _te + + for m in ddp_model[i].modules(): + if isinstance(m, getattr(_te, '_FallbackLinear', tuple())): + m.to(device) + except Exception: + pass + # Ensure model params/buffers have correct dtype for mixed precision + try: + if getattr(args, 'bf16', False): + target_dtype = torch.bfloat16 + elif getattr(args, 'fp16', False): + target_dtype = torch.float16 + else: + target_dtype = None + + if target_dtype is not None: + for p in ddp_model[i].parameters(): + p.data = p.data.to(dtype=target_dtype, device=p.device) + for name, buf in ddp_model[i].named_buffers(recurse=True): + if buf is not None: + try: + buf.data = buf.data.to(dtype=target_dtype, device=buf.device) + except Exception: + pass + except Exception: + pass + # Final dtype enforcement after moving any fallback modules + try: + if getattr(args, 'bf16', False): + target_dtype = torch.bfloat16 + elif getattr(args, 'fp16', False): + target_dtype = torch.float16 + else: + target_dtype = None + + if target_dtype is not None: + mismatches = [] + total = 0 + for name, p in ddp_model[i].named_parameters(): + total += 1 + if p.dtype != target_dtype: + mismatches.append((name, str(p.device), str(p.dtype))) + p.data = p.data.to(device=p.device, dtype=target_dtype) + for name, b in ddp_model[i].named_buffers(recurse=True): + if isinstance(b, torch.Tensor) and b.dtype != target_dtype: + mismatches.append((f'buffer:{name}', str(b.device), str(b.dtype))) + try: + b.data = b.data.to(device=b.device, dtype=target_dtype) + except Exception: + pass + if mismatches: + print_rank_0(f'Post-load converted {len(mismatches)}/{total} params/buffers to {target_dtype}. Examples: {mismatches[:10]}') + except Exception: + pass # Fix up query/key/value matrix ordering if needed. checkpoint_version = get_checkpoint_version() diff --git a/aiak_megatron/megatron/training/checkpointing_org.py b/aiak_megatron/megatron/training/checkpointing_org.py new file mode 100644 index 00000000..033e2bb4 --- /dev/null +++ b/aiak_megatron/megatron/training/checkpointing_org.py @@ -0,0 +1,1444 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + +"""Input/output checkpointing.""" + +import contextlib +import os +import random +import shutil +import sys +import threading +from enum import Enum, auto +from logging import getLogger +from pathlib import Path + +import numpy as np +from time import time + +import torch + +from megatron.core import mpu, tensor_parallel, dist_checkpointing +from megatron.core.dist_checkpointing.mapping import ShardedObject +from megatron.core.dist_checkpointing.serialization import get_default_load_sharded_strategy +from megatron.core.dist_checkpointing.strategies.fully_parallel import \ + FullyParallelSaveStrategyWrapper, FullyParallelLoadStrategyWrapper +from megatron.core.num_microbatches_calculator import update_num_microbatches +from megatron.core.fp8_utils import is_float8tensor, dequantize_fp8_tensor +from megatron.core.rerun_state_machine import get_rerun_state_machine +from .async_utils import schedule_async_save, is_empty_async_queue +from .global_vars import get_args, get_one_logger +from .utils import unwrap_model, print_rank_0, append_to_progress_log, is_last_rank +from ..core.dist_checkpointing.serialization import \ + get_default_save_sharded_strategy +from .one_logger_utils import on_save_checkpoint_start, on_save_checkpoint_success +from . import wandb_utils +from . import ft_integration + +# [ModelOpt]: Import +try: + from modelopt.torch.opt.plugins import ( + save_modelopt_state, + save_sharded_modelopt_state, + restore_modelopt_state, + restore_sharded_modelopt_state, + ) + has_nvidia_modelopt = True +except Exception: + has_nvidia_modelopt = False + +_CHECKPOINT_VERSION = None + +logger = getLogger(__name__) +_NON_PERSISTENT_CKPT_SUBDIR = 'non_persistent' + + +def set_checkpoint_version(value): + """set the current checkpoint version""" + global _CHECKPOINT_VERSION + if _CHECKPOINT_VERSION is not None: + assert _CHECKPOINT_VERSION == value, \ + "checkpoint versions do not match" + _CHECKPOINT_VERSION = value + + +def get_checkpoint_version(): + """get the current checkpoint version""" + global _CHECKPOINT_VERSION + return _CHECKPOINT_VERSION + + +def check_checkpoint_args(checkpoint_args): + """Ensure fixed arguments for a model are the same for the input + arguments and the one retrieved from checkpoint.""" + args = get_args() + + def _compare(arg_name, old_arg_name=None, default=None): + if old_arg_name is not None: + ckpt_arg_name = old_arg_name + else: + ckpt_arg_name = arg_name + if default is not None: + checkpoint_value = getattr(checkpoint_args, ckpt_arg_name, default) + else: + checkpoint_value = getattr(checkpoint_args, ckpt_arg_name) + args_value = getattr(args, arg_name) + error_message = '{} value from checkpoint ({}) is not equal to the ' \ + 'input argument value ({}).'.format( + arg_name, checkpoint_value, args_value) + assert checkpoint_value == args_value, error_message + + _compare('num_layers') + _compare('hidden_size') + _compare('num_attention_heads') + _compare('add_position_embedding', default=False) + if args.vocab_file: + _compare('max_position_embeddings') + _compare('make_vocab_size_divisible_by') + if not args.use_dist_ckpt: + _compare('padded_vocab_size') + _compare('tokenizer_type') + if args.data_parallel_random_init: + _compare('data_parallel_random_init') + if get_checkpoint_version() < 3.0: + _compare('tensor_model_parallel_size', + old_arg_name='model_parallel_size') + if get_checkpoint_version() >= 3.0 and not args.use_dist_ckpt: + _compare('tensor_model_parallel_size') + _compare('pipeline_model_parallel_size') + + +def ensure_directory_exists(filename, check_parent=True): + """Build filename's path if it does not already exists.""" + dirname = os.path.dirname(filename) if check_parent else filename + os.makedirs(dirname, exist_ok=True) + + +def get_checkpoint_name(checkpoints_path, iteration, release=False, + pipeline_parallel=None, + tensor_rank=None, pipeline_rank=None, + expert_parallel=None, expert_rank=None, + return_base_dir=False, + basename="model_optim_rng.pt"): + """Determine the directory name for this rank's checkpoint.""" + if release: + directory = 'release' + else: + directory = 'iter_{:07d}'.format(iteration) + if return_base_dir: + common_path = os.path.join(checkpoints_path, directory) + return common_path + + # Use both the tensor and pipeline MP rank. + if pipeline_parallel is None: + pipeline_parallel = (mpu.get_pipeline_model_parallel_world_size() > 1) + if tensor_rank is None: + tensor_rank = mpu.get_tensor_model_parallel_rank() + if pipeline_rank is None: + pipeline_rank = mpu.get_pipeline_model_parallel_rank() + if expert_parallel is None: + expert_parallel = (mpu.get_expert_model_parallel_world_size() > 1) + if expert_rank is None: + expert_rank = mpu.get_expert_model_parallel_rank() + + # Use both the tensor and pipeline MP rank. If using the distributed + # optimizer, then the optimizer's path must additionally include the + # data parallel rank. + if not pipeline_parallel: + common_path = os.path.join(checkpoints_path, directory, + f'mp_rank_{tensor_rank:02d}') + else: + common_path = os.path.join(checkpoints_path, directory, + f'mp_rank_{tensor_rank:02d}_{pipeline_rank:03d}') + + if expert_parallel: + common_path = common_path + f'_{expert_rank:03d}' + + return os.path.join(common_path, basename) + + +def get_distributed_optimizer_checkpoint_name(model_checkpoint_name): + return os.path.join(os.path.dirname(model_checkpoint_name), + "distrib_optim.pt") + + +def find_checkpoint_rank_0(checkpoints_path, iteration, release=False): + """Finds the checkpoint for rank 0 without knowing if we are using + pipeline parallelism/expert parallelism or not. + + Since the checkpoint naming scheme changes if pipeline or expert + parallelism is present, we need to look for both naming schemes if + we don't know if the checkpoint has pipeline or expert parallelism. + """ + + # Look for checkpoint with no pipelining and no expert parallelism + filename = get_checkpoint_name(checkpoints_path, iteration, release, + pipeline_parallel=False, + tensor_rank=0, pipeline_rank=0, + expert_parallel=False, expert_rank=0) + if os.path.isfile(filename): + return filename + + # Look for checkpoint with no pipelining and expert parallelism + filename = get_checkpoint_name(checkpoints_path, iteration, release, + pipeline_parallel=False, + tensor_rank=0, pipeline_rank=0, + expert_parallel=True, expert_rank=0) + if os.path.isfile(filename): + return filename + + # Look for checkpoint with pipelining and no expert parallelism + filename = get_checkpoint_name(checkpoints_path, iteration, release, + pipeline_parallel=True, + tensor_rank=0, pipeline_rank=0, + expert_parallel=False, expert_rank=0) + if os.path.isfile(filename): + return filename + + # Look for checkpoint with pipelining and expert parallelism + filename = get_checkpoint_name(checkpoints_path, iteration, release, + pipeline_parallel=True, + tensor_rank=0, pipeline_rank=0, + expert_parallel=True, expert_rank=0) + if os.path.isfile(filename): + return filename + + # Look for a distributed checkpoint + filename = get_checkpoint_name(checkpoints_path, iteration, release, + pipeline_parallel=True, + return_base_dir=True) + if dist_checkpointing.check_is_distributed_checkpoint(filename): + return filename + + return None + + +def get_checkpoint_tracker_filename(checkpoints_path): + + """Tracker file rescords the latest chckpoint during + training to restart from.""" + return os.path.join(checkpoints_path, 'latest_checkpointed_iteration.txt') + + +def checkpoint_exists(checkpoints_path): + """Check if a checkpoint exists.""" + if checkpoints_path is None: + return False + load_step = 'latest_checkpointed_iteration.txt' + return os.path.exists(os.path.join(checkpoints_path, load_step)) + + +def read_metadata(tracker_filename): + # Read the tracker file and either set the iteration or + # mark it as a release checkpoint. + iteration = 0 + release = False + with open(tracker_filename, 'r') as f: + metastring = f.read().strip() + try: + iteration = int(metastring) + except ValueError: + release = metastring == 'release' + if not release: + print_rank_0('ERROR: Invalid metadata file {}. Exiting'.format( + tracker_filename)) + sys.exit() + assert iteration > 0 or release, 'error parsing metadata file {}'.format( + tracker_filename) + + # Get the max iteration retrieved across the ranks. + if torch.distributed.is_initialized(): + iters_cuda = torch.tensor([iteration], dtype=torch.long, device='cuda') + torch.distributed.all_reduce(iters_cuda, op=torch.distributed.ReduceOp.MAX) + max_iter = iters_cuda[0].item() + + # We should now have all the same iteration. + # If not, print a warning and chose the maximum + # iteration across all ranks. + if iteration != max_iter: + rank = torch.distributed.get_rank() + print('WARNING: on rank {} found iteration {} in the ' + 'metadata while max iteration across the ranks ' + 'is {}, replacing it with max iteration.'.format( + rank, iteration, max_iter), flush=True) + else: + # When loading a checkpoint outside of training (for example, + # when editing it), we might not have torch distributed + # initialized, in this case, just assume we have the latest + max_iter = iteration + return max_iter, release + + +def get_rng_state(use_dist_ckpt: bool = False): + """ collect rng state across data parallel ranks """ + args = get_args() + rng_state = { + 'random_rng_state': random.getstate(), + 'np_rng_state': np.random.get_state(), + 'torch_rng_state': torch.get_rng_state(), + 'cuda_rng_state': torch.cuda.get_rng_state(), + 'rng_tracker_states': tensor_parallel.get_cuda_rng_tracker().get_states()} + + rng_state_list = None + if torch.distributed.is_initialized() and \ + mpu.get_data_parallel_world_size() > 1 and \ + args.data_parallel_random_init: + rng_state_list = \ + [None for i in range(mpu.get_data_parallel_world_size())] + torch.distributed.all_gather_object( + rng_state_list, + rng_state, + group=mpu.get_data_parallel_group()) + else: + rng_state_list = [rng_state] + + if use_dist_ckpt: + pp_rank = mpu.get_pipeline_model_parallel_rank() + pp_size = mpu.get_pipeline_model_parallel_world_size() + tp_rank = mpu.get_tensor_model_parallel_rank() + tp_size = mpu.get_tensor_model_parallel_world_size() + rng_state_list = ShardedObject('rng_state', rng_state_list, (pp_size, tp_size), (pp_rank, tp_rank), + replica_id=mpu.get_data_parallel_rank(with_context_parallel=True)) + + return rng_state_list + + +class CheckpointType(Enum): + """Checkpoint type""" + LEGACY = auto() + LOCAL = auto() + GLOBAL = auto() + + +def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floating_point_operations_so_far, + checkpointing_context=None, save_arg='save', pipeline_rank=None, expert_rank=None, + tensor_rank=None, pipeline_parallel=None, expert_parallel=None, non_persistent_ckpt=False, + train_data_iterator=None, preprocess_common_state_dict_fn=None): + """Save a model, optimizer and optionally dataloader checkpoint. + + Checkpointing context is used to persist some checkpointing state + throughout a single job. Must be initialized externally (not used if None). + + If non_persistent_ckpt is True, + the checkpoint will be saved with special functionality for removing old checkpoints. + There are several types of non-persistent checkpoints: + "global" - Saved as a standard checkpoint (e.g., on Lustre) with old checkpoints being removed. + "local" - Each rank saves a portion of the checkpoint locally (e.g., on SSD/ramdisk). + + Dataloader checkpoint is only saved if the dataloader supports it. Currently this applies only + to the Megatron Energon dataloader (multimodal) and not the built-in Megatron dataloader (text-only). + """ + start_ckpt = time() + args = get_args() + save_dir = getattr(args, save_arg) + + if args.async_save and not is_empty_async_queue(): + print_rank_0('WARNING: Starting a checkpoint save before previous has finished. ' + 'Consider increasing the checkpoint interval.') + + # Prepare E2E metrics at start of save checkpoint + productive_metrics = on_save_checkpoint_start(args.async_save) + + # Monitor for the checkpointing timeout (no-op if FT is not enabled) + ft_integration.on_checkpointing_start() + + # Only rank zero of the data parallel writes to the disk. + model = unwrap_model(model) + + # Handle non_persistent_ckpt flag. Besides overwriting `args.save` and + # `args.use_dist_ckpt`, non-persistent global ckpt requires no additional logic + ckpt_type = CheckpointType.GLOBAL if args.use_dist_ckpt else CheckpointType.LEGACY + if non_persistent_ckpt: + if args.non_persistent_ckpt_type == 'global': + ckpt_type = CheckpointType.GLOBAL + save_dir = ( + args.non_persistent_global_ckpt_dir + if args.non_persistent_global_ckpt_dir + else os.path.join(save_dir, _NON_PERSISTENT_CKPT_SUBDIR) + ) + # TODO Can we ensure the previous checkpoint is saved? We don't want to allow two saves in parallel. + cleanup_old_non_persistent_checkpoint(save_dir, leave_ckpt_num=1, do_async=args.async_save) + elif args.non_persistent_ckpt_type == 'local': + ckpt_type = CheckpointType.LOCAL + save_dir = checkpointing_context['local_checkpoint_manager'].local_ckpt_dir + else: + assert False, \ + f'Please use local or global non-persistent checkpoints (got: {args.non_persistent_ckpt_type})' + + ckpt_format = args.ckpt_format if ckpt_type == CheckpointType.GLOBAL else 'torch' + print_rank_0('saving checkpoint at iteration {:7d} to {} in {} format'.format(iteration, save_dir, ckpt_format)) + + # Collect rng state across data parallel ranks. + rng_state = get_rng_state(ckpt_type != CheckpointType.LEGACY) + + # Collect rerun state across all ranks + rerun_state_machine = get_rerun_state_machine() + rerun_state = rerun_state_machine.state_dict( + data_iterator=train_data_iterator, use_dist_ckpt=ckpt_type != CheckpointType.LEGACY + ) + + # Checkpoint name. + return_base_dir = (ckpt_type != CheckpointType.LEGACY) + checkpoint_name = get_checkpoint_name(save_dir, iteration, release=False, + pipeline_parallel=pipeline_parallel, tensor_rank=tensor_rank, + pipeline_rank=pipeline_rank, expert_parallel=expert_parallel, + expert_rank=expert_rank, return_base_dir=return_base_dir) + + # Save dataloader state if the dataloader supports it (currently only Megatron Energon). + maybe_save_dataloader_state(train_data_iterator, iteration, getattr(args, "dataloader_save", None)) + + # Save distributed optimizer's custom parameter state. + if ( + args.use_distributed_optimizer + and not args.no_save_optim + and optimizer is not None + and ckpt_type == CheckpointType.LEGACY + ): + optim_checkpoint_name = \ + get_distributed_optimizer_checkpoint_name(checkpoint_name) + ensure_directory_exists(optim_checkpoint_name) + if not optimizer.is_stub_optimizer: + optimizer.save_parameter_state(optim_checkpoint_name) + + async_save_request = None + if args.async_save: + if ckpt_type == CheckpointType.LEGACY: + raise NotImplementedError('Async checkpoint save not implemented for legacy checkpoints') + elif ckpt_type == CheckpointType.GLOBAL and args.ckpt_format != 'torch_dist': + raise NotImplementedError(f'Async checkpoint save not implemented for {args.ckpt_format} ' + f'distributed checkpoint format') + + rank = torch.distributed.get_rank() if torch.distributed.is_initialized() else 0 + + # Collect args, model, RNG. + if not torch.distributed.is_initialized() \ + or mpu.get_expert_data_parallel_rank() == 0 \ + or ckpt_type != CheckpointType.LEGACY: + optim_sd_kwargs = {} + if ckpt_type != CheckpointType.LEGACY and args.use_distributed_optimizer: + optim_sd_kwargs['sharding_type'] = ('fully_sharded_model_space' + if args.ckpt_fully_parallel_save + else 'dp_zero_gather_scatter') + print_rank_0(f'Storing distributed optimizer sharded state of type {optim_sd_kwargs["sharding_type"]}') + + state_dict = generate_state_dict( + args, + model, + optimizer, + opt_param_scheduler, + rng_state, + use_dist_ckpt=ckpt_type != CheckpointType.LEGACY, + iteration=iteration, + optim_sd_kwargs=optim_sd_kwargs, + rerun_state=rerun_state, + ) + + state_dict['num_floating_point_operations_so_far'] = num_floating_point_operations_so_far + if ckpt_type == CheckpointType.GLOBAL: + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + # TODO Handle non-empty directories (e.g., after a crash during saving). + ensure_directory_exists(checkpoint_name, check_parent=False) + if checkpointing_context is not None and 'save_strategy' in checkpointing_context: + save_strategy = checkpointing_context['save_strategy'] + # Already saved once before - don't need to rerun sharding validation + validate_sharding_integrity = not args.ckpt_assume_constant_structure + else: + validate_sharding_integrity = True + save_strategy = get_default_save_sharded_strategy(args.ckpt_format) + if args.ckpt_assume_constant_structure and args.ckpt_format == 'torch_dist': + save_strategy.use_cached_ckpt_structure = args.ckpt_assume_constant_structure + if checkpointing_context is not None and 'load_strategy' in checkpointing_context: + cached_global_metadata = getattr(checkpointing_context['load_strategy'], + 'cached_global_metadata', None) + if cached_global_metadata is not None: + logger.debug("Plugging in the read metadata from the load strategy...") + save_strategy.cached_global_metadata = cached_global_metadata + else: + logger.debug("Failed to plug in the read metadata from the load strategy...") + + if args.ckpt_fully_parallel_save: + save_strategy = FullyParallelSaveStrategyWrapper( + save_strategy, + mpu.get_data_parallel_group(with_context_parallel=True), + args.ckpt_assume_constant_structure + ) + # Store save strategy for future checkpoint saves + if checkpointing_context is not None: + checkpointing_context['save_strategy'] = save_strategy + end_ckpt = time() + logger.debug(f"rank: {rank}, takes {end_ckpt - start_ckpt} to prepare state dict for ckpt ") + async_save_request = dist_checkpointing.save( + state_dict, checkpoint_name, save_strategy, + async_sharded_save=args.async_save, + validate_access_integrity=validate_sharding_integrity, + preprocess_common_before_consistancy_check=preprocess_common_state_dict_fn) + # [ModelOpt]: save sharded modelopt_state + if has_nvidia_modelopt: + save_sharded_modelopt_state(model, checkpoint_name, (args.ckpt_format, 1)) + else: + # [ModelOpt]: Inject modelopt_state into state_dict + if has_nvidia_modelopt: + if ckpt_type == CheckpointType.LOCAL: + print_rank_0('WARNING: Local checkpointing does not support nvidia_modelopt.') + else: + save_modelopt_state(model, state_dict) + + end_ckpt = time() + logger.debug(f"rank: {rank}, takes {end_ckpt - start_ckpt} to prepare state dict for ckpt ") + if ckpt_type == CheckpointType.LOCAL: + try: + from megatron.core.dist_checkpointing.tensor_aware_state_dict import MCoreTensorAwareStateDict + except ModuleNotFoundError: + raise RuntimeError("The 'nvidia_resiliency_ext' module is required for local " + "checkpointing but was not found. Please ensure it is installed.") + + algo = args.non_persistent_local_ckpt_algo + cached_metadata = None + if args.ckpt_assume_constant_structure and 'local_checkpoint_cache' in checkpointing_context: + cached_metadata = checkpointing_context['local_checkpoint_cache'] + state_dict_for_save, cacheable_metadata = MCoreTensorAwareStateDict.from_state_dict( + state_dict, algo=algo, cached_metadata=cached_metadata, + parallelization_group=mpu.get_data_parallel_group(with_context_parallel=True) + ) + async_save_request = checkpointing_context['local_checkpoint_manager'].save( + state_dict_for_save, iteration, is_async=bool(args.async_save) + ) + checkpointing_context['local_checkpoint_cache'] = cacheable_metadata + else: + assert ckpt_type == CheckpointType.LEGACY + # Save. + ensure_directory_exists(checkpoint_name) + torch.save(state_dict, checkpoint_name) + start_misc = time() + if ckpt_type != CheckpointType.LOCAL: + if not args.async_save: + assert async_save_request is None + # Wait so everyone is done (necessary) + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + # And update the latest iteration + if not torch.distributed.is_initialized() \ + or torch.distributed.get_rank() == 0: + tracker_filename = get_checkpoint_tracker_filename(save_dir) + + if ckpt_type == CheckpointType.LOCAL: + def iter_finalize_fn(): + print_rank_0(' successfully saved local checkpoint from iteration {:7d}'.format(iteration)) + if args.log_progress and args.async_save: + append_to_progress_log(f'Saved async local checkpoint\tIteration: {iteration}', barrier=False) + else: + def iter_finalize_fn(): + with open(tracker_filename, 'w') as f: + f.write(str(iteration)) + + _tensor_rank = tensor_rank if tensor_rank is not None else mpu.get_tensor_model_parallel_rank() + _pipeline_rank = pipeline_rank if pipeline_rank is not None else mpu.get_pipeline_model_parallel_rank() + + print_rank_0(f' successfully saved checkpoint from iteration {int(iteration):7d} to {args.save} ' + f'[ t {_tensor_rank + 1}/{mpu.get_tensor_model_parallel_world_size()}, ' + f'p {_pipeline_rank + 1}/{mpu.get_pipeline_model_parallel_world_size()} ]') + if args.log_progress and args.async_save: + append_to_progress_log(f'Saved async checkpoint\tIteration: {iteration}', barrier=False) + + if args.async_save: + assert async_save_request is not None + async_save_request.add_finalize_fn(iter_finalize_fn) + else: + iter_finalize_fn() + + # Additional callback for one_logger (last rank) + if not torch.distributed.is_initialized() \ + or is_last_rank(): + def onelogger_finalize_fn(): + on_save_checkpoint_success(productive_metrics, args.async_save) + if args.async_save: + assert async_save_request is not None + async_save_request.add_finalize_fn(onelogger_finalize_fn) + else: + onelogger_finalize_fn() + + # Additional callback for wandb (last rank) + if not torch.distributed.is_initialized() \ + or is_last_rank(): + def wandb_finalize_fn(): + wandb_utils.on_save_checkpoint_success(checkpoint_name, + get_checkpoint_tracker_filename(save_dir), save_dir, iteration) + if args.async_save: + assert async_save_request is not None + async_save_request.add_finalize_fn(wandb_finalize_fn) + else: + wandb_finalize_fn() + + if args.async_save: + schedule_async_save(async_save_request) + print_rank_0(' scheduled an async checkpoint save at iteration {:7d} to {}' \ + .format(iteration, save_dir)) + + # Wait so everyone is done (not necessary) + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + end_misc = time() + logger.debug(f"rank: {rank}, takes {end_misc - start_misc} to finalize ckpt save ") + + ft_integration.on_checkpointing_end(is_async_finalization=False) + + +def cleanup_old_non_persistent_checkpoint(save_dir, leave_ckpt_num=1, do_async=False): + """Clean up old non-persistent checkpoints""" + if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: + return + save_dir = Path(save_dir) + + iter_prefix = "iter_" + iter_ckpts = save_dir.rglob(f'{iter_prefix}*') + sorted_iter_ckpts = sorted(iter_ckpts, key=lambda ckpt_name: int(ckpt_name.name[len(iter_prefix):])) + if not sorted_iter_ckpts: + return + rm_iter_ckpts = sorted_iter_ckpts[:-leave_ckpt_num] + print_rank_0(f'Non-persistent checkpoints scheduled for removal: {rm_iter_ckpts}') + print_rank_0(f'Non-persistent checkpoints to be kept: {sorted_iter_ckpts[-leave_ckpt_num:]}') + + def remove_iter_ckpts(_iter_ckpts): + for ckpt in _iter_ckpts: + shutil.rmtree(ckpt) + if do_async: + threading.Thread(target=remove_iter_ckpts, args=(rm_iter_ckpts,)).start() + else: + remove_iter_ckpts(rm_iter_ckpts) + + +def maybe_save_dataloader_state(train_iterator, iteration, dataloader_save_path): + """Saves dataloader state if the dataloader supports it. + + Currently, this is only used by Megatron Energon dataloader (multimodal) to store its state at a + specific iteration. The Megatron built-in dataloader (text-only) creates index files upfront + to track its state. + + If the provided dataloader has `save_state` method, then it is called to save the state. + Otherwise, no state is saved. + + Args: + train_iterator (iterable): Train dataloader. + iteration (int): Current iteration. + dataloader_save_path (str): Path where the dataloader state is saved. + """ + # If no dataloader or saving path is provided, exit early, otherwise, raise an error. + if train_iterator is None or dataloader_save_path is None or dataloader_save_path == "": + return + + # If dataloader doesn't support saving state, raise an error. + if not hasattr(train_iterator.iterable, "save_state"): + raise RuntimeError(f"Could not find a save_state for the train_iterator of type {type(train_iterator)}") + + # Save dataloader state for each data parallel rank only once. + first_rank = mpu.is_pipeline_first_stage(ignore_virtual=True) and mpu.get_tensor_model_parallel_rank() == 0 + if not first_rank: + return + + dp_rank = mpu.get_data_parallel_rank() + print(f"saving dataloader checkpoint at iteration {iteration} to {dataloader_save_path}") + train_dataloader_state_dict = train_iterator.iterable.save_state() + data_state_save_path = get_checkpoint_name( + dataloader_save_path, iteration, + basename=f'train_dataloader_dprank{dp_rank:03d}.pt' + ) + + torch.distributed.barrier(group=mpu.get_data_parallel_group()) + + if mpu.get_data_parallel_rank() == 0: + ensure_directory_exists(data_state_save_path) + + torch.distributed.barrier(group=mpu.get_data_parallel_group()) + + dataloader_save_dict = {} + dataloader_save_dict['dataloader_state_dict'] = train_dataloader_state_dict + torch.save(dataloader_save_dict, data_state_save_path) + + +def generate_state_dict(args, model, optimizer, opt_param_scheduler, + rng_state, use_dist_ckpt=False, iteration=None, + optim_sd_kwargs=None, rerun_state=None): + # Arguments, iteration, and model. + state_dict = {} + state_dict['args'] = args + state_dict['checkpoint_version'] = 3.0 + if iteration is not None: + state_dict['iteration'] = iteration + + if len(model) == 1: + state_dict['model'] = (model[0].sharded_state_dict() + if use_dist_ckpt else + model[0].state_dict_for_save_checkpoint()) + else: + for i in range(len(model)): + mpu.set_virtual_pipeline_model_parallel_rank(i) + state_dict['model%d' % i] = ( + model[i].sharded_state_dict() + if use_dist_ckpt else + model[i].state_dict_for_save_checkpoint()) + # Optimizer stuff. + if not args.no_save_optim: + if optimizer is not None and not optimizer.is_stub_optimizer: + state_dict['optimizer'] = (optimizer.sharded_state_dict(state_dict, **(optim_sd_kwargs or {})) + if use_dist_ckpt else + optimizer.state_dict()) + if opt_param_scheduler is not None: + state_dict['opt_param_scheduler'] = \ + opt_param_scheduler.state_dict() + + # Rerun state + state_dict['rerun_state_machine'] = rerun_state + + # RNG states. + if not args.no_save_rng: + state_dict["rng_state"] = rng_state + return state_dict + + +def _transpose_first_dim(t, num_splits, num_splits_first, model): + input_shape = t.size() + # We use a self_attention module but the values extracted aren't + # specific to self attention so should work for cross attention as well + while hasattr(model, 'module'): + model = model.module + attention_module = model.language_model.encoder.layers[0].self_attention + hidden_size_per_attention_head = attention_module.hidden_size_per_attention_head + num_attention_heads_per_partition = attention_module.num_attention_heads_per_partition + if num_splits_first: + """[num_splits * np * hn, h] + -->(view) [num_splits, np, hn, h] + -->(tranpose) [np, num_splits, hn, h] + -->(view) [np * num_splits * hn, h] """ + + intermediate_shape = \ + (num_splits, num_attention_heads_per_partition, + hidden_size_per_attention_head) + input_shape[1:] + + t = t.view(*intermediate_shape) + t = t.transpose(0, 1).contiguous() + else: + """[np * hn * num_splits, h] + -->(view) [np, hn, num_splits, h] + -->(tranpose) [np, num_splits, hn, h] + -->(view) [np * num_splits * hn, h] """ + + intermediate_shape = \ + (num_attention_heads_per_partition, + hidden_size_per_attention_head, num_splits) +\ + input_shape[1:] + + t = t.view(*intermediate_shape) + t = t.transpose(1, 2).contiguous() + t = t.view(*input_shape) + + return t + + +def fix_query_key_value_ordering(model, checkpoint_version): + """Fix up query/key/value matrix ordering if checkpoint + version is smaller than 2.0 + """ + if checkpoint_version < 2.0: + if isinstance(model, list): + assert len(model)==1 + model = model[0] + for name, param in model.named_parameters(): + if name.endswith(('.query_key_value.weight', '.query_key_value.bias')): + if checkpoint_version == 0: + fixed_param = _transpose_first_dim(param.data, 3, True, model) + elif checkpoint_version == 1.0: + fixed_param = _transpose_first_dim(param.data, 3, False, model) + else: + print_rank_0(f"Invalid checkpoint version {checkpoint_version}.") + sys.exit() + param.data.copy_(fixed_param) + if name.endswith(('.key_value.weight', '.key_value.bias')): + if checkpoint_version == 0: + fixed_param = _transpose_first_dim(param.data, 2, True, model) + elif checkpoint_version == 1.0: + fixed_param = _transpose_first_dim(param.data, 2, False, model) + else: + print_rank_0(f"Invalid checkpoint version {checkpoint_version}.") + sys.exit() + param.data.copy_(fixed_param) + print_rank_0(" successfully fixed query-key-values ordering for" + " checkpoint version {}".format(checkpoint_version)) + + +def _get_non_persistent_iteration(non_persistent_global_dir, args, checkpointing_context=None): + if args.non_persistent_ckpt_type is None: + return -1 + elif args.non_persistent_ckpt_type == "global": + tracker_filename = get_checkpoint_tracker_filename(non_persistent_global_dir) + if os.path.isfile(tracker_filename): + iteration, release = read_metadata(tracker_filename) + if release: + raise RuntimeError('Non-persistent checkpoint can\'t be a release checkpoint') + else: + iteration = -1 + print_rank_0('WARNING: could not find the metadata file {}'.format(tracker_filename)) + print_rank_0(' will not load any non-persistent checkpoint') + return iteration + elif args.non_persistent_ckpt_type == "local": + return checkpointing_context['local_checkpoint_manager'].find_latest() + else: + assert False, 'Please use local or global non-persistent checkpoints' \ + f'(got: {args.non_persistent_ckpt_type})' + + +def _load_non_persistent_base_checkpoint( + non_persistent_global_dir, + args, + rank0, + sharded_state_dict, + non_persistent_iteration, + checkpointing_context=None, +): + """ Load the base state_dict from a non-persistent distributed checkpoint. + Depending on the non_persistent_ckpt_type, different logic may be required. + """ + assert args.non_persistent_ckpt_type is not None + if args.non_persistent_ckpt_type == "global": + if not rank0: + print_rank_0( + f'Loading from a non-persistent checkpoint (non-persistent iter {non_persistent_iteration})' + ) + return _load_global_dist_base_checkpoint( + non_persistent_global_dir, args, rank0, sharded_state_dict, non_persistent_iteration, False, + checkpointing_context=checkpointing_context + ) + elif args.non_persistent_ckpt_type == "local": + intermediate_state_dict, checkpoint_name = checkpointing_context[ + 'local_checkpoint_manager' + ].load() + state_dict = intermediate_state_dict.to_state_dict( + sharded_state_dict, + algo=args.non_persistent_local_ckpt_algo, + parallelization_group=mpu.get_data_parallel_group(with_context_parallel=True) + ) + return state_dict, checkpoint_name, False, CheckpointType.LOCAL + else: + assert False, 'Please use local or global non-persistent checkpoints' \ + f'(got: {args.non_persistent_ckpt_type})' + + +def _load_global_dist_base_checkpoint( + load_dir, args, rank0, sharded_state_dict, iteration, release, checkpointing_context=None +): + """ Load the base state_dict from the given directory containing the global distributed checkpoint """ + if rank0: + checkpoint_name = find_checkpoint_rank_0(load_dir, iteration, release) + state_dict = dist_checkpointing.load_common_state_dict(checkpoint_name) + return state_dict, checkpoint_name, release, CheckpointType.GLOBAL + + if sharded_state_dict is None: + assert not args.auto_detect_ckpt_format and not args.use_dist_ckpt, ( + args.auto_detect_ckpt_format, + args.use_dist_ckpt, + ) + raise RuntimeError( + 'Detected load from a distributed checkpoint, ' + 'but neither --use-dist-ckpt nor --auto-detect-ckpt-format is set.' + ) + + checkpoint_name = get_checkpoint_name(load_dir, iteration, release, return_base_dir=True) + load_strategy = get_default_load_sharded_strategy(checkpoint_name) + # NOTE: `args.ckpt_fully_parallel_load` applies to both persistent and non-persistent checkpoints. + if args.ckpt_fully_parallel_load: + load_strategy = FullyParallelLoadStrategyWrapper( + load_strategy, mpu.get_data_parallel_group(with_context_parallel=True) + ) + if checkpointing_context is not None: + checkpointing_context["load_strategy"] = load_strategy + state_dict = dist_checkpointing.load(sharded_state_dict, checkpoint_name, + load_strategy, strict=args.dist_ckpt_strictness) + return state_dict, checkpoint_name, release, CheckpointType.GLOBAL + + +def _load_base_checkpoint( + load_dir, + args, + rank0=False, + sharded_state_dict=None, + checkpointing_context=None, +): + """ Load the base state_dict from the given directory + + If rank0 is true, just loads rank 0 checkpoint, ignoring arguments. + """ + # Try to load non-persistent checkpoint first + non_persistent_global_dir = ( + args.non_persistent_global_ckpt_dir + if args.non_persistent_global_ckpt_dir or load_dir is None + else os.path.join(load_dir, _NON_PERSISTENT_CKPT_SUBDIR) + ) + non_persistent_iteration = _get_non_persistent_iteration( + non_persistent_global_dir, args, checkpointing_context + ) + iteration, release = -1, False + tracker_filename = 'because load directory is not defined' + if load_dir is not None: + tracker_filename = get_checkpoint_tracker_filename(load_dir) + if os.path.isfile(tracker_filename): + iteration, release = read_metadata(tracker_filename) + if non_persistent_iteration != -1: # there is a non-persistent checkpoint + if non_persistent_iteration >= iteration: + return _load_non_persistent_base_checkpoint( + non_persistent_global_dir, + args, + rank0, + sharded_state_dict, + non_persistent_iteration, + checkpointing_context, + ) + else: + print_rank_0('WARNING: non-persistent checkpoints are older than persistent checkpoint') + + # Otherwise we are dealing with global checkpoints + # If no tracker file, return nothing + if iteration == -1: + if not rank0: + print_rank_0('WARNING: could not find the metadata file {}'.format(tracker_filename)) + print_rank_0(' will not load any checkpoints and will start from random') + # Conditionally exit if checkpoint not found. + if args.exit_on_missing_checkpoint: + print_rank_0(">> '--exit-on-missing-checkpoint' set ... exiting. <<") + if torch.distributed.is_initialized(): + torch.distributed.barrier() + sys.exit() + + return None, "", False, None + + # Determine the type of the checkpoint + checkpoint_name = get_checkpoint_name(load_dir, iteration, release, return_base_dir=True) + is_dist_ckpt = dist_checkpointing.check_is_distributed_checkpoint(checkpoint_name) + if not rank0: + dist_infix = "distributed " if is_dist_ckpt else "" + if release: + print_rank_0(f' loading release {dist_infix}checkpoint from {load_dir}') + else: + print_rank_0( + f' loading {dist_infix}checkpoint from {load_dir} at iteration {iteration}' + ) + + # Handle global distributed checkpoint + if is_dist_ckpt: + return _load_global_dist_base_checkpoint( + load_dir, args, rank0, sharded_state_dict, iteration, release, checkpointing_context=checkpointing_context + ) + # Handle global legacy checkpoint + if rank0: + checkpoint_name = find_checkpoint_rank_0(load_dir, iteration, release) + else: + checkpoint_name = get_checkpoint_name(load_dir, iteration, release, return_base_dir=False) + try: + state_dict = torch.load(checkpoint_name, map_location='cpu', weights_only=False) + except ModuleNotFoundError: + from megatron.legacy.fp16_deprecated import loss_scaler + + # For backward compatibility. + if not rank0: + print_rank_0(' > deserializing using the old code structure ...') + sys.modules['fp16.loss_scaler'] = sys.modules['megatron.legacy.fp16_deprecated.loss_scaler'] + sys.modules['megatron.fp16.loss_scaler'] = sys.modules['megatron.legacy.fp16_deprecated.loss_scaler'] + sys.modules['megatron.model'] = sys.modules['megatron.legacy.model'] + state_dict = torch.load(checkpoint_name, map_location='cpu', weights_only=False) + sys.modules.pop('fp16.loss_scaler', None) + sys.modules.pop('megatron.fp16.loss_scaler', None) + sys.modules.pop('megatron.model', None) + except Exception as e: + print('could not load the checkpoint') + print(e) + sys.exit() + + return state_dict, checkpoint_name, release, CheckpointType.LEGACY + + +def load_args_from_checkpoint( + args, load_arg='load', checkpointing_context=None +): + """Set required arguments from the checkpoint specified in the + arguments. + + Will overwrite arguments that have a non-None default value, but + will leave any arguments that default to None as set. + + Returns the same args NameSpace with the new values added/updated. + + If no checkpoint is specified in args, or if the checkpoint is + there but invalid, the arguments will not be modified + + """ + load_dir = getattr(args, load_arg) + + if load_dir is None: + print_rank_0('No load directory specified, using provided arguments.') + return args + + state_dict, checkpoint_name, release, ckpt_type = _load_base_checkpoint( + load_dir, + args, + rank0=True, + checkpointing_context=checkpointing_context, + ) + + # Args. + if not state_dict: + print_rank_0('Checkpoint not found to provide arguments, using provided arguments.') + return args + + if 'args' not in state_dict: + print_rank_0('Checkpoint provided does not have arguments saved, using provided arguments.') + return args + + checkpoint_args = state_dict['args'] + checkpoint_version = state_dict.get('checkpoint_version', 0) + args.iteration = state_dict['iteration'] + + # One-off conversion for foundation models + if hasattr(checkpoint_args, 'disable_bias_linear'): + setattr(checkpoint_args, 'add_bias_linear', not getattr(checkpoint_args, 'disable_bias_linear')) + + def _set_arg(arg_name, old_arg_name=None, force=False): + if not force and getattr(args, arg_name, None) is not None: + return + + if old_arg_name is not None: + checkpoint_value = getattr(checkpoint_args, old_arg_name, None) + else: + checkpoint_value = getattr(checkpoint_args, arg_name, None) + + if checkpoint_value is not None: + print_rank_0(f"Setting {arg_name} to {checkpoint_value} from checkpoint") + setattr(args, arg_name, checkpoint_value) + else: + print_rank_0(f"Checkpoint did not provide arguments {arg_name}") + + # Model args. + _set_arg('num_layers') + _set_arg('hidden_size') + _set_arg('ffn_hidden_size') + _set_arg('seq_length') + _set_arg('num_attention_heads') + _set_arg('num_query_groups', force=True) + _set_arg('group_query_attention', force=True) + _set_arg('kv_channels') + _set_arg('max_position_embeddings') + _set_arg('position_embedding_type', force=True) + _set_arg('add_position_embedding', force=True) + _set_arg('use_rotary_position_embeddings', force=True) + _set_arg('rotary_base', force=True) + _set_arg('rotary_percent', force=True) + _set_arg('rotary_interleaved', force=True) + _set_arg('add_bias_linear', force=True) + _set_arg('add_qkv_bias', force=True) + _set_arg('squared_relu', force=True) + _set_arg('swiglu', force=True) + _set_arg('untie_embeddings_and_output_weights', force=True) + _set_arg('apply_layernorm_1p', force=True) + _set_arg('normalization', force=True) + _set_arg('apply_query_key_layer_scaling', force=True) + _set_arg('attention_dropout', force=True) + _set_arg('hidden_dropout', force=True) + + _set_arg('hybrid_override_pattern', force=True) + _set_arg('spec', force=True) + _set_arg('hybrid_attention_ratio', force=True) + _set_arg('hybrid_mlp_ratio', force=True) + + _set_arg('num_experts', force=True) + _set_arg('moe_layer_freq', force=True) + _set_arg('moe_ffn_hidden_size', force=True) + _set_arg('moe_router_topk', force=True) + _set_arg('moe_token_dispatcher_type', force=True) + _set_arg('moe_router_pre_softmax', force=True) + _set_arg('moe_grouped_gemm', force=True) + _set_arg('moe_shared_expert_intermediate_size', force=True) + + # Tokenizer args. + _set_arg('tokenizer_type', force=True) + # Using checkpoint version might not always be safe (e.g., if running on different cluster). + if args.use_tokenizer_model_from_checkpoint_args: + _set_arg('tokenizer_model', force=True) + _set_arg('tiktoken_pattern', force=True) + _set_arg('padded_vocab_size') + + # Checkpoint args. + _set_arg('ckpt_format') + + # Model parallelism args. + if args.use_mp_args_from_checkpoint_args: + if checkpoint_version < 3.0: + _set_arg('tensor_model_parallel_size', 'model_parallel_size') + else: + _set_arg('tensor_model_parallel_size', force=True) + _set_arg('pipeline_model_parallel_size', force=True) + _set_arg('virtual_pipeline_model_parallel_size', force=True) + _set_arg('num_layers_per_virtual_pipeline_stage') + _set_arg('expert_model_parallel_size', force=True) + + return args, checkpoint_args + + +def fix_fp8_params_lose_precision_when_loading_dist_ckpt(state_dict): + """ + When "--fp8-param-gather" and "--use-dist-ckpt" are both enabled, the state dict read from + dist-checkpoint loses precision (the weights read from checkpoint go through the process of + bf16/fp16 -> fp8 -> bf16/fp16). This function is implemented to solve this problem. + When "--fp8-param-gather" is disabled, this function doesn't modify anything. + """ + for key in state_dict.keys(): + if key.startswith('model'): + for _, sharded_tensor in state_dict[key].items(): + if is_float8tensor(sharded_tensor.data): + sharded_tensor.data = dequantize_fp8_tensor(sharded_tensor.data).cpu() + + +def load_checkpoint(model, optimizer, opt_param_scheduler, load_arg='load', strict=True, + checkpointing_context=None, skip_load_to_model_and_opt=False): + """Load a model checkpoint and return the iteration. + strict (bool): whether to strictly enforce that the keys in + :attr:`state_dict` of the checkpoint match the names of + parameters and buffers in model. + skip_load_to_model_and_opt (bool): whether to call `load_state_dict` + for :attr:`model` and :attr:`optimizer`. In case of running FSDP2 + or other torch features that uses DTensor in state dict, the tensors + are already loaded in-place by `_load_base_checkpoint`. + """ + args = get_args() + load_dir = getattr(args, load_arg) + + # Finetuning directories + pretrained_dir = getattr(args, 'pretrained_checkpoint', None) + if pretrained_dir is not None and not checkpoint_exists(load_dir): + print_rank_0(f'Checkpoint file not found in load directory {load_dir} attempting to finetune ' + f'with checkpoint in {pretrained_dir}') + load_dir = pretrained_dir + if not checkpoint_exists(load_dir): + raise FileNotFoundError("No checkpoint found in load directory or pretrained directory") + args.finetune = True + + ddp_model = model + print("ddp",ddp_model) + model = unwrap_model(ddp_model) + + load_kwargs = {} + is_dist_ckpt = False + if ( + args.auto_detect_ckpt_format + or args.use_dist_ckpt + or args.non_persistent_save_interval is not None + ): + state_dict, checkpoint_name, release, ckpt_type = _load_base_checkpoint( + load_dir, + args, + rank0=True, + checkpointing_context=checkpointing_context, + ) + + is_dist_ckpt = ( + ckpt_type == CheckpointType.LOCAL + or dist_checkpointing.check_is_distributed_checkpoint(checkpoint_name) + ) + if is_dist_ckpt: + ckpt_tp_pp = ( + state_dict['args'].tensor_model_parallel_size, + state_dict['args'].pipeline_model_parallel_size, + getattr(state_dict['args'], 'encoder_tensor_model_parallel_size', 0), + getattr(state_dict['args'], 'encoder_pipeline_model_parallel_size', 0), + ) + run_tp_pp = ( + args.tensor_model_parallel_size, + args.pipeline_model_parallel_size, + # TODO: change this to args.encoder_tensor_model_parallel_size after 30th Nov 24 + getattr(args, 'encoder_tensor_model_parallel_size', 0), + getattr(args, 'encoder_pipeline_model_parallel_size', 0), + ) + mismatch_msg = "(TP, PP, encoder TP, encoder PP) mismatch after resume ({} vs {} from checkpoint)".format( + run_tp_pp, ckpt_tp_pp + ) + + # Determine if RNG state will be loaded + if (ckpt_tp_pp == run_tp_pp and not release and not args.finetune and not args.no_load_rng + and not getattr(state_dict['args'], 'no_save_rng', False)): + gen_sd_rng_state = get_rng_state(True) # we can load the rng state + else: + gen_sd_rng_state = None + if ckpt_tp_pp != run_tp_pp: + print_rank_0("{}: RNG state will be ignored".format(mismatch_msg)) + + optim_sd_kwargs = dict(is_loading=True) + # Determine if optimizer state will be loaded + if (not release and not args.finetune and not args.no_load_optim + and not getattr(state_dict['args'], 'no_save_optim', False)): + gen_sd_optim = optimizer + gen_sd_opt_param_scheduler = opt_param_scheduler + + if args.use_distributed_optimizer: + optim_sd_kwargs['sharding_type'] = ('fully_sharded_model_space' + if getattr(state_dict['args'], + 'ckpt_fully_parallel_save', False) + else 'dp_zero_gather_scatter') + # This is for backwards-compatibility. + # Can be removed once 'fully_sharded_bucket_space' loading is removed + for maybe_dist_opt_optim_state in (state_dict['optimizer'], *state_dict['optimizer'].values()): + if 'param_state_sharding_type' in maybe_dist_opt_optim_state: + if maybe_dist_opt_optim_state['param_state_sharding_type'] == 'fully_sharded_bucket_space': + print_rank_0('Detected deprecated `fully_sharded_bucket_space` ' + 'DistributedOptimizer checkpoint format') + optim_sd_kwargs['sharding_type'] = \ + maybe_dist_opt_optim_state['param_state_sharding_type'] + break + + if ckpt_tp_pp != run_tp_pp and optim_sd_kwargs['sharding_type'] != 'fully_sharded_model_space': + raise RuntimeError(f"{mismatch_msg}: not supported for DistributedOptimizer with " + f"sharding type {optim_sd_kwargs['sharding_type']}." + f" Please use `--ckpt-fully-parallel-save` flag during checkpoint saving.") + else: + gen_sd_optim = None + gen_sd_opt_param_scheduler = None + + # Determine if rerun state will be loaded + if ( + ckpt_tp_pp == run_tp_pp + and not release + and not args.finetune + and 'rerun_state_machine' in state_dict + ): + rerun_state_machine = get_rerun_state_machine() + gen_sd_rerun_state = rerun_state_machine.state_dict( + data_iterator=None, use_dist_ckpt=True + ) + else: + gen_sd_rerun_state = None + if ckpt_tp_pp != run_tp_pp: + print_rank_0("{}: Rerun state will be ignored".format(mismatch_msg)) + + # [ModelOpt]: IMPORTANT! Restoring modelopt_state (sharded or not) must be performed + # after the model instance has been created and before _load_base_checkpoint is called. + if has_nvidia_modelopt: + if ckpt_type == CheckpointType.LOCAL: + print_rank_0('WARNING: Local checkpointing does not support nvidia_modelopt.') + elif ckpt_type == CheckpointType.GLOBAL: + restore_modelopt_state(model, state_dict) + else: + restore_sharded_modelopt_state(model, checkpoint_name) + + # [ModelOpt]: Initial loading from non-resume sharded checkpoint to a Distillation Model + # will result in key mismatch with loss modules potentially containing parameters, since + # it requires generating a state_dict before loading. Here we hide those modules if present. + with contextlib.ExitStack() as stack: # Allows multiple context managers for each model shard + if args.finetune and hasattr(model[0], "hide_loss_modules"): + for m in model: + stack.enter_context(m.hide_loss_modules()) + load_kwargs['sharded_state_dict'] = generate_state_dict( + args, model, gen_sd_optim, gen_sd_opt_param_scheduler, gen_sd_rng_state, + use_dist_ckpt=True, optim_sd_kwargs=optim_sd_kwargs, rerun_state=gen_sd_rerun_state + ) + + # When "--fp8-param-gather" is disabled, this function doesn't modify anything. + fix_fp8_params_lose_precision_when_loading_dist_ckpt(load_kwargs['sharded_state_dict']) + + state_dict, checkpoint_name, release, ckpt_type = _load_base_checkpoint( + load_dir, args, rank0=False, checkpointing_context=checkpointing_context, + **load_kwargs + ) + + # Checkpoint not loaded. + if state_dict is None: + # Iteration and num_floating_point_operations_so_far default to 0. + return 0, 0 + + # Set checkpoint version. + set_checkpoint_version(state_dict.get('checkpoint_version', 0)) + + # Set iteration. + if args.finetune or release: + iteration = 0 + else: + try: + iteration = state_dict['iteration'] + except KeyError: + try: # Backward compatible with older checkpoints + iteration = state_dict['total_iters'] + except KeyError: + print_rank_0('A metadata file exists but unable to load ' + 'iteration from checkpoint {}, exiting'.format(checkpoint_name)) + sys.exit() + num_floating_point_operations_so_far = state_dict.get('num_floating_point_operations_so_far', 0) + + # Check arguments. + if load_arg == 'load': + assert args.consumed_train_samples == 0 + assert args.skipped_train_samples == 0 + assert args.consumed_valid_samples == 0 + if 'args' in state_dict and not args.finetune: + checkpoint_args = state_dict['args'] + check_checkpoint_args(checkpoint_args) + args.consumed_train_samples = getattr(checkpoint_args, 'consumed_train_samples', 0) + args.skipped_train_samples = getattr(checkpoint_args, 'skipped_train_samples', 0) + update_num_microbatches(consumed_samples=args.consumed_train_samples, verbose=True) + args.consumed_valid_samples = getattr(checkpoint_args, 'consumed_valid_samples', 0) + else: + print_rank_0('could not find arguments in the checkpoint ...') + + # Model. + strict = False if args.retro_add_retriever else strict + if not skip_load_to_model_and_opt: + if len(ddp_model) == 1: + ddp_model[0].load_state_dict(state_dict['model'], strict=strict) + else: + for i in range(len(ddp_model)): + mpu.set_virtual_pipeline_model_parallel_rank(i) + ddp_model[i].load_state_dict(state_dict['model%d' % i], strict=strict) + + # Fix up query/key/value matrix ordering if needed. + checkpoint_version = get_checkpoint_version() + print_rank_0(f' checkpoint version {checkpoint_version}') + fix_query_key_value_ordering(model, checkpoint_version) + + # Optimizer. + if not release and not args.finetune and not args.no_load_optim: + try: + # Load state dict. + if not skip_load_to_model_and_opt and optimizer is not None and not optimizer.is_stub_optimizer: + optimizer.load_state_dict(state_dict['optimizer']) + + # Load distributed optimizer's custom parameter state. + # For distributed checkpoint it's already loaded in load_state_dict above + if args.use_distributed_optimizer and not is_dist_ckpt: + # NOTE: this is a manual read of the tracker file. + # This code should not be reached when reading from a non_persistent checkpoint + assert not is_dist_ckpt + tracker_filename = get_checkpoint_tracker_filename(load_dir) + iteration, release = read_metadata(tracker_filename) + model_checkpoint_name = get_checkpoint_name(load_dir, iteration, release) + optim_checkpoint_name = get_distributed_optimizer_checkpoint_name(model_checkpoint_name) + optimizer.load_parameter_state(optim_checkpoint_name, + update_legacy_format=args.ckpt_convert_update_legacy_dist_opt_format) + + # Load scheduler. + if opt_param_scheduler is not None: + if 'lr_scheduler' in state_dict: # backward compatbility + opt_param_scheduler.load_state_dict(state_dict['lr_scheduler']) + else: + opt_param_scheduler.load_state_dict(state_dict['opt_param_scheduler']) + except KeyError as e: + print_rank_0('Unable to load optimizer from checkpoint {}. ' + 'Specify --no-load-optim or --finetune to prevent ' + 'attempting to load the optimizer state, ' + 'exiting ...'.format(checkpoint_name)) + raise e + else: + if (args.fp16 or args.bf16) and optimizer is not None: + optimizer.reload_model_params() + + # rerun state + try: + if 'rerun_state_machine' in state_dict: + get_rerun_state_machine().load_state_dict(state_dict['rerun_state_machine']) + except Exception as e: + print(f"Unable to restore RerunMachine from checkpoint: {e}") + sys.exit() + + # rng states. + if not release and not args.finetune and not args.no_load_rng: + try: + if 'rng_state' in state_dict: + # access rng_state for data parallel rank + if args.data_parallel_random_init: + rng_state = state_dict['rng_state'][mpu.get_data_parallel_rank()] + else: + rng_state = state_dict['rng_state'][0] + random.setstate(rng_state['random_rng_state']) + np.random.set_state(rng_state['np_rng_state']) + torch.set_rng_state(rng_state['torch_rng_state']) + torch.cuda.set_rng_state(rng_state['cuda_rng_state']) + # Check for empty states array + if not rng_state['rng_tracker_states']: + raise KeyError + tensor_parallel.get_cuda_rng_tracker().set_states( + rng_state['rng_tracker_states']) + else: # backward compatability + random.setstate(state_dict['random_rng_state']) + np.random.set_state(state_dict['np_rng_state']) + torch.set_rng_state(state_dict['torch_rng_state']) + torch.cuda.set_rng_state(state_dict['cuda_rng_state']) + # Check for empty states array + if not state_dict['rng_tracker_states']: + raise KeyError + tensor_parallel.get_cuda_rng_tracker().set_states( + state_dict['rng_tracker_states']) + except KeyError: + print_rank_0('Unable to load rng state from checkpoint {}. ' + 'Specify --no-load-rng or --finetune to prevent ' + 'attempting to load the rng state, ' + 'exiting ...'.format(checkpoint_name)) + sys.exit() + + # Some utilities want to load a checkpoint without distributed being initialized + if torch.distributed.is_initialized(): + torch.distributed.barrier() + + print_rank_0(f' successfully loaded checkpoint from {load_dir} ' + f'[ t {mpu.get_tensor_model_parallel_rank() + 1}/{mpu.get_tensor_model_parallel_world_size()}, ' + f'p {mpu.get_pipeline_model_parallel_rank() + 1}/{mpu.get_pipeline_model_parallel_world_size()} ] ' + f'at iteration {iteration}') + + # Additional callback for wandb (last rank) + if not torch.distributed.is_initialized() \ + or is_last_rank(): + wandb_utils.on_load_checkpoint_success(checkpoint_name, load_dir) + + torch.cuda.empty_cache() + + if iteration > 0: + # Notify FT that a checkpoint was loaded. + is_local_chkpt = (ckpt_type == CheckpointType.LOCAL) + ft_integration.on_checkpoint_loaded(is_local_chkpt=is_local_chkpt) + + return iteration, num_floating_point_operations_so_far + + +def load_biencoder_checkpoint(model, only_query_model=False, + only_context_model=False, custom_load_path=None): + """ + selectively load retrieval models for indexing/retrieving + from saved checkpoints + """ + + args = get_args() + + model = unwrap_model(model) + + load_path = custom_load_path if custom_load_path is not None else args.load + + tracker_filename = get_checkpoint_tracker_filename(load_path) + with open(tracker_filename, 'r') as f: + iteration = int(f.read().strip()) + + checkpoint_name = get_checkpoint_name(load_path, iteration, + args.use_distributed_optimizer, + release=False) + + if mpu.get_data_parallel_rank() == 0: + print('global rank {} is loading checkpoint {}'.format( + torch.distributed.get_rank(), checkpoint_name)) + + state_dict = torch.load(checkpoint_name, map_location='cpu', weights_only=False) + ret_state_dict = state_dict['model'] + + if only_query_model: + ret_state_dict.pop('context_model') + if only_context_model: + ret_state_dict.pop('query_model') + + assert len(model) == 1 + model[0].load_state_dict(ret_state_dict) + torch.distributed.barrier() + + if mpu.get_data_parallel_rank() == 0: + print(' successfully loaded {}'.format(checkpoint_name)) + + return model diff --git a/aiak_megatron/megatron_core.egg-info/PKG-INFO b/aiak_megatron/megatron_core.egg-info/PKG-INFO new file mode 100644 index 00000000..53970e3b --- /dev/null +++ b/aiak_megatron/megatron_core.egg-info/PKG-INFO @@ -0,0 +1,332 @@ +Metadata-Version: 2.4 +Name: megatron-core +Version: 0.12.0rc0 +Summary: Megatron Core - a library for efficient and scalable training of transformer based models +Home-page: https://github.com/NVIDIA/Megatron-LM/megatron/core +Download-URL: https://github.com/NVIDIA/Megatron-LM/releases +Author: NVIDIA +Author-email: NVIDIA +Maintainer: NVIDIA +Maintainer-email: NVIDIA +License: The following applies to all files unless otherwise noted: + + # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: + # * Redistributions of source code must retain the above copyright + # notice, this list of conditions and the following disclaimer. + # * Redistributions in binary form must reproduce the above copyright + # notice, this list of conditions and the following disclaimer in the + # documentation and/or other materials provided with the distribution. + # * Neither the name of NVIDIA CORPORATION nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -- + + This repository also contains code from Hugging Face Inc., Google Research, + Facebook (from their Fairseq, Dino, and ParlAI projects), Microsoft (from their + Swin-Transformer project), Philip Popien, the Mamba project (Tri Dao and + Albert Gu), and the Triton language and compiler project (Philippe Tillet and + OpenAI). Files from these organizations have notices at the top of each file. + Below are licenses used in those files, as indicated. + + + -------------------------------------------------------------------------------------- + -- LICENSE FOR Facebook, huggingface, Google Research, LLaVA, Mamba, and vLLM code -- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- + LICENSE FOR + Facebook, Inc. and its affiliates, + Meta Platforms, Inc. and its affiliates, + Microsoft Corporation, + OpenGVLab/InternVL, + Triton language and compiler, + and DeepSeek. + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + +Project-URL: Download, https://github.com/NVIDIA/Megatron-LM/releases +Project-URL: Homepage, https://github.com/NVIDIA/Megatron-LM/megatron/core +Keywords: NLP,NLU,deep,gpu,language,learning,learning,machine,nvidia,pytorch,torch,transformer +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Information Technology +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence +Classifier: Topic :: Scientific/Engineering :: Image Recognition +Classifier: Topic :: Scientific/Engineering :: Mathematics +Classifier: Topic :: Scientific/Engineering +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Utilities +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: einops +Requires-Dist: flask-restful +Requires-Dist: nltk +Requires-Dist: pytest +Requires-Dist: pytest_asyncio +Requires-Dist: pytest-cov +Requires-Dist: pytest_mock +Requires-Dist: pytest-random-order +Requires-Dist: sentencepiece +Requires-Dist: tiktoken +Requires-Dist: wrapt +Requires-Dist: zarr +Requires-Dist: wandb +Requires-Dist: tensorstore!=0.1.46,!=0.1.72 +Requires-Dist: torch +Requires-Dist: nvidia-modelopt[torch]>=0.19.0; sys_platform != "darwin" +Requires-Dist: nvidia-resiliency-ext; sys_platform != "darwin" +Requires-Dist: torch +Requires-Dist: packaging +Dynamic: author +Dynamic: download-url +Dynamic: home-page +Dynamic: license-file +Dynamic: maintainer +Dynamic: requires-dist diff --git a/aiak_megatron/megatron_core.egg-info/SOURCES.txt b/aiak_megatron/megatron_core.egg-info/SOURCES.txt new file mode 100644 index 00000000..7be6c918 --- /dev/null +++ b/aiak_megatron/megatron_core.egg-info/SOURCES.txt @@ -0,0 +1,283 @@ +LICENSE +MANIFEST.in +pyproject.toml +setup.py +megatron/core/README.md +megatron/core/__init__.py +megatron/core/config.py +megatron/core/config_logger.py +megatron/core/enums.py +megatron/core/fp8_utils.py +megatron/core/inference_params.py +megatron/core/jit.py +megatron/core/model_parallel_config.py +megatron/core/num_microbatches_calculator.py +megatron/core/optimizer_param_scheduler.py +megatron/core/package_info.py +megatron/core/packed_seq_params.py +megatron/core/parallel_state.py +megatron/core/requirements.txt +megatron/core/rerun_state_machine.py +megatron/core/timers.py +megatron/core/utils.py +megatron/core/datasets/__init__.py +megatron/core/datasets/bert_dataset.py +megatron/core/datasets/blended_dataset.py +megatron/core/datasets/blended_megatron_dataset_builder.py +megatron/core/datasets/blended_megatron_dataset_config.py +megatron/core/datasets/gpt_dataset.py +megatron/core/datasets/helpers.cpp +megatron/core/datasets/helpers.py +megatron/core/datasets/indexed_dataset.py +megatron/core/datasets/masked_dataset.py +megatron/core/datasets/megatron_dataset.py +megatron/core/datasets/megatron_tokenizer.py +megatron/core/datasets/multimodal_dataset.py +megatron/core/datasets/t5_dataset.py +megatron/core/datasets/utils.py +megatron/core/datasets/utils_s3.py +megatron/core/datasets/retro/__init__.py +megatron/core/datasets/retro/external_libs.py +megatron/core/datasets/retro/utils.py +megatron/core/datasets/retro/config/__init__.py +megatron/core/datasets/retro/config/bert_embedders.py +megatron/core/datasets/retro/config/config.py +megatron/core/datasets/retro/config/gpt_chunk_datasets.py +megatron/core/datasets/retro/config/tokenizers.py +megatron/core/datasets/retro/db/__init__.py +megatron/core/datasets/retro/db/build.py +megatron/core/datasets/retro/db/dataset.py +megatron/core/datasets/retro/db/utils.py +megatron/core/datasets/retro/index/__init__.py +megatron/core/datasets/retro/index/build.py +megatron/core/datasets/retro/index/factory.py +megatron/core/datasets/retro/index/index.py +megatron/core/datasets/retro/index/utils.py +megatron/core/datasets/retro/index/validate.py +megatron/core/datasets/retro/index/indexes/__init__.py +megatron/core/datasets/retro/index/indexes/faiss_base.py +megatron/core/datasets/retro/index/indexes/faiss_par_add.py +megatron/core/datasets/retro/query/__init__.py +megatron/core/datasets/retro/query/gpt_chunk_dataset.py +megatron/core/datasets/retro/query/multi_split_gpt_dataset.py +megatron/core/datasets/retro/query/query.py +megatron/core/datasets/retro/query/retro_dataset.py +megatron/core/datasets/retro/query/utils.py +megatron/core/dist_checkpointing/__init__.py +megatron/core/dist_checkpointing/core.py +megatron/core/dist_checkpointing/dict_utils.py +megatron/core/dist_checkpointing/exchange_utils.py +megatron/core/dist_checkpointing/mapping.py +megatron/core/dist_checkpointing/optimizer.py +megatron/core/dist_checkpointing/serialization.py +megatron/core/dist_checkpointing/state_dict_utils.py +megatron/core/dist_checkpointing/tensor_aware_state_dict.py +megatron/core/dist_checkpointing/utils.py +megatron/core/dist_checkpointing/validation.py +megatron/core/dist_checkpointing/strategies/__init__.py +megatron/core/dist_checkpointing/strategies/async_utils.py +megatron/core/dist_checkpointing/strategies/base.py +megatron/core/dist_checkpointing/strategies/cached_metadata_filesystem_reader.py +megatron/core/dist_checkpointing/strategies/common.py +megatron/core/dist_checkpointing/strategies/filesystem_async.py +megatron/core/dist_checkpointing/strategies/fully_parallel.py +megatron/core/dist_checkpointing/strategies/resharding.py +megatron/core/dist_checkpointing/strategies/state_dict_saver.py +megatron/core/dist_checkpointing/strategies/tensorstore.py +megatron/core/dist_checkpointing/strategies/torch.py +megatron/core/dist_checkpointing/strategies/two_stage.py +megatron/core/dist_checkpointing/strategies/zarr.py +megatron/core/distributed/__init__.py +megatron/core/distributed/data_parallel_base.py +megatron/core/distributed/distributed_data_parallel.py +megatron/core/distributed/distributed_data_parallel_config.py +megatron/core/distributed/finalize_model_grads.py +megatron/core/distributed/param_and_grad_buffer.py +megatron/core/distributed/torch_fully_sharded_data_parallel.py +megatron/core/distributed/custom_fsdp/__init__.py +megatron/core/distributed/custom_fsdp/fully_sharded_data_parallel.py +megatron/core/distributed/custom_fsdp/param_and_grad_buffer.py +megatron/core/export/__init__.py +megatron/core/export/data_type.py +megatron/core/export/export_config.py +megatron/core/export/model_type.py +megatron/core/export/trtllm/__init__.py +megatron/core/export/trtllm/trt_model_config.py +megatron/core/export/trtllm/trt_model_type.py +megatron/core/export/trtllm/trtllm_helper.py +megatron/core/export/trtllm/trtllm_layers.py +megatron/core/export/trtllm/engine_builder/__init__.py +megatron/core/export/trtllm/engine_builder/trtllm_engine_builder.py +megatron/core/export/trtllm/model_to_trllm_mapping/__init__.py +megatron/core/export/trtllm/model_to_trllm_mapping/default_conversion_dict.py +megatron/core/export/trtllm/trtllm_weights_converter/__init__.py +megatron/core/export/trtllm/trtllm_weights_converter/distributed_trtllm_model_weights_converter.py +megatron/core/export/trtllm/trtllm_weights_converter/single_device_trtllm_model_weights_converter.py +megatron/core/extensions/__init__.py +megatron/core/extensions/transformer_engine.py +megatron/core/fusions/__init__.py +megatron/core/fusions/fused_bias_dropout.py +megatron/core/fusions/fused_bias_geglu.py +megatron/core/fusions/fused_bias_gelu.py +megatron/core/fusions/fused_bias_swiglu.py +megatron/core/fusions/fused_cross_entropy.py +megatron/core/fusions/fused_indices_converter.py +megatron/core/fusions/fused_layer_norm.py +megatron/core/fusions/fused_pad_routing_map.py +megatron/core/fusions/fused_softmax.py +megatron/core/inference/__init__.py +megatron/core/inference/async_stream.py +megatron/core/inference/common_inference_params.py +megatron/core/inference/communication_utils.py +megatron/core/inference/inference_request.py +megatron/core/inference/sampling_params.py +megatron/core/inference/scheduler.py +megatron/core/inference/utils.py +megatron/core/inference/ammo_support/__init__.py +megatron/core/inference/ammo_support/gpt/__init__.py +megatron/core/inference/ammo_support/gpt/model_specs.py +megatron/core/inference/ammo_support/gpt/state_dict_hooks.py +megatron/core/inference/engines/__init__.py +megatron/core/inference/engines/abstract_engine.py +megatron/core/inference/engines/mcore_engine.py +megatron/core/inference/gpt/__init__.py +megatron/core/inference/gpt/model_specs.py +megatron/core/inference/gpt/state_dict_hooks.py +megatron/core/inference/model_inference_wrappers/__init__.py +megatron/core/inference/model_inference_wrappers/abstract_model_inference_wrapper.py +megatron/core/inference/model_inference_wrappers/inference_wrapper_config.py +megatron/core/inference/model_inference_wrappers/gpt/__init__.py +megatron/core/inference/model_inference_wrappers/gpt/gpt_inference_wrapper.py +megatron/core/inference/model_inference_wrappers/multimodal/vlm_inference_wrapper.py +megatron/core/inference/model_inference_wrappers/t5/__init__.py +megatron/core/inference/model_inference_wrappers/t5/t5_inference_wrapper.py +megatron/core/inference/modelopt_support/__init__.py +megatron/core/inference/modelopt_support/gpt/__init__.py +megatron/core/inference/modelopt_support/gpt/model_specs.py +megatron/core/inference/modelopt_support/gpt/state_dict_hooks.py +megatron/core/inference/modelopt_support/mamba/__init__.py +megatron/core/inference/modelopt_support/mamba/model_specs.py +megatron/core/inference/text_generation_controllers/__init__.py +megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py +megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py +megatron/core/inference/text_generation_controllers/text_generation_controller.py +megatron/core/inference/text_generation_controllers/vlm_text_generation_controller.py +megatron/core/models/__init__.py +megatron/core/models/T5/__init__.py +megatron/core/models/T5/t5_model.py +megatron/core/models/T5/t5_spec.py +megatron/core/models/bert/__init__.py +megatron/core/models/bert/bert_layer_specs.py +megatron/core/models/bert/bert_lm_head.py +megatron/core/models/bert/bert_model.py +megatron/core/models/bert/pooler.py +megatron/core/models/common/__init__.py +megatron/core/models/common/embeddings/__init__.py +megatron/core/models/common/embeddings/language_model_embedding.py +megatron/core/models/common/embeddings/relative_pos_embedding.py +megatron/core/models/common/embeddings/rope_utils.py +megatron/core/models/common/embeddings/rotary_pos_embedding.py +megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py +megatron/core/models/common/language_module/__init__.py +megatron/core/models/common/language_module/language_module.py +megatron/core/models/common/vision_module/__init__.py +megatron/core/models/common/vision_module/vision_module.py +megatron/core/models/gpt/__init__.py +megatron/core/models/gpt/gpt_layer_specs.py +megatron/core/models/gpt/gpt_model.py +megatron/core/models/gpt/moe_module_specs.py +megatron/core/models/huggingface/__init__.py +megatron/core/models/huggingface/clip_model.py +megatron/core/models/huggingface/module.py +megatron/core/models/huggingface/qwen_model.py +megatron/core/models/mamba/__init__.py +megatron/core/models/mamba/mamba_layer_specs.py +megatron/core/models/mamba/mamba_model.py +megatron/core/models/multimodal/__init__.py +megatron/core/models/multimodal/context_parallel.py +megatron/core/models/multimodal/llava_model.py +megatron/core/models/multimodal/llava_spec.py +megatron/core/models/retro/__init__.py +megatron/core/models/retro/base_attention.py +megatron/core/models/retro/config.py +megatron/core/models/retro/decoder_attention.py +megatron/core/models/retro/decoder_spec.py +megatron/core/models/retro/encoder_attention.py +megatron/core/models/retro/encoder_spec.py +megatron/core/models/retro/model.py +megatron/core/models/retro/utils.py +megatron/core/models/vision/__init__.py +megatron/core/models/vision/clip_vit_model.py +megatron/core/models/vision/multimodal_projector.py +megatron/core/models/vision/radio.py +megatron/core/models/vision/vit_layer_specs.py +megatron/core/optimizer/__init__.py +megatron/core/optimizer/clip_grads.py +megatron/core/optimizer/distrib_optimizer.py +megatron/core/optimizer/grad_scaler.py +megatron/core/optimizer/optimizer.py +megatron/core/optimizer/optimizer_config.py +megatron/core/optimizer/cpu_offloading/__init__.py +megatron/core/optimizer/cpu_offloading/hybrid_optimizer.py +megatron/core/pipeline_parallel/__init__.py +megatron/core/pipeline_parallel/combined_1f1b.py +megatron/core/pipeline_parallel/p2p_communication.py +megatron/core/pipeline_parallel/schedules.py +megatron/core/post_training/__init__.py +megatron/core/post_training/modelopt/__init__.py +megatron/core/post_training/modelopt/layers.py +megatron/core/post_training/modelopt/gpt/__init__.py +megatron/core/post_training/modelopt/gpt/model_specs.py +megatron/core/post_training/modelopt/gpt/state_dict_hooks.py +megatron/core/post_training/modelopt/mamba/__init__.py +megatron/core/post_training/modelopt/mamba/model_specs.py +megatron/core/ssm/__init__.py +megatron/core/ssm/mamba_block.py +megatron/core/ssm/mamba_hybrid_layer_allocation.py +megatron/core/ssm/mamba_layer.py +megatron/core/ssm/mamba_mixer.py +megatron/core/ssm/triton_cache_manager.py +megatron/core/tensor_parallel/__init__.py +megatron/core/tensor_parallel/cross_entropy.py +megatron/core/tensor_parallel/data.py +megatron/core/tensor_parallel/layers.py +megatron/core/tensor_parallel/mappings.py +megatron/core/tensor_parallel/random.py +megatron/core/tensor_parallel/utils.py +megatron/core/transformer/__init__.py +megatron/core/transformer/attention.py +megatron/core/transformer/cuda_graphs.py +megatron/core/transformer/dot_product_attention.py +megatron/core/transformer/enums.py +megatron/core/transformer/identity_op.py +megatron/core/transformer/mlp.py +megatron/core/transformer/module.py +megatron/core/transformer/multi_latent_attention.py +megatron/core/transformer/spec_utils.py +megatron/core/transformer/torch_layer_norm.py +megatron/core/transformer/torch_norm.py +megatron/core/transformer/transformer_block.py +megatron/core/transformer/transformer_config.py +megatron/core/transformer/transformer_layer.py +megatron/core/transformer/utils.py +megatron/core/transformer/custom_layers/__init__.py +megatron/core/transformer/custom_layers/transformer_engine.py +megatron/core/transformer/moe/__init__.py +megatron/core/transformer/moe/experts.py +megatron/core/transformer/moe/fused_a2a.py +megatron/core/transformer/moe/grouped_gemm_util.py +megatron/core/transformer/moe/legacy_a2a_token_dispatcher.py +megatron/core/transformer/moe/moe_layer.py +megatron/core/transformer/moe/moe_utils.py +megatron/core/transformer/moe/router.py +megatron/core/transformer/moe/shared_experts.py +megatron/core/transformer/moe/token_dispatcher.py +megatron/core/transformer/moe/upcycling_utils.py +megatron_core.egg-info/PKG-INFO +megatron_core.egg-info/SOURCES.txt +megatron_core.egg-info/dependency_links.txt +megatron_core.egg-info/requires.txt +megatron_core.egg-info/top_level.txt +requirements/pytorch_24.01/requirements.txt +requirements/pytorch_24.07/requirements.txt +requirements/pytorch_24.10/requirements.txt \ No newline at end of file diff --git a/aiak_megatron/megatron_core.egg-info/dependency_links.txt b/aiak_megatron/megatron_core.egg-info/dependency_links.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/aiak_megatron/megatron_core.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/aiak_megatron/megatron_core.egg-info/requires.txt b/aiak_megatron/megatron_core.egg-info/requires.txt new file mode 100644 index 00000000..9bf8fc9f --- /dev/null +++ b/aiak_megatron/megatron_core.egg-info/requires.txt @@ -0,0 +1,21 @@ +einops +flask-restful +nltk +pytest +pytest_asyncio +pytest-cov +pytest_mock +pytest-random-order +sentencepiece +tiktoken +wrapt +zarr +wandb +tensorstore!=0.1.46,!=0.1.72 +torch +torch +packaging + +[:sys_platform != "darwin"] +nvidia-modelopt[torch]>=0.19.0 +nvidia-resiliency-ext diff --git a/aiak_megatron/megatron_core.egg-info/top_level.txt b/aiak_megatron/megatron_core.egg-info/top_level.txt new file mode 100644 index 00000000..3bfbb11f --- /dev/null +++ b/aiak_megatron/megatron_core.egg-info/top_level.txt @@ -0,0 +1 @@ +megatron diff --git a/aiak_training_llm/data/multimodal/task_encoder.py b/aiak_training_llm/data/multimodal/task_encoder.py index ee3bd692..59a96a82 100644 --- a/aiak_training_llm/data/multimodal/task_encoder.py +++ b/aiak_training_llm/data/multimodal/task_encoder.py @@ -9,7 +9,7 @@ import traceback from dataclasses import dataclass from typing import Dict, List, Optional, Tuple, Union - +import logging from PIL import Image from torchvision.transforms import ToPILImage import numpy as np @@ -309,26 +309,121 @@ def select_samples_to_pack(self, samples: List[ImageTaskSample]) -> List[List[Im return packed_samples + # @stateless + # def pack_selected_samples(self, samples: List[ImageTaskSample]) -> List[ImageTaskSamplePacked]: + # """ + # Function to pack a list of ImageTaskSample into a single ImageTaskSamplePacked. + + # NOTE: Energon dataloader calls this method internally if packing is used. + # Please see https://nvidia.github.io/Megatron-Energon/packing.html + + # Args: + # samples: List of ImageTaskSample instances to pack into one sample. + + # Returns: + # ImageTaskSamplePacked instance. + # """ + + # packing_seq_len = self.args.seq_length + + # packed_tokens = [] + # packed_labels = [] + # packed_masks = [] + # packed_imgs = [] + # packed_videos = [] + + # current_length = 0 + # max_length = 0 + # cu_lengths = [0] + + # # Process each sample and build lists that we will concatenate to create the packed sample. + # for _, sample in enumerate(samples): + # sample_len = sample.total_len + + # # if sample_len > max_length: + # # max_length = sample_len + # # If a single sample exceeds the allowed packing_seq_len, truncate its per-token fields. + # # Note: Do NOT try to slice the sample object itself (it is not subscriptable). + # if sample_len > packing_seq_len: + # logging.getLogger(__name__).warning( + # f"Sample {getattr(sample, '__key__', '')} length {sample_len} > packing_seq_len {packing_seq_len}. Truncating token fields." + # ) + # # Truncate token fields to packing_seq_len (only token-like fields) + # if hasattr(sample, "tokens") and sample.tokens is not None: + # # tokens might be torch.Tensor or numpy array; keep dtype + # sample.tokens = sample.tokens[:packing_seq_len] + # if hasattr(sample, "labels") and sample.labels is not None: + # sample.labels = sample.labels[:packing_seq_len] + # if hasattr(sample, "attn_mask") and sample.attn_mask is not None: + # sample.attn_mask = sample.attn_mask[:packing_seq_len] + + # # If total_len property is used, update it to reflect truncation + # sample_len = min(sample_len, packing_seq_len) + # try: + # sample.total_len = sample_len + # except Exception: + # # If dataclass is frozen or doesn't allow assign, ignore; + # # we will use sample_len variable for logic below. + # pass + + # # update max_length to be the maximum sample length after truncation + # if sample_len > max_length: + # max_length = sample_len + + + + # # If adding this sample exceeds the max length, stop. + # # This should not happen. + # # The select_samples_to_pack method should have already ensured that the samples fit. + # if current_length + sample_len > packing_seq_len: + # print(f"packing_seq_len:{packing_seq_len}----<<<<<----{current_length + sample_len}") + # raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") + + # # Add the sample's tokens and labels + # packed_tokens.append(sample.tokens) + # packed_labels.append(sample.labels) + # packed_masks.append(sample.attn_mask) + + # # Add the images + # if sample.imgs is not None: + # packed_imgs += sample.imgs + # if sample.pixel_values_videos is not None: + # packed_videos += sample.pixel_values_videos + # current_length += sample_len + # cu_lengths.append(current_length) + + # # Concatenate packed tokens and labels. + # packed_tokens = torch.cat(packed_tokens, dim=0) + # packed_labels = torch.cat(packed_labels, dim=0) + # packed_masks = torch.cat(packed_masks, dim=0) + + # return ImageTaskSamplePacked( + # __key__=",".join([s.__key__ for s in samples]), + # __restore_key__=(), # Will be set by energon based on `samples` + # __subflavor__=None, + # __subflavors__=samples[0].__subflavors__, + # tokens=packed_tokens, + # labels=packed_labels, + # attn_mask=packed_masks, + # imgs=packed_imgs, + # pixel_values_videos=packed_videos, + # cu_lengths=torch.tensor(cu_lengths, dtype=torch.int32), + # max_length=max_length, + # num_tiles=[n for s in samples for n in s.num_tiles], + # ) + @stateless - def pack_selected_samples(self, samples: List[ImageTaskSample]) -> List[ImageTaskSamplePacked]: + def pack_selected_samples(self, samples: List[ImageTaskSample]) -> ImageTaskSamplePacked: """ - Function to pack a list of ImageTaskSample into a single ImageTaskSamplePacked. - - NOTE: Energon dataloader calls this method internally if packing is used. - Please see https://nvidia.github.io/Megatron-Energon/packing.html - - Args: - samples: List of ImageTaskSample instances to pack into one sample. - - Returns: - ImageTaskSamplePacked instance. + Pack a list of ImageTaskSample into a single ImageTaskSamplePacked. + Truncates samples as needed so the packed sample never exceeds packing_seq_len. """ - packing_seq_len = self.args.seq_length - packed_tokens = [] - packed_labels = [] - packed_masks = [] + # Lists of per-token pieces (will be concatenated at the end). + packed_tokens_parts = [] + packed_labels_parts = [] + packed_masks_parts = [] packed_imgs = [] packed_videos = [] @@ -336,43 +431,118 @@ def pack_selected_samples(self, samples: List[ImageTaskSample]) -> List[ImageTas max_length = 0 cu_lengths = [0] - # Process each sample and build lists that we will concatenate to create the packed sample. - for _, sample in enumerate(samples): - sample_len = sample.total_len + for sample in samples: + # Determine sample token length (use available fields) + sample_len = int(getattr(sample, "total_len", None) or + (len(sample.tokens) if hasattr(sample, "tokens") and sample.tokens is not None else 0)) + + # If the sample itself is longer than the global packing_seq_len, allow truncation + if sample_len > packing_seq_len: + logging.getLogger(__name__).warning( + f"Sample {getattr(sample, '__key__', '')} length {sample_len} > packing_seq_len {packing_seq_len}. " + "Truncating token fields to packing_seq_len." + ) + sample_len = packing_seq_len + + remaining_capacity = packing_seq_len - current_length + if remaining_capacity <= 0: + # no capacity left (shouldn't normally happen if select_samples_to_pack worked), + # stop adding further samples. + logging.getLogger(__name__).warning( + "No remaining packing capacity; stopping packing of further samples." + ) + break + + # If this sample would overflow the remaining capacity, we will truncate it to fit. + if sample_len > remaining_capacity: + logging.getLogger(__name__).warning( + f"Truncating sample {getattr(sample, '__key__', '')} from {sample_len} to fit remaining capacity {remaining_capacity}." + ) + sample_len = remaining_capacity + + # Slice the per-token fields safely (they might be torch tensors or numpy arrays) + # If a field is missing, create an appropriate dummy slice of length sample_len. + def slice_field(field, length): + if field is None: + # return a zero-length tensor/array (we'll handle concatenation later) + return None + try: + return field[:length] + except Exception: + # fallback: convert to torch tensor then slice + try: + t = torch.as_tensor(field) + return t[:length] + except Exception: + return None + + tokens_slice = slice_field(getattr(sample, "tokens", None), sample_len) + labels_slice = slice_field(getattr(sample, "labels", None), sample_len) + mask_slice = slice_field(getattr(sample, "attn_mask", None), sample_len) + + # Append slices (if None, we will handle later by creating empty tensors) + packed_tokens_parts.append(tokens_slice) + packed_labels_parts.append(labels_slice) + packed_masks_parts.append(mask_slice) + + # For images/videos: keep them as-is (we assume images are not token-aligned) + if getattr(sample, "imgs", None) is not None: + packed_imgs += list(sample.imgs) + if getattr(sample, "pixel_values_videos", None) is not None: + packed_videos += list(sample.pixel_values_videos) - if sample_len > max_length: - max_length = sample_len - - # If adding this sample exceeds the max length, stop. - # This should not happen. - # The select_samples_to_pack method should have already ensured that the samples fit. - if current_length + sample_len > packing_seq_len: - print(f"packing_seq_len:{packing_seq_len}----<<<<<----{current_length + sample_len}") - raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") - - # Add the sample's tokens and labels - packed_tokens.append(sample.tokens) - packed_labels.append(sample.labels) - packed_masks.append(sample.attn_mask) - - # Add the images - if sample.imgs is not None: - packed_imgs += sample.imgs - if sample.pixel_values_videos is not None: - packed_videos += sample.pixel_values_videos current_length += sample_len cu_lengths.append(current_length) + if sample_len > max_length: + max_length = sample_len - # Concatenate packed tokens and labels. - packed_tokens = torch.cat(packed_tokens, dim=0) - packed_labels = torch.cat(packed_labels, dim=0) - packed_masks = torch.cat(packed_masks, dim=0) + # If nothing was added, return a minimal packed sample + if not packed_tokens_parts: + return ImageTaskSamplePacked( + __key__="", + __restore_key__=(), + __subflavor__=None, + __subflavors__=samples[0].__subflavors__ if samples else {}, + tokens=torch.zeros((0,), dtype=torch.int64), + labels=torch.zeros((0,), dtype=torch.int64), + attn_mask=torch.zeros((0,), dtype=torch.bool), + imgs=packed_imgs, + pixel_values_videos=packed_videos, + cu_lengths=torch.tensor(cu_lengths, dtype=torch.int32), + max_length=0, + num_tiles=[n for s in samples for n in s.num_tiles] if samples else [], + ) + + # For concatenation we need to ensure same dtypes. Replace None slices with zero-length tensors. + def to_tensor_or_empty(x, dtype=None): + if x is None: + return torch.zeros((0,), dtype=dtype if dtype is not None else torch.int64) + if isinstance(x, torch.Tensor): + return x + # try to convert numpy/other to tensor + try: + return torch.as_tensor(x) + except Exception: + return torch.zeros((0,), dtype=dtype if dtype is not None else torch.int64) + + # infer dtypes from first available slices + first_tokens = next((p for p in packed_tokens_parts if p is not None), None) + first_labels = next((p for p in packed_labels_parts if p is not None), None) + first_masks = next((p for p in packed_masks_parts if p is not None), None) + + tokens_dtype = first_tokens.dtype if isinstance(first_tokens, torch.Tensor) else torch.int64 + labels_dtype = first_labels.dtype if isinstance(first_labels, torch.Tensor) else torch.int64 + masks_dtype = first_masks.dtype if isinstance(first_masks, torch.Tensor) else torch.bool + + packed_tokens = torch.cat([to_tensor_or_empty(p, dtype=tokens_dtype) for p in packed_tokens_parts], dim=0) + packed_labels = torch.cat([to_tensor_or_empty(p, dtype=labels_dtype) for p in packed_labels_parts], dim=0) + packed_masks = torch.cat([to_tensor_or_empty(p, dtype=masks_dtype) for p in packed_masks_parts], dim=0) return ImageTaskSamplePacked( __key__=",".join([s.__key__ for s in samples]), - __restore_key__=(), # Will be set by energon based on `samples` + __restore_key__=(), # Will be set by energon if needed __subflavor__=None, - __subflavors__=samples[0].__subflavors__, + __subflavors__=samples[0].__subflavors__ if samples else {}, tokens=packed_tokens, labels=packed_labels, attn_mask=packed_masks, @@ -383,7 +553,6 @@ def pack_selected_samples(self, samples: List[ImageTaskSample]) -> List[ImageTas num_tiles=[n for s in samples for n in s.num_tiles], ) - def print_error_handler(exc: Exception, key: Optional[str]): """ Print error handler function called when an exception occurs during loading. """ print( diff --git a/aiak_training_llm/models/custom/common/local_norm.py b/aiak_training_llm/models/custom/common/local_norm.py index d488ee24..c71fdd57 100644 --- a/aiak_training_llm/models/custom/common/local_norm.py +++ b/aiak_training_llm/models/custom/common/local_norm.py @@ -1,75 +1,141 @@ -"""megatron local norm""" +"""megatron local norm - Apex NOT used; pure-PyTorch fallbacks.""" +from typing import Optional +import torch +import torch.nn as nn +from torch.nn.parameter import Parameter + +# Megatron config type (import only for typing/runtime usage) from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.fusions.fused_layer_norm import FusedLayerNorm +# Prefer Megatron's fused op if present (this is NOT Apex): try: - from apex.normalization.fused_layer_norm import FusedRMSNorm as ApexFusedRMSNorm - HAVE_FUSED_RMS_NORM = True -except: - HAVE_FUSED_RMS_NORM = False + from megatron.core.fusions.fused_layer_norm import FusedLayerNorm # optional + HAVE_MEGATRON_FUSED_LAYER_NORM = True +except Exception: + HAVE_MEGATRON_FUSED_LAYER_NORM = False -try: - from apex.normalization.fused_layer_norm import FusedLayerNorm as ApexFusedLayerNorm - HAVE_FUSED_LAYER_NORM = True -except: - HAVE_FUSED_LAYER_NORM = False +# --- Pure PyTorch RMSNorm implementation --- +class RMSNorm(nn.Module): + """Pure-PyTorch RMSNorm implementation. + y = x * (weight / sqrt(mean(x^2, dim=-1, keepdim=True) + eps)) + + Keeps a `weight` attribute compatible with other code expecting per-channel + affine parameters. Sets `weight.sequence_parallel` when a config is provided. + """ -class FusedRMSNorm(ApexFusedRMSNorm): - """Fused RMS Norm""" - def __init__(self, - config: TransformerConfig, - hidden_size: int, - eps=1e-5, - elementwise_affine=True): + def __init__( + self, + config: TransformerConfig, + hidden_size: int, + eps: float = 1e-5, + elementwise_affine: bool = True, + ): + super().__init__() + self.config = config + self.hidden_size = hidden_size + self.eps = eps + self.elementwise_affine = elementwise_affine - if not HAVE_FUSED_RMS_NORM: - # TODO: Add pytorch only rms norm - raise ValueError(f'Apex must currently be installed to use FusedRMSNorm op.') + if self.elementwise_affine: + self.weight = Parameter(torch.ones(hidden_size)) + else: + # keep attribute for compatibility + self.register_buffer("weight", torch.ones(hidden_size)) - super().__init__(hidden_size, - eps=eps, - elementwise_affine=elementwise_affine) + # optional support for megatron sequence_parallel flag on config + self.sequence_parallel = getattr(self.config, "sequence_parallel", False) + try: + setattr(self.weight, "sequence_parallel", self.sequence_parallel) + except Exception: + # some frameworks may not allow setting attributes on Parameter on certain dtypes, + # ignore silently (compatibility only) + pass - self.config = config + def forward(self, x: torch.Tensor): + # compute RMS over last dim + # ms shape: (..., 1) + ms = x.pow(2).mean(dim=-1, keepdim=True) + rs = torch.rsqrt(ms + self.eps) # (..., 1) + if self.elementwise_affine: + # broadcast weight across leading dims + w = self.weight.view(*([1] * (x.dim() - 1)), -1) + return x * rs * w + else: + return x * rs - self.sequence_parallel = self.config.sequence_parallel - # set sequence parallelism flag on weight parameters - setattr(self.weight, 'sequence_parallel', self.sequence_parallel) +# --- LayerNorm wrapper (Megatron fused if available, else torch.nn.LayerNorm) --- +class TorchLayerNormWrapper(nn.Module): + """Wrap torch.nn.LayerNorm so its initializer and attributes match expected API.""" + def __init__(self, hidden_size: int, eps: float = 1e-5, elementwise_affine: bool = True): + super().__init__() + self.layernorm = nn.LayerNorm(hidden_size, eps=eps, elementwise_affine=elementwise_affine) + if elementwise_affine: + # expose weight attribute for compatibility (so callers can set sequence_parallel) + self.weight = self.layernorm.weight + else: + # still expose a weight-like buffer for code that expects it + self.register_buffer("weight", torch.ones(hidden_size)) + + def forward(self, x: torch.Tensor): + return self.layernorm(x) + + +# Pick the best LayerNorm implementation available (but do NOT import Apex anywhere). +if HAVE_MEGATRON_FUSED_LAYER_NORM: + LayerNormClass = FusedLayerNorm +else: + LayerNormClass = TorchLayerNormWrapper + +# --- Factory class: LocalNorm --- class LocalNorm: """ - A conditional wrapper to initialize an instance of Megatron Local `LayerNorm` or `RMSNorm` based on input + Factory to create a normalization module consistent with megatron config. + + Usage: + norm = LocalNorm(config, hidden_size, eps=1e-5, elementwise_affine=True) + + This will return either: + - a LayerNorm-like module (Megatron fused if available, otherwise torch.nn.LayerNorm wrapper), or + - RMSNorm (pure-PyTorch). """ - # TODO should we ditch normalization config and just use spec to choose LayerNorm vs RMSNorm? - def __new__(cls, config: TransformerConfig, hidden_size: int, eps: float = 1e-5, elementwise_affine=True): - if config.normalization == "LayerNorm": - if elementwise_affine: - instance = FusedLayerNorm( - config=config, - hidden_size=hidden_size, - eps=eps, - ) + def __new__( + cls, + config: TransformerConfig, + hidden_size: int, + eps: float = 1e-5, + elementwise_affine: bool = True, + ): + norm_type = getattr(config, "normalization", "LayerNorm") + if norm_type == "LayerNorm": + # prefer megatron fused if it exists, else torch wrapper + # Megatron FusedLayerNorm initializer signature: (config=config, hidden_size=..., eps=...) + if HAVE_MEGATRON_FUSED_LAYER_NORM: + # Create using Megatron fused implementation (keeps config semantics) + try: + instance = FusedLayerNorm(config=config, hidden_size=hidden_size, eps=eps) + except Exception: + # fallback to torch wrapper if megatron fused can't be constructed + instance = LayerNormClass(hidden_size, eps=eps, elementwise_affine=elementwise_affine) else: - assert HAVE_FUSED_LAYER_NORM, "Apex must currently be installed to use FusedLayerNorm op." - instance = ApexFusedLayerNorm( - hidden_size, - eps=eps, - elementwise_affine=elementwise_affine, - ) - elif config.normalization == "RMSNorm": - instance = FusedRMSNorm( - config=config, - hidden_size=hidden_size, - eps=eps, - elementwise_affine=elementwise_affine, - ) + instance = LayerNormClass(hidden_size, eps=eps, elementwise_affine=elementwise_affine) + + # if config exposes sequence_parallel, replicate attribute on weight if present + seq_par = getattr(config, "sequence_parallel", False) + if hasattr(instance, "weight"): + try: + setattr(instance.weight, "sequence_parallel", seq_par) + except Exception: + pass + elif norm_type == "RMSNorm": + instance = RMSNorm(config=config, hidden_size=hidden_size, eps=eps, elementwise_affine=elementwise_affine) else: - raise Exception('Only LayerNorm and RMSNorm are curently supported') + raise ValueError("Only 'LayerNorm' and 'RMSNorm' are supported for LocalNorm") return instance diff --git a/aiak_training_llm/models/custom/common/local_norm_org.py b/aiak_training_llm/models/custom/common/local_norm_org.py new file mode 100644 index 00000000..cbf7e8fc --- /dev/null +++ b/aiak_training_llm/models/custom/common/local_norm_org.py @@ -0,0 +1,77 @@ +"""megatron local norm""" + +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.fusions.fused_layer_norm import FusedLayerNorm + +try: + from apex.normalization.fused_layer_norm import FusedRMSNorm as ApexFusedRMSNorm + HAVE_FUSED_RMS_NORM = True +except Exception as e: + HAVE_FUSED_RMS_NORM = False + print("Failed to import Apex FusedRMSNorm:", repr(e)) +print("HAVE_FUSED_RMS_NORM =", HAVE_FUSED_RMS_NORM) +try: + from apex.normalization.fused_layer_norm import FusedLayerNorm as ApexFusedLayerNorm + HAVE_FUSED_LAYER_NORM = True +except: + HAVE_FUSED_LAYER_NORM = False +print("HAVE_FUSED_LAYER_NORM =", HAVE_FUSED_LAYER_NORM) + +class FusedRMSNorm(ApexFusedRMSNorm): + """Fused RMS Norm""" + def __init__(self, + config: TransformerConfig, + hidden_size: int, + eps=1e-5, + elementwise_affine=True): + + if not HAVE_FUSED_RMS_NORM: + # TODO: Add pytorch only rms norm + raise ValueError(f'Apex must currently be installed to use FusedRMSNorm op.') + + super().__init__(hidden_size, + eps=eps, + elementwise_affine=elementwise_affine) + + self.config = config + + self.sequence_parallel = self.config.sequence_parallel + + # set sequence parallelism flag on weight parameters + setattr(self.weight, 'sequence_parallel', self.sequence_parallel) + + +class LocalNorm: + """ + A conditional wrapper to initialize an instance of Megatron Local `LayerNorm` or `RMSNorm` based on input + """ + + # TODO should we ditch normalization config and just use spec to choose LayerNorm vs RMSNorm? + def __new__(cls, config: TransformerConfig, hidden_size: int, eps: float = 1e-5, elementwise_affine=True): + if config.normalization == "LayerNorm": + if elementwise_affine: + print("LayerNorm") + instance = FusedLayerNorm( + config=config, + hidden_size=hidden_size, + eps=eps, + ) + else: + assert HAVE_FUSED_LAYER_NORM, "Apex must currently be installed to use FusedLayerNorm op." + instance = ApexFusedLayerNorm( + hidden_size, + eps=eps, + elementwise_affine=elementwise_affine, + ) + elif config.normalization == "RMSNorm": + instance = FusedRMSNorm( + config=config, + hidden_size=hidden_size, + eps=eps, + elementwise_affine=elementwise_affine, + ) + + else: + raise Exception('Only LayerNorm and RMSNorm are curently supported') + + return instance diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py index 5a4ccb4b..591797a1 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py @@ -1,10 +1,9 @@ import torch -from megatron.core.extensions.transformer_engine import ( - TEDotProductAttention, TELayerNormColumnParallelLinear, TELinear, - TERowParallelLinear) +from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear +# DotProductAttention is in dot_product_attention.py in this repo. +from megatron.core.transformer.dot_product_attention import DotProductAttention from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add -from megatron.core.transformer.attention import (SelfAttention, - SelfAttentionSubmodules) +from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules from megatron.core.transformer.enums import AttnMaskType from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.mlp import MLP, MLPSubmodules @@ -25,7 +24,7 @@ def get_vision_layer_with_spec() -> ModuleSpec: - """Use this spec for an implementation using transformer, local or multi-accel engine.""" + """Use this spec for a standard (non-Transformer-Engine) implementation.""" return ModuleSpec( module=TransformerLayer, submodules=TransformerLayerSubmodules( @@ -33,9 +32,10 @@ def get_vision_layer_with_spec() -> ModuleSpec: module=SelfAttention, params={"attn_mask_type": AttnMaskType.no_mask}, submodules=SelfAttentionSubmodules( - linear_qkv=TELayerNormColumnParallelLinear, - core_attention=TEDotProductAttention, - linear_proj=TERowParallelLinear, + # use standard (non-TE) parallel linears + linear_qkv=ColumnParallelLinear, + core_attention=DotProductAttention, + linear_proj=RowParallelLinear, apply_rotary_fn=apply_rotary_pos_emb_vision, ), ), @@ -43,8 +43,8 @@ def get_vision_layer_with_spec() -> ModuleSpec: mlp=ModuleSpec( module=MLP, submodules=MLPSubmodules( - linear_fc1=TELayerNormColumnParallelLinear, - linear_fc2=TERowParallelLinear, + linear_fc1=ColumnParallelLinear, + linear_fc2=RowParallelLinear, ), ), mlp_bda=get_bias_dropout_add, @@ -52,6 +52,7 @@ def get_vision_layer_with_spec() -> ModuleSpec: ) + def _get_mlp_module_spec( num_experts: int=None, moe_grouped_gemm: bool=False diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py new file mode 100644 index 00000000..5a4ccb4b --- /dev/null +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py @@ -0,0 +1,177 @@ +import torch +from megatron.core.extensions.transformer_engine import ( + TEDotProductAttention, TELayerNormColumnParallelLinear, TELinear, + TERowParallelLinear) +from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add +from megatron.core.transformer.attention import (SelfAttention, + SelfAttentionSubmodules) +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.mlp import MLP, MLPSubmodules +from megatron.core.transformer.moe.experts import SequentialMLP, TEGroupedMLP +from megatron.core.transformer.moe.moe_layer import MoELayer, MoESubmodules +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.transformer_layer import ( + TransformerLayer, TransformerLayerSubmodules) + +from aiak_training_llm.models.custom.common.local_norm import LocalNorm +from aiak_training_llm.models.dispatch import multiacc_modules +from aiak_training_llm.models.qwen_vl.qwen2_vl_layer_spec import \ + get_adapeter_layer_with_spec +from aiak_training_llm.utils import is_te_min_version + +from .rice_vision_model import apply_rotary_pos_emb_vision + + +def get_vision_layer_with_spec() -> ModuleSpec: + """Use this spec for an implementation using transformer, local or multi-accel engine.""" + return ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + self_attention=ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.no_mask}, + submodules=SelfAttentionSubmodules( + linear_qkv=TELayerNormColumnParallelLinear, + core_attention=TEDotProductAttention, + linear_proj=TERowParallelLinear, + apply_rotary_fn=apply_rotary_pos_emb_vision, + ), + ), + self_attn_bda=get_bias_dropout_add, + mlp=ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=TELayerNormColumnParallelLinear, + linear_fc2=TERowParallelLinear, + ), + ), + mlp_bda=get_bias_dropout_add, + ), + ) + + +def _get_mlp_module_spec( + num_experts: int=None, + moe_grouped_gemm: bool=False +) -> ModuleSpec: + """Helper function to get module spec for MLP/MoE""" + + if num_experts is None: + # Dense MLP w/ TE modules. + return ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=multiacc_modules.TELayerNormColumnParallelLinear, + linear_fc2=multiacc_modules.TERowParallelLinear, + bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, + ), + ) + + # moe mlp + if moe_grouped_gemm: + # use TEGroupedLinear + assert multiacc_modules.TEColumnParallelGroupedLinear is not None + expert_module = TEGroupedMLP + linear_fc1 = multiacc_modules.TEColumnParallelGroupedLinear + linear_fc2 = multiacc_modules.TERowParallelGroupedLinear + else: + expert_module = SequentialMLP + linear_fc1 = multiacc_modules.TEColumnParallelLinear + linear_fc2 = multiacc_modules.TERowParallelLinear + + return ModuleSpec( + module=MoELayer, + submodules=MoESubmodules( + experts=ModuleSpec( + module=expert_module, + submodules=MLPSubmodules( + linear_fc1=linear_fc1, + linear_fc2=linear_fc2, + bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, + ) + ) + ) + ) + + +def get_qwen_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: + """ + Use this spec for an implementation using transformer, local or multi-accel engine + """ + # To simplify the code, temporarily remove the compatibility with MoE/MLA. + # If there is a new version in the future, add and test it separately. + assert not config.multi_latent_attention, "Not supporting multi-latent attention for Qwen model yet." + + mlp = _get_mlp_module_spec( + num_experts=config.num_moe_experts, + moe_grouped_gemm=config.moe_grouped_gemm, + ) + + # TENorm significantly harms convergence when used for QKLayerNorm if TE Version < 1.9; + # we instead use the Apex implementation. + # qk_norm = multiacc_modules.TENorm if is_te_min_version("1.9.0") else multiacc_modules.LocalNorm + qk_norm = multiacc_modules.LocalNorm + + return ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + input_layernorm=IdentityOp, + self_attention=ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=multiacc_modules.TELayerNormColumnParallelLinear, + core_attention=multiacc_modules.DotProductAttention, + linear_proj=multiacc_modules.TERowParallelLinear, + q_layernorm=qk_norm if config.qk_layernorm else IdentityOp, + k_layernorm=qk_norm if config.qk_layernorm else IdentityOp, + apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, + ), + ), + self_attn_bda=multiacc_modules.get_bias_dropout_add, + pre_mlp_layernorm=( + multiacc_modules.TENorm if config.num_moe_experts else IdentityOp + ), + mlp=mlp, + mlp_bda=multiacc_modules.get_bias_dropout_add, + ) + ) + +# def get_qwen_layer_with_spec(qk_layernorm: bool = False) -> ModuleSpec: +# """ +# Use this spec for an implementation using transformer, local or multi-accel engine +# """ +# return ModuleSpec( +# module=TransformerLayer, +# submodules=TransformerLayerSubmodules( +# input_layernorm=IdentityOp, +# self_attention=ModuleSpec( +# module=SelfAttention, +# params={"attn_mask_type": AttnMaskType.causal}, +# submodules=SelfAttentionSubmodules( +# linear_qkv=TELayerNormColumnParallelLinear, +# core_attention=TEDotProductAttention, +# linear_proj=TERowParallelLinear, +# q_layernorm=LocalNorm if qk_layernorm else IdentityOp, +# k_layernorm=LocalNorm if qk_layernorm else IdentityOp, +# apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, +# ), +# ), +# self_attn_bda=get_bias_dropout_add, +# pre_mlp_layernorm=IdentityOp, +# mlp=ModuleSpec( +# module=MLP, +# submodules=MLPSubmodules( +# linear_fc1=TELayerNormColumnParallelLinear, +# linear_fc2=TERowParallelLinear, +# ), +# ), +# mlp_bda=get_bias_dropout_add, +# sharded_state_dict_keys_map={ +# 'input_layernorm.': 'self_attention.linear_qkv.layer_norm_', +# 'pre_mlp_layernorm.': 'mlp.linear_fc1.layer_norm_', +# }, +# ), +# ) \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py b/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py index 5246b2d1..15eb5d7d 100644 --- a/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py +++ b/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py @@ -33,6 +33,7 @@ def apply_rotary_pos_emb_vision(t, freqs, config, cu_seqlens=None, rotary_interl return t.to(orig_dtype) + class PatchEmbed(torch.nn.Module): """" Patch Embedding """ def __init__( diff --git a/apex b/apex new file mode 160000 index 00000000..0da3ffb9 --- /dev/null +++ b/apex @@ -0,0 +1 @@ +Subproject commit 0da3ffb92ee6fbe5336602f0e3989db1cd16f880 diff --git a/data/LLaVA-558K-Webdataset b/data/LLaVA-558K-Webdataset new file mode 160000 index 00000000..28a8d0b2 --- /dev/null +++ b/data/LLaVA-558K-Webdataset @@ -0,0 +1 @@ +Subproject commit 28a8d0b2acd06a07e5b7697ac39270b02559a20f diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 2580ea12..28460b09 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -1,14 +1,19 @@ -AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/workspace/LLaVA-OneVision-1.5}" -AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" -TP="${1:-1}" -PP="${2:-1}" -SEQ_LEN="${3:-32768}" -MBS="${4:-1}" -GBS="${5:-8}" -NSTEP="${6:-2500}" -DATA_PATH=${DATA_PATH:-"/workspace/dataset/LLaVA-558K-Webdataset"} -TOKENIZER_PATH=${TOKENIZER_PATH:-"/workspace/LLaVA-OneVision-1.5/LLaVA-OneVision-1.5-4B-stage0"} -CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/workspace/LLaVA-OneVision-1.5/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1"} +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5}" #Root of the LLaVA-OneVision training repo +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" # Megatron-LM fork used (aiak_megatron) inside the training repo. +# TP="${1:-1}" # Tensor parallel +TP="${1:-2}" +PP="${2:-1}" # pipeline parallel +# Defaults: TP=1, PP=1. +# SEQ_LEN="${3:-32768}" +SEQ_LEN="${3:-2048}" +MBS="${4:-1}" # micro batch size +# GBS="${5:-8}" # global batch size +GBS="${5:-2}" +# NSTEP="${6:-2500}" # number of training iterations +NSTEP="${6:-5}" # number of training iterations +DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} +TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} +CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} #! /bin/bash # The script needs to be run on at least 1 nodes. @@ -76,7 +81,7 @@ TENSORBOARD_PATH="${SAVE_CKPT_PATH}/tensorboard" mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=8 +GPUS_PER_NODE=2 # Change for multinode config MASTER_ADDR=${MASTER_ADDR:-"${list_ip[0]}"} @@ -106,21 +111,23 @@ DATA_ARGS=( --data-path "$DATA_PATH" --dataloader-type external --split 100,0,0 - --num-workers 16 - --chat-template qwen2-vl + --num-workers 1 + --chat-template qwen2-vl # will change this chat template ) TRAINING_ARGS=( --training-phase sft --trainable-modules adapter + --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" + --transformer-impl local --max-position-embeddings 32768 --init-method-std 0.02 --micro-batch-size "${MBS}" --global-batch-size "${GBS}" --lr 1.0e-4 --min-lr 1.0e-6 - --clip-grad 1.0 + --clip-grad 1.0 # gradient norm clipping. --weight-decay 0 --optimizer adam --adam-beta1 0.9 @@ -133,8 +140,8 @@ TRAINING_ARGS=( --lr-warmup-fraction 0.002 --initial-loss-scale 65536 --bf16 - --load "$CHECKPOINT_PATH" - --save "$SAVE_CKPT_PATH" + --load "$CHECKPOINT_PATH" # load initial checkpoint. + --save "$SAVE_CKPT_PATH" # where to save new checkpoints. --save-interval 2000 --ckpt-format torch --dataloader-save "${SAVE_CKPT_PATH}/dataloader" @@ -171,8 +178,10 @@ logfile="${SAVE_CKPT_PATH}/run_${TM}_tp${TP}_pp${PP}_seqlen${SEQ_LEN}_mbs${MBS}_ export OFFLINE_PACKED_DATA='1' export OFFLINE_PACKING_VQA='1' -export PYTORCH_CUDA_ALLOC_CONF=max_split_size_mb:128 -export PYTORCH_CUDA_ALLOC_CONF=garbage_collection_threshold:0.72 +# export PYTORCH_ALLOC_CONF=max_split_size_mb:128 +# export PYTORCH_ALLOC_CONF=garbage_collection_threshold:0.72 +export PYTORCH_ALLOC_CONF=max_split_size_mb:128,garbage_collection_threshold:0.72 + PYTHONPATH="$AIAK_MAGATRON_PATH:$AIAK_TRAINING_PATH:$PYTHONPATH" \ torchrun "${DISTRIBUTED_ARGS[@]}" \ diff --git a/libpod/tmp/pause.pid b/libpod/tmp/pause.pid new file mode 100644 index 00000000..9abfb284 --- /dev/null +++ b/libpod/tmp/pause.pid @@ -0,0 +1 @@ +832214 \ No newline at end of file diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..40c7dfc2 --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log @@ -0,0 +1,1044 @@ +W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] +W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] ***************************************** +W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 8192 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.030 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1222 14:44:31.761652648 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.346 seconds +time to initialize megatron (seconds): 3.999 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-22 14:44:34 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 3False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 FalseFalse + +3 3False + False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc2.linear.weight + module.adapter.layernorm.bias + module.adapter.layernorm.weight + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (7289.26, 7289.29) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 14:45:19 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-22 14:45:19 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (44573.39, 44573.43) + train/valid/test-data-iterators-setup ..........: (840.26, 842.88) +training ... +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-22 14:45:19 +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 356.27 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward\n core_attn_out = self.core_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward\n matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor\n self.buffer[(name, dtype)] = torch.empty(\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward +[rank1]: core_attn_out = self.core_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward +[rank1]: matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor +[rank1]: self.buffer[(name, dtype)] = torch.empty( +[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 356.27 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward\n core_attn_out = self.core_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward\n matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor\n self.buffer[(name, dtype)] = torch.empty(\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward +[rank0]: core_attn_out = self.core_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward +[rank0]: matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor +[rank0]: self.buffer[(name, dtype)] = torch.empty( +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank1]:[W1222 14:45:23.420611800 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1222 14:45:28.727847326 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1222 14:45:29.902000 3342258 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3342267 closing signal SIGTERM +E1222 14:45:30.116000 3342258 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 3342268) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-22_14:45:29 + host : gpu-52 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 3342268) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..77af4bde --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log @@ -0,0 +1,53109 @@ +W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] +W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] ***************************************** +W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 4096 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.066 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1222 21:56:35.174676658 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +> setting tensorboard ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 1.931 seconds +time to initialize megatron (seconds): 19.957 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-22 21:56:53 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.weight + module.adapter.linear_fc1.linear.bias + module.adapter.layernorm.bias + module.adapter.linear_fc2.linear.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (7206.33, 7206.41) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 21:57:38 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not existdataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist + +[after dataloaders are built] datetime: 2025-12-22 21:57:41 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (44392.15, 44392.16) + train/valid/test-data-iterators-setup ..........: (3101.55, 3101.55)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-22 21:57:41 +packing_seq_len:4096----<<<<<----4176 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ + for idx, (sample_idx, inner_sample) in enumerate( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx + x = next(it) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator + sample = next(it) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample + l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples + super().pack_selected_samples(samples), + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 351, in pack_selected_samples + raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") +ValueError: Packed sample exceeds the maximum sequence length of 4096: [Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img000_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, + 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, 198, + 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), total_len=315, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7777, ..., 0.8803, 0.8661, 0.8803], + [-1.7923, -1.7923, -1.7923, ..., -1.4660, -1.4660, -1.4518], + ..., + [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], + [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 46]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img001_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 60424, + 264, 9814, 4008, 315, 279, 2661, 2168, 13, 151645, + 198, 151644, 77091, 198, 1782, 31535, 369, 16301, 55155, + 304, 6303, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 1782, 31535, 369, 16301, 55155, + 304, 6303, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img002_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 1782, 1656, 29428, + 14126, 22361, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 1782, 1656, 29428, + 14126, 22361, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img003_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 276, 2168, 315, 264, 41530, 15616, 28642, 448, + 52484, 12624, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 276, 2168, 315, 264, 41530, 15616, 28642, 448, + 52484, 12624, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img004_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 6750, 264, 2797, 323, 63594, 12126, 315, + 279, 6548, 13, 151645, 198, 151644, 77091, 198, 1782, + 1901, 36780, 15273, 19818, 4621, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, 1782, + 1901, 36780, 15273, 19818, 4621, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.4048, 1.4048, 1.4048, ..., 1.7620, 1.7762, 1.7762], + [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], + [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], + ..., + [ 1.3026, 1.3026, 1.3026, ..., -1.0252, -1.0252, -1.0252], + [-0.1572, 0.6019, 0.4997, ..., -0.9399, -0.9399, -0.9683], + [-0.2740, -0.4784, 0.6457, ..., -0.9683, -0.9683, -0.9683]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img005_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, + 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 78243, 75105, 220, 23, + 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, + 6430, 12136, 56432, 3421, 1947, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 78243, 75105, 220, 23, + 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, + 6430, 12136, 56432, 3421, 1947, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.3835, -1.3835, -1.3835, ..., -0.5275, -0.5275, -0.5275], + [-1.3689, -1.3543, -1.3251, ..., -0.3853, -0.3995, -0.3995], + [-1.3835, -1.3689, -1.3689, ..., -0.4848, -0.4706, -0.4422], + ..., + [-1.3981, -1.4127, -1.4273, ..., -0.4137, -0.3995, -0.3995], + [-1.2375, -0.2594, -1.6609, ..., -0.5275, -0.5133, -0.5133], + [-1.1937, -0.3032, 0.3391, ..., -0.5417, -0.5417, -0.5417]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img006_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, + 264, 9814, 4008, 315, 279, 2168, 13, 151645, 198, + 151644, 77091, 198, 1782, 10550, 29674, 645, 12426, 448, + 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, + 259, 481, 36724, 481, 2953, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 1782, 10550, 29674, 645, 12426, 448, + 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, + 259, 481, 36724, 481, 2953, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.9893, -1.0039, -1.0039, ..., -0.2715, -0.2715, -0.2715], + [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2573], + [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2715], + ..., + [-1.0915, -1.0915, -1.0769, ..., -0.2857, -0.3000, -0.3142], + [-1.1207, -1.1207, -1.1207, ..., -0.3853, -0.3568, -0.3568], + [-1.1061, -1.1061, -1.1061, ..., -0.3142, -0.3000, -0.3000]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img007_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 1782, 42646, 331, 595, 3821, 18, 21, 19, + 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, + 323, 264, 24461, 9509, 27860, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 1782, 42646, 331, 595, 3821, 18, 21, 19, + 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, + 323, 264, 24461, 9509, 27860, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img008_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 67, 20367, 19684, 17, 220, 17, + 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, + 323, 18579, 71738, 11, 4024, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 67, 20367, 19684, 17, 220, 17, + 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, + 323, 18579, 71738, 11, 4024, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + ..., + [-1.2959, -1.3397, -1.7923, ..., 2.0748, 2.0321, 2.1459], + [-1.4273, -1.7193, -0.9602, ..., 1.7620, 1.9184, 1.9610], + [-1.5003, -1.4419, -1.7923, ..., 1.9895, 2.0748, 2.0321]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img009_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, + 198, 151644, 77091, 198, 759, 75242, 12426, 11, 66274, + 32300, 4603, 32300, 4603, 1841, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 759, 75242, 12426, 11, 66274, + 32300, 4603, 32300, 4603, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.2375, -1.5733, -1.7923, ..., -0.8261, -0.6412, -0.6270], + [-1.7923, -1.7923, -1.7777, ..., -0.9683, -0.9683, -0.9683], + [-1.7923, -1.7923, -1.7631, ..., -0.9683, -0.9683, -0.9683], + ..., + [-1.7339, -1.5879, -1.4565, ..., 2.1459, 2.1459, 2.1459], + [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459], + [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img010_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 64, 2766, 1809, 448, + 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, + 279, 18967, 13962, 32372, 4426, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 64, 2766, 1809, 448, + 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, + 279, 18967, 13962, 32372, 4426, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img011_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, + 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, + 151645, 198, 151644, 77091, 198, 1782, 11623, 369, 1435, + 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, + 15786, 369, 279, 5593, 364, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 1782, 11623, 369, 1435, + 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, + 15786, 369, 279, 5593, 364, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3807, -1.3807, -1.3807], + ..., + [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img012_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, + 198, 151644, 77091, 198, 5082, 352, 263, 6837, 39698, + 1899, 69105, 938, 81625, 819, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 5082, 352, 263, 6837, 39698, + 1899, 69105, 938, 81625, 819, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.6536, -1.4419, -1.7923, ..., -1.4518, -1.4518, -1.4518], + [-1.7631, -1.7631, -1.7485, ..., -1.3096, -1.4233, -1.4802], + [-1.7631, -1.7631, -1.7631, ..., -1.4518, -1.4518, -1.4518], + ..., + [-1.7631, -1.7631, -1.7047, ..., -1.4518, -1.4518, -1.4518], + [-0.0113, 0.2953, -1.5733, ..., -1.3238, -1.3807, -1.3949], + [ 0.8063, -0.5368, -1.7777, ..., -1.4518, -1.4518, -1.4518]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img013_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 74785, + 279, 2168, 3529, 285, 974, 13, 151645, 198, 151644, + 77091, 198, 19789, 6078, 304, 279, 6083, 315, 264, + 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, + 5382, 3054, 2114, 10576, 902, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, 19789, 6078, 304, 279, 6083, 315, 264, + 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, + 5382, 3054, 2114, 10576, 902, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5771, 1.5771], + [ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5629, 1.5487], + [ 1.3464, 1.3464, 1.3464, ..., 1.5771, 1.5771, 1.5629], + ..., + [-0.9310, -0.9310, -0.9018, ..., 0.9514, 1.9753, 1.9895], + [-0.7412, -0.7266, -0.7120, ..., -0.7834, -0.7408, -0.7408], + [-0.5806, -0.5660, -0.5368, ..., 0.9230, 2.0606, 2.0748]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img014_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, + 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, + 151645, 198, 151644, 77091, 198, 2277, 481, 61794, 18342, + 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, + 311, 259, 481, 6470, 34957, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 2277, 481, 61794, 18342, + 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, + 311, 259, 481, 6470, 34957, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], + [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7762, 1.7762], + [1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], + ..., + [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7620, 1.7335], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.8281, 1.7114, 1.5800, ..., 2.1317, 2.1317, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img015_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 14593, 15478, 11, 825, 3338, + 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, + 481, 10658, 1419, 19507, 2062, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 14593, 15478, 11, 825, 3338, + 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, + 481, 10658, 1419, 19507, 2062, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + ..., + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img016_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, + 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 500, 5064, 5750, 11, + 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, + 28451, 22601, 11, 12188, 22601, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 500, 5064, 5750, 11, + 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, + 28451, 22601, 11, 12188, 22601, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img017_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 1782, 6623, 60037, + 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, + 738, 448, 19752, 3931, 48051, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 1782, 6623, 60037, + 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, + 738, 448, 19752, 3931, 48051, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img018_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 60424, 264, 9814, 4008, 315, 279, 2661, + 2168, 13, 151645, 198, 151644, 77091, 198, 2024, 3045, + 7821, 2103, 488, 11030, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, 2024, 3045, + 7821, 2103, 488, 11030, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [-0.2886, -0.2302, 0.0179, ..., 2.0890, 2.1175, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img019_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, + 264, 2805, 323, 2797, 16148, 315, 279, 17420, 2168, + 13, 151645, 198, 151644, 77091, 198, 64, 2781, 504, + 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, + 389, 264, 7002, 4034, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 64, 2781, 504, + 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, + 389, 264, 7002, 4034, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., -0.5986, -0.5986, -0.6270], + ..., + [-1.2083, -1.2083, -1.2083, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img020_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, + 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, + 198, 1782, 1081, 72725, 550, 315, 6470, 296, 377, + 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, + 29724, 315, 279, 93767, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 1782, 1081, 72725, 550, 315, 6470, 296, 377, + 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, + 29724, 315, 279, 93767, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img021_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 7985, 264, 50537, 714, + 38219, 12126, 315, 279, 6802, 624, 151652, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, + 198, 151644, 77091, 198, 1782, 3691, 1555, 4013, 281, + 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, + 1555, 2550, 2355, 8149, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 1782, 3691, 1555, 4013, 281, + 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, + 1555, 2550, 2355, 8149, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img022_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 19789, 87361, 14201, + 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, + 2472, 315, 1059, 59511, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 19789, 87361, 14201, + 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, + 2472, 315, 1059, 59511, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[-1.2083, -1.2083, -1.2083, ..., -0.9256, -0.9256, -0.9399], + [-1.1207, -1.1207, -1.0915, ..., -0.8545, -0.8688, -0.8688], + [-1.2959, -1.2959, -1.2813, ..., -1.0821, -1.0678, -1.0678], + ..., + [-0.7704, -0.7850, -0.7704, ..., -0.4990, -0.4990, -0.4990], + [-0.7558, -0.7704, -0.7850, ..., -0.3142, -0.3142, -0.2857], + [-0.8142, -0.8142, -0.8142, ..., -0.2715, -0.2431, -0.2289]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img023_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 3838, 374, 304, 279, + 6548, 5267, 151652, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 18824, 315, 13779, 9317, 1173, + 553, 9080, 567, 84946, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 18824, 315, 13779, 9317, 1173, + 553, 9080, 567, 84946, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., -0.7123, -0.7266, -0.7266], + [ 1.9303, 1.9303, 1.9303, ..., -0.7834, -0.7977, -0.7977], + [-1.0039, -1.3251, -1.2083, ..., -0.7977, -0.7692, -0.7408], + ..., + [ 1.0836, 1.1128, 1.2442, ..., 2.0890, 2.1032, 2.1175], + [ 0.9814, 0.9814, 0.9814, ..., 2.1175, 2.1175, 2.1317], + [ 1.3464, 1.4048, 1.4924, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img024_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 64, 22360, 3745, 448, + 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, + 4158, 67590, 3745, 26334, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 64, 22360, 3745, 448, + 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, + 4158, 67590, 3745, 26334, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.7260, 1.7260, 1.7260, ..., 1.9042, 1.9042, 1.9042], + [1.7260, 1.7260, 1.7260, ..., 1.8899, 1.8899, 1.8899], + [1.6822, 1.6822, 1.6822, ..., 1.8473, 1.8473, 1.8473], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img025_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, + 198, 151644, 77091, 198, 64, 3691, 16049, 8511, 25279, + 11, 825, 7100, 962, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 64, 3691, 16049, 8511, 25279, + 11, 825, 7100, 962, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.7844, 1.7990, 1.8135, ..., 2.0179, 2.0464, 2.0748], + [1.8719, 1.8719, 1.8573, ..., 2.0890, 2.1459, 2.1459], + [1.8427, 1.8427, 1.8573, ..., 2.0606, 2.0606, 2.0748], + ..., + [1.7990, 1.7698, 1.7844, ..., 2.0606, 2.0606, 2.0606], + [1.8427, 1.8427, 1.8427, ..., 2.0321, 2.0321, 2.0464], + [1.7990, 1.7844, 1.7990, ..., 2.0890, 2.1032, 2.1032]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img026_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 1151, 6918, 5636, 304, + 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, + 11, 9710, 5099, 8166, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 1151, 6918, 5636, 304, + 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, + 11, 9710, 5099, 8166, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.7406, 1.7406, 1.7406, ..., 2.0037, 2.0037, 2.0037], + [ 1.7406, 1.7406, 1.6968, ..., 2.0606, 2.0464, 2.0890], + [ 1.7406, 1.7406, 1.7406, ..., 1.7620, 1.5344, 1.5060], + ..., + [ 1.7406, 1.7406, 1.7406, ..., 2.1459, 2.0321, 2.0748], + [ 0.2223, -0.3616, 0.6165, ..., 2.1459, 2.0037, 2.1317], + [ 0.3975, -0.0842, -0.0696, ..., 2.1459, 2.0748, 2.0890]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img027_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, + 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, + 2953, 3198, 52679, 19125, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, + 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, + 2953, 3198, 52679, 19125, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img028_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, + 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 5220, 374, 12233, 458, 220, + 23, 15, 82, 259, 481, 15478, 448, 458, 2168, + 315, 264, 46652, 1841, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 5220, 374, 12233, 458, 220, + 23, 15, 82, 259, 481, 15478, 448, 458, 2168, + 315, 264, 46652, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 1.6055, 1.6055, 1.6055], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317], + [1.3756, 1.4486, 1.6676, ..., 2.1175, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img029_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 1782, 9382, 642, 56727, 11, 448, + 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, + 1632, 438, 1008, 12396, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 1782, 9382, 642, 56727, 11, 448, + 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, + 1632, 438, 1008, 12396, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + ..., + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img030_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, + 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, + 198, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, + 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, + 49496, 8968, 52001, 8778, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, + 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, + 49496, 8968, 52001, 8778, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.7114, 1.4632, ..., 1.6482, 1.6482, 1.6624], + [1.3318, 1.3318, 1.3318, ..., 1.6909, 1.6909, 1.6766], + [1.9303, 1.8135, 1.4340, ..., 1.6055, 1.6055, 1.5913], + ..., + [0.9522, 1.0252, 0.7479, ..., 1.3069, 1.3069, 1.3354], + [1.3026, 1.2734, 1.2588, ..., 1.7193, 1.7477, 1.7477], + [1.5508, 1.5362, 1.5508, ..., 1.7335, 1.7193, 1.7193]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img031_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 9190, 5612, 551, 279, + 9124, 2213, 315, 279, 2168, 624, 151652, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151653, 151645, 198, 151644, 77091, 198, 267, 287, + 10464, 93957, 45993, 1173, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, 267, 287, + 10464, 93957, 45993, 1173, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + ..., + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None)] +---------- +PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.056 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1222 22:03:46.728785705 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +> setting tensorboard ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 0.921 seconds +time to initialize megatron (seconds): 12.880 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-22 22:03:57 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc1.linear.bias + module.adapter.layernorm.bias + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.weight + module.adapter.linear_fc2.linear.bias +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (7090.80, 7090.82) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 22:04:44 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-22 22:04:45 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (46747.78, 46747.84) + train/valid/test-data-iterators-setup ..........: (1066.80, 1066.81)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-22 22:04:45 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ + for idx, (sample_idx, inner_sample) in enumerate( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx + x = next(it) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator + sample = next(it) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample + l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples + super().pack_selected_samples(samples), + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 346, in pack_selected_samples + sample = sample[:max_length] +TypeError: 'Qwen2VLImageTaskSample' object is not subscriptable +---------- +PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.054 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1222 22:12:12.619771483 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +> setting tensorboard ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 1.358 seconds +time to initialize megatron (seconds): 15.380 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-22 22:12:26 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False3 + False +3 False +3 False +33 False +False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc1.linear.bias + module.adapter.layernorm.bias + module.adapter.linear_fc2.linear.weight + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (6811.19, 6811.28) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 22:13:11 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-22 22:13:12 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (45536.07, 45541.33) + train/valid/test-data-iterators-setup ..........: (1021.23, 1021.28)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-22 22:13:12 +packing_seq_len:4096----<<<<<----4176 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ + for idx, (sample_idx, inner_sample) in enumerate( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx + x = next(it) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator + sample = next(it) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample + l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples + super().pack_selected_samples(samples), + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 380, in pack_selected_samples + raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") +ValueError: Packed sample exceeds the maximum sequence length of 4096: [Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img000_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, + 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, 198, + 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), total_len=315, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7777, ..., 0.8803, 0.8661, 0.8803], + [-1.7923, -1.7923, -1.7923, ..., -1.4660, -1.4660, -1.4518], + ..., + [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], + [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 46]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img001_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 60424, + 264, 9814, 4008, 315, 279, 2661, 2168, 13, 151645, + 198, 151644, 77091, 198, 1782, 31535, 369, 16301, 55155, + 304, 6303, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 1782, 31535, 369, 16301, 55155, + 304, 6303, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img002_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 1782, 1656, 29428, + 14126, 22361, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 1782, 1656, 29428, + 14126, 22361, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img003_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 276, 2168, 315, 264, 41530, 15616, 28642, 448, + 52484, 12624, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 276, 2168, 315, 264, 41530, 15616, 28642, 448, + 52484, 12624, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img004_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 6750, 264, 2797, 323, 63594, 12126, 315, + 279, 6548, 13, 151645, 198, 151644, 77091, 198, 1782, + 1901, 36780, 15273, 19818, 4621, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, 1782, + 1901, 36780, 15273, 19818, 4621, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.4048, 1.4048, 1.4048, ..., 1.7620, 1.7762, 1.7762], + [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], + [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], + ..., + [ 1.3026, 1.3026, 1.3026, ..., -1.0252, -1.0252, -1.0252], + [-0.1572, 0.6019, 0.4997, ..., -0.9399, -0.9399, -0.9683], + [-0.2740, -0.4784, 0.6457, ..., -0.9683, -0.9683, -0.9683]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img005_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, + 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 78243, 75105, 220, 23, + 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, + 6430, 12136, 56432, 3421, 1947, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 78243, 75105, 220, 23, + 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, + 6430, 12136, 56432, 3421, 1947, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.3835, -1.3835, -1.3835, ..., -0.5275, -0.5275, -0.5275], + [-1.3689, -1.3543, -1.3251, ..., -0.3853, -0.3995, -0.3995], + [-1.3835, -1.3689, -1.3689, ..., -0.4848, -0.4706, -0.4422], + ..., + [-1.3981, -1.4127, -1.4273, ..., -0.4137, -0.3995, -0.3995], + [-1.2375, -0.2594, -1.6609, ..., -0.5275, -0.5133, -0.5133], + [-1.1937, -0.3032, 0.3391, ..., -0.5417, -0.5417, -0.5417]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img006_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, + 264, 9814, 4008, 315, 279, 2168, 13, 151645, 198, + 151644, 77091, 198, 1782, 10550, 29674, 645, 12426, 448, + 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, + 259, 481, 36724, 481, 2953, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 1782, 10550, 29674, 645, 12426, 448, + 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, + 259, 481, 36724, 481, 2953, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.9893, -1.0039, -1.0039, ..., -0.2715, -0.2715, -0.2715], + [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2573], + [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2715], + ..., + [-1.0915, -1.0915, -1.0769, ..., -0.2857, -0.3000, -0.3142], + [-1.1207, -1.1207, -1.1207, ..., -0.3853, -0.3568, -0.3568], + [-1.1061, -1.1061, -1.1061, ..., -0.3142, -0.3000, -0.3000]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img007_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 1782, 42646, 331, 595, 3821, 18, 21, 19, + 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, + 323, 264, 24461, 9509, 27860, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 1782, 42646, 331, 595, 3821, 18, 21, 19, + 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, + 323, 264, 24461, 9509, 27860, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img008_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 67, 20367, 19684, 17, 220, 17, + 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, + 323, 18579, 71738, 11, 4024, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 67, 20367, 19684, 17, 220, 17, + 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, + 323, 18579, 71738, 11, 4024, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + ..., + [-1.2959, -1.3397, -1.7923, ..., 2.0748, 2.0321, 2.1459], + [-1.4273, -1.7193, -0.9602, ..., 1.7620, 1.9184, 1.9610], + [-1.5003, -1.4419, -1.7923, ..., 1.9895, 2.0748, 2.0321]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img009_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, + 198, 151644, 77091, 198, 759, 75242, 12426, 11, 66274, + 32300, 4603, 32300, 4603, 1841, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 759, 75242, 12426, 11, 66274, + 32300, 4603, 32300, 4603, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.2375, -1.5733, -1.7923, ..., -0.8261, -0.6412, -0.6270], + [-1.7923, -1.7923, -1.7777, ..., -0.9683, -0.9683, -0.9683], + [-1.7923, -1.7923, -1.7631, ..., -0.9683, -0.9683, -0.9683], + ..., + [-1.7339, -1.5879, -1.4565, ..., 2.1459, 2.1459, 2.1459], + [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459], + [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img010_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 64, 2766, 1809, 448, + 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, + 279, 18967, 13962, 32372, 4426, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 64, 2766, 1809, 448, + 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, + 279, 18967, 13962, 32372, 4426, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img011_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, + 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, + 151645, 198, 151644, 77091, 198, 1782, 11623, 369, 1435, + 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, + 15786, 369, 279, 5593, 364, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 1782, 11623, 369, 1435, + 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, + 15786, 369, 279, 5593, 364, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3807, -1.3807, -1.3807], + ..., + [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img012_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, + 198, 151644, 77091, 198, 5082, 352, 263, 6837, 39698, + 1899, 69105, 938, 81625, 819, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 5082, 352, 263, 6837, 39698, + 1899, 69105, 938, 81625, 819, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.6536, -1.4419, -1.7923, ..., -1.4518, -1.4518, -1.4518], + [-1.7631, -1.7631, -1.7485, ..., -1.3096, -1.4233, -1.4802], + [-1.7631, -1.7631, -1.7631, ..., -1.4518, -1.4518, -1.4518], + ..., + [-1.7631, -1.7631, -1.7047, ..., -1.4518, -1.4518, -1.4518], + [-0.0113, 0.2953, -1.5733, ..., -1.3238, -1.3807, -1.3949], + [ 0.8063, -0.5368, -1.7777, ..., -1.4518, -1.4518, -1.4518]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img013_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 74785, + 279, 2168, 3529, 285, 974, 13, 151645, 198, 151644, + 77091, 198, 19789, 6078, 304, 279, 6083, 315, 264, + 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, + 5382, 3054, 2114, 10576, 902, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, 19789, 6078, 304, 279, 6083, 315, 264, + 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, + 5382, 3054, 2114, 10576, 902, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5771, 1.5771], + [ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5629, 1.5487], + [ 1.3464, 1.3464, 1.3464, ..., 1.5771, 1.5771, 1.5629], + ..., + [-0.9310, -0.9310, -0.9018, ..., 0.9514, 1.9753, 1.9895], + [-0.7412, -0.7266, -0.7120, ..., -0.7834, -0.7408, -0.7408], + [-0.5806, -0.5660, -0.5368, ..., 0.9230, 2.0606, 2.0748]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img014_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, + 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, + 151645, 198, 151644, 77091, 198, 2277, 481, 61794, 18342, + 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, + 311, 259, 481, 6470, 34957, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 2277, 481, 61794, 18342, + 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, + 311, 259, 481, 6470, 34957, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], + [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7762, 1.7762], + [1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], + ..., + [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7620, 1.7335], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.8281, 1.7114, 1.5800, ..., 2.1317, 2.1317, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img015_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 14593, 15478, 11, 825, 3338, + 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, + 481, 10658, 1419, 19507, 2062, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 14593, 15478, 11, 825, 3338, + 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, + 481, 10658, 1419, 19507, 2062, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + ..., + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img016_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, + 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 500, 5064, 5750, 11, + 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, + 28451, 22601, 11, 12188, 22601, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 500, 5064, 5750, 11, + 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, + 28451, 22601, 11, 12188, 22601, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img017_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 1782, 6623, 60037, + 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, + 738, 448, 19752, 3931, 48051, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 1782, 6623, 60037, + 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, + 738, 448, 19752, 3931, 48051, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img018_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 60424, 264, 9814, 4008, 315, 279, 2661, + 2168, 13, 151645, 198, 151644, 77091, 198, 2024, 3045, + 7821, 2103, 488, 11030, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, 2024, 3045, + 7821, 2103, 488, 11030, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [-0.2886, -0.2302, 0.0179, ..., 2.0890, 2.1175, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img019_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, + 264, 2805, 323, 2797, 16148, 315, 279, 17420, 2168, + 13, 151645, 198, 151644, 77091, 198, 64, 2781, 504, + 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, + 389, 264, 7002, 4034, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 64, 2781, 504, + 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, + 389, 264, 7002, 4034, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., -0.5986, -0.5986, -0.6270], + ..., + [-1.2083, -1.2083, -1.2083, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img020_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, + 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, + 198, 1782, 1081, 72725, 550, 315, 6470, 296, 377, + 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, + 29724, 315, 279, 93767, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 1782, 1081, 72725, 550, 315, 6470, 296, 377, + 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, + 29724, 315, 279, 93767, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img021_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 7985, 264, 50537, 714, + 38219, 12126, 315, 279, 6802, 624, 151652, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, + 198, 151644, 77091, 198, 1782, 3691, 1555, 4013, 281, + 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, + 1555, 2550, 2355, 8149, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 1782, 3691, 1555, 4013, 281, + 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, + 1555, 2550, 2355, 8149, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img022_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 19789, 87361, 14201, + 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, + 2472, 315, 1059, 59511, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 19789, 87361, 14201, + 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, + 2472, 315, 1059, 59511, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[-1.2083, -1.2083, -1.2083, ..., -0.9256, -0.9256, -0.9399], + [-1.1207, -1.1207, -1.0915, ..., -0.8545, -0.8688, -0.8688], + [-1.2959, -1.2959, -1.2813, ..., -1.0821, -1.0678, -1.0678], + ..., + [-0.7704, -0.7850, -0.7704, ..., -0.4990, -0.4990, -0.4990], + [-0.7558, -0.7704, -0.7850, ..., -0.3142, -0.3142, -0.2857], + [-0.8142, -0.8142, -0.8142, ..., -0.2715, -0.2431, -0.2289]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img023_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 3838, 374, 304, 279, + 6548, 5267, 151652, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 18824, 315, 13779, 9317, 1173, + 553, 9080, 567, 84946, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 18824, 315, 13779, 9317, 1173, + 553, 9080, 567, 84946, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., -0.7123, -0.7266, -0.7266], + [ 1.9303, 1.9303, 1.9303, ..., -0.7834, -0.7977, -0.7977], + [-1.0039, -1.3251, -1.2083, ..., -0.7977, -0.7692, -0.7408], + ..., + [ 1.0836, 1.1128, 1.2442, ..., 2.0890, 2.1032, 2.1175], + [ 0.9814, 0.9814, 0.9814, ..., 2.1175, 2.1175, 2.1317], + [ 1.3464, 1.4048, 1.4924, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img024_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 64, 22360, 3745, 448, + 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, + 4158, 67590, 3745, 26334, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 64, 22360, 3745, 448, + 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, + 4158, 67590, 3745, 26334, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.7260, 1.7260, 1.7260, ..., 1.9042, 1.9042, 1.9042], + [1.7260, 1.7260, 1.7260, ..., 1.8899, 1.8899, 1.8899], + [1.6822, 1.6822, 1.6822, ..., 1.8473, 1.8473, 1.8473], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img025_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, + 198, 151644, 77091, 198, 64, 3691, 16049, 8511, 25279, + 11, 825, 7100, 962, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 64, 3691, 16049, 8511, 25279, + 11, 825, 7100, 962, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.7844, 1.7990, 1.8135, ..., 2.0179, 2.0464, 2.0748], + [1.8719, 1.8719, 1.8573, ..., 2.0890, 2.1459, 2.1459], + [1.8427, 1.8427, 1.8573, ..., 2.0606, 2.0606, 2.0748], + ..., + [1.7990, 1.7698, 1.7844, ..., 2.0606, 2.0606, 2.0606], + [1.8427, 1.8427, 1.8427, ..., 2.0321, 2.0321, 2.0464], + [1.7990, 1.7844, 1.7990, ..., 2.0890, 2.1032, 2.1032]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img026_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 1151, 6918, 5636, 304, + 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, + 11, 9710, 5099, 8166, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 1151, 6918, 5636, 304, + 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, + 11, 9710, 5099, 8166, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.7406, 1.7406, 1.7406, ..., 2.0037, 2.0037, 2.0037], + [ 1.7406, 1.7406, 1.6968, ..., 2.0606, 2.0464, 2.0890], + [ 1.7406, 1.7406, 1.7406, ..., 1.7620, 1.5344, 1.5060], + ..., + [ 1.7406, 1.7406, 1.7406, ..., 2.1459, 2.0321, 2.0748], + [ 0.2223, -0.3616, 0.6165, ..., 2.1459, 2.0037, 2.1317], + [ 0.3975, -0.0842, -0.0696, ..., 2.1459, 2.0748, 2.0890]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img027_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, + 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, + 2953, 3198, 52679, 19125, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, + 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, + 2953, 3198, 52679, 19125, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img028_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, + 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 5220, 374, 12233, 458, 220, + 23, 15, 82, 259, 481, 15478, 448, 458, 2168, + 315, 264, 46652, 1841, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 5220, 374, 12233, 458, 220, + 23, 15, 82, 259, 481, 15478, 448, 458, 2168, + 315, 264, 46652, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 1.6055, 1.6055, 1.6055], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317], + [1.3756, 1.4486, 1.6676, ..., 2.1175, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img029_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 1782, 9382, 642, 56727, 11, 448, + 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, + 1632, 438, 1008, 12396, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 1782, 9382, 642, 56727, 11, 448, + 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, + 1632, 438, 1008, 12396, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + ..., + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img030_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, + 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, + 198, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, + 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, + 49496, 8968, 52001, 8778, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, + 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, + 49496, 8968, 52001, 8778, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.7114, 1.4632, ..., 1.6482, 1.6482, 1.6624], + [1.3318, 1.3318, 1.3318, ..., 1.6909, 1.6909, 1.6766], + [1.9303, 1.8135, 1.4340, ..., 1.6055, 1.6055, 1.5913], + ..., + [0.9522, 1.0252, 0.7479, ..., 1.3069, 1.3069, 1.3354], + [1.3026, 1.2734, 1.2588, ..., 1.7193, 1.7477, 1.7477], + [1.5508, 1.5362, 1.5508, ..., 1.7335, 1.7193, 1.7193]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img031_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 9190, 5612, 551, 279, + 9124, 2213, 315, 279, 2168, 624, 151652, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151653, 151645, 198, 151644, 77091, 198, 267, 287, + 10464, 93957, 45993, 1173, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, 267, 287, + 10464, 93957, 45993, 1173, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + ..., + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None)] +---------- +PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 13:06:02.340272124 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 0.413 seconds +time to initialize megatron (seconds): 7.457 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 13:06:09 +building llava-ov-1.5-4b model ... +3 False +3 False +33 FalseFalse + +33 False False + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +3 False +3 False +33 False False + +3 3False +False +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 False +False +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +3 False +3 False +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +3 False3 + False +33 FalseFalse + +3 False +3 False +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 False +False +33 FalseFalse + +3 False +3 False +33 False + False +33 FalseFalse + +33 False False + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + +3 False +3 False +3 False +3 False +33 FalseFalse + +3 False +3 False +33 FalseFalse + +33 FalseFalse + +33 FalseFalse + + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc2.linear.bias + module.adapter.layernorm.bias + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (8605.30, 8605.43) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:06:57 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 13:06:58 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (47788.52, 47788.53) + train/valid/test-data-iterators-setup ..........: (1018.18, 1018.19)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 13:06:58 +packing_seq_len:4096----<<<<<----4176 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ + for idx, (sample_idx, inner_sample) in enumerate( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx + x = next(it) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator + sample = next(it) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample + l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples + super().pack_selected_samples(samples), + File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 380, in pack_selected_samples + raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") +ValueError: Packed sample exceeds the maximum sequence length of 4096: [Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img000_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, + 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, 198, + 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), total_len=315, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7777, ..., 0.8803, 0.8661, 0.8803], + [-1.7923, -1.7923, -1.7923, ..., -1.4660, -1.4660, -1.4518], + ..., + [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], + [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 46]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img001_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 60424, + 264, 9814, 4008, 315, 279, 2661, 2168, 13, 151645, + 198, 151644, 77091, 198, 1782, 31535, 369, 16301, 55155, + 304, 6303, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 1782, 31535, 369, 16301, 55155, + 304, 6303, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img002_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 1782, 1656, 29428, + 14126, 22361, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 1782, 1656, 29428, + 14126, 22361, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img003_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 276, 2168, 315, 264, 41530, 15616, 28642, 448, + 52484, 12624, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 276, 2168, 315, 264, 41530, 15616, 28642, 448, + 52484, 12624, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img004_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 6750, 264, 2797, 323, 63594, 12126, 315, + 279, 6548, 13, 151645, 198, 151644, 77091, 198, 1782, + 1901, 36780, 15273, 19818, 4621, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, 1782, + 1901, 36780, 15273, 19818, 4621, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.4048, 1.4048, 1.4048, ..., 1.7620, 1.7762, 1.7762], + [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], + [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], + ..., + [ 1.3026, 1.3026, 1.3026, ..., -1.0252, -1.0252, -1.0252], + [-0.1572, 0.6019, 0.4997, ..., -0.9399, -0.9399, -0.9683], + [-0.2740, -0.4784, 0.6457, ..., -0.9683, -0.9683, -0.9683]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img005_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, + 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 78243, 75105, 220, 23, + 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, + 6430, 12136, 56432, 3421, 1947, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 78243, 75105, 220, 23, + 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, + 6430, 12136, 56432, 3421, 1947, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.3835, -1.3835, -1.3835, ..., -0.5275, -0.5275, -0.5275], + [-1.3689, -1.3543, -1.3251, ..., -0.3853, -0.3995, -0.3995], + [-1.3835, -1.3689, -1.3689, ..., -0.4848, -0.4706, -0.4422], + ..., + [-1.3981, -1.4127, -1.4273, ..., -0.4137, -0.3995, -0.3995], + [-1.2375, -0.2594, -1.6609, ..., -0.5275, -0.5133, -0.5133], + [-1.1937, -0.3032, 0.3391, ..., -0.5417, -0.5417, -0.5417]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img006_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, + 264, 9814, 4008, 315, 279, 2168, 13, 151645, 198, + 151644, 77091, 198, 1782, 10550, 29674, 645, 12426, 448, + 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, + 259, 481, 36724, 481, 2953, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 1782, 10550, 29674, 645, 12426, 448, + 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, + 259, 481, 36724, 481, 2953, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.9893, -1.0039, -1.0039, ..., -0.2715, -0.2715, -0.2715], + [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2573], + [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2715], + ..., + [-1.0915, -1.0915, -1.0769, ..., -0.2857, -0.3000, -0.3142], + [-1.1207, -1.1207, -1.1207, ..., -0.3853, -0.3568, -0.3568], + [-1.1061, -1.1061, -1.1061, ..., -0.3142, -0.3000, -0.3000]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img007_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 1782, 42646, 331, 595, 3821, 18, 21, 19, + 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, + 323, 264, 24461, 9509, 27860, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 1782, 42646, 331, 595, 3821, 18, 21, 19, + 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, + 323, 264, 24461, 9509, 27860, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img008_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 67, 20367, 19684, 17, 220, 17, + 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, + 323, 18579, 71738, 11, 4024, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 67, 20367, 19684, 17, 220, 17, + 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, + 323, 18579, 71738, 11, 4024, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], + ..., + [-1.2959, -1.3397, -1.7923, ..., 2.0748, 2.0321, 2.1459], + [-1.4273, -1.7193, -0.9602, ..., 1.7620, 1.9184, 1.9610], + [-1.5003, -1.4419, -1.7923, ..., 1.9895, 2.0748, 2.0321]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img009_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, + 198, 151644, 77091, 198, 759, 75242, 12426, 11, 66274, + 32300, 4603, 32300, 4603, 1841, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 759, 75242, 12426, 11, 66274, + 32300, 4603, 32300, 4603, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-1.2375, -1.5733, -1.7923, ..., -0.8261, -0.6412, -0.6270], + [-1.7923, -1.7923, -1.7777, ..., -0.9683, -0.9683, -0.9683], + [-1.7923, -1.7923, -1.7631, ..., -0.9683, -0.9683, -0.9683], + ..., + [-1.7339, -1.5879, -1.4565, ..., 2.1459, 2.1459, 2.1459], + [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459], + [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img010_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 64, 2766, 1809, 448, + 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, + 279, 18967, 13962, 32372, 4426, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 64, 2766, 1809, 448, + 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, + 279, 18967, 13962, 32372, 4426, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img011_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, + 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, + 151645, 198, 151644, 77091, 198, 1782, 11623, 369, 1435, + 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, + 15786, 369, 279, 5593, 364, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 1782, 11623, 369, 1435, + 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, + 15786, 369, 279, 5593, 364, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3807, -1.3807, -1.3807], + ..., + [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665], + [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img012_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, + 198, 151644, 77091, 198, 5082, 352, 263, 6837, 39698, + 1899, 69105, 938, 81625, 819, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 5082, 352, 263, 6837, 39698, + 1899, 69105, 938, 81625, 819, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[-0.6536, -1.4419, -1.7923, ..., -1.4518, -1.4518, -1.4518], + [-1.7631, -1.7631, -1.7485, ..., -1.3096, -1.4233, -1.4802], + [-1.7631, -1.7631, -1.7631, ..., -1.4518, -1.4518, -1.4518], + ..., + [-1.7631, -1.7631, -1.7047, ..., -1.4518, -1.4518, -1.4518], + [-0.0113, 0.2953, -1.5733, ..., -1.3238, -1.3807, -1.3949], + [ 0.8063, -0.5368, -1.7777, ..., -1.4518, -1.4518, -1.4518]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img013_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 74785, + 279, 2168, 3529, 285, 974, 13, 151645, 198, 151644, + 77091, 198, 19789, 6078, 304, 279, 6083, 315, 264, + 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, + 5382, 3054, 2114, 10576, 902, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, 19789, 6078, 304, 279, 6083, 315, 264, + 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, + 5382, 3054, 2114, 10576, 902, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5771, 1.5771], + [ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5629, 1.5487], + [ 1.3464, 1.3464, 1.3464, ..., 1.5771, 1.5771, 1.5629], + ..., + [-0.9310, -0.9310, -0.9018, ..., 0.9514, 1.9753, 1.9895], + [-0.7412, -0.7266, -0.7120, ..., -0.7834, -0.7408, -0.7408], + [-0.5806, -0.5660, -0.5368, ..., 0.9230, 2.0606, 2.0748]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img014_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, + 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, + 151645, 198, 151644, 77091, 198, 2277, 481, 61794, 18342, + 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, + 311, 259, 481, 6470, 34957, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 2277, 481, 61794, 18342, + 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, + 311, 259, 481, 6470, 34957, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], + [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7762, 1.7762], + [1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], + ..., + [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7620, 1.7335], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.8281, 1.7114, 1.5800, ..., 2.1317, 2.1317, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img015_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 14593, 15478, 11, 825, 3338, + 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, + 481, 10658, 1419, 19507, 2062, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 14593, 15478, 11, 825, 3338, + 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, + 481, 10658, 1419, 19507, 2062, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + ..., + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], + [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img016_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, + 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 500, 5064, 5750, 11, + 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, + 28451, 22601, 11, 12188, 22601, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 500, 5064, 5750, 11, + 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, + 28451, 22601, 11, 12188, 22601, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img017_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 1782, 6623, 60037, + 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, + 738, 448, 19752, 3931, 48051, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 1782, 6623, 60037, + 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, + 738, 448, 19752, 3931, 48051, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img018_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151653, 198, 60424, 264, 9814, 4008, 315, 279, 2661, + 2168, 13, 151645, 198, 151644, 77091, 198, 2024, 3045, + 7821, 2103, 488, 11030, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, 2024, 3045, + 7821, 2103, 488, 11030, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [-0.2886, -0.2302, 0.0179, ..., 2.0890, 2.1175, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img019_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, + 264, 2805, 323, 2797, 16148, 315, 279, 17420, 2168, + 13, 151645, 198, 151644, 77091, 198, 64, 2781, 504, + 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, + 389, 264, 7002, 4034, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 64, 2781, 504, + 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, + 389, 264, 7002, 4034, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., -0.5986, -0.5986, -0.6270], + ..., + [-1.2083, -1.2083, -1.2083, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img020_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, + 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, + 198, 1782, 1081, 72725, 550, 315, 6470, 296, 377, + 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, + 29724, 315, 279, 93767, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 1782, 1081, 72725, 550, 315, 6470, 296, 377, + 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, + 29724, 315, 279, 93767, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img021_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 7985, 264, 50537, 714, + 38219, 12126, 315, 279, 6802, 624, 151652, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, + 198, 151644, 77091, 198, 1782, 3691, 1555, 4013, 281, + 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, + 1555, 2550, 2355, 8149, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 1782, 3691, 1555, 4013, 281, + 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, + 1555, 2550, 2355, 8149, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img022_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, + 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, + 13, 151645, 198, 151644, 77091, 198, 19789, 87361, 14201, + 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, + 2472, 315, 1059, 59511, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, 19789, 87361, 14201, + 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, + 2472, 315, 1059, 59511, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[-1.2083, -1.2083, -1.2083, ..., -0.9256, -0.9256, -0.9399], + [-1.1207, -1.1207, -1.0915, ..., -0.8545, -0.8688, -0.8688], + [-1.2959, -1.2959, -1.2813, ..., -1.0821, -1.0678, -1.0678], + ..., + [-0.7704, -0.7850, -0.7704, ..., -0.4990, -0.4990, -0.4990], + [-0.7558, -0.7704, -0.7850, ..., -0.3142, -0.3142, -0.2857], + [-0.8142, -0.8142, -0.8142, ..., -0.2715, -0.2431, -0.2289]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img023_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 3838, 374, 304, 279, + 6548, 5267, 151652, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 18824, 315, 13779, 9317, 1173, + 553, 9080, 567, 84946, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 18824, 315, 13779, 9317, 1173, + 553, 9080, 567, 84946, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., -0.7123, -0.7266, -0.7266], + [ 1.9303, 1.9303, 1.9303, ..., -0.7834, -0.7977, -0.7977], + [-1.0039, -1.3251, -1.2083, ..., -0.7977, -0.7692, -0.7408], + ..., + [ 1.0836, 1.1128, 1.2442, ..., 2.0890, 2.1032, 2.1175], + [ 0.9814, 0.9814, 0.9814, ..., 2.1175, 2.1175, 2.1317], + [ 1.3464, 1.4048, 1.4924, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img024_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 64, 22360, 3745, 448, + 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, + 4158, 67590, 3745, 26334, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 64, 22360, 3745, 448, + 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, + 4158, 67590, 3745, 26334, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.7260, 1.7260, 1.7260, ..., 1.9042, 1.9042, 1.9042], + [1.7260, 1.7260, 1.7260, ..., 1.8899, 1.8899, 1.8899], + [1.6822, 1.6822, 1.6822, ..., 1.8473, 1.8473, 1.8473], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img025_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, + 198, 151644, 77091, 198, 64, 3691, 16049, 8511, 25279, + 11, 825, 7100, 962, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, 64, 3691, 16049, 8511, 25279, + 11, 825, 7100, 962, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.7844, 1.7990, 1.8135, ..., 2.0179, 2.0464, 2.0748], + [1.8719, 1.8719, 1.8573, ..., 2.0890, 2.1459, 2.1459], + [1.8427, 1.8427, 1.8573, ..., 2.0606, 2.0606, 2.0748], + ..., + [1.7990, 1.7698, 1.7844, ..., 2.0606, 2.0606, 2.0606], + [1.8427, 1.8427, 1.8427, ..., 2.0321, 2.0321, 2.0464], + [1.7990, 1.7844, 1.7990, ..., 2.0890, 2.1032, 2.1032]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img026_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, + 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, + 151645, 198, 151644, 77091, 198, 1151, 6918, 5636, 304, + 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, + 11, 9710, 5099, 8166, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, 1151, 6918, 5636, 304, + 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, + 11, 9710, 5099, 8166, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[ 1.7406, 1.7406, 1.7406, ..., 2.0037, 2.0037, 2.0037], + [ 1.7406, 1.7406, 1.6968, ..., 2.0606, 2.0464, 2.0890], + [ 1.7406, 1.7406, 1.7406, ..., 1.7620, 1.5344, 1.5060], + ..., + [ 1.7406, 1.7406, 1.7406, ..., 2.1459, 2.0321, 2.0748], + [ 0.2223, -0.3616, 0.6165, ..., 2.1459, 2.0037, 2.1317], + [ 0.3975, -0.0842, -0.0696, ..., 2.1459, 2.0748, 2.0890]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img027_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, + 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, + 198, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, + 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, + 2953, 3198, 52679, 19125, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, + 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, + 2953, 3198, 52679, 19125, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img028_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, + 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 64, 5220, 374, 12233, 458, 220, + 23, 15, 82, 259, 481, 15478, 448, 458, 2168, + 315, 264, 46652, 1841, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 64, 5220, 374, 12233, 458, 220, + 23, 15, 82, 259, 481, 15478, 448, 458, 2168, + 315, 264, 46652, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 1.6055, 1.6055, 1.6055], + [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], + ..., + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317], + [1.3756, 1.4486, 1.6676, ..., 2.1175, 2.1459, 2.1459], + [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img029_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, + 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, + 151644, 77091, 198, 1782, 9382, 642, 56727, 11, 448, + 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, + 1632, 438, 1008, 12396, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, 1782, 9382, 642, 56727, 11, 448, + 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, + 1632, 438, 1008, 12396, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + ..., + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], + [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img030_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, + 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, + 198, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, + 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, + 49496, 8968, 52001, 8778, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, + 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, + 49496, 8968, 52001, 8778, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.9303, 1.7114, 1.4632, ..., 1.6482, 1.6482, 1.6624], + [1.3318, 1.3318, 1.3318, ..., 1.6909, 1.6909, 1.6766], + [1.9303, 1.8135, 1.4340, ..., 1.6055, 1.6055, 1.5913], + ..., + [0.9522, 1.0252, 0.7479, ..., 1.3069, 1.3069, 1.3354], + [1.3026, 1.2734, 1.2588, ..., 1.7193, 1.7477, 1.7477], + [1.5508, 1.5362, 1.5508, ..., 1.7335, 1.7193, 1.7193]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img031_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, + 151645, 198, 151644, 872, 198, 9190, 5612, 551, 279, + 9124, 2213, 315, 279, 2168, 624, 151652, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, + 151655, 151653, 151645, 198, 151644, 77091, 198, 267, 287, + 10464, 93957, 45993, 1173, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, -100, -100, + -100, -100, -100, -100, -100, -100, -100, 267, 287, + 10464, 93957, 45993, 1173, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False, False, False, False, False, False, False, + False, False, False, False]), imgs=[tensor([[1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + ..., + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], + [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None)] +---------- +PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.026 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 13:26:14.872313642 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 0.410 seconds +time to initialize megatron (seconds): 4.290 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 13:26:17 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias + module.adapter.layernorm.bias + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (6330.73, 6330.82) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:27:03 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 13:27:04 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (45737.89, 45737.93) + train/valid/test-data-iterators-setup ..........: (849.20, 849.21) +training ... +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 13:27:04 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img020_jpg from 194 to fit remaining capacity 114. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img016_jpg from 265 to fit remaining capacity 22. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img013_jpg from 299 to fit remaining capacity 221. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision\n t = (t * cos_) + (_rotate_half(t) * sin_)\n', 'RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0\n'] +WARNING:megatron.core.utils:The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision\n t = (t * cos_) + (_rotate_half(t) * sin_)\n', 'RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision +[rank1]: t = (t * cos_) + (_rotate_half(t) * sin_) +[rank1]: RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision +[rank0]: t = (t * cos_) + (_rotate_half(t) * sin_) +[rank0]: RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 +[rank1]:[W1224 13:27:07.107477592 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 13:27:12.100223646 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 13:27:14.317000 2671643 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2671665 closing signal SIGTERM +E1224 13:27:14.532000 2671643 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2671666) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_13:27:14 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2671666) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..4e68402f --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log @@ -0,0 +1,1038 @@ +W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] +W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] ***************************************** +W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 4096 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 0 +0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.026 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 13:40:32.550238037 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.347 seconds +time to initialize megatron (seconds): 3.307 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 13:40:34 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 3False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False3 + False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.bias + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (7411.22, 7411.28) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:41:20 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 13:41:21 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (45636.49, 45637.28) + train/valid/test-data-iterators-setup ..........: (967.35, 967.44)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 13:41:21 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img020_jpg from 194 to fit remaining capacity 114. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img016_jpg from 265 to fit remaining capacity 22. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0)\n', 'RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision +[rank1]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0) +[rank1]: RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 +WARNING:megatron.core.utils:repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0)\n', 'RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1\n'] +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img013_jpg from 299 to fit remaining capacity 221. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision +[rank0]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0) +[rank0]: RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 +[rank1]:[W1224 13:41:23.232043330 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 13:41:28.264405723 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 13:41:30.309000 2673230 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2673245 closing signal SIGTERM +E1224 13:41:30.524000 2673230 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2673246) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_13:41:30 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2673246) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..c9f1c9e7 --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log @@ -0,0 +1,1038 @@ +W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] +W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] ***************************************** +W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 4096 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.028 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 13:48:06.611399525 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.221 seconds +time to initialize megatron (seconds): 3.304 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 13:48:08 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False3 False + +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 3 False +False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.layernorm.bias + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc2.linear.bias + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (6690.53, 6690.67) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:48:52 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 13:48:53 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (43774.48, 43778.90) + train/valid/test-data-iterators-setup ..........: (842.12, 859.14) +training ... +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 13:48:53 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img020_jpg from 194 to fit remaining capacity 114. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img016_jpg from 265 to fit remaining capacity 22. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img013_jpg from 299 to fit remaining capacity 221. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision +[rank1]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) +[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision +[rank0]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank1]:[W1224 13:48:56.545373505 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 13:49:01.858122644 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 13:49:02.846000 2674551 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2674566 closing signal SIGTERM +E1224 13:49:03.011000 2674551 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2674567) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_13:49:02 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2674567) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..1bbc0001 --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log @@ -0,0 +1,1038 @@ +W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] +W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] ***************************************** +W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 1024 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 1024 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.024 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 14:21:01.966538345 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 0.355 seconds +time to initialize megatron (seconds): 4.533 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 14:21:05 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False3 False + +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc2.linear.weight + module.adapter.layernorm.weight + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias + module.adapter.layernorm.bias +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (6866.03, 6866.22) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:21:50 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 14:21:51 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (45766.41, 45766.42) + train/valid/test-data-iterators-setup ..........: (860.32, 860.33)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 14:21:51 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img004_jpg from 195 to fit remaining capacity 160. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img004_jpg from 265 to fit remaining capacity 130. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img003_jpg from 299 to fit remaining capacity 139. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision +[rank1]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) +[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision +[rank0]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) +[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank1]:[W1224 14:21:54.192514656 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 14:22:00.551070334 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 14:22:01.564000 2676720 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2676735 closing signal SIGTERM +E1224 14:22:01.728000 2676720 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2676736) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_14:22:01 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2676736) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..d10d8c2f --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log @@ -0,0 +1,1038 @@ +W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] +W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] ***************************************** +W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 1024 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 1024 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.026 seconds +> compiling and loading fused kernels ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 14:23:24.177105851 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.296 seconds +time to initialize megatron (seconds): 3.627 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 14:23:27 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 3 False +False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.bias + module.adapter.linear_fc2.linear.weight + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (6575.26, 6575.33) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:24:11 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 14:24:11 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (43559.41, 43564.47) + train/valid/test-data-iterators-setup ..........: (849.64, 849.72)training ... + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 14:24:11 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img004_jpg from 195 to fit remaining capacity 160. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img004_jpg from 265 to fit remaining capacity 130. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img003_jpg from 299 to fit remaining capacity 139. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0\n'] +WARNING:megatron.core.utils:The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision +[rank1]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) +[rank1]: RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision +[rank0]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) +[rank0]: RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 +[rank1]:[W1224 14:24:14.066149140 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 14:24:19.100043220 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 14:24:21.085000 2677183 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2677198 closing signal SIGTERM +E1224 14:24:21.250000 2677183 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2677199) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_14:24:21 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2677199) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..d4d051ea --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log @@ -0,0 +1,1038 @@ +W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] +W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] ***************************************** +W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 2048 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 2048 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.025 seconds +> compiling and loading fused kernels ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 14:25:00.304975724 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.346 seconds +time to initialize megatron (seconds): 4.384 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 14:25:04 +building llava-ov-1.5-4b model ... +33 False + False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +33 False + False +3 False +3 False +3 False +3 False +3 False +3 False +3 3 False +False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.layernorm.bias + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc2.linear.weight + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (4443.62, 4443.67) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:25:46 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 14:25:47 +done with setup ... +training ...(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (41972.18, 41972.22) + train/valid/test-data-iterators-setup ..........: (840.93, 840.94) + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 14:25:47 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img010_jpg from 195 to fit remaining capacity 14. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img008_jpg from 265 to fit remaining capacity 94. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img006_jpg from 299 to fit remaining capacity 266. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision +[rank1]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) +[rank1]: RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 +WARNING:megatron.core.utils:The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0\n'] +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision +[rank0]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) +[rank0]: RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 +[rank1]:[W1224 14:25:49.461266190 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 14:25:55.544067802 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 14:25:56.532000 2677719 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2677734 closing signal SIGTERM +E1224 14:25:56.747000 2677719 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2677735) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_14:25:56 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2677735) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log new file mode 100644 index 00000000..a40f7bf7 --- /dev/null +++ b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log @@ -0,0 +1,1058 @@ +W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] +W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] ***************************************** +W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] ***************************************** +False +False +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale + warnings.warn( +INFO:datasets:PyTorch version 2.9.1 available. +INFO:datasets:PyTorch version 2.9.1 available. +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' +WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' +-------------- Configure model to llava-ov-1.5-4b -------------- + num_layers = 36 + hidden_size = 2560 + ffn_hidden_size = 9728 + num_attention_heads = 32 + group_query_attention = True + num_query_groups = 8 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = True + vocab_size_in_config_file = 151936 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-06 + rotary_base = 5000000 + kv_channels = 128 + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.flash + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... qwen2-vl + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 36 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 2048 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 2 + ffn_hidden_size ................................. 9728 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 2 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 + hidden_dropout .................................. 0 + hidden_size ..................................... 2560 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 5 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-1.5-4b + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 32 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 36 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 8 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 1 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 4 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 5000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_llava_ov_4b + save_ema ........................................ None + save_interval ................................... 2000 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 2048 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 2 + tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 5 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 4096 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 151936 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 2 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. + > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 2 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.026 seconds +> compiling and loading fused kernels ... +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W1224 14:28:39.475120241 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +>>> done with compiling and loading fused kernels. Compilation time: 0.344 seconds +time to initialize megatron (seconds): 3.439 +/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2025-12-24 14:28:42 +building llava-ov-1.5-4b model ... +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 3 False +False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False3 False + +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False +3 False + > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (27271680 elements, 27271680 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.bias + module.adapter.layernorm.weight +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine + loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 + successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 +(min, max) time across ranks (ms): + load-checkpoint ................................: (6846.22, 6846.29) +[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:29:27 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 10 + validation: 200 + test: 200 +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 +dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist +[after dataloaders are built] datetime: 2025-12-24 14:29:28 +done with setup ... +training ...(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (44510.70, 44515.78) + train/valid/test-data-iterators-setup ..........: (847.39, 868.47) + +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2025-12-24 14:29:28 +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img010_jpg from 195 to fit remaining capacity 14. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img008_jpg from 265 to fit remaining capacity 94. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +WARNING:megatron.core.utils:Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. +cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 + 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 + 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 + 577 577 577 577 625 577 577 577 577 625 577 625 577 577 + 577 577 577 625]. +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision\n raise RuntimeError(\n', 'RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328.\ncu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971\n 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991\n 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577\n 577 577 577 577 625 577 577 577 577 625 577 625 577 577\n 577 577 577 625].\n'] +[rank0]: Traceback (most recent call last): +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank0]: main() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank0]: trainer.train() +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank0]: pretrain( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank0]: iteration, num_floating_point_operations_so_far = train( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank0]: train_step(forward_step_func, +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank0]: losses_reduced = forward_backward_func( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank0]: output_tensor, num_tokens = forward_step( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank0]: output_tensor = model( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank0]: return self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank0]: outputs = self.module(*inputs, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank0]: image_embeddings, window_index = self.vision_model(images, \ +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward +[rank0]: x = self.decoder( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank0]: hidden_states = self._checkpointed_forward( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank0]: hidden_states, context = checkpoint_handler( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank0]: return tensor_parallel.checkpoint( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank0]: outputs = run_function(*args) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank0]: hidden_states, context = layer( +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank0]: attention_output_with_bias = self.self_attention( +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank0]: return self._call_impl(*args, **kwargs) +[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank0]: return forward_call(*args, **kwargs) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision +[rank0]: raise RuntimeError( +[rank0]: RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. +[rank0]: cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 +[rank0]: 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 +[rank0]: 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 +[rank0]: 577 577 577 577 625 577 577 577 577 625 577 625 577 577 +[rank0]: 577 577 577 625]. +WARNING:megatron.core.utils:Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. +cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 + 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 + 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 + 577 577 577 577 625 577 577 577 577 625 577 625 577 577 + 577 577 577 625]. +['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision\n raise RuntimeError(\n', 'RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328.\ncu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971\n 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991\n 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577\n 577 577 577 577 625 577 577 577 577 625 577 625 577 577\n 577 577 577 625].\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in +[rank1]: main() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main +[rank1]: trainer.train() +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train +[rank1]: pretrain( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train +[rank1]: train_step(forward_step_func, +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step +[rank1]: output_tensor = model( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward +[rank1]: image_embeddings, window_index = self.vision_model(images, \ +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward +[rank1]: x = self.decoder( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward +[rank1]: hidden_states = self._checkpointed_forward( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward +[rank1]: hidden_states, context = checkpoint_handler( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler +[rank1]: return tensor_parallel.checkpoint( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint +[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward +[rank1]: outputs = run_function(*args) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward +[rank1]: hidden_states, context = layer( +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward +[rank1]: attention_output_with_bias = self.self_attention( +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward +[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) +[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision +[rank1]: raise RuntimeError( +[rank1]: RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. +[rank1]: cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 +[rank1]: 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 +[rank1]: 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 +[rank1]: 577 577 577 577 625 577 577 577 577 625 577 625 577 577 +[rank1]: 577 577 577 625]. +WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img006_jpg from 299 to fit remaining capacity 266. +WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. +[rank1]:[W1224 14:29:30.229872060 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank0]:[W1224 14:29:35.162733945 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W1224 14:29:37.154000 2678274 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2678289 closing signal SIGTERM +E1224 14:29:37.318000 2678274 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2678290) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 +Traceback (most recent call last): + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in + sys.exit(main()) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper + return f(*args, **kwargs) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main + run(args) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run + elastic_launch( + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-12-24_14:29:37 + host : gpu-51 + rank : 1 (local_rank: 1) + exitcode : 1 (pid: 2678290) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766400270.gpu-52.3342268.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766400270.gpu-52.3342268.0 new file mode 100644 index 0000000000000000000000000000000000000000..83ec48654cb8825f5c98de3a19a2a970208bf7ea GIT binary patch literal 48602 zcmb__2Y6If^EVy#(xfJ|gkF=71Ofz*DhPr!QAEUbv%5)_Y|EBVM8$$g6BWgR9Z|4> zND&1)uYjT=3Zf{AC|E$SD~R9BxpVfOxw{v4`G3*J*9Y_d&fGb3rq4MeD(OFG?i}%; z?O^TALq|;bba3;!MIndBpEIn&?T_@$pOB+vMcn>yAXF4^gj_k{sL$sJ&CHn|@z~v6v zQ$(G!r})Yd>bEa=fl$xa`e~Ui_t{bF#b~WICg67))P8d0kG_Kmx6H~)AAFu$Nzt^)Y)lf}ab@P}_p*V=xSfPJ+-TFc^Mzdaey!ru9P z4A7On6ImA$=u2;&T5fx215w#u>#Ma!UOFPM{);@mN zJlnL51Upi@P|GkNiyY1}x8KEC&?3gu27@CJ{}BCma*DlXZc1XNI+5oL>NhlBv z+RLNv&`g6RoNi_7CWfMlENn^3=rNR_pP$>|5!gJzVo9eZ0}0+A=)6VeLS)~GBao`$e}<`;l{fyUmZPxz&rdhC94rg;j4h=*hY@1$n}JaFBt>BXvp=v+dJ%Favgb- zP*JX%A*$US412r*KL=H|FVCHwt9Tr~te-|Oxz2ctAcb5zf;9CC*}uBgLncLu>N}2PyPBJoXZ|H<~QsmaKML8M9mbPi1zvJn9I!U3Sp4NkhIh?o5mR zMCA5OD<;@pmymf{cdb!)x~IeprW|K(aC7haUz34^HQrs}u$8<`mIR*o!=VRQBj2aV9FLoX7)b%BC^pwo{@CcY8`o&BoAjq1BxioM76bdJS zjF?1u&g?25GZWQ(CxN4hQd~V3XgeUE<0{ULVox3|>U25to!^j{=sUL{H#Y^q0Ue$n z%K&}v5RnMjOlPMZe0_(t3d>a$@H?xv-puSp<97)V?L{Uy5+D+(!;a6Iu!s5h`y?#- z_^rTGJ_peEo^{{H^!XFJh*s3+?NutlggQg^aAc;}bTk^q9Ew_?jnGeu7@GHkaX*~T z!n|wVqXjU`YwGlRg7%V-`CO>hL1pb@*Z%| z1i`0r9PIc7R@zn0SCPLJnUL!YwOI zWp}uyu08z2&dl?7enh0A=ie1hc*qUjvI1=P6x;o7w<~$4bnv*6&sm_TeN1bJKrsXv z10X))Mw$0Vy$(o@n6!#QesR?6H8`F3_r$Ll`RgP!lHbi)>UNgdivy74aG<+u_XfgY zVHe_$m$C*oJGVIJ7c%mv?N!KcW2_Mlddk4PfhD-)DZ*^B1=XsI89JKI+HW5{_K2bpmg59ZSA-XkzzIpu0fZ*zQt)lH z6qpB#{8gEHzx-389@Tr}U?_koKMrQSkMHih;ymU{Rr-w3L>7 zOf@#`ne4lAE~8BwlYMBNt_JDQltVX*c~9BY&9J|!ELE>JZ4beK7=?#VLu6TQE&F5859dE?Qo_shF z*+o%rSt4LdvhOLsCctq2wZ@$uPp~EKcTaWg&e*Z?8=@E4(NVIaC=iH%&2^xcmLQ3* z!26v}#m9N~i-y0k(>VHDg2&U?4poOU7zOVJlGhe#AO)LC-e33IhB54)RqT##UobM$ z9`(Z-vGeYQs+E@?NyXYKCg$Q#zN*2F%Y8?PZjlUfxxpUzfa-9>;{>1GWm47F^0BNu zvx<|%@A4^lOM3a&qxcrevo?xt6_?^P4)*v+ChJZ(dCm0C*4l15M$`_}&evKC=&+~655Xc-=3HKP(mp}M zQv>@>lgKw5E3suACuF-e6%3$JlZJG(3AIPez;*$D62gu=JE)ZsKL*uyuh&uG0K;kT zSp+&DR@pY(wK*$Geq`)ADpwZl@;Sm~pi9Dm;s};yMEi+8`aghd5Q zf1-~PHHRV(oNLUbd2`u{@sn5%1nc1%>PF{Nq}32V^r>MkD7JiRX^sPOc#up<}^MJTt!BX^AM`euNK}Li6J^FR zyS`ET{mgLhl5oOB4@fxS8U%`qy#aGN(VEo1559Jc8N2e|iBvRpoeXTkwHHIoh1C|$ z3njY9Q}dPcPnFzt1?i?nf%UK2(*Gd%vD(F2F3N!(2pJ1u1_ynd6hGvpVHppy?9fYc zC6U}B_g_bjG0c@;YA?+rr^aP0 zHa{aFF*a`@A%hM``X!5b;OYJQm@<0!FCr6_QA?L2;y@8W?i5wxNF$@HlOZmhTW=i*6}|ljntyF68if-CnRNUZ{CR%?$*WeEBNx z+r>=gzm`i0nfF6z4vq}?lX}6=jx|OulGfYU`h)bD%m~aqMW2C2AOkAqQRtTf|Bi@H z7Po(=a1EA)>vWoMMHUXl!XwH^PG=PAN>qtYU=^)N?|Xae)vP__Be?{&hjezigAlNT zZ|bt+(egp;4*vvq+uY}4y~3NP4`0BTGwUCfISA`@mpGg=Q^dXYhf7ZnBH0n8hL&dt z7fTIIeUJ?X*QdmpEVHkB%~6wCn5_~IElg$%&%y;%TuLR3Inw=^XV_`H>R%#KoJIo_ z%BI%g1V7?|5n~vF6{s*K8}7H875#ysK3wS^y?ZbLs@dbWJat=# z93IHj_0YRg6~`t(@;smvptSA{-6c%;$_oCV&JH~xN~EMpw+QdQx2YlIYyQlMR-1H>CO zry~WZiY0>9V8FCRDJ%0Usm|A$yG;J7-SfMZTW9zAjUy;c7&> z$SOFakktc7xx0zlP8GKBjePls*)v$mwoOgK7AadZwvoA=~S6!R}hN)*uW;T1(RN@_Q(SvCDi>2Mhh&^@wy4 z`Uy=VJs&$15?r7fFyF}Gt#|aUR~q+W{BB-fS-n0oRu5jLA~{`?dpz7=Qie5!J}w`d(A4rpOx*nwzVnrP7)$evH|P zCJhJ)EprQRNN=TqkB@64QBa-GpR9Jw?_RunAB)pAOH_>0G68jZ7|aAV=_EpK5<4HY z-7|tv`h@-|)w9nSKIS*pX)&cC5iOdEVnipMKJ4l(uF1G?Mgj`|BnB8xDT8>lZ@=YC zIX~Kn$VBDbhI1h$|2(e1w;wD5eRsvfZzzL{L>?J(9^nq`f-Nt=4-iHBS^{G&8Jo2 zqN#~>-&329v5>8EI)RCh4gJNC$K^&}RD^>*=twSwI5vg!>>DdXA>PR-m%vWO?if&s zDn>AXrtbf+(9;(#YQ!YM-%OE+W;m=9g>1bmjy;ybyQRE!fF0t7%?VK)VgoS|(=AaS zZ*g8P#`D)CD)PJmcBpWz-6^KLTAejXZG-`Z@4tFA6FYYcB3zUPa0v@XLZLbz^tu6= z^ZOr}3{q>ULPA19po*Y{ldnK>*LnCuxA$X8_ZbO^N;h3XhRiMXmVjwfhD}&ON8^RJ zU(KBWwye7sg%M&!F#3+1SulI%>h5<>hu-^eq{8d#K65|G5RW!?oZbGx7a?AOI)Ij*Sc9R;qaCO zxa_3Iq`n^Z7{VA5rTUEXzH0rz$5;+O+Lk^@w6Ye1cNU5IO`1wEG^XyqHot7|!V(i{ z?FddJCLo*_6;;sA#*yMQTeAN5lQ-64l)fUNMYTb289T<3gwwip+rp9sj`l>UNEU#k zWB;BA36f|*;JL7&1qUBx{#%s}ge`(=lP!ckNK?w&n!gu(JGT>S_j_C-W4m8ABKsga z0El}2#Ne{{L(UuPWtw!^`wNr7aUJOcPzDQd84=@Qb%(NKB3*OV<&>S}>{{_+C?G1v%wi17FLqkcTT^uzhfZuGIUJ&bsLIVaNy}m3y7+*UYKM0JFKXjmKV_RawZk@t8nB%75s5CQcLgsTVYrY0 zA{Ux>ft*7q>WoA~NgKo`2TZZCtzDK&pMYE2#yDKqy6%klp$nHYJ0oVp8xt{sGRXQs zdP6h^8>g)HprVfbi22RiJ#(e4FppNs)pE23#lgH@9NAh*9$r>Dgk^~iNGOqoD=Ica zS@QVp%l`R+S;A}ci6~(Su~rCKS}d4?rM1TcThr*;_}#lq)>`!-WFc!}Dgq~UO*^Gf z%9Yg9k2JfAc`lzyJoH>LF>nVzK%cfqfUC6r_bMpp(M#Hy043(0s<%+oyXVwK<*dyut)b!5T93-JjdPW=V(aUIdFt zhqTg};9%iMs!2wbq==HbnetrWOU&x-mSDo_76*Ie!`{cuw{lXg1>gq9%J-PNKfO26 zDmo$bmLdI0g)%pd=*mIsWY<`9RsDS|iQK#ots#=g#vK;2`3r;?zbje%Pi8gEV6~E$ zB|g?lx;tbA9WFwUjRkt!1)i4pg_M2es_yaOH4ZUn_sT;04$)?Sy20-C*;zkSvfuDZ ziy7Cl$Mu-3Khl>77j;8<-^)2HqDcMNN00Ysf;zMx zErNpDB~~yL9R{%&D(TU=CF3tTzxhXX%$8x#O|d5g9(8P0tEnc-Wp3KI?G!7xej}F= z1y{KY+z0c2S>=_gFGg9b&-?-OF{0JSpjvc6S9n4ZY~P4|Rf5vAB7!Li4Be4#7>390Y8Y8GnN($R zOr3F?TXXh3%%9pJ;e|gXS+4k1$#VU+0+*k0z3c*&>n?XO9O~HEpj`};;K&h9xhA{D z9d%3oW&%@h7_ET<(?-V>D?vM*-3#5LrXW@^5{l+qZF{c<^SoSe;O__xu4d@^2ScrWHNv23%QbBw1<_TrV+vpesF-C0qD8J5#OW*FubMiaOULS zFp|Cmo%}ZP_+XmUXD@*{z2tK=W2y)Kv{U#cu5)R75kZUVToOTpZyqT%`oAE!G-ofZ zIrYiM56xgXl{e*bqTs3v<+*%E)xd~F%y@)G(RT>r0WpdgCDxf^*BvjMJ&GB{B*p*L zspXxROWJ%its`7gOqxM2Nr~M(1Fvv{4F`SCp{<#-WZ$9XER1jO{4{wzV2|X;+tkV988sb^! z1GpZ3q@wU?R;Hahj=lrSv@H>y+VG0QuB2t<{+5L+Sk~#a{JDeJ02oxU@k?KNra0oMKdQ2p2rtzm6X_&PQm`bhZpyJ-EQfO^+z{LR|LgMPPc*;$TR%DhtL<6_}`Y9{eUUWH8uNQ35MKQz`XL17m zVowP;H0B|F%O%zxdv6hQDvrq|gj0cf3A*2dK^)fgL0T%#^R!kS4_uwbyD@H`OwsbU3{6@@!)iJOg|EsyDl^w6wFtBQzDQGkRxx$fNRTul|d=X=ke&saOb zsj^m3tosQmkX-!ykfDqXze`YkS_*VJ-ux)8=dm_R+?6`D?1{eYn6Gvbq57Cu0Pgrg zYPri~eJUT=k+qEBO>`@GX>>boiTO|}Ti++IP*IdeYIm5$oQc0Ap6C;nIqQ^hCw#Sy z&m=A{COQSK%<3kA%gy|%=eD0<;qZtO!o_eHZr6fz9$fMP!lmC69LEVo#lw~Bh6V^+ z_}s6IvA;`T+^BTDM;PV@%tj}`8QH_uLI7U+>uB2;uS;aviFTN2(xpxlP9>{D(| zo^pMK|L0EkGYfQqhu}nM5ymf|rvkq@9#f$ZFpV#spL6R*ra_NLNO8Xg`oMbkib>WL zi>sjhd`0^0Y-1KqQ#J<7IOuQVo??%Apx9dCx0b)~81uSv$_Q9^T{^H)phL!To?1gc zc>Jw5*n!_Kp~Ym88%8?BKBaOkUil}7_ArO{7B7)29A3PV5ZXvMy_i)&C4fB*=0ubt z5nQvZzwSxi;nB@U3!#GTEJ7IFY(i?z;}zr4FsKtZ%AI&SS8m+)7CR0fNOVzkH?D-{ zyjG~?=XU4e8Gkaoke{ek@ZztQv&7r+*ql`iuW>-d^SSZP!4&X5zq|LN%v;_h@x=In z{=INgLk@eK>?#~oT#9>xXG)lfDi0Ecx=Vo?@N0va3*N7sK!pn)@s#N! zSzz^W*gZ+q@9tm9r;%!e2o?L%5^rozyUg8dO<9g}8Ic8fc^WmBhhVWs2ulxV^Ti@Us2fa{TVdsTM&uAD){(dmo8yr)vh@}%_FnNL z!yP$Gp*Is!rZBz?eO|%|Pr?0uecnF@SYh&aiHqf6xGYn;c90T4ZzmE6mce|q6Yj5a zddvwqrL3x6$T{$trIpyJzW*Ac9Zz+8V~w!5Y62UzQpnj+kiGfG(Ni_sM2#tq1@Q(YIZ?8ZkDS)YLc!{mexk+4||ECHwWJv^3mKF++J zNX5>_h8|yu-PB{Cd~|-kGJo7^L*Et*srei;lBq0eg*5m?_s^KN+AWb#TQx)LoA{L@ zMQisk{QD2rIuEeo*zy~QFf5KW1#y83TjMN%a&YpL@Tk+uUteLO9lVjiP_$bS^DU-6 zl@DeyQt+nJepVXne-nXXX|TP1Qa}y`dWzV)JNs{77Pk7$ge@$rz?L-_Zk!aE;U4$W ze;<393E{gE5{0lCz6Jrah<1Epfg@Udf7>ft@jIyfbBRV#+r_6=a2AVZX;M%`1IPV( z2kZ1tyM=(!p=%!xoM8M)sXz(-dx~covUTXMhF7s@?`64w2*YUs?5HV18xvnMP*_so zQJxpr{pAjBwCB|RyNJiH5WJwQURSnv(OU@@_YSV>fp9#YN@ai@ z%1>}}fWd*nO-H>f0BJj)07bq)04B;;D<96zLZ`p3sLs^<4v8n!JSM9}>lWBPr%5v@ zADx$9poAcAyJlBmYTmzqXhb#N8p@A2j)Y0-7e-BW|p>!rf|cp)RLd zkvo`ZS4`sAXxe5bcgG|;%3X#)PZqYfapaDT-qnqj_?9doqOru6WkobR1O-RNC9$#5_-(((pNC?R^bH?$Br-Sx19mLs{*#b z8-v8*08{TS=dzaggmld4U7a1hz*3?R&va`9O!lcvPU-A>dws;Rcn$6%V3EZ`V2X8# z$0Gjsxm_O2?!{|TOO||m$I%Sd%CSx^BpTu37oz?$&K>K6es7KKP1#YLcsG3r9>rFO zFM5L^YALSMJ9)p^$0F`!%ZPjtaVzV%;*l+xKL5kL_p(-?KKBqRwgPojQ1vhnZ{CV8lq>Z_ znhG)KZn3G|H%yOwB3BUUL}dkxV)_MmIGCeZuUZPol*UR9zxJpgGqnHC* zZ)g9*SG$>|X>lKcqorx9kc&ace>@(+Ptvt@7CghCw^%{zJ?ioNu-wK;>t6D)guzw! z6V>7#0EKFiDxhFK;oFOrYrHm~@lLiz&E?7(U6nPc`6RX$?qaUEJh^>I>Ai4*57m!K{8i?G*#T;015!oF-M?()h%GB5%2^&*mvxMeL1ZpXpV;+9w3zlw)K1{@l zbPE#e@#71E(>$V;H}0h}OXN)zjfb?L-QW}*@=2l zV&kb5%2%&pKm(r0QK0V`cdxI~R0tDudXuuDet)gnZUjqFPJWc=$0#xb>8IoBg8stf zo3Y=0b;)sNtxri@(Re9vb%)OB2trm89-v4R16uO^(%0wt*vQobj}hJC)(63NFsCt} z12-401dxkyVN}$C>GFoa?HEo4IZ zjl>p%EBFbz*ra^lWXX~}53@rw?{T6U4^c~<`QYoq>u%78mn<(HX}Piv6P3;@&7y*; zF&cEvgtraNR~IRKF!6Zru%V-vYq~??irkLaN_ayA-!Myb)4etD$teSyGK1=Uf(RCM z9yHJ}G#(LoBbTU`z|BYI8*>#OLy3M)e!XBc%R!v?BteUMIJ8ef`wMV4uQ<=tw_r_f zv*p$AWwN$bVxlT(!!Tj4#tE0U@~u;HFwhF@$afgBmc8Pq^(vR<48m*w}>?q?mw1uA~B1M0W4u!+r@fmk3$nEJhTQGgs^f`hQ`VOloV(teLY$;q8 z|4oOwyV!YrQKAadj;N?Y;JzQ&dPuh7FabZ1)@Oau@t+=IMS`o>5XD#|Xo1*pS8O0! zQVNDESb@xrVNGYH77dOw$ki%jD}2oYTt_4~=?9~_ z3>n35@*Mj-Q7L?}A_v^N2#$;x1RI@a$B|IZ07Z%L1X7o+UsPp=yiJmno3c%!vU`65%2gQ&m@y4H>w=`oy*82s5 zMj>mbLPL2w`X3n+UJ9JD^0h2Be6&wui{T?7WWZf`xD7lbWu+w9y=l#gw^)*K_Ie^+ z%n=Boh~43FZc8lt{l|}3@}%VkS^|?NIk<%A4c4b>;NAJNE@Lg`bq=04jCrfC$aO?+ z3D<#(o$(#SvvJBAzf#tX)0ii3-$>sf=C=_a8VJ2`FEY-ZpW7PiPrshQY)REkv<8~^ z=F_1nQ-nQ_*wMmwSc~ygs zw-7MSPxtWnVYFVf#^{X|XawLgAa^;GoX{{PxKeaNzm1~(ra)0Aj2pp|K zrh=L<j*FX^tjM3z@zG`VvuoIy zS@9ZCEG)VqfB3M6RdbI5tyE9Hes5;}HP?AG@SE9pNAYxX{ATl5CeoTFuF=f$N374oj>$Cp&0YHma;ht`+C;D;$vnc zBX1FXqHzepQt{rFZ_K@(g@tL`2oA%-HVEfH38L4N!5WzwjTI^tofXMkvzH~IUy`tx zgl-#yg)IgpgEwKBuenuj{r3f6J`R`;&5H(8UtZX#gz;g@{}etn#T8O{8+U&9+Xpf} z{3GE+?F@{jKaBT3g0}L&yP=k1DOxNHDzA8=a211n=xv3)t&qxdxzC}??q%gnbr0N5 zutIg~*EAWO0^yV{;miKwp*(i{4@q!5{$kibWc9)@*`Ql`j2+{z`AaMrO?qAZCJPsC z+CikFVd#kQsDTa%zErTQY~9wE#iX5g61Ip*LC)afZ29uKy1y zhBiD0?9z8K-(uo1;~hmjGO!s;yyK1@FC-I8eKnRkY|Wn2qnOpnc$Z*BURH<2$01`q zu*otb-R;d{vhlh^7N$W*7THs$K7pEO^45~>;%R64Fn0KMDeMpfUjazw>#gm&lyJfo zkyBD>SNN|Fuu4+;djyM>q%Jzq5|)~c=)qGCuVio7bXA`h`_*8<(N?(t21i+Z0b?8} zr8Rb)&b^bFP5b-w5ok7B>ucx;lcUSJf6iJm22vu~f7^wJfIBbV^ zd`JCd)-H+yUj6m2eqr1BxC9rrMF%(70dAftPoiw6Vp|&CG;|@`&SATi?d+_>!dM_Q zF+z4+XGS8wUJ*l{g<1aiz&lx&?@_sc{$3GG#c;V$khn9k=h6e;GAC`pheRqmY3)j9 z7QqW0%7B=7ota}ZnN$3Le=1kJ&vaeR9)cE4x&)2SilqR$Yg*4Am^kd0pePRQ2^8B> z#LFU3vP)s|==ze0P1v}{^&b(%V%%eQZxaupBzn8n+Iv5&RrF!$r)_c#Oh2{q;cMrZGle;j_^M@EEGJP{T>O!PZ`($C zDCgtE-l&J=Y)bahCt%JdL%>tk_nh!oe|P)O%yQZG5k6Y3=9v0$RFs?>?RP)Z>r2L( z)e>8m6Nm;wazI%KHfy$<8zY6X%&! zrQ9h&vQ#9KUYIa}snW@x65YZ*Re;4qwp@i+ zv!m`syRw*3>GB!DqETrD_nO$@okn|##FigjJ({^6UrSWseyFIRN#Y+?SLlV#3blwc zvfKHeGqJe)KmxHS#`lh#WpNKoD!3*eFYPYRgN(g>K3CZbr6H#sP%()EX%Un0vxdEX z{3IGOClUCY?|I{8=v4+$n|<16^`sc1-=QC-LuFb*Zp4WUszvX zyRQgZ^an$`gt)LUg&UjuUNL?s%Ps7Xh$3-?HVeypSlCVW1phpsR~jQSct|0#xeqQW z68FDh0~5z&j)mf0^%-_UIqOSF`>H5RBe)1#R zxXp4Ik@J;lAcrWUft`h?KRNx@$8N;;OAa#VsfQJrXd#lNVj4SMj1>hxV)#Tkwr$A7 z`}5NKK1OM#YYKl$w^jOvpqEwFmR8m>N{jVmtiX?-(kqD_ld0!&U5@b-%YQssWl&tB z?Y4u=B(0DXi{2EJIea9`pgLJDKXJqK0ORrn-zr>gqc>!nZOGet18Y6;XDxP4j!0;6 zPGr9y+%j(p7!=45IQ~ky=(paP*D!zO)+0o%s3c+sAr@3&Jmzd9TiWLv#x*ZtF-gy( z1dkzRN8<_wOlOeZkH4`%Dd6T67A|Jn_?cWlRECiy5Stn6@LgoU`MGKX zN5;%a)tNz=^Bqx+o<(yQLIl}>aS(X&-NbA~FPBeu`}4CayoT7NoxJm*`{EA?N}qw zPD!X}1XBM0oIXoYICIluix@*gKPU{%KtQmNAO`gHNy1iisdFiVZTO=Z*0^K}n3^Q) zuT?+!i+OPGN=9?ZB>%WP!V^yu2}slO8`#)QSvW3=i&BDaH%4<3~OeqVsz7=%e&U zjq5sd4U4Ep|3ugrQMZi|?eokKrP=t~L6x%$=LD|bG!9vp)APS&0pxKBE?QI(95X@? z2Se2y?y(FfzU0SJl=qE#bpjKPWhaPuQL53!SWK-#8s8*w3QJ7;c=Dg49hnn9@Fby% z*v^B^9Qu7J`i*JHZlkkGC9cUbekNQIBVkB@^>oSdWMN0y=geW!@vDRt(jj|`JaVlW5t?Z6jo$PlC8>jmK)dQ+x7Dq?!;eJT-4=o5d@kdxQ5y9_L?FZ zOWbb`wAss!V8(BRE8<(i#ryA51fIP5?J7*two724hM?(L++mU;>b>vwo6feb{C6T3 zw=RRo)raHb54F;E-Z?L;HU8bjID0>CUb&a)zC=$ z|3u!CCGbDbMxSI}`wJ3Sv<;vn5UjXJEV02z%!nt)J2x?}ebPTfGkWbUgtb5g94rN= zk4u;mi-LMK4%xi$+|VHMBE{Qte@RGexk|%+CU_#z;W5=?m9OWS{WUe#@E5ioIYT7k zp>3^Wouva(>CnknvM28>+IxZ}yaxYEu$b^_Q{f1C^d70$J%!dMj1yciYRDPpogS9R z=$)pK@p1c-j*Wl&>qr&9-i3$&nkY5N(*jN!K4nOz<|P z;=72{z45hg1aEA?lt{a(gs3YKamIDuHW&y%FbR18F6zqJQ6%W@2irW%6!Zy+D|{DS zcj&k-ci2>ZQph#AS}*n4ei5_Ro2wDIWUu`tQG9npPm{%`LoD~{%cK43vpVpt)d?8u zz!~DK3y}w8cyQh%?n;(rN+*X4jJrFe-uJt$;fqBcI`+M4_xIZ-)*$q&v@5li7>l4J zper8t;Cg;6ZtN^xT(ajubBU+K??aDAB4Aw ze7Mihb{OzR<2%PK_qk29?vqRbH?|R~P{0sn!4n}f&gMY&$Wzl8p0{tilBIxOl6WEo zjK(q%o|0;Sp_ZNnx$14M&u>^5VQ12=5si2zTj>_vic&?hLF;*s0;N26=bYgWG2BYE z6X04m}jsF>Y+wahXh>#1eTNPI7G&`7iZ9DLsjFKLSN6&3ZSM8Wq~ z?{_nE4HncT!h~xe0O5lZ0V*{1?xi#-7Obq5r1DG>Ya&KhJkg1 z@eymCtT=Yz9nsyab$_>90bBR8V*U&~@#?l$c*U0K#|(?_!)vJ73^&7IcH%Z5lqdOCU@7;}F-O4mgjfR9QTH!J8jjw))=fOFp)+)Sk&kyz3-Iz~GOuQSj0Cpw)0n~^)|7ySbBE6=1kVNc=}XoOVFcGsOGvPu zzfFPQ4`vTo!ZgPz35=S9il6abUB?)YTV9NhAWejc`yAQ-2IXVAu-$Nr7TrmqxQ6xvk(bDqz6W}fqz`Dn)Pf9ut^m+trdoX>gAbN2H*{YvRSe?Q*u z8OwQF*ZH<#Uv{rkAunKadDD9rJH4S!*~8Mc)*+`i=nLfeYyn4lFzoTz0ww7a1Fn!W zknY6avgW(oPOJVWZ(N&m;%37A;b8|Q1Hp`tb5h7k za6N7)cYW1ss+K8Nx<{FucDIz%uG6l;|32u2C)V~Mv_S6q7c4tpCbXNhj#>k!*Y0yT z1J)!_T^&9?+fV)W$5s()7wsmkfx~%a)QS;WuNmy~I-}Iq`R;H=cfu{Ta_cHfk5>q{ zoz_;XRp@d!oL+0tb(fQWyaJe=+b%N-xM#`1UcCwM>^Ft4TfUcoowN>Gz5INyH4)H` z?Xx>XLEnApmJxjkbi=kk7h9hCFHzZ9>!hV3FKr>%@{r31@J^fK3R{&Og?(yj^+Ur6 z);hHJ49kR93AVr1S4)XP=Gp8;POpQrpl*bxRl0{l-X8kzr02VB!NTYoWp3Ntwi~U{ zyyk`ZmXz0Mja)59t5sBNwHMmFUZ>kYa)k(b*({Y6`$rrdJ%I3LSQaj@?38%jw41d? z1p!~!Z=Dc!21=qV;dCooH#`tlWTE`3_j~sw=+)DkK4+QoI+5BzYp2ymF6+Qz5d*#D zBlMP5HtwLk!tTKYxqbb$|FyJOO^`jc?phO%ZIadR3%bOJ^=ULHi~UF`c`wCt1UJ+y9GlUQVHsKjqB$N`^U;l@`3D@`0m;7!k7zs~aD z8X~=~)>~^8aJp?FSFtn7WpSkQ!XPOj!xyNULJ_|U<6oP}#IM5}v;vA>>o_YQzZ|Y$ z$mX>hKEq609(0YYdpmu_!Q+*-SWe0{ZqfQ_t-?X4HQ)<@c1&s?`q(E@q4pn>Qrk)6FhH&gsX&&`F!HAXOD?C=(0qA=E#L^}g^ zIQCq?RbHeks%h;<8&dldneCcZTyFVFLf))()*29`%L_)r?X(p;4d-2vL&Nvi9`O0& zqX>9bedk)sytPE@U~Qn5Nx(tp1gjgAsaU}8L-7G^m2b0$d;$Ie$_eG#`o++#jhG(n zxsDb>Jy>7gO~2C`wA9>hoJfDqi4o4iROO)vEH*V9064Rev`o$*8(Dek~N9(1v@Ysr+)?%OCmM5$i9GhZS z(3J;9Kd*!@p-|5>ZA0tG(#(+mDHlLPUdQVTc!0}yiEZSFid+vHblYI~#i(2#SocK_ zCfD0H5ZNf#wGq|o^aovTpO=Fw+n42Rlc{(dZ3|kQU~=u>NRUFVZGOMIME76>dY0c5 z*dW)>oNGVBN-Q4IwfTdVx|;}ixHe47)qw-fpvw`qxvh2|m@&H$YS6r)=!%Ihx6N94 zNe^1C_FAr13#8C%b6E?V?r@@r)}_7MGGqMukpwMUxO&6N}M3sF^TYM%L657HOc86RbROseK zjV}KP<%}3a`HSJRComH=c?*37nkdE9bAYx3@;R>J+$i>B*Ps0~<~ujpN|@+7*CjVM z3Bbb}SHH*r_o;xjb#_|8*SDFguuN3}=ifQq!R*Do+lWH67Y)FX0Fg)@cG|8gcQ7Bn z`F4UuAHP0$%2xsU$|wEHGkyM<1Vw$`M5Pi;s6Ai}hDzLqqfs^DP?Q60gnm*es(G94 zZER;@-aYRV)fnd0u)AG;YeB$xE>!EFvi7~s|7*j}ZmSOn4bN^(o5SX@<~c*QEG|h3 zTxNR5C~+?TsixgX6^Jn5kc7i9Aqtnx;grJ!a0=dl#kaW`c@KT4kawdX56lclQ;rXs znA|9No6{!zhe>AcM+AqWUOQOeuzGwRIGJG&hcx@%zn{1~ifQJf5>Uh%9+y4fv*y`C z_QE*9r*a&#qBM4C@Xy`sh|k$UG>Ri$;P&N#l}Z3panW!8X-Ii?fI93XOgum-0h`AU z;g%VuvO8Q;Kem4NGtBcpDPhs`ZwV(n-~?}30k*pGtzM_ok+@SjcKJ`6S)e#;7m+Rk z#Qo=PZ~HMpBl)fDg-&~sHQxtG4ja0= zR<|!06m}u@cqwaev-5bB_X-*LU&}R+{D#prf__&KxHqr_mpnz74YuIui+8>`kj~o0 z{mY)S%-&7pch|aVjiS)NlmfvYV?MzgyXk|wtKCM}6{ZY+%#!m7VfWMeXe}^!3%K%- z?Lmw%tl<5-iiHcAF!|q|y>|k0_YTV?(A}#a-B^z^ROmC9dBrsidtI?n*hAZKKkX(`-`yNMgx18|FQ1$=E}%1GQVUbUS#uBD{bEPC$AN zAT$w|f^Vy-z})cE-5OKx2jw!T-mCiqK1}&>F!Oypab4M$xi7Vsh!$BqLIWG*^us^E zjB;Ql;?Gh%hFLdQ(wU0N_>53dMb&4hG5IDHQ&CZ4cJF$GQF=mR3Q95Msmqr&sv6gU1hI9139eDddL_T5HcrmIms2kgzcg)Q~`f zUYkEy=nGl>wooC5t{w5LD#_nU_q*&xZY2%$SA&}GGf6xz7ePs^7xF`h=^iq$$S&FH4_q(&k-%5wY-S^ffEI&!yURn>Wsgd45(CUb_x8{64bXu#$ z$liK59@%+ecTqfGQ?eiY&jaHa?t&wxxD#UuHpRW=m9%4w9lehdF0!MUWJjLQ7Xq7W zLoY2(5?_J$+U<&ubAHRk1MD>ZAmQ;eHb&K9_lLo|f#kJG8c4zBlJ~>VXGb&aCC7+# z#BS#F_(LVuuou>doOd@=t=#-bD%MspG23gu{s=oReU1~hNCr8aU=KV%bui?zgU{|T zsA_Zhs95hKhn>W4by2l1EI&!$JG2p6C*5nJD6J4mAmG{r(C>5j!A$twF1szLr#_2q zh7U32Q|^#2>ai_ectY9YrjX>cgX0Rmlpjn_$Ok2wr2N3Wb5?9+tm^)i!m3md8gN5` zC0={7bT=Ncegtzx&Prf(MQZ85XqXL_LD^9y_Bys@%}f?5%{xhCVyIMy$b?A5?KI$- zGIqe(qZ?Ut*z0S8#Nf5I+gD)n$qI!^HfYF*Xq@xX}IS~Wf0{lq`JF={xRtmfrR9oF{ zTd@rcr?qV!=zvINJL<0G9DimeW7l4}vS63T7Ayi?67=PVu!NK!4npaHYZGNBxY zb~}#>rkdDl&z^3HZm~{W+>`?EO zxWZi!xRjDh4tSbxQCAinP5w^dLkcCsu~d>2d#^dT>Q?3g=bRx}bb*`cIuv98(vo_V z8yDW3&xcpc-NJHp$0R=H>gu8$6BTPrsTxGW9CXX_6>XTOH~lOTEj&Fe>6!4ELPxi9+F``y`xj4Sf0eZl5uoXijR~A*(hqW7p{i!bX24Es9OJ)_jP$ zu-d|Tp+py1YQA!9gCQS{Cf!uc_vU++?y|Dv%X0J21Uy6=tYzDT*P>er;pDj?qzl;GZl@cpiW_QPVPgY< zDPInKyW;vto^ri=_EB*7D6-!i>P9ztCr(5lDfGc^LYo zz`rBn6UA+Hu{-<|+-+l@kNFDaKL7k6W6rY|Rpub9+gV_bppLW)Ky(5}z{ z;tm_rk?|#NUC*&KS&A(6A40_xS%$7N9ahxYgbEhe=Yhl_WUtX<=gf(}$bH9`-Nlq? zYN@e$tEtE;*u#+314y~M@!C!mw(yO-X-B8kEMl^!-U`{Vh zUm*Ml=oUqF=k_a=4e(eYhK{Vq`DCvD-mr{KV{uwm8CpriX@Y9e(2_Lr{6jsn?;s7h zY<9-OmSYlmkTyU|_xS!leL=f%KMZC{Y*tzN0#sskJZ@X^T={g$^k0}u(e@g`6!{%W znL|PsR0HIujICwn>>m8?*LG&mj!R%PXlecc^sQm`S>l?g^U9Gp_PQ)XOW)M8eFif_ zvE*7JU9<+P*pPwXVqz-w!$T`K*=R3I&3)8i30+5^BedaKTNM=iS@b2uhv?a&P$3jD z3Vm*a->E1z{TnhB%QU!f`m>#A!K(AmoU{zMo)*l|+GtgRP^2{_Ju5p$DU7{#*B5Cl z^lMO#kVNPwG>!CptWZdBfNH>eBZoKN(R-Hu@*Cs#ZxUbd8=+uBu|^0A%$#WGC{fBy zZ*O>V2y4?=ZXu$vO(RVOon$L^fon^3#zd*k>|z_nOJgoKv4t~;Q&(!j^ZG?FN&cIZ!5JLW6hQmrwI(_WXT7^gJ=)QLeb6WF8^ z5BbK(`6%z41JX3`b(Qc$f z5##Pze9T`COn#L`%=cF%OiZreVW7{C)i6p*fpwRCJ#zEkT}TuDr~mnLmaf$ZGF!{k zszFNDYCNq97Y$9U&ED9$f`x3SC71}=&|eI=98UB_ML6hzj^sj!W0OeFUg-5nE7r;Q zXmz3&I~g-Dpb}LKzYk5_|6w85zTUN&M6|z=7D9=rjl()&$ksbz*kdZZtBi(Rky-_tGtGY7^j-rL}pbwJNch!El7PMHy}sMu-)`=-blog4r`ihSxa} zdhdgwVz(z_;6U4m?rm=MI!8cDyU*JuD?K+OD!IXU|GiDy9&N;yWl}C89>P4n&Y(H4Ik{#;Y`m_cf>XwEz+%O>lt^4>Li7!!c?Mqw6k#rw=-Mh(hST1p! zHdJe6zJ$$P;N!AWDWI)vVPN%g`YA?4@ziJZ4g|>jkG@@KIkTc*|f6Ekh%e0}7vaCn#<^+pbkM;~U?1xN3 zq*fq!g}!aP&oGzIp0i4qVQIhj<$@yZCl=%iTsOaFJ7K=k!m2&bFi8rvP$a3T*-EAx z%@4BsEmx0jWV%H0qh3ow*XJxm;Vg-b7~5#e^^Dpr5>NOG@u-b;{giFy)P8q!sm3fP z?QBKVis@Z}ZVSSN1Q5Bok7lPRjAWG45X@G${_yPK~MI2nE^}k(iZcaP3J77cQ z-}0FAe_29{?0|?Wv6(LptdbpI4F}I0X!#J+e9JS5TA}%(UvHi0g4E_jzViwbFa;Zx zp^C1ut60*ZcNW27(xGNy2{>3dl4_7qB`Kn$Zc^?XbPLPnot9w2>gM}%vSIII=36y z7{4n~{O*14^RQaU0f~>bk_?-ypuDyO6T4T-9})Tecc=cBi(b?+|SU zs2i+qkCpX9CHf8iI&HeM$|TGEcb00p&0q+lpI3?ea9!lQF7u77ME^Vw6|6FmCtS@IjMY8mA#wl98u z3X`S1av32@rtqR5kCPO9!xdTLigLy9n&sK&JljF}oOH~VVb4vzD*zsKWL2}NCevju zWTf_C1=pgEw2UaY%4Og_nE%Tv(+@p&h_(8(??kJJR-Y)zF5SK`tel4sa=nFJOaW$;eVk>`<4AgSl2&bQ67)kOVRQsZUkv7uw)q)W1)R zTN#V(reAc&w$<6p2=(kt>xkx6*@fWG)@M_=e3>iux5AF0XSyWoW&%cCB7!Li4Be4V7>390 zY8Y8Gm{etQOr3Gd?%DNO=1(1v@WP*xELZ%hM7chH>Blt~*Gsw*;ezW9XFeS2$k?D& z43pr<5l*=#x<=i8AI)F_liE!Yn1(v0SP9zctZwKYH3YGWkx(>Wm+5!D#dy9=!XnR` zQPm#2b}x2YTJ>9mVs3Cz#GD(?pSC&HFsjFQSEx=AjpV{AlPAl$VG3Jg%FcSMfLyr; z!HNR114cVgC&az;NCD-_c}l^U4h(c{45$tm$9kE9j`Qtk$28ZNo(Id!H*|O z#cXmR+)ZJ{nIHJmV4lJKG(VO@%a>op{H=fFN_x>AR)U&F06X}>0kZp`=MqPJ6BfX2 z+Kvm%$zR@!z6728hVuAen$%-0fH}Ryb2Q)QmFs7v@Jn3h(ziE3i|btCL4$7|Dvb7j zL2zlzUYc{tQzc^}Q(!;Hh?ePvgmC=KO$6=ZrrxJphuL3)i{Bs2P$i4qa!T6+PObLVtTbLxcxr3 z`9&P0sK!-TY!zLUd@*xwbV_}58>`HolBSx6o$^JuK6# z(*B7W;uYruxE_A-%*pYrOq(@;;IT|w582N zs=-a51qDvfVNlzFNn~hD0#$*Ru))7Uxezv*3v9JWV$0GNk`Ur2`YkIywYSTswE{QI z7;o_nrf(UejnX<7IpJ!6e18X=jrRuOMl9HS8&n^x6ANKZ5qizBPN4UDCL;_P<4^Qw zK}YplX;ln5GSMEZu3q$iw9DBM!- zN!HpkW(d)atv!t(g()srl_kJ9I})GT%L`}R%v_4{LkSoCl-hP0JdZ8ZDk-rsY5r_~ znlg?*@vEWluru+d1QqvQ6+&y94K5A<5fWFQ#Zqoyv?8jhhM zQKI6CGdVtQzN-Kn8sm_@=@RE(=>8RRDoWi(O9-a|^%8Wy`~5hq>w&aXjOS^tIxbYL zwU61J7bUc?J;eZO(x_ufZSJZz=UMyApy5QVxCf4=iDO5f%!#!8;r)?`^LdFV+Gi9* z>-ez8pqNeRw7lWVVPZG`c17$OsHpL7c~oRsNl7+&tn?DQ7Vsu=9)|O2tTz zkQGHq=lic6s?SP#Z%9m0(t|p=?%e8JO%{FAro$~5 zYsZXL_e)T$`w2;qoVN2=K4U}aaVi_&`g6Gc0-R>N0m%n7<)0^)WFY z-0_9fa+k^cRCcVA@;$>FWhEL#HH>b@EifKRW$W8!c2sUX{pMniROU?lCGkX`u*_K} zjeBFujYk;n(*F>h0#{~r6TszWe(1Rm%Cm5Iu#Ipr9ERJqAe{%7e1LH2_XNjqLQ(N> z<$6>D)EGFcD`V^*5*Qg@*O&t#~yplqqR>6zCTFw-2rT?Kb3@^o{;(45S=U@_enJdD=-&$`HDI&G$gaXc#ijV|!E4qt6XhIF6zVPoUbGW!t4olLW4qj4nYrK&KW>9vEJG+}I@2}5F z$D`atHkOW?L>5R8oQu|HGs;Y2Jo!h0izwUXfT;}ZoC8e}GsJ4H8`=-4SeYHK=RHKS z@Lvq5aDtQa{twNl{TWl|L%oD90t#pl5|>NIiQN?*b2hBrGwCAJlYdA^)RRr~U{qf( z*n)+?`vx&%<8T=+8Hg{+r+$B|7b~8w@Da6GJk5$iW1b4qLV7cXOhkxkI+sLqw(l>z z(3lyeA%0p4jna+Cm@9UEah~*^|NQmL2>&k8gb@~Zc_#tazVr!xo#m1VL>!&I5IBZO zhY6QgJz0afN<9OFimpDz41He02~WcP9$s(q53DfRAw;-Xn1stRrE3Q%0rYl4 zK7SF+N8924D!a?*kSk?X^+L`)-={yyPW4f_2A=9B(KUkNstIh=N+OH1{N6w8Oyz$S zrY{gGU*EuF*>dc=@_#TVWnnRaihwU#H%Nlj<9b{=#wwlNClV$W^cz4YnR62O&}KI> zD-y@vde}PqUfm74w0Zy3`_xk z&-FgfJ0DAkRP21L>hcs=4Lt_RM`vd%^T&JcIkS!-U3-@i$xs$GL;C26pJp;`^{PZd zZB-ksZ~Rw|6s?^T_3u}{^{Imu$A(WP!mv121H=U`Y>lx1%E8G}!lOlw!C@xa*WXQG zDBAUj`6g4J$_KL;DY$6CWmXzoEkUs~*hD`mAcq1yN$i(~o%onp*xRQN!NS4{Y*~Zh z#z~PG?r}eR{I46C5dJA4Q3z|}YY;GtXvHTMIHKA2fAY$^(%IsO{oYD>#cq zvouL4qRRPxK3+cPUIIpku1PF#obfBA0wwtG$}fq^)=T5w%4L*alnaP3oEE^2nmn{I zu{8sQCB-h~d4Vx?2VBdH%e(gx298F68nm1h@Ek>+Njs9-T_KxOZ?} z4}{~fR4N1PP=11&1EL)Guyp>nEC9)$Mt~w;AOPd#tCbIDXJNwnLz9@AuXI1*2{n(& zYSFp{w$EY0-7e-Lr$!> z!rf|cp)RLdkvo`ZS4`siNv+B-x$8cY&_&a_Ku;95ee}pR57m2~mH19cSS<0iHX|Ay zf`TJsaBmg)%TZ2D{+Vf?ur$Ezhlp}9b3UO+zn(4jb~c3<+-USIxA`{~`ew``MA0>k z;~n73US-Ge{$*?Tvu)fb!9@;;z^Q3Ev4XHtL_2gzYs_?J$!sD$;&kaN7Wi9>*sqom?0qp2)mw18sTCWqW&_@9rJ@eEdQ~U z>?p32_@Z?e@kMViMC}Dmk0Bdi4qNKuFLtozu{-Ay+1NbR3Sr?1O|)4UUW3JUL(>HI zYQ)-D#m}&XCyBY6=#iiJ?jJQ+Zm0C4w2p|lm317k$fiucc7Dbt)(Z5ZM8#I1W(ukv z2I9?Iv4wJ_o=8(6QMx}VckV@|M{aqH=oaZjWd)34`DQAjt{ld>C0REyJ@SJ@L_N}2 zK@?Z1>30-!VDs%<^uX#-%+kz%oXAB>(?|iwAmm>jkKiY1%&HBu7<9)cOrZ51^;mwG zZsSW2U24q|21g_`CJa;|6sZCV=HtG-Xu8IeFIhfhYfPO-ve0l zsPHZm#}@Nx4bhpVtRb!grb*P;Qk1g4c)5<^No;qYEnq9`j#&ZIL39!3)hL9&*EIW; zHB?PlK;M83Rjrk8fQea14so@RsGPpr{)zD{QGMf+w2BzOR#t&NW$*_IL~A9qk0rW5 zX76!frY<+g1w>*57m!K{8i<_v`5ahr5!oF-2j*?B$<&zjDWW@~#_kfDFCM6=Y!2P@ z*9w+yxppC;igXJS>+<3Yg5zDHl{e<4GE?M3+sA#u0`rv;S@i58G8XW4V-4w?f+S+o zJ&l(2VJGU=MMO28D4~4y8U{4ri5vy`o-z0ODouqjF{d{vN7e7Fo&ULyMUfZe8W=^U zApLY)9nfExcr$i-E#H&OTEDQEXcvu_0#|qF>^47SCE)>zcrl8ey zh>6PoBx*!dP&J1A_7Zs8(0Fx`!UqG7>;1F%Gv=Dke3qz1*R+AyN_ayA-!O}J)4e(H z?&`m7WCk_oIRX}S9yHJ}G#(OpBZsJ$z|BX-8*>#OLy3M4dq1|Y9K<0BE$ZRWJ_+qF zz}>9;EJNRdIk~47UALRb+LWb4F3MU%h6!^ucDS^aZ=I5ZfmUE!wk;}a<%+}GS;Dm8 zGQ!4$>5cjo#EB23tZ7xv%EFT!5>CWB@b0t^%D=JW1g6J1FVj=Zazwu6L?7meYQru3 zCWvZus`!A-I@W5*@KT;v@M=MHDsm$+IbFppv3&hsFYL51Yj;NCiTn`YLBlo9f|=kd zWVTbn{~ZSw9b?Q9yrq{Yu`QimcA-KmW+y zvFs*=J6)9SJG) z9ad4q+z%w!lDI6sW7Arh>^$055}`u(BPyy8xbFwH9+Is%Ou!GM`B{HCG;b{{5?uQ- zVPlb?E@H!7vA%FYAsDViK}+r2u$pmpg#<;;idS~P1QkGa-;jndr*mP|_uDbZ0jreF zs*kT(fa{3lCj4MjolgEnOh|u}kir+sv%$TK;K+zUuz^`t90}zNP?QKyAmv|ECX*SG zd9M($Xh>2NVsT2r_>hI7J>jF^Lhq>dyrI-4Co?1+|4YEakmy;XJReM+2v)@Ja8L}1 z5^to{xp;yJ*-;6NLe^M?hVpj!Uos}V6jqiL@a*<$!e2pMn}9&Q8ANLeXC zc9-dt?PW>EX0H*vm?IEE5xK+T>Xul(a`J4JJlQ3ez~o6fE+Kk@^{EPHb#7CAaujM$T)X?b!+Up zTyrV2B|GIBXyWTkgsMy)_B>)o3*TWa(uZCuD)U+I?6fco>R&@7qeZT(!-0DZZ|)io zXk5GzpB@b|yc^$$#S=Fu#@q`S7q9i#<(*73ZIpPTY6&dGP%EBVQuGH;z~sHVr6zOg z^4=s8g;R&Sgd=DL1xefiJ3P4P8g@voeTy*hkl_2g;>pnnE;O3JBJjRzIAOp1+b+Y~ zGVQcdE+L8^QD}Q807X~q2;uv$$Sq#fqx8~tKG83QY19F0iF#46i9+id6Jl~0(&XUr z)!utk`VM9D)8{2H&QIsKyf9iXT4VIa3N!+68IW@Vl$_8o#=Asaak1Jc3ifPv=kY9R zd1f7bf*!TN^JIF19v(ZN&pfZ2#czduxze?>S(dTy+k`8!jF8gBkW=(cMH){MHoK>{ zhVq$JI4{A`Dl||~gr@sfdXo zf?HI!`vk`0&Kp!7Qv$&rcG%EK@*9e>=D4@L{?uCLDx8$K=qfac0?G?7a9rHLWJUgz zj*o>cf8_L>I0uu^PufafC0?4QzF|`Hvk-=|8itZ-3r0=^dguU2CmDkAxvDsF251 z=$<(PFEin)_bvgUaMeV*;zh1XfdIye@&4hcyjE}V;WE~ovqeIR$#sPEz)*~FTua#; zg?&Z)-zi~6()k|ICmM$kEE(_ZvqRm_v9PfGW`e`8upz?PP=e_7WUxktMq`CaMQ1fU z{_6ymgnmuJVw$^A1Qxa!lnma4WxVEAx%J=UgZVgMIy5gvnVM>Sav|fx@GS}-YTydV zyp21--EK1D!|xJK)Xu)+pf3W$=Ff+k-`o!@a2PK zzTVod%OEFQ5jiE5w))RA_p(ZoZwJ9*C8>o@w1lN*BYN;Ai-!Bc5Nch#}9y3>>`peb(iB-)^E?f3FCpVz^u=NZgsob_F)Gz$WJt zLP964abZawywITxh>6#kIX06y#SaKvsP_ZYbw?z$XwoHUd{!(8&`A$1-NnRV=BJ7{ zG$BxIOA#-NK*=tN$)kOl-EC~#BV!MNi*b(E69WF{YpDd+@b$%$dR*NNm-zW+W$3L0tTi zj&IwBa+LG&^!DoISLO|FZb3`2eVw-KEg-KRR>caw&H@Tqy5qA zDwbiaIU=zIYb17X!qvfN{P6bEEFN31pGX%mF9iwMZj5OXW5cQWj{a5p{;G`Zn-3^E z+8_!TKhLB(r4*L7aVw3&O>(w2X4!_XCA{cSQ{V$(Z=Af7sUqorzU~&LN~eELbPM-X z0TvJ0aus6Ejy)4T?ZAvm^MeG7Mx{R7Yhs0W8m&nZJ8e>j<;?xqB~gX@p`wB&iG5gI zp%*$U)FRHY?cEwPvG5;?BNqAi-jTg1=7C8C*Wlx|s`unqjJ*xNP}vKmA-fe&F^L0d z5rgqFhaEEH{go`1d|$$1ESW;L(&!Q#>)ypp8iqU@dDDvbONG*gpqSyh8<$rO{lzv8 zDK=HHCZ`{L{C;*NZ}r0ji&yf-z{YC3+7WG6NyJLxv0y<>G#L7&xVR3>PQEP{!R%zs zfD`2`_8t@^7evWMCAN01aU5e7)^UUgM+;k1@3oCh@G9iG-L_&|8(%TL3Dl--UX`v+V=~7=aj&%9%`39Ll=8<3QS_w*ooJFEloI~8Xi0~>Jos1c}b{(ZJ5a)1RDv^JDkbH(gdR=n*FrnW!t0rD7U8R*V$|->=Vb zIkp{@iJr?B>|91^rpX;I*0TI6p%;|ao+z!Q6z1#6Sb-lqrB@I+CS$MWx}4}!EdTOk zRg~hEPi+2+@3d$Tlk`<*oQ7*3;ed$BS<<2J+E;rO0GOje_ZN7m6rlbyG z=j4Qh7Ux9v`@t>qhJZnV41nXWq>EfF${k>a|KYERT2V>F4ni!b!g$P;NH(?41@F)L zFN;ZX{zvc_lQfH7p@8WO()+PD7AOVWtlX@QIm%4l%ok5~VcU36E+8ty$P$RnqU-Qo zWWd>(Y6Hi*Uu%_Srf>2oq8vSoIxvI?vH{~D@Z`IJ*@|ACFwyDFZd2@zie0rEIzPRT z4$90?dk0vWeM7KwN^7%9YYl>6PjN60uL{u_!bu(*IVrbY)kzUIz6CR4mq{NVREr74 zSwHhZzg?10(Fmmc|2ci8qR{um)@6*Lq0o6rNgkt?Kw z92M|P?btn|-yM}G_WryEzcN;|{!w8?14*)3`Ob9Xnhz-4!*CBUT-4=o5d@kdxQ5y5 zbQ>ZXQ`{CeH*Ld?;IyBJUJ>6CF5Z8iByi=a$CfckYj&Q1g&KmUXK{x~lBgBRtoO04 z+b>aZ>r#kZeKTzXV0BKpI>`_&(B0LN?QZMhOTxPfJmaZu*@xu zhcHGo`bA-cBpCzIWKe7T_qKF)G(M21NNkE;bP+NEPdP)Q0+(M?aoFc5^EHEYT~K%c z2`m`#(nYNh^RNaDVG8?1-ox7&4=P+#cmQd)2<$|c;|jOm@my@_cedTe&ctgH71d57 zB2;`ooQ!xk{3xNb5-;_+{HLGUy4x-hwP@?=A+_S#H>jM(Tu^7qf|DEXYsn7UB?&4H z8azG^*Duqh|M9#hQ{V|N)Evyb_Qk&vwW4hRC4peYMPl&{Mtnv*mT}-~=CybIjiAwM zuPdwtDqw#hIDK5glvothbM%lkyM65WZsbLZx97f}P@}5^XL+Jy!X8p4l&0 zp87Ep*xA1miFgiDb*w9NKq?(N*-G|g^6nnnSi&p)4}uj9p~bduaXbc=5A zzcP+x^_yQNK#|oK{cth#DI3HO>32WfvyGwcWN2cNEOE4F4-{Tt$r1NoL>p#sYUxi6 z#(A4k@m)mfKKiw9FBxrTN~H4Ngs3YKamIDu*6;H{FbR18F6zqJQ6wn8N2Nk0m}?}i z@LhD>q2oH7K|}dTA=lt)^&fC?BU2+I{~>b8UV96|`0j?DCW}pnnC{c)TJ@}~4qUp_ zxEE~0zU%b}EtP7C`WO#7iAnr<*WlWmQ0mj`KQ1AO4Hb46>7ajX{JJ7jVUNU8*C3qcC+(v7KO>l;DsF>Yswans~d*AEKNYuWbaL`E91sr_e3NLAj z@f8*IrbNMMm&cuAuEDz!P`CyH5I#5_ph9E&c1n}t;&UH%VjX6VazrC`n6(C{h`%oo zc9-ClVPM^0e8gNQD~_G_&2N8at^4IIv;wy7w~qKT@WiXrTI?2ErXMpXHX97G(jJ)A zA+ueMdM19Ued&9)&Kh-{jJWGSH`+u{fB)iB^6Q@`-F$LedymnDqoW{ge&aLuKFDQ97J_a_7FEG#WgG~rpOt=px1$mX~W z9!ZUen!h2U)U&Kg<9GTKhYvDM)9VI87On7@_lB5Ve_NYVYOca}yxxBVyBqU6iHUb( z=E6=zngkR%iJh0A|@-swDxD&|%o@-eDtfW}L74kiU!_)xQJSkHP+ zMFJJC70Q>cJt1RIe2OqCPFRD9j`K|6H>ga&!nwm|75rC&Yp-sp$Ozsh zaXG(w=j@VMR}d<1DYOx&l-{x>Ke(}nF(wMIQv> z@Rl0E!~BE~idBX#$oR{&d#m#hHtugvkX-LHcF^UwYL zmfC7QvbsXyurGSnzoIbY@c6T@D|7oJUGj!yYi%QLe>e~-3^+or>~PfQbA-yXCx<){ zcPQJ9zhy7-c-?mWPj3CW|M%Ar?*31E9JMXT^HoLM*7;hlR>vRp*}aZ(cPQL0;+_(* z6I`#$t6W;+;u;rKso3kH{0`Sw(JE^f<9}b(cJt?b2rX3b+S9g??Fp@`)>+GN`<(%o zJ7iB0_3)Ir?tbdG*D4^?tF^9LhRc0^)GK4OUOYJ9cN^3`zd65QPr|LRYSVMJ%@VhR z)?TYy;&Hj$etXz6)y+R%0n97tkYfPe+HKeK*Ad{c?@QL${2hqUE?Ot8VNsFao(yQ` zj(MF7&`rz#DC$d~8{hl;5nJ7k1lmpOqGcj49TC{_h$jH>ZinkUTa{$H=yth0{$dky)fn<6`FR~X zDCoC;e9_`TgnqSi_X=CDu7uu8>!CG`N47`GgVur^3Ir8yw7%!JZ-)_hvtyU8x1EsS zeYNYfHX*my5%H9{4K9l#T^I#PiI~1X)f9^OUH;YRT1@bti7SJN8TYyGq~(XiVd3PeCVrnC=z9Nw_Pst2Ag-qxEI zxpqwU65H=`k3u={zPVnW>+q^4eCyi*J$0e41)Cd!AN-Bjxx9Dyeo2O`u+(WT5KFc z!1L1FuiDN?;KABJEr)=^?n!npC{wXOFo5C%+N#Lmj08gb1C$fWwe>UKj?ZFxaCHy* z0@Qbvq1X7nQ(VejlTA%oFw$ zg3&K5=SwKmGfjJ<<+~R%Lq7Uiq8<%-eSaY211?V$+sF|WxgIuX_+a=&L$1Ge1zu%x z-MBZQqFmQQRJ%JE_ILw+4ytTlt~)D7@i=~6y6Y<@*E=MnkZVUU=q=Yh7=fM}v;;QE z^~_g3z1mJJzM*@|*|sUy5v?P%VOoI>9CC*}uBgLncLuO;$Q)C#n^AccO1$6oCAMw3O{H05yzWAs96)8)1sw%hLTyX=8zBp8j@C%ZkxC1zu2xlpD4g_kmIk#{{UB(#Mu>Wz3nsL;(b zj4uBO<&2m_`MLL74q+zhD+!J!N^$jEpzVNsj;lB~ialvOZr6C`JI@|K1f%cVfZW^^ z09&+fdm97H8>j%*)7fbUU*BP^!g5pv+$;C}rOaL&kbr0}GQg1lkw_i(@+zmgF&}@< zAfgj}{50^CF95W{e;=>F^m*Q30!4k^RHYJ3s54{_N6NjXqfsN~P*eeJgnm-Q(7aBM z=Mfg>9h3`Tm{;5B^#tw3A@jLVt%J(iCGL?c*x8*sgviCSTgTyY`0R!5h$ELvk^+~L z-Ps`S@LMC#_8?Ut!i4rW5FCaH23#J8Qw|frDR`TEKmHIS?})@h@@fh4z|3$o<@lh9 zDKN-u`t9uvm}K5Jl*mI-uNN+M*?j>YoXn_?Lt1_BmLEG_%rtYCVFV~*4WGvu3fK!B z5obw);8Qt{xdx3JU2_hwBYsRmiX&d^4HSZvN&-`H(HAVh&6&E-fFogOR`FMuq$~&Jwq?)Ls;TB!>gtUAs3B z4hy>wf4r16xY_yj=P%bY^81gZHIV#9#v0+Erxe^9Sb|HQBFrXR@OE*lWdrG~{l0tI z6Sm(ZbWg3j*4TgsrW6bQnDYtN*xSx-{9rg?UvbOe`)rSnBC`8weY92>yoEf4$o4Qs z7z)n8r4+`Yb|X$f@q(u|Guxg#Y3lbKgs)2QDAd!)Fgo$Gw~F%$M* zB`^wmBLmnfccq-m_J*|By8bv|6tU#{jSt>!dvpxZJ5ak`%XWgNE5Zv%;Dn^-074UU zDfqTp3e3}Mw?4+yd!Mni46665f}sGW{5Y8PJ_e4AS;Ks(UnH8y;t?9yAa@Y{0cMl~ zD-nOL;xXLN=fMYk_req{-+Tg5jU$UK(r~|bAS{69{;EsXgJP|y85aUP)vtJ>RRrNc3 zCN?X3#`R!YxxL{Hw9123AGyP}Y68)Iop!C(0{ndp*g$g!?7$nR8`Mam+>?UOmB{J` zYFx&Y$k3YzUnmibmxJXHeuI@E-kTzEYXLdEa$OHC(RF^S`H z6qYZ`n!b~%rB@}QP)m6HLSSayz5qJ#ph)0}?x9J*f2v#Kh8D?8|MmU@*m(65(%v71Sv%|VNxBsPo$Aw=}~?13`KuQ*I% zXia)}QSUdHx_e2YqPoj8q1v5ArnHA599;N%e71WT%X*A0ByvSc-3;uo#^pj+Qe|{M zhFi&bK3sEIL7|u9uKc;Xa|j(4Z{Hyg*j7v2-dZoMnVH^D*zStAx7K`a_|5~f8QC|w z5|LdP^_C_Awj}%V@8<7jxaX{KC&v?Pi929jwGS9OR=9~?WJhz!j>1470yfuyURr`A zz5?%eIu#$M(06kcb{dBi5j>v8Ca5}`!6C@;)2b-_NlBP_di4eZfe% zJ?e)wV&~lpRVyz)l8UueOw9HDUTnjT%d%pkTO@;AZm*3We5oLqo=5;nG%5Lk{d#rHV60l|QCO7;LIZ9{xZLkdmF}J| zopv!-q~AmWMpvY+4vdD`WEqqlRbsD8bH;wdLZx3N8iq>siA;z@ylxYoC1ZQ`KCqBQ zhbu~nK#UISc>~23pR7=*L|)p*n<_90hjO!eNswQh{N)yLPpfDWJoiVm}0nP?>Xi-AVfd|CB6S zI#D9eI8kgH=_h2nHUSKvQIm#rv%jXD}f-VUMiXvD-DvE}o^uV==vJ>3w0x0X^AM`euNK}Li6J^FR8|^%NH8b3M zB%E;30}@WS27#g?Z@`>Rv?jG^e#g(5v71pwq@uBFVPF%ky$E71thR7oDA7f(ny-AX zdHAcbq?>96UVht_HJRXtXoIvIlmk5wG8V!N4*EDLe#nYTD$Zrup)04*N+P*M?!S&4 zW0)(i#9oqTN>*4itJ3h7hcS!uhFn8foFG^Pq#WHzldLFv%k|vjPi?<~#pcE3L^j6e z4J2gH0ZG4PG24|tdlgegHK!6LDx-!jN5p|5g5FD9V}g>)hOQ!a2<(v|h_B>rsWFar zvq8m+=GnfJz&B`vwLFLLT69YxoIEdtbRmb=>-K_G@j}fjYHlE~s4Gz=K7m!VCOv=p=f_!lNVi*Q32YB(>2e1lUS)L+pm96WZ??JL7N(~EdBg(PVP}>LD zU~qkkoyjtLLSAuQW?_2YPIzcxGGcfZE}-I4Dq+m05AJ`RoyOA=RGdZw6w0R7;RHY7 zn{NN49Vk%{vw+`*J-@7c77$SOFakktc7 zxx0zlP8GKBjr{sx?o^htt@j_o7AadZwvo{?@-Dd61t!oAU9=hEwg5K zzdHsUWd<$t9s)*#)*={!zBSA~OI!HCV-l3o{fY;=ADvC}2h8)E* z?QfU?3jfFpbYpZcDE!a-W(yE7{NNY)YZeG4p7@IkFm5+sf8zhnl{e-5Go{t?0 z2`*3#m~Z6p);s#l__=)N^0%tj6ssxn288D1C~2ubPjooN?8LbHiBz-` z4ZI<}l?FaOu8~ARbwYo#+A*Kgz2-6&r&V}>P%%!+0MyB0Fca9MlL)z1?0j72o)U!8 zC-hILp8ZX8UOCA+Emlf+(Nq*8I_dOb-#gmg$ha`@L4^xxF~D$28N}Rs&h%i)`IJOO z<=lvKAtmIu{yw^f#RQ8M61^fZ#E=s(!BSCl`K{MAOmSZG5a9~NiMWoC54t%L$JNR) zLpf7U#Ij@6zhjlaBN7-Zfej7d1k@(>624kFXcn^@>lYEJXgAUg#Dsg681sr{4c=oB z^WujI6C-9k4D|W&8b(PeuEi$kw~!*kdWY1;Oci*dZSED3LBg7XvX7(=Ac& z>~{Kn#`8K$6`ohe4i&DoTf~%C+If@IMi{VS*W(A7*zJ}}h(rx8Vc|$9ROf+SHz42n z%$ds|r$0ti3rI*v2viZYaPk#M?mGY7?(c7y(rx-UL5e9T2^liC)LQ}`TU~s3IUS94 zAH1Dy+a>`QRn#7=s8x^83`S$TEp6vTVT4!_jJ_j#D$Jg_+WFm+q4z!b`AmPTw~ zY2q>>83&e{nO1O?f;Ngz-C2so?1c}Ovef8R%LyG*qisy+)UhZQiK`b`u5s#6y$0-1 zzb@B6McUf5h8HFzpmiU=Bl#t&-CjQTCX()bYy5B6+)vSG4A+LneTK4x!&@BSvQsxE z^;cq#A&enWs?Rv@4QF)whUM_9K258MR@Nf$&LUC2NmD6?#?<{cA=CQ-OH8~h;Y4Br z!g*0q1?_AcDNeH`>&L#Jjb)TNS16R$7F@=Tu_WO(e!8rTB@3#oBsh^Q07=LGJrNQl z(SpEpVXl4?|6=~zixOJ|*CtyCeUPS5t+dwF*eYxf)V3{j2kep!g@gX{nx>iH9c z%i<3?Z>*R3&kN46Oa}iamq8gUz-2^?ht(a*l8JQ9d6!dmmb2@~)|_ux)br#j`X~|g zh?+4dAD<@-!9ua6n6ZHGKUSO&R^aNO(a% z!YhhX20VUUwbR@g6*NHzF`N7o+ z`&$~E(P3dFnSJNtVzHOq#O zr0)=EKd~TJ;JW#>+6n8G=4X9#lu1&HHMEkjz*Z|+ZZtp04OjoLqz9QUQT+H=qU&=O zqHvbNMzlY&?;N8xvi9Gojd%T&Z8lL`UTXl$N!Ndg(8ctw;DsX$7ZO0^Lh~+=a|lJ9 zk!UDsgII0RYu(w_ejpdYt!-o+E^J+QM*Pr)%bA@Xv*A|}F@aLZ`apU^GzS}&yhWGF z^}7=D>o-|F-sWFNpOd3yYt@T_xg9vNwUpfEUR9N4iE6$~P$COgSY(E>2eziM$34-#OxDJ|LNp3l6H^g5scYIPg;K7hCOrPcU(9o< z^eW+@=aPYeJNNZAEwTe5s>Ei#1h7hWfHfQp zxV>l<(|m*06SYF~8DDRo?19wgWWMtX6EFp9$WZR)O@FbZ!>2HO8N9v4AC!56d+~64|)JLNW1>ZmkU@#k@~SUuHDE4wb`4r2nuSeSiw+q z7{p?zq(|o-v=!XVeD>`U9)0#)`3)DejPeyve*IDdCQISBR9Uiw7X^8oq~IH_$P!nS z<+=ac#XhIu+q90DIKpfh_S_VCLf}!yR<)XHvRo#+?p@tk!Sy}4j3~IuW#B%T|H~?m zO+VR~wfanahdxHM`WRG;F6c5(D1z-9v9C%{nzsC)Q}3^+&OCvt@6uAD=>V6K_yRU8 zmW(Zx!VcwMmi7*lgqP(?LaT`JPkpLVzt9E`qyBwj+{#>RxBQ|l->(WVBQ$mseGwX= zG}(m^%+qI6xO`bF_FvLkY-Wo6oP=d9ZW30e){DH+uxYNwdX0*UX6#}i$%Eidk^!3u7dLsjcnc9OxbAWn!J&?g4cf&p3631$lxwnUxLU7aaF03+nvYZ>1uzx2gn(Ko=Y6@OW1* zQlGsT=Jb-!(bR18R7X37U*bBKvX6*#ah*#dXz=b<%9kl$kMC+okSm|SKEWzAAp-*#6dDNuEJuw=%VC{S#!hw z>o@7FGMl}NRuh$3bu|$;UhqPXzt(yy)5Dj1LJJ8!EYqyg{)rmmdFKPT9{$Cbv6Gc) zH%oXd(>6qSYQrlEyONfbn)h};#;RVWpAylc)mU%2!Pa8^v_ead12Ujx9&?MgHUF>A zt~Q17zs6?@|J(9Tq;Lsbs)=%sW0etiv1tfdX`aHmDgLq%z3)#i;L|Rz%7svSwum)m zQ^ExB;8=}EYpvA>*&|y&9fk|X&OafJb zm$1RVLAek%nhR{rSYpf89+D8^$M}}d!>dY)XsuA!ITLM_KBu+DX=Ai*rEa(ypeWc0 zXXE{0xDgBX-T~DI`{WXsQ-oe~tP|+{o~Z~!WLLK?Ea+GzSH++s2ko)y8pb3T^dI=_ z&`fXg;u33=+)rm0FhRay4}Duqko9R1Bt7X|!qUv)O;~Hs9}*Q?dm2LuQ(UkrOMnS> zBr&zOuWh}NxfD}h>*DYES_=`qZOIu4$&Rgww}rgw!g@Q^nxwAD2BM=OisXGKtQE#lrpcEyAfly#(Fw!5|Ln`XDV8=XqMIj;$-3RARQL^A`jyY)=`0S~TidQaf~2 z>6@&5=AeWW_rTFKas23$HIW5Z%!x8_p7|vaDcWZgMEk_3&!m_w>3pc`e-<+hi;UeCHjHc32ezNv#(mY@#xmltv1;+^YIxUQ}@HhwYZQT9K@yOn}UK)1F}ZB z3-OP_?ue>b_S7pcVbkIDzb5i`YLT)T}#_|Zh(Q(Gz4D%I+ zDd0q_K?-!vePU@%R?-`HnCKJ*NT`$R&aKYXRMDrp;hu65PH{iB?TE@+L9y;9q(Jgh zD_cCy*&spnX(`a@c=MyUp2ylOaaU^JEgMSMF(30C5vq@g1>lY^q?Wr()~C|;W%dMy z*W`NzuMXXgTWmg*%GS5f>8#v(dgEmk^O!TSL*j`(VVSc|88`an>1Fx;*M={&gP1B6SzCpeB1ii(FT*9{Hue#!mCjIrA#Fm6;` zy+;`42h2$@1&+r=zU z>z@ctlonz90(vU&o8vJR3IWsj;u)VlR+VYcof1;quYo?W-o0Xyb;aT;XkWc}b~xLZ zvSZ4|fEfqwT)p$~T}^RNTISwf4+ zA~%e5h~XTI za8Pk6il!Ys!A#WnQ$(TeQs6~9Mq6EyY^?R%^H(t!{Gwk8RJh;~PpLkV1y&D--IGM^ zz5ccRtTF6aiHdz~En}!CQ(O1|e~|bb{Dj;W1~!9e3SxljL_GTbj!+;mJ=h&w~E$=?-ln$EKI78Uoae3aqN_)DQT z15&0iz6^a{!U<2o{XRD8g}tmW+3Ih?#lj?9mML93NC}{~6A1)MVLsXk_g6VR=7gM5 zR#h+LtnklU5q7F~$~Ev*H#OD>i>oHEQ7eTk&Tj!vF;n@I&(aqNm9KALs%&YJzpE>A zQX2k`Kt)ty)D2Q#_4uB>>at4bCW(m!{S4?Nb58*u+UiDTMdH|751ZC*f1MSrJ^v7? zVw_RJT3?K&C^v3EYM+_vXW5OG=LlIJEds;jgbtCgSqv-z=k$IdjdebLCIPYYv4+Q2 zY&Z27C?B1dr_3K0EZES1Ax*2`*WbO0A(_geR!DDsGq*9*R;?=%5^Ae@XnhmEa-?YO ze8ayl8hYo~tT^_Ggu~)kZ4ei@ur(#Y%%2l?W6|gH8350&*zOQ^elz+VX|W!fun;!omt{S%cxmNs$@uai`xjwU`NE z=wc!lg|HsJ1_85(c6?%iBU*j`*=y{xSm)dGf-Gk=7|`F_QUVrv1Gvi$452lwvUd@WUm>{B@IgMdcV{Io?j2m$1L1f)mC67+l%L?{0E5ua z_qFcI0+2mbh)|I)5P*sD)yjvnv!FeAZUj^FciIR~sCi6Qi`Ff$eNK~RQa(C2uRsYw zTDfkT$kcq6MwqDPGokz#0B;IQFR1V0D|d=6P;*9Yk1n~3d767AEP9$v6vwrM?UiTF=I=lFKPIwSmlL$e90};c zKm=Skcz>rf0XLMLW0|8VLu|Df@NN~ji2yDNx!urvCz=?=l`EVZiULJzi0WhK5zRf%XU@wK%g8XkgzBV%%J75U3oPE7r^ zv!7sTfV$Oa2{Cg%snqH{CWRN=XgqtYOJf%LzAX_&*EEiIfG>N29moHj7kZd&qr1AY zjV%eBnx>O02rEUjkv9kXGo5)^4MLAOUHS?p-YUG{;n?6xsYgti(iQP%Q$ze4|>yeORi%_aY!xt5Yf7e_@Xx$qV{69 z&y)?YhMin_`Fhqo_M3#o=CL*i3r}dG&BE{+EVdV#Ca_l{-o`3^h9f#f%-tAAe%PdM zr?QB9acv@BMBK_cu6Sfirk|d;_#A5my1EXbVk=N{1yv6N@#d}gLb*~;q^S^t?m698 zYD|wDkt>LFqOt-;v3xTXQCAM*tPRVTGCeY{F43&N>t%2p{6%?3F$cEZ&i@=;`W~}1 z?dlP@FuMx57=--G;}QHMt@&=t6b8M=3R>?`kLQQwHs1Q!kh3gdFuguejR^x)2t}%Z zg877RFIui~L;j?xY>mbZlr>r_oT2Vx(g&$rulwnOzjrWkyrChjAv)8PHNIBt zic&sU7Gj0V;2hl~C7orfZbrp|d4OL6i=o_%1s;%-3Ffj|sA+8pZ zmD3fUPUy(unxb@CMGRmot3aPJ_yfhFwG!ILl3k$Zod@_+Zyhpd0g)KN1*DRK1|oM+ z5eHUWM0Ur|m|MgDVQQ>$Cc(zk*i@nU5`kLE=G5<=?!wY7>m;g3w;-_|KfWM1(IZ-U z<6bJWM84|tKBrhW+$q8W5UK$E0nKZ!+-`n zk)uH0GwxnrrKu1m=JY0IL;a2!``%M5ihNbBfl*{S(oe_L1^tD|H)HoZ;K*dw`lcpC zyJ)-=xVl5}8IgE@`)9Jsk~C4gLv3!|bA zOqX}8sPWK6YH{DwU|5UfrCT37iNMKRtK?4m#;}MZJ za*28g+|#*Y(N9_RH7ySHaKqIPWv5OYNJ;1+%hL^V29e87R%oW70W9Z>P= zLUbx}BXK!h#VoOW{dYeu-p7K2nQe(aksks)Xt>5%FbiCT%nnNUe`L?GF>G71vj`Tq zrB0azniZ8FAKt#5X_UPZ4mC>SkpA%c`Mf(7S)JF?Vcj*+eMbq)^j`m?mko zf@=KGFB@38xm7!Y68$+k6b^UCXWXqI>&)qM{0c`+i{SA=!$<1pGi+pY=D&(%xl7g1lV9#v(xj#D=?K1JU9VFkH!k zo@u}JUdGwO5)?TrUfBT?R1DR9QyRjW&cSPCTR_k#WKC3P zC~rsqC1b)%fq&PYRhd1CcJ~2!v3? z?(n#Ym^{hGB}8wqK2-zn&YyP~YcW5w;ns@GTiqqs5xFH?2QGHT zcM#9VDQo-@nGX+Vp8S+f^c`Z781bQj&CdA7RHax>L)Ad~mPgE^|r5I`@QcH^d;0c%t4Hk4_PTeU9DV#doB^*I3E>7VN z*cw}A-og&aqHaVc9uj#fy5BUfSAQ*KA=L^?M188nuZ5t!qq($ze#7gC|yd2VVGL9h;xN?;4^R z=cn^Mei*G6tucCI1sVak49GnRN=|4P6J4UNxL8?=g1zzdqsLj)(y2SGq(?3AJel60 zhsQ1uu+Hn|@>^lQ4fME&Wf@ONT#;pjlrDyxqHikJc#^c)Eqce6&RmZrJ&0(u3K7dS3%VzMHCO2>!$Uf(iyX4+j# z$iku<@`n$5ST*-3&`S06WaBm8vr5w*2`vhS9NMb)UeQrdK~)0JYHMc(Y*=Y)!5s)NHM-RxK_qsF251=7P zs)KYTid>Zf0gM&n{lkX5=D*#~!Y5EYbl$fuy5s#>^E4zRjV(7 ziN+xWOT~NpVYiQmg&QRthJ}p~&Vdp{uP1{wGBp}2R4O{F-jxRzuq5i0$qW%gWYU2v2yp7v(?-u@6;WH9W z)XuutIg~ z*EAWO0^yV{;rnCHkG0tGzkdMHjK^OL8;Go47$zHZOOLT*{55}xC8O@b9ZQ&D=roYf z(J(Z}c+^0L1YauH&vw{$oW-OECANr3LC)afZ29uKZ`HTVb0#Q`8z-#Nm!AW)nV~*$XE|-vdm~+ zy~{gHHiAPG*=VLCi|naWpFmACd231c5089wjdvHCQEStAxc$QY)Qk2}{jJ^x!FnSF$&3y6UyB0+m>B6dFo&V{p`#FJO!VrL;!g zD|2@-vsrf-t$}7UQ(r?*2*Wi!g@N;Affd%wAKLW{QzUOoRMD|wpyI#bupQp<9rc%5 zyC@2HPW!<}*>*aI$8V>>4si2Kc@kwi72C4wx|?@eoR) zw`;Av_a-Czw`S?5$43*zn0`w0;cMrZGle;j_^M@{SWe;!aq&kszHJ-HSI$SL&9Coc zIh#IX2p)4b=>ndzz88eQ#79$GA5$NWvf>M)z2A)KqZw=N8>_NLVuvSP z82qD`q-U{stY93$tMM4N8)KTp+;D2Wqlfx8-owZ~AaQX=GYr7Qc_!5TCfhEFr2 zvO~h6QAvY)P3-Vaqdi4p|Ju8K1Pgk@HxaSI{ZLUslf*x)uFwme6>1SD$J=fX6N?7+ z1Y%Ky?;SZy;~tn)a7{kmwB`%$VeH*1aRqy!G~~1cDkgCtEn+f$*03jUpLIElB?C7T zxfn~P)2%eR1joB~36qA+*5qwsl%_cpL1+Yu8LqqW_~p=FeB+Q}Qx$8n=%=P_*_FIo zBrIOZ8wVS&?drv}T~!e)iN}HkHAXOWC~~wr%TAUS6786stP^shoWDT@K5woy$oP>!MwvOIw8=v4+$n|<1Wsa;s8NLaW)xOXmckQ9RH?Y3Gk0i9{4~BLL zabaN!H#Xmy`Ne!D4^fvQ5B1PyVR;V=yUCv5g-@S7&WLQ}CP*Z*z7H-b68FDh0~5z& zj)mf0jXZJeQP!98p+ppYDPcF7BnYL1|1Db5AukX9l{dAg>u-Ojnug?fMwr$A7^h168%P7q> zt@H17ZSP9xhbn3hRMgT-iu7cxz>lBOD~=tL2^Vr*j`0-BzdTuGP~737=bg+Xm6Q-O zL~jbp96pj|P@OE7XJ^h2F)mm0C|qu&H)Nb|$lH1Ymppc&KRYL{N@$VbmHmEj%e*OI zP#{C#_$%q6f-RT(nc*Kdk*F1wMC>5Mf+~#1oR4Hn`&>S#&tVpmR465Qj7gdsS14dQ zgY7wxX9OO?Lb9vdX-M*k#;U?Zd@%Q09*LWPt6AguSPtHm{TvnKr&&bZB}CX5 zQ8$he?ek0#rP=t~L6x%$=LD|bG!EH&1K$3F1&{|NxM)#DaLfon91K-+xW_V__>vz> zQQlK`@%Jn|m=#upqoFRwVrmuA_$G-{SYq18vd3$OnG@eSLZ~9P^I$WFeqV}yV_LG? zc*wQxS$0B>N?hT@V@QDYbjk8$VY|M3xs6H3mZ&NnvbWd+4P?+gqu(8sEOyCvw@hQK zm{z8+B14jFRlc*_xT6mrsKanuO;&MHm%~L6Xo}z(X208OifAlx$A3Mdh#kS55?92x zgp2pzrwBZHaOnmnX=PK0V4;Sf=~>)ik|JvL!t9D{>zb4+TbEAc>cjEzhgxYnZ<*UR z6aQ{yoV~*pj-O=GwoPKAv}F)BbhX0(L<+ryKmS>qU;5*psxU&5jDct>sEum<^bI4n z)-*yzV$=1ai;xL;${89Jxcpj*!~CMYX$*D)gT=Zb@u39suqF**2|MJA(zlo}Oqj0l z0Mc$T*vTH(d2YYsx!C_bdh-@`CN8>#P*LqPCPKyc!^uc=!;ca=EAi5aoh#^UQwspn4xNfjuqp@Em07Sm)`0R62C>lW39m&nh)mHCTIL9O^d70$J%!dMj1&B6^@4wxcY5_5gpA&49Wp*{U(&Jh!+$QA z%K*Pq0c!&iaMbBAjesaVNKSTV_%XI<^*?8avaJ5RnM9$;>WhB3IQo zTDv<{G%-n*JX*6OkG;r}BcDk$OpesmA4W{@Hl^abh}6CDwbxXAa0F8#Q)Ur;x)Kp* zT=#8*fdB-PkO$zRuACi3f=*pB?OrCBnX?I3_%6Ec&~aVvu&Ml{kZX3es-!=})W~)T zOZM7d9L0Ax^fXy~I>d6H?!9I0r>qVfokR3uk3hOO>q6uK86KQBiMx_z8P~$$0^{xu zsrUW%x_Qmh9y<0NF6-vEHJD52H)-Rwh8T;WB%muE_uzV7EN*NmUR<*0LUW0y*zZG+ zN96kKfiiph!gdCVvAyT^40(k)3~$SYMI<2oF?maih>H03TJMEzN`)~j1L6J;eH3OO znqfH}`lG|b&jHU&u8}PKyVnm&V@C6`c?2&!D31^K8QKm5-e`R1xaB_YS@F>>rhwN< zRH1+&%7Q0Crku}#%+FQR7`>YQu!5z4$J|9!iWD#!%S3oessV;t+85-gx4CAuDnHB4 zo=-ezXzh#uI?tDVd(%N7Xoar1YW;ahQ zv$P!W{BdR^-jq0Kbs7K;zHfz>G{yOfih5I`;30E^eVJ?Eyq5?Qu7LoA4^9NA(Acqq z(xez&(quF1FtaTnNbE3c3r-P#Um)r&$1B6Yy21E}wN6$XyGOcinZ{c8*UA;Jb-!)Q zpMfV{-S#rC*fRZ?L9y9nkd^kp+)gw**c@{qc0JU^CVmcy3rTyrU4 z%2{~5$=+Oc7WzLxV0ael>2|3IvN>LpM^Zbc<}Z&a^<2Bs_+72~qdS?V`B@^1R(Q;N zy>)fM4@P#P_L(J*KXl>=G3%ZMA+!mG=Y!Y}x+?K7yurwQ4C+%el-RgrK7LW&@g}UZtn5*u3OmcLLaM+6Oa^J|!sWLA E2i;~pPXGV_ literal 0 HcmV?d00001 diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766427132.gpu-04.112924.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766427132.gpu-04.112924.0 new file mode 100644 index 0000000000000000000000000000000000000000..077fef1cf8dd2fde6413d41b1aeafda5e528c435 GIT binary patch literal 48602 zcmb__3A~L}_dgmqPa&DdYo6Wtnq|n4c_xYD+;h&o_qb5zcmZBTpPZEwBTlW@zd-?!2BkHl@SyZOf}fZ2KNGY!DAeXAVnLx87_7j3p}7(;}1);enS3JU!8WI#K0$nI!> z{ypQ|{rw1Z=SzRyZyPg~K)Y(4wT8$`M+CM!;t2q}+u^#vR%IKo&wX|7gHZ%)AKqua zt?~5)+h6OaH83Fa9nNC6-^E!_H^$Q{JtGl+Fa3AY3%rhSk+H_K7ISayL2ITUl@*8j4fPBpy!p2KmfL2JBbs|?*JzCkLxE_} zJ|*f7l^QJJbSqmoDil>@;pe3{kG__mx7^a~VOy&k2)d)zUaOB>)`7($CVH=k(OXv8 zxILeJHDU-szP#hIO}6(X$P$e}<`;l}76N?jueyxFPp+ik6GB=CM( zAFWNu?R7*vC2oVu;z;L5K~f^7FHkjwB7Xh+d@oSr41>~2@6OK6iPSa;t ziOZ^~Wt)$tulVTm%6n}yC(u_6(5}h`}R=z%Lqdol!aWFDBQ0U4)8q zT^mvD?qJyC4fr{zvVB?ZcA1LDaht7wJ0{oXB&3jQM=s96)8)1s zw%hLTyX=8zBp8j@r@B3bMP_4YxzN1IlO{54F=jF?B(#Mu>Wz3nsL;(bj4uBO<&2m_ z+4p&ckC}=3OM;_`Qd~V3XgeUE<0{ULVo%<9`P7HZcV1ge1f%a#)>pQGfSf;9gy~CGOW%lAX35fP04IBv&iPT}wZTmrEKK?^Kq7!}m`rs*F z1ZXAy|H4e4_w^Gf>hq>5m0&`hA$vGd>NOpWt78sD1<*$5Cq)d+`|?%I3R#$URxW^H zUJa+$6SNnG%;!S24k~MBZ$AARJG)y0L@u7)nhux4XU}&>99dkF6u8XvP6lxW>k9tu zL8?H62@8V+hhc&Nm(Agn!vt^&-kFKNY-8lPrYPi97vzDN;b_Y7K@*c_koV5hzOhU) zJBA1jMZI>o&}H`pd~h0of#L^p4G}1YAY%Z;N8Bj${;1ah z$q|!QQOGZddc6jxD}5R0$H<>vqLAOlS>$#W+Y17ahE z>ii}nzvWa~1Ice>tPu`+iov~sCAj1%!fdhyH9OuqbP%1j^Y5>D*!H%B?x}Ux8XM5S zltRHDb3VZu`}~VHUo(=hE6y5nk1aZl$i7bNtF^-5E#%2Zwudpou!HySDG@GY(&S&+ z{GsX0-K$qhOQ5?~-`H56J5m%dnR&%Ejru*XM~Z9u=I~!{VAf)v1V&+RWB^;`u9S0` zYe-AqiP_Gv#FFDXm)>D3oKEx((gtYhPVjU^cmWBVkn|itXksn}-&RY3`RdR;8dL9; zX3#RI-m3*e0ZjREFzbE%s%F#2nJ=|fqKPaXp@9u@2jL%JMmew&@n!-Ceue=rs0 zm`OCEimK00 z7}~TU?T$Td+apm2YW=l#;Pit#29EP&@c2QDBO%Owg+x}>@9>$}tn3-rgIUk7dSV!@ zvb6I3^K8YliS|C)wOR}C_c34t%^k1j6Wj#9GOu!{;9e~bNFMt1rwiP(`J^%f@rwj}$m z2k-34aNo1Wof=QDCGMo$e_Jtj%(zuyM{~)J{6HWAHrIh(T7o3L0`GS^6(6Vip5>3Q z)7WS}!Q*Lcf~vzAjDmLq$!m)=kb=!6Z`6YJyBYTDDt2?XFBmDcNByuy?7VxSYUSle zQn9v*i5W2Ky5HDw3Ef6?i)4_?4fenXREHxTC;03xld86skDm|S_^Ok{Z%c92!?wD& z6Y%xg7_GDJHBppS1SJq~Z9?exxq@IOf?ki)5!O?mB@WYvSn}zs$KPAdws@~xQb;@` zIi29Rf-e;W(-R3mi6$jK@S7`g{$i~1EKpe05QGNYkZ`HrnJV2M6y7?Hxgu2-5-_?V zwRB)K%qGjA?5GlZy`S-J9TqC>kZ2ex)gdw=67jlCc$SQPx&Mzg79Fm;g9yanwYE1< zXz|Gkg-YaAd0qQHjKT?b5}csWJ~=7SE1RkjUxZNk`N zM;N<;cdP93Il{%DOTvMI2$qlvqG2dKaBZUO1UEZB?zkh3MFl>|vqQaXF_A0W1%XQ`xzvDnPk8n;<3sl) z3LhF!G8{`KDY3Wqo3lS+F7OEni!N|;U5A1UKw46daudQ^^Lb)MZW)%VTd|bL##~)p zv}2-TjVV==NLYiG-@LvS^YnTwBT(V#VPQuw8j4VEhe^<_QD@!T(a+S#Nr@`5^@utJ z>g9;cW!D<=?P=BTXNvBwtFmUCWX7(_3c^NX*TTRiTzdh;Tv%=4yilTxEHz)bb6U%HZXn%M zHL!J$ZMTFUstwjMQ4aJ#$XEz7IOyY~_#x@{&+5;zL!SHSLqu|m+y|CIz}D~q z0v@If(Xt)FYtb!*aPqtm(uEveuiFb&#S1mBsJVf_k}uc3vSbO9`B&vqLgxJtnu8+) z{-j>;vtx~si=_27)?E4QFU$z|AEeJfBhUaU=27UE0{@PPPZoF6(5wi{!qr$sxFQRO zV&M^GB&Rb9btS6AC$Ngvq>p>!*Y2!6<<3~ciY_O zW4*$OmXG|zm=kzNWe&o6-GvTkX^OZTzRs)BgJegP8frdFxL9ha;e%{2xITr>WSPC@ zXxj?R!n`2y(88p}@GM+F#idljm@m(7>&i}}do>X$PNM+|WmD^Lf}ilxvLBpaR;%I~ z!bGc8Uq*~!2v(rNm~6OT{-w)jGt_+-Ks9^(mZxq@$*+5ux}LL^Xhn6M74!t%Xk5*Q zTO?TiE70Y^=v&Ly{lfg3HjmIs=-<_GWkWevs%b6~iiU!`Hm2%ahK8Ena9(fbk$xy) zg#$0Gzy;tfdGp}Ly>Bs&-}I=mj|~x0G)jSXg$@vJ)SQk?EOEbmGJh9Kk+pt|P%%Z8 zq3cYS9d$OLf&~uvAaMxUYxLMTa}qC7)A>j%kuyu$1k`5?iEf&DcgdKc!VR{O{xyOfVnbKwlvI2|H#l@0J%A%>2v$N6Nf z|IQEJT9(CWGd9vnB2E)ji-wkzk(ZW@W?oMkaLvMuJ8a{hAmqW?KrP)D`2X|;6$X#) z%#>J{O|%Xwu{u7lBXzEv`FmPEb17;)NthzPLn(7e=z?m1+?2Vs%$nUx>OA)wGiYU> zB49LVErKEFTf^+L#I;c8SHy4z*jwwcHtqm9yXRZ#F}(U%Y(qGyXDMNr5n3V2O^r=r;OZ^%?E)57D;r*@_V zuU_`uFub zZK=+fEY)QXzIOxTZ-cE0e=B=Uv6>=pKxk&Bl9sx1+rev?oj4_t(B9YehV)h%`1rU+ z5(U)>{mE*_{F-BHTCg~6`!=Ev=_@wId`p4q=A-S)q|MB}|> zwWVdXD)E`YXpFbT8D11dh!w%;JJP4a?3pXW@16?1_u)v1*OxJ9kYh~Gc0>K{G0@T; z@VCoK&&x0*H=O9dx6AF-PHb5wq8f{}jr;bIjNL;1~zLIPU%@jH@V;^AWrU!6$O{qbi1E?d)AX^D~A@VL)VmT-6r16+1$#iYIx_87t# z5~cc#^M1PF`hzTo|At%*bNJN@z&ndX{U%MN7#dUe->lh=rBqku77Zp{| z&c>1AG+VO%!@1vXWR!M4K+vMvAh?VjV@bl*x@X*EmMp0J8o`NV0Z2Ob?}?Bgi53K& z3%hBqIrS`?W)4A7lprQO}SZ8?+3{U>+_bVmz$wP?k)jYc9B)va_6B-Um+d=Jfq?H4*iQnlUIJ zpC=5#LZPMPSbt8(GB;ObQHA$S`W(^ffS-fCqjXYZ$>n6ZHTjlD53;E3>bD49(2wwn z;uL~9H4uW1zGN4eP&qh(>CJ6&0Ws+5h(hxr_MSND6Hsy;Nk_8Hw{9+A)_0@5vkgJXtW#_2|H0qd~|d#A*eCSLoX&`V4FNEE^qqfTjIT%LPT+ zPb|n4xNd%}cEWn4f6l(}I+LU&@6x9U3v9KLR zB%?}FL`mID95QJcv$}PT5SXyK1;Lzb*!#HoR!*w50DQWrMib`l?~?GM6GCqp(yvq~ zcGHNi9JEe$jZ@Elwu~i_CwxMmAd<+&9Tu|r3xpWID_Q(Te^y`4Y9(bqC48)vWH@96 z9WFwUjRkt!1)i4pg_M2es;=MO$cxO`T`SiSZ3d_t>|URp^+P554K0ptUBxC#`+i2> zfRm*y68A~O-+7W)hR%Jr=_jV6Pf2i5Hr=NnZ0d*-tb`kdg=XU~$~a8b)BUvd8Tv)h?0os-K5S+ax|1$msL;2W;U5?7R0 zG|v2oea^Bkl+Q`WY#H|46nH}5QO8!bnrgCKrqjAvy;#Au)0ec2D7eaH;69lD%PQ5r zT{oPy`W%(3h*lqiYS9H<;t55teIxc&2};wJAN0mOZ{NZ^fqBR1gGAE-E+z2=Y*;KA zTPlSeS}^7JSD7R<{fbr+k|4%E^{GnzLK{4c`uB-(D|4~k@{8WpvgR@)v|p|xnpb5P zLNHsOP2uuot=KEwwC)R`bG*@H2#;BoX=CY8a;y1L02uaW_ z() zC%T0{C0VZcRmpPw&My}8?AhNES#aIuE`UQF8ymEXVG%ld=C4JT8!uYP7*Bgyg60v!E5(px20XbMJVnD7e&mu z@oe8^XFW#sDG4upEzw9WtTK7BoEw&~pR~AjB`YA`^(|2?3dk-P?LeK7@XjLzlq=`= z|0utcfo6Ya2Gs!*ST9S^+MoYji)pUI5>)R>0ZN15#}lPuHn|Ayrm*A85BzC3-{gK; zAIoZg-rS1$TQg4)`FhbFR)U&F2s`+}0dfYQ=MqPJlNP{sEpFbyocus%5+gKr)wGWx$DxHM-ktvO{c=yjSYu%hqj zOGLp{7s`wIj!$>rQOT;$Fdh)2h*4slIdy6yY%CxJ_ z(RW~(wjRP$8(u-!m9(t9GWW*6S=DRwPlPI3jrEorY%SJLE41`DAOl+BF}HYI^MAVti15zHwTWBxDng&1^XqCHk!!(e4gdeXUs?>g1wqrh)SRBY{O3@J=;!Ky3) zCfJe0)K>cDp>oWnnDZM^i+)ONCk>v*7iyK1*!UK~>|hIJ9KZ5kT|QuEqWSLxD(<~1 zg4Q+%TpR!*B(6S-r`*J7MP|7})NT>HRK2XCQ`A6TocC~>G_Yk zJvtoTczK~Q3Z8+zf2!>OOKX24A;tU%#EB3vrZ8B(>E$`6SeMA{{}PGVB~lZwEI}RW zF%7y|?{cYFzfXP2T| zpUV84!p9lj^2>=vQ4OQpaSP3dQrY_4%udR!r(2tU%KL;nmM1*XCoFT;DdYC;dFck$ zfBY50m08^+aJiX3USsErEF6Bj0?~`%Fx;*M={&gP1B6SzCpeB1ii(FT*9{Hu%t!aF zVT_$)Qy5!I?-7Ri0do>e!DEHH-*&X_%`8x!MtGu&3GWiX+mi4m0_8S*WuG$Bvy|&A z+U)zaGqXUSNjOnjgz*dLslacJ$5bc;Oyi5MZL#!ira|XjK@^JnHP8pvyH`xIu2@_J zt=o}S->{8obEUE|V8%gz8}}4@%mc;N5+7MxbQ|-!-jTq<>(YUZ0v$4z^VAypuEB3y z#}0f+MWR_u7P(=hL+n#3*W%s#@KqVi;cZ-rFona5R}w-S38xpcDyRgor@@?vQY3kl|ImDjqNXYB@{1+0~a+WO&;dp3jYU4yJ(TZSrk5<}Ht}LiCC81O0p9 zq=p>!IN4P=sJIlD4W8j+CaTQUgs8g|c+rm0R+l6jyS{kzN9KY*CP9S@9`O|GBUxbe zaM(Rb)amwGFSEw50ab}q>}zWoLq(YiLpuugM~HPynnBqu?(BB!vSB+b9e*zuz|wKk z*aAs{bI~fO^u-d!lf~7DdJ$ziTrib^opYcm;)YnQbwkmNOXjiTl~6>d9vLFsiQ?Y{5d{eS?^> zad}La3?vri2EDHOf)!7j)Ffyuo@N=)n5TlYklu_T6A_}C&Lz>B?IZu!>l$X14#=g@ zC{;(sT(I+t^W>&qsyLYuo>q$p7e-jz<(&fD>a)x5Vv<(BHo?*9i-2R8a+q-I(ldjZ ztMsNsMHj50&P=cn`H;>| z)-c@abrsw+NSVU;GW2<+8ilO5a{Ve+4&Etou`mgjWlGl$QUd7hL;}HLn2&bC{Z&qn z*&$cTs_KQDUo9TnjGgL<^@w&n)lH2x!s4n4Y}862i!*%o_p_MFzob5)3ze^LV5)5C z>Rxq}IVtNTs0hxCxHd_ z(fX`}72}K&*7{;JMY(YU((Z|?GFWzFc^VO{&%l6Tazckl*enK?fV*1?yKF5&0h(j`!MXivE4)5r|wAI2!L=$SO+Gu?fzjCB# z?Ht3uU(~t51Xdi&XiRWe9IFB10vEQ%SpenWWGUg%Wh3kT%S8J_35KFwpO|kk^{IR? zi;)^e)7G=n;LIjOA(jT4>L&%{P@t!Xy|?A5hnR(J+?23|g%#Mc2E&b$A~W3Me%asn zcP51UB_s-AZF~&^W)bc9!~#dO`hMrcmtJ5olcyQcC~CX-)C$gG(JW00iYVvE=3UI8 ztKFP{(V=S^51e59N~u5z{(A~a4cS`oY^8&Y^4H}8A`GVmu%jj)ZA^U4Kw(LVM|oc0 z&o$qE#EeUz1yL^Y5pb6s7(#1UW$z*$ze4cyPj&p1?OolLgo}Fz*Y!X+9#5q*zz*dn zxH-V!K$WT^2D1QUp9I7Jsg?jtl&@AkoSg+*m7PUQ%@?#H0)?8#WVLAB0^8>_X(r{P zv$FG)5M+1RYGJ14ZLJ9t)qF!JKL)^?!qN-syZFkTq6^fV(ZX$mKVzQe6A~6Z&87-0 z^qB_3*xdr27C#aSwA%LMEPRspZH#S06eDzF0WHp1GOyodXXYx;oW1$ysY6U;|CG=| zWCe77AObENyuVYNfE&urvCPrNJsF!B@anciGXggezy%?<8+z|V6Qj6tg>yqu;7Uqm z^@G2^$oSm5ox%(lDS+)0SGQQfTKjOjZC_Mle$WpRT0}PjnkLyJZmhS$-D+^5E~i?N zJD6x!OybUP=gLg(mZmFm*Fc~r3){gsa-WS}x`mbaI%W_omiXFQ5e*MP!I3e!w~GAb zC@1EoYbqRNX@E;IX$diNKBd^|Jtl=0+-U4C`RPj*`finoqH7w*JHVH{$d2RB!fiie z+c+VM$Q3yt0;i_wtM#N3T> zvNTJ<>!$6j!O~cNBAA>+P&t zZPP$zX)1LhaA9^8a128JPW#ykeS#R^*QQIF?`|SKUS6{3`TV(sxe`p z3ZY09P%xkH?M2Hq9w@zHJ6q!yb&b{vXQ=y_^g$}u>pt+?un(CyuIi$QV@qWXaUC#C zqQ;k^l>NoabrescUA|t@u!fx~(VgOrN1^Sf1A1D;9mC!zx>;j#dl* zfyiA@z=0JPk=^m*DO$`Q!Td6Yo;iCB?tL1I0Ad_i!s zN3`vOgo!!5N!d`p@4cT^fklz7ozL@}Tx-y1y^8Nf!a9+&vy)(63NFsCt}12-401dxkyVN}$C z>GF=<&7bVSws+*U=Ix~|fcOoHw(#tPsXbM}#)5O$V12bYi_bHm{8eI$!4>=jU2Iam zZ?bu<&kC6@u%b6nE$Tcv^TF4J*WI8GFIisfn-p%rM5R+7v#6kIj0T;h@V24(>LP^? zCLUk;*FFbxO+S&i=$fX9t%Ns3@C~y>H{DwUmkvJiH#4ZW_a%Zwod*px42?%b-pD2D zC2;eR`Nmwu$55i5Kc5;hn&lvR^&@Ce4~Oytc6Zy4rd9|1=kV1m@uubUqPJwP|ETrch6<lgCwRAhBt|FfS5y~b`*s5O|7@g{|;KEyOhqZQQF zm6vC*bn`9=CHix8C>-vN&$wGbzI%6W9@BTuAw-?fcUVOcb3c$^OX0HkZgpmk{S!vy?5TA%d|S6ssHFyA1tu}DxCvEi=RK(w$33|F$C-ZS^z z%{V)77!iz|6|d}o2`Yr@z9|i1O=tIKb21s^4=QARe9Zz}M0;jMRbyx!kz2xb;9_Td2k}Cj zvc^yElUJ2_@_Qt{m?TDgXdv{$y~sFsesOEe|MB^MnJw{+rEfr6QfDeuW%9A-5j$G= z4r{SK^fHFbw=Mg}S!O}6x}M=?w%aoI2bk96>8AOyLgL8hcjX z#tzBj5)%&zzRxS39F5^ZqX{em@4H5m_RDkk{&WS?PLswH*{Gcw8PLv12#T)Q5yJOh zkz2f|N9m=#^tI~im_}`RBSDLTjRCD|Oo+*0NRxvnR(tIWE56Oo;DAeOvI%QCi^NVp=)2q|3*IYr-8tnnmivui)ruN<=q?@4fUJ<=4^gdx9@DE7gN zE%26?-#FN}rXOCz4o=iggbOc0f1FG2I_Ep7h>0nJ8#1%`UyR3fCaFB81cEc_a-fqG zG!UQQT?2)+WDl|0!<%JhGE^cD7B7aK9$H-5gUc}B!v4bcU7Tu6PeAvUP zxkrIks;8I#@^>RvX{wS>(4t_-p{;uF6&(c?R3-4dW8k^*%+=i{(S@r^ti~}=9N|lL z1JA9`_@`4D{bynM4rgsQIEm(Tt*v%dTv|{ekE_s!rfhwI3D@5e2!*RA(v>K3RSEJewe||E+Q%gEK+$f_c=T@VaPK~b*Fg6MNXwK0g!Iuj5 zvdG$rtZ~g&OxPkO1v!I@v*pX{mRo2t4J{s-c$g8qQG$vf+R%m5d7m^94z8Xs%wrkCL{?tSpAHj;etPYEhL&kbwlV!$g|CcM7Z2TjUg=x@{ zMfTLGPoO56ytSnJv$Z`BF?KxTSJ)v2z5zpRopDL}AT zNou7NEn%tIh#oxU@JjZEO;=r)*Karrj>-pV0Su1X@&$}>pp@3A)a~2vnAzMQ*Fdw` zP+vn&2*Wi!`GE^$ffd%=>+19}QzS#C5aps{#X!Y>!(ltT<2&jvwsuhz@IJ?%9?rJ& zoCFuPMF%(70dAftPoiw6Vq50!8T1|7&U-`3cDB@EVJr}u7$G~ZGb52-uZSVf!t8l@ z)^^tA+c``N=ck6+D)^YGgYrU=30sU>_A(TXK*IIk;M_!s)mZhJn zO{Fz3{Z!wFubpGg6y`wUtClrmIf;tm;*WHE+cuJ;oR5(ku6Um1Y_>~y%-J*$@Raqv zD11lf8$V!{Yuq#?3NMcKU1x*m8Ebx**n%|@J3Qs$;Mbr2{v?aXR+SRz zBIcza0o#o+O=501wcgQj2L{(}Z++m^ja*I;9krw(}~D!Uxl{kFspTZxUYg zs44KFs6Rp8$x@NLHfh}LOqD(`gXk81qyj7+vgInonjI@U<^0KvO23%|i$jM3?uUvBnk4>Vb%kE&tWb+MU#y?Mnu*22Sqa3V0N*=u7RNm> zso$Be>vfbk+67ARO`vwU`3AY6%K3k?zP_vH5wz$JhIR>YVPOh4Hh;3K{1KL0 z*dY-`;tFjRmiMr*o9qd`yS3{SBhq!VLS!8uTv8#Mg#l5OqIHnxyODX?9 zLKJ-|VK9y8w->IZdP7a zryOM_&wa~@(QF$(lM9H-FtP+ME8+3iZahS3ht*=j-0o$zvPPf}B%z`aNcsPB`Yc7EQefP*jG;H( zp)j-o0)mAEF`%zc686i!{&@_x*_~!s9^ifnD?C6!EnZKS8nGP~ zy*-r$r;`^Eu_6(Ns9_xI7Ig_usd9u(PTF%F4)kV6;L5uQRA@4}Q&{12JFu%GS>VEb z505rN>cA%X_)HmKuL}P-bag1o6XPPL@ z#@`OAoLx94aQ&un$Z9`x>~j`C{wu*niz&21H%Xkr64O5B*gdV86F*`xp^DhfgUuZJeJT2lX~}M5rGayIu@mx> z#1%0Th6Gqomn=^fw%5S{Uo+`gv4qGK(m{?2c&1M5p3(1)N)~(F&^iqmE4nUKSdk`4 zwkqFQZd|!qSFK~XUol+Nq9U;k z^rDN933$pG8Wp(wT8hJn&$n!0uygKHcmN4381d3YtqAk5CJkW;+hpm@{OXmaD-|9< z+ARh<)#JLr?RPvE>)YM#G&>WoOH@=ljfqh4{cti8-SDG?&Pu#A@ReLIn_+Bc}2#$8Zn$%2_(UYx=XTI~l2R2(#Td>*b}rc3`5c~6$WxfP!5z`XWd5?Hhi zpd=8ixJWFq!AQ)AMV(sR$GrAQ4-(DjwbvEa0u^wu2%J7HVM;6t>e)DCn>)@~*n_-C z@%CJ~RRoDGTs5(u37$xFcue(Jl4Na?$)x~-OM{ZBazWNtx3km?Mpf~?ligjr3~=? zhlx}KtN}>CQK!Q+0;2dJndzP2$JnADf13I(%j)-7O@Jb+FZ$u)=uWNtQfXg&V7kV#$%)*AQ)(!KtM`jF{kUO2u~(se9vV7gsxefGLsmwS=fE5pl+K z-!>QsKrjh;050mv*-<2@%dmA7m|%V=afR=q>kb{)OKJ<7* zuFoDQvFGMz7%0Z}t{eJZ9p*61dW=3x_jzHZU2C{;8Yw4UeWDdoAJe?Rj$!wqa8T2)*NiWS`Unb}!c3hs!HtIc56qS{8n zMQc$XuT%>_L)d?EA}=eaVUvRUPn4NYfER1we6J1?X))71ZO&jirLLp%PjTR z{8EP*3HuX7AsUIgfP?Q_;U!IRzM`Vulqk4%`mh4#8kE~afWkEpfbhYI02LZLv{#xG zEAJb97wa&4R6=5hSzB<5`1=A;Zz*0G2G$M6N33t(xl%0jVrwI(tLT%kH6+kw}Yw}2H#MJziF{PemR~o;+&+a*ZX_~(zvS@|Jyf?(` z`rF!^Qfn2yzE4*dyBl-=Geqq$xEnJMb}H5+7<=MR{{*a=z`Iue;SeZXj=ovFGK(r! zZ>6O$sz^iQB{~OF0v-0^gq5sk-MNiG#cPG~rNZQCx3n9l9>{6I3~8ljX(cqIwXM4v zYQ^@olHN6uFWM&5>*>Zc3N&L*&C?Paoth?a^6~Chmb5iOiMd{!ZNF}R}_`_dS}ze`+Bu%5q7f#Bmeyj7QJj=j$j!Kk6A_!;ljHIMPQ z-e7zLX%tM{r_25~C?C^>?S@;l;N9z+b|uYmVEft;HqQ=PCriuFYT$ikVt$FAqr^;Z z_WIFWR$lC~ld#0YsThEY9T$;e00MD%OO4=Rej)(HDpMC^;$=LxjSsVN&$`dkGC1y8 z-38-f@egDuP|8u9vSLpbgF0Y_5*wGy$1e+X*upx?{JV%M>@2H>RDlPW4ARxjx?KDJ E02PDBGynhq literal 0 HcmV?d00001 diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766567162.gpu-51.2669674.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766567162.gpu-51.2669674.0 new file mode 100644 index 0000000000000000000000000000000000000000..f8e5e1701a1cedc46563f68cac416665a4d59d62 GIT binary patch literal 48602 zcmb__2bfb;^EU{Jxq?)I1(sd<(w6NM5$Ot21Q8X~m`$>~Vbe(20w|~`DyS$HEEH)1 z0#Yo1NV8D{MUnC<7ElCHv0}mU&73?X>EDlF~WlwO)jK_=DbGTV8y4`VhqJkgH{Djl5xx)orVE27+Be&Wezg z;QH3QxJJF|^)9%$YTpZT3kF}TU8GgVf4?;Msw*3Xm7ZSHX z%hMW{xEv0r*BW$9aPp5=0CVySvZ8>!Dm-fk5a6lfC95s>%_2g3Xx+4CMMYj~BB0&7 z=5&jKUisUO->)Ljjc=TN#*+0If%epTXz9pHTL`v1!tCpVipV-&$! zM+}%|Y4JG04$`jDnnxiEZT3>9*TGrPG{V!meL^8`U;TG7i`=$gNpy|;XI}oTH?5J; z=$FSW@5wdtwOp-nX_?htV)J^PZUf1+BIwn!RaOkrg0Egfc+)ITJz*((f@too_0n1v z2Yg|_b$r+vsEo3N)2(dXs6beeg+{&q_jo^oUUN^I`IeSX5_C7MKubj~>%d|W1HG3- z=&h=3T)N#hWf(!ex#7Ze5=yW(--eLoC?FjOPxj9`6 z6!g7ohPNC_=vUbHFR@IRO$7JVdTXs?k*%Rhzqud>e13%+%geU49ZBGAPStqT(%~rr zze*dRbqqM&wvemL8RfD#(uHA=l#t;IR866X->QPi>zMc*kSn10b&9hB^2^~0hHPHD z;WNy{W&Oz=y|1UQIQUuZZI-*9rmq;RU9EKt2c6b{F9g~#seKq|a|abxwL3CnSbtii z-%XiMTQZ)ZMY?NUwZ^nSp)F`2x^84=YvgB%pXj+M+&$uTCX63SOcchZl4xhZ4#%Dg zxXO!6MKyJ5S5kivky+5@(uJ0Z&k~uvw4PcTLAtzPB-~D0nbUCI6*)9~|Mu1uSvL{z z1F6neEa`Iyc$hXs%Oc>QbG+3J%2X`i_o4WJwkopOL%sn20Of>oZN2Nc{25FS?v)Fn z9!%AD)9AtB?^t1!Hq%JAI5I|s4z*?Jz~ z;`wZ4b2#EiU7|#{9y{lY5ybRopF6zC@~#9QsP)&{du*jnYnjh(D-_lXj!l^>=qd!G zUs%bPP^f2`_V2eVn=(UQIiILUL*B&e3wVIb6T~)hL`AMg4!v#|{9;tDbJ9l4W^&!* zIYLFbPC-zY>?}* z7ayEwB^FV8-l1s6q3FqAMo4T$SMq>-44N zDu4Daw{%`e%XQWAwZ~x0{Mf~}S?~gHN@0W;z*}?H)TfphCf~HLvGX3qj zi+d80J6>Hl(lTKY5t*%Z(dq}wUByN)PoQc4+iWyM9byPe?cv*wjL2AY?6cc~RM;kHnd%pmHJyS!cMIa$CHFh!n~# z3{IER8nxY4o7Z9Wg+u;u$Xf1n6_*%|q3J@KuH1V!(-uRXr-g*J@PyqV7YG%)c~PUw zKSDVp22uX-tK1{ZMExYe(L^b(o&&TUkk4@y=SHz73pccC$$aP8ONn6gotu)In*?Cm zu!%JoVBcj5V2aL8EBN|0a}}1QD&WvFzYb>h;-mybdyxi?1c*fPu(tBGyP1#w&~l;^ zef(7Ll+OdSUaJo4nLf{Zfk088w^peH6KW4wgP}^d;b_#0I20FyHbOrs6xF<~8viko zg?ayz3t*Vn(C&8mt;GT3xlpZx%G$wykMXdxJ7ooti)Xiy&0+Ic3!NcbHkTv?E-SNp zl(;=Zo_@49sR9uuv{^}T7$!vFayXoFm;g?}TYaR)Q;fV1Bp#A?nII3$3`bLr51N?# zD0v%q9LQyoIbju%hoYVmEOuBuJ`bGCu!lpMeQ%@p_V8Gv`HKW7VhxYW9`IQUZ6SL} zoZwSAj@eNf>n&TmmmTq)5>g!TVz;jltW*M+ii`f>N8Ns92Pm|f$ixHGJYe(qA>1;< zRCb4J>e7s=k223cWevfi=idQNc)$tXvI1;%6XcdM0qOjW?<@DBRudHO`+tw=NceIx{?WNWtA0#({K=lz5q7QHhdp8$^AeGLwKh;|kHK5ORfud4VuWD@ z@84A>T*!pUw_Ug^jk$aO$|ca+}@E0(n99W6? zvlWkFdHK#enTqPao={Omr83l*e3Oc)sHjCJCzmivf0USlQcQX3@?{PAtie*3!TVQG z$KDd%w5zP6(?-*#4eNY#t!4EFqVyVVkk%QTesIUYaUKUAKZtQCfZ4Br$f|m69s`?| zJ>z`qW4@p4ZJK35{+E^WUnmibm;IFxeuI@E-Wwuua{=l0^ux1Qu(Df1W3bXhZYAi}Qp3qsOybxqh2;&a zjcLQwQpF}BR;VRBegQDEPLB^Acu*wpcP@42)YkdvhYML6sOjs3jcK431RC_({J|1m z$m+L+N;q^%#Ivd`e=F1PvX{D*G|aJ)rk&fyv-GFMf7a7tXig0k@n_O-2ewOv9xrKm5O5F%-GsopZx9hL24H)j~ zbK)Ay3JSd(_pxz{TMwth;=XG5EX(y-ynEAjjd31*!^MfZXkJWk_J++x#aa5-QZG&{i%xG*6HzwDy?BJ ztPwfyZm3$h`H@tttzu%vFF!n&9hbY`B)Ua1$l(Ng-~p!e26KZZgd0# zY>WRVmlP5YNlrUBuHZ}g!SsZDP@+l74;**6{ZoupliyZYl@3A!Zb-1wYfqN$x{LPi zV6I5UI|PibNMjus4YR>AC_AdeUhm|!N?}Q?_az#JN==ANh(z2@1D+{kd*+5Fu;}pi z9Yi2ThbeAfvB@VZ6e^Kd`H@8{7=;--2~JSxtZ;=)@JuP(^s=*}4FP4A2+A?DoQG2C!g_bU$gRWvP1dC9abGV%e`vecytUhm? zM85O8V#~1IglyGr0RtGVNkcl?fSMaIuyq3Tma;i$S&3?Y5QKz;Igg3PA@% zD%(+aZOiEWn;5(5yh|$!c6n^UQqUzqUr`84NJZfwlpeS?QFelxowI)p-^QYXEfQMP z912}owb*6g}WedDJ7R2 z@LSccY{mG{@I3++sQ^lbW2q!5_Wm&Y)o+;#d|1Mw3*1)Mp&$d0meix%xbWtD?l<@O zuUM|`&i9FI%+)nTJ0>dDm{K)}ggNM-ddo&KPp|bp0u`Pf7Ip-qp$O%)83f%NwRPK1 zpJZy}9f>Nk^@uti>g9;cW!D^X!}S-{V2aN30THTa*!>}w2YMR}Br3v%iL&T1)3$cH zj~VW|9}=8!(R~t5xCXwWBDc?&PBbU=)DwUI%Z%N6iH*jtT@;&etwj)XVYP+xLWwT2 z)qLfODVJxBA>CBp_tF;2&HIV;;o49w3*|r$grW;!1_yndBtN9xN8RhP>`>K@XeE){ zBKKcMjxfxTQ(`U2F(fO@nYE|kRSTG?ERt&oi{l52fRv*#wbQL)RV2`4LcqFxK z*jU=lrd2O^!1C}xBKBHsn3iJ`UW;xigp=onkS<_zyPa;ZDsHHGg^djarhK_`=VdQ2 zna?>yO9`3xLTC<-4EU3J!Ow~{MlO=(+c?l3;xWeoiH$~}IaJKU&@To49TA@>ZtuQR zj#KT0h$iksmxI`JrZVy9Ui7N4NtfD#T$LcgV%GyIR{zpq-dq{@E>4$(Fd{c)N zkCq2wclax~+r~a0^A)B(`{GB;LcS+g5X?bXx3k!0uS^p6iuwipdz0*lQbYJ-q8v*N z4Ly(z2G^(9o+z^`N>6QL7AEBr!b1y_7Qr)d0Tq{031ixCnDjh5jju~kaT=qbP&PFW zC-@0ZS-0v|X0>cbh)}dzsWM^=La+iA#ze#Yqx(Ag80tlznxh&$e$!KTXwtqlOkFRR zsHm>9{Vu-~jjQo+iv-Jm1-d*KUAt#rnE5rsKO_3luW91QfpV@?(_ADJ4F!2^Ow~D} z8mf4sw;S_Ff03}lBb8R*9Pp;R8L{Z8jf~^7KUX-Oj*y~J3bZS9fVjiPbYy&qTfgK( zH%pQA_<~R|Mb<^vnGP%JY(fPK?DIh45VF_kv2*6cU*t^rma0sd9+8WPtb#oZSv`Q1 zyBn|VRACF>$h&Vo{Wwe6-gT5dK%{Jq*wJ);QmZ`k&&_rwm~D^ID#DL|Zc$WsZog95 z0FM=7=*W7UPv-is!5cpLrc=gC*CPdzmYWHg6UmmSbjTB$V0Vj zv`mlh|I-(&oi=4MQ(~*Wrgcz>HSxG@$#Z4rR!=|XQj9u5m?FPJDRW5Zf@*-=l(DtU zoZX?XJHEyY+Mf~_4O%;Y0Q%N2`z&!y)Onpq9D7l=p`~y1sljKNp_q4)NEfZaDmG*w zxR{tq{Wa^mtg+EvR(W`)!*cC61Ugz9rRAxh;LoBjAwERU7KKWnkWu1u8~jd1vFYEC zrC6qvp8cEkpatta{>@R#&vL;oT4$|p5Q?;>q-W>kDuuBz%bPA|q2FVth;$M92~8tC zA1f3R9H1I7-^k(3cl3|XZ=S*UJ?vYR-v|X8iZwz|VCFFXoaoJ1+FdC855;?&#S?8jKB52rv;F|wcUnTO_4VsG%HIAJa&ppxU87S?!q5`*QW?EKZwxn&`tgEe%l1gJ34GNhcoiWs&o7iL=5F zrBCReQa$^b_vCbDofa8C5WHwAiV&S}`mjrlsWllFc1u9v^+W)JNo5dU`L}IrrkuS$ z5}Bx+TW~HUg?xYJ-u2Awz4RwS7KtH-9ES;}iXyA_m2Rdu*GXKVI1$$t@IW_5{J2^< zW+-RMiRfKE>jPE^y#8k*7%PFzqQG&ejqfFN_Z9YHcH?)6h<2lS6fy3e#mDUVd^3$j z%*%cuDluZl!$6-Ot6`Ls0_!fToV}p>6{HDo8}#G1mPdXi$Q&(8s}CtztMRldTr@PX z4%|Mk0SnoBogpw0vZ22ia5KxGPQOLV* znLdX>*85Y1goK1Z6+sIpUxDPV^Q~+B`)zRt zX?f@@(fDLl?Xjv_-Pp`vIKtb~E^ZV?h!w%;+cGD>?3tsB*I5p|_rXw^+tX#p5ZmZJ zorim!qoJkU=k1)GncpQUxxskIg(=UDxI#ah;wm0@9Q&HBs0R-{>v z1rucJy>n6u8CyS)kdmzeGI6$c$ynEd`KZCaiAMBM+k0W2f|5~~-APQixFl4aIptn9 zu+-omS^)=^+89=_mx4BmP2HJ_#h8xi53$tfL5Ys3(T)am>R1$u#MFyS*T{RJ@g8=l zC;dxbfQPz+VGTD-NI>g8en;X<-1N|UpWjN-eQ33JlO?sv41HkoI&DPEXDCb9+{HdF zJB=e!e<}7D!Wa^z`l9E3eX}3NupIstxteHYEduW>6!sc4m11ZN-GB9*X9lvw#BEiH zc#)WZa9&hYK|4o}6sOsg^?&v_n8qlzTtLvG+90@$9b*Z?1&)5Oh9wJ@OE{4%07=LG zJrNQl(1O5oVUPT|WG(aGhEyX`MR0Afh0q6SNO_y{H}jS+wy}1)Sv~#prxkZuZmiF4+k zEW296LcpSG%z$J4EmPP@U!4A%Wj&TkSj>8KWw2pCWD+8^0>LZvZR350xqR;5SEYod z{RV6F9U|=~7UT+CH@{{(VZPGy>u#LLB zft*7iY!8J42^++Ty%&AJwl-Kx+1eJ-!-cKu_K+94a5=M^M{IZlBF0w=SszGmi00s^ zC7+m8aQJd!ev?)!ZniYMlvc{pGPSxz{_FydY%V2-{&?tOmL*y*p+pw0u*e8y%Huz_ z^}m8y!m+i9C}9b)RtQ;IESQ3&wZ;OQ(|E(A2kJ6e`%5AVSrbzcIH_ycDTPw5q*g9i z_6GA@=G7re(Q`?|z#aSmecB=puG0Enke{DhpmqnW*)zG2Isb)q30h;P*x$hp7hd8YX;tVfVS^F_bjTJD0>W;x$^g$bB~jmpsAH7{Juk`B*HSWG%JDyakq z3rA87GO8p+l+;bNg(s&ot2?MZ(I~8Lkv}&F_C995m6K{N0Nw)^Y-jHNDG4t+A@r6Z z{Yr&WCynUJLF+`*QElVOl*nmDkB$1kx;<9b50&UQ^qSJL3!5x0 zZ%E&OlcnwA_esRwc@kfS*0lOy0MpSajR;)S4ds0==dp+)_0u{W8^{E8gIoj!wSA;u zC^`%xF;v2%a|O=XPcWZ-Y-9Qc^x3oJH=Nfp%2(tUuDX}W(%*6!Axoz4q9Bix6nw)K zS>lRv#MeJou+N#FqI^y!X3MbWrpOflk2#CjU0p>hq;sMYQ@vsTN((Wv)O7+czR#6{j?9`ayG-KD>*00@Iq(2Z^QwTuR~# z*q~T4vQ!c~^xWokt(hdWYep*xNf6_o`c$QUp$#5J{rkkYm9f}v`b7gLtoxQ3p}?xi`?O$VXnq}jn`_vw3me>mgcmE z2uaW_(&JA-QVEQ@Ls}kjUw(U7Ps+2~L!^uAT;f54ZyqX%_J2WeY0O@lb830Ddd-;v z>)n=?69rdYD9`6RZoDWfz>LRN5?B}yh*88SvCbT;?s(zsQN$=FDE__khqq%c=|k=4 z1B6S8Ni*mrDYiN*@CrBBaM1T0+MGFEav#~p!kGLF!WLnSo->a%sET$`N;k6tqbgbS zc1VJWBAeSb0eVDfSd9~CdZ2b3Z2lNauU57v`Y^?x2X4O)ZhjF5DXMW57F$IZC11>( z8?8QZcVU&;ln%6-sLZOXiMa8+7n(mPmj`_7NLzH`rRNpH^t;u|Wp3%w=rxHs}A= zjE5#O{?F~C@V^u9L<*L`rJ5-BI93^Q78{0;mF6j|o8m7=qj%>Wfn7{{_UcRvq4sPS zY0RdC3tZY2cicr0TfPBh8@DdMnN-})J%2oG`Ap)E(1vSp6KHX<6Lc8Vc3=`28k0a( z;3aJEPf#v|jphPdKa$w8sjiAs{xAq z-EcPE8-yFNVDD{EeXy37z?>rVnq!?n@Aphb7_K{BX<iy-L<=MtWHq+=h}+GEcm+Of5#C8RLL1*@_I z7-vV~Q(I^G+4q@CQ8Sxx(N9US)8Kh*p;k$Wjcw=8@wZdP@kg}$q#rvIt0btn_o@V1 z+iY-g0Em#d`Ye`m1EUq0)JxF)?)T%ct_RXmF`lQn>KKzh;v;5zX66$0!uFH_s7a%aDYXk5 zRxM%eGg)~ADei%zY2w(?Cvzfy+4QZIiSzpsQMAt}h}Lmok3lh;(mA}K&T=Mpp?pQ` z(p1!Vw>&DctfVB1HdfeK4Y_dvfnp805so`p<1~_~Sy(k)ns@#3oostvlW<~tbU3{6 z@?2vSJOjIS^}_>LTKnd%M4^};fjAKY#v}&I|DAbL3)Us_uf)SHkw$oB3F=UnVbINd zxAT5)aP7mSn7yZters9QjR?HEsy3yn){yL(@%Y>vQTHRqwmY959K@yOoBaH4*L1$Y zS%`lWbcR&L;{K~%KQ3Qtf^quL zLK6C-_b2yHW1#bT5Qzd5WW`035J`akdQ*qmtgEs2&~st)nw6+%o|w7Slj9fm9>Im-A_n@nEq9sBPi02V@c_fC zb)|yWh;GL%HXcf4>+`a@E4Q8&w_o%ab0%Jtc%n~O=B$&(Z8P-98g~NYHTnvZdb}dNf!6hFcT>3r1F`Q6TJY2aR)c^%cUu(}8yQq)C*v5K~ zFw74alVA!SE8M;I?+f2E3pBhh;fXFLyh{LYOTwE7l-ux?eagzrR<5r&8NBxwW`WK~ zI8j=J@eAmwz;BMlR44?@=of$2c*=K7gFe-dC=~Z=pbxBfuNY)qvA7CajXA$|U>nn; zKcR^{1=RnXkiVC=l)H=r#pV)k|7>9z^SVBkz{2a&fujXFWGv^YIrL3We)2Ip@RJ4* z&0@012_qe1pOU#2?}bidYB7hm?Lfj54liCw2yG;sUd*bX62P7YV< z6zp!f5GvRV5yI$Z6H;?7w-}FxL7kXU?)ckzd!5JsVaLI96@7-Nx<{{s=Db#@<>&Uv z8JD^kUdyXv@nWx*GsSB-`I55??=ObuapIkWN#Jc>y!bftmRAfS`o#Ew{=INg12$`n z>?#~oT#CUP?m5j&)QG`^sJj$+(N46jE4m$p^Jb58id5<(s5#Ug~yx?&)w8~AJdb2B_!&}Hia;%uNQ2=Lg0Oan6YuV43`YV z7v)p`EbGpSr)9&5S}dMsN1-uK1!*C@8AB!_L^YjDqB+|qT3Qz|qtxVDS_+NQWyqLw zc7Ac5JU?t>Iy1s=N;F}F#a-S>z>S{Xx;c|H{|F+EPG1Nvk4idB`1{1xyO^uga3rCk ztCX%Y6D&j_q_gAs3}*Iy&crHR8SZ)&w>hRO#GRqoQ(@gcF{G`@M3(igv6psf{9BEKI^>nbNg`lmL1=A)mh#=A-R!f0f;3Ovou^ zRrNy7=BdL6F{Al{Tmw&a>*yLman%GiY9*1yd8yTs1x)3Syq>;5sC<0`lV!`#&SPVl zlk&R+6;VmFZjc14H~sGWkX1UL9!-Q|r85mW$($A7Lz~^mtVkSt>!EAG-=|s8y5|i9 zE5;cmtog-gigM!yWKy8-0hZnPN+RnsFkqPM&><2uih(I$zmK1s#5y1EyOBu6&c}K# zPqEd|W1xI=PL48vJZ(=YuWZG)4L_%$qg4Q?wD@Tgf&W-x_ zJ1eKQWW_Px7$OXdV+}!E;KJ4z3!ogFY$ZJUX2ga76YU0L2@FL$m6&fb^{IR?i;=o5 ztHp1J+#o@*G}u}{DIkXeJxT1%hu*w_S=d`|CW3{971*){!;O<7Gu-3eySUd4KoK+HQ!6-&MYA+XD5A4BE`6Ihbi-~X zO3|Tf9Sa<1{7R`n3I4l^DxL3}M8vJElNj=xJpgGqnHDmZ|67N-zO?Kvw#L+uvPK7mGt_-d`XHI>bqlAS{fUWVr!cJ{I@6Rj#C5sC-c&(!N7UE^q50y0n#$&YyzK{Ay5&!aD$*@Ttjmip2##}!R^FJG z%1n_fAIo3CdUlsq61}2l7m=}muN!Mf=M*Oqn-!@x#~WN@WrhUg708XBR&UiE?fy97vsXHr~||0 z9rd%%c4XUoOfD1IUfKeP-=Js<&rTTHQx$A1I0vJwzxDNYl}sq_xt%^i46fiO=wg%f zeUr!6?OVqVQM)?`S=4!S=7X;bue(7XUZTACeCE?rn5gWMs1Z>?)fo2ME8%TJ*4rpI~KsGoSV&NX)vAm)ft;1+%pL^V29e8B(pZ}k+*IsLBUHHPR^ zUh`Ap~uqAO>{Dd|$rm^#Q z$9+Vo(EW&tDg^HPfvtyRD-ILz18IKN&tA78g%t@}-A~w9Bxs7*a96A^TwDT%D^bwf zZQp#yIQzB)Mb3&>cEAJ`Lv`PfhA^k|?!ougVvwZ|D4UgvuUUZWh~y^xU{uAv;)|G& z)}Bs~!WS#F!M%&%$cRC(A=y?O3FQn>ln75CxtctFoEefe5*ZCibA?!(QZPPbp=eL| zD7er&sy(;OKK(s2B%@{!$-ouA(tZxnvFmD!R#at*X4P0FDvQ;0o} z*wMmwSc~+bSBc8}(&=X}Viwf%2$75yxv35Z?lru*YdoNF@m~9E(+-AL>(N*|af4#a zy?}A?8t-|s8`DfHC7!5S0!uN}il>$o{lOD3^-jEa2XpGKokb)Hrw(@sN6?CklehzR zapyG2SKWii_1*QLryxK7BlkS}LEQPtcX0s&pA_-*$O9uO?R)7D)V$xcR2wTLYAZKuKwtRWrbPElpq2baVUonE$NKg?(N44R3 zU@zLdil4qwixu%`j?G}=9e4D2A(>$4t5JlR>#*&w&dXy~=T`|U^0GQCJ`Ne_flZVd z3wAEPjmgH7ONd}$8gyilJ+tNJ4Y5+VNKx5hubklvPYt#B54;z#ec$KJG|pN z>@78SQ55iQOFzDgZKrQ}>~==k0dAflPoiw6Vp|%!m+-=1-4_T}Ot9!Q!&o3RF+z4+ zXGT1~UJ*lX3)b#>YYyx3eOWG`zgGlPFz47s|g-+Hq8Y*Wqr>J|HE3_|6-PFuf#{o)dW)?wzA^$qy62XeNHmg_|~Yb zk=ViU=LbKx&Y@3PJXU)x!Hbxef&^?g#x#ks;naLbD~=2;W@N9CxVWQfQNZ|lCeEkwqe9NB3tyRDe!@?H%{KkRFRziX!<0kN`H~S!aY@h#Y47Sg_yHr z!kEFwm{FPY648rBB^B;9vBEoz)+C8tKYZyV=6+oHGNB6hLq!Em68o^aLN9bys70Kg zKMyQsVsTU=t71`v?;Y7oV;-1Pa194NYxjl*hu7x{$OaA5Em9EabxqA8;7oB@=)_NMIKVnW?^{`3%iM);QiZoe9DMiBO#H< zCLXw?NZkL14NM%9ITnh0_3yC8(^y~1=uJeV=t~JY(Ii1ACH!yEk`8%!@UPs-Jzalf z;14&mjr&(FBXYhn4P+B#G_bSq^e3m^{Ma2Cocbt(Uhq170YWzw$x<s; zC^@zrm5Gt{4mDapX{L$Ye{F0Tu$iErsH)AXsx>bu(vz_QKXyv5IC4yGIiKrtqEE5> z$CFi2iYtmPE@dX^q@-B%rl8E>BUw?Z6Xo*$7IhjiE>GX0aJhxvka4adZ}ScOYRAD^kTqK*?=T+By z*_VZw!EHn_#w2Z{S14dQgY0vTPL|rh@$A%9jhI1MFM-jZG=U*RkPR3IfhXS$%vSXB_;RN=r*oM*Dt4ED*EHiP zIw<$uwC@^=?G2*$!K&H=RkgGr*i#(L!>dAchH#Q^iJX+{&g-Oz8{dLmVwcGr7gUQ0 z!ww$2gEazO_$Co58iADmp3`S43YOA)DeQUW4FnPDcbIVB#6{0j7dpoFdcHx}B^_$T{cAc$fZ5BW_ z-$~%2MHRs@BLs0URL$WY%V7LVeoRGq^57|-FyS~R@kOad7h^HC3Tb?U#3?K>?BnDy zce$Apf8Q>mS~&48Z06AKOVMvkOLQCW_-NdanJ48*puNl*rVd2E{BUC z&=kQn%wDJ45Yd?8et7HMQ`iyQC~-x6OSpLdeUiW*%&NYeN!rbOiD03Ipy^rMVUi?j z`GIMh+16e39--paH79cQ;rQ4?t+btIWOqu(|F@5xy}23PZfDZAOk$(7r4crCwZi~J z61{~bGj8b17%}91g%Ofu3`CPb4L#Yh8zc66i7JeTUUU&M0Z%zYqXL&-Q*o$!-<3}@ z*a!C!u>uzBhQx;w%)=Tqgeh#*)3<)jc+mL+g$Iy!i@=t<9OttqXie)S|6xhSZ8{-=K0Db3vUc3!FRCpJWHE z!F~c22Mr#dhwGQ=(*JnglPT~8O*Vhby!MwRFvdo0P!b4MTqG9XV8my{oUzv|W?uXC z9}&&ywKo;k0u`{o1e`uDVM;6t>N$GI?i^KeL2vRR#oKegN=R`t8TK>56NxsLp&qMz zJT;L>}fBKH)U zpD<2v-Qt>!n0IVZZlW=VO%gh@ZGr&6z5vd5+5RibQcAH@YMDangGP}c1 zbcUTIyfFi3e`r%^eQ#Obn()94Y2N~L4h9)M-5=XoB-~LBfa-{r!L>neY z8tV@u#(A4k@m)mfKKiv)emF9pDUoI$6QZs}#2MFpTfff-!6f7XxTq^zZYJLCgvtKbaV*7;1C422H4&%EUdYUXY9b&pq$3K1A z&Fa9~5dy|KaC33ig~$UkJUDLj(x!;J-wC{68%=~ zW~~{q&o1-qIqXBEG%md-2I{Max(Q;wOoX8HhGmj)(r}pzw3RGm~p13V(F^eFe;D z&iaf<7ao+$gZm6^hdy^WwsYKcpF7rUKY=OWg3k$6C}4=P;E9lmb2*T?*=ibNQo8LE zmI6L3@k9z3jb%JMCDi~!EqVD_>TRxne|G=l>`YGjf@s7unW|fKGfEZB2Ce70`AT_i zV7+h38E)FqIJhPhE4T$&Ioa6??yk>|l`w0uP2!@pNX0AFe9#c~pPb0g&h3`PuBUoF zZQ#IebZ!<_-hP9{af~SLtaZdDIKw$q%x;caW?6T|n?=k>RR5B2&`2}|9DLsjFKLSL z6&3ZSM8TWq2hy2quvh{L*FXTm2gd_cXzW^`G%22Ze9uhQVK(3^q7ggHI)PKf-xmnG zEAh%Oux>CuVy=@F$FAn@kLR=2{S$HpY~Ak^@n_(PSEseiEw)TQW>9Q47-XeAFuPk; zL9TixYOe174qNBJDY{)Mf^3f4 z;E^se1f zMWEufLiti*;9kXdKzsOZBdaLeVy%UQhS@m|d03sVVxF zz5<<^R&et1?pK%#7Fkw3A!AT{iZCip*toyTUS$fu#&-lPoI8A0!GAuu8OwriGJ=;% zTu!i_zfFSRCJ(&&3ey~=-xI;8IjHy<>(#Z5@VMDfd<1C}Ox$P6{x>Ke(}nF(x9Fky z2T$}Q&9URvg(EEoPt!WtS{JP$-d85(m-sn~&*aN0n|#O0i;w?6SmNPS3_!(>i%8K2 zfjGRSM({8{;e%q8p$jtpGI!kh;oAJlyVe5IMMG%26fm7 lB{nXRkN>3C?n_x`+3cT)D(oz4fK-78m)tdZ;zy4|_U%x&)GE;DG?qD1u#?TY$Fo4!h$ zkW^ZfNcKvoP!uJE)c>64bDn#j^SSqW&+`nar2qW=(2#|e zD&uB)kBr{kuW7B~pv~>e8CdT0g?r|Y&e1xDoxYGiSnRh29XX+h*J}$-Hy!_Hui z6MxI9hMC&Y(3# z)Z&MmwjH8=`;%V~>SbClt+~T_e$?7gS}z{q_c>$KHdsHQupi-8dg1+-EF(7&Zda{P zYf|cVIGjFf$UVc!KVAXMFY1~b16)7n%#?uyc=$l+a?4*5u&35tYvywKtjU1(=$79- z2D)hDHQNRg=sO>reZunmW}>pU)>CVNytIX3%foIzz&mY@^K4aq4EB{Vo4y-Mu+~un zXIaX>B-o+aU@bESS!}bHIeiY!g7hd)>-GzWef{;{$#HpXq0-nI7cCvStS_yR)%cGE zmbP1HjUuf;Yf@HjwU^p_KBvb(a;+%(MR_VK%G>N(IgIdTSsr`XvRUHw)%s|yN`n4K zz&bVJ3{H=+gww5T-PmA6k%gK~(to~!pkKYK?IO#~UlFO@wXRwQa#;r!ix}vw5v8}X zvT^yDe-)1)$PZUmU1rJJN|0A*{j@e-+cayyA99O*b9#%N4u{)UVnD7FMXp+q->s{H zK4E0DZ>}Qr%j{d1SiY3#{k6VYn|NevczVEGkc0k!!i|}KRzE$Oz}p_K_Lk+wZAAKD zZJ^dE==9ja?s8|0%i>5EM?g}-hA&Vxg(7~_Kl=DXCVri^(+VhlofE8p{BpQMVVlox z_zW{~dAV!h$Z_-)JHD&4&azFeaiumy>l6t&twDbnv|~#9Fv#W!DXhBMKh-*b7P(?# z&f}I_cF>3P(7I_&Xn|r|$Ut=6=+0Iv$`wDcbMwOYdrmQ7?6i|GQ5e%D(axY9jy)G} zl@~dRYN|IXW9(2OvuoSh&sw%h$Ua(ctvNxuePAR!PFuOtaNZR;G<^T$pV#)DNWil* zoNrog-9@yH(1vTd1RQcswR%9AiUk6G6d%x5E}K2<5AqLCPAJ#b5B^=<#`IwJZdwTS zV1~Y%0jD)&w|U?oaa42qThzGbFf%SYBrF=27I-csWI}oshIdyPKF>F*20Nd(eof@! z`D|@-I1)%*(nRL*Rq70a^#It;}gH_uFm7!g|56DR+n5#bES{ zr}HHg>Y1i(UA?#(GvvR^1<;T;_4$Kd;PMQyjT}*t>(N(@83Df-lj|)dkKE4W`t5It zY?SLXM725tA-BixyV{AmEvl9dt#Un4zy4#|CN5EsX(OQuX9CU`eW>iwN8(IWP&tvGy!yp9rra8SPpCqN!B$0r?zPac&fQ(&qCX7Unx&`XgbY@0?Cm4Lm^zk#m zQ@#Mu$3|(Vm_C0;f}%ceqf!Yb)E=~k!qYv5qfsyFP*ewPgnm*urg?|`eaCMs%qu=b zRAZRe$nJ3mtR+F?xlpZx%GzhQ9C(S>`8ht=!%!pV$yIi%V5KJfO8GNzduA0a>yYk1xEpx;_-3)@Q*1fR-r z%!|>ucGxB_JK{$rq&VUw9)B@dsU$EJ7d`Oa&YtW5E%}AW!~>KWw0Q#%Zkb^!yTdj0 z2g4ecGtYncuLO&pe@8gsK___23b57fvih7(NAgZ-k9R)k#sbBYat#qE1|eer#D|?I z^S+432FVeFR#C`zMLeDur@iltxs8$k#8HL(PWDo#z0B(JLz2UW?ylA24~2wXh(BJ+ z8r^tex2U%p%LL61tyu zxz;KM4NNH!{4wSe%(1WDyK>DK!mf4mi2qpT|3+jF(FSQ9Fn9~Pi;?Xij4-U={kzMB z3z;Nl8q#Wjui+^|QAYufUO2h*9be=mVi*jvVc z&2m@Dxh#xH%X9ZU-fRM~7)3@E0(n99W6?^AwN4 zy=z5drlRgTK{TR@%3!E*`6d-pQBm(+6AUs+^G_0{pcGS{x_nu~eru@AZSejT)Jaof zn>Ke-%Ms&g(?(?Pf6elZL>;CL)w03q2X_n{=NrJ|2Qdx@G5Zx1Syi9SYhbgoXIu|D zTg}TDNvq7S^TbTc?Wc(Lf!Y;Xd+_%$U<1wVw*qhMPEaF>a!(3AS0cB)-1$wWMB4pH z_(F+byd0Pg;WtiusKXh;#xzh%0uA|Wfl#SGYz^4L zr5rjf>RHv1zm*ek+siyk8mLXyi55%}H_1g%5}Sns5F&cLR)0CF|#tzp%3Om|Kb`<;lVX(P2^wJU}@fCQV-LCjJ zSH9Y5Jv)sxD>c)LLU!Dn|ERJFN$bhy4ze>;ia zt7Y|fSuUwUz}INwwVt}yL{VBPlt93>38LTU2!NRgc-(edNKbv1+YBFK%BSyZr+vz{ z_ATGhRbsEPl};>Vq0+M#6PXw)H6=1367e_szIXup!eS$%o&wg}+ zM85r(63br_*{V$e0~o7GLps`kS`am`9l)Q2up`e3YNf=7LABN6v6b7va9RtCK?g)D z+c9^|x@7T8#;#S>Rd#u8p)$}VA-^k(B_vlQ1f>VAO_ZJBW@qp++xILga9Rjj)EtW4 zaIP^Yn-JApo$SAA`aotDYG?#2tgnF8lc@=TP04*}(9SE_p?+233U@)^Qc5m0;53(3 z%=mCq4TTSxlnlpGNlNTZ|FQqC%mu!xCc&Z$+)me_AOnz=)T7*l@aBB(cg>#7ELWFN zi|{d5myUK!RID+jY7hx?&_xfFJjp!0BN9}2dRW*IjD{kV(`FEKbJPvD{CI?^kxy$A zsiHK3s8gX{j>ueg%^`1nF=I4Sba&SwNIkvb=MEcd*Ra!2}fgT9O7Qze;`Zy_m$nWp9cd_hHi+Z$@NN$n)uOmkp z=EyI#mgXCh73R!JE9`p*6P5LH4PkKtU=fgVbSDk6qU3ri^MeAWT$7%^bF{4MhaKm$=3RC6^6dMb04DqnIF0vOla%qTNid zTxGUpg#;d{jnMLK!fVkjg>dpb5Yh#09*@%lR>cD~uZXdMz?3gp-F6LSGJj1&`UWBM zJ_yahkpX{FFZfxp#>hp|d>c3Bmrr3v;H<<(BajId^9b}yfqzHDCyRTP=Iq6$|czG0S4;kK=mcaIqEQd1y0Xz7n4l5olFU0Qf zS8%tDeLm(ZOzC>>os2ohg)YA_4aGmY@j!ZeTKnYe(8OR0o0-S(^+#7^Tr2`Wxw3>3?~|yguJZ!! zfD?_Y@oJQ!VfY29_quX&{z(U1OJQ%63ObETT*BB5v~$ZKP&&Joj4_YA(` zB=bmb${<+bz)LG|9(YsU?7MRQos8pGW-1(Sfsmq63bZS9fOsOtbYx-}qoXIP5t zltjf8Sr=VrI;^O(2^B1`-wTOD$X=tz&Y6>Vk%A@bzh=sGVRIr~WEJcY$m#*4+}%WN zrwUv6Mn3Y=&L>#Pwtox47AadJb}XHr(kiw4PQ8=~X2aIBitr<#TNKruC!ka|z+;6N zI=UX`lezv|-J$tCEKXY|R}yiWpjtGvq>NnqnR_Q*LmKdz`?}1rl(wO7xJnzQ<#_%7 zpT3~UqWon{iPdUL>!1>A>h;)C=St5#t=?uX#R`cj@;j6=hlDPu2FOhrTg%MZJ!t3s zR%XzyZAZkSL2DlfLf;x@pCzt|I=?g;$6k_WXz9Cd?zGm-P*iG9D~Q%$6&o@TTue-* z{^+U&?QFD{l^%G|VOb(U$7^G?LKPJJS@b2uhv?a&a48frO8p*#->E1z{Tp%>%QX7# zh9C8$1?w&NdB5eVEc%KrTDDd<1Vvg?()02Ql)~8h^xlUr8;A>RF8grzl-sAzgz(MTgPLF)f9OHLUVJKwA8J`HuYzA;?7P)D%$&WPf%~A zfsc=CBvDZ9(4VY!%pW{Cu04y>+I1#WjMJI}YDEal1UBg;LT(s6A2poQ0#N#d{wdY7 zul&@QUaZsNYY8u!ilRg(oj%+#=HB&;3pZvfT*!z5hEmEP&aVE-y-Yc0+aL@}K$)ym-rjVYUyKnKItdp@v0WE=@j9oCG5>Zz=eYywk7r7^WjBHpQ&18zXl$uB1>9D8$JA%(Xsq~nZI0yw3HV55 zZGL5~ZhU4i66I}K7Y_;}#EM|_Z8I~mZaV^J&;S1&SMM^9my~8@`{tUUPscsbGdJ|rB`oSVvIH_?lY7nY@QN7mz^e2sjrPahA@Ui zslM2G|E_pkI?LgICs)H9enS^{XW@v?ps5r?W9a^CaP05{EHN?jG9q3iCLo*_6;;sA zu_MK4Hf8o$wE`(^hfWNi1#Mr1E!2LMsepTxK<{*d#= ze3@&4M<+5F+$onq87#tOM2v^k9mz1^~zhF_%jDGY{BI*$}V^BU` zcL;)o5>v@B|D3B|%{aoMikAIp0nzG!pM$-lbW)?q^rAl*l6j-F`J6GUd^t<|Jv4~EL!|x0 zf?R>?=GSZ|%vWmsal_e6lJW;DlGN60CDV=O2l;<@T@>s~rb`q*wn%h+&O#KX?%8|{bHmC92;=gZUvNmiu(I{k1Ohw?Nu3@JXO1Y96cWv{X%yaow;-Tl# z90Pao1N3Q&1h`7;f7ha-g05XBzVyYp|=d_S1Ob_X+&2J zS|__k*RARx7cMqHn-jNf(=}puDzmT%8T-6QT_S|yj>|Q*Y))8$6s2i*vua)&fCHoCOOkB{IO_nZ^*f?3*K5?H! z{GBI>WvFlUGi{lU9x{dq7j;8<-^&FoqDcKM_Y}@%f_g|Uf`Zy1S}+tH2GJNQ>Cw55 zvhG~TeD-_C(l?;bo+rQIf|gOf;r?hQ zpUZW`#1Ur8u;<3*4uVG=UDa%=$#j{CCnkKy3a;Mq^f{v7Dwl!#VE!+wbeOg5b=K-r zYXYqzT76O@*cG_T4ogh0MNo5JPG zT(NKOo!x~g_U9xlYjKmXGPUmVL_&tS8uK+OJR4qTA<2+yiF6T?pj)O-5<;(KN|Ox- zZeJQ?y5z6~MqMI;DG3bSkxm$f$L?wvSu~hbWpYfNaZBGYc|G%|=1wBIg+C=(uJ~2S za($rl$A1{tyG$lz!F7kz1&2C1HfR;YBsg+}Q?ALbv8vyfk1~PTEZ0DRX{lq1m7tx@ z>VfW2LlCPN2}Sdj{C(M4#`CFDh;-z6JF41)*Y3k^ORIj1P}~hJikNfb>DZV5KBKz9 zbp$VbEzw9WtTK7BoExUF9kxC)k`<8OlCYv+?|{(`)CmdiJW@cpa{g@n#=jZpcq`Ew z1=Rr)ST9r1xn1x0nQ5-m5>)R>0ZIel#}lPuHn|k;rm*775BzDU*x-JeA4}&46W?Y< z`^T>*^7W!UtOPZUAa?MB17!C@&n1rdCM|&WA??D<$?s(&e01_#%HxA+Qm?fH=Jb-! z(G=x7@3&I;C9ZS%T0)EKToOTpZyqj<^?yNdY0O@lbLywjgC{Trc5^X(i72@0LU|$I z@y{sI#^D9b@U$gAx+ctA*h9`{CvnagbsfS7EVLbW!rf%(-#ksZC2+W%gbd zeSoOUs;h~(@q!mRw>|PY)5BRMw2;ulGR-ROpQs_8cRqmY;mzxc9jr{dLBfl<2!y9L zJXgq(w5&XpzHlq6dU;EUXwhn{x7=WBv3^>irN;&t&~mr2#oL_!5qD?~<9}VZ!vD^^ z6Dd>*mujNi<5*?bSz;JMR+^`Ggi{6KH*}8{m&)4Kas6E?98?!0l0+)6r9CuN~ zmT$lx-f@j4lZyLvbHA$PO^)7S5)c~$Qcbtv)h2Tak*n1mP zAFLInFsBH;=2$1t`#n<;hN^p-{g(wDkCo9EV$hL`_E>cdqY@1I4}5lLrZ;$Ti8acZ zZfg!PLB7mGt3(A^pB6#Vlg=d^O5bn`Ywg)1QL(kB6{Il51*@_Im|#Z|Q@iuQexEXz z;z2J_i!Mc)od(b23$;p0Y*PC`exSWFj{jWt)HUo(6!{2L+UQTzwW# zxq;D&%yNh5M5u&x)kSy(k)I`sZ6GuZa@53Adw!{LpW=NqHo8Q8UB zXYTJydQX&44oOHcKLT+g1dJ&RmhYT6{|M_6xi3N_V)IR7ys`v!sM|2;X1?2B&6zs( z0V2?Mc>fVgk8%RMqq25eWvvm}Gvo1@998$DC$+zj9vsA_=9{9T?!&StIE(R*Le8+N zSkel2?Pb&9-%9duI=o&2$4OCBuoa!*e=#dG?DOopObG9&ppO;dlwh1bw2*?nxTNKX zxeT=ZG!+zN#Z8kCDS%$Pw#hfF8-9ZX6(c=DZcKK@^awthv*sa&Iej`2Dli3{Xf;TI z&a=a+Ww4T7%Nc|zT7aNVt~<9nS5rkF^wrlh7;D!{bUpmV@eGuT3aY9k?aOHYT15B&Cem`UETem73)kNcoTtg8@{qnxjA{t^%VpBCt5KJH1~Fb6QxBM zzkr?!{N{K}g+joLees-U`oGOI=#-fRDel)mA6V~RG03`NaTTVHni-^*Ld-Nu1pbBUjS__KSN*Y&al7G9SQ94pWvV>wUFp|`vG^sa%U!>4UEhFo*XKi76ajypj;wNI1QiRY4_yJq^Z0lp+yav;4C0%sWiMK5`ck zj|w(RgfP08!efMS|<^8mjW-^iM7=w$;NR9x?7nG?zor86fSt!U8av@fz`ud_asq|{q#~1 zZw#AFsMyz*6-7mv3PC#x_D6_y44OgNE$-|Ve{^Ytm5!g63y3EMq6;Jm&PB^TbLHiX zCqwU3c+$+~fT;}ZoC8e}H^geL8-|s>*qa@%BNA2kF9uXN!6|wF6(6qja&-n1 zE1vF@&{#api$Pw`ojx?nAIW`cz% zhIDo!pTW$&cj%dR9mD-c#m&TYg}5^moBSPqyWuRGZ_$xS_w-`8i~pn0+Z0D3!Nf1nKJqwR2imECQ0$d$6HdLd^~V0%_mS1oW<$} zDMUKWFaF@QIz&Q7F)#(Z_UwP!v(CpA5)eBd>$$xpRzr`0^3nPE z%KUMk&|hm9(&$Hsv?!9HENX_d(lPoUrmfCMB-B=EXnhmEa-?YOf|!5bI%`UV6~~@@ zlnBG(SR)V@xUeQnh(79+LZl+l2d2KPx& zEDg5NPYTGPKu;07D7^u{25aWyM6j^10$bK#xN%ZshI`ygmbUH6gs|-s1c^eJhOa@u zETR>kSm20e-@oAKys9i_S}y@b%!E&^;4Bu+(xjk>F8TdnCFan1pCn4rp=%QloM8M) zsXz(-yIs>`vej_=nHr4pItyt55r)$O*iloAHYUDips=LetvoL<;;xr}V8-QD2`usv zaF-nzLUUMU@4{}MLh!EAw(Z#7O?XP#JGia~!tr=2l>v4rKf%obF%FFV?&~L60CH9W zVt~{{04B;;D<96z!p%n?FJ)?e;nPH*Q1h6q7Oh)g`|JkIq{`U%69sftoYgyKUDT=4tMcu;^*FQDC9ZG!Vk>7VxzAkx-!3 zwx=DXJ7zQJyA~702;E9Ri*uID>v#FNg~~H$PrqLA857y8X9!w|tbi`|hrxw|_jk$? za6{QSmN{}>e)KE@{zL_CEr4A?rxSYbL=&U9a)on4QQ%5S<*BD!4>M+zJ*zMSMham2 z#MLcku;xCT?c^QvnIF{nzXUC!8v#v|>|rO?Tj6dsxKNiN29mjcd4-R45*ng?AjadXvP1DI0 zgq0%Ns1w)iU^?@lM2|XM`U(c#D!ky~*o()0eUt&texArhz;wJZNE{9@_0FkW))JqP zv8y^|vZI&%0s-QgZh?TwK9$KS{qXU?uPlqVNdk*39s*OWOFS0wzt`z-V|FiIlbW)m zV%wp^td%41B7K2qgo|H@`pY2{gpWrt1Mx*~FhuPoPOl*wU=G{< zK-yx~Joc)D#pbb22n$bWqRqna8Z5R4nkKMUBi_a;eugbFP0ZcIj{JeC8~iNd9{noAP90v(sA*b3B6LDj=Rym>3WP_EPyX(}W}x3=QWPE3zHvW(~! z=|p7(jAHp_Dx$6&#zi{^9AbK;`zwT~zv~s_xVTD9zoVD~n{VgTuiiYuEX~&vT$o*j zTns|~a^`XAA4gw}i1;`GjvT zny&H2{)t`L8f(-wIx3u@?qkvisa&u7_7?j!OdN|}rSA}(Y04VnI$)YajW0zh`-_+B zD4v8Xr!#+r^`h5|D_}Z^F2cMJg)sia=RdH9s{hIru%W86@(nOC3&|m_7Lt|IIZK0= zvAAa73i=K)fUT?oeaheul!(?!Xdg>2h&k#YJRy z{FFZZ{vAw>-Saxp9aUp9gyu^GYATy8v`ag)bW6uK2vww8kXW}5Ul6>(En0cwUMe$1 zUfjfWDGSU$lgOfH7m=}muN!Mf=ai%no1XdZ$C~U!dEX?e@ze_CtJg4~0Z-&8(D#hH z*H>vOgo!!5Njau|7c_feCW|6#zeQ_c6q$+i({Xh`e_`^?*n^L`7BFkQT;htxOM$C9 zbaq<+vXbxsMWPtclwk=i%3g^Y6%|yCk$`XZ3B@j%PWDk0rFIheP`$w7&p%^IUm`z6En~hb;NBg2|eD4Uvno){vLo(E z<<{?+MmZ?q(9E?8>JP7<&%0BR)p`9}=iT-+yGdd8`$Q<-q)^|Bm-0P1loGS#v(yFV#8gr{zyqF7_MYNt9)@Jot4U8 zl%U92@yZUEpc1I=8`2QwbQX4bqB4UV^^vk!8Tgt7xQ<9}(ho*`n(W}4xi{JH(ezJ=Y%zQ!gbcU~54VA5q^y)AyMO$)>uoH_n7)DF#TeV8Q(!XAE(Um zkB|9d81v-+lK5hh81bQj&;$1(u}&+!<)Ot1DX)8Y4G*~3~zq|JaL0!+`WJa@zy?a zY8ul_cYi_jiK->A6ho~!}jv)V1G4kix0MUBVHxl9CkefSvvRp&IOv zd@3>Vkl_2g;>pn{E;O3JBJjRzBx%3=KWiqw!L*ZiGm(wjsbvh>9u7j$6+1%s{ws2e z7xgH;w2LYx6*G<6;!A=?joLZ}t!qq($ze#7gC|yd@7_J5FPoqKPy*xpbb;FkqxGUS zMsKV@BLJ5HIj2I&2@PYSOVkw?D_c>p#lDwcXHiSp7WxD|YJum;^aedVc7DHkUN?{5 z3cKN@3H*+i`d<;Q$TC7o7eh|bHx+F>N!skrdhgn|nN?UN!O#^Vm#R31|T!5(qg&`Amy zin8XoPu4n`!CZxn5*J;CHZeeX;RTM18z~b z732NGF?sFOZEz-Q&RHZO#pF6ddSNKWIIg8^j>5hshb}2*Mza5IqE9pqAy_Kj+efYU z|A&QzKS($X3tJ+b4JC+PPX=pbXf#%+RCLy(Urw;FH22+K6P1|eZWV=vEe0inH(?pC zxm9lc_xfQz4ww$ji!r9|+4uT!#)qtL6h1V<6;gQ{x6Xc7b;gHJC7h_8fzkAZ@cu{8 zR&IDV)Kn}*i$#pe;9D*$gT3KfqEf&jl^1fKL*KEZJFo&#jXeY_RJVRjQ>;@Ul+q=9 zS;IEYWXJy%3695K3>%27UI-=|bW4x1WBfILi6vw9i}}ZyVHoipk&cF;9mbG(#N+aP zig;vVGnjbC9X(!1CK&o^Om*1D&AoPJb!zS>ShPAVbXa^GGTH;1EHkPteEKRT8!IKU zFbz7g$e!Bu3DiWBx2AMYI9_s;v7_{Rg&ktx%MZzXy|rDJ5>B`xa!M-g+A)uwWtF75 zKM*Wdk~-)_OIT_)q6beoypp|P(^aqR?sHn@^~#p=-e^&e(uuE}9S6A1~dRhD=)b)+BYorZ)L1mDzOD?Bz9=(g~7kHGG{i6 z$AGrBf&^?g#x#ks;naLbtL)6lVq_nZxJY*M7+~T&lj@XGSeoro8ik9RI15>} z;ohS}w&+n);DZrgg1nQdBGFufY)qAAA0uGlM=HSLAzQ9O%-NB<{)Qeb7TY9Y(Wqp= zy(U(8r_q`svE#<}n8MtTz;8sXa6eR3&?NB>t1I+EXN6kCu~aD^#l)ii?+L`hh3_5N z%iVoXf9au_Wm18AnF%?$;4x z$xOPHMwj4t_by@5@bvQS)flCxC8V&apqSyh8@Epm{lzy9DK=HHCbOH28_2HYUHAu4 zidXWsG{VMfyR@jbt0Q71@mR3<&tkz)zj0rDz_OErPtYQmoopO*qMXIwgQDbuDA}mQ z)+Y|!_Aaxqze-HBu#NRz+xP^pLaxVSE4O9)%kfR1?84$0xlP`!zL53xJ#>=D75%}` zE+H-~OyS1n$zknJCJ&dLQsf~GZ5EdIu&|r#39kC_#Wfg_-%Cg&vZ)s?DH8X;VFMG# zWR8X6Ufoq%{SDTaa`&G^r07ctIng9RC?))F(UJ~%dGN11sXblaCy?EfZCuW2T1Mo2 zWg5sP%4lF`;ptCKzxlCy=l1^X8T3~wbh=2EifQb4F;*1(kU?YR*mg`N<_(`y;~7db z-Pq$q6HECSqWa;=+PunIW~ob0#tQuSDZP^DF`059*X6{XV)>URt6~(NZgXoTW|Eqm zC5lCF3d$Tlk`<#mSuO|G%wEN~{I&!Z8>lyAoNvh6d;~$dh4a_y?RLNKr||4ni!b!g$R2NH(?4eNIL`XE8~YzX={g%yzLW z6fm7ZdO!Zg0;PbPSCrSIK$*!q-ghG(!g^jVAS%Pi5{S)W>+oGpRVd^a##(aTdSoWA_*a!*X`#(eX~?8oS!+%xgZVU|-8c5Y>D zc4e)32<#~i=HXQ#Izu?gQ=%tj%mtkkapPO4OZ+l9H-yw;!qDT3nzKfr$17#%Hd-_S zDgQmE&r}p19sSz^#?S$k2@n~YiGW}sK@8~YlZ5@Fcu*mOJrxHVyJQNOnj~z-p1&Sq z9^eyIh+5$R3Tp9svebxe?CHFb1*d&4B25?- zUy6QXTC&^tbmpW@?1Ze6xWb9YkO1rHlI6+5?yP#`cNRt8QC*b|a#X-GwPW{;es@%| z*i+9`InCmkp%#S|%_YfZ)2!aB!=5WQ*lw3!$lBiir^Y%pVMQAXiRa7r&J%o zj$rK?ge&4(!o~aVQv}|c9(j&QTK$>?EYuJ*J&QX`Qbg_FWW_1Ab**Xi!MSY;3;QlRN(S!Dh|g#y76KL8@yD7#kwK!p%nA51`S~f z`~D{%)MPvuTvy=%q}`&h6>i6QZolKX*fX<_Z)9g8tsbGG+G$0EitmS$k?4jWC3IHe zr4ub~n$2w8VTp^jt{GA*u6={bY1{>MrY!hzTjQpUHhN{3Fql06xnS5k>3ypBm&(GXg03%d0lspvh0<|m939IE?VCi70$H6eo0J8ewH z$LUQvHr{xs*6$4Pku(Lc5g-9a?KZ;*h~k6f=JbG{*cR35_Fg*6>i2I-fFi3e`r+c} zQ#Obn(nkJu{TSLoh9)M-l1H1|vhE?49GRO=v|)0jiT*HRg10FZ-$kVEV_%z-KYuk- zBKge-QCA}3jO)H_!0(4(67m3C)RnWNNKh}&C0yXd+@$8|VEhVqj_uEEu6 zcVg+yOpV-}LFAIX_LW5N-3>iW7M~6=-KV{c>fXre!0j^$80)~9;;aji2V{6~-XQKu zmSs|Vn*)ryGpOG8yXAqaj=1UAcdgOeXW1apuhS-L%`g^0NkCUT?!oo^Xxx}3UR<)~ zL34?_#OFniN96jf{&H(!ahDiMYd(HRazVh); zu?$4*7KDu%h_+achyLi0@N>X3lWQakKd8;I^~`9lknqBTa(i)~q3zJ`iNtq~o9^@M zjJgMz0-n;6h!zSMqAYkKWZL;0$bvjIjZt{XOJ`ULxN|GQ6DeRcmWl9`R09mP6c**G zx4HiP{xv~%CeKJrJd+u^MK_~V(QMFqUQncz=bpX!HYdYf-JMm^;BOw7k=BF&dsyaZ=PUDZ%fG8S|@CR zGn_-k?B=UwmdqA|&M_mgR^o`e8vqC2x57)B;(SF#y(v*}w_6s3nQL%;J0eWD1_BU1 zI1!*iW4EqKlj5KIcHhZ5%qq1fNbE4{3{DY$Um)U{j#q|(b%XH{bDgX>b`Q^O^cQR0 z?~+9;VC#P8s6PWwygIGr9t@X8NHC(C zF*XIjHJ1XWoP`r#efA1F3-3xWJPT>MU2;J-$7AqF8b#H7&8Sk(vnq|>ZF)JjGfiXd zL}ZFqc+7i4%&xz!%_%ik;TLy1-k#l!S*bH&;@y}65ZF@de=m;Xq!;4r~7Nz?w6TUQ?CnQqf^rwPCnlK3Uk3C%c>`242n+? zM#Twp9qP54Dg5swuyF41S%tub;GXPytPUgi#oUAh>-pOh2!3k!n$Apf%*i8Q)KFCX zjQ8r=MS0xpDtrWKEKJ4Gzvh@-QJIfj%Rp0?8 LgEap6rFs7kQc7In(Eyq1CjX zzwRIUxNTeYqXSQj-Ftajz2cD5hN|)^ zL!O8l%2V;T93>vF>d^jV!Zize^(EXryZRrtZFqb5XvFPUsMwWe{;1F4bylgNaMy@B zCE_5s0T(n|hBQOn^UpU$(~ftqypp(o@MODe*g! z0qxbJpr-+PeEPVNLkM*Jrn8UP9{M*?*+;olX^FgaMqtY$o&dnBPWL&ss=$E#VX0Dm zJi$7~44!8TzDuxIDnpbk1G3oZDp&n(&Vuw9Pa9qyiTDR-zmr$ub%x7~HJY#geocQ` zqkXee5884z(Hcccq0+p(!r>}&`u(ccL~^|tdJVhGia}3L%Nb61^K6SBvVElE^;h~T zIi;aMH0YQdRYO$?e_RfP000Q$TbTK zdUTi2>t6B1xRHc@nd`G>Z6E9C1C;(sn|NeLq$+4F$e}<`;)Zu|j|pQ5yzLLQU$@QO zLZlB-1}mLHs@ECuRHz1*#gQ(Kf}})DUm$A=N&FsvY3eX0e)(Hz1r)!|306RUxjo^C z)9*5UhLyOi{IyTBtLZCt@2~%F+a7(5VaiaYQ#7nPLV*Zq$CUQr3a2+LvFgm~$tMTV zA_K3_d(?K@Hu{iWN)M$uEl}(Xn}}{0+u7Ph`QpboHxtu*$C)s8exEQ=7}Is4)sPF0 zJr{797kQFux_R#xnO72--P>OLlHBLeZ@cPx z0-m3#zGj>A0ns{28KLA8a9Ew}@PaZG3j_lwKA^2ioUTY9#6Lhfps5-(frxy+qN42K^ecQ)Pm~q*o!=iC%iRVIxOh~W9@SZZm-~VFXX?8wm zZYOf_e71JF-3g>FS)y;Qb9VI@V)~LN_q=WE@DYJup$t+w_?+dcqaxsP77ObI$ELy) z_7sEBFRtQCNYpb;+jdFC)69_X))zoSp5_mPe8A_Mj!O zNv?fO7aejCi%0isKiBrN4m@5Ns}yO#AvNrAN1a}WD*$H9C4?F@Z^T$J+2wkCWgjts zmaF=o{{`E!Pv|>(C`C$hkV3!H<0w_V(PR1pGPA>zqt1}(c7Uc$8uHY% z!7ukABDcN%)L7duI;36csx%H)dP>b;(sAYn_t+=Z?EMI9j9THeJ-&;`>!uVaO@W_N z9ARgPNoz`QTssWh)_4jN*ioMn913h^u^NgviamZ*_UI#VCQ7KB$TwYA&t=N3#%F{o zlv@;>F4bY!ZimzFb_Al4U^L>WR6V6-W@Bi%P}Wa_Pcm(>QeQ}D3t!Y5@qkdFn`anZ z{t?m{F^TfV)_3)0CTjfW^c85LBv;Q3+78I)xRP@t*^?7rmN=R3oc;x2qVJqeZf**I zwWsFX!~lPl0b6M7bbzn#v{qsHvI2hckI(Bcd$DylQHb^;8ypD`iPT|>UJOiUKK|oh z5-j@oncyj(2WXpz`wnIL+_#57QJ=SwsRR@13OT}&DzE8iG>SPCbwC@TpA<1P@BCFW z!z|3p-%AT%nAgCX)BPpUwK33K)laTq2Ta0MJrI!pj3;eC@nZ5$)7$5(`hWYSBEH>c3T9V>vUN3{5!%452@fSOTZ3KiNmj|?&O`)CFym(Vu9k#2Z(eL zD25xATpNvlZYmqfi@gVWD8xbHqje!YVPjpTQ7m8q_BM@ayZ98Pq1 z9o|4VEbK!3@sifyW@q)fwAqaOjrtl$ek)^*aL`i@?hP!#B~KD&lP#Du^_jm$5chxi zZ22;q_gfqrjNSM*7Q5V9;#fSbim*(r*3T0h5`RT+^uE1A8R7ruF(X*vy2z z>JZV6!rsaNw#r>e=dzn2EqkZ_y8K#V$@l9YSYYe$AA%jB3{&!4;OUC+0uneO-E#n; ziMbSfTP+2qR^!M?rrvk!%b$>x z%7LYb-!6F!N7lB9G8I*Hm{3tgWir&be3OhRsi?l=sws@pLpr9Q6jPp>d^sWkN4VT$ z^8O{%i8mUXmbGm|&1-1WMs@l2W!u6dMCowlN~H@p{osy)<2(sGeh}kG2(w=ykyZ6O zeI_8{jOx;~~f>2T2wKSnRTqUNoha?5SpA*+yR*>lBxL?0L>&2_+uy}`Dwa9kee~I2f$^fOUnch&?;f}Yr)_nea*2Fg$ z+4X)(#E#;qw>%NBCD~)#&;G`6->}B5j3?LTm_4;N3v-+9C}kVROlA^VgWx47>gRBdPoRsKW^g)hV8c1)e2^ht~|go>}gN zzY~QbdJ&NBsOH%KwgW^u*^i&PtKqC+B*-sS?ptFUdxoIW5$vDsyhH znzT=Fr|s+6lXT=+CrWLXoF!z3aw8Z(qb3dMXcKB-%)oX4e-gqDy93lpsUL%Chu7<@ zaDw4oAs3iyL`@YIp~sbpd^ANq>^YDN)KF{NISvJ&bX5P zr&v^Q%OCVnqUKQSfpd*H*@URp>SW9l8+S6ZkozaW3hOIiwPb2SU`ujq4e$RWJJdUM zTydxcE~VsB1OEM`VqSxu`j^CqEJ}uBsU#)#t~uMx&RpO&e-kXa!0j|03Nip`NiE7v z2ye}2XYWx*Sg!619UpUb>1fA9#TrwpCXuiP-TdeBQs(K+sy0IFm=K;G7Ip-qp$Mfq zO@eNX+WKIRnoNy!txl+-G=iv;pwoGH3}I;56i4@Nvb=xs2OC&Tt^AS zTv%=4ypW;`yPU6lug>DfCXjAw99Z{`?Su}0l`>MvM>)^}A!8xT;GmC_;)m3}yZ2=* zJGA&h`Vf)aBKKcIjxo$#Q06ErFeNLjndP6iZVwZc!8IjONehBSK+4gbG|7szw_MLn z*^}vKvH8zBB*y0HI%LoZNxx(Jj56HLh!aHwy_dMg1SyvdT}5gL z?2#de*DsiO-9*~W^y(MPw@s`~z@wE>N`X^&Et;hePM#M+x{%ZBRlQ(UyioIsni~i# z`Eu~fj)$1c*Q!HH37PjpXbz4H_>)?}&w({YE|S*UxZ&`KrpySe)v?hCWI@F|3jI>x z-x2Z2;&$5k@EVqdE4C5MA`6FN;Spsdmn#Z&C91?Hu!`2CF9^&V%GyI3E3^c*hqQOA zK?vBvH+4JkX!#&^hkt^*ZSM22UZL#QlYcYjysNJun1irhwbbdVN)cCiXY11bBs-$i z;IB)RW2vF353<4F`jonoW%f@w!wxeGlY0^2p@qqg;aRwVl1nLtFiRn!71gyp=n1N5T+N4DBv}3_(B;AC?wQ%0n16R)eWD-zyEJzJlyh}8%|$}e zP>|QgRGr(Ow%ru01`P7_p%hL)6(>)tSF)^(%-m*3TO zfvrt5LLR9MSMq#;|4&~Kd}&4{Q)2a-(>kcc(tKWL>Rh@1VrTA6H6!{%WnL|Ps zR0HIu%&leC?B3OSZ8c`lzHLFoqCv|IhM;c^v(GxNg*tB#i(_lqO)Y($?as!`P^?O$ z6+~;Wj13tGE+&>zUv1elO`Wut)fV3CwoOPU&})?ON;er4{8{uR#D{3vqDUDOGRgv8 zliw*RHtieoCChYX$(`X#X~9Mh9{bi-GlLfFs&r8rhM`DnNxHqDP%4ZSzwrD0EcAO- zM-ri*&@{T|5~BIRS&F{{*kd`BHAu|S;S_bjKU27*Y z<-AEpMdjRzb0MYNDezAmA0$a2tV5|gY7{CdrP3$G~7+1e8vm2XqM6?@O24ccJON^OQ zT(^Wp%>H&nB_>z!Fwp16YZxh|z`DyW&A2H4GSY-MU-|QKTkBkcEKu^5#*mVAm`|(3 zMN<=N(?5FGXCd1T9ZZC5=r4vmZWVn|5f1vGBe@LX*c8&U-7mRoD(hsdY)|xJCu3I( zs6-Vb7(i3^e^{uYGKNb;b_Yo!THvrw6tea1IQCcy?-u(bSF=OBRYw$|i-DMk>6WOU zbeQ`KTN7;nqFdQli5Rs^H(%$o|cXYQ_kwGw*o!;uQFuj_~r z&TB63a+P1b23pz!{x0^sqOOMIh7p-*A|*b#b1FtpYN6wmv)623QGEp*SZZrp!Br00C_Z&(DHfyhx~yfX(M9=0zi6B@p;O1ASR}4qWVyzO zYX@y%hx$@Gt$~NSqiGE{zZ+ZJx}GH_ zx)u_gNK8ODFDj~_osA>KX|`nj##cWb!zkUYLyKyI;4*fMB?*_+tE44M7R=~Il!{~l zNILfKiI5w?HW z$PNIamOnALEdG%5#(J54^&SZ`8JyjnJ^*E~2$vBt9#(fKOD57a=Uh(OS)|M9mnKkIxf^V4>7fa;!fmZTfWziz@c%3y9_({2c5ZrIQ*6XBYMvi-*WmhwM6R>C?XTb6PmL=?vjZ@dMtjGI0EM`4=FxY4iG6}I-f#4PTwuwH& zT0XxY<=o2BepP+wJ4D(~EXWnOW`3=9!g{3#{5|$DNosYOBuQ$)rAhb~;s?5vm#zl4Yhltb1B z(i@^V*f?e0YDJ5B6Z6wrue!lDr9XX6zLKXjED74Xb7X5N`RtPR53(##>&ppBWZ{ZS z%utp*{(0UtmorPaQ%4e(5Nm~yrNx3NSXxIsur-Z4Z*A9+$=WRgh(;l6Vk!bBbxk`Z zQOcFnkr{6`XP!&PfrN*iOEw1X;0I{a771{r*8lEBMTOnv?tq!KdtJz!|Ic-3ksT0G zB{uUVfR(ZXtl^;c(pOe7%{OBZQ7bf`@%4^M52Q9L`OZsBz!a<@L*M*e-kv2Lt{hCT zm~?1XRs{|gj-;AoR7#3Sshba+w|vN~?kOEiSlyCfVFB!Y+txq>Hd54uC6PxBp*2Jj*|@_(Z~g)y#_viN-?^sVK~^g{ zrQ>6*q^nb}pu^?b^zC*Mbpl)z@eGb+SmFzbh zaQ}W6n=Bo6C1K-aX>Q^^iTFED63fsI=d4joNB^vYi@KqH-^+O{B1!%DRx{5sL0vqI zz5oTaL#$vZIt*elRMMk!O4euZGM~NQaDqpl-L8Mbc`YM-#l2bcxGWvfml3jL2`>ur zI7z`bT#_ZOD7US&UB*7=z7f*rF(`Y0D2fUQv5K^91IOq7M>H z2e_1uFJQxB$=FgU>`<+%w*HH0m5x`@NYlpwp)JDMMt-P z$BfV?`Z}U{RqsLw7HG36T)wOod+zc&VW!wCM-%z1#Z8CRQ|l$(XxKDYbB;B-p5V7S zWR9UVL`Z^enKnrXy_P9WHvFsR%UMj9Y}J8Lmxy3W2ZruQ6^7xlyBbCoO(s>E9Fu3< zo*V~K9zPw6a|{HkQReq>0`e>1LMGEU~YTP=Y@9UB{Th+z^OIl?K|WY?%U z>h;fDUv;%cQ z!aI*7P_CRu3~%xa1HJwlqBRDp0Vc3smY_T2~5C8U#O{C>68GWpFoz z1808VPs7C~_tW}VPJ6qJm-$=I=__ePdsqo-8X@f92M5R%fSyYn@l9F)x7=`M8*}nU zT}NMnPJS!>_+XmU=O~3az2tK=cYNw!=AiIPT<3C9hZfhlB!ULtJW^)#e?f3*&R$w` z%H6YC7R#wTay@;CD7b1uc|PB9Uv@ukJO)o7U|~EUMiHaL8gm?)(X z*de+o`C``G7{BlPI;=9g{6_i!QJIxj6LI5tFSO*XO$V4BzVaqoNa$fb&8pi!QA0fE zd;r(Ohc>KQ#mcn*)#1fl1j17rUP;)Uw5-gmbMiM<^?J}jM2l8qt>p$=i?!1VEj>=i zfL3_SE#B7r-(KgZV#fbIH%t8Q%sY|7WpJq`$~}%%M$}T%5VF)fg>_T>Wg~iTbfj~0 zrah193!(PRjWuRd!UZnvN;vK!i7nrNu`~aydjqMs_doi5q3uy8eaIN)Dg|x=EiF|+ zhe2%zCXt~r2~-7M!Uq2aO)J5cOKY7F`rWTyZ8R;4kr% zf-r!q73X;XGHRk*9wk|pQj%{!UdZo+JFSCa4Y?VPJ0uz%mb0*Oy7Y0+dw;O) zSyoOoitW+h@W#t?jZyFn?A)J^Ucu7Z!@LA3=0_k-gn%)H!E!bH>Jio@a#F{`=9^}C zWeMs~k7>}&dbiX5(>HG+DQ5o<-yXLu^$~%$Raa(KSDKPNGasKDV(NbE#N6}g!9iSV zz9}l|IlRlYYBBy%SdGYvCBMP%-Pm;aC_j;h)8UO0I8Kt9g01Kbe;HP2z$-sFm=K=U z(L^{U7^e*_q@XXpI(g@J4D|5;ktjexRy;Hbkpk$7nj0pvuEs$@0u>`YLT)T}#_|X{ zFZyXZ!#u_?1)OL#NP*7BjxX)WN_vYX6P=<32y zF8KiA((Vb4V;(5BmiU(&?VQZKt^+!-@VYc$qdb?VmulKb>c?36L04;FC7Z9p^t}Q7hraUoB^ecVOz88jQXMx5{`v74IBO0k7w<&i$FU{FaU<#t*dbg_9a` zI^twk;-KVGyq~+}LS~}eGl@dYrNE1JjJCQY*?6t(cs=HVH=ISF!Ud0b%C(U!uzEP` zo+RquGsix}8pGbzQAJ}|3>9T64DBe`A0gH;X$EPxxU+k0m(V^|I`+;cvc(evu?3O@ z=b|@M+`v*F-p2P%VBH$RN946F%Vdy&MDowqE zh(s5xrN&IK5XF$rPUJIK+4nh%w(nrL_B$oqEKFBG_a-^O@=7q_p<`cbG4u;Dzb=MA30_g2T0>N^ak9NWRRW6U& zA(zUk@`aokAO1Xro$5Jv(Hf$`)>tDfuA0C`trW62&;99}$5ejt0zwxmU)#V`*>dLU zopYFzQgAncih$3k8>GPMQ=iX2$SR%H?;%XAbY?>*nK}i0Xsa8UC5dBiJzPF>_P#u;^3>x?S|RQJv!2Sd)%^R2Ce&6f(E28Rv|v@kEc=^V2AP( z+#FyKx_ox68(0AHKOGPQn&tv9QNCLGaCR29UvN_?Q}Y`a6M;g_Y@fKg#R}HihnsME$!_KcZP%eibR(c?l0Bkgy%p|OQ_oS-fr)m> zBpzOo&L2#vSW1M8Mk;}xENlM_B=a|*WI&z1>45- zHC zx;>cPi`S%xoFXIByuqb`G?0N_(^(g_LO@V^pz_uptTO@t|e5FZb4!_ zetbc2l1H@i#=TT#i9GT0rK4G3zFS8YJ-djE1$@m|LprB4h1j(I(Q41JQ#<=*q8d+> zP`+9X0~+u|jsktpxO;u2rb3vQ)0&hG^?P{FxEvNmc6f!>z$h{c>8IoBhW^6jo3Y2P zS^GP))}QLQqVZDTY7U*t8HB7PJV22s2DIe+g!g_M!A7pC))C#})(63NFsCt}12-40 z1ke}b!YHW&)8!rS-ktFw+ukm((lW8_r7eK?4T`q#?1ZU3Rl>%CbJ$@08)Gu3GNJrj z#}wcARv4m;qdZHH-rcJafh?5^m zx$kpZ9TuK6dyC*iyaVq}2cY~LKTcqIoI_jw9K>=&n{+_T5w(C@_$?6S=v49nk45G@ z%kcbf6M-V<1ktI;jl|`2C9}lx^%HkoJ%?GlCjTNlksks)XxPA6FbiCX%JV{!){VYdxwzm zCWXd6#575x71Ziw>T;HD-lRi`{u~Vohr8o5?pBZ&Omog-PJ?$NQ77~rR#C*<4?Gsp$V7rcuMS^t1hPz?| z(b6(7T*-oNcHaII<80--L@;tzys`r(s1&OErZj{#ofF*~y$rJ5CJ8bVU$X$$5y?&Z z!Kly^?I$oH{aA+-zF4sn?p*{&Mht?Dusd)hlrun5B0PaK_`ZuyGDA}N9ubR%BugR| zrxeT&SxDLwJ_;`MHniua55ByU8IqjM1S|}RmNhC4z~qTwMdA(z$&g6#M#ZlCgX% zkuK&4giyrp@HoFEu3u9821}ka*+xrX@+1$J5WT_LR1Lg4f6isB#k~0HjBl8?`j)wNw;eVHT@~-#kJ49@Z_|QP;g?o{4?)?1La71tE&TL8D4`>av zC25sVl_|!aN9<_fJFLX|(5o3TKd{k|SC|ETS%*Ulny$ftdkt^ynh$6~ynjD`&p3uR z;X@)%!V@Ne<*!l}bu z!V$F6(iHB1t-3U$gdGy~BO((I3BJ!Oo*a$gLZb;R0`I#crVfo7wY34QX-tU8VMvpMCsun)3R{n5 z^V7v26U{h3UFh+{XuW8S(Hbkz2*71P>SQQ6pB_6+U%}a^U6eK6{hVZ zqR}d3OQ;D$ekoDxgBM%iEib=uu;2EdakGPC-$mHMOVA$Y(z?$1PD)~8ir^kwcSDr% zc)t!QcuWZdSJdr9Cn;zu%39-|T)uxba}^eRN~EHz(8d7NFTB8UaTAjz`I9<6{`bLk z``DT3`57S#i*CptKI~!D+#^9t)zfdf{gBNnO^0-7Q847tR=xL7r9Ph)La~{}Dw28@eg!I8sjCov3+8l{}%ioAJ zVn(w6mjosnhY&0k@2%5_3C&qp_@fSoVPPwTbD{*%>&ak^OpV48m6Fby{NU`dED61M z4^fF}?wlAbY%wSqya~&E&8>9nzb^pualmwFUNo3`oBizLj1NQiN_=RFE2Q!^ZsD!r z(~J+NbU0Bv1Ec8=W@!LR2U z`4zzm)vaCAWONFIQ@Vujy3)kh@`6VN`{(gzz z977wP2llETXI;R=V}p*3;*o{TVB#Hj^mrkeVCt)})M3l}zSfjk9p3>WS>$CkSbQ8Z z)&rX?Gu~ahZXlD5mIn!0mL@1SV`)@VX4`O9z5mnQuc;TSBzv{4Jf<>bl#sZ;< z5whbNGZOjrk{I$VOkm};Zr0_y{ClEXd#?zlVz^vLNZgrta?Kl0Feh!yVM0PDt#w&d zF}%rZ5K*U$tx&%SqG|7k}j8+qRKH z>3leH*Ue`+8}$eJ1kBlF33$r-o)>D>Vvdjaz#=%Qn2N!;2m@2|g6{C&)WlDv~Sq-QSj}(%^rIZsA8tz~UiWu0pKY zv1i@`w=tuV_7lOPQOSgRO&st}qa#IP=hk))VeZEk9aXp=GAd}2_=nXcdZDvIF5=j# zN7}J4ed^B%#G(Y>J93rBJuoTZntZ&dZU5bjy+cmQ?1j>h%K@mE#DTPk$@p2r=6>ET z#A3-)IxNPLS#&FnCc*LUUBaYcvrEQ*&M1BAe-fpwKrzF0Hy*z}^cUYaB-vESnr!Pi zbRoNvcg!h*#VdK^VB@u2iBxi2Da~|CuV0$m0%wTohpH=!sw-J#C0a68;KxttmBxd z;+*LHesIgYDPWKwL*V#J>7v^U2d-p>Kl4w56qQ8mAjELcnz_$?cF+UF&&gUuHQM_ zwo8Y-ySg&Jx{@6Rdy0d3cvXnT5Ki)qv6C|HyiSU^@h#jnewn;UVY!&FqRyX-StHQh z)kcm!F?R3e9YrIM^#AAdS&D-H?>0G%p}ndTATl%y0l`9o7|_=z2^-k?R~dsn90zM$ zG6hUc61L)x+uAY@aM1-st?&Q^wRk;QYQ#=nG2tQ>oc6Cls3H-Es9_xI7Ig_usdR)* zPMYn4@BQou9M?gGCewEcE1aqmyE>8uex}_P-Xgi|LZTMqq?{NY+^tfA7iPzgd`Lvw zyBGG9`lFs3^6VWfq8?R~ura}s6C>K^nIcNF@wbCYXBW;1T)$}?vPU;&UC08+-*s@& zqKe>{5rQ}vs^)NyWjOIAKbE3=`;&{^OgNseB?(7{CdOiF71H=7iIZ4j+Q;1C&0l6t z{J7eLDq=eiHgjnArD!*%CA*E=x<0dkosd6tToEH-NPzWpo#n~Gwr$*(KR~~)jw~H| zZ?Oj&$e?>hyE`gbZ2zPC1~FFLWs_Kut&?n3zO&r8J8#)s$Z&fpGA`YY}8Rv?c@-l z;``xbB)Z{83Z12R>DKE?cQ9L5-jJw8TbF^ z*o-I?S$)wD7e}A8LHv;Zebe$Q8QQSsGMbnqOCD{9yWj_w968&9&@egDTzeQX!P}IQ z?;=w7#@DtlIa`YtI7zjWx33&i6>eAVfB&h250h^d$M$-vb_%536 z&~V*q*i?R!$ThiI>B^+1nHm|LL9k@6{iRWScSB2)#iv6o_i5$@D-N?daK}sn#yW78 zIO{^>0T~{gH;KEHWto`kbc1nML-KvU@4bCwfQOEK_qu)jwgWo)P09^Q2F4;N3FwN) zJ-A*FiyPaE7ndA%XfE-T`hDo}h+Ll|P~qrS+|@uawzp;X`8mvCcrA-QOGE;?Kc?T( zBBCO`z1Dj%!#(^SmVsEBP1u-$Xp7}|=#LHyKL;dD?$|t7@{nAB4oS=>1Nit|FymXw(fV1`7`jutLmumiY?QQ85EmM23cwkwD-*KUMQc5>}RTN zVC&r7p1wpp&XaH*=tipq_4f}xCBOc8$vYQ+(}Ser@R1L$xAp2k>vUE+Dh=aj-K4#g z*iyb{%W>Atcv6QE-HgT*0M}d+m~~73joe2}~#w>!JiZuzw zp7_%~0c$4kuGK#{1QM4gHF$U;iz?RZOJP)zjmArK4yFWZKl0~qtY`iCE=0e0tx$id zFnQW7?Z%!C*SeV@t;(a7(2%yU?rLZr+t-V=u8CsNHX&b6cVEwvTFj|w*Ojo*sc8)- zAMbvJxnPlH)e z8f&X4pmpp@SEVW5S0?6{_&G|<LmR*8Wfd`ljQp?bSqyG=D60}PI literal 0 HcmV?d00001 diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569685.gpu-51.2674567.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569685.gpu-51.2674567.0 new file mode 100644 index 0000000000000000000000000000000000000000..9956aba98c5b814c60e5e43b87837621b50c77fd GIT binary patch literal 48602 zcmb__2Ygi3@;^v(FVbrQDb!FxNFg-oO{EtF6=B)zZjvS2*b<8PL=dH@h=O9lhM*`a zJ_{%!Vi&s)6|hn4hzJ4-BK~L2owN7M-MzTW@1OTR>IdWZ%$+l5`kZrd75(STZ5OY$ zjrgnVxkKak3{0<65^{L_c|$AR{z&h_@p)Qq#O)6ULL~u5$dwn4`h1Si?7W#FPsAO{ zbK`H>OFdq$#V|q(6~D07)_4!0U8wcavfO@Wz~v6v zQ$&5I_*Cd(_1iy@sC~5ywJewW?5K5Pv{oM-@VgCa|9K$r;y}W!vi{Y_ZP$H4)OOdp zY0b+$E|=SH4}0dg`Nu1Og~i?T4Zt_wdgcD11bFcK@{P8)>CU)TI#na1KO)c zVNU~ea-V-zUqqlUzV_EWw%ro6kJekuMqWB1u;md?0N~vY*IBlz(12~Xs3`MNg0+tu zdZTUqS48V2+C^HX0a@a3R=E8x&Vm*(o;Da5iTDTUzmr$$b%e`}HAZAMTQh*x=+Nxg zO55dM(;CHEk=DGT((WvG`2B9LiR3yl^lAkvE51Da;w>Wx??&5Qci7JVhVTYx{j}C) zp+GcfpAmJ3W*aQwbSqnTX(+15!sESfe0DHFKfSQs1Ga4vx~JA%%Rnycz+w>-y|rWX zR#i5xU#9QV(FFPW3pJjwRqQ2d2Wtbhwm!!!doU38h<$VWO584&$6sbbt`|eDQB>HY zyMq3|iH}wtMd*EJhDA9J7_J)p+HdKMpo@_{x_b$+a0X6$@Z!Q zzepRZbq=|`j)5?c&O2qU9s-{rHuTLZURwjNk_7nLiez^%&Kz_MA;fTZU zG<}AZxU9ee%je;z`M z489`oZrcvINH492)|?h7afD4oH;CS#~W1O3&e@&mlgmKolL^uj#3rVy)I9#p-QkN{zU%n8yU>q_1-UoNTWV`Q2B6^rMMC<5tRJiSx0jHxxST8s>m7cJt z1dM*kY`%m-J=3(e&V6t>Gvt^2L<^uHPxl8xKH%~kv5g#2k?ZlJCX9w(G~{~6)X*s= z*V@m7igKNXsCIWS?C}Qt98}r90(X~u#p8JD)f)p$uAh~VLarUbpm(~w62|4Jhp#_8*Eb#60MhN`yO-9{@{GJ$ANPLPA^kqTYxHgbLj} z!|3vlP|k=+l*i1gzn__?tr8qfl;Y~SK-&TN99MB}6nkRdxZ^|SJ3EgM!RR};AU8J! zz*)DA8_WRf9aR9+bavXo*LPT}uzXbkKc7496tfqbBp}+0EN~=1BvOan&}r}j=HuIc zCpyu`&j3&P96(pD8{udAyyh_iMSb2@r4mf2Gh`1(W_wLXqhZXUI3KhT`biN(^Pary z+Rs^-_q1F9!@QcnaUvJbZZn6=;j@>xBaQ+tNeWzk zUN3{VOWw-(WdNxH5hiFS2oA#p1Fn$6DTfK*6uh3rYjYWSTO=Nm*F=y9W`?6F#|KSJ zu|eL1F-vwb$t?SW$U{+23zxatW^VW=0g70|=W&Jt_7X?L zS)L&HRE}eTL1U8>xlgbo{+fgoN4(4%C;=;#1g7GmA6j5tCL?$S;k0y#}ZIPJH|{Bj5R#LVjmwx!YM`FAYGF!-4Lu-5Usp zgrIUOI;Uw3B)^rhMmXrH0QUx#;F6~Zv&j~8nbz;Fk#yFM?|kS1 z+a?J;Q0uR?HlTqiWr9EEe1bLh>c`8{ClGd>Yep}(+5aZ8FV==>9Wi(dc}kG&VT>^B z;Qf0lg$tQ9`RC1jCzH8*HO|ly=|%q~*lCkNQm_mVE!>ikobc{vmotYQwcWCwRIdynqBwNO}$+G%=Tg zZ>y!iWZeJ#0;b;ok;|ZZZyXE-Fy+U=toQNpc9*}*e5og^jM8(0B8x|8V1wL2_zRd( z4y;7{1&YUT=Z7mEU@GeJs)ULvDubcM<(pJYMMX7ATl@v1)OMaqDW*Ji`Lag>_Hc#A z^!e<8O31G`Ok!wF zdY9=r^_aSI*CNtU-DR6l?aoqD+CvczE_~rZ9p|#FN8R%YSftd=zz%C%E_9#l+ZkrK zKb{rWTvkx%<+u-@{^znWbXdF>jk(=+n~mrlq7Bm8nduFM?XGxxYt84=+MX;%c5f{a zJ4&M7ibTMcWWOWa1W9}a z-tTlOK2G~dJuhdcal-{fHlD^ds5+d%D0nxJytYUKDcD@{+TM3CjbXd%DA?`YzF=gw zJ?e)wV&~lpRVyz)l8UueOw7E|14gssQm-yyi)4_?4fenXREHxTC;03xld86sk2$w( zt?MN5d%B|0Zrh6z_;PKc)?4?QC`v1b5(v09A@uuPK`;|RugB>K>#5I5hv`Es`E=~b zTe`9>cGaVA5fTqcPA537;7bL;^h5$sqDjdQTy^T^K8#gE>np6v2B85rBs|;iOqK4? zYrnafE0WcKfYB9ct^=cCHdzK`N0r#?)!Xiw#6qQA5)DJ8bRrWX5wF{XXUW)s<4;v% z(c!HPi9n1F)4YK)i%(W4R3fj3ch~szxxDVAeS*1f z_1!dGB42l?%;s%E$aZZC7(k;Y4e4kTYEjIE2N~2*YJ#cNJ>;yMEb<&26U{S#i z2`y?4B_24}n3GM2YOPN8{&mL7%!0QWQ7o*lfYpNeE$mR&ZBDqtT@bjG zl1mNv@rRzgo$+C#1Qw|PN`_;pBqjDf|H!AGF&FsKG@=(>;P$!>1sQ;}q#oragtzAN zmrw40jpgc2NPNuIwLm*2D%O}%HHm~Z=)+Cwyv01dd(w$$;pt&vM=%9JBsE zF20l*?mJr&g~CM-NI2md1WHT20dqRhn$+lRjjJ(b(k}*o12@g_sMgEu0rh zbWxz@D@Pps^RX*QH#G`8yUn&=!jI8LY56DzdLU#hgc%(4aZ>z{X9m8S&9XxaGwDM_ za*N!59XZA@S7EumywH@aux8d5qgOR%qS7Ht5tZ~HSOlaT-AR+ID0|EG++9^$EN8L# z4he~|c?$^{bU@NCSvZ1TU9Rhn~2;%m= zkvfxUH(OLaZ;7p8D*_&?jn)bs!fVkjg>dq`5YmMlUa#8=R>cc7uc*0!z>+UJw&!nQ zGQUYKC1l6AMHUXl!XwH^PG=PAN>qtYU=^)N&n?;9n6-zjYeS@Cdq@YDI|uQ$kUoJXG~j_ISA`@mpPoXQ^ei;=jqS@k{wZM_*3FysiCP4 zvccf`lsS`S_KKTMf6pw;x^_e-T9~XDo`nmjxRgp5Gp^O*b?h{T+7qZajRq)`O|8QT ze!>^F-^;C5-yFge^-&oyh9Ovi3S+Y2{sluSS2J5!vx7CN+2gl7b$d??9mdr4hY}Ul zbwSV*bfa-KA8wIg`L{rq2csW5^B-q^%{?87e)Mb7U4>B2m1>%cgrcD!uZ^iXm!YAK zTs&tL^GL^ZB3R*(N-J;{cuU^=dd%6Dar~%+#(m62NYN+-+7&uLyis#HGO@({y>b5| zEJb!lXCfF=WL9V8FCRDJ%0Usm|A$yG;J7-SfMc#bp4=+=uy>n?1kyUU;A*%!H`5j7`LqZo+1LUU6t!38i zetG-R@0dZ`BZ1MN&N2QngynozWfK9bYX_#+9D!dv<9o#kb&T0 zVkz~93(vggpuMcJ>{gepT{i-qs9macQ$fL>MPEXEh@LHqltUq-Jm59?or+@9zad|- zOlzxMa;!Hk*l^{~`)xbqf?c&PT7xhYX)Q@FC@fM6WBm@?@FENSe8oh%2>pbnk)Dqo z3JESy4VZ7_@YXx}MfuB{7{8l#SNV-lu%TEZ1O;YIG<1|G<))qCy}ekQ#tRY@+ca`i z&{>X354g5eXH1srrOQrz#Q5vzK~y7u>v>JFnj&vNXnwwumO5{s{S31cHF^>f+KCq4 zklsoIA0O99qM$mVKUwXVf3>Y`J&V&Gm8ck}WdZ8UFqjE!(n*BeBz8V(yJrQV^a=e_ zs%QVm&goyUPKz5=>xlt|Q_3J7%jlKGl=I5oL?$Zd zR-6kdA$R<3eRJma_P&siMPi5{Ct!l5q8OgNY(7(*`y{SVoQUfP`JkI4aa^q&Gn6ys zMEG|6dYV-N=l3Cku@czQ08T({VlTnx^#0B4MowQsM7xn`AST?i#F#ftf4u>VnBSF{ zm|VfbK%XD4VU&~t>n@vc=8{kPk|vyX$&p`dv-%O4g<8JW2vV|k^J!JMXli2JHt_!c zvXCvKKY@vm4gJNC$K^&}RD^>*=twSyI5vg!?C$WdH?mH~FXR%~$=DSGDpAD<2GG>~ z9~P<@vHTR1h&2NgiAcj?ohW4MU2*KO6yBDBnTy#W_6{UOafl7XL`=6tUFyDSG~;=n zK?=_sV229V+Bq@hm2uW2wGjrqI{I2(8aN=A5TyZJ!orbIs4fJ(Za`k%{k032BrhIJ zR0~K*NC;FBv~cnjNbWj+wea#GOzCzSLXct#NAt+H9DKtpYN6wtg6WU={OGUy^|6qjvPeJOw4AFuRkSaB)d^uGc@Gvw@|l7ZJ%g zu++}9g0lj&QGDvoQY=arbpH=ajn=xD&@naI*@R9Vi(-+udXeQCyN1jyVTXFXTmuzp zC({~Un2>n#ZKRF7_D0 z7!sxWjPqWQdHF$}!>V!DZ|iOA_u;=RaLpvfzM(6UhRQbnM>~Awd!?2s{@y z=#mq^Gym;|kwmHpu1&TO`XEgyZ)^VgdzEFgcE7?=gpBQeT@cv^*#SV*^Ct$E#UFCs zSTFNT-}`T8GPqkVgECl*%ZL~ct2>k>6X}|>E~o4)XIIte$xm6-bMm8c3y4++{2c5ZrIQ*_9v08!P75cV`KEql*kG$3YAC~r8HG#fEr2WK# zT!HK6*J>xMSDL$aUn3?-7hbAJQah`aEH|1TrVI!{l zqRB^$+G{WSH)`WuKV_RawF{STx|ZdnTTdi(F}*8z;RwTp1Q5B}jcx6FauM9xR>tAN)^%sZ4_&yN*_kmL-k699R6y1T(i@^V*s$bdH{|ZWkeHv| zX2Vssd6VdK^0hpzL20m{J4d#bl8(<#dYxs7MqEx%A`4eiYKF4p@dNepA7z&CltdDi z5Nm~yrNx3NSXz5Lur-ZahxB=!$=bbF5RF3C#8d=M>Y8>+p_D7B$#cG6$2^yTR}vn2 zE?F43gCC$zTO`0$TK~Hj7Z-I`y91tC>z&G+{{s?QWCui4iOqZoV3q6uYdCo7!N{LX z^DUfA)C$dKe7$|92U43e`OYg$z!a<@Lq$siX)NiGa}~j2(xF-TY;dq}B-JFNN>W5g z-8}O}%>c8yZ%Z&?bxVUqg|PQ=^R1jzYXKN=WqDub?oXdWw2Dp$y=6$hQlY|4Bf4_X zI@vY0E`D?_OCqCuM(xo9bSz2crVdG?JPU1d^ z_&ZM$%h2J@het3Sy-|XTx}m)9oK9)0!#`3>i^jPe!BD_TdGEUhY`uMo0i2`>urI7z`bT#+TNDARxKw}E|5 zA1AFNCXO&$hCMf>o)CD{u~n_6nk<)jb({A+R&f1BE+Y!Aav8V}=Kr$FvHwiFgthuy z=c11htv&|Tq6@mx6N+H_M(nE+l%_2|=*{JWMlw&Jt(%q-O$WG?#22t(v1Dwi6n5zM z&uf-3Nq9r9BqTwMf9g|}`h_-l81?TH<5uQkyX6BE^~zo3j@S&N&5m8tboZ!~P0tFd0A$$J&|^N>WYAwm*#%k)V?=(S8~ zvSHd|%_nhPQclF9E)l_$1cvTNHw?pLcQuSGnoO!PIi}9Iwe%h6!ThP?5?=UIlI4nD zl`Pjs1e$-yxPFgE<+{sV3Wqv2HfR^aBsg+}Q?ALbkrQxT#{_1;bVXoV>6l_AXs5G# zp?lO6#41KY(R?SSfAS!9@{sZakM~rvJ#OURI${ zohcg0g;gd`mUF`rc0?K%Huy?^|2kL}`cOEIATseQ-aQaIObax!64w%4t zS%Tiw=jM}4b6w*jO7*T3pfm`6JW(oUlgr_53OmmHz@LUoOzx-ku`Hglvpe&*vi-D@ zUbKgmpr#SR4t{WeoB`;$#1Y@51#s}8($ATb|C+={C%=_EKA0x;*~?%~FZmqJ_Vo`o zvs3sbu5y+VD!l zuB2t>Ps>n%6fTCAT|Xz6i42DH*+Zt=F}|9xM5@;>ALe=3P?%27l+s$E3vZlm^Uk7gnXFx*^{H^f)c~czo;Vxt55tXEu=ftA zKGXvfx`){w#!7p%$>V1gYuq;I*zmh|ib=2V=JO9-a| z^%8Wy2ZK1Q>w~mZoabq+I=VIOJCE6(Ro4*p!uC`Gs70fWCAFEACw;7aruTmcQrrVa z)5P(kPu4`%Y4>$D6X$OvqG+E{5be{WK9gd$q;t==?f=WfZsE0x*k!4xiEepRWLZf` z_G`GA-wD@Y9)V&FxfzZ-*b_98saaSxU0U+~w`bV)yer|v_ULeUsXgagZYGq%{R^P$`aI}9@C(k^=@~3@Zu`V zh(Q0r{lC~=kbn!SYV)dUP05~_kIz*xbw7S`&bjp9ATBlE6c_g#(Pfgm1pg@Pj;M;o zcfpvGY&yJT0evq{hc`^%I4Nohwjvk)XIP<^3b$2ZLRjm1LK88XV4Oa*kb=JGRFXHC zfj-GVK~_9836TQm(QAhP!MYm9EhI|CNRN;ki=DANf}h?rXd%Nq#xMn(Xf;TI&cTyh z`K+Y(&<#YVXaRycx$fNRTul{y{eUAi7;A^$sP31bSoafBAi4R?H;-U!I3hvyX(`a@ zc=MyUp2ylOaaZah-|KtWF<-fe2-U~L0&vF{Qp;T?>r*-Hvfn!~yxxlyyk>MeZkhQ| zDqG(zzn60BX~XiP2beSQmBbT$!ZK%_GH&&s8gVD(x+O%Xz?E6uByhQz|E12r+gUiA zvy^Z#9ERJqAe{%7e1LH2_XNjrLQ(N><+`B(e!FH(6~@?I5*RnCx!xlT^8@B2n1aU& zcO!B)A7))li*6!9MHds^C4jdj;Y|d}ZTQMQ<>wVB*H?VDx9-c#0u|m&aH6yb;}_6V zf!`dDsZa=*#upF#uwi4SL3c?=alZ!ozC3O0W^ErbfTg9u@C zvk9p=k5`OG!=O&wD0kxRbobc5m>q|m5?xf?jVqx!uN7+fxt;t?*${?z^=(A0f){_a zoF(3>_di?C@Um}L@qBK)b1((Gw0Hlg&b;NYRF-alU;>_ic4`>`-jId z6E%GWQK-8Vc+rm0R+l6j!?&*djk)0URuZUi!6TjueIyI49uB)FiJJ52s8Osj>oo!v)ososW_j-z)F+2V zJZb50!Bhrz&ViI68e1xI8N5Fk#TacH@}~HuYX25?!Tiota=EN+6w` z$Y-#!?@v6w@GFK}bG3q-iRlV)XDBxLy8=$rS+?Gy)iXUi8SVxZHw#jxFun|ZUcw1a z!Tqk*@3s3`VRGDkL@$+Wzw3Vb0-^Ht4NR3ScQ)Cvh&d^JA0SW>l^At{ z6j;6S&b42&O6PYH6ASuT&`IW=1wORZjm(O~v9}(gYj!=$iq=aXBvQpVqlC4-7)?=b z+<;uVdCv%z-RS-hA?u?>V3?fHArdx=fhFJ_746el=i@#Jh@FoOJ-#x#smDP1=)yu} z{`jNbOUphGZ&>S|JT9zWX<(t%@HeB-B=EXnmatUpZ2=c9G%V*Zb+3-&t|& zO9_X?v8Es{aA9kl1yBx7ff63|o&Q!{Cfe7pCE8H5Gl=;XQ=iHQvluBn=)zj8G?=rF zK(RE~RzE2qhXOrC?CX16%kTSpUt$XjE3jn^h8rhEX1K@Qd(M^rV?sFV5h53bFb!XW zfLTO4KC!?Nt-k-=^|w65Vy5gz2~gB_@u?M@#iCi76co{f<$F&vhweQIj1FDfc;E!% zS4ss+@ZVE9+mNlir(3?sD37dHC~rjzU`I^}+L-v7fx?nXkMg|0zKp3ynQ_T@jDSVn z0PeB_Lud`F>|Mm;R|sA?d;T44?_QUR4Z}^6YYvgY;o^QFO$2k zB)Vu?7wE~t_Ari|?ToLImH3wamx#s^U#=C=@DLOn8Iyaf$X}6iVit|O0 z2rET2?=wfnGMzc-IYN&)UHS?p-YUG{;nBo@;Eh4zaDb_I&)~9__=Kb{ z`tv9|dh4Di3i0S=BVe*mWpYZ7RvtgZvUsC65wOVOAuz?d#A6Zv``j)MX7}PXsU=G$ zHQPIwwQ?Mh3yDU!_=TvyjC05OpkLacc3XB7S8k>c5v{w3FM5L^YA5rU~rTh_|tdpW%qk5_31kk?%iqcXbwVe_o&D7!*Uyse>8JEOBg&Sp~XD_3e_T2K*4;%w-+tf zxUS9EtJxZpUncTJkGir3bsv*HNacFn77vxyXX1F?Hd;e;rYUQP>wswzHNF(3>@Qxf zqj(a512d|z71qVAfaxH*2=iPNLh}bcD`yQ=!(X9qz=o<^!u|Oy-SUM*73mfv*5k()1gCpMD{tIOWtPa3 zmwoaz3(WuX8qq6yb`cp1_`0!%bWT|cvFW1^FWAIRRQ~IPjVDSdU%iF_4R|6)fxc(l zy}n9QAxzBaP0EJ)eR%doV_6jWsayl2$V{Z4j;jm$3zKigp0r?cIkVQY-yqsWyNxt^#>-Dfw$-r#NY~kf-W{G-#1yW z#lb1;5H)_Akns?;)R_;yF1+preR#?8;_BNwKEXugMTr^{6;zGUpmR37ZD_u_Na2Ht z$JgIhXCre>%iketMQ%rICA=YmZE0T6z>+r(FoW7-2LX#Z4;p9~8jpy)kxSG| z;N~Oqjk$`Cp+rB={c`dxmV>DAEOqe#&uOLo-DCL6ZpZ|k}C%*Ry zPQ*L#?sNdkzwzS)mdE+@`X)A3r0)Cy0b-6Q4Q}DLKvbhs#RnW!z0q3?Z%YB!c8FQKxgQdJB0mIp&~S~jU>3LvncbD}|HTjHl(TKg`bgQ9W|bCb zR#e`7*M=iZqr5KRP@}XC=?|};&AU^P)p`A4KX=;BZc-@wmAR;Sq|kR*MG^Kt28K5W;o6)vEr?klghd5sQW-Qy~_o6wD7;NAJNE@Lg`g{O~d%v-%?4}FTrE#W$Fu`|Adcs5R1 z<8PgK?|SCRxA}tbMQn`t&_L*gdy#SO{M^=P`r-Jc%$B?@*FY1WJ`<`kCD`+b9W8u^ zwOAi|6+`Ct?LWDZSB-gk{A?U#SnVEoriJFSyTpmu6yKszHLD7s=t2;YB2 zZt zHAZi&KqCN`0l8;D$q5Z(qD#~j7pse+U}v6pFoQ)ctM<_+=urziPo_8M;js$@tn<1B z{8rcnO}2DmS;k@e30GtpA*G8Ur|6rCHJ&7GcKc8JXg#wEha@;!g)9X%VaTr}ihb~6 z3%up!Hx9OYNy9np;Jk2v2*-nyra#W5cb)T{RK&y-!Totd**3=Gso$zRrUZgB>T;ly z6f_lOt#Kbr|7s6&6{>wlxacahH2~#>7dS3%VzMHCO2@~}rQJg8%xsp(!lE1UhYx#L zHTNjcO7(Qd*60aVX`1#u5iJUa9NMb)UeQrdK~)0JYST`?!(823KM=Zbb&1tD28ttm z$!_2-r`EGq75!&vtsX~g8zgF;maEl|OA9LGaTOZ9x&KHeTw{ME@=&;%Azg_gSEWDz zW5sy?upzH)XD?XEnsfe;kYaKjA$>3uV;MY+i&+x-TM3J4?$$9_*kVvJcoUZS znp@@8e_sIR;gbcO86OrORQS*oS4ib;+@rTH9mV+2`4@!`8DKR1VZ8ql zw3P?m4Yd?Y(PCjxS%1xzYzF%=gGDOOucx*= zmv8?+XaR9kDh{r2xloX}Gx5my@8&Wm?K_EtPFkDt*(LBohcX~0UT5amOy(3n;J?o; zxtHm>JN_ghMUyT;s#F`yzm-Q}ZM#WcU^!7vJ_YCZW zMkNF8HL=4xjrJ6Yy~L znMt?O=n@?7-X%;LZhhg{NJi-v2`OwUC}z0s#^aYmfANh&icM9li8r|58+IkHvpP|V zSMs(p!^Ug7w3xQ5Ct@Y>Sg`odj9_TblDqe@?4(wM7QyUfvydC*EdCx8B^N}=MkTgB zals28Fblg`Vxom@ruW*$CwLWdynnbSX4~BLL zabaN!H#T4M_x3-SJk+kG$U_?1EG+L~VK>e-*jae`lhbc~>|Xoh^4A&kW)-@HNS2Ce?07L&6#T`*E|p{3hD`K5nO*ZCN;6&E z>v(h96pg69qpEg$RV}l;R8PhV{P-!ove+@1axT~97*DbM%ac_G#jpJK%+Jgu)vQev zi{2EJIea9`pgLJD*WEX_BjfVp5?E}Y-jH#&A#dvq3?H6WgPoIc7ZA-N!7Kay;Fft) zz@R{e!0}hoMZ3R$dki!DCncn)Bw`047F1z8=4>Qe+UGu#?;pWplGSyHVhl0c8&@b` zI)n6n{EY=l0k@#IpjVMHllObt@NR4yhtyTJF%ww=v6-Gl_Psq`9RH{`D;i|(R>vgnGBN7z=>BYJPHsx7Ii zWre|>;$R+L6{0hQlRPDMQYM_!Nf9@`g}cTtlQ%uA785Sr_9=hJu1|d;R5Su9|2?PA zQWTavd-N4%p$|$xWN0P=f`tSzps!C7c0v2WT^Z~h4T#PdtZ~T{Ff~cozaDvEAM*hF zH6&Q!0SapIda~4rUA^Go-7Gl$R-%eT9HNGCtXtG2IHk%FHaThORZINr2rOm8V=z{J=J(d2bI`vz>QU0L!^}jRWxUi`r94&P*7E`N`#y3fv!V=RyZeG@C7IWecNL1m(d$5^9zb{3< zF)i6`{AG%#B|9ODn-R5mLNYKUz zNWY(VGgf>mv5^&7l4Ps$o#n=T*{aKEhPxn*XcoAr%i$sjG(`z#%q?-NJa*_~b_6@6 z6RwDF2^a6bPZ4<0(%f_=Y41y5p@yL8S=?cgB5Lgo3y-s{o7FWxg=GFuWhN?R6TLsvTtK%~%H_~M2`6Br|2m$1kPNiqhasi3aw zw(l)QYNOZ%G5;`mK(!4j{S;TDJ zdD(=Ewyq^oE3SQm%4ysMb(Sos|Jv&}vV*oxf{KF%kI%#P%XH~~BJar(c+Rk%Cz;ni zvK3K_UVA%~1cDV8i6u4|i5U^8U-&ii+K)+lSPo7fmoOz31@&wkvRxfM z88m>rNO5uM>efW9xJeBAnc#^;hsRWpRlc5Q_Fv4d@h&?D7quZgJha(5)>%3rl@6Uk zC3~{Jdb`Oi;q{Az6%C=4j*v(1k&4|@Xnn#s!4o&%wuiN;-`180M(?y486USV>DYKc zzgY_y;LvsoU{gQ>jyfHt5fH@($O*zINu3`};E`vR@+V zN<^G--M0+}0uW3>9)OFwa&{C6dhd&ePB6h-)PX1!zKgCqbX=D^Y$`t~DN|t4E zj>84U-5pZz`|b6`z;8Wt?7P?QHBgN0-8irH0p>9D=u8WXNI?2y@|G4674hx0-ium8I={m* z5MN4c%s{lmay;}$hlQU5o|#-DS@^Gt(|a?L!+nOf!+L`-S`(naVt(5zl0XZqcnMRWuv4o);A><+&$+TzCmP>n*z`z_p-Q!R?-3SWuwg zUNvI|@6LEt;-a<4z$?`P&=B^ooG30R>Y2~3r`qCM*S{y7n>DkqnPi)uPZW32I%5-@ zX`2+YTd0;2P`&M{KQ=G4;s5d1F-aBE|apoFqmVm-F5P@dp(r-;8V5cST+E5pFL!T5-^PF5VdHkViJ!CLog z7SRgWx}O{KXW)rfx4qIUwoE@}P;53CWTic@pl5#fBK1tnZ2Q}8w$5YfI$aa41Knsd zLH+%UPsy);p7vm`fjvk%e)w_66}Hjc=zDUtPFjQbSvO@b6}I#&a!z91j3*?F=w>vg z0J!E-z?8F)xz~G|c}|ZO6HRy)(sa923fUa5$s=hRQ}Y+ZlzM?(Y5d;a<*)acrun2h zA&XXc%zH!3uD`9#DYaJNANK8YHM<*gUJt^=yD^Jlr(#Wlu_ylYPr#Z9yleF@4uQhu zTR!OaHH#{8deTxDRb-*@5}ku7f%d(@af z7m<&KG|jrJp?Pdy>*`$-C8BLYy`Ju}+-vGGrzWj8VWU&i22MWS{R(rzBFm~LWK4=r z5k|!c`z31_&zig>frWF2&ng7Z1-Jjwm)>UtJ1$H}u%5q7f#A`5GTvsIqir7oMh!*9 z&v>t{eT>H~N8uw#m%_w-o@}dw@-bc5Zn#C0?;5+f4{46KH?0|Odta_opmo)n;(cXe zeu#7v&l^JsHcUYyaFXciBrVgM?3Tttci2*lwnHG+rvi2xL;!{u*P?RT%D59t+prWGxnRDmtJ#%+2?(+LbKR!R0_dRpx%$Yvt3@oSr{Jn7C zO53n|`#<*6s84&>ubLlnc>LM@OWpoR*St~LT4u!U4+lc|0Y}J{9gg~Zj!;?ll#nOl z4rROXx9kNTuiLKw$u+&pULVLR>vRp*}aZ3cPQL8;+`6@ z6I`FmD_>UQk{T5%m+w=dUB|(dwMyD0_}?dFwsj03v`~lV*4mzJLTKHzE?S1$?+m!y zA$y9b7030xWT5)(+nN$;53QS);c{OX^@0 zqt;%lTjX)M++Us>b)Zoe;@sKvJ1S9aFMabk>hXl?L}*} zs&i(EZ9q#}ql4B?t6N-ZcNRJPez(^|a@81m#axvY{XQK0%MikwX7VxqT7 zjNbCf#uZfVI(j%kzVY0pn{9nt6J%elx7N((m}(CO!XB}2ZePCJ(o&}UvkZJ{$YgP!?}N*?Sw?{qxI67#UtAzWkG8}4h4b=H@g1Q_^wd|-u(N^p0zE$ znn=G^>#wy5xxJ2vr_^n5Ssdy7C`d}g^aZM>P{gm~%X?Qb@$1uuRzUH~Ot1p-%jF42 z9Db+iGpxj=PUrbMuBWf~_(-)^Z71X!gSCNLn`qc=4+SEi9aGwe0S<3iVb$V{s;m3a zB7MhaFSjkoqz~z$b<*n60{M=xiRkLFoxQ9>j`%Uo%|ZWnBbYFD%_2+`#s-pTcgP9H zo(s6ji)=+Tb$oB}hCxJT$L3c&X8Te?UZZu_G6>S+2P5HiJ4)TA^RCFD>HFUwJ?%f^ z33zsz`x)DuY@&6zHdMG^pQG4qFAX>y`NDd^u_^V0 zJ^5ht^UL@W3iV9Wjy-zr6f@)>%LUMo*Y^iPKH%~+v5g#2k?T>zMh}NyH01h+Zg*E? za=ko{$VR!Yhp2XUFzoRL{2Wx-zFc=!j^c5wt=;=Llk48?2vW$kBN+6S=^l(g&kb4v zo8-Fppyz{jV)1o7Tis*(S^|&JMrj>%;E+4)aYY?oyE6c0%qfH#G;hRMG1=w9XBK(; z&~jy`{Tppd+S7M*(mH5$K??m2kG;_CjV6n@uw-;TV|LdLDzn2SQAfz_vV*2g8gl8c zZw=~BMDBj}u~D`!BxJ7ER;wAF;wdzPDaV-`+y~Cr&Fx88Bi*GA+q{lMUVAN1s}1~| zY7aXKOj=WcL$pf$iLh;80-G^4+0`J>TO;Wsg1*XQG12iFBu(ew8V=FC?l^ zZc%W$+;+ou+Z}$FJrIoqqY?WQx2LekYz!?IYW>xUvrJpu-I>T2+QJw0Mm!)?=;j$l zmw$wEMogmoUT)==n2E~lLf~kk6j#p$+78I)xQcV5*pru^pZhlRo%cyh^qm`!o0|gQ z_BnGmFu+^85|Id4PiLnce0_(t3d>OyaJ5g}?U}u3+Km9wUSxnH0V0t)?D3m+RbW2; zn-Ug%{50^CF9P&nwHG=ueSULyq80UdGnGm(q0W#!94Ygfjz*1`Ls1#D5&B6HL-Pjb zz4-wP^J?^<1u)F3?euzr_QH_)T&UJTW$lnT?_a{s?(-52&u$%u%i*)lSm|(!=aX95L0i1&OqifqWjJ(Tx5+0Iwl^_qy3`bLr z51N<`26>Yz+Pg8yd|JW@$qW~|?7n~xPG;1{A+5gm_l)rMOf%omi|7-vhR@>+1?>5b zh_fg`@TnZfT!Y5kj0LCI5kD^>#St&`2J*p5C4s59=sORc{+J!0oxO=nJV5Cohc5`> zmKCP5J6uyg-E_f1=K0(E5G;EBSHlSpxxrgjfbE_FyWj10CGV78zWSZBEKsb}m(~!0 zVhAz@KzziFGVhOi9grL`X%&V1f~ePPa60{mW4|%-H%VwDzm2oV?JTwz1R%-bKzG;f z4TQtOF2o-%Wesk2Zhhmo|1k0=^;5`iY^)IudWyllfhD-)DZ*^B1@?aDULQ(l?d<;5 zt85ke6LfE_r`E)P2Bs7W{+RO#*4U+A?D32y?5ej7zu&e_Vh_{?Xst1L3wiR9?O}{C z?BM--N`(uVH2I4^KUkT$dxHiL`RMMY85`?!M~VU_Gq1R&QNIWFNO4V9Z=F%Xg#AYe zjKbd70Jh3qDd)1iAuYZS_N*F5Ecxd7#dB?ouO)hiYJ;_GCwRIdynqBwNO}$+G%=Tg zZ>y!il(eqfo2mD%18Esl@3n%V0H*vnnDsv1YMYqOe5oT6O=R&14Q!A*2>$>x%7K-L zKUeV>ZV5mCB2!U!4I&y*MWr#+xO|g}si>$8GgjZpC~Y^GFa@QU^3>(a9tqgP#U7LQ zub@txY;4;2cec95(xwg1I{K9DONlx}8>D4{(+}<#ILNtAn1@VOGHzi`^m zOo?O-C4AI@jWJ#hmO=OpR)%% zqCmtRbVQ0cbiJ5oRZad@cF^N2_9|(h9tV1U#U$}dxd=*P!$=T9M4!(dD24or!z6~* zq#xXR`T?fyW{n`yQQb8%q1v4VrnHA599;P3j2$|fWj!*lBVdtIHv>DYakQ} z!~NibxaP8gLNCX)kJ$3V2s$j@Yey`yMMo07{j@$>b2Gi6u-z4JZ>{+}Wb!j*jO?sY ziP(`J^%f@rwj}#}@X{X{?tW|BDe(kb;x1X%^D@ScsiPHkw2xQJf6m;s5+d%D0nxJytYUKDcD@{DlgqugJHj}Vz+Snf{`+N z)DLUK&bt??R$hK26>F=Qn9WB^3)peF`Ff&TB!gUTum?V%Ivnvh!Dn}wRJFBy6rNr2 znv=xu$>N%aY}LmS@D18nt*h=eQIu8$B@l3JLg@Fof?y_sUXRle)>EIQ4%3HN@~P&M zZUt$Q=M5BEe6m8J5_#2pa!3_M;a?I?Q0SiOiCExSQds-=$>Yp&ubMy102Cc+;(y4kkg~I_0x%wtf={x~JAdYb2nnM8Z~K1N1IUF#SCm~@FyYc$hCu7DfDAdZTET|r4BHh_V)Rp z17ekJ!(F>)$bX`YUGpcY?D9Fn#h^>Vfr1E@kP4zS<^q2qVbKL{q3ckP0Z2>gQEozbYd(AbSnFiD zx_LJd*_f+qfObq&tTCl(5(#V2HlJUAJM;8%9Rw;oJuK`9Mne(G?Jx=MGr_g;Ti-A3cLYxI?wN(NH^6CJiW_ykBdkjp$*e=P!9A!$XEz7IOyY~_#uUT1~z8dp{w1rl1Ofm`>!L% z80N|=vKQrm>nxFaVIVLI}$Tft;34%pH%F&%P$%?YKT+jWn;C3FHM+%5+jLjQJ z$e;s~e#v59x9;T|nKG(VNSLUM8oC@22Z{)KFL8|tN-i6^irgWvM}{Ci=l`K(BJE~_ z@)c&=c1hstwBcHwLwGH^r4UY@7ecy_!|Qc>!K!$n<`p$J5LohM;qTg6CiAW$`UWBM zehAIMkpX{FFZkK9#>hp|dK*1wPS;~bz~&)rGy>^RF^@vO6!>>Ue6qM-MS8r=vT#pG zT#vvg6V6LF^9y z1b5rq=VQHsxAM+oj5(JUtIR=Iue;FUEK3phSnuGjUL-rB)bNDF#Zp6UA7q2U^(k~F z%j^%MAAi9t%ynL(6D>?e49~&^R9s3WjQRHXmAzP|>{kgYPNM+|WmD^Lf}e2rtjp^% ztF^*Mgre0-lM!PWf)%JRCL8X5zT&Klq4xA!qnbT_%TxF2ujR9ty8cR{qPorvdV+2= zuI9rn5-k4~=<;Cn^%Za2#{8O@0iqxMyZWv?DCbHw%|$}dP>|QgRGrJvP_JB9@fh<+ zGlK*x9C&F3E&y-In>v5A8O}KVk%Y#5Y=n@aQ3|vxbbxrH=5%CYiTib3y$viyR#rj; zV~VV;t}|VB)Y*gz7C7L8#35v_(PQV#NxX>m>SrG?W!fl2i-@d(GYVNffRwwNsO?l? z3*X4&HMGtwW&5VY7AadZwvo(gGyxsJXVOIW9xA~ zS?j;6w~XJ;;{ryFYy2A3;wRv z`yQsm)<@|JP>I#|c^#>9<%;n=W;2)Krm2J}@;j6=hlDPu2FOhXOs=gpyN7?*ViPlH zeaZ+J4O+`!2>RAA`z&!S)cKXMIJQ!*sikjamubtGp*SU15Us%~He?{Um{>}EtyenD zcF$iTQ-fz9;=Pf+N+@8&!R6OK19zJMT(%1Q55i+{7yx&>EDo}Sf=dVJ(qT+ z1#2w%?x?NLEwo@;ElaB&h9a#c>A89Bl)~7jdn$a%Lcik@NrZkv(@4+94uu34s0Pe8 za(L?I!css)90Ve8pGN&=G{i1*rw4^1)b_B z^?++jb;e|=-qpY0Ka9WL|0w*e<~0elB5y!wPL7h6dfdKuF|!k6Zzm+Q6Aio}y_E(& zKCY2OL3KiZvf42}r1yZ2S)6v+bV9{AEdx-eguzT;lTIS!t77M)ihF7hN}td_rF!;P zKDcTV>$G@6!i%P&7|}_m4-fuW;d{n~>+VpvkQM_Br<6f_ykuAzQ_g24Dk|s3oC_%- z-&W_{*O=S8>`tOrB!(Dr0w!20ie+miXEDXudj{bO#fi9%kPo^!635lbF+(|1PK15h z$17MR@T>&JN?=0+I03bZy@YGaCjY|h#^Jk&RJ0rE24ccJON<%3?CCNVG0(f3Ffn4r z!$6-OuVIvw0_!ewMqWJAgEZkygML0~>v0c3=4m-vO-RYw&8Jo2qN#~B;+|ZWg=|M9 zmUkrI%ZuCV(IOv0p# zAt6vj(89@AAi3*2|I-a#rgSIGAu3Vn)|HSUb4$G?V4J=#JoFeHjqPvm%C`L@0Us!@ zEiSKBkIxK7W4tYH>qTLNSP_iABYPUmp1Iok-BY0VJ{&3a`q~a1>KNNQYlPoD7FyZ^ z{;b^W4s8v|4JZ2VS?&8|i7m^aTtp<8XAcGyYw37n;C{x|HFMRS0v;gEdMubETOYc8 z>ko{r*UmGsRX`@s)_s#k4`M#*cM=eN)Yg8Or=VmMW_OYkE-nc*)^@S6fu%Y35y?2P z)ZDa!vlz5deCp0pEY7Fz>C952?dKD^Xq+>lQ^%rMB(7d$xki&n(Vgs2@0V-fp}yL* zh8HFzpmiU=Bl#sdo_K8UjU?TlZ1cZpD_uaJFb)p=D48JK!=R#>46kWywUk=7P&9JImSi=t{>r7WKR=R})c>s2PLu@p-}! zEEHNwj`im}(%62KMHM$KrOy$q4){6PJ4z=tmRwGjTTfj2;}4AfOCBJ2K|jJPic<*c zlt2hN`jTCsM~^0lnBH737Z8J}yk#r>cbM?qhM*hGD74lmdoY7%n zC7FHa^eg*296~Qc$DDe!!QCu4I3X7hRb#;rm?!`z6eh;lur zVI$6eeWoU(_IHVANo~CAr))E)_UBh-)@3>Al`Dx_F}*8z;RwTp1Q5BSItXx}Zxvp$N8b#J(y)Y1;CG3N|)=pLqhAn`tT0bbw1qd;uF4OU9N; zVTX>+J=B~@!d|(O&?;j5Q=h8TFSNnKsDGarw=x&oEx+i4@so3y5ejdiFA~kGvI`-Y zr_ZKv`Lb5*-ODwu#1#8gTM3r6xJg)os~kH#VDvB+tqJkx5Nnq%Xbi*(_c2~p5qRFHxlVj?P+w8Z7{>J>NYo8=| z;ZI4HD}Gh7Trb(ss~qF{WlyPGcexATP{+mw?P8b&M~-mHHQ6;5jfpN}0<&4JfdbQ5 z#}q3;JDuGN-J_-;RxuKa<{PI)?qfWkuwCJK3#!_K*Y3w|OS^uHP}~hJikNfbIj%vY z?-|wqNO<9EiAHi^mC2Lk+^~dQQ@6UG6_8gyO;n2lvI|B#P$wk3^GE^Z%DGCLv=a<; zzz#F04w%4tS%MavDfchFI{A&|@xe5y&t3>~ zddcT#!jFbW*(v-I*SYMK(Be9mM9|=yM~aO8F9SY591As%RIbbW{J>flpcVcEt+>CW>rc z$28~>rC~Krpyh!Izr4K?ORsL0keFU=4{m<|ZhjF5$jfqVI=U8QS z_>1%bqB5(lCgR46Ug+Gh+@(wp|11|m+t)&-S*86GHN*?f2XH-HdF-nGtW3N3CHf95 z(>6qSYQrlCyONfb;*}d8W>v3_FB7V0HP%~hu(en}teBvNtj>^V2z*8CO1AE}Md;3m+*LO19zsO`WcGBhTEs=!Ow z;NPHJ2pi1>wq`7`Wor*f2=QZl%Re*j&MTm`LfvNGY}+l@ny8J}x);0QYJh@ZXPk}q zhv7yn*n0<5AM8_#U``Qw&9P3P_j{%y3@!Wg>&k+T;#cVlG3dxad#t*KF$o6!2R=JA z)0@1w#2Tew!1#eokgs@+R*4C+J}rWzC!I^^b8g%}thMJUiHfZ~O(2CSE?AW%zyv#z znA(Yp_ts@DMak<#E&3_-oHTeIU#L}5ViQ{i^MWmvar`>newxM3M2|NJRNQ-21g&ij zxHterNL+muPq~TFip+9{=;j~G9bpCA%ig4g^nxwAD2BM=OisXG;3))$#yq5NxkQ)g z57%N&#a_9Da4JwQLHBzwh{L)*NK3_ep4O_PaO(cP%=SF^7Ev#3Pbq*}H0oGVdrxVD zXIT4;`)z_0_rTFKas23$HIZ%pxThZz=O()eQMAt}i1wSKK9gd$q;tq#_uowH-jb*& zb{Q&aqFWvnSyob#&)xe|M^-~FeuqfKYIhwRcd#dDBvZ4nYPxjMsH0c1?Wy)(b$fI; zyz%lvV-!3C>-v4>CoHYKO+t$K5r`8ZU`%1K{68Q3(1Ue}jC_|!#O9kicx4IdP>*TQ z&3d>)G}qY1|8 zLklVBiyJPvG@F5b#Xvz;JTwWB0%%^#qo1>`#=G_srDCK<$c@F$SRTO}x;d6J%x3Sa zm;z3;8l*tyD+_GA1!%X#6eT^Vlk3i{&ec@WGq0G&r=>g}sQV=-*8PMONKWf~yaHoG zm3=B3;QDj8{sNq4yaCDDEOA#V`rt9hx9C6mn3%-W$HW3~#}`t|T_)>Od412J(-_{! z4~a%m4Wrv}3(bd8+4|g^Jf(YnTi)nQ=1iQHc%n~O=B!i3J+;32R)+h?M?|Ne!DEHHXSUB8$t=*?14O8pnZ~;W@U|qpi9opxU)iUe>|Evgid$b=lgBL3 zh=T+tN{cXl0X-G?&GDEDg@9>%@rxBp8!!!eT0)BZHP8pvyH`xIu2@_Jt?HcW71_qT z^|7)sV8%gz+hcc$d&~pH))L?KyXS+<>stN^0Sm872Q~_H$XL!(Yv@k*Ui~^d@cj-E zw3sY%!$^nNr&O-R8+~fmcg*4aMPdqv7q29QHWE%RW>ruLU{8ZN5v523*DT{Zj>=*^ z6Aye!#G`_3B|;e8Y(i?z;}zr4FsKtZ%AI&S5AWOBmK}%ApAovyGsczBoYxAq{M`1A zuB^!LK23lZf3=(?-ZP&Rr8B(S4y*L}+<50;3V09QH?M?w%Z>j>cw+oO|6Vw$A%{Iq zb`=gPF2(gfFFVGp|CF&9BsdqX zd9?xu8Bd0Oq41=k!v#|r*f|H9B5sJ)S~rX-J)F*t*Dn%P_%9|@IKe4-|ChhsegRYG zE59U~MZ5cQ5F`^kmjhh|5P07pW^7y@(LH@;xS(~}Y!i>J8;H0G%wEu=SN$V7ywrgKTOX8WRN+IMG0>AtV%E6^xig^ami z=NIS6sMYpC%n0{5Mrgtai@Us2fcsyK`>QfZJ0jue^hLliOgT&_*>d1$1wM(*PN0SHNjH%hp?T>$6*SGu(Go z+zd#W!uT@uc?l;x1^2tz<~ez+FzGu^^kO*}F3Xgz9i#-%+ld5%#V{Z3g!`+U9%$c<+0Z)puY)y<4G!s4n4Y}866i}UISJcIU%L>DSw-@sJa;&fi(XHLpZ zCx~Pb@ELW36j=RX2j@&y=`8;pVPd5-13JmvQ^AL}x{+CtIQG^<e+|tKc=i^BUh@Fo$JibD^ zsmDP1=sY!o`fS`xjUmlHNuyJbmigp??-(u=h`Ct|!ec52} z{j4^c1n5U3us$W?|o#*uugJY*~Zh#z~PG?r}f-Ro7Ka2yZz> zvxs*{J`bD9%MJ{oHLS9C5szOXcuUn=YO}rDC2?`@;JO|N$K$C~2H2te1UCm595^!K z?7J)gah@SUMZQ1)CdyYUAI{D~n_cBDXKMbEvxFzqJSM9}>lWBPr%5v@ADx@mK?y;2 zti0uaOwDhQn5gC(LHQBK5wY}w`Yyh5r|5z%iY{%)ifDKU3XY7)y;bC|opNFhe~{gd zr2$I*B+A9ic}cO=drS&1xY2lQz=yA~(6{DagebbEal8Y3*^BHrKKt9I_G}xUmEa-= zMBvmkom@d!DWcuGbJU+qXO2Hlq{o~teFYP56<+Xgto7ILwqk&Ps(=mf#vpMxz|^}- zxU3~UA!FVf%d^1`{Y@0&nQnxD$v&0IDINLqqrF%bug5>9FhuQz zZl5U|U=3Sm;r_2!^H{U;1dGjMZ4efo&_tVs;Wb!nFEmYHuSUF$Rs0M`bgG!UF^;@e zuJTtH*F$o1-iKcQID-aEfiEe48)ta;tS4TtbkE0-%Lf+mBV;q!k+0&k8GBR`nz5R$Hi4@`W?j_*m^tf`l`|U z%+d_Igvdop(?lT`gOGoDJc6I3@#~#k81#=;(0Y%0JU=YA@ymxsRbvT*`Ii#a;vN8n zYLP0SU_Rm7inJX${JTIYf$$w>4Q|R*X`WRHGqlZCvpwZnWn5Et^=k? z)c8`AvcGt_j^asV&Yj_8D@?tNz9O~)ri17r%!^S7kr@@+vWBX9m1zZRsLE8n0VZZ4 zImFdMvT}NE$&XuDT(eWIA_lOPRiIB9{DDHzS_$oA$u6*IbpBkXE+^RNGelwp7m!K{ z8i?El1sqs$5!oF-t9SGn&(zo-5;msBrU}iL2-H$G4{benH%qrXq!Fr(aI=V-%T=^wV*5L4RTL&DbYS-B`@5 zb(^Y$D;h5auI|t|9YM%S!UGhEVn9p2A9<`#6*h9UN8*cH9|YgQoW^_(++4U4KrY6G zQBen`%R46bf3`K--tZOX?WHY%_zjA-@a%-CJypTRf^*nleYDZ^JSLQNs}Z&sT)|J! z#U|zZCXY?Odj~s2&r4)HL=AQ3gRcv(yFnjbvb+dvFZ`N`%A_mJqJpY18g!Py+lJ<= zixfVXcwD`x^*byo{7d4B+>Y2vctZr=FiUjPy*045W5@z#P*+wbf<>JN4KxgmM?~Jp zCF&(`^O5<+T*b#wqMt4kPAy_Nh-+#Pw5W$e`y{l#0C#f>a!q{;*5u~BzxydBYeys| zs*=VG6Xt51aA_;wIwc1Kt-y{vhaqck_22h1OPEftN%UgE^eX)d;^c=?wmkU8S{9x( zsYP%i-hp?g15o~rA1AOp&h@{3b3e-wy(0lJM^q1P;kQ6kqf^BPthvA2B8FFT6%is@B&0vQej)EpMONqapWIyTPj-_+rMiTSH!0NgA*M+h zt)QOGO}~+)n>R=((VwG3;c$0+#@!0iz47+hOy3Q!N7M;@hgB3Y_X7#G6fTROSab0Y z>^z>5sKT@(Dyk5;?+3OXlC3yQzz?MLS-fiA5F74_4MYozz;GoC zx?$|fv5d3r8W1RQR=lzUCa4gq`=&I6HJzo^c5Pvh2UN&3e9Zz}MS z>6C^M?yIR6eYqFNPBL0brmxtb<+qL4N1B}EKVtyAF@!i zCwvrK=xu1v;j#SF=F5?B}#J!_O7fXNfVio_ibiXlv;m6Bw4LHTEDu_WW_3?g035eT7( z-QjU@OB8STVgXB@^l3y(VDcmzmk_FjI&qe=Va13!4`b*7!ZmrID^hXL)3grMk( z9U*-G6}iQWdX!$;f~U9e$NlbWLDZv0ZE8U48WUo27}Dh6iPheqKdTh8`RSaN1dQ|3 z?L2-Mtrx8^dSe9|0k{mvT>>R1G>nNZQCD27EJeZAt8#h(i&_rImGl4?o+r~A^zhgP z0@iumTz)I;#=~XXS(fp(R`dlT%Lpl53^_&LRIKqNX|sFqm0A6mRcO+hz|ksXD5wcT zekD=tgBM%iEib=uu=k%T$0v05N^IdJ=#O*hUFUo!6)`bIaDlsLY+*buxthooJf;MK zGwO1nlN2-+Wvy|m9=&D*_ruAA`6Rd z$R9rJVb$EDKr7YLmv8ucJ*zaW%_O2l!H`2+_1-Hw3M!~d;Q4*m@=KYkJ2;Ebg{w=f z#xYPF;Y)S{w=a5cMwfE>&z#FT{cQU|qGoHE+Ldu>L4`c7LZ6xO*Ak|c=VlXmC|q@r zu0)ZmQXqh_V!VIYkk^|wPUp9NXSO9sF}aSAJ{XEIk83HLqp)x2n58@p-Xp<8;}C+S z;=QHcHD@&o3;j7n8-|6A5zc`UM6V}y`m8@v2L;T!O_Ubdwk?Y%wSq zya~&E&8>3lzb^pualmwFUNo3mVL`>C%vpF!u7G@~jVq+`HtwZkG8QmCT%V`#Aq|YC zKaBT3g0}L&yP=k1DOxNHDvvJj_Yi~qjlm+77jvJ(xqfd4nCf2Mj;Ix?Tfe5s=oAR2 zbP1pD(+rIr|L*MxTojryViZ}uFibY+mL6lr_-pd0u@2Dp$#tryKKmVvzd6j zDY5bNrDHRgc*h+*UPvaG`f4n7*t*AhPGeTb-HAvRd08D6ABT+fz$VL#@TZ%)FxjZu zS&@zAI|G7SP`-f~goT7YY)0CbD||z+2pQOC)sCniiGi!wVhCfS7olnPW4VQ~ZE=gD)*$ zy6)y{h)B_-OVIeNSPG!GJNJ%d;!vfhA`Z<66x&k7%OX&+OJVXTxT4~2Htw-mf{SsF zGKUXtZ74|g6{{t4uX&!Gx#7KtRFROtIu71wt+&-WpubH#gp%m(T5Iq9ryb{}u=LYu zxdx`6(tP;ZIp$1Z4kW&6Stpj0s46c0$i}yABkh#K=UBD-DVDQY(wja3b2jM$p0d6d zg+HmrSGAet>e7es(Q?(t)Q6+A@ZxAMd+fQ-8EZb1*n%|@J6v*c@YnYD-ofIrd;1dU zYCMMR#+W8CH=J7U=%&8!R$*kf=|{Loc7_3%IM1Xyr4*KCd6h=tnkSDmV)@&>5?=JE zDe$4FKSAEfQjv5zd_R9uJkX!$Mpc@o0E>rgxeBpnNA1!ESFu>E!vKOsqml;qn%Lo; zMth3HK7Px_hnV~Eu|yT_hl&cCB>rJ_gVQsb)C_ViN(}w6Np6tzIWs-j(cEI z!8Q4K+pq1{pINGU16B4yX~<~@R7~PPTEt}htYH_v7U1vhye45WmQ1HxX>N|-vr8PpKp-s+O@Yb>+7pFl%Pd_FtkgE3ky@Yv3cY4-{-L0 z!silEB(BhAVR;V=yUCv5LzBW+G9sTCrVv@*2bUCy``@sEiDNRyLUFGS9k{VG>r1(Q zI3bF@l&~935`ih>_FV2m8wHe|wIt55oBN;BQk z1G_qs1^)H- zhISSwRhmG+BEuKMgS%A<@WSl)kq?FF+>Y%!EB#Rwe#_d$BI=D28xt%|Vnq8qQ$=Yu z{&rC1?7}&L>o<)|Ujq1> z#22L+U5v%lDx~pE5~r}lw2u`JuKtxd@hc_~)grd@U^9n)Uy6QXTC&@izVmdHosh1R z30K5O7!qJTU9vn`*f9qVKft8pQwb}iL-rPXpn(j!XY{+HlEof6=jh9f6}R4~up&c} zY*oIq+_>@|e%y=UHngj_sLSCZ2sA}-4YS|vHAOU*xIO+l@)0|NFG*Yx-x4l_NT~w% zc5UH-=;WJ-V4;Sf=~>)ik|McVRz1*^ZQUgfLdC61Cvx@S`1nJuw4Jxg&1{5!w>Hk+ zM+ZLZz@+VQiH*{hLD@1#Z9Nx!9+VO?#7_iH{{Js+}f8sQ7+38HsNAQ9@@WUTRVK$SG#)rnre(v~>-U zT5;_gR8HeAsIz22#{p-rV+XBK0fCBx29M9f^~-eWe_Va$z4iveTA%_B7J<{pB}|D$K|LFXY{a1DSM?$~DJ~^L{3<4LyX10^3N(x5WKX>7gV(Tx z*EbSYG=!EqLLR+GDt1qy^$Ftyd#~v89P>`M+)MbLe1pvdZrez-XLlnvsC^u3K69%E?lFf=hq zmONUtAT*98M?5~F4Kp}(^@kA?yiKY2E+TbreC?ZuZjCY}V)GNCu0+Hc*L~YyAOOK6 z&-D`!WMpb3YM_hf?kxWq-lOw)CTj_Y!VP30$rT$8KS@a%M+j~)~ta>-u%3#0h% zhMp#iPls6UQ&y!lTUj0Wqy)wufpl@!g~$UkJUDL>cO}a*v8BTW#@!uK@B2Nleo=Q1 z9s7<|y8CVS1&Qn%wMkk-j73lq&=rq+a6K;;H?|TlF4=RTxx`cG_o2rla((tdsl9!E zTLZ<|UdQ4q_b`VcyMz`NzMk~QAtGIPP#z!dGqfEBywUj1am#(4y=GKjrhu!32~{Xyh_c{`kf|4P zAlv1tX^f6;ku$?GFTBc91L-J)Aj zs%SQ7J#W`RDbGD}-`x`!?r({3EhtuSJLcr&<|?>PjNkPavlh#uL@!#4G`vzR01aXP z%83rS?K#0_L*MDMXIyaA%-8RnFqm+=dv^LlTXF7+9+09eSEZ3jQsKt!L5s8CV zrvc#L`&M{KQ=G4;s5d1FZu3IJZ<%W_V+s)_Tmu0JADjqKp|Mj(rAhJUXWo8;jX5`) zN|4xLmI+P~e_tT#EyF9rz`DWsh_y~u9J`*QU$4zt_dk*=VC#Nn%%6cLUfuRmuh=sE zm_f1GWRR8iz}(I`9owmAVr=*2*=(KCGWrtnI8VZLpc`!psK0;lDck3$SNW{{pz6s^ zBpt`U{qJ~N-D$K=rgpVfJ$}|r*-M2jk)c;lVONdqlrW;3(U=0@no9vw&O+wE-SgO4 zn0O1(glD0iZkGxmo8vWkB(-B|{_>bo&$TO!-x=SJJis)~UlLig!eibWU;PlzgL6u) zRe0x3^Tx8fF<0J7)Z*Qk9bl(oO@gr}{`614nhCsX^)C*A!sXo$Pdv$@ik`R8QW#Za zpz#u&gDHV-toCwy*0cVl1Qo9p%9je0r`^(SRR68|W@bot{)fm%Lt4+etD$aeU$4-+ zCh|qwgnB*QZGF0hSZtJkJ7J?!(-clV-u()5!6M75CuB^DPZ37N2|F~b+d8K3D@`Y0 z;oRY~3c-uP?S11(4X!;Uh?6 zVB$Vow$(xTm@aHL+@igm7H#ZKnq&8~kBzd`xs%q()!J&c@xC%Kzr@c`VkQULthtPp z7hjNA;^9;bK*f%WNHG9`IJ~7s@Gw6SfMS)Y3o`LCzptxZk&S!aID<&WanGw0AwUoiL{LCL0qMAIb~l?P+t?Ba~;ek^F?1#NC$_P{cyFWf6{WVY5h?DU2F!6Lsc=*SL5yk1+dJbPNu z9d-t@o%mbUVz)&C@?&&#j&CENoa_50qk_wd<|5VvE#ma8@PMZ8vzt=t(5bqzbG zhphy6Rn5z)*Q-*m(q$E|s+8Y-$YokJtqT74%U3na97Jfrf-S2p_a7m&E3`r_)9JJO z9nPRNMO4S{4ZpZn{q~_>6KWsr3N6#&yf|v@D6Lh7`+d$Bwf?a^miH&z3Txhc-tw!& z?XGpxnv}R54yVr=a?fz`k5>Tm3cBaS0LMRZ_s)R?chO~3eV&lRu3h-d6uP%Eu+6Dn)_*e zwN|d6KN7G`jW~nlF_v(;m8}~cj3}~j^qroQ1`zayJK8>Bxgeo?YTdPTYW#jJbykO#Rf_!^R)fX(QjuN#4wEkKfuWhfuV(3;Q!MYfQE=(^FJtzM8Lew1^wq;?t;#)p0)!ciF0B+<^G9gaO0 zaFrL?ifTH(E2qz3BC~tj+AA%Sj}v5Ht+$p*kZvCs36IlO<}{plMGg($zk9;&x5pFk z{B-9lmYkmnc(`_*mP5cH=Txf)l&M%C;79QRZB=Zuhy6kR0m=#G+PeE3#}1|kE1jT) zP!Fc-yBTm=Lw1`74iZN-r~g>a>>m?Y1Iez2MlCxkK(E zF#1L1d$YG;~!!O3<`rtE5zhQEH z+i4;j<+>@NTAhKA+vE3fP-Xjaon3Mik7H@YmfM+JXZ=EuLauFrfTvvdU<7(@z!caZ z*S5QlWLb&D*I(KGPRkw%JX#y673jc0XUOe{*gRIdAIz9t2sLQlaBRh7mm9xzU7xFH zx$<9p&sjph(s%UG3bZC5g+80x>T-G_$s+#m$+rhGW~cq8GCMRiVhcJQR?xIbLsq)` zil=)Mk-Ijp9BFw&Lgs2+wfdoHZkG{EInLbR`j;%+cqL)o;4HIQY`+tE-LyQd5%6=m zHDoI`XiWuP=TChBhqjwVWR^&Ft>fP9XtI5&zt>Gxsd4CXs;I7<%ZVG_w zek^#10Zurl05;XxX$4>3X0F0=R0Uk=_nAAHy|^F&(OzVNBLO0jI&8HS&oyU0{!@Pv zo#^AIgQt86pnF^;OPD?%_!ohqK5wH^2`1DYw1&dv9>dY77j-Bu18szUQaGl0uV`!y zu`urkxd4WFjqDzG!0HMb&xL9oRMwswcihR&?!A8#xp;OP+Z;Bpwa6K^<#I_<;BvAH zW5hjCw9(&>RDlQ+a?TSRh6ypaJPxNECV*4$mhBpHfRT4V;vsnr1$kg*IGS>N(8LtP z$ou2ThhArrIr9RMhoaszfIitckJ?0j%hSzNm`mIH_ zu)QQf@TnZf+!&1~TmE>49r2wKQXFxY$6o|iDhW)*MIZe93qP_0R9cb9!~>KOw0Q#% zZkb^!yTdhgF#GHz=K0sIM6eimc7ziibb`0609)O~R-e=9NZu)pjH)%A1&SNw8X`~( zLdF1y4?9ujeG!ihk|PGKqL5!4@pxjK&aQQ}n~^`NvO<0*dx_ItYAyCdlEa4XuGQlY zg@j#*KVHfj-0TdU^pTg5e^#!6Hr+_7FxG zR`CAaWx|C_n*5V*Wqr=vy>I0b={YCQC#!ofO7}}lK`Ev@b@{S}{nk*a+u;2x zs1qm0HtmL&y8SqYHf?y9BO5JqG@^8;_n2*1I~5bq6MWGg0dVvfS{ zcMeop%hXc4T7)Rn5+1)Gm|3USj}AO268K*(b>`H5_Ui--O9LH~*q8=tNuVL0Ef6a4 zhphoyxP(JDje1sfNP8&4!G+KLMvF&T*5fM)EK=%5V4FEE7rIEjz!HXg z*X2ZTLR@26L7|u9PAZ>ud;}d9&ov_+vUI6SxL0deX>E=4218ayyuCH&^U0gqZenEb zm+*}2qKKz75wI!Q%iY^QVz}k?60t+zPKzhl6nFEO4j(agWY$;M(N40X$nOt>&9$MI zmLQ3*!29fW#m6b%zI!1%jqgf$JdLeUb=U(D@NOV^ZIT93u({;jm=@^7uzd}Pbi{7w z^ajG^)`$<*h@N*3RINPxNGjG=F)>Rk-p!LijT#cRNCr8aU=O@Nbtvq%gU{|TsA_Zh zIQ{tZjqD_T8%paRu)HpT$7y4EhIVZ;JAV>6#&x{_Ctv#B|oro-NBzQ&7w6{Sk(fA2HcQPxzC;| z-7e?W*~}GLFM-h&X`%z8VK!I>Wk;3R>xa^}e`cZ5*d|0KhDyzdOo&80P6M7PW4B%V z91mWrG$lxk4x4)XE|X7IC{!Y^{GY9lFbda5I6ZPg}&0gTn8AsuZ%&5s(`4&YBh*pX`mwc_$&P;K>iY-KhuoYrnd zpaY_n?U=ikzGm4*#;&2wRd#u8p;FK#A%AfgOGw3$5R@LcHc@tho1M)Ed#+$n!D$IC zY7RwiIMl6sVz5Z;{6gI3mi zhvn*SYe8gVt}YGjn5bA|O4T3|=AiSQp6O?vUdNUMDm*+dQjYGVK~|K#<$7+_*N`_b*A8nUW zZ6fVvTE$B9Ew4!6>$Txpo=tcyx}^|Ko(Dp@pv~iPdcdl9pym}ZHV~NdW#dhixXe#% zPv0P9-Up#MI5OZ*>IFY5))=`+nr~xS%YAP%BT%UWVWSbqfQoqp`lZ0XBjS_AUFRIN zon_&kmbfAdhhpItWhA>j0(B*-#3!(d=A{4Jw#^FG9x}2ck&f*l?H$el1nl6OI;?oK zyb!y?Kf&EL_W78vaHMcS55}Ccas|O0g!MRGHhX!BxY^&g+Srd|N0b^?bRx>J)X>Na z*<%1+~t5>%YV7$}rY&BF

55n~8~6{s*K8}45_h+ov;ieNv*L zy3P%_15Pxq#=|WVEdLbf@?doO+jsoH{F+C)68-4cG;`!ZIajJ_E)t4{g1k1S>Kri* z_1xN14>OOncMib{2VPo%i@=-m=2^%8{>3o zn{Q(&vO9B$U`&y9)pe%BiaMK6!2nDF5JOfVz)^aa9?fNoJ#cbCGsdJ^l!P~B4E=5^S!W8)(N|{4K7gPh}ri`s+ z=IrkM^=pTjLA$Y#fYG331%lAGhS_I{Yog9CkH)dpat$qgEw1i)H8T{|deI7^HCV-l z3h<6EmUTsy{vHmLWgC81RbM|*1D;n;LoBjAwERU7KKZokWu3I82nB}vFYEC zqgbZnO;+Ufq6OGa{6I+-mP7jEpYa3MVk7)mLFxcX>d08`F? zNK{nLEjbrbLVjcSpJC?qF1?EA6^S8+oPY_YisDR_ZOxeCymA2H3dM=IwxAcfITFX! z$}vMZQ%=O=Z|&U5DuG`~V5|f-j{zs3HnEq`@#)o5nBBPbY9bZwMn()V;hrVNZ2i*y zH(12nb|7J5#Egf5K0jW=C@BTjU3Ra1-a~yz6W%=d)K8Y95;9NA(dt7=)@nSh3KtDc ztUK4;Hiw05Zw?~*M97BzV$khyqAx1KK`(S9mp~kwLV9+!tLh)DlhJz(ErFekT``~% zRg8ciP2K-tp-(nlH6EWQs zbzjA2sxqEC2P-_UiybOlYiC83SNcVh)Uhz2%j#LPnAp`ALQ9B54K88gNGMe2fnJY6 z4jO&M8V0$JK|(@8po*Y{ldnK>*SYKZk&~FxyP-O)Z(aHFN;(=Z z?cAPiIVAxfsi-|%QL7uD8H_}ETiVrw!U(Y<7=2sz446H0boDu>LGOJiT;}n1z3w{O znEqWx_?%;)rQPrAlAB%7H72>CME|`@x2w8{Ez6-?L^zOV4FnWxdC;4&lCkx{>(reB z9w5znESMx)7cHOj17mBiVFtDe$mH2t=ixT(nU8u<0-}%F!3Xmcl#IgcPIAJb!RFTQ!JH^v(#vt5rmGZ(M|?*>R1$u#MO&T*Lc7; z@xU{$VmbWFZ=lsgD{C=$XW@v?ps5r?W9a@{dTQ`^mYCQe z;Y4Br!g*0q1??O=Qk-T})`xq3dyY{$exyQaBf(|t7)uiF*LfSBW66R)B%DYVfTUyp zo(Kt&XhGn)u$^zZ2QmNckx@jd2(Asb5c(huDQ|QBK2`D5=d9hY$7n*vcE2u&?1k(A zAnN&(7?;H#a^9FPGvNH8V@w9Wl*^zD7T_`>#>46kWywUk=Az3fJImRX|LobrEb5ta zBYl*JdPL0_l#kaPf?&aADmmt#GojHnEm%~MIffPx%{}-z*gHxmHJV&bmRtLFYiAk# z+aGIpMa8UOFEJ*>~{QB zM*d$C9m#JW{n2ak>;@_2|K1BLT=HL~8|tSLoX&`V4dVJU)6+ z1(xm* ziLTFCh{9P48}aM5+@p-zv6KFZ+IZJb*=9~{UaQbSmXkgs(Z%$x;Ds#&7ZO0^Lh~+= za|lN4;YcuPgIIm-&s*5mE}yJyZOhooY@&s8{UA3@s~o@2htm&IXGs? zYjkQ`;|gMav({@TS^C{XE9Gd}THWG6Zg-AsE+rw`2e~Xu^reInS-7HNBa|tRUCpmA zXO?ibm536S5Nm~yrNx3NSXyg5usMz2)>zk&$y)Q930cURn2NwjUBgZ(lyW6i`Eh3+ z^IZNd@z8V0#K0Z=0Damb0j|>e-@Tw9zq{HUu=pp}Ddzk~Y(%Qa4v44{oB0yJD%k|HlCCyc zL5GVFWMhHec7dlQej#OFxvIC9sK z2AnL-O57(Af9FYJ8G5m6&R(XY(_92D>W1>ZmrGbgk@}A(IFB$v-6j`7LG2JN7>W*q zXbhF~=-h?A-_&G2`{WY(2K3o;p1|x<`XJGCfJ;ey0UHua zMwd!qhca(;)?kv5=Ao5@B#7}(eX3Hw&;}2q{(WNH%2;eS{i5H7-93aEp)GP9G(zdJ z3n7rF&!%wsGFR+}kA5?SDfV$*BA>OmNm!X$FZM)2hPfK^HCn9CS390Y8Y8Gm{etQOr3FCeb>{^Fn_9# zpWuZ*C0VZcRmpO_chK%-T>nxc3$8ny#c-&jV}n*POoAgvIOUq`8fj;4S;7QnPJpOK zfoZ8@ij|<9&gy~gQ9}@`7zsu5?LYMVP{#9iQx%@KqpCf4?LO?bwCcAA#ogedh&eZ& z1+({6W>mi`;f1dy8p(xKCQp`g!xVPy$ia890&+=^s1^le2aI;0PDps?kpjwPPMuO^HDL`oe{CJ{N%qEw>-4s@w`GG$T6&c)5^J6(? zRnb1?Zw(F;`FhbFR)U&F5Igw60kZp{=MqPJlNP{Qf!^;hC;tbDk4}C|d3-QU>b1IH zPA~Z!O-KKxN30ZniR)bMjS%VLI+sMy;G2g_V*OtbTpF{N=A0V0wblZrz_QC|IZ<%c zh4NCqMrh~&-diC~lq7T!n-N5bl!_6<^AjLGU!eXoFqU4L2b0e_%Xbo1G zZ9jun6O~zYH4!&n@K-{vOX<>q$izAaD2GtFV@;KYc8Q;YfmdkVTub@WeG6BjwGgb z#GL(InM*O~R>DO;rKz0;&*KZVN=j^ERv<5srHtcmntpgLI}^twsJQp41X|l{aB%>L zkhuCRo^k`D6`AD@QTwMxyvYi-_uNL*>jhhMQDWkXGdX@=vD*a>jd4idbP3O6hbJ(n zBKvk)LO2zum!SJS5WrzwFQld7JWq4gVXI}`$!yPmB($(SWdLf@sAEd)k_LrmSo_S3 zJBV6w4;)Ps$B#al6S?8oO)e(R>30&MXrECKty3aigJL$N)AP%s>zLT>mZ&IpnJQ|c zTOJizR#K8n&Si~eHDupBA{DFMjd9$;nxK(P&BChb(sAz(<<||?yNlq&_ULeUB-%eq8vxSL4C=9|WNWeMs~w_(uDe79ZA z9$0%nDQ4fXBR^TrO2FGHYI7=TjmVxEkI$s2x*s_)>r#4f5SN;73JQ7-?K0L`gntxr zhE>J#P3t`y*mU@^`SiUw9bPYi6lBFslMpF@UfXHaG1k>M^B$s9jPwY(G1(c@Be?zr*EojR{9YAPz=>9a z6zKf4!q(qeNpG9P6fHndC)b@@ovW#$&(9s#o3VECed>M*igiCB1(G}bJz@=G!vzVd zPfLML$D1F;^*rWgiMvvP*KcKb|5~WvHKyBf zUB*MHY<+G{p3*)4;Db-`YgPt6KzO20Smvx##yzp;#NW(Gsq>(UE3>*u;BqtnZKI`s zuyFV*iHqSd+^z-bJh*>cY0xE$h(d9{2KvBy_liN*6^pB&O=@=jMz%49iwRBSDWLx6 zg#5j{rQB^CC^nb)YkoLXgLz#CC9v?ibl_Nl4jIdNY7Sk=X6?fc{H#ZaW-(dhgpm%h zPpMpsx9#iE{g}hsZV6!uhZnCTgfK?lin)6zrmY>_&6K4L(@Fp*d z$BVyO&J-_gi{oB~SNTyD&+EiH2UEbaA8vCm^Om2Mcw+oO|6Vw$L7O#Bb`=gPE=7km z`<`GXYRF?mq3%-PMLV&!x+K|n+qMtBU@rIx2`XIhu)9UdBqtJs+n9#1jM21(F2kqIH|~lWvSB zha|X&vTY8S%D~P!&=he)tmeAm-{U(y$Bx&nD~M#_zZg*A1gGTvpPt#|38v25JwfOq zpnwJ;ak+GY*j?cQ2xlcl(o@`qLqxyQm7AyqbH;5SD?SFMBjlLeAY| zPW{YI_57!acF|xPTO%Z{n!rY_6tXxUYnEJpH<`JaWA1{c0S z6k=(xjeb%<4h4FO*nf9@t`f7bIWH2nu&@GK)?m1CQe=jE+y}Fpe8Gh9zY-FKuqnO< z0keo!d}4tkntgw}YV*EjG1H87M5BnA@TnD?#iCi76ckbG`4wg`hpyFn0!D|fO+0Xd z@hhbQCHU_yE|1Ap)dN<3KioTV0TG7N0@zVggf=F=W}vX7%&j~x@L6z{jTsl$2BKW# zBj7GOFofo?%HDlI-^Ew%6kSlL=z@(m?Ej2;nyVx%dYWw%Sm-kigs{5>JS~1C z6lk^W>DD(^Z)4CyHW9@L-AX`J8lxNQRRX)9g9k!n&v=CVVUE~jg3kUD- zlqTSYvU4nRwCMZ6{1o5w3ek+ftp#v#(CLKUJJG}_u3X{VP!zb5Qt5p4?tP3IT{kPt zfRO^&K5=!68LYVv_wDMT$Cw|qUqXxMMnKafd)SHfR=8UYF4W~zD{==D?MhOyVC40? znB0}Us>of2Ku;F7N9@Sm@X`nGvJzjTEd+}tzRqSu!$VMTWDM@DB7gbHi5YPH-78oc zV6$98%$!dxHG7Xq;RQDu4WH=v2n&73Z6%^b*EEiIfG>NA9mhR(t>Z!JSqUz3Km<-r z)5#Ttl_J``&&{35bmpV45$REugBB8vaPbRKe;MbF`9ZfX&~9f(@ePTOCLkU0MQ<=f?JlR+kPR@0 z{mH7H8X*-dP&10Pq7M{>Vn}y*uSZoh8O<=D^yp2`-3|nNnn7fG``PnTte9I#4 z3jd~cM8vJEyNksi!uNcS0RciVj#T?jtJCDq{^DwhC^WG$K zh1pfe#USJ#9*^KB>DNj_hcf7_w@je*9`$&Bm~P{kiTi4@gu%NK8WRSp5QjRyaf5$D|KZxn8#*xL_9($IITKHAH8cvWB=0m?lxIdsM@X(>_pYtL)dttg!0vE7|?(xaun!$ z#@*|yG!??coZh4yQ@?*d-0pQ2MZO@{z$h{U>8IoBfd0bdo3Wp;pSz7&>tXK^?V|Bg z;OY*Y-4=kXBs@ToCWO;G;&h)uVRPOkI=#7dBs>VpbUJh>?8m}%=_+a4i zxLSn{=9;$MN4V&kW{RzZH$?CavqU%Dn*(n=aAPHAQ1?h+QRhJe4MXE$kvDRPdI{Wo zWV|s~@iCO>XUlze9c8KJ(*F?6q8<+IlhFPG+|4b{HS{f*lY4VXo8e5>YJ5nTs7hKg zOqi>&!=y#V}v;y1mY%y8edvY7Um;MEbjS16+`W3{<52bwi;2nclcryGWB2~mY z@b0u9%D?gB1g6KiXSaHrS&ryu35YqOrf>_t38EUEsy<-O+)_SV-s@w+6Zs*)gN6;91v9}_$n36!|L1F;{gCCczml-HEse`e(9EcO zeD%&%OrzZX2~mj}rBzUWc>QADorvziOSdrbN(DqY8#+wxCdlAzhjb>1f#_ zt-m9oM1PJBg~Q$P8Fw?tFN+qv!t|ZvGonuDJFKFJxgSWdrEppNutu|->^xTaFQE$4 zj;N?Y;JzQ&dPuh7FabZ1=4ZW5#>ZP(kzkd?#v(x)V#8gr{)npt3|F$CjjL9z9`JD=xj;~pO>xkqg{b1CgSEtQkLOSnrqEh%` zMK-v15gZvY2zFhr6-Poj0~95~6G*-KpSp$_l1>K+84XE>LM%=x7$34wv?qKNT<9Is zp4RHU7cfKez62JAM9&%(`C;-zup)7XgJMXOcw_6~_XaQ_^L(KQS!)#<%G;5D$e8d_ z;E^%YEo}Ix)*-?c!$(5MfV=Q;8+b;_N=dT2#;E5X)#L6iyxP5{{s`Tq)cETU;|cogI=% zUlW;lNbr4L@#JU}7aC1q5qRGbcgrX~Ugz)`W{s`6^Xm8g=#zyYVWVD&%Vs&r~7lk>7z)bgWTNsn6Kc{060509PSZ=To9<+s9S zy?NZivWyRWM_(YajF8gBkW=(cMH^3&HoFVtt-vLtky69kKC?p9G)*kVvJcoUZK znp@@8f3F|rm|jr;5CwJ$I}xK1j3NC%_o3*r5b zpsn2SZm6kPiWZ9)l^f5zbDF_kc1ndsDlg?ehZ{FHwK3KGtb`S+Tfe3$)+rE5=@P#7 znbXd(<3IQ`(JTth7%_^hUI-=|bW4x1WBfILi6x_~?T3|_VK^$$(J-{*=#bz`1^bA1 z${H4v-t`MnE@D!UGq^ZgzPxU_g$C2mW!cC>jNlHx5~v8GW7_Z%u*(*8zn_W6Zi$Vj zF9Vyw#5?Zj@j^1e&{t!s!@k~GonO%C`i)2yd08D6ABT+gz$VL#$&b`bW3o~GcSSbZ z>c}E{YS$-F6HVTl(*4%?&)#S3STE6qGSbmyYr8IkoNz_tlvG-SvdT4CC28UrA{r}6 z9XKpC8_|QO9A3%Zu<5EkFa9=`l{qfR1u!`3%om7_1EsXalHENS(wbd-%Vg$ zzUhC`0{VMJFcrh)LP6rrM3qP1<8vDSmPqKNwJs?yf)_fJ0WtA9Gsk8!r}zOqhHYue zbY0P3M5JiaC1`wBECtZ1hxTMKanSx&#Gws=Vq1!MSp-UUDNG)%ocmF0HiNiPf{SsF za+?=!Z75Fm6{}k|-EU)OZq#`qRU{;^j)ON^>ut3*=x-Abp(J{{*4%sFxp3S!Ed6v& zu7T;NbT7VkjyY491BtI%Hjd^bYKeZQS_eZ6=E zi^rx{B+}J*4BL${O=4^~HQ&*-x13zS$ZlAPaFOiH7+~T&lj@XGSlY#-Gz!n}y|XXN zHoPL?MUR>SAB^}CkXOd%YF%yfSRTGFsF}`CmRQyC};8apeVT@N;WF7wfjGf^H~}s z*Fui1KC6w4eTsD z{mJP!KXwHp`fq2@S}g@SO(aXjGPFL8Il^c1$K3K3`@1QHM%UTJ& zxT5w@MJ=PGSWm_Z{P-z7SM-=nzLe{7Vo$OB!;@7pir=|#^`p!rji^n`5WOiVbNEPB zjOt{$+^VUCH+!CzzJcz{7R_J? z5o80#LEy=E1G5#qJawAWm)E7t6BD~n+7$h@ln%;W<3Ai~c|*e9TTz=|QOgX0J;lL1 zyedRz2q$@R^rVcsq?00Ud<%7rUnYA>NG&EjAMRPm8iDNf>H9?^kn;cM^qGpn=qY8_ zFotRk6ozIXAXrEc1N!CX~XB;pV?gk#;JF2N~PjwSeZ^jPU`ALi3VgcmLCPcVsQAKdf2tgbSRdcwyP<&$x@YveqmspLbF%wKj1@0PY-B~I zB-yNdXS#8_zPafahC8Y`(JXLLm%~L6Xo}z(W}nkzh-gf4w^qtcXGgGII^l}=mT>X@ z`xJq9)P5z6N!rH}Sg0XrdKPz>q=?#ov;Aqdb+=|HTbDuP>cjEzhgxYnZ=c(_1^(S3 zcJ}TX^1$^>A77D4*eGq8gbiKoFaVK4Z{ZNffHxQ;sCJicmN4381d3YtpxM11`S~f`=4R1s*DE{ zTPZw%v|AK*n%i-a+wXWT_W10RUD=sv*qTuBOtd0G#rMO>NOZ%G5;`mK(zu!*KE-U^ zW{Hcot~pXGu6={bY1{>MrYtBwW8KXT+SoQktT<@!_&i*{Oqc#A@}5kA?;VmofW<~< zC9r54KuI82agkVJgOQjKqpzR$2lLwBZA&zx*PbS<1uEb`2{?UR!jxDP)N|~Rm9<`2 zqaS&Z;_bPI+YuzTTs6jiCU_#z<~G!0m9OWS{gKyZe8~j1XBOe%IcT9{U8Dn2>Cnkj zvM00pHvWwzygrw(q9L@*7If=9Qqg+~%}*F7c*T_$a+!BJyFC$%-f3epK2C4avGKYu zJDg>Jxg8Y1Mt}qywc89MAc_x?lU)cuu`L?y3TLpa{>KtfWc5WqTpWGM2Ju5Wx_+B_ z3@zM|$P;K{k}P?&)?=6SWyz5iod^w+BTe*&5fi*ksrW7;bsziMYWHS*%9O|siKr_P zamIDuHsJR|FbR18F6zqJQ6y+#n-}LY!L)ZKN`>#D>kb{);S3qdPYSsPS8LPP!?Ktf zsnvyG$zJ*mtkd+h>`SO=RDsP12fUEP|4Nu6W#o>v_?*vAuY4$(jqzC2p6` ziyn{2^;!L8)^0^zV<@rhU4No;8gm#bccq0zBq04Uc}t6kium@L@5K)#zMd=tu|{HJ z2BIyNWuk%;Xx$!goD8dnhxSLvx69;X%2*xX;jb==Vh8JI77;IWlPE-An-= zm#9JkLzD$igiODf1DT(zrZN6=a|lWRNQbn^t>v?{GQl4wSJAWQK>z^jVHKAC+?VgjDo2%fCo!{?$W-ZF{ ziC(l8>3F4@9~#2`krM^E`8{*k^;Cnpomt$I&dtj5*<&r)-3Ymh)(M;74ChcWyLoDv zCGhj6CCo^ilQ?K~(f|kFx57)B;(SF#y(v-fN9p$!GuPmq0wPSf1_BU1I1!*iV~_4i zlj5SzIr~_L+5O!K5N;H$t^?g@(?I?GgHPEl zN4?5tO-Sd04Kg67xcO^DDHLc;~AU*hK| zF_Q~N^pCLeqOCvCEFMn9095R_h!p)0h{IcI1P}8QekfKMx*!uTQ+dUqpV_!)wX0|u z9QSPKfN`<-2Qm~W2$x6@(lfe^$Rzw?EP)Z)BF%CgS#o1EGR|Bjm~oM}0m=Xj;~kkSF2} zWx4UU?1dh$+phmf?#Ri$`;=?fVepmO6p>@?dYZ-38GvIQE z>?xwYP#Ah+p!)5F_Yi7#t*e&da$g*^a*WpU!vcP{L2aXI6;Agi+%nI;^_(s3Uc&95 zwb$wtdt5HJ-yZf%ck_=|0Q2%YW4KLR}cb@6)JDGAs`>!dX(EcDxx0qxu| zuag0~b=@_a2N3Ayx6d!NZJ9|_cGJ3OjgXg)2yA)8699O(!*!9Z$}?d5w0K_|MX>e} z{bt#A%p%x9+5oMg0a@U1mbm>c&Vu?eo>uD}iTL~Izmrwyb%cwJH7jm)%P z7T9LoM{DG3?X)^2rFLhr!|!){O(a)}pm6J9T^ht{Mh z6o>}xlcVm?G=n9aZe{C6g`$cqXl3Tqy@sG)y0`fX+aU?vN$a4cBbRkxv51M@iZOc2 zDjT=w^DeIpBgh?_E`Q#3#{)#|HCk`2na?rR9t?y%V&B}p0=LWM@fVqptHh8mZd)t*b_I8y3s>d*oOj!U&LzkmS~o3&AU%FC5?;5X)NMNNiX580f6dJ+ejZD} z52U*{+O|sIVcNA?4grVVlkHwmrec9$0L2HiRiVQf3556uC?}L_>kHe~?Pq#$*~9b& zs0Y*a-3+?zVW-0j2Z^Iv)1Q0i{JqS$L?0noG%k(sTu8`-^ePPRq%u6yH}wQNpUoa6 zTs)sm9WGY_sY{mVz2~MrF@l)BY~|rCw)Z7?f32_9%I7F?+e-sZM}e?jaBNCFVNU@V z{eo$H359y5X)_09`Bc~9D4mQ_(emmXTDo+3zO@v z^9dE@x-O#H-NCTO8}M^bW&3j7ZF3Zl<1N=S2bf&{S3(N8b_9doX}Sj^&~t;9z$UrA z)>C7FomhNb&&-*&nG1;4QQAl?Uk47k!yZ@E;k7#hV8)z6s6q2aj1`kz?$(C=UhhN8 zO*`jbYs+0o%XQT9wK^b$euu|iOc#6zm%5mlfclGgGukA@#BiyA9TcgJatG$+|)dqe} zwTB&rCatN!aqZA6XLlbausbCj3T%3TI~1`Oc>JjB(MRG;R8TpQ?`{4ik103bVj@x~ zwdObmVQOJBQRO_I!cJIqw+px3yoTD2K?SoL&UZe^{m@s_>5r<)d0hh<&l*0sY3f_|=$~9x;HC{=0NM0>L9+(-9rW_wM zG5H30AG98H2b0WQ5)MVZZn(&0_XT`#GNV2YY4yE>2JJk^G_!vd(I;XJpT`*r*b5vH zXK{kyQ#p>g292Lr+44O*;&oROq&VV5-ar9ZsU$EJ7yZne=SQ&vv|VE20csd>_<|5_ zSz#)>!!>o8BMWCR&%gLdA{9OVtKo!)+~6%Mz;;ie-S2k0l6Ok&W2XJU0>z3?(HbI9 z3_->Kh>y5Y=KWEx1Ck>qt)h@$81;G$PPhE(OdUr4^AZ}#Z|y90J4@_^0Z4K<(A~9r z1L3f+3-QNGS%aIMu5Y(L#K^zlX@&g8#v0+Erv%&^Sb|HQBFrXR@b;I(uep}a+OG#z zt+4$ep?hmRwI&8MFr`TF$DB{F#=iZJ?t$wGyV9-0942A-j^5bCE`}q0LX@4+ZYM(?CSv*1m8{`hcKfsJ~ zU?t+uRXm2`kF)nM73F`9Xhap2&QRm>O)93MqK?|%n!qS+u#PYVrI_;6<;xxk*uy0r zllQNnj-Oy`n)B~!ok!EA4QqSsW!nykIz$_!wFRdi+%a&RCxOQgVjKx!_A4Z^s(y#h z#AapBxE>7sb>z?MXqEX@mfmG^KTovx)2`84g1?Ue8))u;9eCq(gBnScds6VZ61j2Q zTeX=Ix#|VNM;+K0{+COgHMP6`?sK}l>72||eI^VtKXkY90_ z#L$}b9cMl*Vd`$gOGG-VyGABdyR*=g_E3a_3t#m1&+legk8=`Oq}0v84r^R4bpAP2 zYB1dSFB8EDam{4~glMY;9xKm$E#14TwC7xhQ+5wB!x@Z%cLT|5i!_je%_VQ}$o#$xd%{K{ z9kE-ueZj~yd(;nW#Ll}Hs#acpBo%9`n3!gBYn)}rrR-~jEs{YlH`oIoP#unVoZz#& zOsd*iKF$y5y2VN2_fkoX!?tH6@EC2h)K>#5ID zhv`Es`80EB-deWBgI`y+xH%*_o#42FFBJsS6A3_xCM7?xm#y4~j8#8KcwMt#P0=2n z=69w__nwsADIY!}aZ>SWyr+Ok5c$O5_Ixt`- zv)qlg5TuA+1f(mfdp3Y=2a!(p?!g?un3ho zm)D)NPtf;?_t#C5$akJ8vibf)MBB9qU;vGpG^C?VsO@3~wiWo35O(C+L9G<|F{rkC zy^c}`7*2cp0?+}m%C_OI{qM&Q*E4oi-Kw(7=Lna8E(r$;BUnNzjE14~z_p386Wr{4 z@Z*I4vZ&x?2`y?41s*upn3GM2YOPMzv&#*&% zPU4C~EpRC%ml|-d@gvVOKFoWQ2o|XTN`_;pBqjE)i5}R^T;TTG2^L-87P<}v8Gy8; z9_1#4x8`%%lJIhttNTIXW3H|~+A&eF#+0f_B&@C-G zb0gDNve^7<35l_JeF+(KK+-Q+O!u7eZJ088U>A{z%BX?M5pkf1p!X8jn4sjcp{vLp z0()c#;-t=%?;TIOS-)(#2W(lp3HUl~n3m@dUW;xigp=omkS^r#dfi^IDqg60Ma>Na zmVEJFd9)dm`A_6hLgxJtnu8+){-j>;vtx~si=_27o|u&VIWqz?_Rwdb5oib%^ClHFLWbI_knYvG94#IlfMGoh*6mh@&aZ_P0k{wZM zX!suCVyU6F53<4F`V={nW%gf}P5dvjFgqk3T9}L&o`nmjxRgp5Gjr^T)$BAz-zOr) zX*58gY-$}&@Du*X@d;y?)oSnoVWQPaml0zaf)%JRCL8YmZ|usa8S3_npqf2?%TrhX zt5&O-x}NkQ(TeIiH|Pnv(YTrqw@9%3TcFE>(Y@oZ>%{z;s{3gr^lR$5@}Qh6)if6g zMMFVe8&h>ILqi>FeefIRk#3N%!hx4o;3Dvrym|J|`|XV56FyS*u@ORwMk&y)&;jC& zn$wYqCGLfaO?I;s*<~LSDyGP?b)D(5qs}H&u)qNyBn}~ajUGE^PU1yQI{m*hWx7@_ zBC-n3C}i~jQtob|wo`>Id?RarvUV{`*$zKIA0Se;W^5y!pVBIkML)gI1oIcUitr<# zTNKruH>gxLz+;6NI<_9?lePY9;2!H>aoW;P=zBz*Ca4w-Eh!^E`eMBs#*hYF^z|J}l7;~cb?W#&HYvJL&5Ku2q%wDu|}__OFsh!4@TMUi4CWE2Oy zCcjfrZ2C9kD3)p0t9ie7p#`fi`0kkPSGi!e)>f+)h9a#c>A89Bl)~7wxm~)m&~Mon zM7jw5grI1aZ{+aSJNnSe!-9<8{l8TCjZm_)M*ddunqoCY-hj}Y93?Gv{G-{t z99I&(MhKdTb*6>6tkA)k1JeAj{&BZLA-qF z^DCHgUL#RaIXC89NGUhDv+L&>%iQ36_fD`{$nD%oOJj5?3fr#C3#x z(9MxJu2zm2%9(N^Dv!JOMOF!%cajLkN?-#6I03bZy@a2(H|@ymM*FV`5$#4p12N&A zCB|HK===s2F@GU3F}Z?=fj&Q8!zd{Q)?GI2xyPq=Crx*=twSxI5vg!>=hrskjpw5n|w=4 zU?*cX22`So5e%TI`#&u7*uM08nMCZC3!y~R#bKQ&Wb0jV?6DNy?A15#WrsNU9g!|V z7XvX7(=Ab(R(is4&$o&iw5)uMc1TCC=1(LhYC5PMZV@h|*8KP25K}pDvxuxC`aO}zV zR;{L^@ygC^S++($5a6P++WfLwwfM|nG{)PKY%dBU#EM|_9a+<1_RN*-cTa)d`*5Vx z>&w3OTF2<#ZHN2aqoJie;BT9om7i@$ZaC3@Z`;04Td`$1l#7T2^X$Q(Vl8J@c(f;D z>syj!u~WbUq*;#zlVt0jyVqngwifeV5zxj1!oCpqxjUFrC8iH^NvX@HEKUg^ozzh6FPM)ibdk; zMV4y>PQA2{9qO__(;BEquQsjWg$W60-N)}peu-^e^BPVh>Hc)Re~WFkT;h6dgx1=6 z35T~Rz-6aSOzJCRk0Fd9QL4{4?{oT`=dG-R&d~>nR@OrB&LUC2NmD6?#?<|{zRp{J zv&6(12`3U05YCH=DrjfpNO77iS>JE))6cPL!;)VVN^1))W5-yMaF6B|4`a!KUcVBY zNEU#kWB;BA36f|*;JL7VFa5QW`ETDyY!O_WY$5bPno{1@{9U!C;!UjGZ_aN-HMaY; zMPwgj2LMsepBP*gf5>@bz0CZ3Z@QbwV7LF#GAM)jxQvMLu)0H8GLf#i=yJ-=a&`@> z8LG#kp0DI;BI*$}V^BUmPZ)xQB1_4!{+vBMmN#Ni#mwL7b3}6weh&7I(n*aamy_jI z?++FhG5TBmLGXfpgjW=&5Y#Dw5Onksv)mShDCJMj_ zg~{;*q+0@i(LZv zZ4-TlwR~oHSEsYIUzxvYL6POFTxn>n@jWW4(v%Sq3@K-7xq zUBL@S7%n7$$c5%zAmjUWx(Hv}?vNNrE+|-qrU$5!9n{3s~(n>j6mR7AWnA?FPTT991N2mS4 zvP63%l*qyr6q=zddHl%x8wM~-=qpD=2}_8zLdeo$!4xd5Js#Ma#=~!(InHFQ+GT_+ zWKB#(;H0i;rxZ%Lk~-Yr@DAp=Y>;^9xnyAA4t{_>ZIJ+1Y5nhzpWm*7+8t1Ja+{;f z`5#-JNEO)u5mjO{UjkSqJHQ$a-X68-0@HkdNl2mjjIXy(@jz;G3g3B!37CR4WN1a> zr$(@(!}80CQcOCe6;A^P3rA8-GO8p+l+?|`wHx2YtnRf}5SXyKg~4`tu=jEEt(;VA z0eJQ6v{TI8|4G7&P6)kaNWW5{#7!f*a?m>2H70Z&Rf8pw=T)Fj5J_a?4hz}*1wxG9 zl`MYE$m~*9E9rP8;bW~N+aW9Ha1nxREYRC7@U+A)r0gqKb#or-zJ@uwpUQPan*r(u zyVqxD{ZPq%Lt5W(dp22mi;cbkCreuX!S9u7G2P#o=^naH)3CvpfqjyL8E@z z)tz|)D=O0miKYWwO5zLHuvjv-R0=!Ped6{_OcHumQ6)i)f9g|}`h_-l81?TH<5uQk zyX6;^|8u}-W`s`2b1?%H)_j@m~-Q~;r*I>7}Ygv5xnrVL?gMd%H+v%Zdk%zQGNZDtbn{x z!is{u3r0IoCnUV{NCD-_Sv&dgTn0L}Hqjab)d3S&FH6v(mpVMeG}oUJRPRawN`v6X z6QyD{xft%Iu;a`R{AswrRPC}CmNNx5t1f+sD7flEc`4tqW_$GqnDJ;=kAQ_s3NeZpCDxf^*BvjM zJ&GB{B*p*9O3(hqT++|wI>IHzq#5**6xrQV@d`KCaM1T0+L}4pUp9VI7P zMo-Jf8&pNRD5ab2Eh_W7vRXADFj2bkI;KO9C=IJ|0<8~J%hjt{di4VdiRsn$;PwaL z<`;3042`R>*e<##`C``Gcw@pR%UNYMlujQYDzoZpB5u6og`VoZdkE9RwHnewLJ!L{ ztF(WjhIrBW0Ir8??DaQcW!l#yyqJqXcxuBd47-w+l|p;;Jy!J^mqA2}R%5;823w2u z(+Vv;4##Rw( zZ{%9zwXs^a5;t58P#Ellv+@2g+=vBx?||xqeM&LRDMGI~)(P}}&s2or*>TtGWI@O5 zCiI0EbmX8tR$arG1cUwqpB7+)@2~^yBRSd0d4!Ae~L`YnH7Eif}(TdD+hiG}aDV1hI9kxR2&V$|5_G=@gE*|~gS1qf=V`4vYFuU;!ZL@= zTN1ReJ*5C@(WqldZRa<3{lVI2K9rEyK2wLLiQ`9~tch&()u=0&I8VtWB1QX*f@q%< z^_dj2C7qt=vvwwS>8%JA#V$idO?1nnBFjojGMrO3q~Vn%s6#!bK{xB&Za2oab1o@n|LJ4j+g@!$1m0a%yQ8dDo9vnS_}mmz_an!* zyp$ds#HHq&{QOQs+TQ3cz&{GRBdTI4YMR}fO@}+$5_vcsUOj>1q^K#_iZ<|&mlTRNe}Aex^t^@HC6QTFLr;Kv9@!r zx?h4~-A_n?62mkf8ds6zFuk`B7ZYV{MkWD^+*s4d1Y1{%{@fo!TjQX>>bok@-+6Tc4NHUb+0a!qbCZW6s295>NCA%baz}xYG|W zd!FIm(VplOxH7Ao1THu88?XDC=ko^V6E23saJv?y^Wc&X5H9_m;5bewDju#}H#9(K z*6vKk*ncE2Zd4t;M;PV@%tj}`7(9RG4Qvp}0V5TRmb8t)Ro+mi4m0_8S*WuJ1g za+T{V{Np#aXBMciBf*K%B8*=^PX&H+Jf=b+U>aZikLT4)ra>!pB1mz+2KvBy_lil@ z6^pB&?fiJ_l`K#3vP2Vk3aJ0NA%8D#DfgHMimfI7T$Qg^GOz1~&P1m0x^!TpK!=Rw zJhg_NaP8GI*?}+9g`mY`ksC%j#6G2RE#5cnJukB$@P86hIJ|fzA+(WjdNHelN&tHr z%!w#PBDiMx<#FGYOu@d=m54_Ln<+vV-E2Z?&f^v1(J-hJH_DxOJC`=KwPnX)OgBOo zRd?e`XwGYeT7GVi{QUb8hWB>@y!fl-Eb+>9+FFm{t?aJS=X2wogDK!m9b0u9^OpPe zAUrXCpnor%)R4m-C%Xy<6_=vnbIuSmQKuxL?o!}IJ4Rbwl5EURo6w87;P>|=GKC8s z@s#KzSzz^W*gZ+qI}SB@mNka8?nS8B*OnPWMVSgiI|}wkh;>YwLD?J8;H0G%wEu=SN$V7ywrgKTOW_$UMJC0yR zsZ&2%3XM`NWXwf7zc^1`Uhwb|pFun|ZUcw1a!TpY2J|xTvleGpBE|!DgvP|jPK}rC9 z0`t*MxWCHjF(>4dvZ{I^=Y~nUYq3+kNv?sXx|y*?SX?!Mjan&WalX&%dks_h69&^4 z2$ipIV5)3ckh{~xos=O2DxwmjZjb`2%l`d(C06NNB{8wmnE{<-?y2BITiwX4NF00X zVbx#dy0W76HP;fUVw_RJT3?K&C^v3E9^bL2m}NJ9kjVOI5f~;Xbclq_VqgjQN@l5( zbv`~alt{(S$Lbzmk=@i|pnP;*o-%*D%X!lwhSYwT8Oc-@wL;Py?=@xG>MMza+Nv&E z-^8yRDO$Uo;ora2b#Px+9J_Zo5r)OF+8{1)VQZWPP!3M65*{61_RSAWwA)-qU?|$@ z#C(gXPvwJIjI{s$mQ7e`@IwiTrNL(UNdY+&=qX}v+dW)k0aJ7Y5iBgMz?L-_Zk!aE z;U4#dmM>ReLfB{|L81`W#n&KU7SWDREO11t@BeST+jwnvhXfQc6F#+qvsg4slY$~T z?``d64qfPaq7)svX7RuY#;=qLl;FRoaGD`o>+c@8hedm7qi6vUhSLJrQB#06Ccb8% zu%y(ZJTLJ3yVVHx;<404@x<-Ozg{ni$2EE1VmO0#{Nh zT^g@;F=i|or!WIX3Sj%h)h$-A);`?s5wG6E{GdML30g!q0-7e-BW|p>!rf|cp)RLd zkvo`ZS4`p?KfAtWa`&x77mZW`Jz3a}#*uUE{X4`;d=K43L}Q7sjTO=G5EL94lY6Vk zUpwW*H0ja$6_y5QH-VNAGv|{_tlnc%c)^WEyLV^2%R=9S5>a$b<9G-7vX|I#+_g_Z zMYfI8CMw&QN#N8pom@d!DWaVo(RVM?ndx>yk2ziX3MSqvyx`$jkLtHnXMo#P!1{P& zkT@J*>fMvMtR+4nxBPGF$L#32ZYB!xOgBQnWS`39ls?_?u5&DlSHVHRB8!K>6zdX? zMf~q`yF8fPi`S%?jT`pbx>Ln2z|O zHyEP!BDc?!4X}nSEwLYC&0{}HSZp3^jj-^9CfY0vufbw_p=knpHR5fo;%7LbQ^nkk zapVV7O*_FN?uAYwUqsx>I<9zROQyfKv3yh33e?3#sMreBLP6ETK)iV?zEG~z6KN{M zpgVo#lWmzE`AV)J(uv9n7{&6{ar7Eq(J)R$y+cpr;kA3GDr*X0_bGfi1TTn9{( zsPUyJWqu028y2 z9O7yrSvfuX;%6;bT(fEteTNvpR#t&NW$*`zL~A9qk0ranoHpzJV(M~02`wNJBe;N6 zQqVx;E-d80ii^nZ`00CU!8c5eeJf#OYHYgDe2G9UW%G$@*Q{mfmf2n+R-{{ySdSlH z5S-)@t-NtBm02Q>&3N%g7MN%G2wC*(A~F{6bz=?boT3zB)Ai@8j$|ilzr@BY2vctZr=FiUjPy)|%Ye%fkgs~-;&!J^KC1{#LOBO-6) z67>?e`N(`@uHs`T(a)dFuR6za5Pc#9E$ZRWJ_+qFz}?)!TvOkIHMyhT-}fMswG$E( zRY_xp33D}0xU`jTosxrrR$xb-!;rOicURoW5~dGCiC#>Y*3z#aPJSrGbH|J-EIi37 zB{&i9z`N4{DF4Qf6IdSSy^npjndOM~OF+yK)rDL5EfCe{RPh1#Tz_N{!@GS75h!v_ z5S@zLNL)@=F-t68e^2&-lg!#Ro=SKkKLmKtuz|B+7PtzT9hC4tw?o=XY+H6qSlpJh zQVTRIDl^u0zMW|l|1_c!HA<6^{_y(6ygL1X~K1#os&nu?y@x z{w-03X-8C4A#mRhY&|4fahQM~Nb9qHY)q3ISdn1WEkrRE3F;#@+!Y&$78QfxN*2`d zQ=|{$Z2wyc6gewi*#Q$&1l4_08p4{+r)H1*kU^eSA=B|S3veBg+@v3jTF^5+z=U+} zZA7K;#R?p7?;fWo>UWLy~tpA)_H_s1S=&3g(9_6zvHg z1s8f7+Vl3eHrkmXIVgdJAy2laklk_z(TqaYRE37} zcJyB|CcG55Z2O=m*zi%torEohkA#o`cj4hS@Qjp|l4SRW!S@Vr@x4Tzf+ucJ zjJp>wA>Nt>?M5)o)O{x5iK->A6ho~W)iD;nd+S;Rsq$Q3`j! zjz8p_#SY2LSwtot5`3RmJUJS}g+>!t1m1UzCheDR**C8z(@vT9(GsYg8XM5gNC=9q z*b&0_Uy)n9s7L9g&3XFFKBiF*N@&!mO$}&WV?s<0Lz)~svD(}5?U{LOe)`t?iDsOi zZs+mCXuW8S(Hkq!2*71P?#WPcLc^Ho5_QGJYO5&N9g9xaWl>9u2WTZdz=h|@^aedV zc7cF(UN@KD3j5IO-OjQsU-k zU5O%Br9c2<#d!a)A+H}_vHL7*&bj{)qD@S$Bcu<8V$9=O%H}BSdttz@Rhf}&{V0Kn z#vuet#e199Y|H&DEPPMGVOZE0;T$MI^m;N_BU7WXLZza!CQQA#GfP4T<`I>cgl-ap zg)IgpgEwKBuenuj{r3f6J`R`;&5H(8OK0A2ALB!f`3fIu;|i&~jr;u7w?-KsHc2>9 zI|HNX599rhpshUcZm6YLiWUol$~IfZFJZ7ZEg&idEK+$X_c=5j?VrL__dgO=sBZn5 zCZkgzoYEzHKaV^8Ej#`z7ZS~Q{Kc?=$m)e*vO%}>7(2#a^OsmM&Npe{U@3^DK3&{jiUyY>>8>)TRm|2}qB&^8G>ah4YWUL1^S!N8X zw1;0cddm_;Hk#|mB75r8Cr}ej-dfUKr`qqYGIq3Cs<1;0d<7txueY}AQo;#WL{3Si zJ+yP?JFJp)Si)i@sg+K&gr#O9dhnFPE7==1U3H}MhVd*ox?>s9jloeHzJM_fl+qd} z8mC{+GDNK&r!~-QHqzJ76T)y!PeI@!Szv`V_uUq5z!b?|i7Gl)3{?C#9Ja$dzN7vU zYZpZU?=~j$Ikuht+;c(_knUw@Bnsf;opA}01wABMy*-RYPNKh1qW(10DDdJ@jDA}bj zd6Zf1^k6pbF?bbGEXF;iIec(yLt(P7SS=dY5r=W7}HPbK75oMbEYr{5?{4Ui{&ILiHkq7@NL^jJLPtX7{QCf6qwEt9T@I=mI%KkS)mqjUjL>4045f7 z)+P{(LVWMYSrYfaq=IYm@lI81^*CeiHi;|P3#B2a9Z)fe18EVH@w0~Qx37C9izS`U z5xE#kHl$l=bP0}k?-C{rTRh+M3r6Xc>l8s~42l`9yYcwt&|iGxkYZC6YjR`Oq8aQ; z-t`g|ujGw`jn{T{W7@8Yh?T@+!Gani7`mbPEDy_0j((nK$LwTU$c=Iqe-Db13!-GB z5?hbo_wnL@!4V2=WKyBL>800n|u`bN|`d)m2$QAv; z&@Le^EKK3X=FB>+`5WA$UR30vF4`^9HNW{b{3xgSVcmZ~t+xF)kmHz+waShK!31d0TJbo8K=u$@J#*SBYkk;FbM; zaLc?YU{D}K;P@-)qIokrA7zF=eFH&?N+Na;VnG$gV=hLrrG3s#8~788Nw!OPj7eG; zS14dQgY8u{&h?h7mJg6Zy{ielbXcv;BJ*dyf8a{?Cq2g`?L`SJ)DSd1i#tqGM6G##X(`*fiEk@g*O187 zhvVZ9wbFK;ncJoj{@uzrd+Wa6$^)@-I|v)4ErYP3s~rX)Qs^yws^QuNj1f;tSY(7G z83WN&P?t1bbqzZj1K&}IZKxMrgiOFw&d{j9<=0Xi+RyKC1%v&L!D8Kz_)v^_Sd)gZ zgqO6gEn;+5i1TFJU$QCFVm&}iM%IE z;BLd)9AjR4{oMqNUVC$t1cDV8i6u4|i5X$bzAKM;?OP=@dhPXvwLk?NEC#2KOPCUi zf_gR%S=N^ix_XfpDc+u&w1=q0ma8=EXM!gZ9UfCXR{46K*>5oT$Sx+Zo%RwQ9@<7a z)eMB&Nr)gw- z+`gn^<7WNN?Pq{l?RNPOnWUu{2QG9npPm{%`LoD~H_D63vU_An_Nnq>|XeiFQ5P3j`2j@-V zu4Gxpw{*C`xVuB@eZK>L9MIZB$G$_wZho8nBO-gEc9YfsV-b`Dbj9NyT+fTejhW)b zC3`M3mw1Z&KJ<7*uFoDQwYM+GHc*W1J)Co38*>=Se@qLDNI?2y@|G4674hx0-iyjV z6-{9oh&2)$GZ4+O91s1`Vd3Y1XC~K37QV&t9*3FH9D0CA7e>?L!+nOf!+J?k2A2TR6n+&qj9+=xHr$amSOf1|~>oYbYFzYk=67e`s z!gZhIi=Pryy}|4i`m_n8zm;*jhPQS6>AcVJ@Kc10@h66 zU8{d_2ox^6`$u}PsN&|ML_S6p8ECvj=U_^pTWmMK#Cq1td_|z*wLRR~@RZq}5{yBWc^9ZyKGp1)0j;Ot|*noM(y zIYGdvIjHy<@71-4@wmZId<1C}Ox$P5{x>Ke(}nGZTXgNjW;b^u&9UpX)gx`aPtrQM zTDDdj?<*7YOZ*%qX7Y$ei$<{W;x7_QJe-OFsMv84DFz@Ahqu%S9_A+kP^>a_K_*_N zc+I#d8~0rHHIa;q^Zx+dYcC1_ literal 0 HcmV?d00001 diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766572119.gpu-51.2678290.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766572119.gpu-51.2678290.0 new file mode 100644 index 0000000000000000000000000000000000000000..eb24d08f2861b6250fcd73162aa9119eafdf428b GIT binary patch literal 48602 zcmb__2Ygi3@;_DGtANymKnR^6B&3kgq)P8i1Ob=L?q;)O8(UHk#V!ac_QFF^6vcul zD2gIaMQmV40W0wAsE8Fs{Lh>_XYZN2dvTZFKl<_c!MyL8J7>=HIcIbo{pa|NqgUCA zI(*&o@YH=nvl^9!9o|6xh$?p=+P`ROzScA94n%_CvY;dE%8$hSen+@AzdG!Vy2JTy z{4IOA*XOqDf6^rI_o_jJyZ7B8U)dV9Jf$Pzo>i&kO$7rLpb_Qgb!&&JLxHt>i#Ch6uXfzrarvFZUxz7=)FxL3zx}%OEv_@{mFSpoQ zwV^div|=r@vdZqPa0CKwpNZr~arF9yDl4vDHhtqb!n@jb$E~(~5^sn$NbBec2V)`o zf|xs8Yp{gVt!&-&a7>Yf;_5G+K8K(mzp~3}+aqm>)V^A&mW^E2fyE*wdQXqjTUXh* zZiVr_)@`CNL_Vg8Fwb5FSSj24)2cw`J)7poT4qrrJRr^_=goe{1=gi8# z({_0WTBM)WN6VxI${Z0B(M{t!dumC6_%Y7Su3L5;W5Re_N5Vv5Y$J(whn;Zjxqz#@ z$X8U;gk2dM#t@mMT^g^kHSA1~gS4}?9D?))z)1Mqjw-k5yeo2O`u-7TEPh}X0bi5t ze#-WT1fHmk*9r(Y;$C3)fie{fgn}qOpsmUs&S)^qKR`L5TwA~1@svWQ2RC$~FF-w* zt?y>YZI3t|J~&7m)tdfT-!7HRxXkTJuxMP`-dAO~z1!dKv-4TM8{y*l z?BsB{l1N>uM8DYhtMVzt^#9zm_c_}-2|iLAu66f2D&6*~pwm$%tQQ=cDsRMF21dWE zmM@`D&ou4O&jRh4A)lN})T1HK3IxM`;POJTjT}*t>!}l_O@v=GHHtQ^+b-FfHE75_&?ugeFbNKAeAeb?y5Ngo8QDenam%IIzb2|*9pRVbA;V4J80UJA-9$9Yj`#h z`Re91Q*D!b5|M>kFRgi`+UqfcDaV-`+^#czbq*%1DefwV?H7sFTPxC90Y7W(5l6X6 zYbtPDJLE_DzR3i3RUVOs0-Igt4oB@}-T*3l^pQ9d6;w`S-)%eVG37QcpHPKzi-FVS zwi~wF?g+T-!B{jDi`uK*UQdPD7+NlrG3Lt}rY#Q3g@m^7$9z#Q2o<_{hSB98p_~zu zC~vDZMTYAw~povmkJr`&@AfMwZ&W&PE-uSs7m-){93J4Q@=QiZ#rUCfUo8D;* z@KY7Awa!jE`1%fO6;_}s;H4iw@C>sTmlhI*XfJZWkpPiMANKHg=Q-=TZ1S?e~1n0>wf7Xbll4h9P4B#7EsI z^MRPp0m%`QR#C_=kNJECr)xVqFJa{Gm(WOl4`+qjS!pj1LXyLQ?ylVzj6{T8NIYK3 z8rUm2e?b zCjWt}PN`+?UiH~TKDv9^#>V>H(TbqS%qy;GEZ~JbQe4xf&t1Qt345D?1dPJo!2q_( zT`A|Xw;?U}KVEkJdBl=0x7@tk_FoA*UK^|BJHgWx;RPgc!qRg9p^3W`d|NFA=H`t% zmNNBTHi)Q4_1+>B4r0oWgIVw6^`l}9m@m~}FrkSo9-)B^a);m_U`9Ex67d%*9>dhm z3p+9uwLzkyipplF3Hc@!Q&CaB%`D!=D4jDzr4&=1x_sHAL3^aqYx4dT)Y)^5ON~q(sF=5v2*1I~5bsTqxV3$qptME+KO$m^{1DQTehV%J>3B=IY`2%ed?(GY})e!o3f1^E?+Nerz?Kd)-iPNwdb zjUdud-L*HN+MVU5w1*-bT=*XSbQIA$TpOl!G1D84*j>9Z2avEcZ)S{bt1u*xNGlR`z>R~yfF$px=MDG1%pwrxeoNwk|gmJ_<+-?_&6Dh zJxkeXtUs3E@icZq)!_`qz`KFuwM808!RC_p`O&;V4Eq5UyQ|wDiq_g=0azn`-hEKD z^6?|7SX;%!tpEMmKJ2)RA4hbHWRS}Z_P`HRN1|RQ`0Or|sa)sW`VdP#^{jWXlWp-$qhS_8plpR%Kuj(IKxLByPcp{OBp;8u+36Y4;ZNjr;?3gwK{>P%j9+LqDx&=IdE0Kp^d zWV<#O44_eyhIF(EwK#5IyMsRoVMn1I)QTs7LABlIb5uFNaN2v9fewgQwhedfz^}j7 zGImu?RoUftL@Gg-M1ti}EFqQ0B2aqZ+C4dtWM8#a-ar&n0@%1@5ZrP>=ygOX^W>Qg~}V7v?YD%W`!CW)ME+>e`?k z6BTPrshUK>8uZn5KQCmS-X{`NczRga5sZc+l-pqvbZgW}Ki+kmsgb4U5vd|ukEjcv zUXI9IcC8_Q__)LUOwr}dBuG8O9*TPX(A!`lQ4uaolo`it?U6q|V}|=32`60ipo9~y zL9o2s7c{36tx3&VFw)M9UDYfi72UgT1~%c^%OU2%Y76Iu5?vIk`O12Svbvm4x~X|^ z(>7bX^9g>EHbE;uInV$t`mKb>uk1TtyZ3iXv08 z!kSsv@2fe&M5TPTA}U!Sun0&wx|1eZQTCSWxhLmL@4{kp+Z=+#*u0H|3^^d_mnvqT z7E3%#89gE~Q5m&$Iid~}5%gXX8WWUUHgpxa!(fjLL45oByBE!--E330-Zi!jm@~Xaw3p#XJW6QsCba@u}k8FyiI9EDP7)PPifqhhpIs zWhAFF26ZK>#3!+e)}${Rx8YRQ9&%7Ff$br=E_Vn5cJNJIc05{sh~43z;BK4ye5_aa zt!dUXj5*8ZsmwuGpWEYb)~1O&s?Hs?Lr8W+slnkOTr4%T@z92!xX*58gY-$}&@Du*Te_s5XS*;2u5i07V zGGdHCumTmvRKtB&<>VI`Y9p65s@dbWJawJd_PmFw>kSeW)pcRW8*-y@H6Lz~VEMN| zmj|N*`^~(R`86}#L_hj9S*{`|=SnrrMMBX~kk`gkoy*WryWj0SlzF6oN?75*ODk{! zcuU?mzw>;}IKHY};dpz56pd1#U7-WS7c-|LlS|yY0rVGG~LI)~aeV=3FK zE9e76%GQi+r1R5SrPHfBuVaGw_k3DK_z}=8it5f6QYstZu|fm=E=A7(VT$|?rOY9r3#tKfQ|8t(Yj!{X&y5|KLHkewqe1Hy3Pax- zW}hXlg*rbo9><C2red;Qr~yrwO=@BFY8>t!ez@1 z5$FtUy4G6-1%DQO3GpF%wkTQwg^Y@z&*XP1icSB90>v_&arD!L{b|8ww|slR_Lf|* zmzJkBjX;sslJvr&Vx=(lTI#!PO9wDJ@rOi0d*8+v)>~=d;}aT56jUelC#xOv z!}>RD&*HSzF`^ISv>ZUKj)0lKCY@x+E#v3obazb%N}td_rF!;ftzZ5$>$DhEMew4j zC{A?B>BF1m)VqUm;gAFrUQZk_l2!(>^NK+}rkqz)6Pc)-J8&+fg*+_l*>jlN+q;I4 zMPi5{Ct-r6qImRyLLQ5CtR-BbI1$$o_Cq&E^0-<#W+-RMiFmHA){IpGcS~Tb1hzGR zlTe%7OUOL(*a~JhE?P*WqTOg`AST_jRg6#&P2K-tp#~SPn9d~P4Y?3XL~9(@ zi9xpBmB1cL;T?0@@ZZ=WUT`syE)KDQn2hO`s2v_``wHWEx5Wz2n_`Cw*V^6U$}9VX zNopeuSU2y?kD1u*luL-x04`zSNGMbnfnGNt{~X?DD}xMQLR1S#NJt1&5wvjf6-e$n zzq~8#V@kKxr35Lapd@72+){4|IIjCd&l)-!8(-O$Z+lJx-d0z;xvthUF*6v8^R}{= z4}}q8MKJn~{Dm-k=IRx2S3~c8BwFS3_ZmOmF=J@nq=0({w6q5Ud4>5Uy$s2XB>V4q zy@%zAEi0g0L^M=n4}}zKxw^rC8H}x!mzmg#H0!Zoifp}O{^>0lTeB`Vu~k5(&eo3( zJ3P!s-6{dmN9`Vfc?wEKVRk1q;o_38V#VQkY+%WG1(A#cOI=JWI4eOLC8q8y#p2kg zx2|QWQSC}X7maf!bm~|XizL*GEZ2ygecc*%sMpIiP?4TxTEhnu641Jj-;w$ft8Q*o zb^%HEo(+NLY}2oz&zPo7(Rx@f;qZBaTy`?!Qr{SR3}Fn3QhmmGzo7T;xh#kOhg=PF z_$|x9JB!8wCQYRn8dLY*cNcy!oFyh!UQNV{!~}%%qM{1g**H?1W=qz;5cy{>qjby? zf)>>V!DZ|iOA+qfbw4{Da#~LV^@p5O^-E&8;^+#Qe7#mlCNWxHj2B z=z}z+ysi1Wr^lrItlh8l8bZc)zdS_tLv{cV_56v!W$}lcH`dGiS%1-wOa>3gWl#o7 za2XNfVReVHWFlR2!sV2mHKOW6H~THnmF z9;dD#Sj>9#VX(0fWD?@F0>LZvZIgY5wR~>5(EAQc`>mG?inO0tkSlQA{95gV^-6E; z{b~=Br0F*(lGMd&CCiQG2YG|L`)NbSbcy1}UlLuPvk--|G&W*xlO;Yz?dluO3t3})FmuSG~^b75?Q#iax;`Aj~{>Rirvf-ej$;BCB#}GWNEQr3YONM z2y9Jb$v@?{ElO+R`QI*$684*hpeE(MF_I7KySOi(~`K5vaej# zUG~y*4(9C6`44@EXfr_FVE6g$tRE`XZ@6#Mk~7$3Y28(XjgzI_lJ`j@-g%N-hK6k` zx|iwb2PC+t8_N4$PGS*7>T8#6%3^{#_AdGY6x8nVf}!Xzh{sSVkIo&~aQYDDv;QFB z(PuA|-*8gPC|{AY@XE`WEZuxJeT9%EOL$R`$4Lsl;fgGAMft~;X`|Wa^jl5qh>0W2 zmSN9Lxi<_Rb$nH;sV2*1cJ1;uW(C*%av4!@mCL|=F#nfTzW(UKQLNSH>U-#8M5~WM zwdjJb@`j_>z7hYbB&BJ~54!WhH-BQDz@_)nQljYqmy-AbHX@dcFO|j){W$CMWlR!g ztWhOFjDPA=mHLG?co_BX6XRCqV!P!RJ$U>K4>LlY@1u3l2xZGIgiw(_o5JPGTCv|U z`?=qlVm~ZlS&N&5m8tb|Uo2vptFd0A*X3`IXCcXR_Y>(NBtf@KpCp7{%d{pNJP&Tz z!F0)@wFHd1LC0Kp4?O0rz>t5W6q zrX?F6WL)1akp7$(7yBb;(gb&UhxH1RQksd!Kkm<~FoSP9zc>^|ro zH3hMXkx(>WpV6-jU_3u#ox=03RJ8}MJ%HVocKsHigd1EGG3Unf{YBg6Gpe7E@WR&; zjpV{AlPAl$VF}wZ>*;Q+fPBG2M71c`yI`~fbwbiRj}%a@oIl%q>o*Lv{=;Ta9WaUY zvIKqW!exIm&9zp7>Rl;7X$bsyqEyT#SHRsAcAWWvKaG@`+)wLc=|8ysdFF48euT)^ zi}tV*)HK4_!4D3QGYCDGIO3bK04}RNRLGqCLlPgI{0{Q?V4Bo#_rRQ9>N%S88|*k> zr|?T$=W^qtM7p@nB^fmM=FtkH{|kakbN14jQy&&t1ju*}z#f@T$;{T~9R>oY?>mH*I5H2Yu&7haWV|Ul!6>hNM zpzk@fHFFkD|KU3p#`NDn*dmP4)AET1Rnaa=>E`0x9Y3<@?Q;nxDq=pzLg*2tVKq*m z<$-d(aqL-^UcK^hq7T!ny}|7d!p$$@AQ>80VX<9wQS!yCxv{gw&L`L%GlfskYN9f$ zt|sEflU^uu^T?~19zHG?Lfh9>rdg%^6E(yW&IfQke9?xL#jH&G+D7^gEYr3{cxuBd zkGN8nmC^^^d6ZSXZrentqSaV$xxvAGi|3W*;`0syG;eSuw zi4>`TOEppMajY`x_Lzo{mF6j|o8m7U(fjY-wU4MSLx181r54d{1&9|ApWwtg;JG;^iR|AxX`r>SSAObgH!QMNd z`e3iFfH_6zHOD%E-tU=?Fyudc&MFpkocT1Zia|#K+GEuaqm?Hw6;0m;s6jKarIdu ziXc}U-Ki6`GJSnAqL_ydrivDr&M@9u-+u zQj(`v-S7shA&=fhpjboBz;OqAl14H$3#+C}Z9ldQvF$k|;l%dnaCqb8iN+{+2DWj% z-|k^)?Hm3}6pHx~h!Y`TOk=Qo{_z|7vo4WgFAyF!-(=vGC8$Hara?FB-F~Pv_V)Fp zm;>J(IArVbA^~1rSG%OH){5+z`S{Ft>_7V8CGa>J;x3vgeza7j}d<+dkHE=dW76q?2P3R?DE*LCJb}s%S5Qa6mX)|APqW? zb^LE8E9s5dL71Wi2zd zfa}lU`U`NH@dhMov&3DgD>oG{V8^`lRl?NA#DZ|gmk@Y%DZ2Hkte^42cMR_{6)%Hs z$Mu*GrLy%!1-+HapO0RDydHBVu6T{;6Me!mXPq|gqW0ZxW4PU4S8-)lHw9d7=5yEl zJeh^VZ%bSZhv9ZDNaw*NA0S-%J;4c_P*glzxo&8HCO2KVoH4fM4P~P;^&VlEA227u z6g*bA>r?lGiOd4c-${6)iwW-%z}u4WCIaO)d}W^s@(Y#gEB4&8<5y;Z@^%rNC@sSH z1@u(lHz#5$6auF4#j7`V{fBAL4x#uy&{oup8pSqd(VIk|$WuW5 z&kgx|c}uz1JWy;c@qONYybDX^bbO0|h1aD68wEOKEa#~;^p4z9!tB7mDxt+>ksC%j z#6G2SE#4dV_x3P{xB6`&S2(EQ z*j@Yf)67J<-zN%nmjW-^G1}@LXcT^>EldMbx27?&-`L z!=8|+*w>aDM@5;6KsyTdM~HPynnBqu?(D9paZF&P={k2xEDzwJdI)04*}B(+(59AKCD6<|>{2F`=Rh)?Q~O zSco!6XD9O+tn7P}E2d0hxSy)H?J!*d-J9SvKnB_sbehhx^%m`Us*lNK{6wKQ2U4am zz6^a{=|&;D9O-^md}bP$kSqJK_E+r`PO|D`i#nLe82a zb-!k(`fa&}Xs|Wbh={8uuu&_GEY9dQTYEBtx zI5X-7X+%2Xo;>ALR_T0IVq&E;2Rg~zHQ+;A-N>v+9DD2G{_(ddgshJifnjn&he*UM29|&&qw3tpIv-zhM+JI#wF2 z_a%X1X|S_?Qa}y`dYahtSM4uh7WRILEiA0SmNgh|oD`Yi9`}3W-k-^YaO_t^E(&35 zd<_C-5$*WI0!Os^{(H0$yI9Qhg9H>Y6F#+qvsg4slZGOC_{VZ5bLdunO_ZWT*Etb5 z$@rC0ffD@pme(4xb?%-A>N3iQeM1X~Fq{^^j+!#GF^M$;g(X#9<#~Y?uXo8~M(Z01 zEbmmL^FYglFPqTYZ)@Fj;94P|?G)3?gr!F4?ljwe#746sA_32qKBIPmcw8%tOK z(*HXG#DFGK04B>n+~s`6jI9z_WIhD2 zxVps(*4l@w_doY+<_DD@CYnWbBcN%LJ?h4KE8MLH7wU4V6}f|ncEu#VS=#msCU=d0 zB6QJ6CD2oa?PDCdytZ$yWhK5RB`lWsdRh?;4?)3^F}b&j{1q!FCfKb0dX@&5b%ZDv zGv^B`t=?l&c)^XvJtJQq%0k~iC8Fq>#_xOTyl_1QMw{j;)-xdcv4)2S7N zl_J`N{Gq2ZojLv&LXSIL`U)oAD!ky~*uplAhcmz z(C|Ul%JHRKNHoGFE=2uhoIBPBeNex31K3f#_ILUaJc`+fFM5L^YWKMPrfh&UtY6Fh zi8YTE{z0(VJk|qY;R#K&Sr}e}#r8qd1omnq+E~TUaKviF+>LSMf3@{E%5ppJ$#q1; zt*qlpM7Csl=RGgp$y$M`juQ3Q3e;6W)x$u%c`LC{uGAB0D#W0>#gSv1m>$XelU5Mv zL}dkxV)@9^S7t^)Mb`t&R;|>TAGdu zxfq1}%i|IJB#jtxA9oh&9J7Gdd(;#8VY!WSU%I3ZOBk$?(3mh#g;1mlD40+B_M+t) z|Gd@wVYbG|zlnU&Ev;~dx{pa8q;tLQZ4*Y%VB&a4t|2iuekc4x)=NPevgWjb8f_Yp5#uhgQIbs-DU>z{D&hhqziuRZcrz z^U(7wuGuSB5d+xDD$u74{(wicRzmw&stY{!&7L=zx?Ei6RQ(Zakr=@Rq>_RLB6oQ? z2Uc7}cE`_}!q_cLjdiX|uyHlEP-wnnpq8>Zb=~~6EZy>|L>1{4B-R_i7X;^fMJsQ@ zOJ$bG{qI`*1`EuC^@v{4vx~@Bz}Jm6q;ouJ#HRCJd;TYOqFU4^Y&=my`RX+cXuuOW z3iLe_?)6og3SnYSZ&EhY@7;CUO=VH!7P$sSk?oLvI<7A0FHF4|dtQr!Z!>FM)_`ai zjh6yfcj%ms5M(9c0g7ZXpe5f|uBh`K8@X!OknqK=4}$MtPUAiYZZ2F2AQ$7psHg+e zWEzSqpC)ny)TW z_+aAkxJhqaz+BUDrxCRxwfz8n3GFYy-NN!hQ{RF$xmzyyY%r6x{u*JT ztaV_RFjwP*OI!KYDLEKu1$GoU3|af?wmG|5!t_&#jS171`W3{f52bVryU$?Z$)%?g zsUqHicc+6;{!JVwusqJQ59}*rIik*I5FqA=TEi{;7Kmzes`!BaeDHK0!+S@?%Y^7u zpqK$|?ksks)XxPA6FbiCT%u*%%Uw`YUxolfnHzruzmW(P3 zG%G64Y&!Z$rcs`kaHvr_hV_TnPvqUH$m+cQPWPu<*-Z-0CPXOSq|n@tm?mkof?74i z^DIj@YiAOa=+DuiaJV}$<8B4H`s=;Bn7&&tA%(ufDvFrb@xrVNK_PdF{G0$WhG|$ZUMg0$fKVH{}PTj-0x=2@}$9C8Y4h z${cX-A~-T)5Nv#*9Y;br0~95~6G-RGc4ja`vb+Tmi-x3~LM%=xm>;rGv?qKNTtvXG%thT#Tcgg5yJOhkz2f| zN9m=#BscFk)2Jic5j1MlP6o8DF(D?0Ax#dRT%dVp+ze?Fm<886l;MA*bk@iZ`C5Y19ousNo+hgt@X#b^saNhlZu#_BDe?E^<2+*d~rvW$CN;D##|0`l0v4U ztTpcBoBGseu0qF7gp001X9G}Pc!A^MCMGNLr*wR5y*jpqotal8vasle{NckMR?R&M zv@$1f*Y+n~V3nrI&P22*7;HtIs?!qp{K;}|H8@TIze ztNKjO?N>+tS$1llpKKc?YQEM}J2N3IsF251XqTDCA7R2Zqbrez!j*w^C5v2@0s)K_ ziyuN;>b}4Jl`Ab5I$#sPE!%&QQTua#;g?+aT_^6y2$$PpHeWGy)!P4>GUfFu{ zdn_y*nM-gO7Ir{52TBmVo($H=)M%_wspzbWCk)!clF)}FEGD5l#$jQLLCN4vSmtYP zm0SP)L70yNrbF|h!PHMqA3c=u;l}O?A6nrG>Aa1*E9gJM_)vP5!iQ`ynt=%3{|MU3 z3-5+nilu0=FsNKu_kvOeyNAIdl_ztb!)0xj_h71fQ4gY4sBZn5CZkgzlGY`B#T}bH z$ButaPXfo|FNO_7RxbjR4Z5Yr*fH^%zvPk;-Mr1m48w~O9SuWQj7JT0NbseDeYp1b zOcs-R@`!Q~lY*SV#o6-Zb;~U@nT898wA#uDZjw)+B8WD$;YnZ*to@X~khxi61I|*I=_&J#!6Cm z4ol5O^x!FnSF$&3y6Ub={u^e&QJo@M0E45Rd;wz|D6KVeTQ`21na%s;8fZ4#>ucx< zVYsHJEO>$}u)>-S4LdYrieyYNQ7$@G3{?C#9Ja$dzGHz(YZpZUzvqP?H?i$JB*B&K zG}r-do+(eFY^P#dzPRxr{+`Yay_N0E)oF&YKxkrw?6}U1WPZIOhCB=7>elri*5zAL zLJNqSQgLvF%Y}l(orz=Dyxy2OX&*}@bkaIi)Rw^u9m;^1c%7MJGnrHTfIsfq`6APG zmz5HcqDhya@maAnKz}P3HJ6D)uRe-6bS6-2OA#-Nq&Io=#H?=)v2l+NB)Ay&sCD?^ z)`s#_U$NSF=JwOrnXBnbq>6+D)^YGgYrU=30sU>_A(Uiq*IIk;gR^>F%+gQo`_USh ze#-XaYv-6Vg*lMKs%1tzC(%e;{E?4u+eV9(^Kt&gGxJ%_W`~5woJ~6cPg&oS!oO(l zZ@ZY~^7JS2(Q;*B>cdgxIXT+r_iL2HSYsQYvPNP@7MvXX^WQ!dVe#0b5?;i-6eM7~ zF{Vk(4X4&Sx^>W1D;e1{&Q{3IF#wb2nN+8g!qPmS(kOiDGg)7=Y{M}LFM8Az_;4(c zB=2OYNcL{K?L4MR?-@vRqbki-fWrq(hi`Q;kp}dKo0#SHV!E^Rk0?k+AizNuH?-hMzDA#Zvt$hwrd^Nc1=XA zBpwSE)EL1~s@A$yW)vnOIcrE;c$W${lU;KAucRT$3k(hCJk?RG3!g2Fp>~O zUrNM{CJ90*;eU&kbjZttf8|T>>3Z9B{|T~e-*jae`lhbc~?B;Bp z_ydD}Xp|xoZA7wEOk*dCv7+EdkDM;Ywhfu+usWmXy_9CUu-~tlwn?K2`qsMIO?9<) z73F#|R^TU2>3QPEWbVmamt#D|@-I(T85Hka+4NIpl75yHi{2EJIea9`pgL79zwCOq zh;jM$F$$ME=nWYs8uGT@!2VyH`v*HGgU1rII482-4{n(^1q=#g7#x2kT~zV)<+GUK z|58GVN+Na;VnG$gV@^b}rG0KVbKRXR#JqMKQH&vGSK|r=OlOeZPrR`}Dc}~C6!t4t z=5Q7Uclg;h_8w0Qh{`as1Y$E|9lnbUxTrvF;5ehlC$pJB*&~6`v&e!WM34;_2Z1Nw zP0UvG@`7r2peV1(XNXX1RWccEEaJNc1UYMOY@}Ur2SX$gy>5s}<_R38xqTVdAF~QO?PPE@!BTBQ0 zw}UEY7tRS>ziAw@tKMt&9Sb05Pb0!bizS8RWRw0dVk~oDWrhPp3wCQ`86MwHn6;8Ytn>qCRQuG_sQr*TkC%kbzJ9xv- zC2B>CgdqXe(jmK(Qz-Qv*aD3vSR@%;U3wyT5zq=b} zZ+Y=7p4>bxu~FJ`2phWEVE`hH-ooCGMOrdOtT|s{gd`aQ(R5JXI`-%D>}ZUhO{l_n z=tUPH6Y!KXG%9fUwG@XL_Z2%B?Dq^7>xRUK3e3ZrG=wGW;q}|LFdnR!qwoOIZgJRZ zuj>T2-|<{*Bs$_gb|(7FB~(0X9f?r!{ctjp-SDG?&Pu#w4?Vw&*}9JS8e|FF=y?}@n2Mr#dhwGQ=(*I=MlO^zwL;c=jUVB$N0gJW)lmvnm z7l|b|7|9vY`e+A@dF?wTG~`G%aS7v+=PZ1oJ{>;#3XN1D!z+I-5X!)-?RHHrbHf+h`JIHXI%GfL%|>flaL4C zqOP1BMS_;jEjgVD=EdbisqkHN-J#>U+!0gxNg>zdYW;Uw!3?HGW_k#g>~+8s!*@6I zG+AOg#B!gOwp`wt)qxvT5HR)#v=e7th&&*}gYzbFSF$X#yE$B7+}&aIzTel+Sw7iI z$G-IRvjeuLB>Dx~9IY+JA}9&yiYGj{UKEcTbH$5G_Cjbb@p=M&^ms(B&mOF@_b%&Y zpcva5Ik4nL=IhPz(q{=@Px@o>mKG5e@$I$Vi+-~<V`r!~u0skiPL<$&{L)gD^ zqNK36ZvnfWYE`ylb6+|)Yich#&$cu`6z6F@unEp|4i&Roq?TD6wRb+qjKr`Y;h@!N z1337;6<*Sm;43QXO^JeA9o~18xdw+Mpl}TYAbfB#K!wIWrAm|Hf#$D`V;yEUhKNS& zFzX3U5r1DG=Bvdk!@#=1_=vSmRvf$SJ!dpyt^55K&ByP^S+`$5&s! zKFjuen7*f{c9zyOan?=QONA}p_HEmObu(5+2u5@>8dCsVb17iTSS9_jJQ%Sv@4C@GyiB;#Wc#l~(_`WvQyC%v++k|>O zUE?nMzhO>IW({GZQ_~4fKHmKbbHO6ZswZSjicb+n#R>ai<=;6>;lC<@g>#3`Duhl3 zck@M+E=I7gHYve+{x%JQ4`kmwf@zN43keuC6cs-cy}GV(9=DxlA9ev{raunV4VV=O{UoC+FO_ft438zmRAa52s=P zDt25%ia`j(;Vm_Shxv&h6st^Kkja;^tz0#bjeDN8h?c=|&z3G27fXB~LxEC`;_!VL q*D|QL%ur(EQu+8n7dAh}I?JLL5mne()&i*l4=@#^?=Ss!%KrmbFQOm- literal 0 HcmV?d00001 From 8588f2f8914c44272adc8360b86638ae4d91deb1 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 25 Dec 2025 16:17:45 +0400 Subject: [PATCH 02/39] Add TE-free attention path and training fixes --- .../core/extensions/transformer_engine.py | 98 +++-- aiak_megatron/megatron/core/fp8_utils.py | 8 +- .../models/common/embeddings/rope_utils.py | 6 + .../megatron/core/tensor_parallel/layers.py | 2 - .../megatron/core/transformer/attention.py | 112 ++++- .../core/transformer/dot_product_attention.py | 244 +++++++++-- .../dot_product_attention_no_te.py | 169 ++++++++ aiak_megatron/megatron/training/arguments.py | 4 +- .../data/multimodal/task_encoder.py | 399 ++++++++++++------ .../models/custom/common/local_norm.py | 12 +- aiak_training_llm/models/dispatch.py | 8 +- .../models/llavaov_1_5/llavaov_1_5_model.py | 23 +- .../models/llavaov_1_5/rice_vision_model.py | 120 +++++- aiak_training_llm/models/qwen_vl/adapter.py | 7 +- aiak_training_llm/train/arguments.py | 2 +- ds/src/params.py | 2 + .../stage_1_alignment_llava_ov_4b.sh | 13 +- 17 files changed, 984 insertions(+), 245 deletions(-) create mode 100644 aiak_megatron/megatron/core/transformer/dot_product_attention_no_te.py diff --git a/aiak_megatron/megatron/core/extensions/transformer_engine.py b/aiak_megatron/megatron/core/extensions/transformer_engine.py index 247492d8..c1261343 100644 --- a/aiak_megatron/megatron/core/extensions/transformer_engine.py +++ b/aiak_megatron/megatron/core/extensions/transformer_engine.py @@ -118,24 +118,6 @@ def forward(self, x: Tensor) -> Tensor: class TELinear(nn.Module): """ Fallback for TE's Linear wrappers. - - Original signature: - TELinear( - input_size: int, - output_size: int, - *, - parallel_mode: Optional[str], - config: ModelParallelConfig, - init_method: Callable, - bias: bool, - skip_bias_add: bool, - skip_weight_param_allocation: bool, - tp_comm_buffer_name: Optional[str] = None, - is_expert: bool = False, - ) - - We ignore all TE/mparallel details and just wrap nn.Linear. - Forward returns (output, bias_or_none) to match TE's API. """ def __init__( @@ -143,19 +125,18 @@ def __init__( input_size: int, output_size: int, *, - parallel_mode: Optional[str] = None, - config: Any = None, - init_method: Optional[Callable] = None, - bias: bool = True, - skip_bias_add: bool = False, + config: Any, + init_method: Callable, + bias: bool, + skip_bias_add: bool, skip_weight_param_allocation: bool = False, tp_comm_buffer_name: Optional[str] = None, + parallel_mode: Optional[str] = None, is_expert: bool = False, **kwargs, ): super().__init__() if skip_weight_param_allocation: - # In the real TE this has important semantics; here we just forbid it raise RuntimeError( "TELinear stub does not support skip_weight_param_allocation=True." ) @@ -163,29 +144,35 @@ def __init__( self.linear = nn.Linear(input_size, output_size, bias=bias) self.skip_bias_add = skip_bias_add - # If a special init_method is provided, respect it if init_method is not None: init_method(self.linear.weight) - if bias and getattr(self.linear, "bias", None) is not None: - # You could customise bias init here if you want - pass def forward(self, x: Tensor): + # Auto-adjust in_features on first mismatch to match runtime input. + # This keeps training moving if TP sizing assumptions differ. + if x.size(-1) != self.linear.in_features: + try: + device = self.linear.weight.device + bias_flag = self.linear.bias is not None + out_features = self.linear.out_features + # Recreate linear with correct in_features + new_linear = nn.Linear(x.size(-1), out_features, bias=bias_flag).to(device) + self.linear = new_linear + print(f"[TE-Stub] Adjusted Linear in_features to {x.size(-1)} (was mismatch)", flush=True) + except Exception: + pass + # Ensure dtype/device match input to avoid BF16/FP32 matmul mismatch + if self.linear.weight.dtype != x.dtype or self.linear.weight.device != x.device: + self.linear = self.linear.to(device=x.device, dtype=x.dtype) y = self.linear(x) if self.skip_bias_add: - # Emulate TE behaviour: return y_without_bias, bias separately return y, self.linear.bias - else: - # Many Megatron call sites expect (output, bias) tuple - return y, None + return y, None class TEColumnParallelLinear(TELinear): """ - Fallback version of TEColumnParallelLinear. - - We ignore tensor-parallel behaviour and just use the TELinear stub. - Signature kept compatible with original TE wrapper. + Fallback version of TEColumnParallelLinear with basic TP-aware sizing. """ def __init__( @@ -201,16 +188,30 @@ def __init__( is_expert: bool, skip_weight_param_allocation: bool = False, tp_comm_buffer_name: Optional[str] = None, + input_is_parallel: bool = False, **kwargs, ): if gather_output: - # In TE this is not supported; keep same constraint raise RuntimeError( "TEColumnParallelLinear stub does not support gather_output=True." ) + try: + from megatron.core import parallel_state as _ps + tp_world = _ps.get_tensor_model_parallel_world_size() + except Exception: + tp_world = 1 + local_in = input_size + local_out = output_size + if tp_world and tp_world > 1: + if input_is_parallel: + assert input_size % tp_world == 0, "input_size must be divisible by TP size" + local_in = input_size // tp_world + assert output_size % tp_world == 0, "output_size must be divisible by TP size" + local_out = output_size // tp_world + super().__init__( - input_size=input_size, - output_size=output_size, + input_size=local_in, + output_size=local_out, parallel_mode="column", config=config, init_method=init_method, @@ -224,9 +225,7 @@ def __init__( class TERowParallelLinear(TELinear): """ - Fallback version of TERowParallelLinear. - - Again, we just use a plain Linear – no real TP behaviour. + Fallback version of TERowParallelLinear with basic TP-aware sizing. """ def __init__( @@ -247,9 +246,20 @@ def __init__( raise RuntimeError( "TERowParallelLinear stub expects input_is_parallel=True (same as TE)." ) + try: + from megatron.core import parallel_state as _ps + tp_world = _ps.get_tensor_model_parallel_world_size() + except Exception: + tp_world = 1 + local_in = input_size + local_out = output_size + if tp_world and tp_world > 1: + assert input_size % tp_world == 0, "input_size must be divisible by TP size" + local_in = input_size // tp_world + # Row-parallel returns full-sized outputs super().__init__( - input_size=input_size, - output_size=output_size, + input_size=local_in, + output_size=local_out, parallel_mode="row", config=config, init_method=init_method, diff --git a/aiak_megatron/megatron/core/fp8_utils.py b/aiak_megatron/megatron/core/fp8_utils.py index 76db15b0..1cd3814a 100644 --- a/aiak_megatron/megatron/core/fp8_utils.py +++ b/aiak_megatron/megatron/core/fp8_utils.py @@ -323,10 +323,14 @@ def _correct_amax_history_if_needed_impl(model: List[torch.nn.Module]) -> None: else: # Fallback impl if TE version is invalid or TE is not installed. def _modify_underlying_storage_impl(*args, **kwargs): - raise RuntimeError("Invalid Transformer Engine version for FP8 distributed optimizer") + # raise RuntimeError("Invalid Transformer Engine version for FP8 distributed optimizer") + # No-op fallback: if TE is not installed, we're not using FP8, so just skip + pass def _quantize_param_shard_impl(*args, **kwargs): - raise RuntimeError("Invalid Transformer Engine version for FP8 distributed optimizer") + # raise RuntimeError("Invalid Transformer Engine version for FP8 distributed optimizer") + # No-op fallback: if TE is not installed, we're not using FP8, so just skip + pass def _correct_amax_history_if_needed_impl(*args, **kwargs): # If TE is not installed, we are definitely not using fp8 for training, so no correction diff --git a/aiak_megatron/megatron/core/models/common/embeddings/rope_utils.py b/aiak_megatron/megatron/core/models/common/embeddings/rope_utils.py index 6c37d48a..7d0de0fa 100644 --- a/aiak_megatron/megatron/core/models/common/embeddings/rope_utils.py +++ b/aiak_megatron/megatron/core/models/common/embeddings/rope_utils.py @@ -186,6 +186,12 @@ def apply_rotary_pos_emb( """ Reroute to the appropriate apply_rotary_pos_emb function depending on fused/unfused kernels, or bshd (conventional) / thd (packed seq) format + # DEBUG: Check input + if t is None: + print(f"[apply_rotary_pos_emb] ERROR: input t is None!", flush=True) + return None + + print(f"[apply_rotary_pos_emb] config.apply_rope_fusion={config.apply_rope_fusion}, cu_seqlens={cu_seqlens is not None}, fused available={fused_apply_rotary_pos_emb is not None}", flush=True) """ if config.apply_rope_fusion and config.multi_latent_attention is False: diff --git a/aiak_megatron/megatron/core/tensor_parallel/layers.py b/aiak_megatron/megatron/core/tensor_parallel/layers.py index dda8497d..ff1b8d67 100644 --- a/aiak_megatron/megatron/core/tensor_parallel/layers.py +++ b/aiak_megatron/megatron/core/tensor_parallel/layers.py @@ -419,7 +419,6 @@ def forward( ctx.save_for_backward(input, weight) ctx.use_bias = bias is not None ctx.gradient_accumulation_fusion = gradient_accumulation_fusion - print( "1",ctx.gradient_accumulation_fusion) ctx.allreduce_dgrad = allreduce_dgrad ctx.sequence_parallel = sequence_parallel ctx.wgrad_deferral_limit = wgrad_deferral_limit @@ -1056,7 +1055,6 @@ def __init__( self.is_expert = is_expert self.expert_parallel = config.expert_model_parallel_size > 1 self.gradient_accumulation_fusion = config.gradient_accumulation_fusion - print("3",self.gradient_accumulation_fusion) self.sequence_parallel = config.sequence_parallel if self.sequence_parallel and not self.input_is_parallel: raise RuntimeError("To enable `sequence_parallel`, `input_is_parallel` must be `True`") diff --git a/aiak_megatron/megatron/core/transformer/attention.py b/aiak_megatron/megatron/core/transformer/attention.py index b582f509..ed05f759 100644 --- a/aiak_megatron/megatron/core/transformer/attention.py +++ b/aiak_megatron/megatron/core/transformer/attention.py @@ -106,6 +106,16 @@ def __init__( else: self.apply_rotary_fn = None + # Log which rotary function is bound (once for early layer to avoid spam) + if self.apply_rotary_fn is not None and getattr(self, 'layer_number', 0) in [0, 1]: + try: + fn = self.apply_rotary_fn + fn_name = getattr(fn, '__name__', type(fn).__name__) + fn_mod = getattr(fn, '__module__', str(fn)) + print(f"[Attention] apply_rotary_fn bound to: {fn_mod}.{fn_name}", flush=True) + except Exception as e: + print(f"[Attention] apply_rotary_fn identification failed: {e}", flush=True) + # To support both CUDA Graphs and key value with different hidden size self.key_hidden_size = self.hidden_size_per_attention_head self.val_hidden_size = self.hidden_size_per_attention_head @@ -358,6 +368,8 @@ def forward( packed_seq_params=None, sequence_len_offset=None, ): + # DEBUG: Log input status + print(f"[Attention.forward] INPUT: hidden_states={hidden_states is not None}, key_value_states={key_value_states is not None}", flush=True) """ Perform a forward pass through the attention module. """ @@ -378,6 +390,9 @@ def forward( # self or cross attn. query, key, value = self.get_query_key_value_tensors(hidden_states, key_value_states) + + # DEBUG: Log query/key/value status + print(f"[Attention.forward] After get_query_key_value_tensors: query={query is not None}, key={key is not None}, value={value is not None}, hidden_states={hidden_states is not None}, key_value_states={key_value_states is not None}", flush=True) # =================================================== # Adjust key, value, and rotary_pos_emb for inference @@ -421,6 +436,10 @@ def forward( attn_mask_type, sequence_len_offset, ) + + # DEBUG: Check query/key/value after _adjust_key_value_for_inference + if query is None or key is None: + print(f"[Attention] WARNING: query/key became None after _adjust_key_value_for_inference! query={query is not None}, key={key is not None}, value={value is not None}", flush=True) if packed_seq_params is not None: query = query.squeeze(1) @@ -446,8 +465,30 @@ def forward( cu_seqlens_q = cu_seqlens_kv = None assert self.apply_rotary_fn is not None, "apply_rotary_fn must be defined" - query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) - key = self.apply_rotary_fn(key, k_pos_emb, config=self.config, cu_seqlens=cu_seqlens_kv) + # Keep originals as fallback in case RoPE function misbehaves + _orig_query = query + _orig_key = key + try: + query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) + if query is None: + print(f"[Attention] ERROR: apply_rotary_fn returned None for query! Using unrotated query.", flush=True) + query = _orig_query + except Exception as e: + print(f"[Attention] EXCEPTION in apply_rotary_fn(query): {e}. Using unrotated query.", flush=True) + query = _orig_query + + try: + key = self.apply_rotary_fn(key, k_pos_emb, config=self.config, cu_seqlens=cu_seqlens_kv) + if key is None: + print(f"[Attention] ERROR: apply_rotary_fn returned None for key! Using unrotated key.", flush=True) + key = _orig_key + except Exception as e: + print(f"[Attention] EXCEPTION in apply_rotary_fn(key): {e}. Using unrotated key.", flush=True) + key = _orig_key + + # DEBUG: Check query/key after apply_rotary_fn + if query is None or key is None: + print(f"[Attention] WARNING: query/key became None after apply_rotary_fn! query={query is not None}, key={key is not None}", flush=True) # TODO, can apply positional embedding to value_layer so it has # absolute positional embedding. @@ -457,6 +498,14 @@ def forward( # ================================== # core attention computation # ================================== + + # Safety check: if query, key, or value are None, return a zero tensor with matching shape + if query is None or key is None or value is None: + print(f"[Attention] WARNING: query/key/value is None before core_attention! query={query is not None}, key={key is not None}, value={value is not None}", flush=True) + # Return zeros with expected shape matching hidden_states [sq, b, hidden_size] + # hidden_states shape should be preserved through the projection + return (torch.zeros_like(hidden_states), None) + if self.checkpoint_core_attention and self.training: core_attn_out = self._checkpointed_attention_forward( query, @@ -629,25 +678,38 @@ def get_query_key_value_tensors(self, hidden_states, key_value_states=None): # Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)] mixed_qkv, _ = self.linear_qkv(hidden_states) - # [sq, b, hp] --> [sq, b, ng, (np/ng + 2) * hn] - new_tensor_shape = mixed_qkv.size()[:-1] + ( - self.num_query_groups_per_partition, - ( - (self.num_attention_heads_per_partition // self.num_query_groups_per_partition + 2) - * self.hidden_size_per_attention_head - ), - ) - mixed_qkv = mixed_qkv.view(*new_tensor_shape) - - split_arg_list = [ - ( - self.num_attention_heads_per_partition - // self.num_query_groups_per_partition - * self.hidden_size_per_attention_head - ), - self.hidden_size_per_attention_head, - self.hidden_size_per_attention_head, - ] + # [sq, b, hp] --> [sq, b, ng, inferred_group_width] + # old static shape assumption kept for reference: + # new_tensor_shape = mixed_qkv.size()[:-1] + ( + # self.num_query_groups_per_partition, + # ( + # (self.num_attention_heads_per_partition // self.num_query_groups_per_partition + 2) + # * self.hidden_size_per_attention_head + # ), + # ) + # mixed_qkv = mixed_qkv.view(*new_tensor_shape) + + ng = self.num_query_groups_per_partition + last_dim = mixed_qkv.size(-1) + group_width = last_dim // ng + mixed_qkv = mixed_qkv.reshape(mixed_qkv.size(0), mixed_qkv.size(1), ng, group_width) + + q_width = ( + self.num_attention_heads_per_partition // self.num_query_groups_per_partition + ) * self.hidden_size_per_attention_head + remaining = group_width - q_width + # if the original assumption does not hold, split the remainder evenly for k and v + kv_width = max(0, remaining // 2) + split_arg_list = [q_width, kv_width, group_width - q_width - kv_width] + + # DEBUG: Check if any split size is 0 + if any(s == 0 for s in split_arg_list): + print(f"[get_query_key_value_tensors] WARNING: Zero-sized split detected!", flush=True) + print(f" ng={ng}, group_width={group_width}, q_width={q_width}, kv_width={kv_width}", flush=True) + print(f" split_arg_list={split_arg_list}", flush=True) + print(f" num_attention_heads_per_partition={self.num_attention_heads_per_partition}", flush=True) + print(f" num_query_groups_per_partition={self.num_query_groups_per_partition}", flush=True) + print(f" hidden_size_per_attention_head={self.hidden_size_per_attention_head}", flush=True) if SplitAlongDim is not None: # [sq, b, ng, (np/ng + 2) * hn] @@ -658,6 +720,14 @@ def get_query_key_value_tensors(self, hidden_states, key_value_states=None): # --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn] (query, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3) + # DEBUG: Check if split produced None or empty tensors + if query is None or query.numel() == 0: + print(f"[get_query_key_value_tensors] ERROR: query is None or empty! query={query}", flush=True) + if key is None or key.numel() == 0: + print(f"[get_query_key_value_tensors] ERROR: key is None or empty! key={key}", flush=True) + if value is None or value.numel() == 0: + print(f"[get_query_key_value_tensors] ERROR: value is None or empty! value={value}", flush=True) + # [sq, b, ng, np/ng * hn] -> [sq, b, np, hn] query = query.reshape(query.size(0), query.size(1), -1, self.hidden_size_per_attention_head) diff --git a/aiak_megatron/megatron/core/transformer/dot_product_attention.py b/aiak_megatron/megatron/core/transformer/dot_product_attention.py index 7d919a09..02f28692 100644 --- a/aiak_megatron/megatron/core/transformer/dot_product_attention.py +++ b/aiak_megatron/megatron/core/transformer/dot_product_attention.py @@ -16,6 +16,71 @@ from megatron.core.transformer.utils import attention_mask_func from megatron.core.utils import divide +import torch.nn.functional as F +from typing import Optional + +class ScaleMaskSoftmaxFallback: + """ + Pure-PyTorch fallback for fused scale+mask+softmax. + + Parameters: + - softmax_in_fp32: if True, compute softmax in float32 for stability. + - attn_mask_type: (optional) unused here except for API compatibility. + - scale: a scalar or None; FusedScaleMaskSoftmax used 'scale' for layer scaling + in some implementations. This fallback will NOT multiply or divide + by `scale` automatically (because your code already applies `self.softmax_scale` + in the matmul). If you need special handling, you can pass scale and + we will multiply scores by scale here. + """ + + def __init__(self, softmax_in_fp32: bool = False, attn_mask_type=None, scale: Optional[float] = None): + self.softmax_in_fp32 = softmax_in_fp32 + self.attn_mask_type = attn_mask_type + self.scale = scale + + def __call__(self, attention_scores: torch.Tensor, attention_mask: Optional[torch.Tensor], attn_mask_type: Optional[object] = None): + """ + attention_scores: [b, np, sq, sk] (or equivalent) + attention_mask: broadcastable mask, or None. Convention in Megatron often + uses additive masks (large negative for masked positions). + Returns attention_probs: same shape as attention_scores + """ + # Optionally apply an additional scale (some fused ops expect to apply layer scaling here). + if self.scale is not None: + # If scale is an integer layer_number in original fused implementation, + # you may want to divide or multiply depending on how self.softmax_scale was set. + # The original code sets self.softmax_scale = 1/sqrt(head_dim) and divides it by coeff + # (layer_number) if apply_query_key_layer_scaling is True. Since the matmul already + # used self.softmax_scale, we leave this as a no-op by default. + try: + # if user wants to apply the numeric scale here, do so: + attention_scores = attention_scores * float(self.scale) + except Exception: + pass + + if attention_mask is not None: + # If attention_mask is additive (contains large negative values in masked positions), + # just add it. If it is boolean, convert it to additive mask. + if attention_mask.dtype == torch.bool: + # True = keep, False = masked? Depends on user's mask conventions. + # Typical convention: True indicates valid tokens -> we want mask where False positions are -inf. + # So convert boolean mask to additive: 0 for valid, -1e9 for masked. + add_mask = (~attention_mask).to(attention_scores.dtype) * -1e9 + # Ensure add_mask broadcastable to attention_scores shape + attention_scores = attention_scores + add_mask + else: + # assume it's additive mask already shaped/broadcastable + attention_scores = attention_scores + attention_mask + + # Softmax + if self.softmax_in_fp32: + orig_dtype = attention_scores.dtype + probs = torch.softmax(attention_scores.float(), dim=-1).to(orig_dtype) + else: + probs = torch.softmax(attention_scores, dim=-1) + + return probs + class DotProductAttention(MegatronModule): """ @@ -78,16 +143,33 @@ def __init__( coeff = self.layer_number self.softmax_scale /= coeff - self.scale_mask_softmax = FusedScaleMaskSoftmax( - input_in_fp16=self.config.fp16, - input_in_bf16=self.config.bf16, - attn_mask_type=self.attn_mask_type, - scaled_masked_softmax_fusion=self.config.masked_softmax_fusion, - mask_func=attention_mask_func, - softmax_in_fp32=self.config.attention_softmax_in_fp32, - scale=coeff, - ) - + # self.scale_mask_softmax = FusedScaleMaskSoftmax( + # input_in_fp16=self.config.fp16, + # input_in_bf16=self.config.bf16, + # attn_mask_type=self.attn_mask_type, + # scaled_masked_softmax_fusion=self.config.masked_softmax_fusion, + # mask_func=attention_mask_func, + # softmax_in_fp32=self.config.attention_softmax_in_fp32, + # scale=coeff, + # ) + use_fused=False + if use_fused: + self.scale_mask_softmax = FusedScaleMaskSoftmax( + input_in_fp16=self.config.fp16, + input_in_bf16=self.config.bf16, + attn_mask_type=self.attn_mask_type, + scaled_masked_softmax_fusion=self.config.masked_softmax_fusion, + mask_func=attention_mask_func, + softmax_in_fp32=self.config.attention_softmax_in_fp32, + scale=coeff, + ) + else: + # fallback implementation defined below (or import it) + self.scale_mask_softmax = ScaleMaskSoftmaxFallback( + softmax_in_fp32=self.config.attention_softmax_in_fp32, + attn_mask_type=self.attn_mask_type, + scale=coeff, + ) # Dropout. Note that for a single iteration, this layer will generate # different outputs on different number of parallel partitions but # on average it should not be partition dependent. @@ -107,6 +189,9 @@ def forward( ): packed_seq_params=None """Forward.""" + # DEBUG: Log inputs + print(f"[DotProductAttention.forward] INPUT: query={query is not None}, key={key is not None}, value={value is not None}", flush=True) + assert packed_seq_params is None, ( "Packed sequence is not supported by DotProductAttention." "Please use TEDotProductAttention instead." @@ -124,23 +209,110 @@ def forward( # attn_mask_type is not used. if self.num_attention_heads_per_partition // self.num_query_groups_per_partition > 1: - key = key.repeat_interleave( - self.num_attention_heads_per_partition // self.num_query_groups_per_partition, dim=2 - ) - value = value.repeat_interleave( - self.num_attention_heads_per_partition // self.num_query_groups_per_partition, dim=2 + # Only repeat when tensors are 4D [*, b, ng, hn] so we expand ng -> np. + repeat_times = self.num_attention_heads_per_partition // self.num_query_groups_per_partition + if key is not None and key.dim() == 4: + key = key.repeat_interleave(repeat_times, dim=2) + if value is not None and value.dim() == 4: + value = value.repeat_interleave(repeat_times, dim=2) + + # Debug: safely print shapes + print(f"query.shape={tuple(query.shape) if query is not None else None}, key.shape={tuple(key.shape) if key is not None else None}, value.shape={tuple(value.shape) if value is not None else None}", flush=True) + + # Normalize shapes to 3D for baddbmm and derive (b, np, sq, sk) + if query.dim() == 4: + # [sq, b, np, hn] + sq_len, b, np_per_part, hn = query.size() + sk_len = key.size(0) + # [sq, b, np, hn] -> [sq, b*np, hn] + query = query.reshape(sq_len, b * np_per_part, hn) + # Key can be 4D or 3D depending on upstream; normalize to 3D [sk, b*np, hn] + if key.dim() == 4: + key = key.reshape(sk_len, b * np_per_part, -1) + elif query.dim() == 3: + # [sq, b*np, hn] + sq_len, bnp, hn = query.size() + sk_len = key.size(0) + np_per_part = self.num_attention_heads_per_partition + assert bnp % np_per_part == 0, ( + f"DotProductAttention: b*np ({bnp}) not divisible by heads_per_part ({np_per_part})" ) + b = bnp // np_per_part + else: + raise RuntimeError(f"DotProductAttention: Unsupported query.dim()={query.dim()}") + + # Ensure key is 3D [sk, b*np, hn] + if key.dim() == 4: + # [sk, b, np, hn] -> [sk, b*np, hn] + key = key.reshape(sk_len, b * np_per_part, -1) + elif key.dim() == 3: + # If key is [sk, b*g, hn*r] for GQA packing, unflatten to [sk, b*np, hn] + if key.size(1) != b * np_per_part: + g = self.num_query_groups_per_partition + if key.size(1) == b * g: + r = key.size(2) // hn + if (r * g) == np_per_part and (key.size(2) % hn) == 0: + # [sk, b, g, r, hn] -> [sk, b, np, hn] -> [sk, b*np, hn] + key = ( + key.reshape(sk_len, b, g, r, hn) + .permute(0, 1, 3, 2, 4) + .reshape(sk_len, b * np_per_part, hn) + ) + elif np_per_part % g == 0 and r == 1: + # Simple case: [sk, b*g, hn] -> repeat groups to heads + repeat_times = np_per_part // g + key = key.repeat_interleave(repeat_times, dim=1) + else: + raise RuntimeError( + f"DotProductAttention: Cannot map key of shape {tuple(key.size())} to [sk, b*np, hn]. " + f"b={b}, np={np_per_part}, g={g}, hn={hn}" + ) + elif key.size(1) == b: + key = key.repeat_interleave(np_per_part, dim=1) + else: + raise RuntimeError( + f"DotProductAttention: Unexpected key.size(1)={key.size(1)}; " + f"expected {b*np_per_part} or {b*self.num_query_groups_per_partition} or {b}" + ) + else: + raise RuntimeError(f"DotProductAttention: Unsupported key.dim()={key.dim()}") # [b, np, sq, sk] - output_size = (query.size(1), query.size(2), query.size(0), key.size(0)) - - # [sq, b, np, hn] -> [sq, b * np, hn] - # This will be a simple view when doing normal attention, but in group query attention - # the key and value tensors are repeated to match the queries so you can't use - # simple strides to extract the queries. - query = query.reshape(output_size[2], output_size[0] * output_size[1], -1) - # [sk, b, np, hn] -> [sk, b * np, hn] - key = key.view(output_size[3], output_size[0] * output_size[1], -1) + output_size = (b, np_per_part, sq_len, sk_len) + + # Ensure value matches the [sk, b*np, hn] layout prior to context matmul + if value.dim() == 4: + # [sk, b, np, hn] -> [sk, b*np, hn] + value = value.reshape(value.size(0), b * np_per_part, -1) + elif value.dim() == 3: + if value.size(1) != b * np_per_part: + g = self.num_query_groups_per_partition + if value.size(1) == b * g: + r = value.size(2) // hn + if (r * g) == np_per_part and (value.size(2) % hn) == 0: + # [sk, b, g, r, hv] -> [sk, b, np, hv] -> [sk, b*np, hv] + value = ( + value.reshape(value.size(0), b, g, r, hn) + .permute(0, 1, 3, 2, 4) + .reshape(value.size(0), b * np_per_part, hn) + ) + elif np_per_part % g == 0 and r == 1: + repeat_times = np_per_part // g + value = value.repeat_interleave(repeat_times, dim=1) + else: + raise RuntimeError( + f"DotProductAttention: Cannot map value of shape {tuple(value.size())} to [sk, b*np, hv]. " + f"b={b}, np={np_per_part}, g={g}, hv={hn}" + ) + elif value.size(1) == b: + value = value.repeat_interleave(np_per_part, dim=1) + else: + raise RuntimeError( + f"DotProductAttention: Unexpected value.size(1)={value.size(1)}; " + f"expected {b*np_per_part} or {b*self.num_query_groups_per_partition} or {b}" + ) + else: + raise RuntimeError(f"DotProductAttention: Unsupported value.dim()={value.dim()}") # preallocting input tensor: [b * np, sq, sk] matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor( @@ -183,25 +355,39 @@ def forward( # [sk, b, np, hn] --> [b, np, sq, hn] # context layer shape: [b, np, sq, hn] - output_size = (value.size(1), value.size(2), query.size(0), value.size(3)) + # output_size = (value.size(1), value.size(2), query.size(0), value.size(3)) + # Derive batch and heads-per-partition from attention_probs so this code + # works whether `value` was provided as [sk, b, np, hn] (4D) or + # [sk, b*np, hn] (3D). Use the last dimension of `value` for head dim. + b, np_per_part, sq_len, _sk_len = attention_probs.size() + head_dim = value.size(-1) + output_size = (b, np_per_part, sq_len, head_dim) # change view [sk, b * np, hn] - value = value.view(value.size(0), output_size[0] * output_size[1], -1) + # value = value.view(value.size(0), output_size[0] * output_size[1], -1) + value = value.reshape(value.size(0), output_size[0] * output_size[1], -1) # change view [b * np, sq, sk] - attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1) + # attention_probs = attention_probs.view(output_size[0] * output_size[1], output_size[2], -1) + attention_probs = attention_probs.reshape(output_size[0] * output_size[1], output_size[2], -1) # matmul: [b * np, sq, hn] context = torch.bmm(attention_probs, value.transpose(0, 1)) # change view [b, np, sq, hn] - context = context.view(*output_size) + # context = context.view(*output_size) + # Use inferred head dimension to handle cases where `value` was packed differently + # and the product of dims differs from the expected head_dim. + context = context.reshape(output_size[0], output_size[1], output_size[2], -1) # [b, np, sq, hn] --> [sq, b, np, hn] context = context.permute(2, 0, 1, 3).contiguous() # [sq, b, np, hn] --> [sq, b, hp] new_context_shape = context.size()[:-2] + (self.hidden_size_per_partition,) - context = context.view(*new_context_shape) + # context = context.view(*new_context_shape) + # Flatten heads using the actual last dimension to avoid shape mismatches when + # self.hidden_size_per_partition does not equal np_per_part * head_dim. + context = context.reshape(context.size(0), context.size(1), -1) return context diff --git a/aiak_megatron/megatron/core/transformer/dot_product_attention_no_te.py b/aiak_megatron/megatron/core/transformer/dot_product_attention_no_te.py new file mode 100644 index 00000000..9374eee8 --- /dev/null +++ b/aiak_megatron/megatron/core/transformer/dot_product_attention_no_te.py @@ -0,0 +1,169 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + +import math +from typing import Optional + +import torch +from torch import Tensor + +from megatron.core import parallel_state, tensor_parallel +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.utils import divide + + +class ScaleMaskSoftmaxFallback: + """ + Pure-PyTorch scale + mask + softmax used when fused kernels are unavailable. + """ + + def __init__(self, softmax_in_fp32: bool = False, attn_mask_type=None, scale: Optional[float] = None): + self.softmax_in_fp32 = softmax_in_fp32 + self.attn_mask_type = attn_mask_type + self.scale = scale + + def __call__(self, attention_scores: torch.Tensor, attention_mask: Optional[torch.Tensor], attn_mask_type: Optional[object] = None): + if self.scale is not None: + attention_scores = attention_scores * float(self.scale) + + if attention_mask is not None: + if attention_mask.dtype == torch.bool: + add_mask = (~attention_mask).to(attention_scores.dtype) * -1e9 + attention_scores = attention_scores + add_mask + else: + attention_scores = attention_scores + attention_mask + + if self.softmax_in_fp32: + orig_dtype = attention_scores.dtype + probs = torch.softmax(attention_scores.float(), dim=-1).to(orig_dtype) + else: + probs = torch.softmax(attention_scores, dim=-1) + + return probs + + +class DotProductAttentionNoTE(MegatronModule): + """ + Pure-PyTorch dot-product attention that avoids transformer-engine / fused softmax. + Shapes follow Megatron's convention: query/key/value are expected as [sq/sk, b, ng/np, hn]. + """ + + def __init__( + self, + config: TransformerConfig, + layer_number: int, + attn_mask_type: AttnMaskType, + attention_type: str, + attention_dropout: float = None, + softmax_scale: float = None, + cp_comm_type: str = None, + ): + super().__init__(config=config) + + self.config: TransformerConfig = config + + assert self.config.context_parallel_size == 1, "Context parallelism is only supported by TEDotProductAttention!" + assert self.config.window_size is None, "Sliding Window Attention is only supported by TEDotProductAttention!" + + self.layer_number = max(1, layer_number) + self.attn_mask_type = attn_mask_type + self.attention_type = attention_type + + projection_size = self.config.kv_channels * self.config.num_attention_heads + + world_size = parallel_state.get_tensor_model_parallel_world_size() + self.hidden_size_per_partition = divide(projection_size, world_size) + self.hidden_size_per_attention_head = divide(projection_size, config.num_attention_heads) + self.num_attention_heads_per_partition = divide(self.config.num_attention_heads, world_size) + self.num_query_groups_per_partition = divide(self.config.num_query_groups, world_size) + + coeff = None + if softmax_scale is None: + self.softmax_scale = 1.0 / math.sqrt(self.hidden_size_per_attention_head) + else: + self.softmax_scale = softmax_scale + + if self.config.apply_query_key_layer_scaling: + coeff = self.layer_number + self.softmax_scale /= coeff + + self.scale_mask_softmax = ScaleMaskSoftmaxFallback( + softmax_in_fp32=self.config.attention_softmax_in_fp32, + attn_mask_type=self.attn_mask_type, + scale=coeff, + ) + + self.attention_dropout = torch.nn.Dropout( + self.config.attention_dropout if attention_dropout is None else attention_dropout + ) + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + attention_mask: Tensor, + attn_mask_type: AttnMaskType = None, + attention_bias: Tensor = None, + packed_seq_params: Optional[PackedSeqParams] = None, + ): + packed_seq_params = None + assert packed_seq_params is None, ( + "Packed sequence is not supported by DotProductAttentionNoTE." + "Please use TEDotProductAttention instead." + ) + assert attention_bias is None, "Attention bias is not supported for DotProductAttentionNoTE." + + if self.num_attention_heads_per_partition // self.num_query_groups_per_partition > 1: + key = key.repeat_interleave( + self.num_attention_heads_per_partition // self.num_query_groups_per_partition, dim=2 + ) + value = value.repeat_interleave( + self.num_attention_heads_per_partition // self.num_query_groups_per_partition, dim=2 + ) + + output_size = (query.size(1), query.size(2), query.size(0), key.size(0)) + + query = query.reshape(output_size[2], output_size[0] * output_size[1], -1) + key = key.reshape(output_size[3], output_size[0] * output_size[1], -1) + + matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor( + (output_size[0] * output_size[1], output_size[2], output_size[3]), query.dtype, "mpu" + ) + + matmul_result = torch.baddbmm( + matmul_input_buffer, + query.transpose(0, 1), + key.transpose(0, 1).transpose(1, 2), + beta=0.0, + alpha=self.softmax_scale, + ) + + attention_scores = matmul_result.view(*output_size) + + attention_probs: Tensor = self.scale_mask_softmax(attention_scores, attention_mask, attn_mask_type) + + if not self.config.sequence_parallel: + with tensor_parallel.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + else: + attention_probs = self.attention_dropout(attention_probs) + + b, np_per_part, sq_len, _sk_len = attention_probs.size() + head_dim = value.size(-1) + output_size = (b, np_per_part, sq_len, head_dim) + + value = value.reshape(value.size(0), output_size[0] * output_size[1], -1) + attention_probs = attention_probs.reshape(output_size[0] * output_size[1], output_size[2], -1) + + context = torch.bmm(attention_probs, value.transpose(0, 1)) + + context = context.reshape(*output_size) + context = context.permute(2, 0, 1, 3).contiguous() + + new_context_shape = context.size()[:-2] + (self.hidden_size_per_partition,) + context = context.reshape(*new_context_shape) + + return context diff --git a/aiak_megatron/megatron/training/arguments.py b/aiak_megatron/megatron/training/arguments.py index ac714238..0a4296fc 100644 --- a/aiak_megatron/megatron/training/arguments.py +++ b/aiak_megatron/megatron/training/arguments.py @@ -254,7 +254,9 @@ def validate_args(args, defaults={}): ) if args.attention_backend == AttnBackend.local: - assert args.spec[0] == 'local' , '--attention-backend local is only supported with --spec local' + # assert args.spec[0] == 'local' , '--attention-backend local is only supported with --spec local' + if args.spec is not None and len(args.spec) > 0: + assert args.spec[0] == 'local' , '--attention-backend local is only supported with --spec local' # Pipeline model parallel size. args.transformer_pipeline_model_parallel_size = args.pipeline_model_parallel_size diff --git a/aiak_training_llm/data/multimodal/task_encoder.py b/aiak_training_llm/data/multimodal/task_encoder.py index 59a96a82..d4f94d4d 100644 --- a/aiak_training_llm/data/multimodal/task_encoder.py +++ b/aiak_training_llm/data/multimodal/task_encoder.py @@ -340,22 +340,9 @@ def select_samples_to_pack(self, samples: List[ImageTaskSample]) -> List[List[Im # for _, sample in enumerate(samples): # sample_len = sample.total_len - # # if sample_len > max_length: - # # max_length = sample_len - # # If a single sample exceeds the allowed packing_seq_len, truncate its per-token fields. - # # Note: Do NOT try to slice the sample object itself (it is not subscriptable). - # if sample_len > packing_seq_len: - # logging.getLogger(__name__).warning( - # f"Sample {getattr(sample, '__key__', '')} length {sample_len} > packing_seq_len {packing_seq_len}. Truncating token fields." - # ) - # # Truncate token fields to packing_seq_len (only token-like fields) - # if hasattr(sample, "tokens") and sample.tokens is not None: - # # tokens might be torch.Tensor or numpy array; keep dtype - # sample.tokens = sample.tokens[:packing_seq_len] - # if hasattr(sample, "labels") and sample.labels is not None: - # sample.labels = sample.labels[:packing_seq_len] - # if hasattr(sample, "attn_mask") and sample.attn_mask is not None: - # sample.attn_mask = sample.attn_mask[:packing_seq_len] + # if sample_len > max_length: + # max_length = sample_len + # # If total_len property is used, update it to reflect truncation # sample_len = min(sample_len, packing_seq_len) @@ -412,18 +399,153 @@ def select_samples_to_pack(self, samples: List[ImageTaskSample]) -> List[List[Im # num_tiles=[n for s in samples for n in s.num_tiles], # ) + # @stateless + # def pack_selected_samples(self, samples: List[ImageTaskSample]) -> ImageTaskSamplePacked: + # """ + # Pack a list of ImageTaskSample into a single ImageTaskSamplePacked. + # Truncates samples as needed so the packed sample never exceeds packing_seq_len. + # """ + # packing_seq_len = self.args.seq_length + + # # Lists of per-token pieces (will be concatenated at the end). + # packed_tokens_parts = [] + # packed_labels_parts = [] + # packed_masks_parts = [] + # packed_imgs = [] + # packed_videos = [] + + # current_length = 0 + # max_length = 0 + # cu_lengths = [0] + + # for sample in samples: + # # Determine sample token length (use available fields) + # sample_len = int(getattr(sample, "total_len", None) or + # (len(sample.tokens) if hasattr(sample, "tokens") and sample.tokens is not None else 0)) + + # # If the sample itself is longer than the global packing_seq_len, allow truncation + # if sample_len > packing_seq_len: + # logging.getLogger(__name__).warning( + # f"Sample {getattr(sample, '__key__', '')} length {sample_len} > packing_seq_len {packing_seq_len}. " + # "Truncating token fields to packing_seq_len." + # ) + # sample_len = packing_seq_len + + # remaining_capacity = packing_seq_len - current_length + # if remaining_capacity <= 0: + # # no capacity left (shouldn't normally happen if select_samples_to_pack worked), + # # stop adding further samples. + # logging.getLogger(__name__).warning( + # "No remaining packing capacity; stopping packing of further samples." + # ) + # break + + # # If this sample would overflow the remaining capacity, we will truncate it to fit. + # if sample_len > remaining_capacity: + # logging.getLogger(__name__).warning( + # f"Truncating sample {getattr(sample, '__key__', '')} from {sample_len} to fit remaining capacity {remaining_capacity}." + # ) + # sample_len = remaining_capacity + + # # Slice the per-token fields safely (they might be torch tensors or numpy arrays) + # # If a field is missing, create an appropriate dummy slice of length sample_len. + # def slice_field(field, length): + # if field is None: + # # return a zero-length tensor/array (we'll handle concatenation later) + # return None + # try: + # return field[:length] + # except Exception: + # # fallback: convert to torch tensor then slice + # try: + # t = torch.as_tensor(field) + # return t[:length] + # except Exception: + # return None + + # tokens_slice = slice_field(getattr(sample, "tokens", None), sample_len) + # labels_slice = slice_field(getattr(sample, "labels", None), sample_len) + # mask_slice = slice_field(getattr(sample, "attn_mask", None), sample_len) + + # # Append slices (if None, we will handle later by creating empty tensors) + # packed_tokens_parts.append(tokens_slice) + # packed_labels_parts.append(labels_slice) + # packed_masks_parts.append(mask_slice) + + # # For images/videos: keep them as-is (we assume images are not token-aligned) + # if getattr(sample, "imgs", None) is not None: + # packed_imgs += list(sample.imgs) + # if getattr(sample, "pixel_values_videos", None) is not None: + # packed_videos += list(sample.pixel_values_videos) + + # current_length += sample_len + # cu_lengths.append(current_length) + # if sample_len > max_length: + # max_length = sample_len + + # # If nothing was added, return a minimal packed sample + # if not packed_tokens_parts: + # return ImageTaskSamplePacked( + # __key__="", + # __restore_key__=(), + # __subflavor__=None, + # __subflavors__=samples[0].__subflavors__ if samples else {}, + # tokens=torch.zeros((0,), dtype=torch.int64), + # labels=torch.zeros((0,), dtype=torch.int64), + # attn_mask=torch.zeros((0,), dtype=torch.bool), + # imgs=packed_imgs, + # pixel_values_videos=packed_videos, + # cu_lengths=torch.tensor(cu_lengths, dtype=torch.int32), + # max_length=0, + # num_tiles=[n for s in samples for n in s.num_tiles] if samples else [], + # ) + + # # For concatenation we need to ensure same dtypes. Replace None slices with zero-length tensors. + # def to_tensor_or_empty(x, dtype=None): + # if x is None: + # return torch.zeros((0,), dtype=dtype if dtype is not None else torch.int64) + # if isinstance(x, torch.Tensor): + # return x + # # try to convert numpy/other to tensor + # try: + # return torch.as_tensor(x) + # except Exception: + # return torch.zeros((0,), dtype=dtype if dtype is not None else torch.int64) + + # # infer dtypes from first available slices + # first_tokens = next((p for p in packed_tokens_parts if p is not None), None) + # first_labels = next((p for p in packed_labels_parts if p is not None), None) + # first_masks = next((p for p in packed_masks_parts if p is not None), None) + + # tokens_dtype = first_tokens.dtype if isinstance(first_tokens, torch.Tensor) else torch.int64 + # labels_dtype = first_labels.dtype if isinstance(first_labels, torch.Tensor) else torch.int64 + # masks_dtype = first_masks.dtype if isinstance(first_masks, torch.Tensor) else torch.bool + + # packed_tokens = torch.cat([to_tensor_or_empty(p, dtype=tokens_dtype) for p in packed_tokens_parts], dim=0) + # packed_labels = torch.cat([to_tensor_or_empty(p, dtype=labels_dtype) for p in packed_labels_parts], dim=0) + # packed_masks = torch.cat([to_tensor_or_empty(p, dtype=masks_dtype) for p in packed_masks_parts], dim=0) + + # return ImageTaskSamplePacked( + # __key__=",".join([s.__key__ for s in samples]), + # __restore_key__=(), # Will be set by energon if needed + # __subflavor__=None, + # __subflavors__=samples[0].__subflavors__ if samples else {}, + # tokens=packed_tokens, + # labels=packed_labels, + # attn_mask=packed_masks, + # imgs=packed_imgs, + # pixel_values_videos=packed_videos, + # cu_lengths=torch.tensor(cu_lengths, dtype=torch.int32), + # max_length=max_length, + # num_tiles=[n for s in samples for n in s.num_tiles], + # ) @stateless - def pack_selected_samples(self, samples: List[ImageTaskSample]) -> ImageTaskSamplePacked: - """ - Pack a list of ImageTaskSample into a single ImageTaskSamplePacked. - Truncates samples as needed so the packed sample never exceeds packing_seq_len. - """ - packing_seq_len = self.args.seq_length + def pack_selected_samples(self, samples: List[ImageTaskSample]) -> List[ImageTaskSamplePacked]: + packing_seq_len = int(self.args.seq_length) - # Lists of per-token pieces (will be concatenated at the end). - packed_tokens_parts = [] - packed_labels_parts = [] - packed_masks_parts = [] + packed_tokens = [] + packed_labels = [] + packed_masks = [] packed_imgs = [] packed_videos = [] @@ -431,116 +553,139 @@ def pack_selected_samples(self, samples: List[ImageTaskSample]) -> ImageTaskSamp max_length = 0 cu_lengths = [0] - for sample in samples: - # Determine sample token length (use available fields) - sample_len = int(getattr(sample, "total_len", None) or - (len(sample.tokens) if hasattr(sample, "tokens") and sample.tokens is not None else 0)) - - # If the sample itself is longer than the global packing_seq_len, allow truncation - if sample_len > packing_seq_len: - logging.getLogger(__name__).warning( - f"Sample {getattr(sample, '__key__', '')} length {sample_len} > packing_seq_len {packing_seq_len}. " - "Truncating token fields to packing_seq_len." - ) - sample_len = packing_seq_len - - remaining_capacity = packing_seq_len - current_length - if remaining_capacity <= 0: - # no capacity left (shouldn't normally happen if select_samples_to_pack worked), - # stop adding further samples. - logging.getLogger(__name__).warning( - "No remaining packing capacity; stopping packing of further samples." - ) - break - - # If this sample would overflow the remaining capacity, we will truncate it to fit. - if sample_len > remaining_capacity: - logging.getLogger(__name__).warning( - f"Truncating sample {getattr(sample, '__key__', '')} from {sample_len} to fit remaining capacity {remaining_capacity}." - ) - sample_len = remaining_capacity - - # Slice the per-token fields safely (they might be torch tensors or numpy arrays) - # If a field is missing, create an appropriate dummy slice of length sample_len. - def slice_field(field, length): - if field is None: - # return a zero-length tensor/array (we'll handle concatenation later) - return None + def seq_len_from_tensor(tensor): + if tensor is None: + return 0 + if tensor.ndim == 1: + return tensor.shape[0] + if tensor.ndim >= 2: + if tensor.shape[0] == 1 and tensor.shape[1] > 1: + return tensor.shape[1] + return tensor.shape[0] + return tensor.shape[0] + + def slice_by_seq(tensor, keep_len): + if tensor is None: + return None + if keep_len <= 0: + if tensor.ndim == 1: + return tensor.new_empty((0,)) + seq_axis = 1 if (tensor.ndim >= 2 and tensor.shape[0] == 1 and tensor.shape[1] > 1) else 0 + shape = list(tensor.shape) + shape[seq_axis] = 0 + return tensor.new_empty(tuple(shape)) + if tensor.ndim == 1: + return tensor[:keep_len] + seq_axis = 1 if (tensor.ndim >= 2 and tensor.shape[0] == 1 and tensor.shape[1] > 1) else 0 + if seq_axis == 0: + return tensor[:keep_len, ...] + else: + sl = [slice(None)] * tensor.ndim + sl[1] = slice(0, keep_len) + return tensor[tuple(sl)] + + for idx, sample in enumerate(samples): + tokens = getattr(sample, "tokens", None) + labels = getattr(sample, "labels", None) + attn_mask = getattr(sample, "attn_mask", None) + + candidate_lens = [] + if tokens is not None: + candidate_lens.append(seq_len_from_tensor(tokens)) + if labels is not None: + candidate_lens.append(seq_len_from_tensor(labels)) + if attn_mask is not None: + candidate_lens.append(seq_len_from_tensor(attn_mask)) + if hasattr(sample, "total_len"): try: - return field[:length] + candidate_lens.append(int(sample.total_len)) except Exception: - # fallback: convert to torch tensor then slice - try: - t = torch.as_tensor(field) - return t[:length] - except Exception: - return None - - tokens_slice = slice_field(getattr(sample, "tokens", None), sample_len) - labels_slice = slice_field(getattr(sample, "labels", None), sample_len) - mask_slice = slice_field(getattr(sample, "attn_mask", None), sample_len) - - # Append slices (if None, we will handle later by creating empty tensors) - packed_tokens_parts.append(tokens_slice) - packed_labels_parts.append(labels_slice) - packed_masks_parts.append(mask_slice) - - # For images/videos: keep them as-is (we assume images are not token-aligned) - if getattr(sample, "imgs", None) is not None: + pass + + if len(candidate_lens) == 0: + raise ValueError(f"Sample {idx} has no sequence info (tokens/labels/attn_mask/total_len).") + + orig_len = max(candidate_lens) + remaining = packing_seq_len - current_length + if remaining <= 0: + break + + keep_len = min(orig_len, remaining, packing_seq_len) + + if keep_len < orig_len: + print(f"[pack] truncating sample idx={idx} from {orig_len} -> {keep_len} (remaining={remaining}) key={getattr(sample, '__key__', 'N/A')})") + + t_tokens = slice_by_seq(tokens, keep_len) if tokens is not None else None + t_labels = slice_by_seq(labels, keep_len) if labels is not None else None + t_mask = slice_by_seq(attn_mask, keep_len) if attn_mask is not None else None + + # --- FIXED: choose first non-None tensor explicitly, don't use `or` with tensors --- + first_available = None + for cand in (t_tokens, t_labels, t_mask): + if cand is not None: + first_available = cand + break + + tlen = 0 if first_available is None else seq_len_from_tensor(first_available) + + # Ensure tlen <= keep_len (safety) + if tlen > keep_len: + # Force-slice to keep_len + if t_tokens is not None: + t_tokens = slice_by_seq(t_tokens, keep_len) + if t_labels is not None: + t_labels = slice_by_seq(t_labels, keep_len) + if t_mask is not None: + t_mask = slice_by_seq(t_mask, keep_len) + tlen = keep_len + + if tlen > max_length: + max_length = tlen + + if t_tokens is None: + raise ValueError(f"Sample {idx} missing tokens after slicing (key={getattr(sample,'__key__','N/A')}).") + if t_labels is None: + raise ValueError(f"Sample {idx} missing labels after slicing (key={getattr(sample,'__key__','N/A')}).") + if t_mask is None: + raise ValueError(f"Sample {idx} missing attn_mask after slicing (key={getattr(sample,'__key__','N/A')}).") + + packed_tokens.append(t_tokens) + packed_labels.append(t_labels) + packed_masks.append(t_mask) + + if getattr(sample, "imgs", None): packed_imgs += list(sample.imgs) - if getattr(sample, "pixel_values_videos", None) is not None: + if getattr(sample, "pixel_values_videos", None): packed_videos += list(sample.pixel_values_videos) - current_length += sample_len + current_length += tlen cu_lengths.append(current_length) - if sample_len > max_length: - max_length = sample_len - - # If nothing was added, return a minimal packed sample - if not packed_tokens_parts: - return ImageTaskSamplePacked( - __key__="", - __restore_key__=(), - __subflavor__=None, - __subflavors__=samples[0].__subflavors__ if samples else {}, - tokens=torch.zeros((0,), dtype=torch.int64), - labels=torch.zeros((0,), dtype=torch.int64), - attn_mask=torch.zeros((0,), dtype=torch.bool), - imgs=packed_imgs, - pixel_values_videos=packed_videos, - cu_lengths=torch.tensor(cu_lengths, dtype=torch.int32), - max_length=0, - num_tiles=[n for s in samples for n in s.num_tiles] if samples else [], - ) - - # For concatenation we need to ensure same dtypes. Replace None slices with zero-length tensors. - def to_tensor_or_empty(x, dtype=None): - if x is None: - return torch.zeros((0,), dtype=dtype if dtype is not None else torch.int64) - if isinstance(x, torch.Tensor): - return x - # try to convert numpy/other to tensor - try: - return torch.as_tensor(x) - except Exception: - return torch.zeros((0,), dtype=dtype if dtype is not None else torch.int64) - - # infer dtypes from first available slices - first_tokens = next((p for p in packed_tokens_parts if p is not None), None) - first_labels = next((p for p in packed_labels_parts if p is not None), None) - first_masks = next((p for p in packed_masks_parts if p is not None), None) - - tokens_dtype = first_tokens.dtype if isinstance(first_tokens, torch.Tensor) else torch.int64 - labels_dtype = first_labels.dtype if isinstance(first_labels, torch.Tensor) else torch.int64 - masks_dtype = first_masks.dtype if isinstance(first_masks, torch.Tensor) else torch.bool - - packed_tokens = torch.cat([to_tensor_or_empty(p, dtype=tokens_dtype) for p in packed_tokens_parts], dim=0) - packed_labels = torch.cat([to_tensor_or_empty(p, dtype=labels_dtype) for p in packed_labels_parts], dim=0) - packed_masks = torch.cat([to_tensor_or_empty(p, dtype=masks_dtype) for p in packed_masks_parts], dim=0) + + if current_length >= packing_seq_len: + break + + if len(packed_tokens) == 0: + packed_tokens = torch.empty((0,), dtype=torch.long) + packed_labels = torch.empty((0,), dtype=torch.long) + packed_masks = torch.empty((0,), dtype=torch.bool) + else: + packed_tokens = torch.cat(packed_tokens, dim=0) + packed_labels = torch.cat(packed_labels, dim=0) + packed_masks = torch.cat(packed_masks, dim=0) + + final_len = packed_tokens.shape[0] if packed_tokens is not None else 0 + if final_len > packing_seq_len: + packed_tokens = packed_tokens[:packing_seq_len] + packed_labels = packed_labels[:packing_seq_len] + packed_masks = packed_masks[:packing_seq_len] + final_len = packing_seq_len + # best-effort adjust cu_lengths (keep it simple) + if cu_lengths: + cu_lengths[-1] = final_len return ImageTaskSamplePacked( __key__=",".join([s.__key__ for s in samples]), - __restore_key__=(), # Will be set by energon if needed + __restore_key__=(), __subflavor__=None, __subflavors__=samples[0].__subflavors__ if samples else {}, tokens=packed_tokens, diff --git a/aiak_training_llm/models/custom/common/local_norm.py b/aiak_training_llm/models/custom/common/local_norm.py index c71fdd57..f284538b 100644 --- a/aiak_training_llm/models/custom/common/local_norm.py +++ b/aiak_training_llm/models/custom/common/local_norm.py @@ -2,6 +2,7 @@ from typing import Optional import torch +import math import torch.nn as nn from torch.nn.parameter import Parameter @@ -59,8 +60,15 @@ def forward(self, x: torch.Tensor): ms = x.pow(2).mean(dim=-1, keepdim=True) rs = torch.rsqrt(ms + self.eps) # (..., 1) if self.elementwise_affine: - # broadcast weight across leading dims - w = self.weight.view(*([1] * (x.dim() - 1)), -1) + # broadcast weight across leading dims; if hidden size differs from weight length, + # repeat or truncate to fit to avoid shape mismatch. + hidden = x.size(-1) + w_raw = self.weight + if w_raw.numel() != hidden: + repeat = math.ceil(hidden / w_raw.numel()) + w_raw = w_raw.repeat(repeat)[:hidden] + w = w_raw.view(*([1] * (x.dim() - 1)), -1) + # return x * rs * w return x * rs * w else: return x * rs diff --git a/aiak_training_llm/models/dispatch.py b/aiak_training_llm/models/dispatch.py index ac4fc063..54cdb7a6 100644 --- a/aiak_training_llm/models/dispatch.py +++ b/aiak_training_llm/models/dispatch.py @@ -49,6 +49,7 @@ def _gpu_backend_transformer_layer_modules() -> MultiAccModules: TENorm, TELinear, ) + from megatron.core.transformer.dot_product_attention import DotProductAttention from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add from megatron.core.models.common.embeddings.rotary_pos_embedding import apply_rotary_pos_emb @@ -56,6 +57,11 @@ def _gpu_backend_transformer_layer_modules() -> MultiAccModules: args = get_args() + # Use local DotProductAttention if transformer_impl is "local", otherwise use TE version + # DotProductAttention=TEDotProductAttention (old hardcoded version, now commented) + use_te_attn = args.transformer_impl != "local" + attn_module = TEDotProductAttention if use_te_attn else DotProductAttention + return MultiAccModules( # dense linear TELayerNormColumnParallelLinear=TELayerNormColumnParallelLinear, @@ -68,7 +74,7 @@ def _gpu_backend_transformer_layer_modules() -> MultiAccModules: ColumnParallelLinear=ColumnParallelLinear, RowParallelLinear=RowParallelLinear, # attn - DotProductAttention=TEDotProductAttention, + DotProductAttention=attn_module, # norm TENorm=TENorm, LocalNorm=LocalNorm, diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index e831ae12..f063ea37 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -328,9 +328,26 @@ def forward( n_image_tokens = (input_ids == self.config.image_token_id).sum().item() n_image_features = image_embeddings.shape[0] if n_image_tokens != n_image_features: - raise ValueError( - f"Image features {n_image_features} != image tokens {n_image_tokens}" - ) + # raise ValueError( + # f"Image features {n_image_features} != image tokens {n_image_tokens}" + # ) + if n_image_features > n_image_tokens: + # Trim extra vision features if the model produced more than requested tokens. + image_embeddings = image_embeddings[:n_image_tokens] + if window_index is not None: + window_index = window_index[:n_image_tokens] + else: + # Pad missing vision features with zeros to align with token count. + pad_len = n_image_tokens - n_image_features + pad_emb = torch.zeros( + (pad_len, image_embeddings.size(1)), + device=image_embeddings.device, + dtype=image_embeddings.dtype, + ) + image_embeddings = torch.cat([image_embeddings, pad_emb], dim=0) + if window_index is not None: + pad_idx = window_index[-1:].repeat(pad_len, *([1] * (window_index.dim() - 1))) + window_index = torch.cat([window_index, pad_idx], dim=0) # If running inference, the language model KV cache will be updated for image token positions. # Here we store the image tokens sequence length, which can be used as an offset to the KV cache later. diff --git a/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py b/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py index 15eb5d7d..ea9ab0c9 100644 --- a/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py +++ b/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py @@ -18,20 +18,130 @@ def _rotate_half(x): return torch.cat((-x2, x1), dim=-1) +# def apply_rotary_pos_emb_vision(t, freqs, config, cu_seqlens=None, rotary_interleaved=False): +# """" Apply rotation to positional embedding """ +# orig_dtype = t.dtype +# t = t.float() +# if cu_seqlens is not None: +# freqs = freqs.squeeze(1) +# cos_ = freqs.cos().float().repeat(1, 1, 2) +# sin_ = freqs.sin().float().repeat(1, 1, 2) +# else: +# cos_ = freqs.cos().float().repeat(1, 1, 1, 2) +# sin_ = freqs.sin().float().repeat(1, 1, 1, 2) +# t = (t * cos_) + (_rotate_half(t) * sin_) +# return t.to(orig_dtype) + def apply_rotary_pos_emb_vision(t, freqs, config, cu_seqlens=None, rotary_interleaved=False): - """" Apply rotation to positional embedding """ + """ + Apply rotary positional embeddings to t in a robust way: + - build cos_/sin_ from freqs as before + - slice cos_/sin_ to match t sequence length + - expand/unsqueeze so broadcasting works for common layouts + """ + # DEBUG: Check input + if t is None: + print(f"[apply_rotary_pos_emb_vision] ERROR: input t is None!", flush=True) + return None + orig_dtype = t.dtype t = t.float() + + # Build cos_/sin_ as in your original code if cu_seqlens is not None: - freqs = freqs.squeeze(1) - cos_ = freqs.cos().float().repeat(1, 1, 2) - sin_ = freqs.sin().float().repeat(1, 1, 2) + # freqs shape expected something like [B, 1, S, C] or similar; you squeezed axis 1 + freqs_work = freqs.squeeze(1) + cos_ = freqs_work.cos().float().repeat(1, 1, 2) # -> [?, ?, 2*?] depending on freqs_work layout + sin_ = freqs_work.sin().float().repeat(1, 1, 2) else: cos_ = freqs.cos().float().repeat(1, 1, 1, 2) sin_ = freqs.sin().float().repeat(1, 1, 1, 2) + + # --- Robust alignment code: make cos_/sin_ match t's sequence axis & length ---- + # Determine candidate sequence axis in t. Prefer axis 0, then 1, then others. + # We'll detect the likely seq axis by comparing sizes; fallback to axis 0. + def _find_seq_axis_and_len(t, cos_): + t_shape = tuple(t.shape) + # Try to find an axis i where cos_.shape[0] matches t.shape[i] + for axis in range(len(t_shape)): + if cos_.shape[0] == t_shape[axis]: + return axis, t_shape[axis] + # If no exact match, prefer axis 0 if it's smaller or cos_ is larger and can be sliced + return 0, t_shape[0] + + seq_axis, seq_len = _find_seq_axis_and_len(t, cos_) + + # Slice cos_/sin_ on their first dimension to match seq_len (safe even if cos_ is smaller) + if cos_.shape[0] != seq_len: + # if cos_ is shorter than seq_len, we'll let broadcasting fail later (better to raise) + if cos_.shape[0] < seq_len: + # Recompute or raise — here we raise a helpful error so you can detect upstream bug + raise RuntimeError( + f"freqs/cos_ length ({cos_.shape[0]}) is smaller than the tensor sequence length ({seq_len}). " + "Recompute freqs for the required length or provide larger cached freqs." + ) + # Slice axis 0 + sl = [slice(None)] * cos_.ndim + sl[0] = slice(0, seq_len) + cos_ = cos_[tuple(sl)].contiguous() + sin_ = sin_[tuple(sl)].contiguous() + + # Now make cos_/sin_ broadcast-compatible with t. + # Typical cases: + # - t: [S, B, D] and cos_: [S, D] -> unsqueeze(1) -> [S,1,D] + # - t: [B, S, D] and cos_: [S, D] -> unsqueeze(0) -> [1,S,D] + # - t: [B, H, S, D] and cos_: [S, D] -> unsqueeze to [1,1,S,1,D...] not common + # We'll handle the common 2/3-dim cases and otherwise attempt to add singleton dims to the left. + if cos_.ndim == 2 and t.ndim == 3: + # cos_: [S, D] + if seq_axis == 0: + # t: [S, B, D] -> need [S, 1, D] + cos_ = cos_.unsqueeze(1) + sin_ = sin_.unsqueeze(1) + else: + # seq_axis == 1: t: [B, S, D] -> need [1, S, D] + cos_ = cos_.unsqueeze(0) + sin_ = sin_.unsqueeze(0) + elif cos_.ndim == 2 and t.ndim == 2: + # both [S, D] vs [S, D] -> ok + pass + elif cos_.ndim == 3 and t.ndim == 3: + # cos_ might already be [S, 1, D] or [B, S, D] style; try to match by permuting if necessary + # If cos_.shape[0] == seq_len then assume first dim is seq and cos_ probably is [S,1,D] + # else attempt no-op; broadcasting should handle remaining dims + pass + else: + # Generic approach: try to prefix cos_ with singleton dims so that cos_.ndim == t.ndim or cos_.ndim == t.ndim-1 + # We want sequence dim at position seq_axis in t; ensure cos_ has seq at dim 0 currently (we sliced it) + # So add singleton dims to the left until cos_.ndim matches t.ndim with seq at that position. + if cos_.ndim < t.ndim: + # number of dims to add on left + to_add = t.ndim - cos_.ndim + for _ in range(to_add): + cos_ = cos_.unsqueeze(0) + sin_ = sin_.unsqueeze(0) + # If still not aligned, rely on broadcasting but print a debug warning: + # (you can remove/disable the print in production) + # debug print + # print(f"[rotary] after align: t.shape={tuple(t.shape)}, cos_.shape={tuple(cos_.shape)}") + + # Final safety check: ensure cos_ can broadcast over t along seq_axis + # Map cos_ seq dim (we assumed it's axis 0) to the seq_axis in t via unsqueezing above. + # If the seq axis in t is not axis 0, the unsqueeze logic handled typical 3-D case above. + + # Debug print (optional; remove when satisfied) + # print("rotary debug: t.shape", tuple(t.shape), "cos.shape", tuple(cos_.shape), "sin.shape", tuple(sin_.shape)) + + # Apply rotary t = (t * cos_) + (_rotate_half(t) * sin_) - return t.to(orig_dtype) + result = t.to(orig_dtype) + + # DEBUG: Check output + if result is None: + print(f"[apply_rotary_pos_emb_vision] ERROR: result is None after conversion!", flush=True) + + return result class PatchEmbed(torch.nn.Module): diff --git a/aiak_training_llm/models/qwen_vl/adapter.py b/aiak_training_llm/models/qwen_vl/adapter.py index 57544bbc..65f47a6e 100644 --- a/aiak_training_llm/models/qwen_vl/adapter.py +++ b/aiak_training_llm/models/qwen_vl/adapter.py @@ -25,7 +25,9 @@ def __init__(self, spatial_merge_size: int = 2 ) -> None: super().__init__(config=config) - self.hidden_size = input_size * (spatial_merge_size**2) + # self.hidden_size = input_size * (spatial_merge_size**2) # OLD: assumes spatial merging + # Since input is already [seq_len, input_size], no spatial merging needed + self.hidden_size = input_size # Use input_size directly self.layernorm = build_module( submodules.layernorm, @@ -62,7 +64,8 @@ def __init__(self, def forward(self, x: torch.Tensor, window_index: torch.LongTensor = None) -> torch.Tensor: """ Forward pass.""" - x = self.layernorm(x).view(-1, self.hidden_size) + # Input is already [seq_len, input_size], no spatial merging needed + x = self.layernorm(x) x, _ = self.linear_fc1(x) x = self.activation_func(x) x, _ = self.linear_fc2(x) diff --git a/aiak_training_llm/train/arguments.py b/aiak_training_llm/train/arguments.py index acdf5b24..348734b1 100644 --- a/aiak_training_llm/train/arguments.py +++ b/aiak_training_llm/train/arguments.py @@ -79,7 +79,7 @@ def _add_extra_training_rice_vl_args(parser: argparse.ArgumentParser) -> argpars group.add_argument( '--training-rice-vl-max-answer-length', type=int, - default=4096, + default=512, # Changed from 4096 to 512 to match seq_length help=( "The maximum number of characters allowed in an answer during training. " "Answers longer than this will be truncated." diff --git a/ds/src/params.py b/ds/src/params.py index 1eb4dc97..f900558a 100644 --- a/ds/src/params.py +++ b/ds/src/params.py @@ -26,6 +26,7 @@ class TrainingArguments(HFTrainingArguments): max_seq_length: int = field( default=32768, # This is the default value of the qwen2-vl model + # default=2048, metadata={ "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)." @@ -73,6 +74,7 @@ class DPOArguments(DPOConfigTRL): max_seq_length: int = field( default=32768, # This is the default value of the qwen2-vl model + # default=2048, metadata={ "help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)." diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 28460b09..354e50fc 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -5,12 +5,12 @@ TP="${1:-2}" PP="${2:-1}" # pipeline parallel # Defaults: TP=1, PP=1. # SEQ_LEN="${3:-32768}" -SEQ_LEN="${3:-2048}" +SEQ_LEN="${3:-512}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size -GBS="${5:-2}" +GBS="${5:-8}" # NSTEP="${6:-2500}" # number of training iterations -NSTEP="${6:-5}" # number of training iterations +NSTEP="${6:-20}" # number of training iterations DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} @@ -111,7 +111,7 @@ DATA_ARGS=( --data-path "$DATA_PATH" --dataloader-type external --split 100,0,0 - --num-workers 1 + --num-workers 16 --chat-template qwen2-vl # will change this chat template ) @@ -120,6 +120,8 @@ TRAINING_ARGS=( --trainable-modules adapter --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" + --no-rope-fusion + --training-rice-vl-max-answer-length 512 --transformer-impl local --max-position-embeddings 32768 --init-method-std 0.02 @@ -153,7 +155,8 @@ TRAINING_ARGS=( ) MODEL_PARALLEL_ARGS=( - --attention-backend flash + # --attention-backend flash + --attention-backend local --pipeline-model-parallel-size "${PP}" --tensor-model-parallel-size "${TP}" --use-distributed-optimizer From 4692f438c8e72cc1e28170bef8b7c1a819c13d3d Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 25 Dec 2025 19:31:55 +0400 Subject: [PATCH 03/39] Track Stage1/alignment.sh in outer repo (de-nest Stage1) --- Stage1 | 1 - Stage1/alignment.sh | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) delete mode 160000 Stage1 create mode 100755 Stage1/alignment.sh diff --git a/Stage1 b/Stage1 deleted file mode 160000 index b16e06e0..00000000 --- a/Stage1 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b16e06e0c10c901bd35f1aad770e94ab6c6f33f3 diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh new file mode 100755 index 00000000..00733319 --- /dev/null +++ b/Stage1/alignment.sh @@ -0,0 +1,44 @@ +#!/bin/bash +#SBATCH --job-name=llava_stage1_4b +#SBATCH --time=72:00:00 +#SBATCH --nodes=1 +#SBATCH -p long +#SBATCH -q gpu-12 +#SBATCH --gres=gpu:4 # 4 A100 GPUs on a single node +#SBATCH --mem=230G # node RAM, not GPU RAM +#SBATCH --ntasks-per-node=4 # one task per GPU +#SBATCH --cpus-per-task=16 # adjust if you want fewer CPU cores +#SBATCH --output=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out # logs/llava_stage1_4b-.out + +# ---- ENV SETUP ---- +source ~/.bashrc +conda activate llava-ov-4b-clean +# conda activate apex_cuda120 +export APEX_CUDA_EXT=1 + +# Go to repo root +cd /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 +# ============================================================ +# Required environment variables: +# AIAK_TRAINING_PATH Root directory of the AIAK-Training-LLM project +# DATA_PATH Directory with WebDataset shards (.tar) for pretraining +# TOKENIZER_PATH Hugging Face tokenizer directory +# CHECKPOINT_PATH Megatron-formatted checkpoint directory (e.g., mcore TP1/PP1) +# SAVE_CKPT_PATH Output directory for saving training checkpoints +export CUDA_HOME=/apps/local/nvidia/cuda-12.0 +export PATH="$CUDA_HOME/bin:$PATH" +export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH + +AIAK_TRAINING_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 \ +DATA_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset \ +TOKENIZER_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 \ +CHECKPOINT_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ + +echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" +echo "DATA_PATH=${DATA_PATH}" +echo "TOKENIZER_PATH=${TOKENIZER_PATH}" +echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" +echo "SLURM_NODELIST=${SLURM_NODELIST}" + + +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh From f46815e8deaaa61d42ea69b27e07e4bc90af249d Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 25 Dec 2025 19:35:05 +0400 Subject: [PATCH 04/39] Track apex in outer repo (de-nest apex submodule) --- apex | 1 - apex/.github/ISSUE_TEMPLATE/bug_report.md | 23 + apex/.gitignore | 147 + apex/.gitmodules | 7 + apex/.nojekyll | 0 apex/LICENSE | 11 + apex/README.md | 182 + apex/apex/RNN/README.md | 3 + apex/apex/RNN/RNNBackend.py | 365 ++ apex/apex/RNN/__init__.py | 3 + apex/apex/RNN/cells.py | 84 + apex/apex/RNN/models.py | 56 + apex/apex/__init__.py | 68 + apex/apex/_autocast_utils.py | 26 + apex/apex/amp/README.md | 72 + apex/apex/amp/__init__.py | 5 + apex/apex/amp/__version__.py | 2 + apex/apex/amp/_amp_state.py | 59 + apex/apex/amp/_initialize.py | 265 ++ apex/apex/amp/_process_optimizer.py | 489 ++ apex/apex/amp/amp.py | 183 + apex/apex/amp/compat.py | 46 + apex/apex/amp/frontend.py | 446 ++ apex/apex/amp/handle.py | 281 ++ apex/apex/amp/lists/__init__.py | 0 apex/apex/amp/lists/functional_overrides.py | 80 + apex/apex/amp/lists/tensor_overrides.py | 63 + apex/apex/amp/lists/torch_overrides.py | 115 + apex/apex/amp/opt.py | 103 + apex/apex/amp/rnn_compat.py | 53 + apex/apex/amp/scaler.py | 217 + apex/apex/amp/utils.py | 210 + apex/apex/amp/wrap.py | 276 ++ apex/apex/contrib/__init__.py | 0 apex/apex/contrib/bottleneck/__init__.py | 2 + apex/apex/contrib/bottleneck/bottleneck.py | 749 +++ .../contrib/bottleneck/halo_exchangers.py | 180 + apex/apex/contrib/bottleneck/test.py | 71 + apex/apex/contrib/clip_grad/__init__.py | 1 + apex/apex/contrib/clip_grad/clip_grad.py | 128 + apex/apex/contrib/conv_bias_relu/__init__.py | 2 + .../contrib/conv_bias_relu/conv_bias_relu.py | 104 + .../contrib/csrc/bottleneck/bottleneck.cpp | 4073 +++++++++++++++++ .../csrc/conv_bias_relu/conv_bias_relu.cpp | 2149 +++++++++ .../apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp | 131 + .../contrib/csrc/cudnn_gbn/norm_sample.cpp | 474 ++ .../apex/contrib/csrc/cudnn_gbn/norm_sample.h | 79 + apex/apex/contrib/csrc/fmha/fmha_api.cpp | 365 ++ apex/apex/contrib/csrc/fmha/src/fmha.h | 163 + apex/apex/contrib/csrc/fmha/src/fmha/gemm.h | 314 ++ .../contrib/csrc/fmha/src/fmha/gmem_tile.h | 456 ++ .../csrc/fmha/src/fmha/kernel_traits.h | 97 + apex/apex/contrib/csrc/fmha/src/fmha/mask.h | 81 + .../contrib/csrc/fmha/src/fmha/smem_tile.h | 1286 ++++++ .../apex/contrib/csrc/fmha/src/fmha/softmax.h | 395 ++ apex/apex/contrib/csrc/fmha/src/fmha/utils.h | 1038 +++++ .../src/fmha_dgrad_fp16_128_64_kernel.sm80.cu | 60 + .../src/fmha_dgrad_fp16_256_64_kernel.sm80.cu | 60 + .../src/fmha_dgrad_fp16_384_64_kernel.sm80.cu | 60 + .../src/fmha_dgrad_fp16_512_64_kernel.sm80.cu | 105 + .../fmha/src/fmha_dgrad_kernel_1xN_reload.h | 558 +++ .../src/fmha_dgrad_kernel_1xN_reload_nl.h | 569 +++ apex/apex/contrib/csrc/fmha/src/fmha_fill.cu | 69 + .../src/fmha_fprop_fp16_128_64_kernel.sm80.cu | 79 + .../src/fmha_fprop_fp16_256_64_kernel.sm80.cu | 79 + .../src/fmha_fprop_fp16_384_64_kernel.sm80.cu | 79 + .../src/fmha_fprop_fp16_512_64_kernel.sm80.cu | 137 + .../csrc/fmha/src/fmha_fprop_kernel_1xN.h | 531 +++ apex/apex/contrib/csrc/fmha/src/fmha_kernel.h | 179 + .../csrc/fmha/src/fmha_noloop_reduce.cu | 177 + apex/apex/contrib/csrc/fmha/src/fmha_utils.h | 92 + .../csrc/focal_loss/focal_loss_cuda.cpp | 70 + .../csrc/focal_loss/focal_loss_cuda_kernel.cu | 267 ++ apex/apex/contrib/csrc/groupbn/batch_norm.cu | 328 ++ apex/apex/contrib/csrc/groupbn/batch_norm.h | 735 +++ .../csrc/groupbn/batch_norm_add_relu.cu | 340 ++ .../csrc/groupbn/batch_norm_add_relu.h | 682 +++ apex/apex/contrib/csrc/groupbn/cuda_utils.h | 20 + apex/apex/contrib/csrc/groupbn/interface.cpp | 175 + apex/apex/contrib/csrc/groupbn/ipc.cu | 129 + .../csrc/groupbn/nhwc_batch_norm_kernel.h | 2685 +++++++++++ .../csrc/index_mul_2d/index_mul_2d_cuda.cpp | 139 + .../index_mul_2d/index_mul_2d_cuda_kernel.cu | 479 ++ apex/apex/contrib/csrc/layer_norm/ln.h | 202 + apex/apex/contrib/csrc/layer_norm/ln_api.cpp | 246 + .../csrc/layer_norm/ln_bwd_kernels.cuh | 315 ++ .../layer_norm/ln_bwd_semi_cuda_kernel.cu | 240 + .../csrc/layer_norm/ln_fwd_cuda_kernel.cu | 228 + .../csrc/layer_norm/ln_fwd_kernels.cuh | 110 + .../csrc/layer_norm/ln_kernel_traits.h | 159 + .../apex/contrib/csrc/layer_norm/ln_utils.cuh | 733 +++ .../additive_masked_softmax_dropout_cuda.cu | 113 + .../contrib/csrc/multihead_attn/dropout.cuh | 272 ++ .../encdec_multihead_attn_cuda.cu | 368 ++ .../encdec_multihead_attn_norm_add_cuda.cu | 445 ++ .../csrc/multihead_attn/layer_norm.cuh | 636 +++ .../masked_softmax_dropout_cuda.cu | 124 + .../multihead_attn_frontend.cpp | 835 ++++ .../contrib/csrc/multihead_attn/philox.cuh | 126 + ..._multihead_attn_bias_additive_mask_cuda.cu | 306 ++ .../self_multihead_attn_bias_cuda.cu | 312 ++ .../self_multihead_attn_cuda.cu | 307 ++ .../self_multihead_attn_norm_add_cuda.cu | 385 ++ .../contrib/csrc/multihead_attn/softmax.cuh | 3142 +++++++++++++ .../multihead_attn/strided_batched_gemm.cuh | 565 +++ apex/apex/contrib/csrc/nccl_p2p/nccl_p2p.cpp | 25 + .../contrib/csrc/nccl_p2p/nccl_p2p_cuda.cu | 211 + .../contrib/csrc/nccl_p2p/nccl_p2p_cuda.cuh | 45 + .../contrib/csrc/nccl_p2p/nccl_version.cpp | 11 + .../csrc/nccl_p2p/nccl_version_check.cu | 10 + .../csrc/optimizers/fused_adam_cuda.cpp | 86 + .../csrc/optimizers/fused_adam_cuda_kernel.cu | 1037 +++++ .../csrc/optimizers/fused_lamb_cuda.cpp | 21 + .../csrc/optimizers/fused_lamb_cuda_kernel.cu | 294 ++ .../optimizers/multi_tensor_distopt_adam.cpp | 40 + .../multi_tensor_distopt_adam_kernel.cu | 430 ++ .../optimizers/multi_tensor_distopt_lamb.cpp | 36 + .../multi_tensor_distopt_lamb_kernel.cu | 506 ++ .../contrib/csrc/peer_memory/peer_memory.cpp | 29 + .../csrc/peer_memory/peer_memory_cuda.cu | 741 +++ .../csrc/peer_memory/peer_memory_cuda.cuh | 48 + .../csrc/transducer/transducer_joint.cpp | 98 + .../transducer/transducer_joint_kernel.cu | 979 ++++ .../csrc/transducer/transducer_loss.cpp | 109 + .../csrc/transducer/transducer_loss_kernel.cu | 767 ++++ apex/apex/contrib/csrc/xentropy/interface.cpp | 52 + .../contrib/csrc/xentropy/xentropy_kernel.cu | 718 +++ apex/apex/contrib/cudnn_gbn/__init__.py | 1 + apex/apex/contrib/cudnn_gbn/batch_norm.py | 144 + .../func_test_multihead_attn.py | 108 + .../perf_test_multihead_attn.py | 115 + apex/apex/contrib/fmha/__init__.py | 1 + apex/apex/contrib/fmha/fmha.py | 76 + apex/apex/contrib/focal_loss/__init__.py | 9 + apex/apex/contrib/focal_loss/focal_loss.py | 60 + apex/apex/contrib/groupbn/__init__.py | 9 + apex/apex/contrib/groupbn/batch_norm.py | 225 + apex/apex/contrib/index_mul_2d/__init__.py | 1 + .../apex/contrib/index_mul_2d/index_mul_2d.py | 144 + apex/apex/contrib/layer_norm/__init__.py | 1 + apex/apex/contrib/layer_norm/layer_norm.py | 53 + apex/apex/contrib/multihead_attn/MHA_bwd.png | Bin 0 -> 86630 bytes apex/apex/contrib/multihead_attn/MHA_fwd.png | Bin 0 -> 84392 bytes apex/apex/contrib/multihead_attn/README.md | 60 + apex/apex/contrib/multihead_attn/__init__.py | 3 + .../multihead_attn/encdec_multihead_attn.py | 188 + .../encdec_multihead_attn_func.py | 356 ++ .../fast_encdec_multihead_attn_func.py | 121 + ...ast_encdec_multihead_attn_norm_add_func.py | 159 + .../fast_self_multihead_attn_func.py | 243 + .../fast_self_multihead_attn_norm_add_func.py | 135 + .../mask_softmax_dropout_func.py | 64 + .../multihead_attn/self_multihead_attn.py | 254 + .../self_multihead_attn_func.py | 308 ++ apex/apex/contrib/optimizers/__init__.py | 3 + .../optimizers/distributed_fused_adam.py | 2136 +++++++++ .../optimizers/distributed_fused_lamb.py | 1061 +++++ .../apex/contrib/optimizers/fp16_optimizer.py | 243 + apex/apex/contrib/optimizers/fused_adam.py | 206 + apex/apex/contrib/optimizers/fused_lamb.py | 208 + apex/apex/contrib/optimizers/fused_sgd.py | 211 + apex/apex/contrib/peer_memory/__init__.py | 3 + .../peer_memory/peer_halo_exchanger_1d.py | 74 + apex/apex/contrib/peer_memory/peer_memory.py | 87 + apex/apex/contrib/sparsity/COPYRIGHT | 1 + apex/apex/contrib/sparsity/README.md | 138 + apex/apex/contrib/sparsity/__init__.py | 2 + apex/apex/contrib/sparsity/asp.py | 311 ++ apex/apex/contrib/sparsity/permutation_lib.py | 1683 +++++++ .../permutation_search_kernels.cu | 701 +++ .../permutation_search_kernels/__init__.py | 2 + .../call_permutation_search_kernels.py | 73 + .../channel_swap.py | 213 + .../exhaustive_search.py | 368 ++ .../permutation_utilities.py | 556 +++ .../sparsity/permutation_tests/README.md | 155 + .../permutation_tests/ablation_studies.sh | 87 + .../permutation_tests/permutation_test.py | 270 ++ .../permutation_tests/runtime_table.sh | 78 + .../permutation_tests/unstructured_study.sh | 93 + apex/apex/contrib/sparsity/sparse_masklib.py | 187 + .../sparsity/test/checkpointing_test_part1.py | 94 + .../sparsity/test/checkpointing_test_part2.py | 79 + .../test/checkpointing_test_reference.py | 96 + .../test/test_permutation_application.py | 792 ++++ .../apex/contrib/sparsity/test/toy_problem.py | 87 + apex/apex/contrib/test/__init__.py | 0 apex/apex/contrib/test/bottleneck/__init__.py | 0 .../test/bottleneck/test_bottleneck_module.py | 326 ++ apex/apex/contrib/test/clip_grad/__init__.py | 0 .../contrib/test/clip_grad/test_clip_grad.py | 172 + .../contrib/test/conv_bias_relu/__init__.py | 0 .../conv_bias_relu/test_conv_bias_relu.py | 129 + apex/apex/contrib/test/cudnn_gbn/__init__.py | 0 .../cudnn_gbn/test_cudnn_gbn_with_two_gpus.py | 147 + apex/apex/contrib/test/fmha/__init__.py | 0 apex/apex/contrib/test/fmha/test_fmha.py | 149 + apex/apex/contrib/test/focal_loss/__init__.py | 0 .../test/focal_loss/test_focal_loss.py | 74 + .../test/fused_dense/test_fused_dense.py | 62 + .../contrib/test/index_mul_2d/__init__.py | 0 .../test/index_mul_2d/test_index_mul_2d.py | 104 + apex/apex/contrib/test/layer_norm/__init__.py | 0 .../test/layer_norm/test_fast_layer_norm.py | 279 ++ .../contrib/test/multihead_attn/__init__.py | 0 .../test_encdec_multihead_attn.py | 161 + .../test_encdec_multihead_attn_norm_add.py | 87 + .../test_fast_self_multihead_attn_bias.py | 93 + .../multihead_attn/test_mha_fused_softmax.py | 55 + .../test_self_multihead_attn.py | 155 + .../test_self_multihead_attn_norm_add.py | 79 + apex/apex/contrib/test/optimizers/__init__.py | 0 .../contrib/test/optimizers/test_dist_adam.py | 470 ++ .../optimizers/test_distributed_fused_lamb.py | 125 + .../apex/contrib/test/peer_memory/__init__.py | 0 .../test_peer_halo_exchange_module.py | 328 ++ apex/apex/contrib/test/transducer/__init__.py | 0 .../test/transducer/test_transducer_joint.py | 167 + .../test/transducer/test_transducer_loss.py | 139 + apex/apex/contrib/test/xentropy/__init__.py | 0 .../test/xentropy/test_label_smoothing.py | 136 + apex/apex/contrib/transducer/__init__.py | 3 + .../contrib/transducer/_transducer_ref.py | 109 + apex/apex/contrib/transducer/transducer.py | 195 + apex/apex/contrib/xentropy/__init__.py | 6 + .../apex/contrib/xentropy/softmax_xentropy.py | 30 + apex/apex/fp16_utils/README.md | 16 + apex/apex/fp16_utils/__init__.py | 16 + apex/apex/fp16_utils/fp16_optimizer.py | 557 +++ apex/apex/fp16_utils/fp16util.py | 189 + apex/apex/fp16_utils/loss_scaler.py | 188 + apex/apex/fused_dense/__init__.py | 1 + apex/apex/fused_dense/fused_dense.py | 95 + apex/apex/mlp/__init__.py | 1 + apex/apex/mlp/mlp.py | 86 + apex/apex/multi_tensor_apply/__init__.py | 4 + .../multi_tensor_apply/multi_tensor_apply.py | 30 + apex/apex/normalization/__init__.py | 2 + apex/apex/normalization/fused_layer_norm.py | 437 ++ apex/apex/normalization/instance_norm.py | 151 + apex/apex/optimizers/__init__.py | 6 + apex/apex/optimizers/fused_adagrad.py | 122 + apex/apex/optimizers/fused_adam.py | 305 ++ apex/apex/optimizers/fused_lamb.py | 215 + .../optimizers/fused_mixed_precision_lamb.py | 256 ++ apex/apex/optimizers/fused_novograd.py | 214 + apex/apex/optimizers/fused_sgd.py | 227 + apex/apex/parallel/LARC.py | 107 + apex/apex/parallel/README.md | 66 + apex/apex/parallel/__init__.py | 97 + apex/apex/parallel/distributed.py | 643 +++ apex/apex/parallel/multiproc.py | 35 + .../apex/parallel/optimized_sync_batchnorm.py | 85 + .../optimized_sync_batchnorm_kernel.py | 119 + apex/apex/parallel/sync_batchnorm.py | 136 + apex/apex/parallel/sync_batchnorm_kernel.py | 87 + apex/apex/transformer/README.md | 81 + apex/apex/transformer/__init__.py | 23 + apex/apex/transformer/_data/__init__.py | 8 + apex/apex/transformer/_data/_batchsampler.py | 180 + apex/apex/transformer/_ucc_util.py | 9 + apex/apex/transformer/amp/__init__.py | 6 + apex/apex/transformer/amp/grad_scaler.py | 125 + apex/apex/transformer/enums.py | 35 + apex/apex/transformer/functional/__init__.py | 5 + .../transformer/functional/fused_softmax.py | 301 ++ apex/apex/transformer/layers/__init__.py | 11 + apex/apex/transformer/layers/layer_norm.py | 99 + apex/apex/transformer/log_util.py | 18 + apex/apex/transformer/microbatches.py | 195 + apex/apex/transformer/parallel_state.py | 796 ++++ .../transformer/pipeline_parallel/__init__.py | 8 + .../transformer/pipeline_parallel/_timers.py | 83 + .../pipeline_parallel/p2p_communication.py | 684 +++ .../pipeline_parallel/schedules/__init__.py | 35 + .../pipeline_parallel/schedules/common.py | 403 ++ .../schedules/fwd_bwd_no_pipelining.py | 124 + .../fwd_bwd_pipelining_with_interleaving.py | 744 +++ ...fwd_bwd_pipelining_without_interleaving.py | 597 +++ .../transformer/pipeline_parallel/utils.py | 357 ++ .../transformer/tensor_parallel/__init__.py | 75 + .../tensor_parallel/cross_entropy.py | 134 + apex/apex/transformer/tensor_parallel/data.py | 122 + .../transformer/tensor_parallel/layers.py | 813 ++++ .../transformer/tensor_parallel/mappings.py | 312 ++ .../transformer/tensor_parallel/memory.py | 151 + .../transformer/tensor_parallel/random.py | 311 ++ .../apex/transformer/tensor_parallel/utils.py | 64 + apex/apex/transformer/testing/__init__.py | 0 apex/apex/transformer/testing/arguments.py | 977 ++++ apex/apex/transformer/testing/commons.py | 296 ++ .../testing/distributed_test_base.py | 126 + apex/apex/transformer/testing/global_vars.py | 272 ++ .../transformer/testing/standalone_bert.py | 255 ++ .../transformer/testing/standalone_gpt.py | 111 + .../testing/standalone_transformer_lm.py | 1574 +++++++ apex/apex/transformer/utils.py | 54 + apex/csrc/amp_C_frontend.cpp | 205 + apex/csrc/compat.h | 9 + apex/csrc/flatten_unflatten.cpp | 18 + apex/csrc/fused_dense.cpp | 193 + apex/csrc/fused_dense_cuda.cu | 1930 ++++++++ apex/csrc/instance_norm_nvfuser.cpp | 35 + apex/csrc/instance_norm_nvfuser_kernel.cu | 286 ++ apex/csrc/layer_norm_cuda.cpp | 442 ++ apex/csrc/layer_norm_cuda_kernel.cu | 1170 +++++ .../megatron/fused_weight_gradient_dense.cpp | 21 + ...d_weight_gradient_dense_16bit_prec_cuda.cu | 155 + .../fused_weight_gradient_dense_cuda.cu | 195 + .../generic_scaled_masked_softmax.cpp | 83 + .../megatron/generic_scaled_masked_softmax.h | 384 ++ .../generic_scaled_masked_softmax_cuda.cu | 114 + apex/csrc/megatron/scaled_masked_softmax.cpp | 105 + apex/csrc/megatron/scaled_masked_softmax.h | 735 +++ .../megatron/scaled_masked_softmax_cuda.cu | 121 + apex/csrc/megatron/scaled_softmax.cpp | 75 + apex/csrc/megatron/scaled_softmax_cuda.cu | 104 + .../scaled_upper_triang_masked_softmax.cpp | 72 + .../scaled_upper_triang_masked_softmax.h | 538 +++ ...scaled_upper_triang_masked_softmax_cuda.cu | 98 + apex/csrc/mlp.cpp | 166 + apex/csrc/mlp_cuda.cu | 1678 +++++++ apex/csrc/multi_tensor_adagrad.cu | 100 + apex/csrc/multi_tensor_adam.cu | 478 ++ apex/csrc/multi_tensor_apply.cuh | 133 + apex/csrc/multi_tensor_axpby_kernel.cu | 157 + apex/csrc/multi_tensor_l2norm_kernel.cu | 448 ++ apex/csrc/multi_tensor_l2norm_kernel_mp.cu | 216 + apex/csrc/multi_tensor_l2norm_scale_kernel.cu | 326 ++ apex/csrc/multi_tensor_lamb.cu | 413 ++ apex/csrc/multi_tensor_lamb_mp.cu | 496 ++ apex/csrc/multi_tensor_lamb_stage_1.cu | 151 + apex/csrc/multi_tensor_lamb_stage_2.cu | 125 + apex/csrc/multi_tensor_novograd.cu | 188 + apex/csrc/multi_tensor_scale_kernel.cu | 136 + apex/csrc/multi_tensor_sgd_kernel.cu | 280 ++ apex/csrc/syncbn.cpp | 109 + apex/csrc/type_shim.h | 419 ++ apex/csrc/welford.cu | 1510 ++++++ apex/docs/Makefile | 32 + .../docs/source/_static/css/pytorch_theme.css | 118 + apex/docs/source/_static/img/nv-pytorch2.png | Bin 0 -> 6502 bytes apex/docs/source/_templates/layout.html | 51 + apex/docs/source/advanced.rst | 219 + apex/docs/source/amp.rst | 288 ++ apex/docs/source/conf.py | 248 + apex/docs/source/fp16_utils.rst | 59 + apex/docs/source/index.rst | 53 + apex/docs/source/layernorm.rst | 17 + apex/docs/source/optimizers.rst | 23 + apex/docs/source/parallel.rst | 25 + apex/examples/README.md | 4 + apex/examples/dcgan/README.md | 41 + apex/examples/dcgan/main_amp.py | 274 ++ apex/examples/docker/Dockerfile | 16 + apex/examples/docker/README.md | 40 + apex/examples/imagenet/README.md | 183 + apex/examples/imagenet/main_amp.py | 543 +++ apex/examples/simple/distributed/README.md | 13 + .../distributed/distributed_data_parallel.py | 65 + apex/examples/simple/distributed/run.sh | 2 + apex/requirements.txt | 6 + apex/requirements_dev.txt | 3 + apex/setup.py | 806 ++++ apex/tests/L0/run_amp/__init__.py | 0 apex/tests/L0/run_amp/test_add_param_group.py | 148 + apex/tests/L0/run_amp/test_basic_casts.py | 143 + apex/tests/L0/run_amp/test_cache.py | 137 + apex/tests/L0/run_amp/test_checkpointing.py | 267 ++ apex/tests/L0/run_amp/test_fused_sgd.py | 794 ++++ apex/tests/L0/run_amp/test_larc.py | 53 + .../L0/run_amp/test_multi_tensor_axpby.py | 180 + .../L0/run_amp/test_multi_tensor_l2norm.py | 87 + .../L0/run_amp/test_multi_tensor_scale.py | 126 + .../test_multiple_models_optimizers_losses.py | 762 +++ apex/tests/L0/run_amp/test_promotion.py | 75 + apex/tests/L0/run_amp/test_rnn.py | 116 + apex/tests/L0/run_amp/utils.py | 21 + .../run_deprecated/test_deprecated_warning.py | 56 + apex/tests/L0/run_fp16util/__init__.py | 0 apex/tests/L0/run_fp16util/test_fp16util.py | 75 + .../test_fused_layer_norm.py | 273 ++ .../L0/run_instance_norm_nvfuser/__init__.py | 0 .../test_instance_norm_nvfuser.py | 107 + apex/tests/L0/run_mlp/test_mlp.py | 206 + apex/tests/L0/run_optimizers/__init__.py | 0 apex/tests/L0/run_optimizers/test_adam.py | 238 + .../L0/run_optimizers/test_fused_novograd.py | 170 + .../L0/run_optimizers/test_fused_optimizer.py | 311 ++ apex/tests/L0/run_optimizers/test_lamb.py | 336 ++ apex/tests/L0/run_test.py | 94 + apex/tests/L0/run_transformer/__init__.py | 0 .../L0/run_transformer/gpt_scaling_test.py | 116 + .../L0/run_transformer/test_batch_sampler.py | 141 + .../L0/run_transformer/test_bert_minimal.py | 250 + .../L0/run_transformer/test_cross_entropy.py | 97 + apex/tests/L0/run_transformer/test_data.py | 64 + .../run_transformer/test_dynamic_batchsize.py | 220 + .../L0/run_transformer/test_fused_softmax.py | 371 ++ .../L0/run_transformer/test_gpt_minimal.py | 232 + apex/tests/L0/run_transformer/test_layers.py | 561 +++ apex/tests/L0/run_transformer/test_mapping.py | 87 + .../L0/run_transformer/test_microbatches.py | 85 + .../tests/L0/run_transformer/test_p2p_comm.py | 122 + .../L0/run_transformer/test_parallel_state.py | 187 + .../test_pipeline_parallel_fwd_bwd.py | 716 +++ apex/tests/L0/run_transformer/test_random.py | 117 + .../run_transformer/test_transformer_utils.py | 40 + apex/tests/L1/common/compare.py | 64 + apex/tests/L1/common/main_amp.py | 526 +++ apex/tests/L1/common/run_test.sh | 144 + apex/tests/L1/cross_product/run.sh | 6 + .../tests/L1/cross_product_distributed/run.sh | 4 + .../pipeline_parallel_fwd_bwd_ucc_async.py | 219 + .../DDP/ddp_race_condition_test.py | 69 + apex/tests/distributed/DDP/run_race_test.sh | 3 + .../amp_master_params/amp_master_params.py | 70 + .../distributed/amp_master_params/compare.py | 28 + .../distributed/amp_master_params/run.sh | 4 + .../python_single_gpu_unit_test.py | 111 + .../synced_batchnorm/single_gpu_unit_test.py | 159 + .../synced_batchnorm/test_batchnorm1d.py | 18 + .../synced_batchnorm/test_groups.py | 185 + .../two_gpu_test_different_batch_size.py | 158 + .../synced_batchnorm/two_gpu_unit_test.py | 180 + .../distributed/synced_batchnorm/unit_test.sh | 8 + apex/tests/docker_extension_builds/run.sh | 73 + 427 files changed, 101562 insertions(+), 1 deletion(-) delete mode 160000 apex create mode 100644 apex/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 apex/.gitignore create mode 100644 apex/.gitmodules create mode 100644 apex/.nojekyll create mode 100644 apex/LICENSE create mode 100644 apex/README.md create mode 100644 apex/apex/RNN/README.md create mode 100644 apex/apex/RNN/RNNBackend.py create mode 100644 apex/apex/RNN/__init__.py create mode 100644 apex/apex/RNN/cells.py create mode 100644 apex/apex/RNN/models.py create mode 100644 apex/apex/__init__.py create mode 100644 apex/apex/_autocast_utils.py create mode 100644 apex/apex/amp/README.md create mode 100644 apex/apex/amp/__init__.py create mode 100644 apex/apex/amp/__version__.py create mode 100644 apex/apex/amp/_amp_state.py create mode 100644 apex/apex/amp/_initialize.py create mode 100644 apex/apex/amp/_process_optimizer.py create mode 100644 apex/apex/amp/amp.py create mode 100644 apex/apex/amp/compat.py create mode 100644 apex/apex/amp/frontend.py create mode 100644 apex/apex/amp/handle.py create mode 100644 apex/apex/amp/lists/__init__.py create mode 100644 apex/apex/amp/lists/functional_overrides.py create mode 100644 apex/apex/amp/lists/tensor_overrides.py create mode 100644 apex/apex/amp/lists/torch_overrides.py create mode 100644 apex/apex/amp/opt.py create mode 100644 apex/apex/amp/rnn_compat.py create mode 100644 apex/apex/amp/scaler.py create mode 100644 apex/apex/amp/utils.py create mode 100644 apex/apex/amp/wrap.py create mode 100644 apex/apex/contrib/__init__.py create mode 100644 apex/apex/contrib/bottleneck/__init__.py create mode 100644 apex/apex/contrib/bottleneck/bottleneck.py create mode 100644 apex/apex/contrib/bottleneck/halo_exchangers.py create mode 100644 apex/apex/contrib/bottleneck/test.py create mode 100644 apex/apex/contrib/clip_grad/__init__.py create mode 100644 apex/apex/contrib/clip_grad/clip_grad.py create mode 100644 apex/apex/contrib/conv_bias_relu/__init__.py create mode 100644 apex/apex/contrib/conv_bias_relu/conv_bias_relu.py create mode 100644 apex/apex/contrib/csrc/bottleneck/bottleneck.cpp create mode 100644 apex/apex/contrib/csrc/conv_bias_relu/conv_bias_relu.cpp create mode 100644 apex/apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp create mode 100644 apex/apex/contrib/csrc/cudnn_gbn/norm_sample.cpp create mode 100644 apex/apex/contrib/csrc/cudnn_gbn/norm_sample.h create mode 100644 apex/apex/contrib/csrc/fmha/fmha_api.cpp create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/gemm.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/gmem_tile.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/kernel_traits.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/mask.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/smem_tile.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/softmax.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha/utils.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_128_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_256_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_384_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_512_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload_nl.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_fill.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_128_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_256_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_384_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_512_64_kernel.sm80.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_fprop_kernel_1xN.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_kernel.h create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_noloop_reduce.cu create mode 100644 apex/apex/contrib/csrc/fmha/src/fmha_utils.h create mode 100644 apex/apex/contrib/csrc/focal_loss/focal_loss_cuda.cpp create mode 100644 apex/apex/contrib/csrc/focal_loss/focal_loss_cuda_kernel.cu create mode 100644 apex/apex/contrib/csrc/groupbn/batch_norm.cu create mode 100644 apex/apex/contrib/csrc/groupbn/batch_norm.h create mode 100644 apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.cu create mode 100644 apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.h create mode 100644 apex/apex/contrib/csrc/groupbn/cuda_utils.h create mode 100644 apex/apex/contrib/csrc/groupbn/interface.cpp create mode 100644 apex/apex/contrib/csrc/groupbn/ipc.cu create mode 100644 apex/apex/contrib/csrc/groupbn/nhwc_batch_norm_kernel.h create mode 100644 apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda.cpp create mode 100644 apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda_kernel.cu create mode 100644 apex/apex/contrib/csrc/layer_norm/ln.h create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_api.cpp create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_bwd_kernels.cuh create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_bwd_semi_cuda_kernel.cu create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_fwd_cuda_kernel.cu create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_fwd_kernels.cuh create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_kernel_traits.h create mode 100644 apex/apex/contrib/csrc/layer_norm/ln_utils.cuh create mode 100644 apex/apex/contrib/csrc/multihead_attn/additive_masked_softmax_dropout_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/dropout.cuh create mode 100644 apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/layer_norm.cuh create mode 100644 apex/apex/contrib/csrc/multihead_attn/masked_softmax_dropout_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/multihead_attn_frontend.cpp create mode 100644 apex/apex/contrib/csrc/multihead_attn/philox.cuh create mode 100644 apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_additive_mask_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu create mode 100644 apex/apex/contrib/csrc/multihead_attn/softmax.cuh create mode 100644 apex/apex/contrib/csrc/multihead_attn/strided_batched_gemm.cuh create mode 100644 apex/apex/contrib/csrc/nccl_p2p/nccl_p2p.cpp create mode 100644 apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cu create mode 100644 apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cuh create mode 100644 apex/apex/contrib/csrc/nccl_p2p/nccl_version.cpp create mode 100644 apex/apex/contrib/csrc/nccl_p2p/nccl_version_check.cu create mode 100644 apex/apex/contrib/csrc/optimizers/fused_adam_cuda.cpp create mode 100644 apex/apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu create mode 100644 apex/apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp create mode 100644 apex/apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu create mode 100644 apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam.cpp create mode 100644 apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam_kernel.cu create mode 100644 apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb.cpp create mode 100644 apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb_kernel.cu create mode 100644 apex/apex/contrib/csrc/peer_memory/peer_memory.cpp create mode 100644 apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cu create mode 100644 apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cuh create mode 100755 apex/apex/contrib/csrc/transducer/transducer_joint.cpp create mode 100755 apex/apex/contrib/csrc/transducer/transducer_joint_kernel.cu create mode 100644 apex/apex/contrib/csrc/transducer/transducer_loss.cpp create mode 100755 apex/apex/contrib/csrc/transducer/transducer_loss_kernel.cu create mode 100644 apex/apex/contrib/csrc/xentropy/interface.cpp create mode 100644 apex/apex/contrib/csrc/xentropy/xentropy_kernel.cu create mode 100644 apex/apex/contrib/cudnn_gbn/__init__.py create mode 100644 apex/apex/contrib/cudnn_gbn/batch_norm.py create mode 100644 apex/apex/contrib/examples/multihead_attn/func_test_multihead_attn.py create mode 100644 apex/apex/contrib/examples/multihead_attn/perf_test_multihead_attn.py create mode 100644 apex/apex/contrib/fmha/__init__.py create mode 100644 apex/apex/contrib/fmha/fmha.py create mode 100644 apex/apex/contrib/focal_loss/__init__.py create mode 100644 apex/apex/contrib/focal_loss/focal_loss.py create mode 100644 apex/apex/contrib/groupbn/__init__.py create mode 100644 apex/apex/contrib/groupbn/batch_norm.py create mode 100644 apex/apex/contrib/index_mul_2d/__init__.py create mode 100644 apex/apex/contrib/index_mul_2d/index_mul_2d.py create mode 100644 apex/apex/contrib/layer_norm/__init__.py create mode 100644 apex/apex/contrib/layer_norm/layer_norm.py create mode 100644 apex/apex/contrib/multihead_attn/MHA_bwd.png create mode 100644 apex/apex/contrib/multihead_attn/MHA_fwd.png create mode 100644 apex/apex/contrib/multihead_attn/README.md create mode 100644 apex/apex/contrib/multihead_attn/__init__.py create mode 100644 apex/apex/contrib/multihead_attn/encdec_multihead_attn.py create mode 100644 apex/apex/contrib/multihead_attn/encdec_multihead_attn_func.py create mode 100644 apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py create mode 100644 apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py create mode 100644 apex/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py create mode 100644 apex/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py create mode 100644 apex/apex/contrib/multihead_attn/mask_softmax_dropout_func.py create mode 100644 apex/apex/contrib/multihead_attn/self_multihead_attn.py create mode 100644 apex/apex/contrib/multihead_attn/self_multihead_attn_func.py create mode 100644 apex/apex/contrib/optimizers/__init__.py create mode 100644 apex/apex/contrib/optimizers/distributed_fused_adam.py create mode 100644 apex/apex/contrib/optimizers/distributed_fused_lamb.py create mode 100755 apex/apex/contrib/optimizers/fp16_optimizer.py create mode 100644 apex/apex/contrib/optimizers/fused_adam.py create mode 100644 apex/apex/contrib/optimizers/fused_lamb.py create mode 100644 apex/apex/contrib/optimizers/fused_sgd.py create mode 100644 apex/apex/contrib/peer_memory/__init__.py create mode 100644 apex/apex/contrib/peer_memory/peer_halo_exchanger_1d.py create mode 100644 apex/apex/contrib/peer_memory/peer_memory.py create mode 100644 apex/apex/contrib/sparsity/COPYRIGHT create mode 100644 apex/apex/contrib/sparsity/README.md create mode 100644 apex/apex/contrib/sparsity/__init__.py create mode 100644 apex/apex/contrib/sparsity/asp.py create mode 100644 apex/apex/contrib/sparsity/permutation_lib.py create mode 100644 apex/apex/contrib/sparsity/permutation_search_kernels/CUDA_kernels/permutation_search_kernels.cu create mode 100644 apex/apex/contrib/sparsity/permutation_search_kernels/__init__.py create mode 100644 apex/apex/contrib/sparsity/permutation_search_kernels/call_permutation_search_kernels.py create mode 100644 apex/apex/contrib/sparsity/permutation_search_kernels/channel_swap.py create mode 100644 apex/apex/contrib/sparsity/permutation_search_kernels/exhaustive_search.py create mode 100644 apex/apex/contrib/sparsity/permutation_search_kernels/permutation_utilities.py create mode 100644 apex/apex/contrib/sparsity/permutation_tests/README.md create mode 100755 apex/apex/contrib/sparsity/permutation_tests/ablation_studies.sh create mode 100644 apex/apex/contrib/sparsity/permutation_tests/permutation_test.py create mode 100755 apex/apex/contrib/sparsity/permutation_tests/runtime_table.sh create mode 100755 apex/apex/contrib/sparsity/permutation_tests/unstructured_study.sh create mode 100644 apex/apex/contrib/sparsity/sparse_masklib.py create mode 100644 apex/apex/contrib/sparsity/test/checkpointing_test_part1.py create mode 100644 apex/apex/contrib/sparsity/test/checkpointing_test_part2.py create mode 100644 apex/apex/contrib/sparsity/test/checkpointing_test_reference.py create mode 100644 apex/apex/contrib/sparsity/test/test_permutation_application.py create mode 100644 apex/apex/contrib/sparsity/test/toy_problem.py create mode 100644 apex/apex/contrib/test/__init__.py create mode 100644 apex/apex/contrib/test/bottleneck/__init__.py create mode 100644 apex/apex/contrib/test/bottleneck/test_bottleneck_module.py create mode 100644 apex/apex/contrib/test/clip_grad/__init__.py create mode 100644 apex/apex/contrib/test/clip_grad/test_clip_grad.py create mode 100644 apex/apex/contrib/test/conv_bias_relu/__init__.py create mode 100644 apex/apex/contrib/test/conv_bias_relu/test_conv_bias_relu.py create mode 100644 apex/apex/contrib/test/cudnn_gbn/__init__.py create mode 100644 apex/apex/contrib/test/cudnn_gbn/test_cudnn_gbn_with_two_gpus.py create mode 100644 apex/apex/contrib/test/fmha/__init__.py create mode 100644 apex/apex/contrib/test/fmha/test_fmha.py create mode 100644 apex/apex/contrib/test/focal_loss/__init__.py create mode 100644 apex/apex/contrib/test/focal_loss/test_focal_loss.py create mode 100644 apex/apex/contrib/test/fused_dense/test_fused_dense.py create mode 100644 apex/apex/contrib/test/index_mul_2d/__init__.py create mode 100644 apex/apex/contrib/test/index_mul_2d/test_index_mul_2d.py create mode 100644 apex/apex/contrib/test/layer_norm/__init__.py create mode 100644 apex/apex/contrib/test/layer_norm/test_fast_layer_norm.py create mode 100644 apex/apex/contrib/test/multihead_attn/__init__.py create mode 100644 apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn.py create mode 100644 apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn_norm_add.py create mode 100644 apex/apex/contrib/test/multihead_attn/test_fast_self_multihead_attn_bias.py create mode 100644 apex/apex/contrib/test/multihead_attn/test_mha_fused_softmax.py create mode 100644 apex/apex/contrib/test/multihead_attn/test_self_multihead_attn.py create mode 100644 apex/apex/contrib/test/multihead_attn/test_self_multihead_attn_norm_add.py create mode 100644 apex/apex/contrib/test/optimizers/__init__.py create mode 100644 apex/apex/contrib/test/optimizers/test_dist_adam.py create mode 100644 apex/apex/contrib/test/optimizers/test_distributed_fused_lamb.py create mode 100644 apex/apex/contrib/test/peer_memory/__init__.py create mode 100644 apex/apex/contrib/test/peer_memory/test_peer_halo_exchange_module.py create mode 100644 apex/apex/contrib/test/transducer/__init__.py create mode 100755 apex/apex/contrib/test/transducer/test_transducer_joint.py create mode 100755 apex/apex/contrib/test/transducer/test_transducer_loss.py create mode 100644 apex/apex/contrib/test/xentropy/__init__.py create mode 100644 apex/apex/contrib/test/xentropy/test_label_smoothing.py create mode 100755 apex/apex/contrib/transducer/__init__.py create mode 100755 apex/apex/contrib/transducer/_transducer_ref.py create mode 100755 apex/apex/contrib/transducer/transducer.py create mode 100644 apex/apex/contrib/xentropy/__init__.py create mode 100644 apex/apex/contrib/xentropy/softmax_xentropy.py create mode 100644 apex/apex/fp16_utils/README.md create mode 100644 apex/apex/fp16_utils/__init__.py create mode 100755 apex/apex/fp16_utils/fp16_optimizer.py create mode 100644 apex/apex/fp16_utils/fp16util.py create mode 100644 apex/apex/fp16_utils/loss_scaler.py create mode 100644 apex/apex/fused_dense/__init__.py create mode 100644 apex/apex/fused_dense/fused_dense.py create mode 100644 apex/apex/mlp/__init__.py create mode 100644 apex/apex/mlp/mlp.py create mode 100644 apex/apex/multi_tensor_apply/__init__.py create mode 100644 apex/apex/multi_tensor_apply/multi_tensor_apply.py create mode 100644 apex/apex/normalization/__init__.py create mode 100644 apex/apex/normalization/fused_layer_norm.py create mode 100644 apex/apex/normalization/instance_norm.py create mode 100644 apex/apex/optimizers/__init__.py create mode 100644 apex/apex/optimizers/fused_adagrad.py create mode 100644 apex/apex/optimizers/fused_adam.py create mode 100644 apex/apex/optimizers/fused_lamb.py create mode 100644 apex/apex/optimizers/fused_mixed_precision_lamb.py create mode 100644 apex/apex/optimizers/fused_novograd.py create mode 100644 apex/apex/optimizers/fused_sgd.py create mode 100644 apex/apex/parallel/LARC.py create mode 100644 apex/apex/parallel/README.md create mode 100644 apex/apex/parallel/__init__.py create mode 100644 apex/apex/parallel/distributed.py create mode 100644 apex/apex/parallel/multiproc.py create mode 100644 apex/apex/parallel/optimized_sync_batchnorm.py create mode 100644 apex/apex/parallel/optimized_sync_batchnorm_kernel.py create mode 100644 apex/apex/parallel/sync_batchnorm.py create mode 100644 apex/apex/parallel/sync_batchnorm_kernel.py create mode 100644 apex/apex/transformer/README.md create mode 100644 apex/apex/transformer/__init__.py create mode 100644 apex/apex/transformer/_data/__init__.py create mode 100644 apex/apex/transformer/_data/_batchsampler.py create mode 100644 apex/apex/transformer/_ucc_util.py create mode 100644 apex/apex/transformer/amp/__init__.py create mode 100644 apex/apex/transformer/amp/grad_scaler.py create mode 100644 apex/apex/transformer/enums.py create mode 100644 apex/apex/transformer/functional/__init__.py create mode 100644 apex/apex/transformer/functional/fused_softmax.py create mode 100644 apex/apex/transformer/layers/__init__.py create mode 100644 apex/apex/transformer/layers/layer_norm.py create mode 100644 apex/apex/transformer/log_util.py create mode 100644 apex/apex/transformer/microbatches.py create mode 100644 apex/apex/transformer/parallel_state.py create mode 100644 apex/apex/transformer/pipeline_parallel/__init__.py create mode 100644 apex/apex/transformer/pipeline_parallel/_timers.py create mode 100644 apex/apex/transformer/pipeline_parallel/p2p_communication.py create mode 100644 apex/apex/transformer/pipeline_parallel/schedules/__init__.py create mode 100644 apex/apex/transformer/pipeline_parallel/schedules/common.py create mode 100644 apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_no_pipelining.py create mode 100644 apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_with_interleaving.py create mode 100644 apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_without_interleaving.py create mode 100644 apex/apex/transformer/pipeline_parallel/utils.py create mode 100644 apex/apex/transformer/tensor_parallel/__init__.py create mode 100644 apex/apex/transformer/tensor_parallel/cross_entropy.py create mode 100644 apex/apex/transformer/tensor_parallel/data.py create mode 100644 apex/apex/transformer/tensor_parallel/layers.py create mode 100644 apex/apex/transformer/tensor_parallel/mappings.py create mode 100644 apex/apex/transformer/tensor_parallel/memory.py create mode 100644 apex/apex/transformer/tensor_parallel/random.py create mode 100644 apex/apex/transformer/tensor_parallel/utils.py create mode 100644 apex/apex/transformer/testing/__init__.py create mode 100644 apex/apex/transformer/testing/arguments.py create mode 100644 apex/apex/transformer/testing/commons.py create mode 100644 apex/apex/transformer/testing/distributed_test_base.py create mode 100644 apex/apex/transformer/testing/global_vars.py create mode 100644 apex/apex/transformer/testing/standalone_bert.py create mode 100644 apex/apex/transformer/testing/standalone_gpt.py create mode 100644 apex/apex/transformer/testing/standalone_transformer_lm.py create mode 100644 apex/apex/transformer/utils.py create mode 100644 apex/csrc/amp_C_frontend.cpp create mode 100644 apex/csrc/compat.h create mode 100644 apex/csrc/flatten_unflatten.cpp create mode 100644 apex/csrc/fused_dense.cpp create mode 100644 apex/csrc/fused_dense_cuda.cu create mode 100644 apex/csrc/instance_norm_nvfuser.cpp create mode 100644 apex/csrc/instance_norm_nvfuser_kernel.cu create mode 100644 apex/csrc/layer_norm_cuda.cpp create mode 100644 apex/csrc/layer_norm_cuda_kernel.cu create mode 100644 apex/csrc/megatron/fused_weight_gradient_dense.cpp create mode 100644 apex/csrc/megatron/fused_weight_gradient_dense_16bit_prec_cuda.cu create mode 100644 apex/csrc/megatron/fused_weight_gradient_dense_cuda.cu create mode 100644 apex/csrc/megatron/generic_scaled_masked_softmax.cpp create mode 100644 apex/csrc/megatron/generic_scaled_masked_softmax.h create mode 100644 apex/csrc/megatron/generic_scaled_masked_softmax_cuda.cu create mode 100644 apex/csrc/megatron/scaled_masked_softmax.cpp create mode 100644 apex/csrc/megatron/scaled_masked_softmax.h create mode 100644 apex/csrc/megatron/scaled_masked_softmax_cuda.cu create mode 100644 apex/csrc/megatron/scaled_softmax.cpp create mode 100644 apex/csrc/megatron/scaled_softmax_cuda.cu create mode 100644 apex/csrc/megatron/scaled_upper_triang_masked_softmax.cpp create mode 100644 apex/csrc/megatron/scaled_upper_triang_masked_softmax.h create mode 100644 apex/csrc/megatron/scaled_upper_triang_masked_softmax_cuda.cu create mode 100644 apex/csrc/mlp.cpp create mode 100644 apex/csrc/mlp_cuda.cu create mode 100644 apex/csrc/multi_tensor_adagrad.cu create mode 100644 apex/csrc/multi_tensor_adam.cu create mode 100644 apex/csrc/multi_tensor_apply.cuh create mode 100644 apex/csrc/multi_tensor_axpby_kernel.cu create mode 100644 apex/csrc/multi_tensor_l2norm_kernel.cu create mode 100644 apex/csrc/multi_tensor_l2norm_kernel_mp.cu create mode 100644 apex/csrc/multi_tensor_l2norm_scale_kernel.cu create mode 100644 apex/csrc/multi_tensor_lamb.cu create mode 100644 apex/csrc/multi_tensor_lamb_mp.cu create mode 100644 apex/csrc/multi_tensor_lamb_stage_1.cu create mode 100644 apex/csrc/multi_tensor_lamb_stage_2.cu create mode 100644 apex/csrc/multi_tensor_novograd.cu create mode 100644 apex/csrc/multi_tensor_scale_kernel.cu create mode 100644 apex/csrc/multi_tensor_sgd_kernel.cu create mode 100644 apex/csrc/syncbn.cpp create mode 100644 apex/csrc/type_shim.h create mode 100644 apex/csrc/welford.cu create mode 100644 apex/docs/Makefile create mode 100644 apex/docs/source/_static/css/pytorch_theme.css create mode 100644 apex/docs/source/_static/img/nv-pytorch2.png create mode 100644 apex/docs/source/_templates/layout.html create mode 100644 apex/docs/source/advanced.rst create mode 100644 apex/docs/source/amp.rst create mode 100644 apex/docs/source/conf.py create mode 100644 apex/docs/source/fp16_utils.rst create mode 100644 apex/docs/source/index.rst create mode 100644 apex/docs/source/layernorm.rst create mode 100644 apex/docs/source/optimizers.rst create mode 100644 apex/docs/source/parallel.rst create mode 100644 apex/examples/README.md create mode 100644 apex/examples/dcgan/README.md create mode 100644 apex/examples/dcgan/main_amp.py create mode 100644 apex/examples/docker/Dockerfile create mode 100644 apex/examples/docker/README.md create mode 100644 apex/examples/imagenet/README.md create mode 100644 apex/examples/imagenet/main_amp.py create mode 100644 apex/examples/simple/distributed/README.md create mode 100644 apex/examples/simple/distributed/distributed_data_parallel.py create mode 100644 apex/examples/simple/distributed/run.sh create mode 100644 apex/requirements.txt create mode 100644 apex/requirements_dev.txt create mode 100644 apex/setup.py create mode 100644 apex/tests/L0/run_amp/__init__.py create mode 100644 apex/tests/L0/run_amp/test_add_param_group.py create mode 100644 apex/tests/L0/run_amp/test_basic_casts.py create mode 100644 apex/tests/L0/run_amp/test_cache.py create mode 100644 apex/tests/L0/run_amp/test_checkpointing.py create mode 100644 apex/tests/L0/run_amp/test_fused_sgd.py create mode 100644 apex/tests/L0/run_amp/test_larc.py create mode 100644 apex/tests/L0/run_amp/test_multi_tensor_axpby.py create mode 100644 apex/tests/L0/run_amp/test_multi_tensor_l2norm.py create mode 100644 apex/tests/L0/run_amp/test_multi_tensor_scale.py create mode 100644 apex/tests/L0/run_amp/test_multiple_models_optimizers_losses.py create mode 100644 apex/tests/L0/run_amp/test_promotion.py create mode 100644 apex/tests/L0/run_amp/test_rnn.py create mode 100644 apex/tests/L0/run_amp/utils.py create mode 100644 apex/tests/L0/run_deprecated/test_deprecated_warning.py create mode 100644 apex/tests/L0/run_fp16util/__init__.py create mode 100644 apex/tests/L0/run_fp16util/test_fp16util.py create mode 100644 apex/tests/L0/run_fused_layer_norm/test_fused_layer_norm.py create mode 100644 apex/tests/L0/run_instance_norm_nvfuser/__init__.py create mode 100644 apex/tests/L0/run_instance_norm_nvfuser/test_instance_norm_nvfuser.py create mode 100644 apex/tests/L0/run_mlp/test_mlp.py create mode 100644 apex/tests/L0/run_optimizers/__init__.py create mode 100644 apex/tests/L0/run_optimizers/test_adam.py create mode 100755 apex/tests/L0/run_optimizers/test_fused_novograd.py create mode 100644 apex/tests/L0/run_optimizers/test_fused_optimizer.py create mode 100644 apex/tests/L0/run_optimizers/test_lamb.py create mode 100644 apex/tests/L0/run_test.py create mode 100644 apex/tests/L0/run_transformer/__init__.py create mode 100644 apex/tests/L0/run_transformer/gpt_scaling_test.py create mode 100644 apex/tests/L0/run_transformer/test_batch_sampler.py create mode 100644 apex/tests/L0/run_transformer/test_bert_minimal.py create mode 100644 apex/tests/L0/run_transformer/test_cross_entropy.py create mode 100644 apex/tests/L0/run_transformer/test_data.py create mode 100644 apex/tests/L0/run_transformer/test_dynamic_batchsize.py create mode 100644 apex/tests/L0/run_transformer/test_fused_softmax.py create mode 100644 apex/tests/L0/run_transformer/test_gpt_minimal.py create mode 100644 apex/tests/L0/run_transformer/test_layers.py create mode 100644 apex/tests/L0/run_transformer/test_mapping.py create mode 100644 apex/tests/L0/run_transformer/test_microbatches.py create mode 100644 apex/tests/L0/run_transformer/test_p2p_comm.py create mode 100644 apex/tests/L0/run_transformer/test_parallel_state.py create mode 100644 apex/tests/L0/run_transformer/test_pipeline_parallel_fwd_bwd.py create mode 100644 apex/tests/L0/run_transformer/test_random.py create mode 100644 apex/tests/L0/run_transformer/test_transformer_utils.py create mode 100644 apex/tests/L1/common/compare.py create mode 100644 apex/tests/L1/common/main_amp.py create mode 100644 apex/tests/L1/common/run_test.sh create mode 100644 apex/tests/L1/cross_product/run.sh create mode 100644 apex/tests/L1/cross_product_distributed/run.sh create mode 100644 apex/tests/L1/transformer/pipeline_parallel_fwd_bwd_ucc_async.py create mode 100644 apex/tests/distributed/DDP/ddp_race_condition_test.py create mode 100644 apex/tests/distributed/DDP/run_race_test.sh create mode 100644 apex/tests/distributed/amp_master_params/amp_master_params.py create mode 100644 apex/tests/distributed/amp_master_params/compare.py create mode 100644 apex/tests/distributed/amp_master_params/run.sh create mode 100644 apex/tests/distributed/synced_batchnorm/python_single_gpu_unit_test.py create mode 100644 apex/tests/distributed/synced_batchnorm/single_gpu_unit_test.py create mode 100644 apex/tests/distributed/synced_batchnorm/test_batchnorm1d.py create mode 100644 apex/tests/distributed/synced_batchnorm/test_groups.py create mode 100755 apex/tests/distributed/synced_batchnorm/two_gpu_test_different_batch_size.py create mode 100644 apex/tests/distributed/synced_batchnorm/two_gpu_unit_test.py create mode 100755 apex/tests/distributed/synced_batchnorm/unit_test.sh create mode 100644 apex/tests/docker_extension_builds/run.sh diff --git a/apex b/apex deleted file mode 160000 index 0da3ffb9..00000000 --- a/apex +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0da3ffb92ee6fbe5336602f0e3989db1cd16f880 diff --git a/apex/.github/ISSUE_TEMPLATE/bug_report.md b/apex/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..63d76737 --- /dev/null +++ b/apex/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Create a report to help us improve apex +title: '' +labels: bug +assignees: '' + +--- + +**Describe the Bug** + +**Minimal Steps/Code to Reproduce the Bug** + + +**Expected Behavior** + + +**Environment** + diff --git a/apex/.gitignore b/apex/.gitignore new file mode 100644 index 00000000..d30f85c3 --- /dev/null +++ b/apex/.gitignore @@ -0,0 +1,147 @@ +apex.egg-info +dist +build +docs/build +*~ +__pycache__ +.vscode + +# Copied from https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/apex/.gitmodules b/apex/.gitmodules new file mode 100644 index 00000000..6479428d --- /dev/null +++ b/apex/.gitmodules @@ -0,0 +1,7 @@ +[submodule "apex/contrib/csrc/multihead_attn/cutlass"] + path = apex/contrib/csrc/multihead_attn/cutlass + url = https://github.com/NVIDIA/cutlass.git + branch = v1.2.0 +[submodule "apex/contrib/csrc/cudnn-frontend"] + path = apex/contrib/csrc/cudnn-frontend + url = https://github.com/NVIDIA/cudnn-frontend.git diff --git a/apex/.nojekyll b/apex/.nojekyll new file mode 100644 index 00000000..e69de29b diff --git a/apex/LICENSE b/apex/LICENSE new file mode 100644 index 00000000..3d1e9454 --- /dev/null +++ b/apex/LICENSE @@ -0,0 +1,11 @@ +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/apex/README.md b/apex/README.md new file mode 100644 index 00000000..32fdd927 --- /dev/null +++ b/apex/README.md @@ -0,0 +1,182 @@ +# Introduction + +This repository holds NVIDIA-maintained utilities to streamline mixed precision and distributed training in Pytorch. +Some of the code here will be included in upstream Pytorch eventually. +The intent of Apex is to make up-to-date utilities available to users as quickly as possible. + +## Full API Documentation: [https://nvidia.github.io/apex](https://nvidia.github.io/apex) + +## [GTC 2019](https://github.com/mcarilli/mixed_precision_references/tree/master/GTC_2019) and [Pytorch DevCon 2019](https://github.com/mcarilli/mixed_precision_references/tree/master/Pytorch_Devcon_2019) Slides + +# Contents + +## 1. Amp: Automatic Mixed Precision + +**Deprecated. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)** + +`apex.amp` is a tool to enable mixed precision training by changing only 3 lines of your script. +Users can easily experiment with different pure and mixed precision training modes by supplying +different flags to `amp.initialize`. + +[Webinar introducing Amp](https://info.nvidia.com/webinar-mixed-precision-with-pytorch-reg-page.html) +(The flag `cast_batchnorm` has been renamed to `keep_batchnorm_fp32`). + +[API Documentation](https://nvidia.github.io/apex/amp.html) + +[Comprehensive Imagenet example](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) + +[DCGAN example coming soon...](https://github.com/NVIDIA/apex/tree/master/examples/dcgan) + +[Moving to the new Amp API](https://nvidia.github.io/apex/amp.html#transition-guide-for-old-api-users) (for users of the deprecated "Amp" and "FP16_Optimizer" APIs) + +## 2. Distributed Training + +**`apex.parallel.DistributedDataParallel` is deprecated. Use [`torch.nn.parallel.DistributedDataParallel`](https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html?highlight=distributeddataparallel#torch.nn.parallel.DistributedDataParallel)** + +`apex.parallel.DistributedDataParallel` is a module wrapper, similar to +`torch.nn.parallel.DistributedDataParallel`. It enables convenient multiprocess distributed training, +optimized for NVIDIA's NCCL communication library. + +[API Documentation](https://nvidia.github.io/apex/parallel.html) + +[Python Source](https://github.com/NVIDIA/apex/tree/master/apex/parallel) + +[Example/Walkthrough](https://github.com/NVIDIA/apex/tree/master/examples/simple/distributed) + +The [Imagenet example](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) +shows use of `apex.parallel.DistributedDataParallel` along with `apex.amp`. + +### Synchronized Batch Normalization + +**Deprecated. Use [`torch.nn.SyncBatchNorm`](https://pytorch.org/docs/stable/generated/torch.nn.SyncBatchNorm.html)** + +`apex.parallel.SyncBatchNorm` extends `torch.nn.modules.batchnorm._BatchNorm` to +support synchronized BN. +It allreduces stats across processes during multiprocess (DistributedDataParallel) training. +Synchronous BN has been used in cases where only a small +local minibatch can fit on each GPU. +Allreduced stats increase the effective batch size for the BN layer to the +global batch size across all processes (which, technically, is the correct +formulation). +Synchronous BN has been observed to improve converged accuracy in some of our research models. + +### Checkpointing + +To properly save and load your `amp` training, we introduce the `amp.state_dict()`, which contains all `loss_scalers` and their corresponding unskipped steps, +as well as `amp.load_state_dict()` to restore these attributes. + +In order to get bitwise accuracy, we recommend the following workflow: +```python +# Initialization +opt_level = 'O1' +model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level) + +# Train your model +... +with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() +... + +# Save checkpoint +checkpoint = { + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict() +} +torch.save(checkpoint, 'amp_checkpoint.pt') +... + +# Restore +model = ... +optimizer = ... +checkpoint = torch.load('amp_checkpoint.pt') + +model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level) +model.load_state_dict(checkpoint['model']) +optimizer.load_state_dict(checkpoint['optimizer']) +amp.load_state_dict(checkpoint['amp']) + +# Continue training +... +``` + +Note that we recommend restoring the model using the same `opt_level`. Also note that we recommend calling the `load_state_dict` methods after `amp.initialize`. + +# Installation +Each [`apex.contrib`](./apex/contrib) module requires one or more install options other than `--cpp_ext` and `--cuda_ext`. +Note that contrib modules do not necessarily support stable PyTorch releases. + +## Containers +NVIDIA PyTorch Containers are available on NGC: https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch. +The containers come with all the custom extensions available at the moment. + +See [the NGC documentation](https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/index.html) for details such as: +- how to pull a container +- how to run a pulled container +- release notes + +## From Source + +To install Apex from source, we recommend using the nightly Pytorch obtainable from https://github.com/pytorch/pytorch. + +The latest stable release obtainable from https://pytorch.org should also work. + +### Linux +For performance and full functionality, we recommend installing Apex with +CUDA and C++ extensions via +```bash +git clone https://github.com/NVIDIA/apex +cd apex +pip install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./ +``` + +APEX also supports a Python-only build via +```bash +pip install -v --disable-pip-version-check --no-cache-dir ./ +``` +A Python-only build omits: +- Fused kernels required to use `apex.optimizers.FusedAdam`. +- Fused kernels required to use `apex.normalization.FusedLayerNorm` and `apex.normalization.FusedRMSNorm`. +- Fused kernels that improve the performance and numerical stability of `apex.parallel.SyncBatchNorm`. +- Fused kernels that improve the performance of `apex.parallel.DistributedDataParallel` and `apex.amp`. +`DistributedDataParallel`, `amp`, and `SyncBatchNorm` will still be usable, but they may be slower. + + +### [Experimental] Windows +`pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" .` may work if you were able to build Pytorch from source +on your system. A Python-only build via `pip install -v --no-cache-dir .` is more likely to work. +If you installed Pytorch in a Conda environment, make sure to install Apex in that same environment. + + +## Custom C++/CUDA Extensions and Install Options + +If a requirement of a module is not met, then it will not be built. + +| Module Name | Install Option | Misc | +|---------------|------------------|--------| +| `apex_C` | `--cpp_ext` | | +| `amp_C` | `--cuda_ext` | | +| `syncbn` | `--cuda_ext` | | +| `fused_layer_norm_cuda` | `--cuda_ext` | [`apex.normalization`](./apex/normalization) | +| `mlp_cuda` | `--cuda_ext` | | +| `scaled_upper_triang_masked_softmax_cuda` | `--cuda_ext` | | +| `generic_scaled_masked_softmax_cuda` | `--cuda_ext` | | +| `scaled_masked_softmax_cuda` | `--cuda_ext` | | +| `fused_weight_gradient_mlp_cuda` | `--cuda_ext` | Requires CUDA>=11 | +| `permutation_search_cuda` | `--permutation_search` | [`apex.contrib.sparsity`](./apex/contrib/sparsity) | +| `bnp` | `--bnp` | [`apex.contrib.groupbn`](./apex/contrib/groupbn) | +| `xentropy` | `--xentropy` | [`apex.contrib.xentropy`](./apex/contrib/xentropy) | +| `focal_loss_cuda` | `--focal_loss` | [`apex.contrib.focal_loss`](./apex/contrib/focal_loss) | +| `fused_index_mul_2d` | `--index_mul_2d` | [`apex.contrib.index_mul_2d`](./apex/contrib/index_mul_2d) | +| `fused_adam_cuda` | `--deprecated_fused_adam` | [`apex.contrib.optimizers`](./apex/contrib/optimizers) | +| `fused_lamb_cuda` | `--deprecated_fused_lamb` | [`apex.contrib.optimizers`](./apex/contrib/optimizers) | +| `fast_layer_norm` | `--fast_layer_norm` | [`apex.contrib.layer_norm`](./apex/contrib/layer_norm). different from `fused_layer_norm` | +| `fmhalib` | `--fmha` | [`apex.contrib.fmha`](./apex/contrib/fmha) | +| `fast_multihead_attn` | `--fast_multihead_attn` | [`apex.contrib.multihead_attn`](./apex/contrib/multihead_attn) | +| `transducer_joint_cuda` | `--transducer` | [`apex.contrib.transducer`](./apex/contrib/transducer) | +| `transducer_loss_cuda` | `--transducer` | [`apex.contrib.transducer`](./apex/contrib/transducer) | +| `cudnn_gbn_lib` | `--cudnn_gbn` | Requires cuDNN>=8.5, [`apex.contrib.cudnn_gbn`](./apex/contrib/cudnn_gbn) | +| `peer_memory_cuda` | `--peer_memory` | [`apex.contrib.peer_memory`](./apex/contrib/peer_memory) | +| `nccl_p2p_cuda` | `--nccl_p2p` | Requires NCCL >= 2.10, [`apex.contrib.nccl_p2p`](./apex/contrib/nccl_p2p) | +| `fast_bottleneck` | `--fast_bottleneck` | Requires `peer_memory_cuda` and `nccl_p2p_cuda`, [`apex.contrib.bottleneck`](./apex/contrib/bottleneck) | +| `fused_conv_bias_relu` | `--fused_conv_bias_relu` | Requires cuDNN>=8.4, [`apex.contrib.conv_bias_relu`](./apex/contrib/conv_bias_relu) | diff --git a/apex/apex/RNN/README.md b/apex/apex/RNN/README.md new file mode 100644 index 00000000..82c4eb68 --- /dev/null +++ b/apex/apex/RNN/README.md @@ -0,0 +1,3 @@ +**This module will be removed by the end of February 2023** + +Under construction... diff --git a/apex/apex/RNN/RNNBackend.py b/apex/apex/RNN/RNNBackend.py new file mode 100644 index 00000000..a9382e60 --- /dev/null +++ b/apex/apex/RNN/RNNBackend.py @@ -0,0 +1,365 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable + +import torch.nn.functional as F + +import math + + +def is_iterable(maybe_iterable): + return isinstance(maybe_iterable, list) or isinstance(maybe_iterable, tuple) + + +def flatten_list(tens_list): + """ + flatten_list + """ + if not is_iterable(tens_list): + return tens_list + + return torch.cat(tens_list, dim=0).view(len(tens_list), *tens_list[0].size() ) + + +#These modules always assumes batch_first +class bidirectionalRNN(nn.Module): + """ + bidirectionalRNN + """ + def __init__(self, inputRNN, num_layers=1, dropout = 0): + super(bidirectionalRNN, self).__init__() + self.dropout = dropout + self.fwd = stackedRNN(inputRNN, num_layers=num_layers, dropout = dropout) + self.bckwrd = stackedRNN(inputRNN.new_like(), num_layers=num_layers, dropout = dropout) + self.rnns = nn.ModuleList([self.fwd, self.bckwrd]) + + #collect hidden option will return all hidden/cell states from entire RNN + def forward(self, input, collect_hidden=False): + """ + forward() + """ + seq_len = input.size(0) + bsz = input.size(1) + + fwd_out, fwd_hiddens = list(self.fwd(input, collect_hidden = collect_hidden)) + bckwrd_out, bckwrd_hiddens = list(self.bckwrd(input, reverse=True, collect_hidden = collect_hidden)) + + output = torch.cat( [fwd_out, bckwrd_out], -1 ) + hiddens = tuple( torch.cat(hidden, -1) for hidden in zip( fwd_hiddens, bckwrd_hiddens) ) + + return output, hiddens + + def reset_parameters(self): + """ + reset_parameters() + """ + for rnn in self.rnns: + rnn.reset_parameters() + + def init_hidden(self, bsz): + """ + init_hidden() + """ + for rnn in self.rnns: + rnn.init_hidden(bsz) + + def detach_hidden(self): + """ + detach_hidden() + """ + for rnn in self.rnns: + rnn.detachHidden() + + def reset_hidden(self, bsz): + """ + reset_hidden() + """ + for rnn in self.rnns: + rnn.reset_hidden(bsz) + + def init_inference(self, bsz): + """ + init_inference() + """ + for rnn in self.rnns: + rnn.init_inference(bsz) + + +#assumes hidden_state[0] of inputRNN is output hidden state +#constructor either takes an RNNCell or list of RNN layers +class stackedRNN(nn.Module): + """ + stackedRNN + """ + def __init__(self, inputRNN, num_layers=1, dropout=0): + super(stackedRNN, self).__init__() + + self.dropout = dropout + + if isinstance(inputRNN, RNNCell): + self.rnns = [inputRNN] + for i in range(num_layers-1): + self.rnns.append(inputRNN.new_like(inputRNN.output_size)) + elif isinstance(inputRNN, list): + assert len(inputRNN) == num_layers, "RNN list length must be equal to num_layers" + self.rnns=inputRNN + else: + raise RuntimeError() + + self.nLayers = len(self.rnns) + + self.rnns = nn.ModuleList(self.rnns) + + + ''' + Returns output as hidden_state[0] Tensor([sequence steps][batch size][features]) + If collect hidden will also return Tuple( + [n_hidden_states][sequence steps] Tensor([layer][batch size][features]) + ) + If not collect hidden will also return Tuple( + [n_hidden_states] Tensor([layer][batch size][features]) + ''' + def forward(self, input, collect_hidden=False, reverse=False): + """ + forward() + """ + seq_len = input.size(0) + bsz = input.size(1) + inp_iter = reversed(range(seq_len)) if reverse else range(seq_len) + + hidden_states = [[] for i in range(self.nLayers)] + outputs = [] + + for seq in inp_iter: + for layer in range(self.nLayers): + + if layer == 0: + prev_out = input[seq] + + outs = self.rnns[layer](prev_out) + + if collect_hidden: + hidden_states[layer].append(outs) + elif seq == seq_len-1: + hidden_states[layer].append(outs) + + prev_out = outs[0] + + outputs.append(prev_out) + + if reverse: + outputs = list(reversed(outputs)) + ''' + At this point outputs is in format: + list( [seq_length] x Tensor([bsz][features]) ) + need to convert it to: + list( Tensor([seq_length][bsz][features]) ) + ''' + output = flatten_list(outputs) + + ''' + hidden_states at this point is in format: + list( [layer][seq_length][hidden_states] x Tensor([bsz][features]) ) + need to convert it to: + For not collect hidden: + list( [hidden_states] x Tensor([layer][bsz][features]) ) + For collect hidden: + list( [hidden_states][seq_length] x Tensor([layer][bsz][features]) ) + ''' + if not collect_hidden: + seq_len = 1 + n_hid = self.rnns[0].n_hidden_states + new_hidden = [ [ [ None for k in range(self.nLayers)] for j in range(seq_len) ] for i in range(n_hid) ] + + + for i in range(n_hid): + for j in range(seq_len): + for k in range(self.nLayers): + new_hidden[i][j][k] = hidden_states[k][j][i] + + hidden_states = new_hidden + #Now in format list( [hidden_states][seq_length][layer] x Tensor([bsz][features]) ) + #Reverse seq_length if reverse + if reverse: + hidden_states = list( list(reversed(list(entry))) for entry in hidden_states) + + #flatten layer dimension into tensor + hiddens = list( list( + flatten_list(seq) for seq in hidden ) + for hidden in hidden_states ) + + #Now in format list( [hidden_states][seq_length] x Tensor([layer][bsz][features]) ) + #Remove seq_length dimension if not collect_hidden + if not collect_hidden: + hidden_states = list( entry[0] for entry in hidden_states) + return output, hidden_states + + def reset_parameters(self): + """ + reset_parameters() + """ + for rnn in self.rnns: + rnn.reset_parameters() + + def init_hidden(self, bsz): + """ + init_hidden() + """ + for rnn in self.rnns: + rnn.init_hidden(bsz) + + def detach_hidden(self): + """ + detach_hidden() + """ + for rnn in self.rnns: + rnn.detach_hidden() + + def reset_hidden(self, bsz): + """ + reset_hidden() + """ + for rnn in self.rnns: + rnn.reset_hidden(bsz) + + def init_inference(self, bsz): + """ + init_inference() + """ + for rnn in self.rnns: + rnn.init_inference(bsz) + +class RNNCell(nn.Module): + """ + RNNCell + gate_multiplier is related to the architecture you're working with + For LSTM-like it will be 4 and GRU-like will be 3. + Always assumes input is NOT batch_first. + Output size that's not hidden size will use output projection + Hidden_states is number of hidden states that are needed for cell + if one will go directly to cell as tensor, if more will go as list + """ + def __init__(self, gate_multiplier, input_size, hidden_size, cell, n_hidden_states = 2, bias = False, output_size = None): + super(RNNCell, self).__init__() + + self.gate_multiplier = gate_multiplier + self.input_size = input_size + self.hidden_size = hidden_size + self.cell = cell + self.bias = bias + self.output_size = output_size + if output_size is None: + self.output_size = hidden_size + + self.gate_size = gate_multiplier * self.hidden_size + self.n_hidden_states = n_hidden_states + + self.w_ih = nn.Parameter(torch.empty(self.gate_size, self.input_size)) + self.w_hh = nn.Parameter(torch.empty(self.gate_size, self.output_size)) + + #Check if there's recurrent projection + if(self.output_size != self.hidden_size): + self.w_ho = nn.Parameter(torch.empty(self.output_size, self.hidden_size)) + + self.b_ih = self.b_hh = None + if self.bias: + self.b_ih = nn.Parameter(torch.empty(self.gate_size)) + self.b_hh = nn.Parameter(torch.empty(self.gate_size)) + + #hidden states for forward + self.hidden = [ None for states in range(self.n_hidden_states)] + + self.reset_parameters() + + def new_like(self, new_input_size=None): + """ + new_like() + """ + if new_input_size is None: + new_input_size = self.input_size + + return type(self)(self.gate_multiplier, + new_input_size, + self.hidden_size, + self.cell, + self.n_hidden_states, + self.bias, + self.output_size) + + + #Use xavier where we can (weights), otherwise use uniform (bias) + def reset_parameters(self, gain=1): + """ + reset_parameters() + """ + stdev = 1.0 / math.sqrt(self.hidden_size) + for param in self.parameters(): + param.data.uniform_(-stdev, stdev) + ''' + Xavier reset: + def reset_parameters(self, gain=1): + stdv = 1.0 / math.sqrt(self.gate_size) + + for param in self.parameters(): + if (param.dim() > 1): + torch.nn.init.xavier_normal(param, gain) + else: + param.data.uniform_(-stdv, stdv) + ''' + def init_hidden(self, bsz): + """ + init_hidden() + """ + for param in self.parameters(): + if param is not None: + a_param = param + break + + for i, _ in enumerate(self.hidden): + if(self.hidden[i] is None or self.hidden[i].data.size()[0] != bsz): + + if i==0: + hidden_size = self.output_size + else: + hidden_size = self.hidden_size + + tens = a_param.data.new(bsz, hidden_size).zero_() + self.hidden[i] = Variable(tens, requires_grad=False) + + + def reset_hidden(self, bsz): + """ + reset_hidden() + """ + for i, _ in enumerate(self.hidden): + self.hidden[i] = None + self.init_hidden(bsz) + + def detach_hidden(self): + """ + detach_hidden() + """ + for i, _ in enumerate(self.hidden): + if self.hidden[i] is None: + raise RuntimeError("Must initialize hidden state before you can detach it") + for i, _ in enumerate(self.hidden): + self.hidden[i] = self.hidden[i].detach() + + def forward(self, input): + """ + forward() + if not inited or bsz has changed this will create hidden states + """ + self.init_hidden(input.size()[0]) + + hidden_state = self.hidden[0] if self.n_hidden_states == 1 else self.hidden + self.hidden = self.cell(input, hidden_state, self.w_ih, self.w_hh, b_ih=self.b_ih, b_hh=self.b_hh) + if(self.n_hidden_states > 1): + self.hidden = list(self.hidden) + else: + self.hidden=[self.hidden] + + if self.output_size != self.hidden_size: + self.hidden[0] = F.linear(self.hidden[0], self.w_ho) + + return tuple(self.hidden) diff --git a/apex/apex/RNN/__init__.py b/apex/apex/RNN/__init__.py new file mode 100644 index 00000000..d7067466 --- /dev/null +++ b/apex/apex/RNN/__init__.py @@ -0,0 +1,3 @@ +from .models import LSTM, GRU, ReLU, Tanh, mLSTM + +__all__ = ['models'] diff --git a/apex/apex/RNN/cells.py b/apex/apex/RNN/cells.py new file mode 100644 index 00000000..09b08581 --- /dev/null +++ b/apex/apex/RNN/cells.py @@ -0,0 +1,84 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .RNNBackend import RNNCell + +from torch.nn._functions.thnn import rnnFusedPointwise as fusedBackend + +import math + + +class mLSTMRNNCell(RNNCell): + """ + mLSTMRNNCell + """ + + def __init__(self, input_size, hidden_size, bias = False, output_size = None): + gate_multiplier = 4 + super(mLSTMRNNCell, self).__init__(gate_multiplier, input_size, hidden_size, mLSTMCell, n_hidden_states = 2, bias = bias, output_size = output_size) + + self.w_mih = nn.Parameter(torch.empty(self.output_size, self.input_size)) + self.w_mhh = nn.Parameter(torch.empty(self.output_size, self.output_size)) + + self.reset_parameters() + + def forward(self, input): + """ + mLSTMRNNCell.forward() + """ + #if not inited or bsz has changed this will create hidden states + self.init_hidden(input.size()[0]) + + hidden_state = self.hidden[0] if self.n_hidden_states == 1 else self.hidden + + self.hidden = list( + self.cell(input, hidden_state, self.w_ih, self.w_hh, self.w_mih, self.w_mhh, + b_ih=self.b_ih, b_hh=self.b_hh) + ) + + if self.output_size != self.hidden_size: + self.hidden[0] = F.linear(self.hidden[0], self.w_ho) + return tuple(self.hidden) + + + def new_like(self, new_input_size=None): + if new_input_size is None: + new_input_size = self.input_size + + return type(self)( + new_input_size, + self.hidden_size, + self.bias, + self.output_size) + +def mLSTMCell(input, hidden, w_ih, w_hh, w_mih, w_mhh, b_ih=None, b_hh=None): + """ + mLSTMCell + """ + + if input.is_cuda: + igates = F.linear(input, w_ih) + m = F.linear(input, w_mih) * F.linear(hidden[0], w_mhh) + hgates = F.linear(m, w_hh) + + state = fusedBackend.LSTMFused.apply + return state(igates, hgates, hidden[1], b_ih, b_hh) + + hx, cx = hidden + + m = F.linear(input, w_mih) * F.linear(hidden[0], w_mhh) + gates = F.linear(input, w_ih, b_ih) + F.linear(m, w_hh, b_hh) + + ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1) + + ingate = F.sigmoid(ingate) + forgetgate = F.sigmoid(forgetgate) + cellgate = F.tanh(cellgate) + outgate = F.sigmoid(outgate) + + cy = (forgetgate * cx) + (ingate * cellgate) + hy = outgate * F.tanh(cy) + + return hy, cy + diff --git a/apex/apex/RNN/models.py b/apex/apex/RNN/models.py new file mode 100644 index 00000000..d661aa0d --- /dev/null +++ b/apex/apex/RNN/models.py @@ -0,0 +1,56 @@ +import torch + +from torch.nn._functions.rnn import LSTMCell, RNNReLUCell, RNNTanhCell, GRUCell + +from apex import deprecated_warning +from .RNNBackend import bidirectionalRNN, stackedRNN, RNNCell +from .cells import mLSTMRNNCell, mLSTMCell + +def toRNNBackend(inputRNN, num_layers, bidirectional=False, dropout = 0): + """ + :class:`toRNNBackend` + """ + + deprecated_warning("`apex.RNN` is deprecated and will be removed by the end of February 2023.") + if bidirectional: + return bidirectionalRNN(inputRNN, num_layers, dropout = dropout) + else: + return stackedRNN(inputRNN, num_layers, dropout = dropout) + + +def LSTM(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None): + """ + :class:`LSTM` + """ + inputRNN = RNNCell(4, input_size, hidden_size, LSTMCell, 2, bias, output_size) + return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout) + +def GRU(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None): + """ + :class:`GRU` + """ + inputRNN = RNNCell(3, input_size, hidden_size, GRUCell, 1, bias, output_size) + return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout) + +def ReLU(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None): + """ + :class:`ReLU` + """ + inputRNN = RNNCell(1, input_size, hidden_size, RNNReLUCell, 1, bias, output_size) + return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout) + +def Tanh(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None): + """ + :class:`Tanh` + """ + inputRNN = RNNCell(1, input_size, hidden_size, RNNTanhCell, 1, bias, output_size) + return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout) + +def mLSTM(input_size, hidden_size, num_layers, bias=True, batch_first=False, dropout=0, bidirectional=False, output_size = None): + """ + :class:`mLSTM` + """ + inputRNN = mLSTMRNNCell(input_size, hidden_size, bias=bias, output_size=output_size) + return toRNNBackend(inputRNN, num_layers, bidirectional, dropout=dropout) + + diff --git a/apex/apex/__init__.py b/apex/apex/__init__.py new file mode 100644 index 00000000..74851f5b --- /dev/null +++ b/apex/apex/__init__.py @@ -0,0 +1,68 @@ +import logging +import warnings + +# May help avoid undefined symbol errors https://pytorch.org/cppdocs/notes/faq.html#undefined-symbol-errors-from-pytorch-aten +import torch + + +__all__ = ["amp", "fp16_utils", "optimizers", "normalization", "transformer"] + + +if torch.distributed.is_available(): + from . import parallel + __all__.append("parallel") + +from . import amp +from . import fp16_utils + +# For optimizers and normalization there is no Python fallback. +# Absence of cuda backend is a hard error. +# I would like the errors from importing fused_adam_cuda or fused_layer_norm_cuda +# to be triggered lazily, because if someone has installed with --cpp_ext and --cuda_ext +# so they expect those backends to be available, but for some reason they actually aren't +# available (for example because they built improperly in a way that isn't revealed until +# load time) the error message is timely and visible. +from . import optimizers +from . import normalization +from . import transformer + + +# Logging utilities for apex.transformer module +class RankInfoFormatter(logging.Formatter): + + def format(self, record): + from apex.transformer.parallel_state import get_rank_info + record.rank_info = get_rank_info() + return super().format(record) + + +_library_root_logger = logging.getLogger(__name__) +handler = logging.StreamHandler() +handler.setFormatter(RankInfoFormatter("%(asctime)s - PID:%(process)d - rank:%(rank_info)s - %(filename)s:%(lineno)d - %(levelname)s - %(message)s", "%y-%m-%d %H:%M:%S")) +_library_root_logger.addHandler(handler) +_library_root_logger.propagate = False + + +def check_cudnn_version_and_warn(global_option: str, required_cudnn_version: int) -> bool: + cudnn_available = torch.backends.cudnn.is_available() + cudnn_version = torch.backends.cudnn.version() if cudnn_available else None + if not (cudnn_available and (cudnn_version >= required_cudnn_version)): + warnings.warn( + f"`{global_option}` depends on cuDNN {required_cudnn_version} or later, " + f"but {'cuDNN is not available' if not cudnn_available else cudnn_version}" + ) + return False + return True + + +class DeprecatedFeatureWarning(FutureWarning): + pass + + +def deprecated_warning(msg: str) -> None: + if ( + not torch.distributed.is_available + or not torch.distributed.is_initialized() + or (torch.distributed.is_initialized() and torch.distributed.get_rank() == 0) + ): + warnings.warn(msg, DeprecatedFeatureWarning) diff --git a/apex/apex/_autocast_utils.py b/apex/apex/_autocast_utils.py new file mode 100644 index 00000000..3a92a83f --- /dev/null +++ b/apex/apex/_autocast_utils.py @@ -0,0 +1,26 @@ +from typing import Optional, Sequence + +import torch + + +__all__ = ["_cast_if_autocast_enabled"] + + +def _get_autocast_dtypes() -> Sequence[torch.dtype]: + if torch.cuda.is_bf16_supported(): + return [torch.half, torch.bfloat16] + return [torch.half] + + +def _get_current_dtype(dtype: Optional[torch.dtype] = None) -> torch.dtype: + if not torch.is_autocast_enabled(): + return torch.float or dtype + else: + return torch.get_autocast_gpu_dtype() + + +def _cast_if_autocast_enabled(*args): + if not torch.is_autocast_enabled(): + return args + else: + return torch.cuda.amp.autocast_mode._cast(args, torch.get_autocast_gpu_dtype()) diff --git a/apex/apex/amp/README.md b/apex/apex/amp/README.md new file mode 100644 index 00000000..a87b5010 --- /dev/null +++ b/apex/apex/amp/README.md @@ -0,0 +1,72 @@ +# amp: Automatic Mixed Precision + +## Annotating User Functions + +Nearly all PyTorch user code needs nothing more than the two steps +above to use amp. After all, custom layers are built out of simpler +PyTorch components, and amp already can see those. + +However, any custom C++ or CUDA code is outside of amp's (default) +view of things. For example, suppose I implemented a new recurrent +cell called a "forgetful recurrent unit" that calls directly into a +CUDA backend: + +```python +from backend import FRUBackend + +def fru(input, hidden, weight, bias): + # call to CUDA code + FRUBackend(input, hidden, weight, bias) +``` + +In this case, it is possible to get a runtime type mismatch. For +example, you might have `input` in fp16, and `weight` in fp32, and amp +doesn't have the visibility to insert an appropriate cast. + +amp exposes two ways to handle "invisible" backend code: function +annotations and explicit registration. + +#### Function annotation + +The first way to handle backend code is a set of function annotations: + +- `@amp.half_function` +- `@amp.float_function` +- `@amp.promote_function` + +These correspond to: + +- Cast all arguments to fp16 +- Cast all argumnets fo fp32 +- If there are any type mismatches, cast everything to the widest type + +In our example, we believe that the FRU unit is fp16-safe and will get +performance gains from casting its arguments to fp16, so we write: + +```python +@amp.half_function +def fru(input, hidden, weight, bias): + #... +``` + +#### Explicit registration + +The other way to handle backend code is with explicit function +registration: + +- `amp.register_half_function(module, function_name)` +- `amp.register_float_function(module, function_name)` +- `amp.register_promote_function(module, function_name)` + +When using this API, `module` is the containing class or module for +the function, and `function_name` is the _string_ name of the +function. Note that the function must be registered before the call to +`amp.initalize()`. + +For our FRU unit, we can register the backend function directly: + +```python +import backend + +amp.register_half_function(backend, 'FRUBackend') +``` diff --git a/apex/apex/amp/__init__.py b/apex/apex/amp/__init__.py new file mode 100644 index 00000000..34d080a6 --- /dev/null +++ b/apex/apex/amp/__init__.py @@ -0,0 +1,5 @@ +from .amp import init, half_function, float_function, promote_function,\ + register_half_function, register_float_function, register_promote_function +from .handle import scale_loss, disable_casts +from .frontend import initialize, state_dict, load_state_dict +from ._amp_state import master_params, _amp_state diff --git a/apex/apex/amp/__version__.py b/apex/apex/amp/__version__.py new file mode 100644 index 00000000..3a83701b --- /dev/null +++ b/apex/apex/amp/__version__.py @@ -0,0 +1,2 @@ +VERSION = (0, 1, 0) +__version__ = '.'.join(map(str, VERSION)) diff --git a/apex/apex/amp/_amp_state.py b/apex/apex/amp/_amp_state.py new file mode 100644 index 00000000..7e8a329f --- /dev/null +++ b/apex/apex/amp/_amp_state.py @@ -0,0 +1,59 @@ +# This is a "header object" that allows different amp modules to communicate. +# I'm a C++ guy, not a python guy. I decided this approach because it seemed most C++-like. +# But apparently it's ok: +# http://effbot.org/pyfaq/how-do-i-share-global-variables-across-modules.htm +import torch + + +class AmpState(object): + def __init__(self): + self.hard_override=False + self.allow_incoming_model_not_fp32 = False + self.verbosity=1 + + +# Attribute stash. Could also just stash things as global module attributes. +_amp_state = AmpState() + + +def warn_or_err(msg): + if _amp_state.hard_override: + print("Warning: " + msg) + else: + raise RuntimeError(msg) + # I'm not sure if allowing hard_override is a good idea. + # + " If you're sure you know what you're doing, supply " + + # "hard_override=True to amp.initialize.") + + +def maybe_print(msg, rank0=False): + distributed = torch.distributed.is_available() and \ + torch.distributed.is_initialized() and \ + torch.distributed.get_world_size() > 1 + if _amp_state.verbosity > 0: + if rank0: + if distributed: + if torch.distributed.get_rank() == 0: + print(msg) + else: + print(msg) + else: + print(msg) + + +# def iter_params(param_groups): +# for group in param_groups: +# for p in group['params']: +# yield p + + +def master_params(optimizer): + """ + Generator expression that iterates over the params owned by ``optimizer``. + + Args: + optimizer: An optimizer previously returned from ``amp.initialize``. + """ + for group in optimizer.param_groups: + for p in group['params']: + yield p diff --git a/apex/apex/amp/_initialize.py b/apex/apex/amp/_initialize.py new file mode 100644 index 00000000..3ae6fded --- /dev/null +++ b/apex/apex/amp/_initialize.py @@ -0,0 +1,265 @@ +import collections.abc as container_abcs +from types import MethodType +import functools +import sys +import warnings + +import numpy as np +import torch + +from ._amp_state import _amp_state, warn_or_err +from .handle import disable_casts +from .scaler import LossScaler +from ._process_optimizer import _process_optimizer +from apex.fp16_utils import convert_network +from ..fp16_utils import FP16_Optimizer as FP16_Optimizer_general +from ..contrib.optimizers import FP16_Optimizer as FP16_Optimizer_for_fused + +if torch.distributed.is_available(): + from ..parallel import DistributedDataParallel as apex_DDP + from ..parallel.LARC import LARC + + +def to_type(dtype, t): + if isinstance(t, torch.Tensor): + if not t.is_cuda: + # This should not be a hard error, since it may be legitimate. + warnings.warn("An input tensor was not cuda.") + # GANs require this. + # if t.requires_grad: + # warn_or_err("input data requires grad. Since input data is not a model parameter,\n" + # "its gradients will not be properly allreduced by DDP.") + if t.is_floating_point(): + return t.to(dtype) + return t + else: + # Trust the user's custom batch type, that's all I can do here. + return t.to(dtype) + + +# Modified from torch.optim.optimizer.py. This is a bit more general than casted_args in utils.py. +def applier(value, fn): + if isinstance(value, torch.Tensor): + return fn(value) + elif isinstance(value, str): + return value + elif isinstance(value, np.ndarray): + return value + elif hasattr(value, "to"): # Allow handling of custom batch classes + return fn(value) + elif isinstance(value, container_abcs.Mapping): + return {applier(k, fn) : applier(v, fn) for k, v in value.items()} + elif isinstance(value, container_abcs.Iterable): + return type(value)(applier(v, fn) for v in value) + else: + # Do I want this to fire off even if someone chooses to pass something ordinary like + # an int or float? May be more annoying than it's worth. + # print("Warning: unrecognized type in applier. If your input data is a custom class, " + # "provide it with a .to(dtype) method which converts its floating-point Tensors to dtype. " + # "Amp will check for your custom to() and invoke it to cast the batch's " + # "floating-point Tensors to the appropriate type. " + # "Also, if your data is a custom class, it is your responsibility to ensure that " + # "any Tensors you want to be cuda are already cuda." + return value + + +def check_models(models): + for model in models: + parallel_type = None + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + parallel_type = "torch.nn.parallel.DistributedDataParallel" + if ('apex_DDP' in sys.modules) and isinstance(model, apex_DDP): + parallel_type = "apex.parallel.DistributedDataParallel" + if isinstance(model, torch.nn.parallel.DataParallel): + parallel_type = "torch.nn.parallel.DataParallel" + if parallel_type is not None: + raise RuntimeError("Incoming model is an instance of {}. ".format(parallel_type) + + "Parallel wrappers should only be applied to the model(s) AFTER \n" + "the model(s) have been returned from amp.initialize.") + + +def check_params_fp32(models): + for model in models: + for name, param in model.named_parameters(): + if param.is_floating_point(): + if 'Half' in param.type(): + warn_or_err("Found param {} with type {}, expected torch.cuda.FloatTensor.\n" + "When using amp.initialize, you do not need to call .half() on your model\n" + "before passing it, no matter what optimization level you choose.".format( + name, param.type())) + elif not param.is_cuda: + warn_or_err("Found param {} with type {}, expected torch.cuda.FloatTensor.\n" + "When using amp.initialize, you need to provide a model with parameters\n" + "located on a CUDA device before passing it no matter what optimization level\n" + "you chose. Use model.to('cuda') to use the default device.".format( + name, param.type())) + + # Backward compatibility for PyTorch 0.4 + if hasattr(model, 'named_buffers'): + buf_iter = model.named_buffers() + else: + buf_iter = model._buffers + for obj in buf_iter: + if type(obj)==tuple: + name, buf = obj + else: + name, buf = obj, buf_iter[obj] + if buf.is_floating_point(): + if 'Half' in buf.type(): + warn_or_err("Found buffer {} with type {}, expected torch.cuda.FloatTensor.\n" + "When using amp.initialize, you do not need to call .half() on your model\n" + "before passing it, no matter what optimization level you choose.".format( + name, buf.type())) + elif not buf.is_cuda: + warn_or_err("Found buffer {} with type {}, expected torch.cuda.FloatTensor.\n" + "When using amp.initialize, you need to provide a model with buffers\n" + "located on a CUDA device before passing it no matter what optimization level\n" + "you chose. Use model.to('cuda') to use the default device.".format( + name, buf.type())) + + +def check_optimizers(optimizers): + for optim in optimizers: + bad_optim_type = None + if isinstance(optim, FP16_Optimizer_general): + bad_optim_type = "apex.fp16_utils.FP16_Optimizer" + if isinstance(optim, FP16_Optimizer_for_fused): + bad_optim_type = "apex.optimizers.FP16_Optimizer" + if bad_optim_type is not None: + raise RuntimeError("An incoming optimizer is an instance of {}. ".format(bad_optim_type) + + "The optimizer(s) passed to amp.initialize() must be bare \n" + "instances of either ordinary Pytorch optimizers, or Apex fused \n" + "optimizers.\n") + + +class O2StateDictHook(object): + def __init__(self, fn): + self.fn = fn + + def __call__(self, module, state_dict, prefix, local_metadata): + for key in state_dict: + param = state_dict[key] + if 'Half' in param.type(): + param = param.to(torch.float32) + state_dict[key] = param + + +def _initialize(models, optimizers, properties, num_losses=1, cast_model_outputs=None): + from .amp import init as amp_init + + optimizers_was_list = False + if isinstance(optimizers, torch.optim.Optimizer) or ('LARC' in globals() and isinstance(optimizers, LARC)): + optimizers = [optimizers] + elif optimizers is None: + optimizers = [] + elif isinstance(optimizers, list): + optimizers_was_list = True + check_optimizers(optimizers) + else: + check_optimizers([optimizers]) + raise TypeError("optimizers must be either a single optimizer or a list of optimizers.") + + if isinstance(models, torch.nn.Module): + models_was_list = False + models = [models] + elif isinstance(models, list): + models_was_list = True + else: + raise TypeError("models must be either a single model or a list of models.") + + check_models(models) + + if not _amp_state.allow_incoming_model_not_fp32: + check_params_fp32(models) + + # In the future, when FP16_Optimizer can be deprecated and master weights can + # become an attribute, remember to stash master weights before casting the model. + + if properties.cast_model_type: + if properties.keep_batchnorm_fp32: + for model in models: + convert_network(model, properties.cast_model_type) + else: + for model in models: + model.to(properties.cast_model_type) + + input_caster = functools.partial(to_type, properties.cast_model_type) + if cast_model_outputs is not None: + output_caster = functools.partial(to_type, cast_model_outputs) + else: + output_caster = functools.partial(to_type, torch.float32) + + for model in models: + # Patch the forward method to cast incoming data to the correct type, and + # outgoing data to float32, so "the user never needs to call .half()." + # I like writing things explicitly more than decorators. + def patch_forward(old_fwd): + def new_fwd(*args, **kwargs): + output = old_fwd(*applier(args, input_caster), + **applier(kwargs, input_caster)) + return applier(output, output_caster) + return new_fwd + + model.forward = patch_forward(model.forward) + + # State dict trick to recast any preexisting per-param state tensors + for optimizer in optimizers: + optimizer.load_state_dict(optimizer.state_dict()) + + # patch model.state_dict() to return float32 params + for model in models: + for module in model.modules(): + module._register_state_dict_hook(O2StateDictHook(functools.partial(to_type, torch.float32))) + + elif cast_model_outputs is not None: + output_caster = functools.partial(to_type, cast_model_outputs) + + for model in models: + def patch_forward(old_fwd): + def new_fwd(*args, **kwargs): + output = old_fwd(*args, **kwargs) + return applier(output, output_caster) + return new_fwd + + model.forward = patch_forward(model.forward) + + for i, optimizer in enumerate(optimizers): + optimizers[i] = _process_optimizer(optimizer, properties) + + _amp_state.loss_scalers = [] + for _ in range(num_losses): + _amp_state.loss_scalers.append(LossScaler(properties.loss_scale, + min_loss_scale=_amp_state.min_loss_scale, + max_loss_scale=_amp_state.max_loss_scale)) + + if properties.patch_torch_functions: + # handle is unused here. It's accessible later through a global value anyway. + handle = amp_init(loss_scale=properties.loss_scale, verbose=(_amp_state.verbosity == 2)) + for optimizer in optimizers: + # Disable Amp casting for the optimizer step, because it should only be + # applied to FP32 master params anyway. + def patch_step(old_step): + def new_step(self, *args, **kwargs): + with disable_casts(): + output = old_step(*args, **kwargs) + return output + return new_step + + optimizer.step = MethodType(patch_step(optimizer.step), optimizer) + + if optimizers_was_list: + if models_was_list: + return models, optimizers + else: + return models[0], optimizers + else: + if models_was_list: + if len(optimizers) == 0: + return models + else: + return models, optimizers[0] + else: + if len(optimizers) == 0: + return models[0] + else: + return models[0], optimizers[0] diff --git a/apex/apex/amp/_process_optimizer.py b/apex/apex/amp/_process_optimizer.py new file mode 100644 index 00000000..471289bb --- /dev/null +++ b/apex/apex/amp/_process_optimizer.py @@ -0,0 +1,489 @@ +import types +from ..fp16_utils import master_params_to_model_params +from ..multi_tensor_apply import multi_tensor_applier +from ._amp_state import maybe_print +import torch +from ..optimizers import FusedSGD + + +class AmpOptimizerState(object): + def __init__(self): + pass + + +def _master_params_to_model_params(self): + stash = self._amp_stash + if multi_tensor_applier.available: + if len(stash.all_fp16_params) > 0: + multi_tensor_applier( + stash.multi_tensor_scale, + stash.dummy_overflow_buf, + [stash.all_fp32_from_fp16_params, stash.all_fp16_params], + 1.0) + else: + for fp16_group, fp32_from_fp16_group in zip(stash.fp16_groups, stash.fp32_from_fp16_groups): + master_params_to_model_params(fp16_group, fp32_from_fp16_group) + + +def lazy_init_with_master_weights(self): + stash = self._amp_stash + stash.fp16_groups = [] + stash.fp32_from_fp16_groups = [] + stash.fp32_from_fp32_groups = [] + for i, param_group in enumerate(self.param_groups): + # maybe_print("FP16_Optimizer processing param group {}:".format(i)) + fp16_params_this_group = [] + fp32_params_this_group = [] + fp32_from_fp16_params_this_group = [] + for i, param in enumerate(param_group['params']): + if param.requires_grad: + if param.type() == 'torch.cuda.HalfTensor': + # maybe_print("FP16_Optimizer received torch.cuda.HalfTensor with {}" + # .format(param.size())) + fp16_params_this_group.append(param) + master_param = param.detach().clone().float() + master_param.requires_grad = True + param_group['params'][i] = master_param + fp32_from_fp16_params_this_group.append(master_param) + # Reset existing state dict key to the new master param. + # We still need to recast per-param state tensors, if any, to FP32. + if param in self.state: + self.state[master_param] = self.state.pop(param) + elif param.type() == 'torch.cuda.FloatTensor': + # maybe_print("FP16_Optimizer received torch.cuda.FloatTensor with {}" + # .format(param.size())) + fp32_params_this_group.append(param) + param_group['params'][i] = param + else: + raise TypeError("Optimizer's parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + stash.fp16_groups.append(fp16_params_this_group) + stash.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group) + stash.fp32_from_fp32_groups.append(fp32_params_this_group) + + stash.all_fp16_params = [] + for group in stash.fp16_groups: + stash.all_fp16_params += group + + stash.all_fp32_from_fp16_params = [] + for group in stash.fp32_from_fp16_groups: + stash.all_fp32_from_fp16_params += group + + stash.all_fp32_from_fp32_params = [] + for group in stash.fp32_from_fp32_groups: + stash.all_fp32_from_fp32_params += group + + # all_fp16_grad_stash is only needed for fused optimizers. + stash.all_fp16_grad_stash = [None for _ in stash.all_fp16_params] + # stash.all_fp32_from_fp16_grad_stash = [None for _ in stash.all_fp32_from_fp16_params] + stash.all_fp32_from_fp32_grad_stash = [None for _ in stash.all_fp32_from_fp32_params] + + for param in stash.all_fp32_from_fp16_params: + param.grad = None + + for param in stash.all_fp32_from_fp32_params: + param.grad = None + + # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors + self.load_state_dict(self.state_dict()) + + +def post_backward_models_are_masters(scaler, params, stashed_grads, scale_override=None): + grads_have_scale, stashed_have_scale, out_scale = scaler.loss_scale(), 1.0, 1.0 + + # not much to do if scale == 1.0 and static scaling + if scaler.loss_scale() == 1.0 and not scaler.dynamic: + # Clear the stash. + for i in range(len(stashed_grads)): + stashed_grads[i] = None + return + + if scale_override is not None: + grads_have_scale, stashed_have_scale, out_scale = scale_override + + # This is a lot of python overhead... + grads_needing_unscale = [] + grads_needing_unscale_with_stash = [] + stashed = [] + for param, stashed_grad in zip(params, stashed_grads): + if param.grad is None and stashed_grad is not None: + param.grad = stashed_grad + elif param.grad is not None and stashed_grad is None: + grads_needing_unscale.append(param.grad) + elif param.grad is not None and stashed_grad is not None: + grads_needing_unscale_with_stash.append(param.grad) + stashed.append(stashed_grad) + else: # param.grad is None and stashed_grad is None + continue + + # unscale() implements grads*(1/scale), so "scale" should be grads_have_scale/out_scale. + if len(grads_needing_unscale) > 0: + scaler.unscale( + grads_needing_unscale, + grads_needing_unscale, + None, # unused_scale, currently present to avoid API breakage elsewhere + models_are_masters=True, + scale_override=grads_have_scale/out_scale) + + if len(grads_needing_unscale_with_stash) > 0: + scaler.unscale_with_stashed( + grads_needing_unscale_with_stash, + stashed, + grads_needing_unscale_with_stash, + scale_override=(grads_have_scale, stashed_have_scale, out_scale)) + + # Clear the stash. + for i in range(len(stashed_grads)): + stashed_grads[i] = None + + +def prepare_backward_with_master_weights(self): + stash = self._amp_stash + + self._amp_lazy_init() + + for i, param in enumerate(stash.all_fp16_params): + # Set up to leverage grad copy elision. + # This may behave differently from an unpatched optimizer if zero_grad is used and the param is unused. + param.grad = None + + # for i, param in enumerate(stash.all_fp32_from_fp16_params): + # stash.all_fp32_from_fp16_grad_stash[i] = param.grad + + for i, param in enumerate(stash.all_fp32_from_fp32_params): + stash.all_fp32_from_fp32_grad_stash[i] = param.grad + # Set up to leverage grad copy elision: + param.grad = None + + +def post_backward_with_master_weights(self, scaler): + stash = self._amp_stash + + self._amp_lazy_init() + + # This is a lot of python overhead... + fp16_grads_needing_unscale = [] + new_fp32_grads = [] + fp16_grads_needing_unscale_with_stash = [] + preexisting_fp32_grads = [] + for fp16_param, fp32_param in zip(stash.all_fp16_params, + stash.all_fp32_from_fp16_params): + if fp16_param.grad is None and fp32_param.grad is not None: + continue + elif fp16_param.grad is not None and fp32_param.grad is None: + fp32_param.grad = torch.empty_like(fp32_param) + fp16_grads_needing_unscale.append(fp16_param.grad) + new_fp32_grads.append(fp32_param.grad) + elif fp16_param.grad is not None and fp32_param.grad is not None: + fp16_grads_needing_unscale_with_stash.append(fp16_param.grad) + preexisting_fp32_grads.append(fp32_param.grad) + else: # fp16_param.grad is None and fp32_param.grad is None: + continue + + if len(fp16_grads_needing_unscale) > 0: + scaler.unscale( + fp16_grads_needing_unscale, + new_fp32_grads, + scaler.loss_scale(), + models_are_masters=False) + + if len(fp16_grads_needing_unscale_with_stash) > 0: + scaler.unscale_with_stashed( + fp16_grads_needing_unscale_with_stash, + preexisting_fp32_grads, + preexisting_fp32_grads) + + # fp32 params can be treated as they would be in the "no_master_weights" case. + post_backward_models_are_masters( + scaler, + stash.all_fp32_from_fp32_params, + stash.all_fp32_from_fp32_grad_stash) + + +def lazy_init_no_master_weights(self): + stash = self._amp_stash + stash.all_fp16_params = [] + stash.all_fp32_params = [] + for i, param_group in enumerate(self.param_groups): + for i, param in enumerate(param_group['params']): + if param.type() == 'torch.cuda.HalfTensor': + stash.all_fp16_params.append(param) + elif param.type() == 'torch.cuda.FloatTensor': + stash.all_fp32_params.append(param) + else: + raise TypeError("Optimizer's parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + stash.all_fp16_grad_stash = [None for _ in stash.all_fp16_params] + stash.all_fp32_grad_stash = [None for _ in stash.all_fp32_params] + + +def prepare_backward_no_master_weights(self): + stash = self._amp_stash + + self._amp_lazy_init() + + for i, param in enumerate(stash.all_fp16_params): + stash.all_fp16_grad_stash[i] = param.grad + # Set up to leverage grad copy elision: + param.grad = None + + for i, param in enumerate(stash.all_fp32_params): + stash.all_fp32_grad_stash[i] = param.grad + # Set up to leverage grad copy elision: + param.grad = None + + +def post_backward_no_master_weights(self, scaler): + stash = self._amp_stash + + self._amp_lazy_init() + + split_types = ((stash.all_fp16_params, stash.all_fp16_grad_stash), + (stash.all_fp32_params, stash.all_fp32_grad_stash)) + + for params, stashed_grads in split_types: + post_backward_models_are_masters(scaler, params, stashed_grads) + + +##################################################################################### +# FusedSGD versions +##################################################################################### + +# FusedSGD never explicitly materializes the fp32 gradients for "fp32 from fp16" master params +# outside the kernel, so we must accumulate directly into the model grads. +def prepare_backward_with_master_weights_FusedSGD(self): + if self.materialize_master_grads: + prepare_backward_with_master_weights(self) + else: + stash = self._amp_stash + + self._amp_lazy_init() + + for i, param in enumerate(stash.all_fp16_params): + stash.all_fp16_grad_stash[i] = param.grad + # Set up to leverage grad copy elision: + param.grad = None + + for i, param in enumerate(stash.all_fp32_from_fp32_params): + stash.all_fp32_from_fp32_grad_stash[i] = param.grad + # Set up to leverage grad copy elision: + param.grad = None + + +def post_backward_with_master_weights_FusedSGD(self, scaler): + if self.materialize_master_grads: + post_backward_with_master_weights(self, scaler) + else: + stash = self._amp_stash + + self._amp_lazy_init() + + grads_have_scale = scaler.loss_scale() + stashed_have_scale = self.most_recent_scale + out_scale = grads_have_scale + if self.scale_set_by_backward: + out_scale = min(grads_have_scale, self.most_recent_scale) + + split_types = ((stash.all_fp16_params, stash.all_fp16_grad_stash), + (stash.all_fp32_from_fp32_params, stash.all_fp32_from_fp32_grad_stash)) + + + # unscale_with_stashed() implements grads*1/scale + stashed_grads*1. + # stashed_grads are scaled by self.most_recent_scale. + for params, stashed_grads in split_types: + post_backward_models_are_masters(scaler, params, stashed_grads, + (grads_have_scale, stashed_have_scale, out_scale)) + + self.most_recent_scale = out_scale + self.scale_set_by_backward = True + + +def prepare_backward_no_master_weights_FusedSGD(self): + prepare_backward_no_master_weights(self) + + +def post_backward_no_master_weights_FusedSGD(self, scaler): + post_backward_no_master_weights(self, scaler) + + +def _amp_lazy_init(self): + stash = self._amp_stash + + if not stash.lazy_init_called: + self._lazy_init_maybe_master_weights() + stash.lazy_init_called = True + + +def _process_optimizer(optimizer, properties): + if hasattr(optimizer, "_amp_stash"): + raise RuntimeError("A given optimizer should only be passed through amp.initialize once.") + else: + optimizer._amp_stash = AmpOptimizerState() + + optimizer._amp_stash.lazy_init_called = False + optimizer._amp_stash.already_patched = False + optimizer._amp_stash.params_have_scaled_gradients = False + + for name in ("_lazy_init_maybe_master_weights", + "_master_params_to_model_params", + "_prepare_amp_backward", + "_post_amp_backward", + "_amp_lazy_init"): + if hasattr(optimizer, name): + raise RuntimeError("Incoming optimizer already has {} defined.".format(name)) + + # TODO: Centralize exposure and import error checking for the C backend. + if multi_tensor_applier.available: + import amp_C + optimizer._amp_stash.multi_tensor_scale = amp_C.multi_tensor_scale + optimizer._amp_stash.multi_tensor_l2norm = amp_C.multi_tensor_l2norm + optimizer._amp_stash.dummy_overflow_buf = torch.cuda.IntTensor([0]); + + if properties.master_weights: + optimizer._lazy_init_maybe_master_weights = types.MethodType( + lazy_init_with_master_weights, optimizer) + + optimizer._master_params_to_model_params = types.MethodType( + _master_params_to_model_params, optimizer) + + old_step = optimizer.step + def new_step(self, closure=None): + if closure is not None: + raise RuntimeError("Currently, Amp does not support closure use with optimizers.") + retval = old_step() + if not isinstance(self, FusedSGD): + self._master_params_to_model_params() + # Clear the master grads that wouldn't be zeroed by model.zero_grad() + for param in self._amp_stash.all_fp32_from_fp16_params: + param.grad = None + return retval + optimizer.step = types.MethodType(new_step, optimizer) + + old_zero_grad = optimizer.zero_grad + def new_zero_grad(self): + stash = self._amp_stash + self._amp_lazy_init() + # Zero the model grads. + for param in stash.all_fp16_params: + if param.grad is not None: + param.grad.detach_() + param.grad.zero_() + for param in stash.all_fp32_from_fp32_params: + if param.grad is not None: + param.grad.detach_() + param.grad.zero_() + # Clear the master grads that are independent of model grads + for param in self._amp_stash.all_fp32_from_fp16_params: + param.grad = None + optimizer.zero_grad = types.MethodType(new_zero_grad, optimizer) + + if isinstance(optimizer, FusedSGD): + optimizer._prepare_amp_backward = types.MethodType( + prepare_backward_with_master_weights_FusedSGD, optimizer) + optimizer._post_amp_backward = types.MethodType( + post_backward_with_master_weights_FusedSGD, optimizer) + else: + optimizer._prepare_amp_backward = types.MethodType( + prepare_backward_with_master_weights, optimizer) + optimizer._post_amp_backward = types.MethodType( + post_backward_with_master_weights, optimizer) + else: + optimizer._lazy_init_maybe_master_weights = types.MethodType( + lazy_init_no_master_weights, optimizer) + + if isinstance(optimizer, FusedSGD): + optimizer._prepare_amp_backward = types.MethodType( + prepare_backward_no_master_weights_FusedSGD, optimizer) + optimizer._post_amp_backward = types.MethodType( + post_backward_no_master_weights_FusedSGD, optimizer) + else: + optimizer._prepare_amp_backward = types.MethodType( + prepare_backward_no_master_weights, optimizer) + optimizer._post_amp_backward = types.MethodType( + post_backward_no_master_weights, optimizer) + + optimizer._amp_lazy_init = types.MethodType(_amp_lazy_init, optimizer) + + old_add_param_group = optimizer.add_param_group + + def new_add_param_group(self, new_group): + stash = self._amp_stash + + if not stash.lazy_init_called: + self._lazy_init_maybe_master_weights() + stash.lazy_init_called = True + + assert isinstance(new_group, dict), "param group must be a dict" + + new_params = new_group['params'] + if isinstance(new_params, torch.Tensor): + new_group['params'] = [new_params] + elif isinstance(new_params, set): + raise TypeError('optimizer parameters need to be organized in ordered collections, but ' + 'the ordering of tensors in sets will change between runs. Please use a list instead.') + else: + new_group['params'] = list(new_params) + + if properties.master_weights: + # Mutate new_group in-place to use FP32 master params + fp16_params_this_group = [] + fp32_params_this_group = [] + fp32_from_fp16_params_this_group = [] + for i, param in enumerate(new_group['params']): + if param.requires_grad: + if param.type() == 'torch.cuda.HalfTensor': + fp16_params_this_group.append(param) + master_param = param.detach().clone().float() + master_param.requires_grad = True + new_group['params'][i] = master_param + fp32_from_fp16_params_this_group.append(master_param) + elif param.type() == 'torch.cuda.FloatTensor': + fp32_params_this_group.append(param) + new_group['params'][i] = param + else: + raise TypeError("Optimizer's parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + stash.fp16_groups.append(fp16_params_this_group) + stash.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group) + stash.fp32_from_fp32_groups.append(fp32_params_this_group) + + stash.all_fp16_params += fp16_params_this_group + stash.all_fp32_from_fp16_params += fp32_from_fp16_params_this_group + stash.all_fp32_from_fp32_params += fp32_params_this_group + + # stash.all_fp32_from_fp16_grad_stash = [None for _ in stash.all_fp32_from_fp16_params] + stash.all_fp32_from_fp32_grad_stash += [None for _ in fp32_params_this_group] + + # It should be ok to let params be added with existing .grad attributes. + # for param in fp16_params_this_group: + # param.grad = None + + # for param in fp32_from_fp16_params_this_group: + # param.grad = None + + # for param in stash.fp32_params_this_group: + # param.grad = None + else: + for param in new_group['params']: + if param.type() == 'torch.cuda.HalfTensor': + stash.all_fp16_params.append(param) + stash.all_fp16_grad_stash.append(None) + elif param.type() == 'torch.cuda.FloatTensor': + stash.all_fp32_params.append(param) + stash.all_fp32_grad_stash.append(None) + else: + raise TypeError("Optimizer's parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + old_add_param_group(new_group) + + optimizer.add_param_group = types.MethodType(new_add_param_group, optimizer) + + return optimizer diff --git a/apex/apex/amp/amp.py b/apex/apex/amp/amp.py new file mode 100644 index 00000000..1a604666 --- /dev/null +++ b/apex/apex/amp/amp.py @@ -0,0 +1,183 @@ +import functools +import itertools + +import torch + +from . import compat, rnn_compat, utils, wrap +from .handle import AmpHandle, NoOpHandle +from .lists import functional_overrides, torch_overrides, tensor_overrides +from ._amp_state import _amp_state +from .frontend import * + + +_DECORATOR_HANDLE = None +_USER_CAST_REGISTRY = set() +_USER_PROMOTE_REGISTRY = set() + + +def _decorator_helper(orig_fn, cast_fn, wrap_fn): + def wrapper(*args, **kwargs): + handle = _DECORATOR_HANDLE + if handle is None or not handle.is_active(): + return orig_fn(*args, **kwargs) + inner_cast_fn = utils.verbosify(cast_fn, orig_fn.__name__, + handle.verbose) + return wrap_fn(orig_fn, inner_cast_fn, handle)(*args, **kwargs) + return wrapper + + +# Decorator form +def half_function(fn): + from apex import deprecated_warning + deprecated_warning("apex.amp is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + wrap_fn = functools.partial(wrap.make_cast_wrapper, try_caching=True) + return _decorator_helper(fn, utils.maybe_half, wrap_fn) + + +def float_function(fn): + from apex import deprecated_warning + deprecated_warning("apex.amp is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + wrap_fn = functools.partial(wrap.make_cast_wrapper, try_caching=False) + return _decorator_helper(fn, utils.maybe_float, wrap_fn) + + +def promote_function(fn): + from apex import deprecated_warning + deprecated_warning("apex.amp is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + wrap_fn = functools.partial(wrap.make_promote_wrapper) + return _decorator_helper(fn, utils.maybe_float, wrap_fn) + + +# Registry form +def register_half_function(module, name): + if not hasattr(module, name): + raise ValueError('No function named {} in module {}.'.format( + name, module)) + _USER_CAST_REGISTRY.add((module, name, utils.maybe_half)) + + +def register_float_function(module, name): + if not hasattr(module, name): + raise ValueError('No function named {} in module {}.'.format( + name, module)) + _USER_CAST_REGISTRY.add((module, name, utils.maybe_float)) + + +def register_promote_function(module, name): + if not hasattr(module, name): + raise ValueError('No function named {} in module {}.'.format( + name, module)) + _USER_PROMOTE_REGISTRY.add((module, name)) + + +# Top-level function to insert _all_ the hooks. +def init(enabled=True, loss_scale="dynamic", enable_caching=True, verbose=False, allow_banned=False): + global _DECORATOR_HANDLE + + if not enabled: + handle = NoOpHandle() + _DECORATOR_HANDLE = handle + return handle + + handle = AmpHandle(loss_scale, enable_caching, verbose) + + # 0) Force-{fp16, fp32} for user-annotated functions + for mod, fn, cast_fn in _USER_CAST_REGISTRY: + try_caching = (cast_fn == utils.maybe_half) + wrap.cached_cast(mod, fn, cast_fn, handle, + try_caching, verbose) + _USER_CAST_REGISTRY.clear() + + # 0.5) Force-promote for user-annotated functions + for mod, fn in _USER_PROMOTE_REGISTRY: + wrap.promote(mod, fn, handle, verbose) + _USER_PROMOTE_REGISTRY.clear() + + # 1) Force-{fp16, fp32} on white- / black-list functions + override_modules = [functional_overrides, + torch_overrides, + tensor_overrides] + cast_table = [('FP16_FUNCS', utils.maybe_half), + ('FP32_FUNCS', utils.maybe_float)] + for module, (list_name, cast_fn) in itertools.product(override_modules, + cast_table): + for fn in getattr(module, list_name): + try_caching = (cast_fn == utils.maybe_half) + wrap.cached_cast(module.MODULE, fn, cast_fn, handle, + try_caching, verbose) + + # 1.5) Pre-0.4, put the blacklist methods on HalfTensor and whitelist + # methods on FloatTensor, since they're distinct types. + if compat.tensor_is_float_tensor(): + for fn in tensor_overrides.FP16_FUNCS: + wrap.cached_cast(torch.cuda.FloatTensor, fn, utils.maybe_half, + handle, try_caching=True, verbose=verbose) + for fn in tensor_overrides.FP32_FUNCS: + wrap.cached_cast(torch.cuda.HalfTensor, fn, utils.maybe_float, + handle, try_caching=False, verbose=verbose) + + # 2) Enable type-promotion on multi-arg functions and methods. + # NB: special handling for sequence fns (e.g. `torch.cat`). + promote_modules = [torch_overrides, tensor_overrides] + promote_table = [('CASTS', wrap.promote), + ('SEQUENCE_CASTS', wrap.sequence_promote)] + for promote_mod, (list_name, promote_fn) in itertools.product(promote_modules, + promote_table): + for fn in getattr(promote_mod, list_name): + promote_fn(promote_mod.MODULE, fn, handle, verbose) + + # 2.5) Pre-0.4, add blacklist methods directly to HalfTensor and FloatTensor types + if compat.tensor_is_float_tensor(): + for cls, (list_name, promote_fn) in itertools.product([torch.cuda.FloatTensor, + torch.cuda.HalfTensor], + promote_table): + for fn in getattr(tensor_overrides, list_name): + promote_fn(cls, fn, handle, verbose) + + # 3) For any in-place version of a blacklist function, error if any input is fp16. + # NB: this is overly conservative. + for fn in utils.as_inplace(torch_overrides.FP32_FUNCS): + wrap.err_if_any_half(torch_overrides.MODULE, fn, handle) + + # 3.5) For any in-place blacklist method, error if called on fp16 tensor + for fn in utils.as_inplace(tensor_overrides.FP32_FUNCS): + wrap.err_if_arg0_half(tensor_overrides.MODULE, fn, handle, verbose) + if compat.tensor_is_float_tensor(): + wrap.err_if_arg0_half(torch.cuda.HalfTensor, fn, handle, verbose) + + # 4) For other in-place methods, match the type of self tensor + for fn in utils.as_inplace(itertools.chain( + tensor_overrides.FP16_FUNCS, + tensor_overrides.CASTS)): + wrap.promote_match_arg0(tensor_overrides.MODULE, fn, handle, verbose) + if compat.tensor_is_float_tensor(): + wrap.promote_match_arg0(torch.cuda.HalfTensor, fn, handle, verbose) + wrap.promote_match_arg0(torch.cuda.FloatTensor, fn, handle, verbose) + + # 5) RNNs + RNN cells are whitelisted specially + if rnn_compat.has_old_rnns(): + wrap.rnn_cast(torch.nn.backends.thnn.backend, 'RNN', handle, verbose) + if not rnn_compat.has_old_rnns(): + # Patch in our own indirection of `_VF` in modules/rnn s.t. it is mutable. + torch.nn.modules.rnn._VF = rnn_compat.VariableFunctionsShim() + # Wrap all the rnns + for x in rnn_compat.RNN_NAMES: + wrap.new_rnn_cast(x.upper(), handle, verbose) + + # Wrap all the RNN cells + rnn_compat.whitelist_rnn_cells(handle, verbose) + + # 6) Place error+print message on banned functions. + # Or, if allow_banned, then cast to FP32. + for fn, err_msg in functional_overrides.BANNED_FUNCS: + if allow_banned: + wrap.cached_cast(functional_overrides.MODULE, fn, utils.maybe_float, + handle, try_caching=True, verbose=verbose) + else: + wrap.err_if_any_half(functional_overrides.MODULE, fn, handle, err_msg) + + _DECORATOR_HANDLE = handle + + _amp_state.handle = handle + + return handle diff --git a/apex/apex/amp/compat.py b/apex/apex/amp/compat.py new file mode 100644 index 00000000..22276bd4 --- /dev/null +++ b/apex/apex/amp/compat.py @@ -0,0 +1,46 @@ +import torch + +# True for post-0.4, when Variables/Tensors merged. +def variable_is_tensor(): + v = torch.autograd.Variable() + return isinstance(v, torch.Tensor) + +def tensor_is_variable(): + x = torch.Tensor() + return type(x) == torch.autograd.Variable + +# False for post-0.4 +def tensor_is_float_tensor(): + x = torch.Tensor() + return type(x) == torch.FloatTensor + +# Akin to `torch.is_tensor`, but returns True for Variable +# objects in pre-0.4. +def is_tensor_like(x): + return torch.is_tensor(x) or isinstance(x, torch.autograd.Variable) + +# Wraps `torch.is_floating_point` if present, otherwise checks +# the suffix of `x.type()`. +def is_floating_point(x): + if hasattr(torch, 'is_floating_point'): + return torch.is_floating_point(x) + try: + torch_type = x.type() + return torch_type.endswith('FloatTensor') or \ + torch_type.endswith('HalfTensor') or \ + torch_type.endswith('DoubleTensor') + except AttributeError: + return False + +def scalar_python_val(x): + if hasattr(x, 'item'): + return x.item() + else: + if isinstance(x, torch.autograd.Variable): + return x.data[0] + else: + return x[0] + +# Accounts for the possibility that some ops may be removed from a namespace. +def filter_attrs(module, attrs): + return list(attrname for attrname in attrs if hasattr(module, attrname)) diff --git a/apex/apex/amp/frontend.py b/apex/apex/amp/frontend.py new file mode 100644 index 00000000..616fb113 --- /dev/null +++ b/apex/apex/amp/frontend.py @@ -0,0 +1,446 @@ +from collections import OrderedDict + +import torch + +from ._initialize import _initialize +from ._amp_state import _amp_state, warn_or_err, maybe_print + + +class Properties(object): + """ + This class has two purposes: to establish a set of default properties, + and to route setting of these attributes through __setattr__ so that (in theory) + they can be checked for consistency with other existing args. + """ + def __init__(self): + self.options = { + "enabled" : False, + "opt_level" : None, + "cast_model_type" : None, + "patch_torch_functions" : False, + "keep_batchnorm_fp32" : None, + "master_weights" : None, + "loss_scale" : 1.0, + # Reserved for future functionality + # "fused_optimizer" : False, + # "enable_ddp_interop" : False, + } + + """ + This function allows updating several options at a time without routing through + __setattr__ checks, to avoid "you can't get there from here" scenarios. + Currently not intended to be exposed; users are expected to select an opt_level + and apply consistent modifications. + """ + def _update_options_dict(self, new_options): + for k, v in new_options: + if k in self.options: + self.options[k] = v + else: + raise ValueError("Tried to set unexpected option {}".format(k)) + """ + The members of "options" are not direct attributes of self, so access attempts + will roll down to __getattr__. This borrows from the logic in torch.nn.Module. + """ + def __getattr__(self, name): + if "options" in self.__dict__: + options = self.__dict__["options"] + if name in options: + return options[name] + raise AttributeError("'{}' object has no attribute '{}'".format( + type(self).__name__, name)) + + def __setattr__(self, name, value): + if "options" in self.__dict__: + if name in self.options: + # print("setting {} {}".format(name, value)) + if name == "cast_model_type": + if self.opt_level == "O1" and value is not None: + if value is not False: + if value is not torch.float32: + warn_or_err("O1 inserts casts around Torch functions rather than " + "model weights, so with O1, the model weights themselves " + "should remain FP32. If you wish to cast the model to a " + "different type, use opt_level='O2' or 'O3'. " + + "cast_model_type was {}".format(value)) + self.options[name] = value + elif name == "patch_torch_functions": + if self.opt_level != "O1" and value: + warn_or_err("Currently, patch_torch_functions=True should only be set by " + "selecting opt_level='O1'.") + self.options[name] = value + elif name == "keep_batchnorm_fp32": + if self.opt_level == "O1" and value is not None: + warn_or_err("With opt_level O1, batchnorm functions are automatically patched " + "to run in FP32, so keep_batchnorm_fp32 should be None." + + " keep_batchnorm_fp32 was {}".format(value)) + if value == "False": + self.options[name] = False + elif value == "True": + self.options[name] = True + else: + assert (value is True or value is False or value is None),\ + "keep_batchnorm_fp32 must be a boolean, the string 'True' or 'False', "\ + "or None, found keep_batchnorm_fp32={}".format(value) + self.options[name] = value + elif name == "master_weights": + if self.opt_level == "O1" and value is not None: + warn_or_err("It doesn't make sense to use master_weights with O1. " + "With O1, your model weights themselves should be FP32.") + self.options[name] = value + elif name == "loss_scale": + if value == "dynamic": + self.options[name] = value + else: + self.options[name] = float(value) + else: + self.options[name] = value + else: + super(Properties, self).__setattr__(name, value) + + +""" O0-O3 are convenience wrappers to establish defaults for typically used mixed precision options. """ + +class O3: + brief = "O3: Pure FP16 training." + more = "Calls .half() on your model, converting the entire model to FP16.\n"\ + "A casting operation is also inserted to cast incoming Tensors to FP16,\n"\ + "so you don't need to change your data pipeline.\n"\ + "This mode is useful for establishing a performance ceiling.\n"\ + "It's also possible training may 'just work' in this mode.\n"\ + "If not, try other optimization levels." + + def __call__(self, properties): + properties.enabled = True + properties.opt_level = "O3" + properties.cast_model_type = torch.float16 + properties.patch_torch_functions = False + properties.keep_batchnorm_fp32 = False + properties.master_weights = False + properties.loss_scale = 1.0 + # properties.fused_optimizer = False + # properties.enable_ddp_interop = False + return properties # modified in place so this isn't really necessary + + +class O2: + brief = "O2: FP16 training with FP32 batchnorm and FP32 master weights.\n" + more = "Calls .half() on your model, converting the entire model (except for batchnorms)\n"\ + "to FP16. Batchnorms are retained in FP32 for additional stability.\n"\ + "The forward pass is patched to cast incoming Tensors to FP16, so you don't need to change\n"\ + "your data pipeline.\n"\ + "O2 creates FP32 master weights outside the model and patches any optimizers to update\n"\ + "these master weights, then copy the master weights into the FP16 model weights.\n"\ + "Master weights can also improve convergence and stability." + + def __call__(self, properties): + properties.enabled = True + properties.opt_level = "O2" + properties.cast_model_type = torch.float16 + properties.patch_torch_functions = False + properties.keep_batchnorm_fp32 = True + properties.master_weights = True + properties.loss_scale = "dynamic" + # properties.fused_optimizer = False + # properties.enable_ddp_interop = False + return properties # modified in place so this isn't really necessary + + +class O1: + brief = "O1: Insert automatic casts around Pytorch functions and Tensor methods.\n" + more = "The type of your model's weights is not altered. However, internally,\n"\ + "Pytorch functions are patched to cast any Tensor Core-friendly ops to FP16 for speed,\n"\ + "while operations that might benefit from the additional stability of FP32 are patched\n"\ + "to cast their inputs to fp32.\n"\ + "O1 is the safest way to try mixed precision training, and is recommended when\n"\ + "trying mixed precision training for the first time." + + def __call__(self, properties): + properties.enabled = True + properties.opt_level = "O1" + properties.cast_model_type = None + properties.patch_torch_functions = True + properties.keep_batchnorm_fp32 = None + properties.master_weights = None + properties.loss_scale = "dynamic" + # properties.fused_optimizer = False + # properties.enable_ddp_interop = False + return properties # modified in place so this isn't really necessary + + +class O0: + brief = "O0: Pure FP32 training.\n" + more = "Your models are checked to make sure parameters are FP32, but otherwise the\n"\ + "types of weights and internal Pytorch operations are not altered. This mode disables any\n"\ + "FP16 arithmetic, although other optimizations like DDP interop may still be requested.\n" + + def __call__(self, properties): + properties.enabled = True + properties.opt_level = "O0" + properties.cast_model_type = torch.float32 + properties.patch_torch_functions = False + properties.keep_batchnorm_fp32 = None + properties.master_weights = False + properties.loss_scale = 1.0 + # properties.fused_optimizer = False + # properties.enable_ddp_interop = False + return properties # modified in place so this isn't really necessary + + +opt_levels = {"O3": O3(), + "O2": O2(), + "O1": O1(), + "O0": O0()} + + +# allow user to directly pass Properties struct as well? +def initialize( + models, + optimizers=None, + enabled=True, + opt_level="O1", + cast_model_type=None, + patch_torch_functions=None, + keep_batchnorm_fp32=None, + master_weights=None, + loss_scale=None, + cast_model_outputs=None, + num_losses=1, + verbosity=1, + min_loss_scale=None, + max_loss_scale=2.**24 + ): + """ + Initialize your models, optimizers, and the Torch tensor and functional namespace according to the + chosen ``opt_level`` and overridden properties, if any. + + ``amp.initialize`` should be called **after** you have finished + constructing your model(s) and + optimizer(s), but **before** you send your model through any DistributedDataParallel wrapper. + See `Distributed training`_ in the Imagenet example. + + Currently, ``amp.initialize`` should only be called **once**, + although it can process an arbitrary number of + models and optimizers (see the corresponding `Advanced Amp Usage topic`_). + If you think your use case requires ``amp.initialize`` to be called more than once, + `let us know`_. + + Any property keyword argument that is not ``None`` will be interpreted as a manual override. + + To prevent having to rewrite anything else in your script, name the returned models/optimizers + to replace the passed models/optimizers, as in the code sample below. + + Args: + models (torch.nn.Module or list of torch.nn.Modules): Models to modify/cast. + optimizers (optional, torch.optim.Optimizer or list of torch.optim.Optimizers): Optimizers to modify/cast. + REQUIRED for training, optional for inference. + enabled (bool, optional, default=True): If False, renders all Amp calls no-ops, so your script + should run as if Amp were not present. + opt_level (str, optional, default="O1"): Pure or mixed precision optimization level. Accepted values are + "O0", "O1", "O2", and "O3", explained in detail above. + cast_model_type (``torch.dtype``, optional, default=None): Optional property override, see + above. + patch_torch_functions (bool, optional, default=None): Optional property override. + keep_batchnorm_fp32 (bool or str, optional, default=None): Optional property override. If + passed as a string, must be the string "True" or "False". + master_weights (bool, optional, default=None): Optional property override. + loss_scale (float or str, optional, default=None): Optional property override. If passed as a string, + must be a string representing a number, e.g., "128.0", or the string "dynamic". + cast_model_outputs (torch.dtype, optional, default=None): Option to ensure that the outputs + of your model(s) are always cast to a particular type regardless of ``opt_level``. + num_losses (int, optional, default=1): Option to tell Amp in advance how many losses/backward + passes you plan to use. When used in conjunction with the ``loss_id`` argument to + ``amp.scale_loss``, enables Amp to use a different loss scale per loss/backward pass, + which can improve stability. See "Multiple models/optimizers/losses" + under `Advanced Amp Usage`_ for examples. If ``num_losses`` is left to 1, Amp will still + support multiple losses/backward passes, but use a single global loss scale + for all of them. + verbosity (int, default=1): Set to 0 to suppress Amp-related output. + min_loss_scale (float, default=None): Sets a floor for the loss scale values that can be chosen by dynamic + loss scaling. The default value of None means that no floor is imposed. + If dynamic loss scaling is not used, `min_loss_scale` is ignored. + max_loss_scale (float, default=2.**24): Sets a ceiling for the loss scale values that can be chosen by + dynamic loss scaling. If dynamic loss scaling is not used, `max_loss_scale` is ignored. + + Returns: + Model(s) and optimizer(s) modified according to the ``opt_level``. + If either the ``models`` or ``optimizers`` args were lists, the corresponding return value will + also be a list. + + Permissible invocations:: + + model, optim = amp.initialize(model, optim,...) + model, [optim1, optim2] = amp.initialize(model, [optim1, optim2],...) + [model1, model2], optim = amp.initialize([model1, model2], optim,...) + [model1, model2], [optim1, optim2] = amp.initialize([model1, model2], [optim1, optim2],...) + + # This is not an exhaustive list of the cross product of options that are possible, + # just a set of examples. + model, optim = amp.initialize(model, optim, opt_level="O0") + model, optim = amp.initialize(model, optim, opt_level="O0", loss_scale="dynamic"|128.0|"128.0") + + model, optim = amp.initialize(model, optim, opt_level="O1") # uses "loss_scale="dynamic" default + model, optim = amp.initialize(model, optim, opt_level="O1", loss_scale=128.0|"128.0") + + model, optim = amp.initialize(model, optim, opt_level="O2") # uses "loss_scale="dynamic" default + model, optim = amp.initialize(model, optim, opt_level="O2", loss_scale=128.0|"128.0") + model, optim = amp.initialize(model, optim, opt_level="O2", keep_batchnorm_fp32=True|False|"True"|"False") + + model, optim = amp.initialize(model, optim, opt_level="O3") # uses loss_scale=1.0 default + model, optim = amp.initialize(model, optim, opt_level="O3", loss_scale="dynamic"|128.0|"128.0") + model, optim = amp.initialize(model, optim, opt_level="O3", keep_batchnorm_fp32=True|False|"True"|"False") + + The `Imagenet example`_ demonstrates live use of various opt_levels and overrides. + + .. _`Distributed training`: + https://github.com/NVIDIA/apex/tree/master/examples/imagenet#distributed-training + + .. _`Imagenet example`: + https://github.com/NVIDIA/apex/tree/master/examples/imagenet + + .. _`Advanced Amp Usage`: + https://nvidia.github.io/apex/advanced.html + + .. _`Advanced Amp Usage topic`: + https://nvidia.github.io/apex/advanced.html#multiple-models-optimizers-losses + + .. _`let us know`: + https://github.com/NVIDIA/apex/issues + """ + from apex import deprecated_warning + deprecated_warning("apex.amp is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + _amp_state.opt_properties = Properties() + _amp_state.verbosity = verbosity + + if not enabled: + if optimizers is None: + return models + else: + return models, optimizers + + if not torch.backends.cudnn.enabled: + raise RuntimeError( + "Amp requires torch.backends.cudnn.enabled = True") + + if opt_level not in opt_levels: + raise RuntimeError( + "Unexpected optimization level {}. ".format(opt_level) + + "Options are 'O0', 'O1', 'O2', 'O3'. Note that in `O0`, `O1`, etc., the prefix O is the letter O, " + + "not the number zero.") + else: + _amp_state.opt_properties = opt_levels[opt_level](_amp_state.opt_properties) + maybe_print("Selected optimization level {}".format(opt_levels[opt_level].brief), True) + maybe_print("Defaults for this optimization level are:", True) + for k, v in _amp_state.opt_properties.options.items(): + maybe_print("{:22} : {}".format(k, v), True) + + _amp_state.min_loss_scale = min_loss_scale + _amp_state.max_loss_scale = max_loss_scale + + maybe_print("Processing user overrides (additional kwargs that are not None)...", True) + # I chose to have the keyword arguments listed directly in the argument list, + # instead of **kwargs, so I can't use kwargs.items() here. + if enabled is not None: + _amp_state.opt_properties.enabled = enabled + if opt_level is not None: + _amp_state.opt_properties.opt_level = opt_level + if cast_model_type is not None: + _amp_state.opt_properties.cast_model_type = cast_model_type + if patch_torch_functions is not None: + _amp_state.opt_properties.patch_torch_functions = patch_torch_functions + if keep_batchnorm_fp32 is not None: + _amp_state.opt_properties.keep_batchnorm_fp32 = keep_batchnorm_fp32 + if master_weights is not None: + _amp_state.opt_properties.master_weights = master_weights + if loss_scale is not None: + _amp_state.opt_properties.loss_scale = loss_scale + + maybe_print("After processing overrides, optimization options are:", True) + for k, v in _amp_state.opt_properties.options.items(): + maybe_print("{:22} : {}".format(k, v), True) + + return _initialize(models, optimizers, _amp_state.opt_properties, num_losses, cast_model_outputs) + + +def state_dict(destination=None): + if destination is None: + destination = OrderedDict() + + for idx, loss_scaler in enumerate(_amp_state.loss_scalers): + destination['loss_scaler%d' % idx] = { + 'loss_scale': loss_scaler.loss_scale(), + 'unskipped': loss_scaler._unskipped, + } + return destination + + +def load_state_dict(state_dict): + # Check if state_dict containes the same number of loss_scalers as current setup + if len(state_dict) != len(_amp_state.loss_scalers): + print('Warning: state_dict contains {} entries, while {} loss_scalers are used'.format( + len(state_dict), len(_amp_state.loss_scalers))) + + state_dict = state_dict.copy() + + nb_loss_scalers = len(_amp_state.loss_scalers) + unexpected_keys = [] + # Initialize idx outside, since unexpected_keys will increase it if enumerate is used + idx = 0 + for key in state_dict: + if 'loss_scaler' not in key: + unexpected_keys.append(key) + else: + if idx > (nb_loss_scalers - 1): + print('Skipping loss_scaler[{}], since num_losses was set to {}'.format( + idx, nb_loss_scalers)) + break + _amp_state.loss_scalers[idx]._loss_scale = state_dict[key]['loss_scale'] + _amp_state.loss_scalers[idx]._unskipped = state_dict[key]['unskipped'] + idx += 1 + + if len(unexpected_keys) > 0: + raise RuntimeError( + 'Error(s) in loading state_dict. Unexpected key(s) in state_dict: {}. '.format( + ', '.join('"{}"'.format(k) for k in unexpected_keys))) + + +# TODO: is this necessary/useful? +# def check_option_consistency(enabled=True, +# opt_level=None, +# cast_model_type=None, +# patch_torch_functions=None, +# keep_batchnorm_fp32=None, +# master_weights=None, +# loss_scale=None, +# enable_ddp_interop=None, +# hard_override=False): +# """ +# Utility function that enables users to quickly check if the option combination they intend +# to use is permitted. ``check_option_consistency`` does not require models or optimizers +# to be constructed, and can be called at any point in the script. ``check_option_consistency`` +# is totally self-contained; it does not set any amp global state or affect anything outside +# of itself. +# """ +# +# if not enabled: +# return +# +# if opt_level not in opt_levels: +# raise RuntimeError("Unexpected optimization level. Options are 'O0', 'O1', 'O2', 'O3'.") +# else: +# opt_properties = opt_levels[opt_level](Properties()) +# print("Selected optimization level {}", opt_levels[opt_level].brief) +# print("Defaults for this optimization level are:") +# for k, v in opt_properties.options: +# print("{:22} : {}".format(k, v)) +# +# print("Processing user overrides (additional kwargs that are not None)...") +# for k, v in kwargs: +# if k not in _amp_state.opt_properties.options: +# raise RuntimeError("Unexpected kwarg {}".format(k)) +# if v is not None: +# setattr(opt_properties, k, v) +# +# print("After processing overrides, optimization options are:") +# for k, v in opt_properties.options: +# print("{:22} : {}".format(k, v)) diff --git a/apex/apex/amp/handle.py b/apex/apex/amp/handle.py new file mode 100644 index 00000000..0be567ca --- /dev/null +++ b/apex/apex/amp/handle.py @@ -0,0 +1,281 @@ +import contextlib +import warnings +import sys +import torch + +from . import utils +from .opt import OptimWrapper +from .scaler import LossScaler +from ._amp_state import _amp_state, master_params, maybe_print + +if torch.distributed.is_available(): + from ..parallel.LARC import LARC + + +# There's no reason to expose the notion of a "handle". Everything can happen through amp.* calls. +@contextlib.contextmanager +def scale_loss(loss, + optimizers, + loss_id=0, + model=None, + delay_unscale=False, + delay_overflow_check=False): + """ + On context manager entrance, creates ``scaled_loss = (loss.float())*current loss scale``. + ``scaled_loss`` is yielded so that the user can call ``scaled_loss.backward()``:: + + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + On context manager exit (if ``delay_unscale=False``), the gradients are checked for infs/NaNs + and unscaled, so that ``optimizer.step()`` can be called. + + .. note:: + If Amp is using explicit FP32 master params (which is the default for ``opt_level=O2``, and + can also be manually enabled by supplying ``master_weights=True`` to ``amp.initialize``) + any FP16 gradients are copied to FP32 master gradients before being unscaled. + ``optimizer.step()`` will then apply the unscaled master gradients to the master params. + + .. warning:: + If Amp is using explicit FP32 master params, only the FP32 master gradients will be + unscaled. The direct ``.grad`` attributes of any FP16 + model params will remain scaled after context manager exit. + This subtlety affects gradient clipping. See "Gradient clipping" under + `Advanced Amp Usage`_ for best practices. + + Args: + loss(Tensor): Typically a scalar Tensor. The ``scaled_loss`` that the context + manager yields is simply ``loss.float()*loss_scale``, so in principle + ``loss`` could have more than one element, as long as you call + ``backward()`` on ``scaled_loss`` appropriately within the context manager body. + optimizers: All optimizer(s) for which the current backward pass is creating gradients. + Must be an optimizer or list of optimizers returned from an earlier call + to ``amp.initialize``. For example use with multiple optimizers, see + "Multiple models/optimizers/losses" under `Advanced Amp Usage`_. + loss_id(int, optional, default=0): When used in conjunction with the ``num_losses`` argument + to ``amp.initialize``, enables Amp to use a different loss scale per loss. ``loss_id`` + must be an integer between 0 and ``num_losses`` that tells Amp which loss is + being used for the current backward pass. See "Multiple models/optimizers/losses" + under `Advanced Amp Usage`_ for examples. If ``loss_id`` is left unspecified, Amp + will use the default global loss scaler for this backward pass. + model(torch.nn.Module, optional, default=None): Currently unused, reserved to enable future + optimizations. + delay_unscale(bool, optional, default=False): ``delay_unscale`` is never necessary, and + the default value of ``False`` is strongly recommended. + If ``True``, Amp will not unscale the gradients or perform model->master + gradient copies on context manager exit. + ``delay_unscale=True`` is a minor ninja performance optimization and can result + in weird gotchas (especially with multiple models/optimizers/losses), + so only use it if you know what you're doing. + "Gradient accumulation across iterations" under `Advanced Amp Usage`_ + illustrates a situation where this CAN (but does not need to) be used. + + .. warning:: + If ``delay_unscale`` is ``True`` for a given backward pass, ``optimizer.step()`` cannot be + called yet after context manager exit, and must wait for another, later backward context + manager invocation with ``delay_unscale`` left to False. + + .. _`Advanced Amp Usage`: + https://nvidia.github.io/apex/advanced.html + """ + if not hasattr(_amp_state, "opt_properties"): + raise RuntimeError("Invoked 'with amp.scale_loss`, but internal Amp state has not been initialized. " + "model, optimizer = amp.initialize(model, optimizer, opt_level=...) must be called " + "before `with amp.scale_loss`.") + + if not _amp_state.opt_properties.enabled: + yield loss + return + + if isinstance(optimizers, torch.optim.Optimizer) or ('LARC' in globals() and isinstance(optimizers, LARC)): + optimizers = [optimizers] + + loss_scaler = _amp_state.loss_scalers[loss_id] + loss_scale = loss_scaler.loss_scale() + + if ((not _amp_state.opt_properties.master_weights) + and (not loss_scaler.dynamic) + and loss_scale == 1.0): + yield loss.float() + # Needing to drop the cache here as well is an ugly gotcha. + # But for now I think it's necessary to short-circuit. + # Probably ok to skip this if not delay_unscale + if _amp_state.opt_properties.patch_torch_functions: + _amp_state.handle._clear_cache() + return + + if not delay_unscale: + if isinstance(optimizers, list): + for optimizer in optimizers: + if not optimizer._amp_stash.params_have_scaled_gradients: + optimizer._prepare_amp_backward() + + yield (loss.float())*loss_scale + + if delay_unscale: + for optimizer in optimizers: + optimizer._amp_stash.params_have_scaled_gradients = True + else: + # FusedSGD may take care of unscaling as part of their step() methods. + # if not isinstance(optimizers, FP16_Optimizer_for_fused): + loss_scaler.clear_overflow_state() + for optimizer in optimizers: + optimizer._post_amp_backward(loss_scaler) + optimizer._amp_stash.params_have_scaled_gradients = False + # For future fused optimizers that enable sync-free dynamic loss scaling, + # should_skip will always be False. + should_skip = False if delay_overflow_check else loss_scaler.update_scale() + if should_skip: + for optimizer in optimizers: + if not optimizer._amp_stash.already_patched: + # Close on loss_scaler and loss_id as well, to be safe. Probably not + # necessary because amp.scale_loss is already creating a temporary scope. + def patch_step(opt, loss_scaler, loss_id): + opt_step = opt.step + def skip_step(closure=None): + if closure is not None: + raise RuntimeError("Currently, Amp does not support closure use with optimizers.") + maybe_print(("Gradient overflow. Skipping step, loss scaler " + + "{} reducing loss scale to {}").format(loss_id, + loss_scaler.loss_scale())) + # TODO: I don't like the special casing for different optimizer implementations. + # Maybe skip should delegate to a method owned by the optimizers themselves. + if hasattr(opt._amp_stash, "all_fp32_from_fp16_params"): + # Clear the master grads that wouldn't be zeroed by model.zero_grad() + for param in opt._amp_stash.all_fp32_from_fp16_params: + param.grad = None + if hasattr(opt, "most_recent_scale"): + opt.most_recent_scale = 1.0 + opt.scale_set_by_backward = False + opt.step = opt_step + opt._amp_stash.already_patched = False + return skip_step + optimizer.step = patch_step(optimizer, loss_scaler, loss_id) + optimizer._amp_stash.already_patched = True + + # Probably ok to skip this if not delay_unscale + if _amp_state.opt_properties.patch_torch_functions: + _amp_state.handle._clear_cache() + + +# Free function version of AmpHandle.disable_casts, another step on the +# path to removing the concept of "AmpHandle" +@contextlib.contextmanager +def disable_casts(): + _amp_state.handle._is_active = False + yield + _amp_state.handle._is_active = True + + +class AmpHandle(object): + def __init__(self, loss_scale="dynamic", enable_caching=True, verbose=False): + self._enable_caching = enable_caching + self._verbose = verbose + self._cache = dict() + self._default_scaler = LossScaler(loss_scale) + self._is_active = True + self._all_wrappers = [] + + def is_active(self): + return self._is_active + + @contextlib.contextmanager + def _disable_casts(self): + self._is_active = False + yield + self._is_active = True + + def wrap_optimizer(self, optimizer, num_loss=1): + self._default_scaler = None + return OptimWrapper(optimizer, self, num_loss) + + @contextlib.contextmanager + def scale_loss(self, loss, optimizer): + raise RuntimeError("The old Amp API is no longer supported. Please move to the new API, " + "documented here: https://nvidia.github.io/apex/amp.html. Transition guide: " + "https://nvidia.github.io/apex/amp.html#transition-guide-for-old-api-users") + + if not self.is_active(): + yield loss + return + + if self._default_scaler is None: + raise RuntimeError( + 'After calling `handle.wrap_optimizer()`, you must explicitly ' + + 'use `optimizer.scale_loss(loss)`.') + + # TODO: this code block is duplicated here and `opt.py`. Unify. + loss_scale = self._default_scaler.loss_scale() + yield loss * loss_scale + + self._default_scaler.clear_overflow_state() + self._default_scaler.unscale( + master_params(optimizer), + master_params(optimizer), + loss_scale) + should_skip = self._default_scaler.update_scale() + if should_skip: + optimizer_step = optimizer.step + def skip_step(): + maybe_print('Gradient overflow, skipping update') + optimizer.step = optimizer_step + optimizer.step = skip_step + + self._clear_cache() + + def _clear_cache(self): + self._cache.clear() + + # Experimental support for saving / restoring uncasted versions of functions + def _save_func(self, mod, fn, func): + self._all_wrappers.append((mod, fn, func)) + + def _deactivate(self): + for mod, fn, func in self._all_wrappers: + utils.set_func(mod, fn, func) + self._all_wrappers = [] + + @property + def has_cache(self): + return self._enable_caching + + @property + def cache(self): + return self._cache + + def remove_cache(self, param): + if self.has_cache and param in self.cache: + del self.cache[param] + + @property + def verbose(self): + return self._verbose + +class NoOpHandle(object): + def is_active(self): + return False + + @contextlib.contextmanager + def _disable_casts(self): + yield + + def wrap_optimizer(self, optimizer, num_loss=1): + return OptimWrapper(optimizer, self, num_loss) + + @contextlib.contextmanager + def scale_loss(self, loss, optimizer): + yield loss + + @property + def has_cache(self): + return False + + @property + def verbose(self): + return False + + def _clear_cache(self): + pass + + def _deactivate(self): + pass diff --git a/apex/apex/amp/lists/__init__.py b/apex/apex/amp/lists/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/amp/lists/functional_overrides.py b/apex/apex/amp/lists/functional_overrides.py new file mode 100644 index 00000000..dd009cec --- /dev/null +++ b/apex/apex/amp/lists/functional_overrides.py @@ -0,0 +1,80 @@ + +# TODO: think about the following two. They do weird things. +# - torch.nn.utils.clip_grad (but it should always be fp32 anyway) +# - torch.nn.utils.weight_norm + +# Notes: +# F.instance_norm uses batch_norm internally. Which correctly handles +# fp16 in/out with fp32 weights. So we shouldn't do anything for +# either of these. +# F.normalize calls `input.norm()` internally, so it's redundant, but +# kept here in case impl. changes. +# F.cosine_similarity is same: calls `x.norm()` internally. + +import torch.nn.functional + +MODULE = torch.nn.functional + +FP16_FUNCS = [ + 'conv1d', + 'conv2d', + 'conv3d', + 'conv_transpose1d', + 'conv_transpose2d', + 'conv_transpose3d', + 'conv_tbc', # Undocumented / maybe new? + 'linear', +] + +FP32_FUNCS = [ + + # Interpolation/Upsampling TODO: Remove for 1.2 + 'interpolate', + 'grid_sample', + + # Pointwise + 'softplus', + 'softmin', + 'log_softmax', + 'softmax', + 'gelu', + + # Normalization + 'layer_norm', + 'group_norm', + 'local_response_norm', + 'normalize', + 'cosine_similarity', + + # Loss functions + # TODO: which of these can be fp16? + 'poisson_nll_loss', + 'cosine_embedding_loss', + 'cross_entropy', + 'hinge_embedding_loss', + 'kl_div', + 'l1_loss', + 'mse_loss', + 'margin_ranking_loss', + 'multilabel_margin_loss', + 'multilabel_soft_margin_loss', + 'multi_margin_loss', + 'nll_loss', + 'binary_cross_entropy_with_logits', + 'smooth_l1_loss', + 'soft_margin_loss', + 'triplet_margin_loss', + 'ctc_loss' +] + +BANNED_FUNCS = [ + ('binary_cross_entropy', + ("\namp does not work out-of-the-box with `F.binary_cross_entropy` or `torch.nn.BCELoss.` " + "It requires that the output of the previous function be already a FloatTensor. \n\n" + "Most models have a Sigmoid right before BCELoss. In that case, you can use\n" + " torch.nn.BCEWithLogitsLoss\nto combine Sigmoid+BCELoss into a single layer " + "that is compatible with amp.\nAnother option is to add\n" + " amp.register_float_function(torch, 'sigmoid')\nbefore calling `amp.init()`.\n" + "If you _really_ know what you are doing, you can disable this warning by passing " + "allow_banned=True to `amp.init()`.")) +] diff --git a/apex/apex/amp/lists/tensor_overrides.py b/apex/apex/amp/lists/tensor_overrides.py new file mode 100644 index 00000000..18f3e5dc --- /dev/null +++ b/apex/apex/amp/lists/tensor_overrides.py @@ -0,0 +1,63 @@ +from .. import compat +from . import torch_overrides + +import importlib + +import torch + +# if compat.variable_is_tensor() and not compat.tensor_is_variable(): +MODULE = torch.Tensor +# else: +# MODULE = torch.autograd.Variable + + +FP16_FUNCS = compat.filter_attrs(MODULE, [ + '__matmul__', +]) + +FP32_FUNCS = compat.filter_attrs(MODULE, [ + '__ipow__', + '__pow__', + '__rpow__', + + # Cast to fp32 before transfer to CPU + 'cpu', +]) + +CASTS = compat.filter_attrs(MODULE, [ + '__add__', + '__div__', + '__eq__', + '__ge__', + '__gt__', + '__iadd__', + '__idiv__', + '__imul__', + '__isub__', + '__itruediv__', + '__le__', + '__lt__', + '__mul__', + '__ne__', + '__radd__', + '__rdiv__', + '__rmul__', + '__rsub__', + '__rtruediv__', + '__sub__', + '__truediv__', +]) + +# None of these, but here to make code cleaner. +SEQUENCE_CASTS = [] + +# We need to grab all the methods from torch_overrides and add them to +# the Tensor lists as well, as almost all methods are duplicated +# between `torch` and `torch.Tensor` (and check with `hasattr`, +# because a few random ones aren't defined on Tensor) +_self_mod = importlib.import_module(__name__) +for attrname in ['FP16_FUNCS', 'FP32_FUNCS', 'CASTS', 'SEQUENCE_CASTS']: + lst = getattr(_self_mod, attrname) + for fn in getattr(torch_overrides, attrname): + if hasattr(MODULE, fn): + lst.append(fn) diff --git a/apex/apex/amp/lists/torch_overrides.py b/apex/apex/amp/lists/torch_overrides.py new file mode 100644 index 00000000..7dedb05a --- /dev/null +++ b/apex/apex/amp/lists/torch_overrides.py @@ -0,0 +1,115 @@ +import torch + +from .. import utils + +MODULE = torch + +FP16_FUNCS = [ + # Low level functions wrapped by torch.nn layers. + # The wrapper layers contain the weights which are then passed in as a parameter + # to these functions. + 'conv1d', + 'conv2d', + 'conv3d', + 'conv_transpose1d', + 'conv_transpose2d', + 'conv_transpose3d', + 'conv_tbc', + 'prelu', + + # BLAS + 'addmm', + 'addmv', + 'addr', + 'matmul', + 'mm', + 'mv', +] + +FP32_FUNCS = [ + # Pointwise + 'acos', + 'asin', + 'cosh', + 'erfinv', + 'exp', + 'expm1', + 'log', + 'log10', + 'log2', + 'reciprocal', + 'rsqrt', + 'sinh', + 'tan', + + # Other math + 'pow', + + # Reduction + 'cumprod', + 'cumsum', + 'dist', + # 'mean', + 'norm', + 'prod', + 'std', + 'sum', + 'var', + + # Misc + 'renorm' +] + +version_strings = torch.__version__.split('.') +version_major = version_strings[0] +version_minor = version_strings[1] +version_num = float(version_major + "." + version_minor) +# Before torch 1.1, mean must be blacklisted. +if version_num < 1.1: + FP32_FUNCS.append('mean') + +# Before CUDA 9.1, batched matmul was missing fast FP16 kernels. We +# check the CUDA version -- if at least 9.1, then put the bmm +# functions on the fp16 list. Otherwise, put them on the fp32 list. +_bmms = ['addbmm', + 'baddbmm', + 'bmm'] + +if utils.is_cuda_enabled(): + # workaround https://github.com/facebookresearch/maskrcnn-benchmark/issues/802 + if utils.get_cuda_version() >= (9, 1, 0): + FP16_FUNCS.extend(_bmms) + else: + FP32_FUNCS.extend(_bmms) + +# Multi-tensor fns that may need type promotion +CASTS = [ + # Multi-tensor math + 'addcdiv', + 'addcmul', + 'atan2', + 'cross', + 'bilinear', + 'dot', + + # Element-wise _or_ tensor-wise math + 'add', + 'div', + 'mul', + + # Comparison + 'eq', + 'equal', + 'ge', + 'gt', + 'le', + 'lt', + 'ne' +] + +# Functions that take sequence arguments. We need to inspect the whole +# sequence and cast to the widest type. +SEQUENCE_CASTS = [ + 'cat', + 'stack' +] diff --git a/apex/apex/amp/opt.py b/apex/apex/amp/opt.py new file mode 100644 index 00000000..baf31168 --- /dev/null +++ b/apex/apex/amp/opt.py @@ -0,0 +1,103 @@ +import contextlib +import warnings + +from .scaler import LossScaler, master_params +from ._amp_state import maybe_print + +import numpy as np + +class OptimWrapper(object): + def __init__(self, optimizer, amp_handle, num_loss): + self._optimizer = optimizer + self._amp_handle = amp_handle + self._num_loss = num_loss + self._loss_idx = 0 + self._skip_next = [False] * num_loss + self._loss_scaler = [LossScaler('dynamic') for _ in range(num_loss)] + + @contextlib.contextmanager + def scale_loss(self, loss): + if not self._amp_handle.is_active(): + yield loss + return + + # When there are multiple losses per-optimizer, we need + # to save out current grad accumulation, since we won't be + # able to unscale this particulare loss once the grads are + # all mixed together. + cached_grads = [] + if self._loss_idx > 0: + for p in master_params(self._optimizer): + if p.grad is not None: + cached_grads.append(p.grad.data.detach().clone()) + else: + cached_grads.append(None) + self._optimizer.zero_grad() + + loss_scale = self._cur_loss_scaler().loss_scale() + yield loss * loss_scale + + self._cur_loss_scaler().clear_overflow_state() + self._cur_loss_scaler().unscale( + master_params(self._optimizer), + master_params(self._optimizer), + loss_scale) + self._skip_next[self._loss_idx] = self._cur_loss_scaler().update_scale() + self._loss_idx += 1 + + if len(cached_grads) > 0: + for p, cached_grad in zip(master_params(self._optimizer), + cached_grads): + if cached_grad is not None: + p.grad.data.add_(cached_grad) + cached_grads = [] + + def _cur_loss_scaler(self): + assert 0 <= self._loss_idx < self._num_loss + return self._loss_scaler[self._loss_idx] + + def step(self, closure=None): + if not self._amp_handle.is_active(): + return self._optimizer.step(closure=closure) + + self._loss_idx = 0 + + for group in self._optimizer.param_groups: + for p in group['params']: + self._amp_handle.remove_cache(p) + + if closure is not None: + raise NotImplementedError( + 'The `closure` argument is unsupported by the amp ' + + 'optimizer wrapper.') + if any(self._skip_next): + maybe_print('Gradient overflow, skipping update') + self._skip_next = [False] * self._num_loss + else: + return self._optimizer.step(closure=closure) + + # Forward any attribute lookups + def __getattr__(self, attr): + return getattr(self._optimizer, attr) + + # Forward all torch.optim.Optimizer methods + def __getstate__(self): + return self._optimizer.__getstate__() + + def __setstate__(self): + return self._optimizer.__setstate__() + + def __repr__(self): + return self._optimizer.__repr__() + + def state_dict(self): + return self._optimizer.state_dict() + + def load_state_dict(self, state_dict): + return self._optimizer.load_state_dict(state_dict) + + def zero_grad(self): + return self._optimizer.zero_grad() + + def add_param_group(self, param_group): + return self._optimizer.add_param_group(param_group) diff --git a/apex/apex/amp/rnn_compat.py b/apex/apex/amp/rnn_compat.py new file mode 100644 index 00000000..d062ae26 --- /dev/null +++ b/apex/apex/amp/rnn_compat.py @@ -0,0 +1,53 @@ +from . import utils, wrap + +import torch +_VF = torch._C._VariableFunctions +RNN_NAMES = ['rnn_relu', 'rnn_tanh', 'gru', 'lstm'] + +def _gen_VF_wrapper(name): + def wrapper(*args, **kwargs): + return getattr(_VF, name)(*args, **kwargs) + return wrapper + +# Some python magic to generate an object that has the rnn cell functions +# defined on it, all of which call into corresponding _VF version. +# Intended to patch torch.nn.modules.rnn._VF (aka, the ref named "_VF" +# imported at module scope within torch.nn.modules.rnn). This should +# not affect third-party importers of _VF.py. +class VariableFunctionsShim(object): + def __init__(self): + for name in RNN_NAMES: + for suffix in ['', '_cell']: + fn_name = name + suffix + setattr(self, fn_name, _gen_VF_wrapper(fn_name)) + +def has_old_rnns(): + try: + torch.nn.backends.thnn.backend.LSTMCell + return True + except: + return False + +def whitelist_rnn_cells(handle, verbose): + # Different module + function names in old/new RNN cases + if has_old_rnns(): + fn_names = ['RNNReLUCell', 'RNNTanhCell', 'LSTMCell', 'GRUCell'] + mod = torch.nn.backends.thnn.backend + else: + fn_names = [x + '_cell' for x in RNN_NAMES] + mod = torch.nn.modules.rnn._VF + assert isinstance(mod, VariableFunctionsShim) + + # Insert casts on cell functions + for fn in fn_names: + wrap.cached_cast(mod, fn, utils.maybe_half, handle, + try_caching=True, verbose=verbose) + + if has_old_rnns(): + # Special handling of `backward` for fused gru / lstm: + # The `backward` method calls Tensor.sum() (blacklist) internally, + # and then the resulting grad_input has the wrong type. + # TODO: where else is this a problem? + for rnn_type in ['GRUFused', 'LSTMFused']: + mod = getattr(torch.nn._functions.thnn.rnnFusedPointwise, rnn_type) + wrap.disable_casts(mod, 'backward', handle) diff --git a/apex/apex/amp/scaler.py b/apex/apex/amp/scaler.py new file mode 100644 index 00000000..99888bc6 --- /dev/null +++ b/apex/apex/amp/scaler.py @@ -0,0 +1,217 @@ +import torch +from ..multi_tensor_apply import multi_tensor_applier +from ._amp_state import _amp_state, master_params, maybe_print +from itertools import product + +def scale_check_overflow_python(model_grad, master_grad, scale, check_overflow=False): + # Exception handling for 18.04 compatibility + if check_overflow: + cpu_sum = float(model_grad.float().sum()) + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + + if master_grad is not model_grad: # copy_ probably internally short-circuits this + master_grad.copy_(model_grad) + if scale != 1.0: + master_grad.mul_(scale) + return False + +def axpby_check_overflow_python(model_grad, stashed_grad, master_grad, a, b, check_overflow=False): + # Exception handling for 18.04 compatibility + if check_overflow: + cpu_sum = float(model_grad.float().sum()) + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + + # if master_grad is not model_grad: # copy_ probably internally short-circuits this + # master_grad.copy_(model_grad) + assert stashed_grad.dtype == master_grad.dtype + converted_model_grad = model_grad.data.to(master_grad.dtype) + master_grad.data = a*converted_model_grad.data + b*stashed_grad.data + return False + +class LossScaler(object): + warned_no_fused_kernel = False + warned_unscaling_non_fp32_grad = False + has_fused_kernel = False + + def __init__(self, + loss_scale, + init_scale=2.**16, + scale_factor=2., + scale_window=2000, + min_loss_scale=None, + max_loss_scale=2.**24): + if loss_scale == "dynamic": + self.dynamic = True + self._loss_scale = min(max_loss_scale, init_scale) + else: + self.dynamic = False + self._loss_scale = loss_scale + self._max_loss_scale = max_loss_scale + self._min_loss_scale = min_loss_scale + self._scale_seq_len = scale_window + self._unskipped = 0 + self._has_overflow = False + self._overflow_buf = torch.cuda.IntTensor([0]) + if multi_tensor_applier.available: + import amp_C + LossScaler.has_fused_kernel = multi_tensor_applier.available + LossScaler.multi_tensor_scale_cuda = amp_C.multi_tensor_scale + LossScaler.multi_tensor_axpby_cuda = amp_C.multi_tensor_axpby + else: + if not LossScaler.warned_no_fused_kernel: + maybe_print( + "Warning: multi_tensor_applier fused unscale kernel is unavailable, " + "possibly because apex was installed without --cuda_ext --cpp_ext. " + "Using Python fallback. Original ImportError was: " + + repr(multi_tensor_applier.import_err), + True) + LossScaler.has_fused_kernel = False + LossScaler.warned_no_fused_kernel = True + + def loss_scale(self): + return self._loss_scale + + def unscale_python(self, model_grads, master_grads, scale): + for model, master in zip(model_grads, master_grads): + if model is not None: + if not LossScaler.warned_unscaling_non_fp32_grad: + if master.dtype != torch.float32: + maybe_print( + "Attempting to unscale a grad with type {} ".format(master.type()) + + "Unscaling non-fp32 grads may indicate an error. " + "When using Amp, you don't need to call .half() on your model.") + LossScaler.warned_unscaling_non_fp32_grad = True + self._has_overflow = scale_check_overflow_python(model, + master, + 1./scale, + self.dynamic) + if self._has_overflow and self.dynamic: + break + + # unused_scale keeps some of the old API alive for hopefully a short time. + def unscale(self, model_grads, master_grads, unused_scale, models_are_masters=False, scale_override=None): + if self._has_overflow: + return + + scale = self._loss_scale + if scale_override is not None: + scale = scale_override + + if scale == 1.0 and models_are_masters and not self.dynamic: + return + + if LossScaler.has_fused_kernel: + # if (not LossScaler.warned_unscaling_non_fp32_grad + # and master_grads[0].dtype == torch.float16): + # print("Warning: unscaling grads that are not FP32. " + # "Unscaling non-fp32 grads may indicate an error. " + # "When using Amp, you don't need to call .half() on your model.") + # # Setting this to True unconditionally allows the possibility of an escape + # # if never-before-seen non-fp32 grads are created in some later iteration. + # LossScaler.warned_unscaling_non_fp32_grad = True + multi_tensor_applier(LossScaler.multi_tensor_scale_cuda, + self._overflow_buf, + [model_grads, master_grads], + 1./scale) + else: + self.unscale_python(model_grads, master_grads, scale) + + # Defer to update_scale + # If the fused kernel is available, we only need one D2H memcopy and sync. + # if LossScaler.has_fused_kernel and self.dynamic and not self._has_overflow: + # self._has_overflow = self._overflow_buf.item() + + def unscale_with_stashed_python(self, + model_grads, + stashed_master_grads, + master_grads, + a, + b): + for model, stashed, master in zip(model_grads, stashed_master_grads, master_grads): + if model is None and stashed is None: + continue + else: + if not LossScaler.warned_unscaling_non_fp32_grad: + if master.dtype != torch.float32: + maybe_print( + "Attempting to unscale a grad with type {} ".format(master.type()) + + "Unscaling non-fp32 grads may indicate an error. " + "When using Amp, you don't need to call .half() on your model.") + LossScaler.warned_unscaling_non_fp32_grad = True + self._has_overflow = axpby_check_overflow_python(model, + stashed, + master, + a, + b, + self.dynamic) + if self._has_overflow and self.dynamic: + break + + def unscale_with_stashed(self, + model_grads, + stashed_master_grads, + master_grads, + scale_override=None): + if self._has_overflow: + return + + grads_have_scale, stashed_have_scale, out_scale = self._loss_scale, 1.0, 1.0 + if scale_override is not None: + grads_have_scale, stashed_have_scale, out_scale = scale_override + + if LossScaler.has_fused_kernel: + if (not LossScaler.warned_unscaling_non_fp32_grad + and master_grads[0].dtype == torch.float16): + print("Warning: unscaling grads that are not FP32. " + "Unscaling non-fp32 grads may indicate an error. " + "When using Amp, you don't need to call .half() on your model.") + # Setting this to True unconditionally allows the possibility of an escape + # if never-before-seen non-fp32 grads are created in some later iteration. + LossScaler.warned_unscaling_non_fp32_grad = True + multi_tensor_applier(LossScaler.multi_tensor_axpby_cuda, + self._overflow_buf, + [model_grads, stashed_master_grads, master_grads], + out_scale/grads_have_scale, # 1./scale, + out_scale/stashed_have_scale, # 1.0, + 0) # check only arg 0, aka the incoming model grads, for infs + else: + self.unscale_with_stashed_python(model_grads, + stashed_master_grads, + master_grads, + out_scale/grads_have_scale, + out_scale/stashed_have_scale) + + # Defer to update_scale + # If the fused kernel is available, we only need one D2H memcopy and sync. + # if LossScaler.has_fused_kernel and self.dynamic and not self._has_overflow: + # self._has_overflow = self._overflow_buf.item() + + def clear_overflow_state(self): + self._has_overflow = False + if self.has_fused_kernel: + self._overflow_buf.zero_() + + # Separate so unscale() can be called more that once before updating. + def update_scale(self): + # If the fused kernel is available, we only need one D2H memcopy and sync. + if LossScaler.has_fused_kernel and self.dynamic and not self._has_overflow: + self._has_overflow = self._overflow_buf.item() + + if self._has_overflow and self.dynamic: + should_skip = True + if(self._min_loss_scale): + self._loss_scale = max(self._min_loss_scale, self._loss_scale/2.) + else: + self._loss_scale = self._loss_scale/2. + self._unskipped = 0 + else: + should_skip = False + self._unskipped += 1 + + if self._unskipped == self._scale_seq_len and self.dynamic: + self._loss_scale = min(self._max_loss_scale, self._loss_scale*2.) + self._unskipped = 0 + + return should_skip diff --git a/apex/apex/amp/utils.py b/apex/apex/amp/utils.py new file mode 100644 index 00000000..0590cd70 --- /dev/null +++ b/apex/apex/amp/utils.py @@ -0,0 +1,210 @@ +from . import compat + +import functools +import itertools + +import torch + +def is_cuda_enabled(): + return torch.version.cuda is not None + +def get_cuda_version(): + return tuple(int(x) for x in torch.version.cuda.split('.')) + +def is_fp_tensor(x): + if is_nested(x): + # Fast-fail version of all(is_fp_tensor) + for y in x: + if not is_fp_tensor(y): + return False + return True + return compat.is_tensor_like(x) and compat.is_floating_point(x) + +def is_nested(x): + return isinstance(x, tuple) or isinstance(x, list) + +def should_cache(x): + if is_nested(x): + # Fast-fail version of all(should_cache) + for y in x: + if not should_cache(y): + return False + return True + return isinstance(x, torch.nn.parameter.Parameter) and \ + type_string(x) == 'FloatTensor' + +def collect_fp_tensor_types(args, kwargs): + def collect_types(x, types): + if is_nested(x): + for y in x: + collect_types(y, types) + else: + types.add(type_string(x)) + + all_args = itertools.chain(args, kwargs.values()) + types = set() + for x in all_args: + if is_fp_tensor(x): + collect_types(x, types) + return types + +def type_string(x): + return x.type().split('.')[-1] + +def maybe_half(x, name='', verbose=False): + if is_nested(x): + return type(x)([maybe_half(y) for y in x]) + + if not x.is_cuda or type_string(x) == 'HalfTensor': + return x + else: + if verbose: + print('Float->Half ({})'.format(name)) + return x.half() + +def maybe_float(x, name='', verbose=False): + if is_nested(x): + return type(x)([maybe_float(y) for y in x]) + + if not x.is_cuda or type_string(x) == 'FloatTensor': + return x + else: + if verbose: + print('Half->Float ({})'.format(name)) + return x.float() + +# NB: returneds casted `args`, mutates `kwargs` in-place +def casted_args(cast_fn, args, kwargs): + new_args = [] + for x in args: + if is_fp_tensor(x): + new_args.append(cast_fn(x)) + else: + new_args.append(x) + for k in kwargs: + val = kwargs[k] + if is_fp_tensor(val): + kwargs[k] = cast_fn(val) + return new_args + +def cached_cast(cast_fn, x, cache): + if is_nested(x): + return type(x)([cached_cast(y) for y in x]) + if x in cache: + cached_x = cache[x] + if x.requires_grad and cached_x.requires_grad: + # Make sure x is actually cached_x's autograd parent. + if cached_x.grad_fn.next_functions[1][0].variable is not x: + raise RuntimeError("x and cache[x] both require grad, but x is not " + "cache[x]'s parent. This is likely an error.") + # During eval, it's possible to end up caching casted weights with + # requires_grad=False. On the next training iter, if cached_x is found + # and reused from the cache, it will not actually have x as its parent. + # Therefore, we choose to invalidate the cache (and force refreshing the cast) + # if x.requires_grad and cached_x.requires_grad do not match. + # + # During eval (i.e. running under with torch.no_grad()) the invalidation + # check would cause the cached value to be dropped every time, because + # cached_x would always be created with requires_grad=False, while x would + # still have requires_grad=True. This would render the cache effectively + # useless during eval. Therefore, if we are running under the no_grad() + # context manager (torch.is_grad_enabled=False) we elide the invalidation + # check, and use the cached value even though its requires_grad flag doesn't + # match. During eval, we don't care that there's no autograd-graph + # connection between x and cached_x. + if torch.is_grad_enabled() and x.requires_grad != cached_x.requires_grad: + del cache[x] + else: + return cached_x + + casted_x = cast_fn(x) + cache[x] = casted_x + return casted_x + +def verbosify(cast_fn, fn_name, verbose): + if verbose: + return functools.partial(cast_fn, name=fn_name, verbose=verbose) + else: + return cast_fn + +def as_inplace(fns): + for x in fns: + yield x + '_' + +def has_func(mod, fn): + if isinstance(mod, dict): + return fn in mod + else: + return hasattr(mod, fn) + +def get_func(mod, fn): + if isinstance(mod, dict): + return mod[fn] + else: + return getattr(mod, fn) + +def set_func(mod, fn, new_fn): + if isinstance(mod, dict): + mod[fn] = new_fn + else: + setattr(mod, fn, new_fn) + +def set_func_save(handle, mod, fn, new_fn): + cur_fn = get_func(mod, fn) + handle._save_func(mod, fn, cur_fn) + set_func(mod, fn, new_fn) + +# A couple problems get solved here: +# - The flat_weight buffer is disconnected from autograd graph, +# so the fp16 weights need to be derived from the input weights +# to this forward call, not the flat buffer. +# - The ordering of weights in the flat buffer is...idiosyncratic. +# First problem is solved with combination of set_ (to set up +# correct storage) and copy_ (so the fp16 weight derives from the +# fp32 one in autograd. +# Second is solved by doing ptr arithmetic on the fp32 weights +# to derive the correct offset. +# +# TODO: maybe this should actually use +# `torch._cudnn_rnn_flatten_weight`? But then I need to call +# on first iter and cache the right offsets. Ugh. +def synthesize_flattened_rnn_weights(fp32_weights, + fp16_flat_tensor, + rnn_fn='', + verbose=False): + fp16_weights = [] + fp32_base_ptr = fp32_weights[0][0].data_ptr() + for layer_weights in fp32_weights: + fp16_layer_weights = [] + for w_fp32 in layer_weights: + w_fp16 = w_fp32.new().half() + offset = (w_fp32.data_ptr() - fp32_base_ptr) // w_fp32.element_size() + w_fp16.set_(fp16_flat_tensor.storage(), + offset, + w_fp32.shape) + w_fp16.copy_(w_fp32) + if verbose: + print('Float->Half ({})'.format(rnn_fn)) + fp16_layer_weights.append(w_fp16) + fp16_weights.append(fp16_layer_weights) + return fp16_weights + +# Roughly same as above, just the `fp32_weights` aren't nested. +# Code kept separate for readability. +def new_synthesize_flattened_rnn_weights(fp32_weights, + fp16_flat_tensor, + rnn_fn='', + verbose=False): + fp16_weights = [] + fp32_base_ptr = fp32_weights[0].data_ptr() + for w_fp32 in fp32_weights: + w_fp16 = w_fp32.new().half() + offset = (w_fp32.data_ptr() - fp32_base_ptr) // w_fp32.element_size() + w_fp16.set_(fp16_flat_tensor.storage(), + offset, + w_fp32.shape) + w_fp16.copy_(w_fp32) + if verbose: + print('Float->Half ({})'.format(rnn_fn)) + fp16_weights.append(w_fp16) + return fp16_weights diff --git a/apex/apex/amp/wrap.py b/apex/apex/amp/wrap.py new file mode 100644 index 00000000..559d0558 --- /dev/null +++ b/apex/apex/amp/wrap.py @@ -0,0 +1,276 @@ +from . import compat +from . import utils +from ._amp_state import _amp_state +from . import rnn_compat + +import functools + +import torch + +def make_cast_wrapper(orig_fn, cast_fn, handle, + try_caching=False): + @functools.wraps(orig_fn) + def wrapper(*args, **kwargs): + if not handle.is_active(): + return orig_fn(*args, **kwargs) + + if try_caching and handle.has_cache: + args = list(args) + for i in range(len(args)): + if utils.should_cache(args[i]): + args[i] = utils.cached_cast(cast_fn, args[i], handle.cache) + for k in kwargs: + if utils.should_cache(kwargs[k]): + kwargs[k] = utils.cached_cast(cast_fn, kwargs[k], handle.cache) + new_args = utils.casted_args(cast_fn, + args, + kwargs) + return orig_fn(*new_args, **kwargs) + return wrapper + +def cached_cast(mod, fn, cast_fn, handle, + try_caching=False, verbose=False): + if not utils.has_func(mod, fn): + return + + orig_fn = utils.get_func(mod, fn) + cast_fn = utils.verbosify(cast_fn, fn, verbose) + wrapper = make_cast_wrapper(orig_fn, cast_fn, handle, try_caching) + utils.set_func_save(handle, mod, fn, wrapper) + +# `handle` arg is unused, but simplifies API to make `make_cast_wrapper` +# Annoyingly, make_promote_wrapper still uses the global handle. Once everyone +# is on the new API and I am free to get rid of handle, I can clean this up. +def make_promote_wrapper(orig_fn, cast_fn, handle=None): + @functools.wraps(orig_fn) + def wrapper(*args, **kwargs): + if not _amp_state.handle.is_active(): + return orig_fn(*args, **kwargs) + + types = utils.collect_fp_tensor_types(args, kwargs) + + if len(types) <= 1: + return orig_fn(*args, **kwargs) + elif len(types) == 2 and types == set(['HalfTensor', 'FloatTensor']): + new_args = utils.casted_args(cast_fn, + args, + kwargs) + return orig_fn(*new_args, **kwargs) + else: + raise NotImplementedError('Do not know how to handle ' + + 'these types to promote: {}' + .format(types)) + return wrapper + +def promote(mod, fn, handle, verbose=False): + orig_fn = utils.get_func(mod, fn) + maybe_float = utils.verbosify(utils.maybe_float, fn, verbose) + wrapper = make_promote_wrapper(orig_fn, maybe_float) + utils.set_func_save(handle, mod, fn, wrapper) + +def sequence_promote(mod, fn, handle, verbose=False): + orig_fn = utils.get_func(mod, fn) + maybe_float = utils.verbosify(utils.maybe_float, fn, verbose) + @functools.wraps(orig_fn) + def wrapper(seq, *args, **kwargs): + if not _amp_state.handle.is_active(): + return orig_fn(seq, *args, **kwargs) + + types = set([utils.type_string(x) for x in seq]) + if len(types) <= 1: + return orig_fn(seq, *args, **kwargs) + elif types == set(['HalfTensor', 'FloatTensor']): + cast_seq = utils.casted_args(maybe_float, + seq, {}) + return orig_fn(cast_seq, *args, **kwargs) + else: + # TODO: other mixed-type cases aren't due to amp. + # Just pass through? + return orig_fn(seq, *args, **kwargs) + utils.set_func_save(handle, mod, fn, wrapper) + +def promote_match_arg0(mod, fn, handle, verbose=False): + if not utils.has_func(mod, fn): + return + + orig_fn = utils.get_func(mod, fn) + @functools.wraps(orig_fn) + def wrapper(arg0, *args, **kwargs): + assert compat.is_tensor_like(arg0) + if not _amp_state.handle.is_active(): + return orig_fn(arg0, *args, **kwargs) + + if utils.type_string(arg0) == 'HalfTensor': + cast_fn = utils.maybe_half + elif utils.type_string(arg0) == 'FloatTensor': + cast_fn = utils.maybe_float + else: + return orig_fn(arg0, *args, **kwargs) + cast_fn = utils.verbosify(cast_fn, fn, verbose) + new_args = utils.casted_args(cast_fn, args, kwargs) + return orig_fn(arg0, *new_args, **kwargs) + utils.set_func_save(handle, mod, fn, wrapper) + +def err_if_any_half(mod, fn, handle, custom_err_msg=None): + if not utils.has_func(mod, fn): + return + + orig_fn = utils.get_func(mod, fn) + @functools.wraps(orig_fn) + def wrapper(*args, **kwargs): + types = utils.collect_fp_tensor_types(args, kwargs) + if 'HalfTensor' in types: + if custom_err_msg: + raise NotImplementedError(custom_err_msg) + else: + raise NotImplementedError('Cannot call in-place function ' + + '{} with fp16 arguments.'.format(fn)) + else: + return orig_fn(*args, **kwargs) + utils.set_func_save(handle, mod, fn, wrapper) + +def err_if_arg0_half(mod, fn, handle, verbose=False): + if not utils.has_func(mod, fn): + return + + orig_fn = utils.get_func(mod, fn) + @functools.wraps(orig_fn) + def wrapper(arg0, *args, **kwargs): + assert compat.is_tensor_like(arg0) + if utils.type_string(arg0) == 'HalfTensor': + raise NotImplementedError('Cannot call in-place method ' + + '{} on fp16 Tensors.'.format(fn)) + else: + cast_fn = utils.verbosify(utils.maybe_float, fn, verbose) + new_args = utils.casted_args(cast_fn, args, kwargs) + return orig_fn(arg0, *new_args, **kwargs) + utils.set_func_save(handle, mod, fn, wrapper) + +# Current RNN approach: +# - Wrap top-level `RNN` function in thnn backend +# - Will call into either CudnnRNN or AutogradRNN +# - Each of these are factory functions that return a per-iter +# `forward` function +# - We interpose on the factory function to: +# 1) Interpose on the actual forward function and put in casts +# 2) Insert an fp16 `flat_weight` if necessary +def rnn_cast(backend, fn, handle, verbose=False): + orig_rnn = utils.get_func(backend, fn) + @functools.wraps(orig_rnn) + def rnn_wrapper(*args, **kwargs): + flat_weight = kwargs.get('flat_weight') + if flat_weight is not None: + # We replace `flat_weight` with an uninitialized fp16 + # Tensor. The "actual" weight tensors (provided in `forward`), + # will then be set up as ptrs into the buffer and have the + # corresponding fp32 values copied in. + # We need to call `copy` on the "actual" weights so that the + # autograd graph correctly backprops from the wgrads computed + # inside cuDNN (on fp16 weights) into the fp32 weights. + assert utils.type_string(flat_weight) == 'FloatTensor' + if compat.tensor_is_float_tensor() or compat.tensor_is_variable(): + # Pre-0.4. A little slower, since it zeros out memory. + flat_weight_fp16 = flat_weight.new().half().resize_(flat_weight.shape) + else: + flat_weight_fp16 = torch.empty_like(flat_weight, + dtype=torch.float16) + kwargs['flat_weight'] = flat_weight_fp16 + else: + flat_weight_fp16 = None + + forward = orig_rnn(*args, **kwargs) + @functools.wraps(forward) + def fwd_wrapper(*fargs, **fkwargs): + assert len(fargs) == 3 or len(fargs) == 4 + inputs, weights, hiddens = fargs[:3] + assert utils.is_fp_tensor(inputs) + assert isinstance(weights, list) + cast_fn = utils.verbosify(utils.maybe_half, + fn, + verbose) + new_args = [] + + # 0) Inputs + new_args.append(cast_fn(inputs)) + + # 1) Weights + if flat_weight_fp16 is not None: + fp16_weights = utils.synthesize_flattened_rnn_weights( + weights, flat_weight_fp16, fn, verbose) + else: + fp16_weights = [[cast_fn(w) for w in layer] + for layer in weights] + new_args.append(fp16_weights) + + # 2) Inputs: either a tuple (for LSTM) or single tensor + if isinstance(hiddens, tuple): + new_args.append(tuple(cast_fn(x) for x in hiddens)) + elif utils.is_fp_tensor(hiddens): + new_args.append(cast_fn(hiddens)) + else: + # Hiddens can, in principle, be `None` -- pass through + new_args.append(hiddens) + + # 3) Batch sizes (0.4 or later only) + if len(fargs) == 4: + new_args.append(fargs[3]) + + return forward(*new_args, **fkwargs) + return fwd_wrapper + utils.set_func_save(handle, backend, fn, rnn_wrapper) + +def new_rnn_cast(fn, handle, verbose=False): + # Forward+backward compatibility around https://github.com/pytorch/pytorch/pull/15744 + # For rnn backend calls that route through _rnn_impls, we must patch the ref + # that _rnn_impls stashed. For rnn backend calls that directly invoke + # _VF., e.g. _VF.lstm, we can patch onto VariableFunctionsShim, + # which in turn has patched the ref named "_VF" in torch.nn.modules.rnn. + if utils.has_func(torch.nn.modules.rnn._rnn_impls, fn): + mod = torch.nn.modules.rnn._rnn_impls + else: + mod = torch.nn.modules.rnn._VF + assert isinstance(mod, rnn_compat.VariableFunctionsShim) + fn = fn.lower() + orig_fn = utils.get_func(mod, fn) + cast_fn = utils.verbosify(utils.maybe_half, fn, verbose) + @functools.wraps(orig_fn) + def wrapper(*args, **kwargs): + # Exact call signature from modules/rnn.py + assert len(args) == 9 + assert len(kwargs) == 0 + + if not _amp_state.handle.is_active(): + return orig_fn(*args, **kwargs) + + if isinstance(args[6], bool): + params_idx = 2 # Not PackedSequence case + else: + params_idx = 3 # PackedSequence case + + new_args = [] + for i, arg in enumerate(args): + if i == params_idx: + num_params = sum([x.numel() for x in arg]) + fp16_weight_buf = args[0].new_empty((num_params,), + dtype=torch.half) + casted_weights = utils.new_synthesize_flattened_rnn_weights( + arg, fp16_weight_buf, fn, verbose) + new_args.append(casted_weights) + elif utils.is_fp_tensor(arg): + new_args.append(cast_fn(arg)) + else: + new_args.append(arg) + + return orig_fn(*new_args) + utils.set_func_save(handle, mod, fn, wrapper) + +def disable_casts(mod, fn, handle): + if not utils.has_func(mod, fn): + return + + orig_fn = utils.get_func(mod, fn) + @functools.wraps(orig_fn) + def wrapper(*args, **kwargs): + with handle._disable_casts(): + return orig_fn(*args, **kwargs) + utils.set_func_save(handle, mod, fn, wrapper) diff --git a/apex/apex/contrib/__init__.py b/apex/apex/contrib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/bottleneck/__init__.py b/apex/apex/contrib/bottleneck/__init__.py new file mode 100644 index 00000000..300b7c36 --- /dev/null +++ b/apex/apex/contrib/bottleneck/__init__.py @@ -0,0 +1,2 @@ +from .bottleneck import Bottleneck, SpatialBottleneck +from .halo_exchangers import HaloExchangerNoComm, HaloExchangerAllGather, HaloExchangerSendRecv, HaloExchangerPeer diff --git a/apex/apex/contrib/bottleneck/bottleneck.py b/apex/apex/contrib/bottleneck/bottleneck.py new file mode 100644 index 00000000..5ea5694c --- /dev/null +++ b/apex/apex/contrib/bottleneck/bottleneck.py @@ -0,0 +1,749 @@ +import functools as func + +import torch +import torch.distributed as dist +from torch import nn + +from apex import check_cudnn_version_and_warn +import fast_bottleneck +import nccl_p2p_cuda as inc + + +assert check_cudnn_version_and_warn(__name__, 8400) + + +def kaiming_uniform_(tensor, a=0, mode='fan_in', nonlinearity='leaky_relu'): + weight_tensor_nchw = tensor + nn.init.kaiming_uniform_(weight_tensor_nchw, a=a, mode=mode, nonlinearity=nonlinearity) + +def compute_scale_bias_one(nhwc, weight, bias, running_mean, running_var, w_scale, w_bias): + scale = weight * running_var.rsqrt() + bias = bias - running_mean * scale + w_scale.copy_(scale) + w_bias.copy_(bias) + +def compute_scale_bias_method(nhwc, args): + for arg in args: + # arg is tuple of (weight, bias, running_mean, running_var, w_scale, w_bias) + compute_scale_bias_one(nhwc, *arg) + +class FrozenBatchNorm2d(torch.jit.ScriptModule): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed + """ + def __init__(self, n): + super(FrozenBatchNorm2d, self).__init__() + self.register_buffer("weight", torch.ones(n)) + self.register_buffer("bias", torch.zeros(n)) + self.register_buffer("running_mean", torch.zeros(n)) + self.register_buffer("running_var", torch.ones(n)) + + @torch.jit.script_method + def get_scale_bias(self, nhwc): + # type: (bool) -> List[torch.Tensor] + scale = self.weight * self.running_var.rsqrt() + bias = self.bias - self.running_mean * scale + if nhwc: + scale = scale.reshape(1, 1, 1, -1) + bias = bias.reshape(1, 1, 1, -1) + else: + scale = scale.reshape(1, -1, 1, 1) + bias = bias.reshape(1, -1, 1, 1) + return scale, bias + + @torch.jit.script_method + def forward(self, x): + scale, bias = self.get_scale_bias(False) + return x * scale + bias + +@torch.jit.script +def drelu_dscale1(grad_o, output, scale1): + relu_mask = (output>0) + dx_relu = relu_mask * grad_o + g1 = dx_relu * scale1 + return g1, dx_relu + +@torch.jit.script +def drelu_dscale2(grad_o, output, scale1, scale2): + relu_mask = (output>0) + dx_relu = relu_mask * grad_o + g1 = dx_relu * scale1 + g2 = dx_relu * scale2 + return g1, g2 + +class BottleneckFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, nhwc, stride_1x1, scale, bias, x, *conv): + # TODO: clean up order of tensors + args = [x, *conv[0:3], *scale[0:3], *bias[0:3]] + ctx.downsample = len(conv) > 3 + if ctx.downsample: + args.append(conv[3]) + args.append(scale[3]) + args.append(bias[3]) + + # weight buffers are always in nhwc while shape can be nhwc or channels_last + # here we pass in flag and let c++ handle it + # alternatively, we can put all sizes into a fixed format and pass it in + outputs = fast_bottleneck.forward(nhwc, stride_1x1, args) + ctx.save_for_backward(*(args+outputs)) + # save relu outputs for drelu + ctx.nhwc = nhwc + ctx.stride_1x1 = stride_1x1 + return outputs[2] + + # backward relu is not exposed, MUL with mask used now + # only support dgrad + @staticmethod + def backward(ctx, grad_o): + outputs = ctx.saved_tensors[-3:] + + if ctx.downsample: + grad_conv3, grad_conv4 = drelu_dscale2(grad_o, outputs[2], ctx.saved_tensors[6], ctx.saved_tensors[11]) + else: + grad_conv3, grad_conv4 = drelu_dscale1(grad_o, outputs[2], ctx.saved_tensors[6]) + + # create input vector for backward + t_list = [*ctx.saved_tensors[0:10]] + t_list.append(grad_conv3) + t_list.append(grad_conv4) + + # outputs used for wgrad and generating drelu mask + t_list.append(outputs[0]) + t_list.append(outputs[1]) + + # in case there is downsample + if ctx.downsample: + t_list.append(ctx.saved_tensors[10]) + + grads = fast_bottleneck.backward(ctx.nhwc, ctx.stride_1x1, t_list) + + return (None, None, None, None, *grads) + +bottleneck_function = BottleneckFunction.apply + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + +class Bottleneck(torch.nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + # here we put it at 1x1 + + def __init__(self, in_channels, bottleneck_channels, out_channels, stride=1, groups=1, + dilation=1, norm_func=None, use_cudnn=False, explicit_nhwc=False): + super(Bottleneck, self).__init__() + if groups != 1: + raise RuntimeError('Only support groups == 1') + if dilation != 1: + raise RuntimeError('Only support dilation == 1') + if norm_func == None: + norm_func = FrozenBatchNorm2d + else: + raise RuntimeError('Only support frozen BN now.') + + if stride != 1 or in_channels != out_channels: + self.downsample = nn.Sequential( + conv1x1(in_channels, out_channels, stride), + norm_func(out_channels), + ) + else: + self.downsample = None + + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(in_channels, bottleneck_channels, stride) + self.conv2 = conv3x3(bottleneck_channels, bottleneck_channels) + self.conv3 = conv1x1(bottleneck_channels, out_channels) + self.relu = nn.ReLU(inplace=True) + self.stride = stride + + self.bn1 = norm_func(bottleneck_channels) + self.bn2 = norm_func(bottleneck_channels) + self.bn3 = norm_func(out_channels) + self.w_scale = None + + self.use_cudnn = use_cudnn + + # setup conv weights + self.w_conv = [self.conv1.weight, self.conv2.weight, self.conv3.weight] + if self.downsample is not None: + self.w_conv.append(self.downsample[0].weight) + + # init weight in nchw format before possible transpose + for w in self.w_conv: + kaiming_uniform_(w, a=1) + + # TODO: prevent unsupported case usage + # support cases + # native cudnn + # normal yes no + # channel_last yes yes + # explicit_nhwc no yes + self.explicit_nhwc = explicit_nhwc + if self.explicit_nhwc: + for p in self.parameters(): + with torch.no_grad(): + p.data = p.data.permute(0,2,3,1).contiguous() + + return + + # Returns single callable that recomputes scale and bias for all frozen batch-norms. + # This method must be called before cuda graphing. + # The callable it returns can be called anytime. + # Calling this method will prevent these from being computed every forward call. + def get_scale_bias_callable(self): + self.w_scale, self.w_bias, args = [], [], [] + batch_norms = [self.bn1, self.bn2, self.bn3] + if self.downsample is not None: + batch_norms.append(self.downsample[1]) + for bn in batch_norms: + s = torch.empty_like(bn.weight) + b = torch.empty_like(s) + args.append( (bn.weight, bn.bias, bn.running_mean, bn.running_var, s, b) ) + if self.explicit_nhwc: + self.w_scale.append( s.reshape(1, 1, 1, -1) ) + self.w_bias.append( b.reshape(1, 1, 1, -1) ) + else: + self.w_scale.append( s.reshape(1, -1, 1, 1) ) + self.w_bias.append( b.reshape(1, -1, 1, 1) ) + return func.partial(compute_scale_bias_method, self.explicit_nhwc, args) + + def forward(self, x): + if self.use_cudnn: + if self.w_scale is None: + # calculate scale/bias from registered buffers + # TODO: make this better + s1, b1 = self.bn1.get_scale_bias(self.explicit_nhwc) + s2, b2 = self.bn2.get_scale_bias(self.explicit_nhwc) + s3, b3 = self.bn3.get_scale_bias(self.explicit_nhwc) + w_scale = [s1, s2, s3] + w_bias = [b1, b2, b3] + if self.downsample is not None: + s4, b4 = self.downsample[1].get_scale_bias(self.explicit_nhwc) + w_scale.append(s4) + w_bias.append(b4) + out = bottleneck_function(self.explicit_nhwc, self.stride, w_scale, w_bias, x, *self.w_conv) + else: + out = bottleneck_function(self.explicit_nhwc, self.stride, self.w_scale, self.w_bias, x, *self.w_conv) + return out + + if self.explicit_nhwc: + raise RuntimeError('explicit nhwc with native ops is not supported.') + + # fallback to native ops + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class SpatialBottleneckFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, spatial_group_size, spatial_group_rank, spatial_communicator, spatial_halo_exchanger, spatial_method, use_delay_kernel, explicit_nhwc, stride_1x1, scale, bias, thresholdTop, thresholdBottom, x, *conv): + if spatial_group_size > 1: + stream1 = spatial_halo_exchanger.stream1 + stream2 = spatial_halo_exchanger.stream2 + stream3 = spatial_halo_exchanger.stream3 + + # TODO: clean up order of tensors + args = [x, *conv[0:3], *scale[0:3], *bias[0:3]] + ctx.downsample = len(conv) > 3 + if ctx.downsample: + args.append(conv[3]) + args.append(scale[3]) + args.append(bias[3]) + + # weight buffers are always in explicit_nhwc while shape can be explicit_nhwc or channels_last + # here we pass in flag and let c++ handle it + # alternatively, we can put all sizes into a fixed format and pass it in + outputs = fast_bottleneck.forward_init(explicit_nhwc, stride_1x1, args) + fast_bottleneck.forward_out1(explicit_nhwc, stride_1x1, args, outputs) + + if spatial_group_size > 1: + out1 = outputs[0] + if explicit_nhwc: + N,Hs,W,C = list(out1.shape) + memory_format = torch.contiguous_format + out1_pad = torch.empty([N,Hs+2,W,C], dtype=out1.dtype, device='cuda') + else: + N,C,Hs,W = list(out1.shape) + memory_format = torch.channels_last if out1.is_contiguous(memory_format=torch.channels_last) else torch.contiguous_format + out1_pad = torch.empty([N,C,Hs+2,W], dtype=out1.dtype, device='cuda', memory_format=memory_format) + stream1.wait_stream(torch.cuda.current_stream()) + if spatial_method != 2: stream3.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream1): + if explicit_nhwc: + top_out1_halo = out1_pad[:,:1,:,:] + btm_out1_halo = out1_pad[:,Hs+1:Hs+2,:,:] + spatial_halo_exchanger.left_right_halo_exchange(out1[:,:1,:,:], out1[:,Hs-1:,:,:], top_out1_halo, btm_out1_halo) + else: + top_out1_halo = out1_pad[:,:,:1,:] + btm_out1_halo = out1_pad[:,:,Hs+1:Hs+2,:] + spatial_halo_exchanger.left_right_halo_exchange(out1[:,:,:1,:], out1[:,:,Hs-1:,:], top_out1_halo, btm_out1_halo) + if spatial_method == 1: + # overlap mid convolution with halo transfer + if spatial_group_rank < spatial_group_size-1: + stream2.wait_stream(stream1) + with torch.cuda.stream(stream2): + if explicit_nhwc: + btm_fat_halo = torch.empty((N,3,W,C),dtype=out1.dtype,device=out1.device) + btm_fat_halo[:,0:2,:,:].copy_(out1[:,Hs-2:,:,:]) + btm_fat_halo[:,2:,:,:].copy_(btm_out1_halo) + else: + btm_fat_halo = torch.empty((N,C,3,W),dtype=out1.dtype,device=out1.device) + btm_fat_halo[:,:,0:2,:].copy_(out1[:,:,Hs-2:,:]) + btm_fat_halo[:,:,2:,:].copy_(btm_out1_halo) + btm_out2 = fast_bottleneck.forward_out2_halo(explicit_nhwc, btm_fat_halo, args) + if spatial_group_rank > 0: + with torch.cuda.stream(stream1): + if explicit_nhwc: + top_fat_halo = torch.empty((N,3,W,C),dtype=out1.dtype,device=out1.device) + top_fat_halo[:,:1,:,:].copy_(top_out1_halo) + top_fat_halo[:,1:3,:,:].copy_(out1[:,:2,:,:]) + else: + top_fat_halo = torch.empty((N,C,3,W),dtype=out1.dtype,device=out1.device) + top_fat_halo[:,:,:1,:].copy_(top_out1_halo) + top_fat_halo[:,:,1:3,:].copy_(out1[:,:,:2,:]) + top_out2 = fast_bottleneck.forward_out2_halo(explicit_nhwc, top_fat_halo, args) + if use_delay_kernel: inc.add_delay(10) + elif spatial_method != 2 and spatial_method != 3: + assert(False), "spatial_method must be 1, 2 or 3" + + if spatial_group_size <= 1: + fast_bottleneck.forward_out2(explicit_nhwc, stride_1x1, args, outputs) + elif spatial_method == 1: + fast_bottleneck.forward_out2(explicit_nhwc, stride_1x1, args, outputs) + with torch.cuda.stream(stream3): + if explicit_nhwc: + out1_pad[:,1:Hs+1,:,:].copy_(out1) + else: + out1_pad[:,:,1:Hs+1,:].copy_(out1) + elif spatial_method == 2: + # wait for halo transfer to finish before doing a full convolution of padded x + if explicit_nhwc: + out1_pad[:,1:Hs+1,:,:].copy_(out1) + else: + out1_pad[:,:,1:Hs+1,:].copy_(out1) + torch.cuda.current_stream().wait_stream(stream1) + fast_bottleneck.forward_out2_pad(explicit_nhwc, stride_1x1, args, outputs, out1_pad) + elif spatial_method == 3: + fast_bottleneck.forward_out2_mask(explicit_nhwc, stride_1x1, args, outputs, thresholdTop, thresholdBottom) + with torch.cuda.stream(stream3): + if explicit_nhwc: + out1_pad[:,1:Hs+1,:,:].copy_(out1) + else: + out1_pad[:,:,1:Hs+1,:].copy_(out1) + + # compute halo cells for outputs[1] (out2) + if spatial_group_size > 1: + out2 = outputs[1] + if explicit_nhwc: + top_out2_halo = out2[:,:1,:,:] + btm_out2_halo = out2[:,Hs-1:,:,:] + else: + top_out2_halo = out2[:,:,:1,:] + btm_out2_halo = out2[:,:,Hs-1:,:] + if spatial_method == 1: + if spatial_group_rank > 0: + torch.cuda.current_stream().wait_stream(stream1) + top_out2_halo.copy_(top_out2) + if spatial_group_rank < spatial_group_size-1: + torch.cuda.current_stream().wait_stream(stream2) + btm_out2_halo.copy_(btm_out2) + elif spatial_method == 3: + # Note + # out2 halo correction cannot overlap with anything since it has + # to wait for out2_mask to finish, but itself has to finish before + # the first kernel of _forward_rest can launch. + # At least we can overlap the two halo correction kernels. + if spatial_group_rank < spatial_group_size-1: + stream2.wait_stream(stream1) # wait for halo transfers to finish + stream2.wait_stream(torch.cuda.current_stream()) # wait for *_out2_mask to finish + with torch.cuda.stream(stream2): + w1by3 = args[2][:,2:3,:,:].clone() + btm_out1_halo = btm_out1_halo.clone() + btm_out2 = fast_bottleneck.forward_out2_halo_corr(explicit_nhwc, btm_out1_halo, args, w1by3, btm_out2_halo.clone()) + btm_out2_halo.copy_(btm_out2) + if spatial_group_rank > 0: + stream1.wait_stream(torch.cuda.current_stream()) # wait for *_out2_mask to finish + with torch.cuda.stream(stream1): + w1by3 = args[2][:,:1,:,:].clone() + top_out1_halo = top_out1_halo.clone() + top_out2 = fast_bottleneck.forward_out2_halo_corr(explicit_nhwc, top_out1_halo, args, w1by3, top_out2_halo.clone()) + top_out2_halo.copy_(top_out2) + if spatial_group_rank < spatial_group_size-1: + torch.cuda.current_stream().wait_stream(stream2) + if spatial_group_rank > 0: + torch.cuda.current_stream().wait_stream(stream1) + + fast_bottleneck.forward_rest(explicit_nhwc, stride_1x1, args, outputs) + # save halos for backward pass + if spatial_group_size > 1: + if spatial_method != 2: + # make sure copy of mid-section of out1 into out1_pad is done before exiting + torch.cuda.current_stream().wait_stream(stream3) + ctx.save_for_backward(*(args+outputs+[out1_pad,])) + else: + ctx.save_for_backward(*(args+outputs)) + # save relu outputs for drelu + ctx.explicit_nhwc = explicit_nhwc + ctx.stride_1x1 = stride_1x1 + ctx.spatial_group_size = spatial_group_size + if spatial_group_size > 1: + ctx.spatial_group_rank = spatial_group_rank + ctx.spatial_halo_exchanger = spatial_halo_exchanger + ctx.spatial_method = spatial_method + ctx.use_delay_kernel = use_delay_kernel + ctx.thresholdTop = thresholdTop + ctx.thresholdBottom = thresholdBottom + ctx.stream1 = stream1 + ctx.stream2 = stream2 + ctx.stream3 = stream3 + return outputs[2] + + # backward relu is not exposed, MUL with mask used now + # only support dgrad + @staticmethod + def backward(ctx, grad_o): + if ctx.spatial_group_size > 1: + out1_pad = ctx.saved_tensors[-1] + outputs = ctx.saved_tensors[-4:-1] + else: + outputs = ctx.saved_tensors[-3:] + + if ctx.downsample: + grad_conv3, grad_conv4 = drelu_dscale2(grad_o, outputs[2], ctx.saved_tensors[6], ctx.saved_tensors[11]) + else: + grad_conv3, grad_conv4 = drelu_dscale1(grad_o, outputs[2], ctx.saved_tensors[6]) + + # create input vector for backward + t_list = [*ctx.saved_tensors[0:10]] + t_list.append(grad_conv3) + t_list.append(grad_conv4) + + # outputs used for wgrad and generating drelu mask + t_list.append(outputs[0]) + t_list.append(outputs[1]) + + # in case there is downsample + if ctx.downsample: + t_list.append(ctx.saved_tensors[10]) + + grads = fast_bottleneck.backward_init(ctx.explicit_nhwc, ctx.stride_1x1, t_list) + wgrad3_stream = torch.cuda.Stream() + wgrad3_stream.wait_stream(torch.cuda.current_stream()) + grad_out2 = fast_bottleneck.backward_grad_out2(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads) + wgrad2_stream = torch.cuda.Stream() + wgrad2_stream.wait_stream(torch.cuda.current_stream()) + # do halo exchange of grad_out2 here + # compute halo cells for grad_out1 + if ctx.spatial_group_size > 1: + if ctx.explicit_nhwc: + N,Hs,W,C = list(grad_out2.shape) + else: + N,C,Hs,W = list(grad_out2.shape) + relu1 = t_list[12] + ctx.stream1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(ctx.stream1): + top_halo, btm_halo = ctx.spatial_halo_exchanger.left_right_halo_exchange(grad_out2[:,:1,:,:], grad_out2[:,Hs-1:,:,:]) + # copy halos to send buffer + if ctx.spatial_method == 1 or ctx.spatial_method == 2: + # 1 -> halo recompute approach + # 2 -> wait for concatenated halos, then do single conv on full input (not implemented yet for bprop) + if ctx.spatial_group_rank < ctx.spatial_group_size-1: + ctx.stream2.wait_stream(ctx.stream1) + with torch.cuda.stream(ctx.stream2): + if ctx.explicit_nhwc: + btm_fat_halo = torch.empty((N,3,W,C),dtype=grad_out2.dtype,device=grad_out2.device) + btm_fat_halo[:,:2,:,:].copy_(grad_out2[:,Hs-2:,:,:]) + btm_fat_halo[:,2:,:,:].copy_(btm_halo) + btm_fat_relu_halo = torch.empty((N,3,W,C),dtype=grad_out2.dtype,device=grad_out2.device) + btm_fat_relu_halo[:,:2,:,:].copy_(relu1[:,Hs-2:,:,:]) + btm_fat_relu_halo[:,2:,:,:].zero_() + else: + btm_fat_halo = torch.empty((N,C,3,W),dtype=grad_out2.dtype,device=grad_out2.device) + btm_fat_halo[:,:,:2,:].copy_(grad_out2[:,:,Hs-2:,:]) + btm_fat_halo[:,:,2:,:].copy_(btm_halo) + btm_fat_relu_halo = torch.empty((N,C,3,W),dtype=grad_out2.dtype,device=grad_out2.device) + btm_fat_relu_halo[:,:,:2,:].copy_(relu1[:,:,Hs-2:,:]) + btm_fat_relu_halo[:,:,2:,:].zero_() + btm_grad_out1_halo = fast_bottleneck.backward_grad_out1_halo(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, btm_fat_halo, btm_fat_relu_halo) + if ctx.explicit_nhwc: + btm_grad_out1_halo = btm_grad_out1_halo[:,1:2,:,:] + else: + btm_grad_out1_halo = btm_grad_out1_halo[:,:,1:2,:] + if ctx.spatial_group_rank > 0: + with torch.cuda.stream(ctx.stream1): + if ctx.explicit_nhwc: + top_fat_halo = torch.empty((N,3,W,C),dtype=grad_out2.dtype,device=grad_out2.device) + top_fat_halo[:,:1,:,:].copy_(top_halo) + top_fat_halo[:,1:,:,:].copy_(grad_out2[:,:2,:,:]) + top_fat_relu_halo = torch.empty((N,3,W,C),dtype=grad_out2.dtype,device=grad_out2.device) + top_fat_relu_halo[:,:1,:,:].zero_() + top_fat_relu_halo[:,1:,:,:].copy_(relu1[:,:2,:,:]) + else: + top_fat_halo = torch.empty((N,C,3,W),dtype=grad_out2.dtype,device=grad_out2.device) + top_fat_halo[:,:,:1,:].copy_(top_halo) + top_fat_halo[:,:,1:,:].copy_(grad_out2[:,:,:2,:]) + top_fat_relu_halo = torch.empty((N,C,3,W),dtype=grad_out2.dtype,device=grad_out2.device) + top_fat_relu_halo[:,:,:1,:].zero_() + top_fat_relu_halo[:,:,1:,:].copy_(relu1[:,:,:2,:]) + top_grad_out1_halo = fast_bottleneck.backward_grad_out1_halo(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, top_fat_halo, top_fat_relu_halo) + if ctx.explicit_nhwc: + top_grad_out1_halo = top_grad_out1_halo[:,1:2,:,:] + else: + top_grad_out1_halo = top_grad_out1_halo[:,:,1:2,:] + if ctx.use_delay_kernel: inc.add_delay(10) + elif ctx.spatial_method != 3: + assert(False), "spatial_method must be 1, 2 or 3" + + # compute grad_out1 for internal cells + if ctx.spatial_group_size <= 1 or ctx.spatial_method == 1 or ctx.spatial_method == 2: + grad_out1 = fast_bottleneck.backward_grad_out1(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, grad_out2) + elif ctx.spatial_group_size > 1 and ctx.spatial_method == 3: + grad_out1 = fast_bottleneck.backward_grad_out1_mask(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, grad_out2, ctx.thresholdTop, ctx.thresholdBottom) + + # apply halo cells to grad_out1 + if ctx.spatial_group_size > 1: + w = t_list[2] + z = t_list[4] + relu1 = t_list[12] + #print("w.shape = %s, z.shape = %s, relu1.shape = %s" % (str(list(w.shape)), str(list(z.shape)), str(list(relu1.shape)))) + if ctx.spatial_method == 1 or ctx.spatial_method == 2: + if ctx.spatial_group_rank < ctx.spatial_group_size-1: + torch.cuda.current_stream().wait_stream(ctx.stream2) + if ctx.explicit_nhwc: + grad_out1[:,Hs-1:,:,:].copy_(btm_grad_out1_halo) + else: + grad_out1[:,:,Hs-1:,:].copy_(btm_grad_out1_halo) + #print("ctx.spatial_group_rank = %d, apply grad_out1 btm halo (grad_out1.shape = %s)" % (ctx.spatial_group_rank, str(list(grad_out1.shape)))) + if ctx.spatial_group_rank > 0: + torch.cuda.current_stream().wait_stream(ctx.stream1) + if ctx.explicit_nhwc: + grad_out1[:,:1,:,:].copy_(top_grad_out1_halo) + else: + grad_out1[:,:,:1,:].copy_(top_grad_out1_halo) + #print("ctx.spatial_group_rank = %d, apply grad_out1 top halo (grad_out1.shape = %s)" % (ctx.spatial_group_rank, str(list(grad_out1.shape)))) + elif ctx.spatial_method == 3: + if ctx.spatial_group_rank < ctx.spatial_group_size-1: + if ctx.explicit_nhwc: + btm_relu_halo = relu1[:,Hs-1:,:,:].clone() + btm_grad_out1 = grad_out1[:,Hs-1:,:,:] + else: + btm_relu_halo = relu1[:,:,Hs-1:,:].clone() + btm_grad_out1 = grad_out1[:,:,Hs-1:,:] + w1by3 = w[:,:1,:,:].clone() + ctx.stream2.wait_stream(ctx.stream1) # wait for halo transfers to finish + ctx.stream2.wait_stream(torch.cuda.current_stream()) # wait for backward_grad_out1_mask to finish before launching halo correction kernel + with torch.cuda.stream(ctx.stream2): + btm_grad_out1_halo = fast_bottleneck.backward_grad_out1_halo_corr(ctx.explicit_nhwc, ctx.stride_1x1, t_list, w1by3, grads, btm_halo, btm_relu_halo, btm_grad_out1.clone()) + btm_grad_out1.copy_(btm_grad_out1_halo) + if ctx.spatial_group_rank > 0: + if ctx.explicit_nhwc: + top_relu_halo = relu1[:,:1,:,:].clone() + top_grad_out1 = grad_out1[:,:1,:,:] + else: + top_relu_halo = relu1[:,:,:1,:].clone() + top_grad_out1 = grad_out1[:,:,:1,:] + w1by3 = w[:,2:,:,:].clone() + ctx.stream1.wait_stream(torch.cuda.current_stream()) # wait for backward_grad_out1_mask to finish before launching halo correction kernel + with torch.cuda.stream(ctx.stream1): + top_grad_out1_halo = fast_bottleneck.backward_grad_out1_halo_corr(ctx.explicit_nhwc, ctx.stride_1x1, t_list, w1by3, grads, top_halo, top_relu_halo, top_grad_out1.clone()) + top_grad_out1.copy_(top_grad_out1_halo) + if ctx.spatial_group_rank < ctx.spatial_group_size-1: + torch.cuda.current_stream().wait_stream(ctx.stream2) # wait for halo correction to finish + if ctx.spatial_group_rank > 0: + torch.cuda.current_stream().wait_stream(ctx.stream1) + + wgrad1_stream = torch.cuda.Stream() + wgrad1_stream.wait_stream(torch.cuda.current_stream()) + fast_bottleneck.backward_rest(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, grad_out2, grad_out1) + with torch.cuda.stream(wgrad3_stream): + fast_bottleneck.backward_wgrad3(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads) + with torch.cuda.stream(wgrad2_stream): + if ctx.spatial_group_size > 1: + fast_bottleneck.backward_wgrad2_pad(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, out1_pad, grad_out2) + else: + fast_bottleneck.backward_wgrad2(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, grad_out2) + with torch.cuda.stream(wgrad1_stream): + fast_bottleneck.backward_wgrad1(ctx.explicit_nhwc, ctx.stride_1x1, t_list, grads, grad_out1) + torch.cuda.current_stream().wait_stream(wgrad3_stream) + torch.cuda.current_stream().wait_stream(wgrad2_stream) + torch.cuda.current_stream().wait_stream(wgrad1_stream) + + return (None, None, None, None, None, None, None, None, None, None, None, None, *grads) + +spatial_bottleneck_function = SpatialBottleneckFunction.apply + +class SpatialBottleneck(torch.nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + # here we put it at 1x1 + + def __init__(self, in_channels, bottleneck_channels, out_channels, stride=1, groups=1, + dilation=1, norm_func=None, use_cudnn=False, explicit_nhwc=False, + spatial_parallel_args=None): + super(SpatialBottleneck, self).__init__() + if groups != 1: + raise RuntimeError('Only support groups == 1') + if dilation != 1: + raise RuntimeError('Only support dilation == 1') + if norm_func == None: + norm_func = FrozenBatchNorm2d + else: + raise RuntimeError('Only support frozen BN now.') + + if stride != 1 or in_channels != out_channels: + self.downsample = nn.Sequential( + conv1x1(in_channels, out_channels, stride), + norm_func(out_channels), + ) + else: + self.downsample = None + + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(in_channels, bottleneck_channels, stride) + self.conv2 = conv3x3(bottleneck_channels, bottleneck_channels) + self.conv3 = conv1x1(bottleneck_channels, out_channels) + self.relu = nn.ReLU(inplace=True) + self.stride = stride + + self.bn1 = norm_func(bottleneck_channels) + self.bn2 = norm_func(bottleneck_channels) + self.bn3 = norm_func(out_channels) + self.w_scale = None + + self.use_cudnn = use_cudnn + + # setup conv weights + self.w_conv = [self.conv1.weight, self.conv2.weight, self.conv3.weight] + if self.downsample is not None: + self.w_conv.append(self.downsample[0].weight) + + # init weight in nchw format before possible transpose + for w in self.w_conv: + kaiming_uniform_(w, a=1) + + self.thresholdTop, self.thresholdBottom = None, None + + # TODO: prevent unsupported case usage + # support cases + # native cudnn + # normal yes no + # channel_last yes yes + # explicit_nhwc no yes + self.explicit_nhwc = explicit_nhwc + if self.explicit_nhwc: + for p in self.parameters(): + with torch.no_grad(): + p.data = p.data.permute(0,2,3,1).contiguous() + + # spatial communicator + if spatial_parallel_args is None: + self.spatial_parallel_args = (1, 0, None, None, 0, False) + else: + self.spatial_parallel_args = spatial_parallel_args + return + + # Returns single callable that recomputes scale and bias for all frozen batch-norms. + # This method must be called before cuda graphing. + # The callable it returns can be called anytime. + # Calling this method will prevent these from being computed every forward call. + def get_scale_bias_callable(self): + self.w_scale, self.w_bias, args = [], [], [] + batch_norms = [self.bn1, self.bn2, self.bn3] + if self.downsample is not None: + batch_norms.append(self.downsample[1]) + for bn in batch_norms: + s = torch.empty_like(bn.weight) + b = torch.empty_like(s) + args.append( (bn.weight, bn.bias, bn.running_mean, bn.running_var, s, b) ) + if self.explicit_nhwc: + self.w_scale.append( s.reshape(1, 1, 1, -1) ) + self.w_bias.append( b.reshape(1, 1, 1, -1) ) + else: + self.w_scale.append( s.reshape(1, -1, 1, 1) ) + self.w_bias.append( b.reshape(1, -1, 1, 1) ) + return func.partial(compute_scale_bias_method, self.explicit_nhwc, args) + + def forward(self, x): + if self.use_cudnn: + if self.thresholdTop is None: + spatial_group_size, spatial_group_rank, _, _, _, _ = self.spatial_parallel_args + if self.explicit_nhwc: + N,H,W,C = list(x.shape) + else: + N,C,H,W = list(x.shape) + self.thresholdTop = torch.tensor([1 if spatial_group_rank > 0 else 0], dtype=torch.int32, device='cuda') + self.thresholdBottom = torch.tensor([H-2 if spatial_group_rank < spatial_group_size - 1 else H-1], dtype=torch.int32, device='cuda') + + if self.w_scale is None: + # calculate scale/bias from registered buffers + # TODO: make this better + s1, b1 = self.bn1.get_scale_bias(self.explicit_nhwc) + s2, b2 = self.bn2.get_scale_bias(self.explicit_nhwc) + s3, b3 = self.bn3.get_scale_bias(self.explicit_nhwc) + w_scale = [s1, s2, s3] + w_bias = [b1, b2, b3] + if self.downsample is not None: + s4, b4 = self.downsample[1].get_scale_bias(self.explicit_nhwc) + w_scale.append(s4) + w_bias.append(b4) + out = spatial_bottleneck_function(*self.spatial_parallel_args, self.explicit_nhwc, self.stride, w_scale, w_bias, self.thresholdTop, self.thresholdBottom, x, *self.w_conv) + else: + out = spatial_bottleneck_function(*self.spatial_parallel_args, self.explicit_nhwc, self.stride, self.w_scale, self.w_bias, self.thresholdTop, self.thresholdBottom, x, *self.w_conv) + return out + + if self.explicit_nhwc: + raise RuntimeError('explicit nhwc with native ops is not supported.') + + # fallback to native ops + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + diff --git a/apex/apex/contrib/bottleneck/halo_exchangers.py b/apex/apex/contrib/bottleneck/halo_exchangers.py new file mode 100644 index 00000000..3299e823 --- /dev/null +++ b/apex/apex/contrib/bottleneck/halo_exchangers.py @@ -0,0 +1,180 @@ +import torch +import torch.distributed as dist +from torch import nn +import nccl_p2p_cuda as inc +import peer_memory_cuda as pm + +# Communication free halo exchanger. +# NB! This halo exchanger does not exchange halos with neighbors as it should, it merely swaps the inputs +# NB! This is only useful for performance testing. +# NB! Do not use for actual production runs +class HaloExchanger(object): + def __init__(self, ranks, rank_in_group): + self.stream1 = torch.cuda.Stream() + self.stream2 = torch.cuda.Stream() + self.stream3 = torch.cuda.Stream() + self.group_size = len(ranks) + self.ranks = ranks + self.rank_in_group = rank_in_group + self.wrap_around_left_rank_in_group = (rank_in_group + self.group_size - 1) % self.group_size + self.wrap_around_right_rank_in_group = (rank_in_group + 1) % self.group_size + self.left_rank = ranks[rank_in_group-1] if rank_in_group > 0 else -1 + self.left_zero = True if rank_in_group == 0 else False + self.right_rank = ranks[rank_in_group+1] if rank_in_group < self.group_size - 1 else -1 + self.right_zero = True if rank_in_group == self.group_size - 1 else False + +class HaloExchangerNoComm(HaloExchanger): + def __init__(self, ranks, rank_in_group): + super(HaloExchangerNoComm, self).__init__(ranks, rank_in_group) + + def left_right_halo_exchange(self, left_output_halo, right_output_halo, left_input_halo=None, right_input_halo=None): + if left_input_halo is None: + return right_output_halo, left_output_halo + else: + left_input_halo.copy_(right_output_halo) + right_input_halo.copy_(left_output_halo) + +class HaloExchangerAllGather(HaloExchanger): + def __init__(self, ranks, rank_in_group, comm): + super(HaloExchangerAllGather, self).__init__(ranks, rank_in_group) + # self.comm must be NCCL process_group created with torch.distributed.new_group(ranks=ranks) + self.comm = comm + + def left_right_halo_exchange(self, left_output_halo, right_output_halo, left_input_halo=None, right_input_halo=None): + N,Hh,W,C = list(left_output_halo.shape) + send_halos = torch.empty((N,2*Hh,W,C),dtype=left_output_halo.dtype,device=left_output_halo.device) + send_halos[:,:Hh,:,:].copy_(left_output_halo) + send_halos[:,Hh:,:,:].copy_(right_output_halo) + all_halos = torch.empty((N,2*Hh*self.group_size,W,C),dtype=left_output_halo.dtype,device=left_output_halo.device) + all_halos = [all_halos[:,i*2*Hh:(i+1)*2*Hh,:,:] for i in range(self.group_size)] + torch.distributed.all_gather(all_halos,send_halos,group=self.comm,no_copy=True) + ag_left_input_halo = all_halos[self.wrap_around_left_rank_in_group][:,Hh:,:,:] + ag_right_input_halo = all_halos[self.wrap_around_right_rank_in_group][:,:Hh,:,:] + if left_input_halo is None: + if self.left_zero: + ag_left_input_halo.zero_() + if self.right_zero: + ag_right_input_halo.zero_() + return ag_left_input_halo, ag_right_input_halo + else: + if self.left_zero: + left_input_halo.zero_() + else: + left_input_halo.copy_(ag_left_input_halo) + if self.right_zero: + right_input_halo.zero_() + else: + right_input_halo.copy_(ag_right_input_halo) + +class HaloExchangerSendRecv(HaloExchanger): + def __init__(self, ranks, rank_in_group): + super(HaloExchangerSendRecv, self).__init__(ranks, rank_in_group) + nccl_id = inc.get_unique_nccl_id(1).cuda() + torch.distributed.broadcast(nccl_id, 0) + nccl_id = nccl_id.cpu() + print("%d :: nccl_id = %s" % (torch.distributed.get_rank(), str(nccl_id))) + # Create another global nccl communicator in addition to the one created by torch.distributed.init_process_group("nccl") + # This is unavoidable because the underlying NCCL communicator torch.distributed creates is a protected variable, hence + # it cannot be accessed from another class. + # TODO: Figure out a way to avoid creating a second global communicator + assert(torch.distributed.get_rank() == self.ranks[self.rank_in_group]), "ranks[%d](%d) != torch.distributed.get_rank()(%d)" % (self.rank_in_group, self.ranks[self.rank_in_group], torch.distributed.get_rank()) + self.handle = inc.init_nccl_comm(nccl_id, torch.distributed.get_rank(), torch.distributed.get_world_size()) + + def left_right_halo_exchange(self, left_output_halo, right_output_halo, left_input_halo=None, right_input_halo=None): + if left_input_halo is None: + left_input_halo, right_input_halo = inc.left_right_halo_exchange(self.handle, self.left_rank, self.right_rank , left_output_halo, right_output_halo) + return left_input_halo, right_input_halo + else: + inc.left_right_halo_exchange_inplace(self.handle, self.left_rank, self.right_rank, left_output_halo, right_output_halo, left_input_halo, right_input_halo) + +class HaloExchangerPeer(HaloExchanger): + def __init__(self, ranks, rank_in_group, peer_pool, explicit_nhwc, numSM=0): + super(HaloExchangerPeer, self).__init__(ranks, rank_in_group) + self.diagnostics = False + self.explicit_nhwc = explicit_nhwc + self.numSM = numSM + self.peer_pool = peer_pool + + def _allocate_peer_tensor(self, halo): + + # Compute size in bytes + # Note: Pad buffer so each CUDA block gets required buffer size + size = 4 * halo.numel() * halo.element_size() + size_per_block = 128 * 2 * 16 # 128 threads each require two 128b buffers + size = (size + size_per_block - 1) // size_per_block * size_per_block + + # Construct dtype peer buffer with desired size + shape = [1, 1, 1, size // halo.element_size()] + return self.peer_pool.allocate_peer_tensors(shape, halo.dtype, False, True) + + def left_right_halo_exchange(self, left_output_halo, right_output_halo, left_input_halo=None, right_input_halo=None): + inplace = False if left_input_halo is None and right_input_halo is None else True + if not inplace: + left_input_halo = torch.empty_like(right_output_halo) + right_input_halo = torch.empty_like(left_output_halo) + channels_last = left_output_halo.is_contiguous(memory_format=torch.channels_last) and not self.explicit_nhwc + left_tx = self._allocate_peer_tensor(left_input_halo) + right_tx = self._allocate_peer_tensor(right_input_halo) + pm.push_pull_halos_1d( + self.diagnostics, self.explicit_nhwc, self.numSM, self.rank_in_group, + self.left_zero, left_output_halo, left_tx[self.rank_in_group], right_tx[self.wrap_around_left_rank_in_group], left_input_halo, + self.right_zero, right_output_halo, right_tx[self.rank_in_group], left_tx[self.wrap_around_right_rank_in_group], right_input_halo, + ) + if not inplace: + return left_input_halo, right_input_halo + +# Class that combines input volume with halos from neighbors (1d). +class HaloPadder: + def __init__(self, halo_ex): + self.halo_ex = halo_ex + self.stream1 = torch.cuda.Stream() + self.stream2 = torch.cuda.Stream() + + def __call__(self, y, half_halo, explicit_nhwc, H_split): + channels_last = not explicit_nhwc and y.is_contiguous(memory_format=torch.channels_last) + if explicit_nhwc: + N,H,W,C = list(y.shape) + if H_split: + padded_shape = [N,H+2*half_halo,W,C] + ypad = torch.empty(shape=padded_shape, dtype=y.dtype, device=y.device, memory_format=torch.contiguous_format) + yleft = ypad[:,:half_halo,:,:] + ymid = ypad[:,half_halo:H+half_halo,:,:] + yright = ypad[:,H+half_halo:H+2*half_halo,:,:] + oleft = y[:,:half_halo,:,:] + oright = y[:,H-half_halo:,:,:] + else: + padded_shape = [N,H,W+2*half_halo,C] + ypad = torch.empty(shape=padded_shape, dtype=y.dtype, device=y.device, memory_format=torch.contiguous_format) + yleft = ypad[:,:,:half_halo,:] + ymid = ypad[:,:,half_halo:W+half_halo,:] + yright = ypad[:,:,W+half_halo:W+2*half_halo,:] + oleft = y[:,:,:half_halo,:] + oright = y[:,:,W-half_halo:,:] + else: + N,C,H,W = list(y.shape) + if H_split: + padded_shape = [N,C,H+2*half_halo,W] + ypad = torch.empty(shape=padded_shape, dtype=y.dtype, device=y.device, memory_format=torch.channels_last) + yleft = ypad[:,:,:half_halo,:] + ymid = ypad[:,:,half_halo:H+half_halo,:] + yright = ypad[:,:,H+half_halo:H+2*half_halo,:] + oleft = y[:,:,:half_halo,:] + oright = y[:,:,H-half_halo:,:] + else: + padded_shape = [N,C,H,W+2*half_halo] + ypad = torch.empty(shape=padded_shape, dtype=y.dtype, device=y.device, memory_format=torch.channels_last) + yleft = ypad[:,:,:,:half_halo] + ymid = ypad[:,:,:,half_halo:W+half_halo] + yright = ypad[:,:,:,W+half_halo:W+2*half_halo] + oleft = y[:,:,:,:half_halo] + oright = y[:,:,:,W-half_halo:] + with torch.cuda.stream(self.stream1): + self.halo_ex(oleft, oright, yleft, yright) + with torch.cuda.stream(self.stream2): + ymid.copy_(y) + return ypad + + def wait(self): + current_stream = torch.cuda.current_stream() + current_stream.wait_stream(self.stream1) + current_stream.wait_stream(self.stream2) diff --git a/apex/apex/contrib/bottleneck/test.py b/apex/apex/contrib/bottleneck/test.py new file mode 100644 index 00000000..2c3c6213 --- /dev/null +++ b/apex/apex/contrib/bottleneck/test.py @@ -0,0 +1,71 @@ +import torch +from bottleneck import Bottleneck +torch.manual_seed(23337) + +# use True to print layerwise sum for all outputs in reference code path +DEBUG = False#True + +for stride, o_channel in [(1,32), (1,128), (2,32)]: + print("testing stride ==", stride, ", in_channel == 32 , out_channel ==", o_channel) + a_ = torch.randn(17,32,28,28) + + a = a_.cuda().half().to(memory_format=torch.channels_last).requires_grad_() + model = Bottleneck(32,8,o_channel,stride=stride).cuda().half().to(memory_format=torch.channels_last) + + # test model + b = model(a) + b.mean().backward() + d_grad = a.grad.float() + a.grad = None + torch.cuda.synchronize() + + if DEBUG: + print("[DEBUG] ref dx :", d_grad.sum().item()) + # print wgrad. we don't need to reset since later cpp print before accumulation + for i, w in enumerate(model.w_conv): + print("[DEBUG] ref wgrad{} :".format(i+1), w.grad.sum().item()) + + wgrads = [] + for w in model.w_conv: + wgrads.append(w.grad.float()) + + model.use_cudnn = True + model.zero_grad() + c = model(a) + c.mean().backward() + + torch.cuda.synchronize() + print("comparing native and channels_last:") + print("max error fprop:", (b-c).abs().max().item(), "max elem:", b.abs().max().item()) + print("max error dgrad:", (d_grad-a.grad.float()).abs().max().item(), "max elem:", d_grad.abs().max().item()) + for i, (w, wgrad) in enumerate(zip(model.w_conv, wgrads)): + print("max error wgrad{}:".format(i+1), (wgrad - w.grad.float()).abs().max().item(), "max elem:", wgrad.abs().max().item()) + + nhwc_a = a_.permute(0,2,3,1).contiguous().cuda().half().requires_grad_() + nhwc_model = Bottleneck(32,8,o_channel,stride=stride,explicit_nhwc=True, use_cudnn=True).cuda().half() + for p,q in zip(model.parameters(), nhwc_model.parameters()): + # model's storage is already in nhwc, we clone and assign to explicit nhwc model + q.data.copy_(p.data.permute(0,2,3,1).contiguous()) + for p,q in zip(model.buffers(), nhwc_model.buffers()): + q.data.copy_(p.data) + + d = nhwc_model(nhwc_a) + d.mean().backward() + torch.cuda.synchronize() + + # reset reference to cudnn channels_last permute + #c_s = c.storage().tolist() + #d_s = d.storage().tolist() + #print(max([x-y for x,y in zip(c_s,d_s)])) + c = c.contiguous(memory_format=torch.contiguous_format).permute(0,2,3,1).contiguous() + d_grad = a.grad.float().permute(0,2,3,1).contiguous() + wgrads = [] + for w in model.w_conv: + wgrads.append(w.grad.float().permute(0,2,3,1).contiguous()) + + torch.cuda.synchronize() + print("comparing nhwc and channels_last:") + print("max error fprop:", (d-c).abs().max().item(), "max elem:", c.abs().max().item()) + print("max error dgrad:", (d_grad-nhwc_a.grad.float()).abs().max().item(), "max elem:", d_grad.abs().max().item()) + for i, (w, wgrad) in enumerate(zip(nhwc_model.w_conv, wgrads)): + print("max error wgrad{}:".format(i+1), (wgrad - w.grad.float()).abs().max().item(), "max elem:", wgrad.abs().max().item()) diff --git a/apex/apex/contrib/clip_grad/__init__.py b/apex/apex/contrib/clip_grad/__init__.py new file mode 100644 index 00000000..cc9f501b --- /dev/null +++ b/apex/apex/contrib/clip_grad/__init__.py @@ -0,0 +1 @@ +from .clip_grad import clip_grad_norm_ diff --git a/apex/apex/contrib/clip_grad/clip_grad.py b/apex/apex/contrib/clip_grad/clip_grad.py new file mode 100644 index 00000000..b6411352 --- /dev/null +++ b/apex/apex/contrib/clip_grad/clip_grad.py @@ -0,0 +1,128 @@ +from typing import Union, Iterable + +import torch + +_kernel_import_succeeded = False +try: + import amp_C + from apex.multi_tensor_apply import multi_tensor_applier + _kernel_import_succeeded = True +except ImportError: + _kernel_import_succeeded = False + +_tensor_or_tensors = Union[torch.Tensor, Iterable[torch.Tensor]] + + +def clip_grad_norm_( + parameters: _tensor_or_tensors, max_norm: float, norm_type: float = 2.0, + error_if_nonfinite: bool = False) -> torch.Tensor: + r"""Clips gradient norm of an iterable of parameters. + + The norm is computed over all gradients together, as if they were + concatenated into a single vector. Gradients are modified in-place. + + This is identical to torch.nn.utils.clip_grad_norm_, except it + uses a fused CUDA kernel when computing the 2-norm of GPU tensors + in float32 and float16. + + Args: + parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a + single Tensor that will have gradients normalized + max_norm (float or int): max norm of the gradients + norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for + infinity norm. + error_if_nonfinite (bool): if True, an error is thrown if the total + norm of the gradients from :attr:`parameters` is ``nan``, + ``inf``, or ``-inf``. Default: False (will switch to True in the future) + + Returns: + Total norm of the parameters (viewed as a single vector). + + """ + if isinstance(parameters, torch.Tensor): + parameters = [parameters] + parameters = [p for p in parameters if p.grad is not None] + max_norm = float(max_norm) + norm_type = float(norm_type) + + # Trivial case + if len(parameters) == 0: + return torch.tensor(0.) + + # Fallback implementation + if not (_kernel_import_succeeded + and norm_type == 2.0 + and any(p.is_cuda for p in parameters)): + return torch.nn.utils.clip_grad_norm_( + parameters, + max_norm, + norm_type=norm_type, + error_if_nonfinite = error_if_nonfinite, + ) + + # Find fp32 and fp16 gradients on GPU + device = next(p.device for p in parameters if p.is_cuda) + grads_fp32, grads_fp16, grads_misc = [], [], [] + for p in parameters: + grad = p.grad.detach() + if p.dtype == torch.float32 and p.device == device: + grads_fp32.append(grad) + elif p.dtype == torch.float16 and p.device == device: + grads_fp16.append(grad) + else: + grads_misc.append(grad) + + # Compute gradient L2 norms + norms = [] + dummy_overflow_buf = torch.zeros([1], dtype=torch.int32, device=device) + if grads_fp32: + norms.append( + multi_tensor_applier( + amp_C.multi_tensor_l2norm, + dummy_overflow_buf, + [grads_fp32], + False, + )[0] + ) + if grads_fp16: + norms.append( + multi_tensor_applier( + amp_C.multi_tensor_l2norm, + dummy_overflow_buf, + [grads_fp16], + False, + )[0], + ) + for g in grads_misc: + norms.append(torch.linalg.norm(g).unsqueeze(0).to(device)) + total_norm = torch.linalg.norm(torch.cat(norms)) + + # Check for non-finite values + if error_if_nonfinite and torch.logical_or(total_norm.isnan(), total_norm.isinf()): + raise RuntimeError( + f'The total norm of order {norm_type} for gradients from ' + '`parameters` is non-finite, so it cannot be clipped. To disable ' + 'this error and scale the gradients by the non-finite norm anyway, ' + 'set `error_if_nonfinite=False`') + + # Scale gradients + clip_coef = max_norm / (total_norm + 1e-6) + clip_coef_clamped = torch.clamp(clip_coef, max=1.0) + if grads_fp32: + multi_tensor_applier( + amp_C.multi_tensor_scale, + dummy_overflow_buf, + [grads_fp32, grads_fp32], + clip_coef_clamped, + ) + if grads_fp16: + multi_tensor_applier( + amp_C.multi_tensor_scale, + dummy_overflow_buf, + [grads_fp16, grads_fp16], + clip_coef_clamped, + ) + for g in grads_misc: + g.mul_(clip_coef_clamped.to(g.device)) + + return total_norm diff --git a/apex/apex/contrib/conv_bias_relu/__init__.py b/apex/apex/contrib/conv_bias_relu/__init__.py new file mode 100644 index 00000000..6e449ba2 --- /dev/null +++ b/apex/apex/contrib/conv_bias_relu/__init__.py @@ -0,0 +1,2 @@ +from .conv_bias_relu import ConvBiasReLU, ConvBias, ConvBiasMaskReLU, ConvFrozenScaleBiasReLU + diff --git a/apex/apex/contrib/conv_bias_relu/conv_bias_relu.py b/apex/apex/contrib/conv_bias_relu/conv_bias_relu.py new file mode 100644 index 00000000..c873ebe1 --- /dev/null +++ b/apex/apex/contrib/conv_bias_relu/conv_bias_relu.py @@ -0,0 +1,104 @@ +import pdb + +import torch +from torch.autograd import gradcheck + +from apex import check_cudnn_version_and_warn +import fused_conv_bias_relu + +check_cudnn_version_and_warn(__name__, 8400) + + +class ConvBiasReLU_(torch.autograd.Function): + @staticmethod + @torch.cuda.amp.custom_fwd(cast_inputs=torch.half) + def forward(ctx, x, weight, bias, padding, stride): + outputs = fused_conv_bias_relu.forward([x, weight, bias], padding, stride) + ctx.save_for_backward(x, weight, outputs[0]) + ctx.padding = padding + ctx.stride = stride + + return outputs[0] + + @staticmethod + @torch.cuda.amp.custom_bwd + def backward(ctx, grad_output): + bwd_args = [*ctx.saved_tensors, grad_output] + padding = ctx.padding + stride = ctx.stride + grads = fused_conv_bias_relu.backward(bwd_args, padding, stride) + + return grads[0], grads[1], grads[2], None, None + + +class ConvBiasMaskReLU_(torch.autograd.Function): + @staticmethod + @torch.cuda.amp.custom_fwd(cast_inputs=torch.half) + def forward(ctx, x, weight, bias, mask, padding, stride): + outputs = fused_conv_bias_relu.forward_mask([x, weight, bias, mask], padding, stride) + ctx.save_for_backward(x, weight, outputs[0]) + ctx.padding = padding + ctx.stride = stride + + return outputs[0] + + @staticmethod + @torch.cuda.amp.custom_bwd + def backward(ctx, grad_output): + bwd_args = [*ctx.saved_tensors, grad_output] + padding = ctx.padding + stride = ctx.stride + grads = fused_conv_bias_relu.backward(bwd_args, padding, stride) + + return grads[0], grads[1], grads[2], None, None, None + + +class ConvBias_(torch.autograd.Function): + @staticmethod + @torch.cuda.amp.custom_fwd(cast_inputs=torch.half) + def forward(ctx, x, weight, bias, padding, stride): + outputs = fused_conv_bias_relu.forward_no_relu([x, weight, bias], padding, stride) + ctx.save_for_backward(x, weight) + ctx.padding = padding + ctx.stride = stride + + return outputs[0] + + @staticmethod + @torch.cuda.amp.custom_bwd + def backward(ctx, grad_output): + bwd_args = [*ctx.saved_tensors, grad_output] + padding = ctx.padding + stride = ctx.stride + grads = fused_conv_bias_relu.backward_no_relu(bwd_args, padding, stride) + + return grads[0], grads[1], grads[2], None, None + + +class ConvFrozenScaleBiasReLU_(torch.autograd.Function): + @staticmethod + @torch.cuda.amp.custom_fwd(cast_inputs=torch.half) + def forward(ctx, x, weight, scale, bias, padding, stride): + output = fused_conv_bias_relu.forward_cscale_cbias_relu([x, weight, scale, bias], padding, stride) + ctx.save_for_backward(x, weight, scale, output) + ctx.padding = padding + ctx.stride = stride + + return output + + @staticmethod + @torch.cuda.amp.custom_bwd + def backward(ctx, grad_output): + bwd_args = [*ctx.saved_tensors, grad_output] + padding = ctx.padding + stride = ctx.stride + grads = fused_conv_bias_relu.backward_cscale_cbias_relu(bwd_args, padding, stride) + + return grads[0], grads[1], None, None, None, None + + +ConvBiasReLU = ConvBiasReLU_.apply +ConvBiasMaskReLU = ConvBiasMaskReLU_.apply +ConvBias = ConvBias_.apply +ConvFrozenScaleBiasReLU = ConvFrozenScaleBiasReLU_.apply + diff --git a/apex/apex/contrib/csrc/bottleneck/bottleneck.cpp b/apex/apex/contrib/csrc/bottleneck/bottleneck.cpp new file mode 100644 index 00000000..9a0c3403 --- /dev/null +++ b/apex/apex/contrib/csrc/bottleneck/bottleneck.cpp @@ -0,0 +1,4073 @@ +#include +#include // for getcudnnhandle +#include +#include +#include +#include + +#include + +#ifdef DEBUG +#define DEBUG_MSG(str) do { std::cout << str << std::endl; } while( false ) +#else +#define DEBUG_MSG(str) do { } while ( false ) +#endif + +#ifdef DEBUG_CUDNN +#define DEBUG_CUDNN_MSG(buf, str) do { buf << str << std::endl; } while( false ) +#else +#define DEBUG_CUDNN_MSG(buf, str) do { } while ( false ) +#endif + +#define checkCudnnErr(...) \ + do { \ + int err = checkCudnnError(__VA_ARGS__, #__VA_ARGS__, __FILE__, __LINE__); \ + if (err) { \ + return; \ + } \ + } while (0) + + +int checkCudnnError(cudnnStatus_t code, const char* expr, const char* file, int line) { + if (code) { + printf("CUDNN error at %s:%d, code=%d (%s) in '%s'\n", file, line, (int)code, cudnnGetErrorString(code), expr); + return 1; + } + return 0; +} + +void checkError(cudaError_t code, char const * func, const char *file, const int line, bool abort = true); +#define checkCUDAError(val) { checkError((val), #val, __FILE__, __LINE__); } // in-line regular function + +void checkError(cudaError_t code, char const * func, const char *file, const int line, bool abort) +{ + if (code != cudaSuccess) + { + const char * errorMessage = cudaGetErrorString(code); + fprintf(stderr, "CUDA error returned from \"%s\" at %s:%d, Error code: %d (%s)\n", func, file, line, code, errorMessage); + if (abort){ + cudaDeviceReset(); + exit(code); + } + } +} + +void generateStrides(const int64_t* dimA, int64_t* strideA, int nbDims, cudnnTensorFormat_t filterFormat) { + // For INT8x4 and INT8x32 we still compute standard strides here to input + // into the cuDNN functions. We will manually scale by resizeFactor in the cpu ref. + if (filterFormat == CUDNN_TENSOR_NCHW) { + strideA[nbDims - 1] = 1; + for (int64_t d = nbDims - 2; d >= 0; d--) { + strideA[d] = strideA[d + 1] * dimA[d + 1]; + } + } else { + // Here we assume that the format is CUDNN_TENSOR_NHWC + strideA[1] = 1; + strideA[nbDims - 1] = strideA[1] * dimA[1]; + for (int64_t d = nbDims - 2; d >= 2; d--) { + strideA[d] = strideA[d + 1] * dimA[d + 1]; + } + strideA[0] = strideA[2] * dimA[2]; + } +} + + +int getFwdConvDilatedFilterDim(int filterDim, int dilation) { + return ((filterDim - 1) * dilation) + 1; +} + +int getFwdConvPaddedImageDim(int tensorDim, int pad) { + return tensorDim + (2 * pad); +} + +int getFwdConvOutputDim( + int tensorDim, + int pad, + int filterDim, + int stride, + int dilation) +{ + int p = (getFwdConvPaddedImageDim(tensorDim, pad) - getFwdConvDilatedFilterDim(filterDim, dilation)) / stride + 1; + return (p); +} + +enum { + X_TENSOR, + Y_TENSOR, + W_TENSOR, + Z_TENSOR, + B_TENSOR, + AFTERADD_TENSOR, + AFTERBIAS_TENSOR, + AFTERCONV_TENSOR, + OPTIONAL, + AFTEROPT_TENSOR, +}; + +using common_conv_descriptors = + std::tuple; + + +common_conv_descriptors +create_common_descriptors(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + cudnnConvolutionMode_t mode) { + const int convDim = 2; + + int64_t strideA_padded[4]; + int64_t outstrideA_padded[4]; + int64_t filterstrideA_padded[4]; + + generateStrides(w_dim_padded, filterstrideA_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(x_dim_padded, strideA_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(y_dim_padded, outstrideA_padded, 4, CUDNN_TENSOR_NHWC); + + return common_conv_descriptors(cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, strideA_padded) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, outstrideA_padded) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, w_dim_padded) + .setStrides(4, filterstrideA_padded) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(mode) + .setNDims(convDim) + .setStrides(convDim, convstrideA) + .setPrePadding(convDim, padA) + .setPostPadding(convDim, padA) + .setDilation(convDim, dilationA) + .build()); +} + +using common_convbias_descriptors = std::tuple; + +common_convbias_descriptors +create_conv_bias_add_act_descriptors(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType) { + const int convDim = 2; + + int64_t b_dim_padded[4]; + b_dim_padded[0] = 1; + b_dim_padded[1] = y_dim_padded[1]; + b_dim_padded[2] = 1; + b_dim_padded[3] = 1; + + int64_t x_stride_padded[4]; + int64_t y_stride_padded[4]; + int64_t w_stride_padded[4]; + int64_t b_stride_padded[4]; + + generateStrides(w_dim_padded, w_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(x_dim_padded, x_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(y_dim_padded, y_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(b_dim_padded, b_stride_padded, 4, CUDNN_TENSOR_NHWC); + + return common_convbias_descriptors(cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, w_dim_padded) + .setStrides(4, w_stride_padded) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('z') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('b') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setVirtual() + .setId('A') // after add + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setVirtual() + .setId('B') // after bias + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('C') // after conv + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('i') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('D') // after optional add + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build()); +} + +// tensor descriptors used for dgrad +enum { + X_OR_DX_TENSOR, + DY_TENSOR, + W_OR_DW_TENSOR, + SCALE_TENSOR, + RELU_TENSOR, + AFTER_DCONV_TENSOR, + AFTER_DRELU_TENSOR, +}; + +using dconv_descriptors = std::tuple; + +dconv_descriptors +create_dconv_descriptors(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType) { + const int convDim = 2; + + int64_t b_dim_padded[4]; + b_dim_padded[0] = 1; + b_dim_padded[1] = x_dim_padded[1]; + b_dim_padded[2] = 1; + b_dim_padded[3] = 1; + + int64_t x_stride_padded[4]; + int64_t y_stride_padded[4]; + int64_t w_stride_padded[4]; + int64_t b_stride_padded[4]; + + generateStrides(w_dim_padded, w_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(x_dim_padded, x_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(y_dim_padded, y_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(b_dim_padded, b_stride_padded, 4, CUDNN_TENSOR_NHWC); + + return dconv_descriptors(cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, w_dim_padded) + .setStrides(4, w_stride_padded) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('s') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('r') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setVirtual() + .setId('A') // after dconv + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setVirtual() + .setId('B') // after drelu + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build()); +} + +// create a cache for plan +std::unordered_map plan_cache; + +// TODO: better name +std::string getConvFusionString(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + cudnnDataType_t dataType, + std::string fusion_string) { + + for(int i=0;i<4;i++) { + fusion_string += 'X'; + fusion_string += std::to_string(x_dim_padded[i]); + } + for(int i=0;i<4;i++) { + fusion_string += 'W'; + fusion_string += std::to_string(w_dim_padded[i]); + } + for(int i=0;i<2;i++) { + fusion_string += 'P'; + fusion_string += std::to_string(padA[i]); + } + for(int i=0;i<2;i++) { + fusion_string += 'S'; + fusion_string += std::to_string(convstrideA[i]); + } + for(int i=0;i<2;i++) { + fusion_string += 'D'; + fusion_string += std::to_string(dilationA[i]); + } + fusion_string += 'T'; + fusion_string += std::to_string(dataType); + return fusion_string; +} + +cudnn_frontend::ExecutionPlan& getOrCreatePlan(cudnnHandle_t handle_, + std::stringstream& log_buf, + cudnn_frontend::OperationGraph& opGraph, + std::string cache_string, + bool use_heuristic = true){ + auto it = plan_cache.find(cache_string); + if (it != plan_cache.end()) { + DEBUG_CUDNN_MSG(log_buf, "Found plan in cache"); + return it->second; + } else { + if (use_heuristic){ + // TODO: confirm which mode to use + auto heuristics = cudnn_frontend::EngineHeuristicsBuilder() + .setOperationGraph(opGraph) + .setHeurMode(CUDNN_HEUR_MODE_INSTANT) + .build(); + // try 3 times for now as WAR for no heuristic training + int max_tries = 3, count = 0; + auto& engine_configs = heuristics.getEngineConfig(max_tries); + while(true) { + try { + plan_cache.emplace(cache_string, std::move(cudnn_frontend::ExecutionPlanBuilder() + .setHandle(handle_) + .setEngineConfig(engine_configs[count], opGraph.getTag()) + .build())); + break; + } catch (cudnn_frontend::cudnnException e) { + if (++count == max_tries) throw e; + } + } + }else{ + DEBUG_CUDNN_MSG(log_buf, "No plan in cache"); + // How many engines support this operation graph ? + auto total_engines = opGraph.getEngineCount(); + DEBUG_CUDNN_MSG(log_buf, opGraph.describe() << " has " << total_engines << " engines."); + // We have to randomly pick one engine from [0, total_engines) + // Selecting "0" by default + auto engine = cudnn_frontend::EngineBuilder().setGlobalEngineIdx(0).setOperationGraph(opGraph).build(); + DEBUG_CUDNN_MSG(log_buf, engine.describe()); + auto& knobs = engine.getSupportedKnobs(); + for (auto it = std::begin(knobs); it != std::end(knobs); ++it) { + DEBUG_CUDNN_MSG(log_buf, it->describe()); + } + if (knobs.begin() != knobs.end()) { + DEBUG_CUDNN_MSG(log_buf, "Updated knob choice"); + knobs.begin()->setChoice(knobs.begin()->getMinValue() + 1); + DEBUG_CUDNN_MSG(log_buf, knobs.begin()->describe()); + } + + // Createmplacee the requisite engine config + auto engine_config = cudnn_frontend::EngineConfigBuilder().setEngine(engine).build(); + DEBUG_CUDNN_MSG(log_buf, engine_config.describe()); + plan_cache.emplace(cache_string, std::move(cudnn_frontend::ExecutionPlanBuilder().setHandle(handle_).setEngineConfig(engine_config).build())); + } + + return plan_cache.find(cache_string)->second; + } +} + +void +run_conv_scale_bias_add_activation(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrB, + at::Half* devPtrI) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + common_convbias_descriptors tensors = create_conv_bias_add_act_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the add operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // optional add + auto addDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, addDesc.describe()); + + // Define the activation operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_FWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a Add Node with scaling parameters. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create a Bias Node. + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(scale_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create a optional add Node. + auto add_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(bias_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(addDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, add_op.describe()); + + + // Create an Activation Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(devPtrI ? add_op.getOutputTensor() : bias_op.getOutputTensor()) + .setyDesc(std::get(tensors)) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &scale_op, &bias_op, devPtrI ? &add_op : &act_op, &act_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(devPtrI ? ops.size() : 4, ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrB, devPtrI}; + int64_t uids[] = {'x', 'y', 'w', 'z', 'b', 'i'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(devPtrI ? 6 : 5, data_ptrs) + .setUids(devPtrI ? 6 : 5, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +void +run_conv_scale_bias(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrB) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + common_convbias_descriptors tensors = create_conv_bias_add_act_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the add operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Define the bias operation + auto addDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, addDesc.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a Add Node with scaling parameters. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) // TODO: change enum to aftermul + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create a Bias Node. + auto add_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(scale_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(addDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, add_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &scale_op, &add_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrB}; + int64_t uids[] = {'x', 'y', 'w', 'z', 'b'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(5, data_ptrs) + .setUids(5, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_dconv_drelu_dscale(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrR) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + dconv_descriptors tensors = create_dconv_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the activation backward operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_BWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the scale backward operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) + .setdxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setdyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // TODO: do we need getOutputTensor(), and what it returns in backward case? + // Create an relu backward Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setdyDesc(std::get(tensors)) + .setxDesc(std::get(tensors)) + .setdxDesc(std::get(tensors)) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create a Scale Node. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &act_op, &scale_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrR}; + int64_t uids[] = {'x', 'y', 'w', 's', 'r'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(5, data_ptrs) + .setUids(5, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +void +run_dconv(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + cudnnBackendDescriptorType_t mode) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + dconv_descriptors tensors = create_dconv_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + // mode should be one of following + // CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR + // CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR + auto conv_op_builder = cudnn_frontend::OperationBuilder(mode); + if (mode == CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) { + conv_op_builder.setdxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setdyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta); + } + else { + conv_op_builder.setxDesc(std::get(tensors)) + .setdwDesc(std::get(tensors)) + .setdyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta); + } + auto conv_op = conv_op_builder.build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW}; + int64_t uids[] = {'x', 'y', 'w'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(3, data_ptrs) + .setUids(3, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +void +run_dconv_add(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrR) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + dconv_descriptors tensors = create_dconv_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the add backward operation + auto addDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, addDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) + .setdxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setdyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // TODO: do we need getOutputTensor(), and what it returns in backward case? + // Create add Node. + auto add_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(addDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, add_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &add_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrR}; + int64_t uids[] = {'x', 'y', 'w', 'r'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(4, data_ptrs) + .setUids(4, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +// inputs contains x,w,z,b,(i) +std::vector bottleneck_forward(bool explicit_nhwc, int stride_1X1, std::vector inputs) { + + std::cout << std::fixed; + // create output vector + std::vector outputs; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t dimA[] = {0, 0, 0, 0}; + int64_t filterdimA1[] = {0, 0, 0, 0}; + int64_t filterdimA2[] = {0, 0, 0, 0}; + int64_t filterdimA3[] = {0, 0, 0, 0}; + int64_t filterdimA4[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] {0,1,2,3}; + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 3; + axis[2] = 1; + axis[3] = 2; + } + for (int dim=0;dim<4;dim++) { + dimA[dim] = inputs[0].size(axis[dim]); + filterdimA1[dim] = inputs[1].size(axis[dim]); + filterdimA2[dim] = inputs[2].size(axis[dim]); + filterdimA3[dim] = inputs[3].size(axis[dim]); + } + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]) { + for (int dim=0;dim<4;dim++) { + filterdimA4[dim] = inputs[10].size(axis[dim]); + } + } + + // output dim in n,c,h,w used by backend + int64_t outdimA1[] = {0, 0, 0, 0}; // Computed Below + int64_t outdimA2[] = {0, 0, 0, 0}; // Computed Below + int64_t outdimA3[] = {0, 0, 0, 0}; // Computed Below + + // use these fixed value for test run + int64_t padA[] = {0, 0}; + int64_t padA1[] = {1, 1}; + int64_t dilationA[] = {1, 1}; + int64_t convstrideA[] = {1, 1}; + int64_t convstride1X1[] = {stride_1X1, stride_1X1}; + + // compute output from pad/stride/dilation + outdimA1[0] = dimA[0]; + outdimA1[1] = filterdimA1[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA1[dim + 2] = getFwdConvOutputDim(dimA[dim + 2], padA[dim], filterdimA1[dim + 2], convstride1X1[dim], dilationA[dim]); + } + + outdimA2[0] = outdimA1[0]; + outdimA2[1] = filterdimA2[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA2[dim + 2] = getFwdConvOutputDim(outdimA1[dim + 2], padA1[dim], filterdimA2[dim + 2], convstrideA[dim], dilationA[dim]); + } + + outdimA3[0] = outdimA2[0]; + outdimA3[1] = filterdimA3[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA3[dim + 2] = getFwdConvOutputDim(outdimA2[dim + 2], padA[dim], filterdimA3[dim + 2], convstrideA[dim], dilationA[dim]); + } + + // Create output tensor in the correct shape in pytorch's view + int64_t outdim1[] = {0, 0, 0, 0}; + int64_t outdim2[] = {0, 0, 0, 0}; + int64_t outdim3[] = {0, 0, 0, 0}; + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 2; + axis[2] = 3; + axis[3] = 1; + } + for (int dim=0;dim<4;dim++) { + outdim1[dim] = outdimA1[axis[dim]]; + outdim2[dim] = outdimA2[axis[dim]]; + outdim3[dim] = outdimA3[axis[dim]]; + } + + // run + at::Half* x = inputs[0].data_ptr(); + at::Half* w = inputs[1].data_ptr(); + at::Half* z = inputs[4].data_ptr(); + at::Half* b = inputs[7].data_ptr(); + auto out1 = at::empty(outdim1, inputs[0].type(), output_format); + at::Half* y1 = out1.data_ptr(); + + run_conv_scale_bias_add_activation(dimA, + padA, + convstride1X1, + dilationA, + filterdimA1, + outdimA1, + CUDNN_DATA_HALF, + x, + w, + y1, + z, + b, + nullptr); + + DEBUG_MSG("[DEBUG] new relu1 : " << out1.to(at::kFloat).sum().item()); + + w = inputs[2].data_ptr(); + z = inputs[5].data_ptr(); + b = inputs[8].data_ptr(); + auto out2 = at::empty(outdim2, inputs[0].type(), output_format); + at::Half* y2 = out2.data_ptr(); + + run_conv_scale_bias_add_activation(outdimA1, + padA1, + convstrideA, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + y1, + w, + y2, + z, + b, + nullptr); + DEBUG_MSG("[DEBUG] new relu2 : " << out2.to(at::kFloat).sum().item()); + + // create output of conv3 + auto out3 = at::empty(outdim3, inputs[0].type(), output_format); + at::Half* y3 = out3.data_ptr(); + + // create output of conv4 that may exist + auto identity = at::empty_like(out3); + at::Half* yi = identity.data_ptr(); + + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]){ + + w = inputs[10].data_ptr(); + z = inputs[11].data_ptr(); + b = inputs[12].data_ptr(); + run_conv_scale_bias(dimA, + padA, + convstride1X1, + dilationA, + filterdimA4, + outdimA3, + CUDNN_DATA_HALF, + x, + w, + yi, + z, + b); + DEBUG_MSG("[DEBUG] new downsample : " << identity.to(at::kFloat).sum().item()); + } + else { + yi = x; + } + + w = inputs[3].data_ptr(); + z = inputs[6].data_ptr(); + b = inputs[9].data_ptr(); + + run_conv_scale_bias_add_activation(outdimA2, + padA, + convstrideA, + dilationA, + filterdimA3, + outdimA3, + CUDNN_DATA_HALF, + y2, + w, + y3, + z, + b, + yi); + DEBUG_MSG("[DEBUG] new relu3 : " << out3.to(at::kFloat).sum().item()); + + outputs.push_back(out1); + outputs.push_back(out2); + outputs.push_back(out3); + + return outputs; +} + +std::vector bottleneck_backward(bool explicit_nhwc, int stride_1X1, std::vector inputs) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + // create output vector + std::vector outputs; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t dimA[] = {0, 0, 0, 0}; + int64_t filterdimA1[] = {0, 0, 0, 0}; + int64_t filterdimA2[] = {0, 0, 0, 0}; + int64_t filterdimA3[] = {0, 0, 0, 0}; + int64_t filterdimA4[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] {0,1,2,3}; + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 3; + axis[2] = 1; + axis[3] = 2; + } + for (int dim=0;dim<4;dim++) { + dimA[dim] = inputs[0].size(axis[dim]); + filterdimA1[dim] = inputs[1].size(axis[dim]); + filterdimA2[dim] = inputs[2].size(axis[dim]); + filterdimA3[dim] = inputs[3].size(axis[dim]); + } + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]) { + for (int dim=0;dim<4;dim++) { + filterdimA4[dim] = inputs[14].size(axis[dim]); + } + } + + // output dim in n,c,h,w used by backend + int64_t outdimA1[] = {0, 0, 0, 0}; // Computed Below + int64_t outdimA2[] = {0, 0, 0, 0}; // Computed Below + int64_t outdimA3[] = {0, 0, 0, 0}; // Computed Below + + // use these fixed value for test run + int64_t padA[] = {0, 0}; + int64_t padA1[] = {1, 1}; + int64_t dilationA[] = {1, 1}; + int64_t convstrideA[] = {1, 1}; + int64_t convstride1X1[] = {stride_1X1, stride_1X1}; + + // compute output from pad/stride/dilation + outdimA1[0] = dimA[0]; + outdimA1[1] = filterdimA1[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA1[dim + 2] = getFwdConvOutputDim(dimA[dim + 2], padA[dim], filterdimA1[dim + 2], convstride1X1[dim], dilationA[dim]); + } + + outdimA2[0] = outdimA1[0]; + outdimA2[1] = filterdimA2[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA2[dim + 2] = getFwdConvOutputDim(outdimA1[dim + 2], padA1[dim], filterdimA2[dim + 2], convstrideA[dim], dilationA[dim]); + } + + outdimA3[0] = outdimA2[0]; + outdimA3[1] = filterdimA3[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA3[dim + 2] = getFwdConvOutputDim(outdimA2[dim + 2], padA[dim], filterdimA3[dim + 2], convstrideA[dim], dilationA[dim]); + } + + // Create output tensor in the correct shape in pytorch's view + int64_t outdim1[] = {0, 0, 0, 0}; + int64_t outdim2[] = {0, 0, 0, 0}; + int64_t outdim3[] = {0, 0, 0, 0}; + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 2; + axis[2] = 3; + axis[3] = 1; + } + for (int dim=0;dim<4;dim++) { + outdim1[dim] = outdimA1[axis[dim]]; + outdim2[dim] = outdimA2[axis[dim]]; + outdim3[dim] = outdimA3[axis[dim]]; + } + + // dconv3+drelu2+dscale2 + at::Half* conv_in = inputs[13].data_ptr(); + at::Half* dy3 = inputs[10].data_ptr(); + + DEBUG_MSG("[DEBUG] new dconv3 : " << inputs[10].to(at::kFloat).sum().item()); + + // wgrad + auto wgrad3 = at::empty_like(inputs[3]); + at::Half* dw3 = wgrad3.data_ptr(); + run_dconv(outdimA2, + padA, + convstrideA, + dilationA, + filterdimA3, + outdimA3, + CUDNN_DATA_HALF, + conv_in, + dw3, + dy3, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + // dgrad + auto grad_out2 = at::empty(outdim2, inputs[0].type(), output_format); + at::Half* dy2 = grad_out2.data_ptr(); + at::Half* w = inputs[3].data_ptr(); + at::Half* z = inputs[5].data_ptr(); + + at::Half* relu2 = inputs[13].data_ptr(); + + run_dconv_drelu_dscale(outdimA2, + padA, + convstrideA, + dilationA, + filterdimA3, + outdimA3, + CUDNN_DATA_HALF, + dy2, + w, + dy3, + z, + relu2); + + DEBUG_MSG("[DEBUG] new dconv2 : " << grad_out2.to(at::kFloat).sum().item()); + + // dconv2+drelu1+dscale1 + conv_in = inputs[12].data_ptr(); + + // wgrad + auto wgrad2 = at::empty_like(inputs[2]); + at::Half* dw2 = wgrad2.data_ptr(); + run_dconv(outdimA1, + padA1, + convstrideA, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + conv_in, + dw2, + dy2, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + // dgrad + auto grad_out1 = at::empty(outdim1, inputs[0].type(), output_format); + at::Half* dy1 = grad_out1.data_ptr(); + w = inputs[2].data_ptr(); + z = inputs[4].data_ptr(); + + at::Half* relu1 = inputs[12].data_ptr(); + // fused dgrad + run_dconv_drelu_dscale(outdimA1, + padA1, + convstrideA, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + z, + relu1); + +/* + // backward strided conv cannot be fused + // if stride == 1 but channel changes, we can fuse here + if (stride_1X1 != 1){ + // dgrad + run_dconv(outdimA1, + padA1, + convstride1X1, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + + // mul fused mask + grad_out1.mul_(inputs[15]); + } + else { + at::Half* relu1 = inputs[12].data_ptr(); + // fused dgrad + run_dconv_drelu_dscale(outdimA1, + padA1, + convstride1X1, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + z, + relu1); + } +*/ + DEBUG_MSG("[DEBUG] new dconv1 : " << grad_out1.to(at::kFloat).sum().item()); + + // create grads of conv4 that may exist + auto grad_x_conv4 = at::empty_like(inputs[0]); + at::Half* dx_conv4 = grad_x_conv4.data_ptr(); + at::Tensor wgrad4; + + // x used for dconv1 and dconv4 wgrad + at::Half* x = inputs[0].data_ptr(); + + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]){ + w = inputs[14].data_ptr(); + at::Half* dy_conv4 = inputs[11].data_ptr(); + if (requires_grad) { + run_dconv(dimA, + padA, + convstride1X1, + dilationA, + filterdimA4, + outdimA3, + CUDNN_DATA_HALF, + dx_conv4, + w, + dy_conv4, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + // we don't print here since we can't hook out this grad in pytorch alone to compare, due to addition with dx + // DEBUG_MSG("[DEBUG] new dx_identity : " << grad_x_conv4.to(at::kFloat).sum().item()); + } + // wgrad + wgrad4 = at::empty_like(inputs[14]); + at::Half* dw4 = wgrad4.data_ptr(); + run_dconv(dimA, + padA, + convstride1X1, + dilationA, + filterdimA4, + outdimA3, + CUDNN_DATA_HALF, + x, + dw4, + dy_conv4, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + } + else { + // if there is no downsample, dx_conv4 is fork of drelu3 + dx_conv4 = inputs[11].data_ptr(); + } + + // dconv1+add + // wgrad + auto wgrad1 = at::empty_like(inputs[1]); + at::Half* dw1 = wgrad1.data_ptr(); + run_dconv(dimA, + padA, + convstride1X1, + dilationA, + filterdimA1, + outdimA1, + CUDNN_DATA_HALF, + x, + dw1, + dy1, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + // dgrad + w = inputs[1].data_ptr(); + auto grad_x = at::empty_like(inputs[0]); + at::Half* dx = grad_x.data_ptr(); + + // backward strided conv cannot be fused + // if stride == 1 but channel changes, we can fuse here + if (requires_grad){ + if (stride_1X1 != 1){ + run_dconv(dimA, + padA, + convstride1X1, + dilationA, + filterdimA1, + outdimA1, + CUDNN_DATA_HALF, + dx, + w, + dy1, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + // add 2 together + grad_x.add_(grad_x_conv4); + } + else { + run_dconv_add(dimA, + padA, + convstride1X1, + dilationA, + filterdimA1, + outdimA1, + CUDNN_DATA_HALF, + dx, + w, + dy1, + dx_conv4); + } + } + + DEBUG_MSG("[DEBUG] new dx : " << grad_x.to(at::kFloat).sum().item()); + DEBUG_MSG("[DEBUG] new wgrad1 : " << wgrad1.to(at::kFloat).sum().item()); + DEBUG_MSG("[DEBUG] new wgrad2 : " << wgrad2.to(at::kFloat).sum().item()); + DEBUG_MSG("[DEBUG] new wgrad3 : " << wgrad3.to(at::kFloat).sum().item()); + outputs.push_back(grad_x); + outputs.push_back(wgrad1); + outputs.push_back(wgrad2); + outputs.push_back(wgrad3); + + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]) { + DEBUG_MSG("[DEBUG] new wgrad4 : " << wgrad4.to(at::kFloat).sum().item()); + outputs.push_back(wgrad4); + } + + return outputs; +} + +namespace { + +enum { + X_TENSOR, + Y_TENSOR, + W_TENSOR, + Z_TENSOR, + B_TENSOR, + AFTERADD_TENSOR, + AFTERBIAS_TENSOR, + AFTERCONV_TENSOR, + OPTIONAL, + AFTEROPT_TENSOR, + AFTERACT_TENSOR, + GEN_INDEX_TENSOR, + MASK_TOP_TENSOR, + MASK_BOTTOM_TENSOR, + MASK_TENSOR, + THRESHOLD_TOP_TENSOR, + THRESHOLD_BOTTOM_TENSOR, +}; + +using masked_convbias_descriptors = std::tuple; + +masked_convbias_descriptors +create_conv_bias_add_act_mask_descriptors(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + int64_t* threshold_dim, + cudnnDataType_t dataType) { + const int convDim = 2; + + int64_t b_dim_padded[4]; + b_dim_padded[0] = 1; + b_dim_padded[1] = y_dim_padded[1]; + b_dim_padded[2] = 1; + b_dim_padded[3] = 1; + + int64_t x_stride_padded[4]; + int64_t y_stride_padded[4]; + int64_t w_stride_padded[4]; + int64_t b_stride_padded[4]; + int64_t threshold_stride[4]; + + generateStrides(w_dim_padded, w_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(x_dim_padded, x_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(y_dim_padded, y_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(b_dim_padded, b_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(threshold_dim, threshold_stride, 4, CUDNN_TENSOR_NHWC); + + return masked_convbias_descriptors(cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, w_dim_padded) + .setStrides(4, w_stride_padded) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('z') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('b') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setVirtual() + .setId('A') // after add + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setVirtual() + .setId('B') // after bias + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('C') // after conv + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('i') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('D') // after optional add + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('E') // after act for masked + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('I') // output of the gen index operation + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_INT32) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('m') // top half of the mask created after the less than + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_BOOLEAN) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('n') // bottom half of the mask + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_BOOLEAN) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('M') // OR of the top and bottom masks + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_BOOLEAN) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, threshold_dim) + .setStrides(4, threshold_stride) + .setId('t') // threshold for creating the top mask + .setAlignment(16) + .setDataType(CUDNN_DATA_INT32) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, threshold_dim) + .setStrides(4, threshold_stride) + .setId('u') // threshold for creating the bottom mask + .setAlignment(16) + .setDataType(CUDNN_DATA_INT32) + .build()); +} + +// tensor descriptors used for dgrad +enum { + X_OR_DX_TENSOR, + DY_TENSOR, + W_OR_DW_TENSOR, + SCALE_TENSOR, + RELU_TENSOR, + AFTER_DCONV_TENSOR, + AFTER_DRELU_TENSOR, + DGRAD_INPUT_TENSOR, + DGRAD_OPTIONAL_TENSOR, + DGRAD_GEN_INDEX_TENSOR, + DGRAD_MASK_TOP_TENSOR, + DGRAD_MASK_BOTTOM_TENSOR, + DGRAD_MASK_TENSOR, + DGRAD_THRESHOLD_TOP_TENSOR, + DGRAD_THRESHOLD_BOTTOM_TENSOR, +}; + +using dconv_add_descriptors = std::tuple; + +dconv_add_descriptors +create_dconv_add_descriptors(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType) { + const int convDim = 2; + + int64_t b_dim_padded[4]; + b_dim_padded[0] = 1; + b_dim_padded[1] = x_dim_padded[1]; + b_dim_padded[2] = 1; + b_dim_padded[3] = 1; + + int64_t x_stride_padded[4]; + int64_t y_stride_padded[4]; + int64_t w_stride_padded[4]; + int64_t b_stride_padded[4]; + + generateStrides(w_dim_padded, w_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(x_dim_padded, x_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(y_dim_padded, y_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(b_dim_padded, b_stride_padded, 4, CUDNN_TENSOR_NHWC); + + return dconv_add_descriptors(cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, w_dim_padded) + .setStrides(4, w_stride_padded) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('s') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('r') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setVirtual() + .setId('A') // after dconv + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setVirtual() + .setId('B') // after drelu + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('i') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('D') // after optional add + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build()); +} + +using dconv_mask_descriptors = std::tuple; + +dconv_mask_descriptors +create_dconv_mask_descriptors(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + int64_t* threshold_dim, + cudnnDataType_t dataType) { + const int convDim = 2; + + int64_t b_dim_padded[4]; + b_dim_padded[0] = 1; + b_dim_padded[1] = x_dim_padded[1]; + b_dim_padded[2] = 1; + b_dim_padded[3] = 1; + + int64_t x_stride_padded[4]; + int64_t y_stride_padded[4]; + int64_t w_stride_padded[4]; + int64_t b_stride_padded[4]; + int64_t threshold_stride[4]; + + generateStrides(w_dim_padded, w_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(x_dim_padded, x_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(y_dim_padded, y_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(b_dim_padded, b_stride_padded, 4, CUDNN_TENSOR_NHWC); + generateStrides(threshold_dim, threshold_stride, 4, CUDNN_TENSOR_NHWC); + + return dconv_mask_descriptors(cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, w_dim_padded) + .setStrides(4, w_stride_padded) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, b_dim_padded) + .setStrides(4, b_stride_padded) + .setId('s') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setId('r') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setVirtual() + .setId('A') // after dconv + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, x_dim_padded) + .setStrides(4, x_stride_padded) + .setVirtual() + .setId('B') // after drelu + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('i') + .setAlignment(16) + .setDataType(dataType) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('D') // after optional add + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_FLOAT) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('I') // output of the gen index operation + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_INT32) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('m') // top half of the mask created after the less than + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_BOOLEAN) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('n') // bottom half of the mask + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_BOOLEAN) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, y_dim_padded) + .setStrides(4, y_stride_padded) + .setId('M') // OR of the top and bottom masks + .setAlignment(16) + .setVirtual() + .setDataType(CUDNN_DATA_BOOLEAN) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, threshold_dim) + .setStrides(4, threshold_stride) + .setId('t') // threshold for creating the top mask + .setAlignment(16) + .setDataType(CUDNN_DATA_INT32) + .build(), + cudnn_frontend::TensorBuilder() + .setDim(4, threshold_dim) + .setStrides(4, threshold_stride) + .setId('u') // threshold for creating the bottom mask + .setAlignment(16) + .setDataType(CUDNN_DATA_INT32) + .build()); +} + +void +run_conv_add_scale_bias_activation(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrB, + at::Half* devPtrI) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + common_convbias_descriptors tensors = create_conv_bias_add_act_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the add operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // optional add + auto addDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, addDesc.describe()); + + // Define the activation operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_FWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // create an add node. + auto add_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(addDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, add_op.describe()); + + // Create a Add Node with scaling parameters. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(add_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create a Bias Node. + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(scale_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Activation Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(bias_op.getOutputTensor()) + .setyDesc(std::get(tensors)) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &add_op, &scale_op, &bias_op, &act_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrB, devPtrI}; + int64_t uids[] = {'x', 'y', 'w', 'z', 'b', 'i'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(6, data_ptrs) + .setUids(6, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +void +run_conv_scale_bias_add_activation_mask(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + int64_t* threshold_dim, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrB, + at::Half* devPtrI, + int* devPtrT, + int* devPtrU, + int axis) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + masked_convbias_descriptors tensors = create_conv_bias_add_act_mask_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, threshold_dim, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the add operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // optional add + auto addDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, addDesc.describe()); + + // Define the activation operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_FWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the genIndex descriptor + auto genIndexDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_GEN_INDEX) + .setMathPrecision(CUDNN_DATA_FLOAT) + .setAxis(axis) + .build(); + DEBUG_CUDNN_MSG(log_buf, genIndexDesc.describe()); + + // Define the lessThan descriptor + auto lessThanDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_CMP_LT) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, lessThanDesc.describe()); + + // Define the greaterThan descriptor + auto greaterThanDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_CMP_GT) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, greaterThanDesc.describe()); + + // Define the logical_or descriptor + auto logicalOrDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_LOGICAL_OR) + .setMathPrecision(CUDNN_DATA_BOOLEAN) + .build(); + DEBUG_CUDNN_MSG(log_buf, logicalOrDesc.describe()); + + // Define the binary_selection descriptor + auto selectionDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_BINARY_SELECT) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, selectionDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a Add Node with scaling parameters. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create a Bias Node. + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(scale_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create a optional add Node. + auto add_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(bias_op.getOutputTensor()) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(addDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, add_op.describe()); + + + // Create an Activation Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(devPtrI ? add_op.getOutputTensor() : bias_op.getOutputTensor()) + .setyDesc(std::get(tensors)) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create a Gen_Index Node. + auto genIndex_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(genIndexDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, genIndex_op.describe()); + + // Create a LessThan Node. + auto lessThan_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(lessThanDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, lessThan_op.describe()); + + // Create a GreaterThan Node. + auto greaterThan_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(greaterThanDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, greaterThan_op.describe()); + + // Create a LogicalOr Node. + auto logicalOr_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(logicalOrDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, logicalOr_op.describe()); + + // Create a Binary_Selection Node. + auto selection_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .settDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(selectionDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, selection_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + if (devPtrI) { + + std::array ops = {&conv_op, &scale_op, &bias_op, &add_op, &act_op, &genIndex_op, &lessThan_op, &greaterThan_op, &logicalOr_op, &selection_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrB, devPtrI, devPtrT, devPtrU}; + int64_t uids[] = {'x', 'y', 'w', 'z', 'b', 'i', 't', 'u'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(8, data_ptrs) + .setUids(8, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } else { + + std::array ops = {&conv_op, &scale_op, &bias_op, &act_op, &genIndex_op, &lessThan_op, &greaterThan_op, &logicalOr_op, &selection_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrB, devPtrT, devPtrU}; + int64_t uids[] = {'x', 'y', 'w', 'z', 'b', 't', 'u'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(7, data_ptrs) + .setUids(7, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +void +run_dconv_add_drelu_dscale(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrR, + at::Half* devPtrI) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + dconv_add_descriptors tensors = create_dconv_add_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // optional add + auto addDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, addDesc.describe()); + + // Define the activation backward operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_BWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the scale backward operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) + .setdxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setdyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create add Node. + auto add_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(addDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, add_op.describe()); + + // TODO: do we need getOutputTensor(), and what it returns in backward case? + // Create an relu backward Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setdyDesc(std::get(tensors)) + .setxDesc(std::get(tensors)) + .setdxDesc(std::get(tensors)) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create a Scale Node. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &add_op, &act_op, &scale_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrR, devPtrI}; + int64_t uids[] = {'x', 'y', 'w', 's', 'r', 'i'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(6, data_ptrs) + .setUids(6, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +void +run_dconv_drelu_dscale_mask(int64_t* x_dim_padded, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + int64_t* w_dim_padded, + int64_t* y_dim_padded, + int64_t* threshold_dim, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + at::Half* devPtrZ, + at::Half* devPtrR, + int* devPtrT, + int* devPtrU, + int axis) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + + // Creates the necessary tensor descriptors + dconv_mask_descriptors tensors = create_dconv_mask_descriptors( + x_dim_padded, pad, convstride, dilation, w_dim_padded, y_dim_padded, threshold_dim, dataType); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + DEBUG_CUDNN_MSG(log_buf, std::get(tensors).describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the activation backward operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_BWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the scale backward operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Define the genIndex descriptor + auto genIndexDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_GEN_INDEX) + .setMathPrecision(CUDNN_DATA_FLOAT) + .setAxis(axis) + .build(); + DEBUG_CUDNN_MSG(log_buf, genIndexDesc.describe()); + + // Define the lessThan descriptor + auto lessThanDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_CMP_LT) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, lessThanDesc.describe()); + + // Define the greaterThan descriptor + auto greaterThanDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_CMP_GT) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, greaterThanDesc.describe()); + + // Define the logical_or descriptor + auto logicalOrDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_LOGICAL_OR) + .setMathPrecision(CUDNN_DATA_BOOLEAN) + .build(); + DEBUG_CUDNN_MSG(log_buf, logicalOrDesc.describe()); + + // Define the binary_selection descriptor + auto selectionDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_BINARY_SELECT) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, selectionDesc.describe()); + + float alpha = 1.0f; + float beta = 0.0f; + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) + .setdxDesc(std::get(tensors)) + .setwDesc(std::get(tensors)) + .setdyDesc(std::get(tensors)) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // TODO: do we need getOutputTensor(), and what it returns in backward case? + // Create an relu backward Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setdyDesc(std::get(tensors)) + .setxDesc(std::get(tensors)) + .setdxDesc(std::get(tensors)) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create a Scale Node. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create a Gen_Index Node. + auto genIndex_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(genIndexDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, genIndex_op.describe()); + + // Create a LessThan Node. + auto lessThan_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(lessThanDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, lessThan_op.describe()); + + // Create a GreaterThan Node. + auto greaterThan_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(greaterThanDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, greaterThan_op.describe()); + + // Create a LogicalOr Node. + auto logicalOr_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(logicalOrDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, logicalOr_op.describe()); + + // Create a Binary_Selection Node. + auto selection_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(std::get(tensors)) + .setbDesc(std::get(tensors)) + .settDesc(std::get(tensors)) + .setyDesc(std::get(tensors)) + .setpwDesc(selectionDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, selection_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op, &act_op, &scale_op, &genIndex_op, &lessThan_op, &greaterThan_op, &logicalOr_op, &selection_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim_padded, pad, convstride, dilation, w_dim_padded, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY, devPtrW, devPtrZ, devPtrR, devPtrT, devPtrU}; + int64_t uids[] = {'x', 'y', 'w', 's', 'r', 't', 'u'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(7, data_ptrs) + .setUids(7, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + +struct bottleneck_forward_status { + + int64_t dimA[4]; + int64_t filterdimA1[4]; + int64_t filterdimA2[4]; + int64_t filterdimA2hh[4]; + int64_t filterdimA3[4]; + int64_t filterdimA4[4]; + + int64_t threshdim[4]; + + int axis[4]; + + int64_t outdimA0[4]; + int64_t outdimA1[4]; + int64_t outdimA1b[4]; // out1_pad + int64_t outdimA2[4]; + int64_t outdimA3[4]; + int64_t outdimA4[4]; + + int64_t padA[2]; + int64_t padA1[2]; + int64_t padA2[2]; // halo padding + int64_t dilationA[2]; + int64_t convstrideA[2]; + int64_t convstride1X1[2]; + + int64_t outdim0[4]; // halo input shape + int64_t outdim1[4]; + int64_t outdim1b[4]; + int64_t outdim2[4]; + int64_t outdim3[4]; + int64_t outdim4[4]; // halo output shape + + void init(bool explicit_nhwc, int stride_1X1, std::vector inputs) { + dimA[0] = dimA[1] = dimA[2] = dimA[3] = 0; + filterdimA1[0] = filterdimA1[1] = filterdimA1[2] = filterdimA1[3] = 0; + filterdimA2[0] = filterdimA2[1] = filterdimA2[2] = filterdimA2[3] = 0; + filterdimA2hh[0] = filterdimA2hh[1] = filterdimA2hh[2] = filterdimA2hh[3] = 0; + filterdimA3[0] = filterdimA3[1] = filterdimA3[2] = filterdimA3[3] = 0; + filterdimA4[0] = filterdimA4[1] = filterdimA4[2] = filterdimA4[3] = 0; + threshdim[0] = threshdim[1] = threshdim[2] = threshdim[3] = 1; + + // All dim calculation after this order of n,c,h,w + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 3; + axis[2] = 1; + axis[3] = 2; + } else { + axis[0] = 0; + axis[1] = 1; + axis[2] = 2; + axis[3] = 3; + } + + for (int dim=0;dim<4;dim++) { + dimA[dim] = inputs[0].size(axis[dim]); + filterdimA1[dim] = inputs[1].size(axis[dim]); + filterdimA2[dim] = inputs[2].size(axis[dim]); + filterdimA3[dim] = inputs[3].size(axis[dim]); + } + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]) { + for (int dim=0;dim<4;dim++) { + filterdimA4[dim] = inputs[10].size(axis[dim]); + } + } + for (int dim=0;dim<4;dim++) { + if (dim == 2) { + filterdimA2hh[dim] = 1; + } else { + filterdimA2hh[dim] = filterdimA2[dim]; + } + } + + // output dim in n,c,h,w used by backend + outdimA0[0] = outdimA0[1] = outdimA0[2] = outdimA0[3] = 0; + outdimA1[0] = outdimA1[1] = outdimA1[2] = outdimA1[3] = 0; + outdimA1b[0] = outdimA1b[1] = outdimA1b[2] = outdimA1b[3] = 0; + outdimA2[0] = outdimA2[1] = outdimA2[2] = outdimA2[3] = 0; + outdimA3[0] = outdimA3[1] = outdimA3[2] = outdimA3[3] = 0; + outdimA4[0] = outdimA4[1] = outdimA4[2] = outdimA4[3] = 0; + + // use these fixed value for test run + padA[0] = 0; padA[1] = 0; + padA1[0] = 1; padA1[1] = 1; + padA2[0] = 0; padA2[1] = 1; + dilationA[0] = 1; dilationA[1] = 1; + convstrideA[0] = 1; convstrideA[1] = 1; + convstride1X1[0] = stride_1X1; convstride1X1[1] = stride_1X1; + + // compute output from pad/stride/dilation + outdimA1[0] = dimA[0]; + outdimA1[1] = filterdimA1[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA1[dim + 2] = getFwdConvOutputDim(dimA[dim + 2], padA[dim], filterdimA1[dim + 2], convstride1X1[dim], dilationA[dim]); + } + for (int dim = 0; dim < 4; dim++) { + if (dim == 2) { + outdimA1b[dim] = outdimA1[dim] + 2; + } else { + outdimA1b[dim] = outdimA1[dim]; + } + } + + outdimA2[0] = outdimA1[0]; + outdimA2[1] = filterdimA2[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA2[dim + 2] = getFwdConvOutputDim(outdimA1[dim + 2], padA1[dim], filterdimA2[dim + 2], convstrideA[dim], dilationA[dim]); + } + + for (int dim = 0; dim < 4; dim++) { + if (dim == 2) { + outdimA0[dim] = 3; + outdimA4[dim] = 1; + } else { + outdimA0[dim] = outdimA1[dim]; + outdimA4[dim] = outdimA2[dim]; + } + } + + outdimA3[0] = outdimA2[0]; + outdimA3[1] = filterdimA3[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA3[dim + 2] = getFwdConvOutputDim(outdimA2[dim + 2], padA[dim], filterdimA3[dim + 2], convstrideA[dim], dilationA[dim]); + } + + // Create output tensor in the correct shape in pytorch's view + outdim1[0] = outdim1[1] = outdim1[2] = outdim1[3] = 0; + outdim1b[0] = outdim1b[1] = outdim1b[2] = outdim1b[3] = 0; + outdim2[0] = outdim2[1] = outdim2[2] = outdim2[3] = 0; + outdim3[0] = outdim3[1] = outdim3[2] = outdim3[3] = 0; + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 2; + axis[2] = 3; + axis[3] = 1; + } + for (int dim=0;dim<4;dim++) { + outdim0[dim] = outdimA0[axis[dim]]; + outdim1[dim] = outdimA1[axis[dim]]; + outdim1b[dim] = outdimA1b[axis[dim]]; + outdim2[dim] = outdimA2[axis[dim]]; + outdim3[dim] = outdimA3[axis[dim]]; + outdim4[dim] = outdimA4[axis[dim]]; + } + } +}; + +bottleneck_forward_status forward_state; + +} // end of anonymous namespace + +std::vector bottleneck_forward_init(bool explicit_nhwc, int stride_1X1, std::vector inputs) { + // NB! Bottleneck_forward and bottleneck_backward are NOT thread safe method. + // NB! We use a global object to store state. + forward_state.init(explicit_nhwc, stride_1X1, inputs); + + // create output vector + std::vector outputs; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + //printf("outdim1 = (%d,%d,%d,%d)\n",forward_state.outdim1[0],forward_state.outdim1[1],forward_state.outdim1[2],forward_state.outdim1[3]); + auto out1 = at::empty(forward_state.outdim1, inputs[0].type(), output_format); + auto out2 = at::empty(forward_state.outdim2, inputs[0].type(), output_format); + auto out3 = at::empty(forward_state.outdim3, inputs[0].type(), output_format); + + outputs.push_back(out1); + outputs.push_back(out2); + outputs.push_back(out3); + + return outputs; +} + +// inputs contains x,w,z,b,(i) +void bottleneck_forward_out1(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs) { + + std::cout << std::fixed; + + // run + at::Half* x = inputs[0].data_ptr(); + at::Half* w = inputs[1].data_ptr(); + at::Half* z = inputs[4].data_ptr(); + at::Half* b = inputs[7].data_ptr(); + auto out1 = outputs[0]; + at::Half* y1 = out1.data_ptr(); + + run_conv_scale_bias_add_activation(forward_state.dimA, + forward_state.padA, + forward_state.convstride1X1, + forward_state.dilationA, + forward_state.filterdimA1, + forward_state.outdimA1, + CUDNN_DATA_HALF, + x, + w, + y1, + z, + b, + nullptr); + + DEBUG_MSG("[DEBUG] new relu1 : " << out1.to(at::kFloat).sum().item()); +} + +// computes halo (top or bottom) from fat halo input. +// fat halo input is 3 pixels wide in H. +at::Tensor bottleneck_forward_out2_halo(bool explicit_nhwc, at::Tensor fat_halo_y1, std::vector inputs) { + + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // run + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[5].data_ptr(); + at::Half* b = inputs[8].data_ptr(); + + at::Half* y1 = fat_halo_y1.data_ptr(); + + auto halo_y2 = at::empty(forward_state.outdim4, inputs[0].type(), output_format); + at::Half* y2 = halo_y2.data_ptr(); + + run_conv_scale_bias_add_activation(forward_state.outdimA0, + forward_state.padA2, + forward_state.convstrideA, + forward_state.dilationA, + forward_state.filterdimA2, + forward_state.outdimA4, + CUDNN_DATA_HALF, + y1, + w, + y2, + z, + b, + nullptr); + + return halo_y2; +} + +// compute halo correction term (top or bottom) from slim halo input (N,C,1,W). +// slim halo input is 1 pixel wide in H. +at::Tensor bottleneck_forward_out2_halo_corr(bool explicit_nhwc, at::Tensor slim_halo_y1, std::vector inputs, at::Tensor w1by3, at::Tensor out2_part_halo) { + + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // run + at::Half* w = w1by3.data_ptr(); // C,C,1,3 + at::Half* z = inputs[5].data_ptr(); + at::Half* b = inputs[8].data_ptr(); + + at::Half* y1 = slim_halo_y1.data_ptr(); + + at::Half* prev_out2 = out2_part_halo.data_ptr(); + + auto halo_y2 = at::empty(forward_state.outdim4, inputs[0].type(), output_format); + at::Half* y2 = halo_y2.data_ptr(); + + run_conv_add_scale_bias_activation(forward_state.outdimA4, + forward_state.padA2, + forward_state.convstrideA, + forward_state.dilationA, + forward_state.filterdimA2hh, + forward_state.outdimA4, + CUDNN_DATA_HALF, + y1, + w, + y2, + z, + b, + prev_out2); + + return halo_y2; +} + +void bottleneck_forward_out2(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs) { + + std::cout << std::fixed; + + // from _out1 method + at::Half* x = inputs[0].data_ptr(); + auto out1 = outputs[0]; + at::Half* y1 = out1.data_ptr(); + + // run + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[5].data_ptr(); + at::Half* b = inputs[8].data_ptr(); + auto out2 = outputs[1]; + at::Half* y2 = out2.data_ptr(); + + //printf("forward_state.outdimA1 = {%d,%d,%d,%d}\n",forward_state.outdimA1[0],forward_state.outdimA1[1],forward_state.outdimA1[2],forward_state.outdimA1[3]); + //printf("forward_state.padA1 = {%d,%d}\n",forward_state.padA1[0],forward_state.padA1[1]); + //printf("forward_state.convstrideA = {%d,%d}\n",forward_state.convstrideA[0],forward_state.convstrideA[1]); + //printf("forward_state.dilationA = {%d,%d}\n",forward_state.dilationA[0],forward_state.dilationA[1]); + //printf("forward_state.filterdimA2 = {%d,%d,%d,%d}\n",forward_state.filterdimA2[0],forward_state.filterdimA2[1],forward_state.filterdimA2[2],forward_state.filterdimA2[3]); + //printf("forward_state.outdimA2 = {%d,%d,%d,%d}\n",forward_state.outdimA2[0],forward_state.outdimA2[1],forward_state.outdimA2[2],forward_state.outdimA2[3]); + run_conv_scale_bias_add_activation(forward_state.outdimA1, + forward_state.padA1, + forward_state.convstrideA, + forward_state.dilationA, + forward_state.filterdimA2, + forward_state.outdimA2, + CUDNN_DATA_HALF, + y1, + w, + y2, + z, + b, + nullptr); + DEBUG_MSG("[DEBUG] new relu2 : " << out2.to(at::kFloat).sum().item()); +} + +void bottleneck_forward_out2_mask(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor thresholdTop, at::Tensor thresholdBottom) { + + std::cout << std::fixed; + + // from _out1 method + at::Half* x = inputs[0].data_ptr(); + auto out1 = outputs[0]; + at::Half* y1 = out1.data_ptr(); + + // run + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[5].data_ptr(); + at::Half* b = inputs[8].data_ptr(); + auto out2 = outputs[1]; + at::Half* y2 = out2.data_ptr(); + + //printf("forward_state.outdimA1 = {%d,%d,%d,%d}\n",forward_state.outdimA1[0],forward_state.outdimA1[1],forward_state.outdimA1[2],forward_state.outdimA1[3]); + //printf("forward_state.padA1 = {%d,%d}\n",forward_state.padA1[0],forward_state.padA1[1]); + //printf("forward_state.convstrideA = {%d,%d}\n",forward_state.convstrideA[0],forward_state.convstrideA[1]); + //printf("forward_state.dilationA = {%d,%d}\n",forward_state.dilationA[0],forward_state.dilationA[1]); + //printf("forward_state.filterdimA2 = {%d,%d,%d,%d}\n",forward_state.filterdimA2[0],forward_state.filterdimA2[1],forward_state.filterdimA2[2],forward_state.filterdimA2[3]); + //printf("forward_state.outdimA2 = {%d,%d,%d,%d}\n",forward_state.outdimA2[0],forward_state.outdimA2[1],forward_state.outdimA2[2],forward_state.outdimA2[3]); + run_conv_scale_bias_add_activation_mask(forward_state.outdimA1, + forward_state.padA1, + forward_state.convstrideA, + forward_state.dilationA, + forward_state.filterdimA2, + forward_state.outdimA2, + forward_state.threshdim, + CUDNN_DATA_HALF, + y1, + w, + y2, + z, + b, + nullptr, + thresholdTop.data_ptr(), + thresholdBottom.data_ptr(), + 2); // axis == 1 -> Does this assume explicit NHWC? + DEBUG_MSG("[DEBUG] new relu2 : " << out2.to(at::kFloat).sum().item()); +} + +void bottleneck_forward_out2_pad(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor out1_pad) { + + std::cout << std::fixed; + + // from _out1 method + at::Half* x = inputs[0].data_ptr(); + auto out1 = outputs[0]; + at::Half* y1 = out1_pad.data_ptr(); + + // run + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[5].data_ptr(); + at::Half* b = inputs[8].data_ptr(); + auto out2 = outputs[1]; + at::Half* y2 = out2.data_ptr(); + + //printf("forward_state.outdimA1 = {%d,%d,%d,%d}\n",forward_state.outdimA1[0],forward_state.outdimA1[1],forward_state.outdimA1[2],forward_state.outdimA1[3]); + //printf("forward_state.padA1 = {%d,%d}\n",forward_state.padA1[0],forward_state.padA1[1]); + //printf("forward_state.convstrideA = {%d,%d}\n",forward_state.convstrideA[0],forward_state.convstrideA[1]); + //printf("forward_state.dilationA = {%d,%d}\n",forward_state.dilationA[0],forward_state.dilationA[1]); + //printf("forward_state.filterdimA2 = {%d,%d,%d,%d}\n",forward_state.filterdimA2[0],forward_state.filterdimA2[1],forward_state.filterdimA2[2],forward_state.filterdimA2[3]); + //printf("forward_state.outdimA2 = {%d,%d,%d,%d}\n",forward_state.outdimA2[0],forward_state.outdimA2[1],forward_state.outdimA2[2],forward_state.outdimA2[3]); + run_conv_scale_bias_add_activation(forward_state.outdimA1b, + forward_state.padA2, + forward_state.convstrideA, + forward_state.dilationA, + forward_state.filterdimA2, + forward_state.outdimA2, + CUDNN_DATA_HALF, + y1, + w, + y2, + z, + b, + nullptr); + DEBUG_MSG("[DEBUG] new relu2 : " << out2.to(at::kFloat).sum().item()); +} + +void bottleneck_forward_rest(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs) { + + std::cout << std::fixed; + + // from _out1 method + at::Half* x = inputs[0].data_ptr(); + + // create output of conv3 + auto out3 = outputs[2]; + at::Half* y3 = out3.data_ptr(); + + // create output of conv4 that may exist + auto identity = at::empty_like(out3); + at::Half* yi = identity.data_ptr(); + + at::Half *w, *z, *b; + + if (stride_1X1 != 1 || forward_state.filterdimA3[0] != forward_state.dimA[1]){ + + w = inputs[10].data_ptr(); + z = inputs[11].data_ptr(); + b = inputs[12].data_ptr(); + run_conv_scale_bias(forward_state.dimA, + forward_state.padA, + forward_state.convstride1X1, + forward_state.dilationA, + forward_state.filterdimA4, + forward_state.outdimA3, + CUDNN_DATA_HALF, + x, + w, + yi, + z, + b); + DEBUG_MSG("[DEBUG] new downsample : " << identity.to(at::kFloat).sum().item()); + } + else { + yi = x; + } + + auto out2 = outputs[1]; + at::Half* y2 = out2.data_ptr(); + + w = inputs[3].data_ptr(); + z = inputs[6].data_ptr(); + b = inputs[9].data_ptr(); + + run_conv_scale_bias_add_activation(forward_state.outdimA2, + forward_state.padA, + forward_state.convstrideA, + forward_state.dilationA, + forward_state.filterdimA3, + forward_state.outdimA3, + CUDNN_DATA_HALF, + y2, + w, + y3, + z, + b, + yi); + DEBUG_MSG("[DEBUG] new relu3 : " << out3.to(at::kFloat).sum().item()); +} + +namespace { + +struct bottleneck_backward_state { + + int64_t dimA[4]; + int64_t filterdimA1[4]; + int64_t filterdimA2[4]; + int64_t filterdimA3[4]; + int64_t filterdimA4[4]; + int64_t filterdimA2hh[4]; // Cin,Cout,1,3 + int64_t threshdim[4]; + + int axis[4]; + + int64_t outdimA1[4]; // grad_out1 + int64_t outdimA1b[4]; // out1_pad + int64_t outdimA2[4]; // grad_out2 + int64_t outdimA3[4]; + int64_t outdimA1h[4]; // output: grad_out1 halo (H=3) + int64_t outdimA2h[4]; // input : grad_out2 halo cells (H=3) + int64_t outdimA1hh[4]; // input: grad_out2 halo (H=1) + int64_t outdimA2hh[4]; // input: out1 halo (H=1) + + int64_t padA[2]; + int64_t padA1[2]; + int64_t padA2[2]; + int64_t dilationA[2]; + int64_t convstrideA[2]; + int64_t convstride1X1[2]; + + int64_t filterdim2hh[4]; // Cin,1,3,Cout + + int64_t outdim1[4]; + int64_t outdim1b[4]; + int64_t outdim2[4]; + int64_t outdim3[4]; + int64_t outdim1h[4]; + int64_t outdim1hh[4]; + + void init(bool explicit_nhwc, int stride_1X1, std::vector inputs) { + // setup dimensions + dimA[0] = dimA[1] = dimA[2] = dimA[3] = 0; + filterdimA1[0] = filterdimA1[1] = filterdimA1[2] = filterdimA1[3] = 0; + filterdimA2[0] = filterdimA2[1] = filterdimA2[2] = filterdimA2[3] = 0; + filterdimA3[0] = filterdimA3[1] = filterdimA3[2] = filterdimA3[3] = 0; + filterdimA4[0] = filterdimA4[1] = filterdimA4[2] = filterdimA4[3] = 0; + filterdimA2hh[0] = filterdimA2hh[1] = filterdimA2hh[2] = filterdimA2hh[3] = 0; + threshdim[0] = threshdim[1] = threshdim[2] = threshdim[3] = 1; + + // All dim calculation after this order of n,c,h,w + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 3; + axis[2] = 1; + axis[3] = 2; + } else { + axis[0] = 0; + axis[1] = 1; + axis[2] = 2; + axis[3] = 3; + } + + for (int dim=0;dim<4;dim++) { + dimA[dim] = inputs[0].size(axis[dim]); + filterdimA1[dim] = inputs[1].size(axis[dim]); + filterdimA2[dim] = inputs[2].size(axis[dim]); + filterdimA3[dim] = inputs[3].size(axis[dim]); + } + if (stride_1X1 != 1 || filterdimA3[0] != dimA[1]) { + for (int dim=0;dim<4;dim++) { + filterdimA4[dim] = inputs[14].size(axis[dim]); + } + } + + for (int dim=0;dim<4;dim++) { + if (dim == 2) { + filterdimA2hh[dim] = 1; + } else { + filterdimA2hh[dim] = filterdimA2[dim]; + } + } + + // output dim in n,c,h,w used by backend + outdimA1[0] = outdimA1[1] = outdimA1[2] = outdimA1[3] = 0; + outdimA1b[0] = outdimA1b[1] = outdimA1b[2] = outdimA1b[3] = 0; + outdimA2[0] = outdimA2[1] = outdimA2[2] = outdimA2[3] = 0; + outdimA3[0] = outdimA3[1] = outdimA3[2] = outdimA3[3] = 0; + outdimA1h[0] = outdimA1h[1] = outdimA1h[2] = outdimA1h[3] = 0; + outdimA2h[0] = outdimA2h[1] = outdimA2h[2] = outdimA2h[3] = 0; + outdimA1hh[0] = outdimA1hh[1] = outdimA1hh[2] = outdimA1hh[3] = 0; + outdimA2hh[0] = outdimA2hh[1] = outdimA2hh[2] = outdimA2hh[3] = 0; + + // use these fixed value for test run + padA[0] = 0; padA[1] = 0; + padA1[0] = 1; padA1[1] = 1; + padA2[0] = 0; padA2[1] = 1; + dilationA[0] = 1; dilationA[1] = 1; + convstrideA[0] = 1; convstrideA[1] = 1; + convstride1X1[0] = stride_1X1; convstride1X1[1] = stride_1X1; + + // compute output from pad/stride/dilation + outdimA1[0] = dimA[0]; + outdimA1[1] = filterdimA1[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA1[dim + 2] = getFwdConvOutputDim(dimA[dim + 2], padA[dim], filterdimA1[dim + 2], convstride1X1[dim], dilationA[dim]); + } + for (int dim = 0; dim < 4; dim++) { + if (dim == 2) { + outdimA1b[dim] = outdimA1[dim] + 2; + } else { + outdimA1b[dim] = outdimA1[dim]; + } + } + + outdimA2[0] = outdimA1[0]; + outdimA2[1] = filterdimA2[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA2[dim + 2] = getFwdConvOutputDim(outdimA1[dim + 2], padA1[dim], filterdimA2[dim + 2], convstrideA[dim], dilationA[dim]); + } + + outdimA3[0] = outdimA2[0]; + outdimA3[1] = filterdimA3[0]; + for (int dim = 0; dim < 2; dim++) { + outdimA3[dim + 2] = getFwdConvOutputDim(outdimA2[dim + 2], padA[dim], filterdimA3[dim + 2], convstrideA[dim], dilationA[dim]); + } + + for (int dim = 0; dim < 4; dim++) { + if (dim == 2) { + outdimA1h[dim] = 3; + outdimA2h[dim] = 3; + outdimA1hh[dim] = 1; + outdimA2hh[dim] = 1; + } else { + outdimA1h[dim] = outdimA1[dim]; + outdimA2h[dim] = outdimA2[dim]; + outdimA1hh[dim] = outdimA1[dim]; + outdimA2hh[dim] = outdimA2[dim]; + } + } + + // Create output tensor in the correct shape in pytorch's view + outdim1[0] = outdim1[1] = outdim1[2] = outdim1[3] = 0; + outdim1b[0] = outdim1b[1] = outdim1b[2] = outdim1b[3] = 0; + outdim2[0] = outdim2[1] = outdim2[2] = outdim2[3] = 0; + outdim3[0] = outdim3[1] = outdim3[2] = outdim3[3] = 0; + outdim1h[0] = outdim1h[1] = outdim1h[2] = outdim1h[3] = 0; + outdim1hh[0] = outdim1hh[1] = outdim1hh[2] = outdim1hh[3] = 0; + filterdim2hh[0] = filterdim2hh[1] = filterdim2hh[2] = filterdim2hh[3] = 0; + if (explicit_nhwc) { + axis[0] = 0; + axis[1] = 2; + axis[2] = 3; + axis[3] = 1; + } + for (int dim=0;dim<4;dim++) { + outdim1[dim] = outdimA1[axis[dim]]; + outdim1b[dim] = outdimA1b[axis[dim]]; + outdim2[dim] = outdimA2[axis[dim]]; + outdim3[dim] = outdimA3[axis[dim]]; + outdim1h[dim] = outdimA1h[axis[dim]]; + outdim1hh[dim] = outdimA1hh[axis[dim]]; + filterdim2hh[dim] = filterdimA2hh[axis[dim]]; + } + } +}; + +bottleneck_backward_state backward_state; + +} + +std::vector bottleneck_backward_init(bool explicit_nhwc, int stride_1X1, std::vector inputs) { + + std::cout << std::fixed; + + backward_state.init(explicit_nhwc, stride_1X1, inputs); + + // create output vector + std::vector outputs; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + auto grad_x = at::empty_like(inputs[0]); + auto wgrad1 = at::empty_like(inputs[1]); + auto wgrad2 = at::empty_like(inputs[2]); + auto wgrad3 = at::empty_like(inputs[3]); + + outputs.push_back(grad_x); + outputs.push_back(wgrad1); + outputs.push_back(wgrad2); + outputs.push_back(wgrad3); + if (stride_1X1 != 1 || backward_state.filterdimA3[0] != backward_state.dimA[1]) { + auto wgrad4 = at::empty_like(inputs[14]); + outputs.push_back(wgrad4); + } + + return outputs; +} + +void bottleneck_backward_wgrad3(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs) { + + // dconv3+drelu2+dscale2 + at::Half* conv_in = inputs[13].data_ptr(); + at::Half* dy3 = inputs[10].data_ptr(); + + // wgrad + auto wgrad3 = outputs[3]; + at::Half* dw3 = wgrad3.data_ptr(); + run_dconv(backward_state.outdimA2, + backward_state.padA, + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA3, + backward_state.outdimA3, + CUDNN_DATA_HALF, + conv_in, + dw3, + dy3, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + DEBUG_MSG("[DEBUG] new wgrad3 : " << wgrad3.to(at::kFloat).sum().item()); + +} + +at::Tensor bottleneck_backward_grad_out2(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dconv3+drelu2+dscale2 + at::Half* conv_in = inputs[13].data_ptr(); + at::Half* dy3 = inputs[10].data_ptr(); + + DEBUG_MSG("[DEBUG] new dconv3 : " << inputs[10].to(at::kFloat).sum().item()); + + // dgrad + auto grad_out2 = at::empty(backward_state.outdim2, inputs[0].type(), output_format); + at::Half* dy2 = grad_out2.data_ptr(); + at::Half* w = inputs[3].data_ptr(); + at::Half* z = inputs[5].data_ptr(); + + at::Half* relu2 = inputs[13].data_ptr(); + + run_dconv_drelu_dscale(backward_state.outdimA2, + backward_state.padA, + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA3, + backward_state.outdimA3, + CUDNN_DATA_HALF, + dy2, + w, + dy3, + z, + relu2); + + // do halo exchange of dy2 here + + DEBUG_MSG("[DEBUG] new dconv2 : " << grad_out2.to(at::kFloat).sum().item()); + + return grad_out2; +} + +at::Tensor bottleneck_backward_grad_out1(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor grad_out2) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2 = grad_out2.data_ptr(); + + // dgrad + auto grad_out1 = at::empty(backward_state.outdim1, inputs[0].type(), output_format); + at::Half* dy1 = grad_out1.data_ptr(); + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[4].data_ptr(); + + at::Half* relu1 = inputs[12].data_ptr(); + //printf("relu.shape = [%d,%d,%d,%d]\n",inputs[12].size(0),inputs[12].size(1),inputs[12].size(2),inputs[12].size(3)); + + // fused dgrad + //printf("backward_state.outdim1 = {%d,%d,%d,%d}\n",backward_state.outdim1[0],backward_state.outdim1[1],backward_state.outdim1[2],backward_state.outdim1[3]); + run_dconv_drelu_dscale(backward_state.outdimA1, + backward_state.padA1, + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2, + backward_state.outdimA2, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + z, + relu1); + + return grad_out1; +} + +at::Tensor bottleneck_backward_grad_out1_mask(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor grad_out2, at::Tensor thresholdTop, at::Tensor thresholdBottom) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2 = grad_out2.data_ptr(); + + // dgrad + auto grad_out1 = at::empty(backward_state.outdim1, inputs[0].type(), output_format); + at::Half* dy1 = grad_out1.data_ptr(); + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[4].data_ptr(); + + at::Half* relu1 = inputs[12].data_ptr(); + //printf("relu.shape = [%d,%d,%d,%d]\n",inputs[12].size(0),inputs[12].size(1),inputs[12].size(2),inputs[12].size(3)); + + // fused dgrad + run_dconv_drelu_dscale_mask(backward_state.outdimA1, + backward_state.padA1, + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2, + backward_state.outdimA2, + backward_state.threshdim, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + z, + relu1, + thresholdTop.data_ptr(), + thresholdBottom.data_ptr(), + 2); + + return grad_out1; +} + +// perform backward data 1x3 convolution (grad_out * w_rot180) on grad_out2 input of shape [N,1,W,C] with padding=(0,1) to produce output of shape [N,1,W,C] +at::Tensor bottleneck_backward_grad_out1_halo_corr(bool explicit_nhwc, int stride_1X1, std::vector inputs, at::Tensor w1by3, std::vector outputs, at::Tensor grad_out2_halo, at::Tensor relu1_halo, at::Tensor part_grad_out1) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2h = grad_out2_halo.data_ptr(); + + // dgrad + auto grad_out1_halo = at::empty(backward_state.outdim1hh, inputs[0].type(), output_format); + at::Half* dy1h = grad_out1_halo.data_ptr(); + //at::Half* w = inputs[2].data_ptr(); // use w1by3 instead, which is a sliced version of inputs[2] + at::Half* w = w1by3.data_ptr(); + at::Half* z = inputs[4].data_ptr(); + at::Half* relu1h = relu1_halo.data_ptr(); + at::Half* pdy1h = part_grad_out1.data_ptr(); + + //printf("relu.shape = [%d,%d,%d,%d]\n",relu1_halo.size(0),relu1_halo.size(1),relu1_halo.size(2),relu1_halo.size(3)); + // fused dgrad + //printf("backward_state.outdimA1h = {%d,%d,%d,%d}\n",backward_state.outdimA1h[0],backward_state.outdimA1h[1],backward_state.outdimA1h[2],backward_state.outdimA1h[3]); + //printf("backward_state.outdimA2h = {%d,%d,%d,%d}\n",backward_state.outdimA2h[0],backward_state.outdimA2h[1],backward_state.outdimA2h[2],backward_state.outdimA2h[3]); + //printf("backward_state.filterdimA2 = {%d,%d,%d,%d}\n",backward_state.filterdimA2[0],backward_state.filterdimA2[1],backward_state.filterdimA2[2],backward_state.filterdimA2[3]); + run_dconv_add_drelu_dscale(backward_state.outdimA1hh, + backward_state.padA2, // 0,1 + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2hh, // C,1,3,C + backward_state.outdimA2hh, + CUDNN_DATA_HALF, + dy1h, + w, + dy2h, + z, + relu1h, + pdy1h); + + return grad_out1_halo; +} + +// perform backward data 3x3 convolution (grad_out * w_rot180) on grad_out2 input of shape [N,3,W,C] with padding=(1,1) to produce output of shape [N,3,W,C] +at::Tensor bottleneck_backward_grad_out1_halo(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor grad_out2_halo, at::Tensor relu1_halo) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2h = grad_out2_halo.data_ptr(); + + // dgrad + auto grad_out1_halo = at::empty(backward_state.outdim1h, inputs[0].type(), output_format); + at::Half* dy1h = grad_out1_halo.data_ptr(); + at::Half* w = inputs[2].data_ptr(); + at::Half* z = inputs[4].data_ptr(); + + at::Half* relu1h = relu1_halo.data_ptr(); + //printf("relu.shape = [%d,%d,%d,%d]\n",relu1_halo.size(0),relu1_halo.size(1),relu1_halo.size(2),relu1_halo.size(3)); + // fused dgrad + //printf("backward_state.outdimA1h = {%d,%d,%d,%d}\n",backward_state.outdimA1h[0],backward_state.outdimA1h[1],backward_state.outdimA1h[2],backward_state.outdimA1h[3]); + //printf("backward_state.outdimA2h = {%d,%d,%d,%d}\n",backward_state.outdimA2h[0],backward_state.outdimA2h[1],backward_state.outdimA2h[2],backward_state.outdimA2h[3]); + //printf("backward_state.filterdimA2 = {%d,%d,%d,%d}\n",backward_state.filterdimA2[0],backward_state.filterdimA2[1],backward_state.filterdimA2[2],backward_state.filterdimA2[3]); + run_dconv_drelu_dscale(backward_state.outdimA1h, + backward_state.padA1, + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2, + backward_state.outdimA2h, + CUDNN_DATA_HALF, + dy1h, + w, + dy2h, + z, + relu1h); + + return grad_out1_halo; +} + +void bottleneck_backward_wgrad2_pad(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor input, at::Tensor grad_out2) { + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2 = grad_out2.data_ptr(); + + // dconv2+drelu1+dscale1 + at::Half* conv_in = input.data_ptr(); + + // wgrad + auto wgrad2 = outputs[2]; + at::Half* dw2 = wgrad2.data_ptr(); + + //printf("outdimA1b = (%d,%d,%d,%d)\n",backward_state.outdimA1b[0],backward_state.outdimA1b[1],backward_state.outdimA1b[2],backward_state.outdimA1b[3]); + //printf("backward_state.padA2 = {%d,%d}\n",backward_state.padA2[0],backward_state.padA2[1]); + run_dconv(backward_state.outdimA1b, // conv_in.shape (including H halos) + backward_state.padA2, // 0, 1 + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2, // dw2.shape + backward_state.outdimA2, // dy2.shape + CUDNN_DATA_HALF, + conv_in, + dw2, + dy2, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + DEBUG_MSG("[DEBUG] new wgrad2 : " << wgrad2.to(at::kFloat).sum().item()); +} + +void bottleneck_backward_wgrad2(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor grad_out2) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2 = grad_out2.data_ptr(); + + // dconv2+drelu1+dscale1 + at::Half* conv_in = inputs[12].data_ptr(); + + // wgrad + auto wgrad2 = outputs[2]; + at::Half* dw2 = wgrad2.data_ptr(); + + //printf("outdimA1 = (%d,%d,%d,%d)\n",backward_state.outdimA1[0],backward_state.outdimA1[1],backward_state.outdimA1[2],backward_state.outdimA1[3]); + run_dconv(backward_state.outdimA1, + backward_state.padA1, + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2, + backward_state.outdimA2, + CUDNN_DATA_HALF, + conv_in, + dw2, + dy2, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + DEBUG_MSG("[DEBUG] new wgrad2 : " << wgrad2.to(at::kFloat).sum().item()); +} + +// compute halo cells for input volume of dimension [N,1,W,C] with padding=(0,1) to produce output volume of dimension [N,1,W,C] +// input and grad_out2_halo tensors are all of same shape +// output tensor is of shape [Cin,1,3,Cout] (regular filter dims are [Cin,3,3,Cout] +at::Tensor bottleneck_backward_wgrad2_halo(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor input, at::Tensor grad_out2_halo) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2 = grad_out2_halo.data_ptr(); + + // dconv2+drelu1+dscale1 + at::Half* conv_in = input.data_ptr(); + + // wgrad + auto wgrad2_halo = at::empty(backward_state.filterdim2hh, input.type(), output_format); + at::Half* dw2 = wgrad2_halo.data_ptr(); + + //printf("backward_state.outdimA1hh = {%d,%d,%d,%d}\n",backward_state.outdimA1hh[0],backward_state.outdimA1hh[1],backward_state.outdimA1hh[2],backward_state.outdimA1hh[3]); + //printf("backward_state.outdimA2hh = {%d,%d,%d,%d}\n",backward_state.outdimA2hh[0],backward_state.outdimA2hh[1],backward_state.outdimA2hh[2],backward_state.outdimA2hh[3]); + //printf("backward_state.filterdim2hh = {%d,%d,%d,%d}\n",backward_state.filterdim2hh[0],backward_state.filterdim2hh[1],backward_state.filterdim2hh[2],backward_state.filterdim2hh[3]); + //printf("backward_state.filterdimA2hh = {%d,%d,%d,%d}\n",backward_state.filterdimA2hh[0],backward_state.filterdimA2hh[1],backward_state.filterdimA2hh[2],backward_state.filterdimA2hh[3]); + //printf("backward_state.padA2 = {%d,%d}\n",backward_state.padA2[0],backward_state.padA2[1]); + run_dconv(backward_state.outdimA1hh, // N,C,1,W + backward_state.padA2, // 0, 1 + backward_state.convstrideA, + backward_state.dilationA, + backward_state.filterdimA2hh, // Cin,Cout,1,3 + backward_state.outdimA2hh, // N,C,1,W + CUDNN_DATA_HALF, + conv_in, + dw2, + dy2, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + return wgrad2_halo; +} + +void bottleneck_backward_wgrad1(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor grad_out1) { + + at::Half* x = inputs[0].data_ptr(); + at::Half* dy1 = grad_out1.data_ptr(); + + // dconv1+add + // wgrad + auto wgrad1 = outputs[1]; + at::Half* dw1 = wgrad1.data_ptr(); + run_dconv(backward_state.dimA, + backward_state.padA, + backward_state.convstride1X1, + backward_state.dilationA, + backward_state.filterdimA1, + backward_state.outdimA1, + CUDNN_DATA_HALF, + x, + dw1, + dy1, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + +} + +void bottleneck_backward_rest(bool explicit_nhwc, int stride_1X1, std::vector inputs, std::vector outputs, at::Tensor grad_out2, at::Tensor grad_out1) { + + bool requires_grad = inputs[0].requires_grad(); + + std::cout << std::fixed; + auto output_format = explicit_nhwc ? at::MemoryFormat::Contiguous : at::MemoryFormat::ChannelsLast; + + // dgrad + at::Half* dy2 = grad_out2.data_ptr(); + at::Half* dy1 = grad_out1.data_ptr(); + +/* + // backward strided conv cannot be fused + // if stride == 1 but channel changes, we can fuse here + if (stride_1X1 != 1){ + // dgrad + run_dconv(outdimA1, + padA1, + convstride1X1, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + + // mul fused mask + grad_out1.mul_(inputs[15]); + } + else { + at::Half* relu1 = inputs[12].data_ptr(); + // fused dgrad + run_dconv_drelu_dscale(outdimA1, + padA1, + convstride1X1, + dilationA, + filterdimA2, + outdimA2, + CUDNN_DATA_HALF, + dy1, + w, + dy2, + z, + relu1); + } +*/ + DEBUG_MSG("[DEBUG] new dconv1 : " << grad_out1.to(at::kFloat).sum().item()); + + // create grads of conv4 that may exist + auto grad_x_conv4 = at::empty_like(inputs[0]); + at::Half* dx_conv4 = grad_x_conv4.data_ptr(); + at::Tensor wgrad4; + + // x used for dconv1 and dconv4 wgrad + at::Half* x = inputs[0].data_ptr(); + + at::Half* w = NULL; + + if (stride_1X1 != 1 || backward_state.filterdimA3[0] != backward_state.dimA[1]){ + w = inputs[14].data_ptr(); + at::Half* dy_conv4 = inputs[11].data_ptr(); + if (requires_grad) { + run_dconv(backward_state.dimA, + backward_state.padA, + backward_state.convstride1X1, + backward_state.dilationA, + backward_state.filterdimA4, + backward_state.outdimA3, + CUDNN_DATA_HALF, + dx_conv4, + w, + dy_conv4, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + // we don't print here since we can't hook out this grad in pytorch alone to compare, due to addition with dx + // DEBUG_MSG("[DEBUG] new dx_identity : " << grad_x_conv4.to(at::kFloat).sum().item()); + } + // wgrad + wgrad4 = outputs[4]; + at::Half* dw4 = wgrad4.data_ptr(); + run_dconv(backward_state.dimA, + backward_state.padA, + backward_state.convstride1X1, + backward_state.dilationA, + backward_state.filterdimA4, + backward_state.outdimA3, + CUDNN_DATA_HALF, + x, + dw4, + dy_conv4, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + } + else { + // if there is no downsample, dx_conv4 is fork of drelu3 + dx_conv4 = inputs[11].data_ptr(); + } + + // dgrad + w = inputs[1].data_ptr(); + auto grad_x = outputs[0]; + at::Half* dx = grad_x.data_ptr(); + + // backward strided conv cannot be fused + // if stride == 1 but channel changes, we can fuse here + if (requires_grad){ + if (stride_1X1 != 1){ + run_dconv(backward_state.dimA, + backward_state.padA, + backward_state.convstride1X1, + backward_state.dilationA, + backward_state.filterdimA1, + backward_state.outdimA1, + CUDNN_DATA_HALF, + dx, + w, + dy1, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + // add 2 together + grad_x.add_(grad_x_conv4); + } + else { + run_dconv_add(backward_state.dimA, + backward_state.padA, + backward_state.convstride1X1, + backward_state.dilationA, + backward_state.filterdimA1, + backward_state.outdimA1, + CUDNN_DATA_HALF, + dx, + w, + dy1, + dx_conv4); + } + } + + DEBUG_MSG("[DEBUG] new dx : " << grad_x.to(at::kFloat).sum().item()); + DEBUG_MSG("[DEBUG] new wgrad1 : " << wgrad1.to(at::kFloat).sum().item()); + + if (stride_1X1 != 1 || backward_state.filterdimA3[0] != backward_state.dimA[1]) { + DEBUG_MSG("[DEBUG] new wgrad4 : " << wgrad4.to(at::kFloat).sum().item()); + } +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &bottleneck_forward, "Bottleneck block forward"); + m.def("backward", &bottleneck_backward, "Bottleneck block backward"); + m.def("forward_init", &bottleneck_forward_init, "Bottleneck block init"); + m.def("forward_out1", &bottleneck_forward_out1, "Bottleneck block forward"); + m.def("forward_out2", &bottleneck_forward_out2, "Bottleneck block forward"); + m.def("forward_out2_mask", &bottleneck_forward_out2_mask, "Bottleneck block forward"); + m.def("forward_out2_halo", &bottleneck_forward_out2_halo, "Bottleneck block forward"); + m.def("forward_out2_halo_corr", &bottleneck_forward_out2_halo_corr, "Bottleneck block forward"); + m.def("forward_out2_pad", &bottleneck_forward_out2_pad, "Bottleneck block forward"); + m.def("forward_rest", &bottleneck_forward_rest, "Bottleneck block forward"); + m.def("backward_init", &bottleneck_backward_init, "Bottleneck block backward init"); + m.def("backward_grad_out2", &bottleneck_backward_grad_out2, "Bottleneck block backward"); + m.def("backward_grad_out1", &bottleneck_backward_grad_out1, "Bottleneck block backward"); + m.def("backward_grad_out1_mask", &bottleneck_backward_grad_out1_mask, "Bottleneck block backward"); + m.def("backward_grad_out1_halo", &bottleneck_backward_grad_out1_halo, "Bottleneck block backward"); + m.def("backward_grad_out1_halo_corr", &bottleneck_backward_grad_out1_halo_corr, "Bottleneck block backward"); + m.def("backward_wgrad2_pad", &bottleneck_backward_wgrad2_pad, "Bottleneck block backward"); + m.def("backward_wgrad2", &bottleneck_backward_wgrad2, "Bottleneck block backward"); + m.def("backward_wgrad2_halo", &bottleneck_backward_wgrad2_halo, "Bottleneck block backward"); + m.def("backward_wgrad3", &bottleneck_backward_wgrad3, "Bottleneck block backward"); + m.def("backward_wgrad1", &bottleneck_backward_wgrad1, "Bottleneck block backward"); + m.def("backward_rest", &bottleneck_backward_rest, "Bottleneck block backward"); +} diff --git a/apex/apex/contrib/csrc/conv_bias_relu/conv_bias_relu.cpp b/apex/apex/contrib/csrc/conv_bias_relu/conv_bias_relu.cpp new file mode 100644 index 00000000..1f947a34 --- /dev/null +++ b/apex/apex/contrib/csrc/conv_bias_relu/conv_bias_relu.cpp @@ -0,0 +1,2149 @@ +#include +#include // for getcudnnhandle +#include +#include +#include +#include + +#include + +#ifdef DEBUG +#define DEBUG_MSG(str) do { std::cout << str << std::endl; } while( false ) +#else +#define DEBUG_MSG(str) do { } while ( false ) +#endif + +#ifdef DEBUG_CUDNN +#define DEBUG_CUDNN_MSG(buf, str) do { buf << str << std::endl; } while( false ) +#else +#define DEBUG_CUDNN_MSG(buf, str) do { } while ( false ) +#endif + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(at::MemoryFormat::ChannelsLast), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +#define checkCudnnErr(...) \ + do { \ + int err = checkCudnnError(__VA_ARGS__, #__VA_ARGS__, __FILE__, __LINE__); \ + if (err) { \ + return; \ + } \ + } while (0) + + +int checkCudnnError(cudnnStatus_t code, const char* expr, const char* file, int line) { + if (code) { + printf("CUDNN error at %s:%d, code=%d (%s) in '%s'\n", file, line, (int)code, cudnnGetErrorString(code), expr); + return 1; + } + return 0; +} + +void checkError(cudaError_t code, char const * func, const char *file, const int line, bool abort = true); +#define checkCUDAError(val) { checkError((val), #val, __FILE__, __LINE__); } // in-line regular function + +void checkError(cudaError_t code, char const * func, const char *file, const int line, bool abort) { + if (code != cudaSuccess) + { + const char * errorMessage = cudaGetErrorString(code); + fprintf(stderr, "CUDA error returned from \"%s\" at %s:%d, Error code: %d (%s)\n", func, file, line, code, errorMessage); + if (abort){ + cudaDeviceReset(); + exit(code); + } + } +} + +void generateStrides(const int64_t* dimA, int64_t* strideA, int nbDims, cudnnTensorFormat_t filterFormat) { + // For INT8x4 and INT8x32 we still compute standard strides here to input + // into the cuDNN functions. We will manually scale by resizeFactor in the cpu ref. + if (filterFormat == CUDNN_TENSOR_NCHW) { + strideA[nbDims - 1] = 1; + for (int64_t d = nbDims - 2; d >= 0; d--) { + strideA[d] = strideA[d + 1] * dimA[d + 1]; + } + } else { + // Here we assume that the format is CUDNN_TENSOR_NHWC + strideA[1] = 1; + strideA[nbDims - 1] = strideA[1] * dimA[1]; + for (int64_t d = nbDims - 2; d >= 2; d--) { + strideA[d] = strideA[d + 1] * dimA[d + 1]; + } + strideA[0] = strideA[2] * dimA[2]; + } +} + + +int getFwdConvDilatedFilterDim(int filterDim, int dilation) { + return ((filterDim - 1) * dilation) + 1; +} + + +int getFwdConvPaddedImageDim(int tensorDim, int pad) { + return tensorDim + (2 * pad); +} + + +int getFwdConvOutputDim(int tensorDim, + int pad, + int filterDim, + int stride, + int dilation) { + int p = (getFwdConvPaddedImageDim(tensorDim, pad) - getFwdConvDilatedFilterDim(filterDim, dilation)) / stride + 1; + return (p); +} + + +// create a cache for plan +std::unordered_map plan_cache; + + +std::string getConvFusionString(int64_t* x_dim_padded, + int64_t* padA, + int64_t* convstrideA, + int64_t* dilationA, + int64_t* w_dim_padded, + cudnnDataType_t dataType, + std::string fusion_string) { + + for(int i=0;i<4;i++) { + fusion_string += 'X'; + fusion_string += std::to_string(x_dim_padded[i]); + } + for(int i=0;i<4;i++) { + fusion_string += 'W'; + fusion_string += std::to_string(w_dim_padded[i]); + } + for(int i=0;i<2;i++) { + fusion_string += 'P'; + fusion_string += std::to_string(padA[i]); + } + for(int i=0;i<2;i++) { + fusion_string += 'S'; + fusion_string += std::to_string(convstrideA[i]); + } + for(int i=0;i<2;i++) { + fusion_string += 'D'; + fusion_string += std::to_string(dilationA[i]); + } + fusion_string += 'T'; + fusion_string += std::to_string(dataType); + return fusion_string; +} + + +cudnn_frontend::ExecutionPlan& getOrCreatePlan(cudnnHandle_t handle_, + std::stringstream& log_buf, + cudnn_frontend::OperationGraph& opGraph, + std::string cache_string, + bool use_heuristic = true){ + auto it = plan_cache.find(cache_string); + if (it != plan_cache.end()) { + DEBUG_CUDNN_MSG(log_buf, "Found plan in cache"); + return it->second; + } else { + if (use_heuristic){ + // TODO: confirm which mode to use + auto heuristics = cudnn_frontend::EngineHeuristicsBuilder() + .setOperationGraph(opGraph) + .setHeurMode(CUDNN_HEUR_MODE_INSTANT) + .build(); + // try 3 times for now as WAR for no heuristic training + int max_tries = 3, count = 0; + auto& engine_configs = heuristics.getEngineConfig(max_tries); + while(true) { + try { + plan_cache.emplace(cache_string, std::move(cudnn_frontend::ExecutionPlanBuilder() + .setHandle(handle_) + .setEngineConfig(engine_configs[count], opGraph.getTag()) + .build())); + break; + } catch (cudnn_frontend::cudnnException e) { + if (++count == max_tries) throw e; + } + } + }else{ + DEBUG_CUDNN_MSG(log_buf, "No plan in cache"); + // How many engines support this operation graph ? + auto total_engines = opGraph.getEngineCount(); + DEBUG_CUDNN_MSG(log_buf, opGraph.describe() << " has " << total_engines << " engines."); + // We have to randomly pick one engine from [0, total_engines) + // Selecting "0" by default + auto engine = cudnn_frontend::EngineBuilder().setGlobalEngineIdx(0).setOperationGraph(opGraph).build(); + DEBUG_CUDNN_MSG(log_buf, engine.describe()); + auto& knobs = engine.getSupportedKnobs(); + for (auto it = std::begin(knobs); it != std::end(knobs); ++it) { + DEBUG_CUDNN_MSG(log_buf, it->describe()); + } + if (knobs.begin() != knobs.end()) { + DEBUG_CUDNN_MSG(log_buf, "Updated knob choice"); + knobs.begin()->setChoice(knobs.begin()->getMinValue() + 1); + DEBUG_CUDNN_MSG(log_buf, knobs.begin()->describe()); + } + + // Createmplacee the requisite engine config + auto engine_config = cudnn_frontend::EngineConfigBuilder().setEngine(engine).build(); + DEBUG_CUDNN_MSG(log_buf, engine_config.describe()); + plan_cache.emplace(cache_string, std::move(cudnn_frontend::ExecutionPlanBuilder().setHandle(handle_).setEngineConfig(engine_config).build())); + } + + return plan_cache.find(cache_string)->second; + } +} + + +void +run_conv_bias(int64_t* x_dim, + int64_t* w_dim, + int64_t* y_dim, + int64_t* conv_pad, + int64_t* convstride, + int64_t* dilation, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrB, + at::Half* devPtrY) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int convDim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + // Creates the necessary tensor descriptors + int64_t stride[4]; + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto xTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, xTensor.describe()); + + generateStrides(w_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto wTensor = cudnn_frontend::TensorBuilder() + .setDim(4, w_dim) + .setStrides(4, stride) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterConvTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('c') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterConvTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto bTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('b') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, bTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterBiasTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, afterBiasTensor.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, conv_pad) + .setPostPadding(convDim, conv_pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(xTensor) + .setwDesc(wTensor) + .setyDesc(afterConvTensor) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a Bias Node. + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(bTensor) + .setyDesc(afterBiasTensor) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Operation Graph. In this case it is convolution bias activation + std::array ops = {&conv_op, &bias_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(2, ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim, conv_pad, convstride, dilation, w_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrW, devPtrB, devPtrY}; + int64_t uids[] = {'x', 'w', 'b', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(4, data_ptrs) + .setUids(4, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_conv_bias_mask_relu(int64_t* x_dim, + int64_t* w_dim, + int64_t* y_dim, + int64_t* conv_pad, + int64_t* conv_stride, + int64_t* conv_dilation, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrB, + int8_t* devPtrM, + at::Half* devPtrY) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int conv_dim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + // Creates the necessary tensor descriptors + int64_t stride[4]; + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto xTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, xTensor.describe()); + + generateStrides(w_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto wTensor = cudnn_frontend::TensorBuilder() + .setDim(4, w_dim) + .setStrides(4, stride) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto mTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('m') + .setAlignment(16) + .setDataType(CUDNN_DATA_INT8) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterConvTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('c') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterConvTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto bTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('b') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, bTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterBiasTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('B') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterBiasTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterMaskTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('M') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterBiasTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterReLUTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, afterReLUTensor.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(conv_dim) + .setStrides(conv_dim, conv_stride) + .setPrePadding(conv_dim, conv_pad) + .setPostPadding(conv_dim, conv_pad) + .setDilation(conv_dim, conv_dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Define the mask operation + auto maskDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + + // Define the activation operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_FWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(xTensor) + .setwDesc(wTensor) + .setyDesc(afterConvTensor) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a Bias Node + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(bTensor) + .setyDesc(afterBiasTensor) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // create a Mask Node + auto mask_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(bias_op.getOutputTensor()) + .setbDesc(mTensor) + .setyDesc(afterMaskTensor) + .setpwDesc(maskDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, mask_op.describe()); + + // Create an Activation Node + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(mask_op.getOutputTensor()) + .setyDesc(afterReLUTensor) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create an Operation Graph. In this case it is convolution bias activation + std::array ops = {&conv_op, &bias_op, &mask_op, &act_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(4, ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim, conv_pad, conv_stride, conv_dilation, w_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrW, devPtrB, devPtrM, devPtrY}; + int64_t uids[] = {'x', 'w', 'b', 'm', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(5, data_ptrs) + .setUids(5, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_conv_cscale_cbias_relu(int64_t* x_dim, + int64_t* w_dim, + int64_t* y_dim, + int64_t* conv_pad, + int64_t* conv_stride, + int64_t* conv_dilation, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrS, + at::Half* devPtrB, + at::Half* devPtrY) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int conv_dim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t s_dim[] = {1, y_dim[1], 1, 1}; + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + // Creates the necessary tensor descriptors + int64_t stride[4]; + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto xTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, xTensor.describe()); + + generateStrides(w_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto wTensor = cudnn_frontend::TensorBuilder() + .setDim(4, w_dim) + .setStrides(4, stride) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterConvTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('c') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterConvTensor.describe()); + + generateStrides(s_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto sTensor = cudnn_frontend::TensorBuilder() + .setDim(4, s_dim) + .setStrides(4, stride) + .setId('s') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, sTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterScaleTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('S') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterScaleTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto bTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('b') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, bTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterBiasTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('B') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterBiasTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterReLUTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, afterReLUTensor.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(conv_dim) + .setStrides(conv_dim, conv_stride) + .setPrePadding(conv_dim, conv_pad) + .setPostPadding(conv_dim, conv_pad) + .setDilation(conv_dim, conv_dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the scale operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Define the activation operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_FWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(xTensor) + .setwDesc(wTensor) + .setyDesc(afterConvTensor) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a scale Node. + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(sTensor) + .setyDesc(afterScaleTensor) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create a Bias Node. + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(scale_op.getOutputTensor()) + .setbDesc(bTensor) + .setyDesc(afterBiasTensor) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Activation Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(bias_op.getOutputTensor()) + .setyDesc(afterReLUTensor) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create an Operation Graph. In this case it is convolution bias activation + std::array ops = {&conv_op, &scale_op, &bias_op, &act_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim, conv_pad, conv_stride, conv_dilation, w_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrW, devPtrS, devPtrB, devPtrY}; + int64_t uids[] = {'x', 'w', 's', 'b', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(5, data_ptrs) + .setUids(5, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_conv_bias_relu(int64_t* x_dim, + int64_t* w_dim, + int64_t* y_dim, + int64_t* conv_pad, + int64_t* conv_stride, + int64_t* conv_dilation, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrB, + at::Half* devPtrY) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int conv_dim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + // Creates the necessary tensor descriptors + int64_t stride[4]; + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto xTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, xTensor.describe()); + + generateStrides(w_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto wTensor = cudnn_frontend::TensorBuilder() + .setDim(4, w_dim) + .setStrides(4, stride) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterConvTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('c') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterConvTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto bTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('b') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, bTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterBiasTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('B') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, afterBiasTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto afterReLUTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, afterReLUTensor.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(conv_dim) + .setStrides(conv_dim, conv_stride) + .setPrePadding(conv_dim, conv_pad) + .setPostPadding(conv_dim, conv_pad) + .setDilation(conv_dim, conv_dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the bias operation + auto biasDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_ADD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Define the activation operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_FWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_FORWARD_DESCRIPTOR) + .setxDesc(xTensor) + .setwDesc(wTensor) + .setyDesc(afterConvTensor) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create a Bias Node. + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(conv_op.getOutputTensor()) + .setbDesc(bTensor) + .setyDesc(afterBiasTensor) + .setpwDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Activation Node. + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(bias_op.getOutputTensor()) + .setyDesc(afterReLUTensor) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create an Operation Graph. In this case it is convolution bias activation + std::array ops = {&conv_op, &bias_op, &act_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(3, ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim, conv_pad, conv_stride, conv_dilation, w_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrW, devPtrB, devPtrY}; + int64_t uids[] = {'x', 'w', 'b', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(4, data_ptrs) + .setUids(4, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_drelu_dscale(int64_t* dy_dim, + cudnnDataType_t dataType, + at::Half* devPtrDY, + at::Half* devPtrR, + at::Half* devPtrS, + at::Half* devPtrDX) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int convDim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t s_dim[] = {1, dy_dim[1], 1, 1}; + + // Creates the necessary tensor descriptors + int64_t stride[4]; + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto dyTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, dyTensor.describe()); + + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto rTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('r') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, rTensor.describe()); + + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto inActGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('R') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, inActGradTensor.describe()); + + generateStrides(s_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto scaleTensor = cudnn_frontend::TensorBuilder() + .setDim(4, s_dim) + .setStrides(4, stride) + .setId('s') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleTensor.describe()); + + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto dxTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, dxTensor.describe()); + + // Define the activation backward operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_BWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the bias backward operation + auto scaleDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_MUL) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, scaleDesc.describe()); + + // Create an relu backward Node + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setdyDesc(dyTensor) + .setxDesc(rTensor) + .setdxDesc(inActGradTensor) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create bias node + auto scale_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setxDesc(inActGradTensor) + .setbDesc(scaleTensor) + .setyDesc(dxTensor) + .setpwDesc(scaleDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, scale_op.describe()); + + // Create an Operation Graph. In this case it is bias only + std::array ops = {&act_op, &scale_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + // creating unique dummy values + int64_t pad_dummy[] = {40, 40}; + int64_t stride_dummy[] = {40, 40}; + int64_t dilation_dummy[] = {40, 40}; + auto cache_string = getConvFusionString(dy_dim, pad_dummy, stride_dummy, dilation_dummy, s_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrDY, devPtrR, devPtrS, devPtrDX}; + int64_t uids[] = {'y', 'r', 's', 'x'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(4, data_ptrs) + .setUids(4, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_drelu_dbias(int64_t* dy_dim, + cudnnDataType_t dataType, + at::Half* devPtrDY, + at::Half* devPtrR, + at::Half* devPtrDR, + float* devPtrDB) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int convDim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t b_dim[] = {1, dy_dim[1], 1, 1}; + + // Creates the necessary tensor descriptors + int64_t stride[4]; + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto dyTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, dyTensor.describe()); + + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto rTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('r') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, rTensor.describe()); + + generateStrides(dy_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto inActGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, dy_dim) + .setStrides(4, stride) + .setId('R') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, inActGradTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto biasGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasGradTensor.describe()); + + // Define the activation backward operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_BWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the bias backward operation + auto biasDesc = cudnn_frontend::ReductionDescBuilder() + .setMathPrecision(CUDNN_DATA_FLOAT) + .setReductionOp(CUDNN_REDUCE_TENSOR_ADD) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Create an relu backward Node + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setdyDesc(dyTensor) + .setxDesc(rTensor) + .setdxDesc(inActGradTensor) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create bias node + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR) + .setxDesc(inActGradTensor) + .setyDesc(biasGradTensor) + .setreductionDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Operation Graph. In this case it is bias only + std::array ops = {&act_op, &bias_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + // creating unique dummy values + int64_t pad_dummy[] = {20, 20}; + int64_t stride_dummy[] = {20, 20}; + int64_t dilation_dummy[] = {20, 20}; + auto cache_string = getConvFusionString(dy_dim, pad_dummy, stride_dummy, dilation_dummy, b_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrDY, devPtrR, devPtrDR, devPtrDB}; + int64_t uids[] = {'x', 'r', 'R', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(4, data_ptrs) + .setUids(4, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_dconv_drelu_dbias(int64_t* x_dim, + int64_t* w_dim, + int64_t* y_dim, + int64_t* pad, + int64_t* convstride, + int64_t* dilation, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrR, + at::Half* devPtrRg, + float* devPtrY) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + float alpha = 1.0f; + float beta = 0.0f; + int64_t b_dim[] = {1, x_dim[1], 1, 1}; + + int64_t stride[4]; + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto outConvGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, outConvGradTensor.describe()); + + generateStrides(w_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto wTensor = cudnn_frontend::TensorBuilder() + .setDim(4, w_dim) + .setStrides(4, stride) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto inConvGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('A') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .setVirtual() + .build(); + DEBUG_CUDNN_MSG(log_buf, inConvGradTensor.describe()); + + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto rTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('r') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, rTensor.describe()); + + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto inReLUGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('R') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, inReLUGradTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto inBiasGradTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, inBiasGradTensor.describe()); + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(convDim) + .setStrides(convDim, convstride) + .setPrePadding(convDim, pad) + .setPostPadding(convDim, pad) + .setDilation(convDim, dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Define the activation backward operation + auto actDesc = cudnn_frontend::PointWiseDescBuilder() + .setMode(CUDNN_POINTWISE_RELU_BWD) + .setMathPrecision(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, actDesc.describe()); + + // Define the bias backward operation + auto biasDesc = cudnn_frontend::ReductionDescBuilder() + .setMathPrecision(CUDNN_DATA_FLOAT) + .setReductionOp(CUDNN_REDUCE_TENSOR_ADD) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Create a convolution Node + auto conv_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) + .setdyDesc(outConvGradTensor) + .setwDesc(wTensor) + .setdxDesc(inConvGradTensor) + .setcDesc(convDesc) + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create an relu backward Node + auto act_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_POINTWISE_DESCRIPTOR) + .setdyDesc(inConvGradTensor) + .setxDesc(rTensor) + .setdxDesc(inReLUGradTensor) + .setpwDesc(actDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, act_op.describe()); + + // Create bias node + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR) + .setxDesc(inReLUGradTensor) + .setyDesc(inBiasGradTensor) + .setreductionDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Operation Graph. In this case it is bias only + std::array ops = {&conv_op, &act_op, &bias_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim, pad, convstride, dilation, w_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrW, devPtrR, devPtrRg, devPtrY}; + int64_t uids[] = {'x', 'w', 'r', 'R', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(5, data_ptrs) + .setUids(5, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } + +} + + +void +run_dconv(int64_t* x_dim, + int64_t* w_dim, + int64_t* y_dim, + int64_t* conv_pad, + int64_t* conv_stride, + int64_t* conv_dilation, + cudnnDataType_t dataType, + at::Half* devPtrX, + at::Half* devPtrW, + at::Half* devPtrY, + cudnnBackendDescriptorType_t mode) { + + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + + try { + int conv_dim = 2; + float alpha = 1.0f; + float beta = 0.0f; + + // Define the convolution problem + int64_t stride[4]; + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto xTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, xTensor.describe()); + + generateStrides(w_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto wTensor = cudnn_frontend::TensorBuilder() + .setDim(4, w_dim) + .setStrides(4, stride) + .setId('w') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, wTensor.describe()); + + generateStrides(y_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto yTensor = cudnn_frontend::TensorBuilder() + .setDim(4, y_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, yTensor.describe()); + + + // Define the convolution problem + auto convDesc = cudnn_frontend::ConvDescBuilder() + .setDataType(CUDNN_DATA_FLOAT) + .setMathMode(CUDNN_CROSS_CORRELATION) + .setNDims(conv_dim) + .setStrides(conv_dim, conv_stride) + .setPrePadding(conv_dim, conv_pad) + .setPostPadding(conv_dim, conv_pad) + .setDilation(conv_dim, conv_dilation) + .build(); + DEBUG_CUDNN_MSG(log_buf, convDesc.describe()); + + // Create a convolution node + // mode should be one of following + // CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR + // CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR + auto conv_op_builder = cudnn_frontend::OperationBuilder(mode); + if (mode == CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR) { + conv_op_builder.setdxDesc(xTensor) + .setwDesc(wTensor) + .setdyDesc(yTensor) + .setcDesc(convDesc); + } + else { + conv_op_builder.setxDesc(xTensor) + .setdwDesc(wTensor) + .setdyDesc(yTensor) + .setcDesc(convDesc); + } + auto conv_op = conv_op_builder + .setAlpha(alpha) + .setBeta(beta) + .build(); + DEBUG_CUDNN_MSG(log_buf, conv_op.describe()); + + // Create an Operation Graph. In this case it is convolution add bias activation + std::array ops = {&conv_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + auto cache_string = getConvFusionString(x_dim, conv_pad, conv_stride, conv_dilation, w_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrW, devPtrY}; + int64_t uids[] = {'x', 'w', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(3, data_ptrs) + .setUids(3, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } +} + + +void +run_dbias(int64_t* x_dim, + cudnnDataType_t dataType, + at::Half* devPtrX, + float* devPtrY) { + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + std::stringstream log_buf; + try { + int convDim = 2; + int64_t b_dim[] = {1, x_dim[1], 1, 1}; + + int64_t stride[4]; + generateStrides(x_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto xTensor = cudnn_frontend::TensorBuilder() + .setDim(4, x_dim) + .setStrides(4, stride) + .setId('x') + .setAlignment(16) + .setDataType(dataType) + .build(); + DEBUG_CUDNN_MSG(log_buf, xTensor.describe()); + + generateStrides(b_dim, stride, 4, CUDNN_TENSOR_NHWC); + auto yTensor = cudnn_frontend::TensorBuilder() + .setDim(4, b_dim) + .setStrides(4, stride) + .setId('y') + .setAlignment(16) + .setDataType(CUDNN_DATA_FLOAT) + .build(); + DEBUG_CUDNN_MSG(log_buf, yTensor.describe()); + + // Define the bias backward operation + auto biasDesc = cudnn_frontend::ReductionDescBuilder() + .setMathPrecision(CUDNN_DATA_FLOAT) + .setReductionOp(CUDNN_REDUCE_TENSOR_ADD) + .build(); + DEBUG_CUDNN_MSG(log_buf, biasDesc.describe()); + + // Create bias node + auto bias_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_REDUCTION_DESCRIPTOR) + .setxDesc(xTensor) + .setyDesc(yTensor) + .setreductionDesc(biasDesc) + .build(); + DEBUG_CUDNN_MSG(log_buf, bias_op.describe()); + + // Create an Operation Graph. In this case it is bias only + std::array ops = {&bias_op}; + + auto opGraph = cudnn_frontend::OperationGraphBuilder() + .setHandle(handle_) + .setOperationGraph(ops.size(), ops.data()) + .build(); + + // Create string encoding for plan caching + int64_t pad_dummy[] = {10, 10}; + int64_t stride_dummy[] = {10, 10}; + int64_t dilation_dummy[] = {10, 10}; + auto cache_string = getConvFusionString(x_dim, pad_dummy, stride_dummy, dilation_dummy, b_dim, dataType, opGraph.getTag()); + DEBUG_CUDNN_MSG(log_buf, "[convstring] " << cache_string); + + auto& plan = getOrCreatePlan(handle_, log_buf, opGraph, cache_string); + DEBUG_CUDNN_MSG(log_buf, "Plan tag: " << plan.getTag()); + + auto workspace_size = plan.getWorkspaceSize(); + DEBUG_CUDNN_MSG(log_buf, plan.describe() << " requires workspace " << workspace_size); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + void* data_ptrs[] = {devPtrX, devPtrY}; + int64_t uids[] = {'x', 'y'}; + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(2, data_ptrs) + .setUids(2, uids) + .build(); + DEBUG_CUDNN_MSG(log_buf, "variantPack " << variantPack.describe()); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + checkCudnnErr(status); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); + } catch (cudnn_frontend::cudnnException e) { + std::cout << log_buf.str() << "[ERROR] Exception " << e.what() << std::endl; + } + +} + + +std::vector conv_bias_mask_relu_forward(std::vector inputs, int64_t padding, int64_t stride) { + std::cout << std::fixed; + + // create output vector + std::vector outputs; + auto output_format = at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + } + + // output dim in n,c,h,w used by backend + int64_t y_dim[] = {0, 0, 0, 0}; + + // use these fixed values + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // compute output from pad/stride/dilation + y_dim[0] = x_dim[0]; + y_dim[1] = w_dim[0]; + for (int dim = 0; dim < 2; dim++) { + y_dim[dim + 2] = getFwdConvOutputDim(x_dim[dim + 2], conv_pad[dim], w_dim[dim + 2], conv_stride[dim], conv_dilation[dim]); + } + + // run + at::Half* x = inputs[0].data_ptr(); + at::Half* w = inputs[1].data_ptr(); + at::Half* b = inputs[2].data_ptr(); + int8_t* m = inputs[3].data_ptr(); + auto out = at::empty(y_dim, inputs[0].type(), output_format); + at::Half* y = out.data_ptr(); + + run_conv_bias_mask_relu(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + w, + b, + m, + y); + + DEBUG_MSG("[DEBUG] conv-bias-mask-relu : " << y.to(at::kFloat).sum().item()); + + outputs.push_back(out); + + return outputs; +} + + +at::Tensor conv_cscale_cbias_relu_forward(std::vector inputs, int64_t padding, int64_t stride) { + std::cout << std::fixed; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + } + + // output dim in n,c,h,w used by backend + int64_t y_dim[] = {0, 0, 0, 0}; + + // use these fixed values + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // compute output from pad/stride/dilation + y_dim[0] = x_dim[0]; + y_dim[1] = w_dim[0]; + for (int dim = 0; dim < 2; dim++) { + y_dim[dim + 2] = getFwdConvOutputDim(x_dim[dim + 2], conv_pad[dim], w_dim[dim + 2], conv_stride[dim], conv_dilation[dim]); + } + + // run + at::Half* x = inputs[0].data_ptr(); + at::Half* w = inputs[1].data_ptr(); + at::Half* s = inputs[2].data_ptr(); + at::Half* b = inputs[3].data_ptr(); + auto out = at::empty(y_dim, inputs[0].type(), at::MemoryFormat::ChannelsLast); + at::Half* y = out.data_ptr(); + + run_conv_cscale_cbias_relu(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + w, + s, + b, + y); + + DEBUG_MSG("[DEBUG] conv-cscale-cbias-relu : " << y.to(at::kFloat).sum().item()); + + return out; +} + + +std::vector conv_cscale_cbias_relu_backward(std::vector inputs, int64_t padding, int64_t stride) { + bool requires_grad = inputs[0].requires_grad(); + + for (int i = 0; i <= 4; i++) { + CHECK_INPUT(inputs[i]); + } + + std::cout << std::fixed; + + // create output vector + std::vector outputs; + auto output_format = at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + int64_t y_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + y_dim[dim] = inputs[3].size(axis[dim]); + } + + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // run + // drelu-dbias + at::Half* dy = inputs[4].data_ptr(); + at::Half* r = inputs[3].data_ptr(); + auto s = inputs[2].data_ptr(); + auto dscale = at::empty_like(inputs[4]); + at::Half* ds = dscale.data_ptr(); + + auto options = at::TensorOptions().dtype(at::kFloat).layout(inputs[0].layout()).device(inputs[0].device()).requires_grad(false); + run_drelu_dscale(y_dim, + CUDNN_DATA_HALF, + dy, + r, + s, + ds); + + // conv wgrad + at::Half* x = inputs[0].data_ptr(); + auto wgrad = at::empty_like(inputs[1]); + at::Half* dw = wgrad.data_ptr(); + run_dconv(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + dw, + ds, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + // conv dgrad + at::Half* w = inputs[1].data_ptr(); + auto dgrad = at::empty_like(inputs[0]); + at::Half* dx = dgrad.data_ptr(); + run_dconv(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + dx, + w, + ds, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + + outputs.push_back(dgrad); + outputs.push_back(wgrad); + + return outputs; +} + + +std::vector conv_bias_relu_forward(std::vector inputs, int64_t padding, int64_t stride) { + std::cout << std::fixed; + + // create output vector + std::vector outputs; + auto output_format = at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + } + + // output dim in n,c,h,w used by backend + int64_t y_dim[] = {0, 0, 0, 0}; + + // use these fixed values + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // compute output from pad/stride/dilation + y_dim[0] = x_dim[0]; + y_dim[1] = w_dim[0]; + for (int dim = 0; dim < 2; dim++) { + y_dim[dim + 2] = getFwdConvOutputDim(x_dim[dim + 2], conv_pad[dim], w_dim[dim + 2], conv_stride[dim], conv_dilation[dim]); + } + + // run + at::Half* x = inputs[0].data_ptr(); + at::Half* w = inputs[1].data_ptr(); + at::Half* b = inputs[2].data_ptr(); + auto out = at::empty(y_dim, inputs[0].type(), output_format); + at::Half* y = out.data_ptr(); + + run_conv_bias_relu(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + w, + b, + y); + + DEBUG_MSG("[DEBUG] conv-bias-relu : " << y.to(at::kFloat).sum().item()); + + outputs.push_back(out); + + return outputs; +} + + +std::vector conv_bias_relu_backward(std::vector inputs, int64_t padding, int64_t stride) { + bool requires_grad = inputs[0].requires_grad(); + + for (int i = 0; i <= 3; i++) { + CHECK_INPUT(inputs[i]); + } + + std::cout << std::fixed; + + // create output vector + std::vector outputs; + auto output_format = at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + int64_t y_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + y_dim[dim] = inputs[3].size(axis[dim]); + } + + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // run + // drelu-dbias + at::Half* dy = inputs[3].data_ptr(); + at::Half* r = inputs[2].data_ptr(); + auto drelu = at::empty_like(inputs[2]); + at::Half* dr = drelu.data_ptr(); + auto options = at::TensorOptions().dtype(at::kFloat).layout(inputs[0].layout()).device(inputs[0].device()).requires_grad(false); + auto bgrad = at::empty(b_dim, options, output_format); + float* db = bgrad.data_ptr(); + run_drelu_dbias(y_dim, + CUDNN_DATA_HALF, + dy, + r, + dr, + db); + + // conv wgrad + at::Half* x = inputs[0].data_ptr(); + auto wgrad = at::empty_like(inputs[1]); + at::Half* dw = wgrad.data_ptr(); + run_dconv(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + dw, + dr, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + // conv dgrad + at::Half* w = inputs[1].data_ptr(); + auto dgrad = at::empty_like(inputs[0]); + at::Half* dx = dgrad.data_ptr(); + run_dconv(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + dx, + w, + dr, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + + outputs.push_back(dgrad); + outputs.push_back(wgrad); + outputs.push_back(bgrad); + + return outputs; + +} + +std::vector conv_bias_forward(std::vector inputs, int64_t padding, int64_t stride) { + std::cout << std::fixed; + + // create output vector + std::vector outputs; + auto output_format = at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + } + + // output dim in n,c,h,w used by backend + int64_t y_dim[] = {0, 0, 0, 0}; + + // use these fixed values + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // compute output from pad/stride/dilation + y_dim[0] = x_dim[0]; + y_dim[1] = w_dim[0]; + for (int dim = 0; dim < 2; dim++) { + y_dim[dim + 2] = getFwdConvOutputDim(x_dim[dim + 2], conv_pad[dim], w_dim[dim + 2], conv_stride[dim], conv_dilation[dim]); + } + + // run + at::Half* x = inputs[0].data_ptr(); + at::Half* w = inputs[1].data_ptr(); + at::Half* b = inputs[2].data_ptr(); + auto out = at::empty(y_dim, inputs[0].type(), output_format); + at::Half* y = out.data_ptr(); + + run_conv_bias(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + w, + b, + y); + + DEBUG_MSG("[DEBUG] conv-bias : " << y.to(at::kFloat).sum().item()); + + outputs.push_back(out); + + return outputs; +} + + +std::vector conv_bias_backward(std::vector inputs, int64_t padding, int64_t stride) { + bool requires_grad = inputs[0].requires_grad(); + + for (int i = 0; i <= 2; i++) { + CHECK_INPUT(inputs[i]); + } + + std::cout << std::fixed; + + // create output vector + std::vector outputs; + auto output_format = at::MemoryFormat::ChannelsLast; + + // setup dimensions + int64_t x_dim[] = {0, 0, 0, 0}; + int64_t w_dim[] = {0, 0, 0, 0}; + int64_t y_dim[] = {0, 0, 0, 0}; + + // All dim calculation after this order of n,c,h,w + int axis[] = {0, 1, 2, 3}; + for (int dim = 0; dim < 4; dim++) { + x_dim[dim] = inputs[0].size(axis[dim]); + w_dim[dim] = inputs[1].size(axis[dim]); + y_dim[dim] = inputs[2].size(axis[dim]); + } + + int64_t b_dim[] = {1, y_dim[1], 1, 1}; + + int64_t conv_pad[] = {padding, padding}; + int64_t conv_stride[] = {stride, stride}; + int64_t conv_dilation[] = {1, 1}; + + // run + // dbias + at::Half* dy = inputs[2].data_ptr(); + auto options = at::TensorOptions().dtype(at::kFloat).layout(inputs[0].layout()).device(inputs[0].device()).requires_grad(false); + auto bgrad = at::empty(b_dim, options, output_format); + float* db = bgrad.data_ptr(); + run_dbias(y_dim, + CUDNN_DATA_HALF, + dy, + db); + + // conv wgrad + at::Half* x = inputs[0].data_ptr(); + auto wgrad = at::empty_like(inputs[1]); + at::Half* dw = wgrad.data_ptr(); + run_dconv(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + x, + dw, + dy, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_FILTER_DESCRIPTOR); + + // conv dgrad + at::Half* w = inputs[1].data_ptr(); + auto dgrad = at::empty_like(inputs[0]); + at::Half* dx = dgrad.data_ptr(); + run_dconv(x_dim, + w_dim, + y_dim, + conv_pad, + conv_stride, + conv_dilation, + CUDNN_DATA_HALF, + dx, + w, + dy, + CUDNN_BACKEND_OPERATION_CONVOLUTION_BACKWARD_DATA_DESCRIPTOR); + + outputs.push_back(dgrad); + outputs.push_back(wgrad); + outputs.push_back(bgrad); + + return outputs; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &conv_bias_relu_forward, "Fused Conv-Bias-ReLU forward"); + m.def("backward", &conv_bias_relu_backward, "Fused Conv-Bias-ReLU backward"); + m.def("forward_no_relu", &conv_bias_forward, "Fused Conv-Bias forward"); + m.def("backward_no_relu", &conv_bias_backward, "Fused Conv-Bias backward"); + m.def("forward_mask", &conv_bias_mask_relu_forward, "Fused Conv-Bias-Mask-ReLU forward"); + m.def("forward_cscale_cbias_relu", &conv_cscale_cbias_relu_forward, "Fused Conv-(const)Scale-(const)Bias-ReLU"); + m.def("backward_cscale_cbias_relu", &conv_cscale_cbias_relu_backward, "Fused Conv-(const)Scale-(const)Bias-ReLU backward"); +} + diff --git a/apex/apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp b/apex/apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp new file mode 100644 index 00000000..ded444a6 --- /dev/null +++ b/apex/apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp @@ -0,0 +1,131 @@ +#include +#include +#include +#include + +#include + +#include "norm_sample.h" + +at::Tensor gbn_forward(const at::Tensor& x, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const float momentum, + const float epsilon, + const int64_t bn_group, + const int rank_id, + const std::vector &peer_buffers) { + + int64_t N = x.size(0); + int64_t C = x.size(1); + int64_t H = x.size(2); + int64_t W = x.size(3); + + int64_t tensorDims[] = {N, C, H, W}; + int64_t peerDims[] = {bn_group, 4*C, 1, 1}; + int64_t perChannelDims[] = {1, C, 1, 1}; + int64_t epsilonDims[] = {1, 1, 1, 1}; + + // Allocate output tensor + at::Tensor y = at::empty_like(x); + + std::vector void_peer_buffers; + for (int64_t addr : peer_buffers) { + void_peer_buffers.push_back((void*)addr); + } + + assert(bn_group == void_peer_buffers.size()); + run_batch_norm_forward( + perChannelDims, + epsilonDims, + tensorDims, + peerDims, + x.data_ptr(), + y.data_ptr(), + scale.data_ptr(), + bias.data_ptr(), + running_mean.data_ptr(), + running_var.data_ptr(), + running_mean.data_ptr(), + running_var.data_ptr(), + minibatch_mean.data_ptr(), + minibatch_inv_var.data_ptr(), + void_peer_buffers, + epsilon, + momentum, + rank_id + ); + + return y; +} + +std::vector gbn_backward( + const at::Tensor& x, + const at::Tensor& dy, + const at::Tensor& scale, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const float epsilon, + const int64_t bn_group, + const int rank_id, + const std::vector &peer_buffers) { + + int64_t N = x.size(0); + int64_t C = x.size(1); + int64_t H = x.size(2); + int64_t W = x.size(3); + + int64_t tensorDims[] = {N, C, H, W}; + int64_t peerDims[] = {bn_group, 4*C, 1, 1}; + int64_t perChannelDims[] = {1, C, 1, 1}; + int64_t epsilonDims[] = {1, 1, 1, 1}; + + // Allocate output tensor + // outputs + at::Tensor x_grad, scale_grad, bias_grad; + + // Allocate outputs + x_grad = at::empty_like(x); + scale_grad = at::empty_like(scale); + bias_grad = at::empty_like(scale); + + std::vector void_peer_buffers; + for (int64_t addr : peer_buffers) { + void_peer_buffers.push_back((void*)addr); + } + + assert(bn_group == void_peer_buffers.size()); + + run_batch_norm_backward( + perChannelDims, + epsilonDims, + tensorDims, + peerDims, + x.data_ptr(), + dy.data_ptr(), + scale.data_ptr(), + minibatch_mean.data_ptr(), + minibatch_inv_var.data_ptr(), + x_grad.data_ptr(), + scale_grad.data_ptr(), + bias_grad.data_ptr(), + void_peer_buffers, + epsilon, + rank_id); + + + + return std::vector{x_grad, scale_grad, bias_grad}; +} + + + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &gbn_forward, "Group batch norm forward"); + m.def("backward", &gbn_backward, "Group batch backward"); +} diff --git a/apex/apex/contrib/csrc/cudnn_gbn/norm_sample.cpp b/apex/apex/contrib/csrc/cudnn_gbn/norm_sample.cpp new file mode 100644 index 00000000..d4835190 --- /dev/null +++ b/apex/apex/contrib/csrc/cudnn_gbn/norm_sample.cpp @@ -0,0 +1,474 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "norm_sample.h" +#include +#include "cudnn_backend.h" +#include // for getcudnnhandle +#include +#include + +#define FatalError(s) { \ + std::stringstream _where, _message; \ + _where << __FILE__ << ':' << __LINE__; \ + _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__;\ + std::cerr << _message.str() << "\nAborting...\n"; \ + exit(EXIT_FAILURE); \ +} + +#define checkCUDNN(status) { \ + std::stringstream _error; \ + if (status != CUDNN_STATUS_SUCCESS) { \ + _error << "CUDNN failure\nError: " << cudnnGetErrorString(status); \ + FatalError(_error.str()); \ + } \ +} + +#define checkCudaErrors(status) { \ + std::stringstream _error; \ + if (status != 0) { \ + _error << "Cuda failure\nError: " << cudaGetErrorString(status); \ + FatalError(_error.str()); \ + } \ +} + +int checkCudnnError(cudnnStatus_t code, const char* expr, const char* file, int line) { + if (code) { + printf("CUDNN error at %s:%d, code=%d (%s) in '%s'\n", file, line, (int)code, cudnnGetErrorString(code), expr); + return 1; + } + return 0; +} + +#define checkCudnnErr(...) \ + do { \ + int err = checkCudnnError(__VA_ARGS__, #__VA_ARGS__, __FILE__, __LINE__); \ + if (err) { \ + return; \ + } \ + } while (0) + +bool +AllowAll(cudnnBackendDescriptor_t engine_config) { + (void)engine_config; + return false; +} + +void generateStrides(const int64_t* dimA, int64_t* strideA, int nbDims, cudnnTensorFormat_t filterFormat) { + // For INT8x4 and INT8x32 we still compute standard strides here to input + // into the cuDNN functions. We will manually scale by resizeFactor in the cpu ref. + if (filterFormat == CUDNN_TENSOR_NCHW) { + strideA[nbDims - 1] = 1; + for (int64_t d = nbDims - 2; d >= 0; d--) { + strideA[d] = strideA[d + 1] * dimA[d + 1]; + } + } else { + // Here we assume that the format is CUDNN_TENSOR_NHWC + strideA[1] = 1; + strideA[nbDims - 1] = strideA[1] * dimA[1]; + for (int64_t d = nbDims - 2; d >= 2; d--) { + strideA[d] = strideA[d + 1] * dimA[d + 1]; + } + strideA[0] = strideA[2] * dimA[2]; + } +} + +void +run_batch_norm_forward( + int64_t *perChannelSum, + int64_t *epsilon, + int64_t *tensorDims, + int64_t *peerDims, + + void *xDevPtr, + void *yDevPtr, + void *scaledevPtr, + void *biasdevPtr, + void *in_meandevPtr, + void *in_vardevPtr, + void *out_meandevPtr, + void *out_vardevPtr, + void *saved_meandevPtr, + void *saved_inv_vardevPtr, + const std::vector &peer_devPtrs, + double epsilon_val, + double exponential_decay_factor, + int rank_id) +{ + cudaStream_t stream; + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + cudnnGetStream(handle_, &stream); + + try { + + // Creates the necessary tensor descriptors + int64_t tensor_stride[4]; + int64_t stride[4]; + int64_t peer_stride[4]; + + generateStrides(tensorDims, tensor_stride, 4, CUDNN_TENSOR_NHWC); + generateStrides(peerDims, peer_stride, 4, CUDNN_TENSOR_NHWC); + + auto tensor_create = [&tensor_stride, &tensorDims](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, tensorDims) + .setStrides(4, tensor_stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .build(); + }; + + auto peer_tensor_create = [&peer_stride, &tensorDims](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, tensorDims) + .setStrides(4, peer_stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .build(); + }; + + + generateStrides(perChannelSum, stride, 4, CUDNN_TENSOR_NHWC); + + auto per_channel_tensor_create = [&stride, &perChannelSum](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, perChannelSum) + .setStrides(4, stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .build(); + }; + + + auto xTensor = tensor_create(CUDNN_DATA_HALF, 100); + auto yTensor = tensor_create(CUDNN_DATA_HALF, 101); + auto scaleTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 102); + auto biasTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 103); + auto inMeanTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 104); + auto inVarTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 105); + auto outMeanTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 106); + auto outVarTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 107); + auto savedMeanTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 108); + auto savedInvVarTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 109); + + + int64_t epsilon_stride[4]; + generateStrides(epsilon, epsilon_stride, 4, CUDNN_TENSOR_NHWC); + auto scalar_tensor_create = [&epsilon_stride, &epsilon](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, epsilon) + .setStrides(4, epsilon_stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .setByValue(true) + .build(); + }; + + auto epsilonTensor = scalar_tensor_create(CUDNN_DATA_DOUBLE, 110); + auto expDecayTensor = scalar_tensor_create(CUDNN_DATA_DOUBLE, 111); + + std::vector peerStatTensors; + for (size_t i = 112; i < 112 + peer_devPtrs.size(); ++i) { + peerStatTensors.push_back(peer_tensor_create(CUDNN_DATA_FLOAT, i)); + } + +#if (CUDNN_VERSION >= 8500) + // Batch normalization + cudnnBackendNormMode_t normalizationMode = CUDNN_BATCH_NORM; + + // Forward training + cudnnBackendNormFwdPhase_t phase = CUDNN_NORM_FWD_TRAINING; + + //Create a Finalize node + auto batch_norm_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_NORM_FORWARD_DESCRIPTOR) + .setNormalizationMode(normalizationMode) + .setNormFwdPhase(phase) + .setxDesc(xTensor) + .setScaleAndBias(scaleTensor, biasTensor) + .setPrevRunningMeanAndVar(inMeanTensor, inVarTensor) + .setNextRunningMeanAndVar(outMeanTensor, outVarTensor) + .setSavedMeanAndInvVar(savedMeanTensor, savedInvVarTensor) + .setEpsilonTensor(epsilonTensor) + .setExpDecayFactorTensor(expDecayTensor) + .setPeerStatTensor(peerStatTensors) + .setyDesc(yTensor) + .build(); + + std::array ops = {&batch_norm_op}; + auto opGraph = cudnn_frontend::OperationGraphBuilder().setHandle(handle_).setOperationGraph(ops.size(), ops.data()).build(); + + cudnn_frontend::EngineConfigList filtered_configs; + auto statuses = + cudnn_frontend::get_heuristics_list<2>({"heuristics_instant" + , "heuristics_fallback" + }, opGraph,::AllowAll, filtered_configs, true); + + auto plan_builder = [&filtered_configs, &opGraph, &handle_]() { + for (size_t i = 0; i < filtered_configs.size(); i++) { + try { + auto plan = cudnn_frontend::ExecutionPlanBuilder().setHandle(handle_).setEngineConfig(filtered_configs[i], opGraph.getTag()).build(); + return plan; + } catch (cudnn_frontend::cudnnException &e) { + continue; + } + } + return cudnn_frontend::ExecutionPlanBuilder().setHandle(handle_).setEngineConfig(filtered_configs[0], opGraph.getTag()).build(); + }; + + CHECK(filtered_configs.size() > 0); + auto plan = plan_builder(); + + auto workspace_size = plan.getWorkspaceSize(); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + std::vector data_ptrs {xDevPtr, yDevPtr, scaledevPtr, biasdevPtr, + in_meandevPtr, in_vardevPtr, out_meandevPtr, out_vardevPtr, + saved_meandevPtr, saved_inv_vardevPtr, + &epsilon_val, &exponential_decay_factor}; + data_ptrs.insert(data_ptrs.end(), peer_devPtrs.begin(), peer_devPtrs.end()); + std::vector uids; + for (size_t i = 100; i < 100 + data_ptrs.size(); ++i) { + uids.push_back(i); + } + + assert(data_ptrs.size() == uids.size()); + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(data_ptrs.size(), data_ptrs.data()) + .setUids(uids.size(), uids.data()) + .build(); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); +#endif + + size_t peer_size = 1; + for (size_t i = 0; i < 4; ++i){ + peer_size *= peerDims[i]; + } + // Reset local communication buffer + cudaMemsetAsync(peer_devPtrs[rank_id], 0, peer_size*4, stream); + } catch (cudnn_frontend::cudnnException &e) { + struct cudaDeviceProp prop; + checkCudaErrors(cudaGetDeviceProperties(&prop, 0)); + if (prop.major == 8) { + std::cout << "[ERROR] Exception " << e.what() << std::endl; + CHECK(false); + } + } +} + +void +run_batch_norm_backward( + int64_t *perChannelSum, + int64_t *epsilon, + int64_t *tensorDims, + int64_t *peerDims, + + void *xDevPtr, + void *dyDevPtr, + void *scaledevPtr, + void *saved_meandevPtr, + void *saved_inv_vardevPtr, + void *dxDevPtr, + void *dscaledevPtr, + void *dbiasdevPtr, + const std::vector &peer_devPtrs, + double epsilon_val, + int rank_id) + +{ + cudaStream_t stream; + cudnnHandle_t handle_ = torch::native::getCudnnHandle(); + cudnnGetStream(handle_, &stream); + try { + + // Creates the necessary tensor descriptors + int64_t tensor_stride[4]; + int64_t stride[4]; + int64_t peer_stride[4]; + + generateStrides(tensorDims, tensor_stride, 4, CUDNN_TENSOR_NHWC); + generateStrides(peerDims, peer_stride, 4, CUDNN_TENSOR_NHWC); + + auto tensor_create = [&tensor_stride, &tensorDims](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, tensorDims) + .setStrides(4, tensor_stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .build(); + }; + + auto peer_tensor_create = [&peer_stride, &tensorDims](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, tensorDims) + .setStrides(4, peer_stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .build(); + }; + + generateStrides(perChannelSum, stride, 4, CUDNN_TENSOR_NHWC); + + auto per_channel_tensor_create = [&stride, &perChannelSum](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, perChannelSum) + .setStrides(4, stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .build(); + }; + + auto xTensor = tensor_create(CUDNN_DATA_HALF, 100); + auto dyTensor = tensor_create(CUDNN_DATA_HALF, 101); + auto scaleTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 102); + auto savedMeanTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 103); + auto savedInvVarTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 104); + auto dxTensor = tensor_create(CUDNN_DATA_HALF, 105); + auto dScaleTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 106); + auto dBiasTensor = per_channel_tensor_create(CUDNN_DATA_FLOAT, 107); + + + int64_t epsilon_stride[4]; + generateStrides(epsilon, epsilon_stride, 4, CUDNN_TENSOR_NHWC); + auto scalar_tensor_create = [&epsilon_stride, &epsilon](cudnnDataType_t type, + int64_t id) { + return cudnn_frontend::TensorBuilder() + .setDim(4, epsilon) + .setStrides(4, epsilon_stride) + .setId(id) + .setAlignment(16) + .setDataType(type) + .setByValue(true) + .build(); + }; + + auto epsilonTensor = scalar_tensor_create(CUDNN_DATA_DOUBLE, 108); + + + std::vector peerStatTensors; + for (size_t i = 109; i < 109 + peer_devPtrs.size(); ++i) { + peerStatTensors.push_back(peer_tensor_create(CUDNN_DATA_FLOAT, i)); + } + +#if (CUDNN_VERSION >= 8500) + // Batch normalization + cudnnBackendNormMode_t normalizationMode = CUDNN_BATCH_NORM; + + //Create a Finalize node + auto batch_norm_op = cudnn_frontend::OperationBuilder(CUDNN_BACKEND_OPERATION_NORM_BACKWARD_DESCRIPTOR) + .setNormalizationMode(normalizationMode) + .setxDesc(xTensor) + .setSavedMeanAndInvVar(savedMeanTensor, savedInvVarTensor) + .setdyDesc(dyTensor) + .setScale(scaleTensor) + .setEpsilonTensor(epsilonTensor) + .setDScaleAndDBias(dScaleTensor, dBiasTensor) + .setdxDesc(dxTensor) + .setPeerStatTensor(peerStatTensors) + .build(); + + std::array ops = {&batch_norm_op}; + auto opGraph = cudnn_frontend::OperationGraphBuilder().setHandle(handle_).setOperationGraph(ops.size(), ops.data()).build(); + + cudnn_frontend::EngineConfigList filtered_configs; + auto statuses = + cudnn_frontend::get_heuristics_list<2>({"heuristics_instant" + , "heuristics_fallback" + }, opGraph,::AllowAll, filtered_configs, true); + + auto plan_builder = [&filtered_configs, &opGraph, &handle_]() { + for (size_t i = 0; i < filtered_configs.size(); i++) { + try { + auto plan = cudnn_frontend::ExecutionPlanBuilder().setHandle(handle_).setEngineConfig(filtered_configs[i], opGraph.getTag()).build(); + return plan; + } catch (cudnn_frontend::cudnnException &e) { + continue; + } + } + return cudnn_frontend::ExecutionPlanBuilder().setHandle(handle_).setEngineConfig(filtered_configs[0], opGraph.getTag()).build(); + }; + + CHECK(filtered_configs.size() > 0); + auto plan = plan_builder(); + + auto workspace_size = plan.getWorkspaceSize(); + + void* workspace_ptr = nullptr; + auto workspace_tensor = at::empty({(workspace_size+3)/4}, at::TensorOptions(at::kCUDA).dtype(at::kFloat)); + if (workspace_size > 0) { + workspace_ptr = workspace_tensor.data_ptr(); + } + std::vector data_ptrs {xDevPtr, dyDevPtr, scaledevPtr, + saved_meandevPtr, saved_inv_vardevPtr, + dxDevPtr, dscaledevPtr, dbiasdevPtr, &epsilon_val}; + data_ptrs.insert(data_ptrs.end(), peer_devPtrs.begin(), peer_devPtrs.end()); + std::vector uids; + for (size_t i = 100; i < 100 + data_ptrs.size(); ++i) { + uids.push_back(i); + } + + assert(data_ptrs.size() == uids.size()); + + auto variantPack = cudnn_frontend::VariantPackBuilder() + .setWorkspacePointer(workspace_ptr) + .setDataPointers(data_ptrs.size(), data_ptrs.data()) + .setUids(uids.size(), uids.data()) + .build(); + cudnnStatus_t status = cudnnBackendExecute(handle_, plan.get_raw_desc(), variantPack.get_raw_desc()); + cudnn_frontend::throw_if([status]() { return (status != CUDNN_STATUS_SUCCESS); }, "Plan execute error", status); +#endif + + size_t peer_size = 1; + for (size_t i = 0; i < 4; ++i){ + peer_size *= peerDims[i]; + } + // Reset local communication buffer + cudaMemsetAsync(peer_devPtrs[rank_id], 0, peer_size*4, stream); + + } catch (cudnn_frontend::cudnnException &e) { + struct cudaDeviceProp prop; + checkCudaErrors(cudaGetDeviceProperties(&prop, 0)); + if (prop.major == 8) { + std::cout << "[ERROR] Exception " << e.what() << std::endl; + CHECK(false); + } + } +} \ No newline at end of file diff --git a/apex/apex/contrib/csrc/cudnn_gbn/norm_sample.h b/apex/apex/contrib/csrc/cudnn_gbn/norm_sample.h new file mode 100644 index 00000000..26d51bac --- /dev/null +++ b/apex/apex/contrib/csrc/cudnn_gbn/norm_sample.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +void +run_batch_norm_forward( + int64_t *perChannelSum, + int64_t *epsilon, + int64_t *tensorDims, + int64_t *peerDims, + + void *xDevPtr, + void *yDevPtr, + void *scaledevPtr, + void *biasdevPtr, + void *in_meandevPtr, + void *in_vardevPtr, + void *out_meandevPtr, + void *out_vardevPtr, + void *saved_meandevPtr, + void *saved_inv_vardevPtr, + const std::vector &peer_devPtrs, + double epsilon_val, + double exponential_decay_factor, + int rank_id +); + +void +run_batch_norm_backward( + int64_t *perChannelSum, + int64_t *epsilon, + int64_t *tensorDims, + int64_t *peerDims, + + void *xDevPtr, + void *dyDevPtr, + void *scaledevPtr, + void *saved_meandevPtr, + void *saved_inv_vardevPtr, + void *dxDevPtr, + void *dscaledevPtr, + void *dbiasdevPtr, + const std::vector &peer_devPtrs, + double epsilon_val, + int rank_id +); \ No newline at end of file diff --git a/apex/apex/contrib/csrc/fmha/fmha_api.cpp b/apex/apex/contrib/csrc/fmha/fmha_api.cpp new file mode 100644 index 00000000..46032867 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/fmha_api.cpp @@ -0,0 +1,365 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include +#include + +#include "fmha.h" + +extern at::Tensor & mha_fill(at::Tensor &self, const at::Tensor &start_index); +void set_params(Fused_multihead_attention_fprop_params ¶ms, + // sizes + const size_t b, + const size_t s, + const size_t h, + const size_t d, + // device pointers + void *qkv_packed_d, + void *cu_seqlens_d, + void *o_packed_d, + void *s_d, + float p_dropout) { + + Data_type acc_type = DATA_TYPE_FP32; + Data_type data_type = DATA_TYPE_FP16; + + // Reset the parameters + memset(¶ms, 0, sizeof(params)); + + // Set the pointers and strides. + params.qkv_ptr = qkv_packed_d; + params.qkv_stride_in_bytes = get_size_in_bytes(h * 3 * d, data_type); + params.o_ptr = o_packed_d; + params.o_stride_in_bytes = get_size_in_bytes(h * d, data_type); + + params.cu_seqlens = static_cast(cu_seqlens_d); + + // S = softmax(P) + params.s_ptr = s_d; + params.s_stride_in_bytes = get_size_in_bytes(b * h * s, data_type); + + // Set the dimensions. + params.b = b; + params.h = h; + params.s = s; + params.d = d; + + // Set the different scale values. + const float scale_bmm1 = 1.f / sqrtf(d); + constexpr float scale_softmax = 1.f; + constexpr float scale_bmm2 = 1.f; + + set_alpha(params.scale_bmm1, scale_bmm1, data_type); + set_alpha(params.scale_softmax, scale_softmax, acc_type); + set_alpha(params.scale_bmm2, scale_bmm2, data_type); + + // Set this to probability of keeping an element to simplify things. + params.p_dropout = 1.f - p_dropout; + params.rp_dropout = 1.f / params.p_dropout; + TORCH_CHECK(p_dropout < 1.f); + set_alpha(params.scale_dropout, params.rp_dropout, data_type); +} + +std::vector +mha_fwd(const at::Tensor &qkv, // total x num_heads x 3 x head_size, total := \sum_{i=0}^{b} s_i + const at::Tensor &cu_seqlens, // b+1 + const float p_dropout, + const int max_seq_len, + const bool is_training, + const bool is_nl, + const bool zero_tensors, + c10::optional gen_) { + + using namespace torch::indexing; + auto dprops = at::cuda::getCurrentDeviceProperties(); + TORCH_CHECK((dprops->major == 8 && dprops->minor == 0) || + (dprops->major == 9 && dprops->minor == 0)); + auto stream = at::cuda::getCurrentCUDAStream().stream(); + Launch_params launch_params(dprops, stream, is_training, is_nl); + + int seq_len = 512; + auto launch = &run_fmha_fp16_512_64_sm80; + if( max_seq_len <= 128 ) { + seq_len = 128; + launch = &run_fmha_fp16_128_64_sm80; + } else if( max_seq_len <= 256 ) { + seq_len = 256; + launch = &run_fmha_fp16_256_64_sm80; + } else if( max_seq_len <= 384 ) { + seq_len = 384; + launch = &run_fmha_fp16_384_64_sm80; + } else if( max_seq_len <= 512 ) { + seq_len = 512; + launch = &run_fmha_fp16_512_64_sm80; + } else { + TORCH_CHECK(false); + } + + TORCH_CHECK(qkv.is_cuda()) + TORCH_CHECK(cu_seqlens.is_cuda()) + + TORCH_CHECK(qkv.is_contiguous()) + TORCH_CHECK(cu_seqlens.is_contiguous()) + + TORCH_CHECK(cu_seqlens.dim() == 1); + TORCH_CHECK(qkv.dim() == 4); + + const auto sizes = qkv.sizes(); + + TORCH_CHECK(sizes[THREE_DIM] == 3); + + const int batch_size = cu_seqlens.numel() - 1; + const int total = sizes[TOTAL_DIM]; + const int num_heads = sizes[H_DIM]; + const int head_size = sizes[D_DIM]; + TORCH_CHECK(batch_size > 0); + TORCH_CHECK(head_size == 64); + auto opts = qkv.options(); + + auto ctx = torch::empty({ total, num_heads, head_size }, opts); + + auto s = torch::empty({ batch_size, num_heads, seq_len, seq_len }, opts); + + if( zero_tensors ) { + mha_fill(ctx, cu_seqlens.index({Slice(-1,None)})); + } + + auto gen = at::get_generator_or_default( + gen_, at::cuda::detail::getDefaultCUDAGenerator()); + + + set_params(launch_params.params, + batch_size, + seq_len, + num_heads, + head_size, + qkv.data_ptr(), + cu_seqlens.data_ptr(), + ctx.data_ptr(), + s.data_ptr(), + p_dropout); + + launch(launch_params, /*configure=*/ true); + // number of times random will be generated per thread, to offset philox counter in thc random + // state + int64_t counter_offset = launch_params.elts_per_thread; + at::PhiloxCudaState rng_engine_inputs; + + if( is_training ) { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen->mutex_); + launch_params.params.philox_args = gen->philox_cuda_state(counter_offset); + } + + launch(launch_params, /*configure=*/ false); + + return { ctx, s }; +} + + +std::vector +mha_bwd(const at::Tensor &dout, // total x num_heads, x head_size + const at::Tensor &qkv, // total x num_heads x 3 x head_size, total := \sum_{i=0}^{b} s_i + at::Tensor &softmax, // b x h x s x s softmax and dmask - will be overwritten with dP + const at::Tensor &cu_seqlens, // b+1 + const float p_dropout, // probability to drop + const int max_seq_len, // max sequence length to choose the kernel + const bool zero_tensors +) { + using namespace torch::indexing; + auto dprops = at::cuda::getCurrentDeviceProperties(); + TORCH_CHECK((dprops->major == 8 && dprops->minor == 0) || + (dprops->major == 9 && dprops->minor == 0)); + int seq_len = 512; + auto launch = &run_fmha_dgrad_fp16_512_64_sm80; + if( max_seq_len <= 128 ) { + seq_len = 128; + launch = &run_fmha_dgrad_fp16_128_64_sm80; + } else if( max_seq_len <= 256 ) { + seq_len = 256; + launch = &run_fmha_dgrad_fp16_256_64_sm80; + } else if( max_seq_len <= 384 ) { + seq_len = 384; + launch = &run_fmha_dgrad_fp16_384_64_sm80; + } else if( max_seq_len <= 512 ) { + seq_len = 512; + launch = &run_fmha_dgrad_fp16_512_64_sm80; + } else { + TORCH_CHECK(false); + } + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + + TORCH_CHECK(qkv.dtype() == torch::kFloat16); + TORCH_CHECK(dout.dtype() == torch::kFloat16); + TORCH_CHECK(softmax.dtype() == torch::kFloat16); + TORCH_CHECK(cu_seqlens.dtype() == torch::kInt32); + + TORCH_CHECK(qkv.is_cuda()); + TORCH_CHECK(cu_seqlens.is_cuda()); + + TORCH_CHECK(qkv.is_contiguous()); + TORCH_CHECK(cu_seqlens.is_contiguous()); + + TORCH_CHECK(cu_seqlens.dim() == 1); + TORCH_CHECK(qkv.dim() == 4); + + const auto sizes = qkv.sizes(); + + TORCH_CHECK(sizes[THREE_DIM] == 3); + + const int batch_size = cu_seqlens.numel() - 1; + const int num_heads = sizes[H_DIM]; + const int head_size = sizes[D_DIM]; + TORCH_CHECK(batch_size > 0); + TORCH_CHECK(head_size == 64); + + auto dqkv = torch::empty_like(qkv); + + if( zero_tensors ) { + mha_fill(dqkv, cu_seqlens.index({Slice(-1,None)})); + } + + Fused_multihead_attention_fprop_params params; + + set_params(params, + batch_size, + seq_len, + num_heads, + head_size, + qkv.data_ptr(), + cu_seqlens.data_ptr(), + dout.data_ptr(), // we set o_ptr to dout + softmax.data_ptr(), // softmax gets overwritten by dP! + p_dropout); + + // we're re-using these scales + Data_type acc_type = DATA_TYPE_FP32; + set_alpha(params.scale_bmm1, 1.f, acc_type); + set_alpha(params.scale_softmax, 1.f / sqrtf(head_size), acc_type); + set_alpha(params.scale_bmm2, 1.f, DATA_TYPE_FP16); + params.dqkv_ptr = dqkv.data_ptr(); + + launch(params, stream); + return { dqkv, softmax }; +} + +std::vector mha_bwd_nl(const at::Tensor &dout, // total x num_heads, x head_size + const at::Tensor &qkv, // total x num_heads x 3 x head_size, total := \sum_{i=0}^{b} s_i + at::Tensor &softmax, // b x h x s x s softmax and dmask - will be overwritten with dP + const at::Tensor &cu_seqlens, // b+1 + const float p_dropout, // probability to drop + const int max_seq_len, // max sequence length to choose the kernel + const bool zero_tensors +) { + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + + TORCH_CHECK(qkv.is_cuda()) + TORCH_CHECK(cu_seqlens.is_cuda()) + + TORCH_CHECK(qkv.is_contiguous()) + TORCH_CHECK(cu_seqlens.is_contiguous()) + + TORCH_CHECK(cu_seqlens.dim() == 1); + + TORCH_CHECK(qkv.dim() == 4); + + const auto sizes = qkv.sizes(); + + TORCH_CHECK(sizes[THREE_DIM] == 3); + + const int batch_size = cu_seqlens.numel() - 1; + + const int total = sizes[TOTAL_DIM]; + const int num_heads = sizes[H_DIM]; + const int head_size = sizes[D_DIM]; + TORCH_CHECK(batch_size > 0); + TORCH_CHECK(head_size == 64); + + int seq_len = 512; + auto launch = &run_fmha_dgrad_fp16_512_64_sm80_nl; + + auto opts = qkv.options(); + + auto dqkv = torch::empty_like(qkv); + + if( zero_tensors ) { + dqkv.zero_(); + } + + int num_chunks = 2; + if( batch_size == 1 ) { + num_chunks = 4; + }else if( batch_size == 2 ) { + num_chunks = 3; + } + auto dkv = torch::empty({total, num_chunks, 2, num_heads, head_size}, opts); + + Fused_multihead_attention_fprop_params params; + + set_params(params, + batch_size, + seq_len, + num_heads, + head_size, + qkv.data_ptr(), + cu_seqlens.data_ptr(), + dout.data_ptr(), // o_ptr = dout + softmax.data_ptr(), // softmax gets overwritten by dP! + p_dropout); + + params.dkv_ptr = dkv.data_ptr(); + + Data_type acc_type = DATA_TYPE_FP32; + set_alpha(params.scale_bmm1, 1.f, acc_type); + set_alpha(params.scale_softmax, 1.f / sqrtf(head_size), acc_type); + set_alpha(params.scale_bmm2, 1.f, DATA_TYPE_FP16); + params.dqkv_ptr = dqkv.data_ptr(); + + launch(params, num_chunks, stream); + + //SPLIT-K reduction of num_chunks dK, dV parts + + // The equivalent of the following Pytorch code: + // using namespace torch::indexing; + // at::Tensor view_out = dqkv.index({Slice(), Slice(1, None, None)}); + // torch::sum_out(view_out, dkv, 1); + + const int hidden_size = num_heads * head_size; + fmha_run_noloop_reduce( + dqkv.data_ptr(), dkv.data_ptr(), cu_seqlens.data_ptr(), hidden_size, batch_size, total, num_chunks, stream); + + return { dqkv, softmax, dkv }; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.doc() = "Fused Multi-head Self-attention for BERT"; + m.def("fwd", &mha_fwd, "Forward pass"); + m.def("bwd", &mha_bwd, "Backward pass"); + m.def("bwd_nl", &mha_bwd_nl, "Backward pass (small-batch)"); +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha.h b/apex/apex/contrib/csrc/fmha/src/fmha.h new file mode 100644 index 00000000..d01a9150 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha.h @@ -0,0 +1,163 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include +#include + +#ifdef OLD_GENERATOR_PATH +#include +#else +#include +#endif + +#include + +#include + + +constexpr int TOTAL_DIM = 0; +constexpr int THREE_DIM = 1; +constexpr int H_DIM = 2; +constexpr int D_DIM = 3; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Qkv_params { + // The QKV matrices. + void * __restrict__ qkv_ptr; + + // The stride between rows of the Q, K and V matrices. + size_t qkv_stride_in_bytes; + + // The number of heads. + int h; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Fused_multihead_attention_fprop_params : public Qkv_params { + + // The dQKV matrices. + void * __restrict__ dqkv_ptr; + + // Temporary for dKV. + void * __restrict__ dkv_ptr; + + // The O matrix (output). + void * __restrict__ o_ptr; + + // The stride between rows of O. + int64_t o_stride_in_bytes; + + // The pointer to the S matrix, overwritten by the dP matrix (bwd). + void * __restrict__ s_ptr; + // The stride between rows of the S matrix. + int64_t s_stride_in_bytes; + + // The dimensions. + int b, s, d; + + // The scaling factors for the kernel. + uint32_t scale_bmm1, scale_softmax, scale_bmm2; + + // array of length b+1 holding starting offset of each sequence. + int * __restrict__ cu_seqlens; + + // The dropout probability (probability of keeping an activation). + float p_dropout; + + // Scale factor of 1 / (1 - p_dropout). + float rp_dropout; + + // Scale factor of 1 / (1 - p_dropout), in half2. + uint32_t scale_dropout; + + // Random state. + at::PhiloxCudaState philox_args; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Launch_params{ + Launch_params(cudaDeviceProp * props_, + cudaStream_t stream_, + bool is_training_, + bool is_nl_) + : elts_per_thread(0) + , props(props_) + , stream(stream_) + , is_training(is_training_) + , is_nl(is_nl_) { + } + + size_t elts_per_thread; + + cudaDeviceProp * props; + + cudaStream_t stream; + + bool is_training; + + Kernel_params params; + int num_full_heads; + int num_main_groups; + int heads_last_wave; + int main_steps; + int rest_steps; + bool is_nl; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +void run_fmha_fp16_128_64_sm80(Launch_params &launch_params, const bool configure); +void run_fmha_fp16_256_64_sm80(Launch_params &launch_params, const bool configure); +void run_fmha_fp16_384_64_sm80(Launch_params &launch_params, const bool configure); +void run_fmha_fp16_512_64_sm80(Launch_params &launch_params, const bool configure); + +void run_fmha_dgrad_fp16_128_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream); +void run_fmha_dgrad_fp16_256_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream); +void run_fmha_dgrad_fp16_384_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream); +void run_fmha_dgrad_fp16_512_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream); + +void run_fmha_fp16_512_64_sm80_nl(const Fused_multihead_attention_fprop_params ¶ms, const bool is_training, const int num_chunks, cudaStream_t stream); + +void run_fmha_dgrad_fp16_512_64_sm80_nl(const Fused_multihead_attention_fprop_params ¶ms, const int num_chunks, cudaStream_t stream); + +void fmha_run_noloop_reduce(void *out, + const void *in, + const int *cu_seqlens, + const int hidden_size, + const int batch_size, + const int total, + const int num_chunks, + cudaStream_t stream); + + diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/gemm.h b/apex/apex/contrib/csrc/fmha/src/fmha/gemm.h new file mode 100644 index 00000000..62529a2c --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/gemm.h @@ -0,0 +1,314 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include + +#define FMHA_DIV_UP(m, n) (((m) + (n)-1) / (n)) + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Data_type_, int NUM_ELTS_, int BITS_PER_ELT_, int ALIGNMENT_ > +struct Fragment_base_ { + + // The data type. + using Data_type = Data_type_; + // default input type + using Input_type_ = Data_type_; + // Does it store the array of elements. + enum { HAS_ELTS = BITS_PER_ELT_ >= 8 }; + // The number of elements. + enum { NUM_ELTS = NUM_ELTS_ }; + // The size of element in bits. + enum { BITS_PER_ELT = BITS_PER_ELT_ }; + // The size of byte of a single register. + enum { BYTES_PER_REG = 4 }; + // The size in bits. + enum { BITS_PER_REG = BYTES_PER_REG * 8 }; + // The number of registers needed to store the fragment. + enum { NUM_REGS = Div_up::VALUE }; + // The size in bytes (as returned by sizeof(Fragment_base<>). + enum { SIZE_IN_BYTES = NUM_REGS * BYTES_PER_REG }; + // The alignment. + enum { ALIGNMENT = ALIGNMENT_ > 0 ? ALIGNMENT_ : Min::VALUE }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The type of the elements. + typename Data_type_, + // The number of elements. + int NUM_ELTS_, + // The alignment if you want to force a value -- use 0 otherwise. + int ALIGNMENT_ = 0, + // The base class. + typename Base_ = Fragment_base_ +> +struct alignas(static_cast(Base_::ALIGNMENT)) Fragment : public Base_ { + + // The size of a load/store. + enum { BYTES_PER_LOAD_STORE = Base_::NUM_REGS * sizeof(uint32_t) }; + + // Clear the fragment. Using PTX in that code seems to produce better SASS... + inline __device__ void clear() { + #pragma unroll + for( int ii = 0; ii < Base_::NUM_REGS; ++ii ) { + asm volatile("mov.u32 %0, 0; \n" : "=r"(this->reg(ii)) : ); + } + } + + // Immutable access to a register. + inline __device__ const uint32_t& reg(int ii) const { + return this->regs_[ii]; + } + + // Mutable access to a register. + inline __device__ uint32_t& reg(int ii) { + return this->regs_[ii]; + } + + uint32_t regs_[Base_::NUM_REGS]; + + // Immutable access to the elements. + inline __device__ const Data_type_& elt(int ii) const { + return reinterpret_cast(&this->regs_[0])[ii]; + } + + // Mutable access to the elements. + inline __device__ Data_type_& elt(int ii) { + return reinterpret_cast(&this->regs_[0])[ii]; + } + + // Immutable access to the elements with a cast. + template< typename Cast_type > + inline __device__ const Cast_type& elt_as(int ii) const { + return reinterpret_cast(&this->regs_[0])[ii]; + } + + // Mutable access to the elements. + template< typename Cast_type > + inline __device__ Cast_type& elt_as(int ii) { + return reinterpret_cast(&this->regs_[0])[ii]; + } + + // Add another fragment. + inline __device__ void add(const Fragment &other) { + #pragma unroll + for( int ii = 0; ii < NUM_ELTS_; ++ii ) { + this->elt(ii) += other.elt(ii); + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Layout > +struct Fragment_a : public Fragment { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Layout > +struct Fragment_b : public Fragment { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Fragment_accumulator : public Fragment { + + // The base class. + using Base = Fragment; + + // Add two fragments. + template< typename Other_fragment_ > + inline __device__ void add(const Other_fragment_ &other) { + for( int ii = 0; ii < Base::NUM_ELTS; ++ii ) { + this->elt(ii) = this->elt(ii) + other.elt(ii); + } + } + + // Do the HMMA. + template< typename Layout_a, typename Layout_b > + inline __device__ void mma(const Fragment_a &a, + const Fragment_b &b) { + asm volatile( \ + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 \n" \ + " {%0, %1, %2, %3}, \n" \ + " {%4, %5, %6, %7}, \n" \ + " {%8, %9}, \n" \ + " {%0, %1, %2, %3}; \n" \ + : "+f"( elt(0)), "+f"( elt(1)), "+f"( elt(2)), "+f"( elt(3)) + : "r"(a.reg(0)), "r"(a.reg(1)), "r"(a.reg(2)), "r"(a.reg(3)) + , "r"(b.reg(0)), "r"(b.reg(1))); + asm volatile( \ + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 \n" \ + " {%0, %1, %2, %3}, \n" \ + " {%4, %5, %6, %7}, \n" \ + " {%8, %9}, \n" \ + " {%0, %1, %2, %3}; \n" \ + : "+f"( elt(4)), "+f"( elt(5)), "+f"( elt(6)), "+f"( elt(7)) + : "r"(a.reg(0)), "r"(a.reg(1)), "r"(a.reg(2)), "r"(a.reg(3)) + , "r"(b.reg(2)), "r"(b.reg(3))); + } + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Fragment, int M, int N > +inline __device__ void clear(Fragment (&frag)[M][N]) { + #pragma unroll + for( int mi = 0; mi < M; ++mi ) { + #pragma unroll + for( int ni = 0; ni < N; ++ni ) { + frag[mi][ni].clear(); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Accumulator_type, int WARPS_K > +struct Clear_accumulator { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int WARPS_K > +struct Clear_accumulator { + template< typename Acc, int M, int N > + static inline __device__ void apply(Acc (&acc)[M][N], bool = false) { + fmha::clear(acc); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ void gemm(Acc (&acc)[M][N], const A (&a)[M], const B (&b)[N]) { + + #pragma unroll + for( int mi = 0; mi < M; ++mi ) { + #pragma unroll + for( int ni = 0; ni < N; ++ni ) { + acc[mi][ni].mma(a[mi], b[ni]); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The number of rows in the CTA tile. + int M_, + // The number of cols in the CTA tile. + int N_, + // The number of elements in the the K dimension of the GEMM loop. + int K_, + // The number of rows of warps. + int WARPS_M_, + // The number of cols of warps. + int WARPS_N_, + // The number of warps in the K dimension of the GEMM loop. + int WARPS_K_> +struct Cta_tile_ { + + enum { M = M_, N = N_, K = K_ }; + // The number of warps. + enum { WARPS_M = WARPS_M_, WARPS_N = WARPS_N_, WARPS_K = WARPS_K_ }; + // The number of warps per CTA. + enum { WARPS_PER_CTA = WARPS_M * WARPS_N * WARPS_K }; + // The number of threads per warp. + enum { THREADS_PER_WARP = 32 }; + // The number of threads per CTA. + enum { THREADS_PER_CTA = WARPS_PER_CTA * THREADS_PER_WARP }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Hmma_tile { + // The number of elements computed with a single warp-MMA. + enum { M_PER_MMA = 16, N_PER_MMA = 16, K_PER_MMA = 16 }; + + // The number of elements computed with a single CTA-MMA. + enum { + M_PER_MMA_PER_CTA = M_PER_MMA * Cta_tile::WARPS_M, + N_PER_MMA_PER_CTA = N_PER_MMA * Cta_tile::WARPS_N, + K_PER_MMA_PER_CTA = K_PER_MMA * Cta_tile::WARPS_K + }; + + // The number of MMAs needed to compute the GEMM. + enum { + MMAS_M = Div_up::VALUE, + MMAS_N = Div_up::VALUE, + MMAS_K = Div_up::VALUE, + }; + + // The number of elements computed per warp. + enum { + M_PER_WARP = MMAS_M * M_PER_MMA, + N_PER_WARP = MMAS_N * N_PER_MMA, + K_PER_WARP = MMAS_K * K_PER_MMA, + }; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +using A_type = uint16_t; +using B_type = uint16_t; +using C_type = uint16_t; +using Accumulator_type = float; +using Epilogue_type = float; + +constexpr int BITS_PER_ELEMENT_A = sizeof(A_type) * 8; +constexpr int BITS_PER_ELEMENT_B = sizeof(B_type) * 8; +constexpr int BITS_PER_ELEMENT_C = sizeof(C_type) * 8; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +using Cta_tile_extd = Cta_tile_; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +using Cta_tile_with_k_with_padding = Cta_tile_extd::VALUE, + Cta_tile_::WARPS_M, + Cta_tile_::WARPS_N, + Cta_tile_::WARPS_K>; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/gmem_tile.h b/apex/apex/contrib/csrc/fmha/src/fmha/gmem_tile.h new file mode 100644 index 00000000..5c86dd84 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/gmem_tile.h @@ -0,0 +1,456 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The number of bits per element. + int BITS_PER_ELEMENT, + // The number of rows of Q, K or V loaded by this tile. + int ROWS, + // The number of columns. + int COLS, + // The number of matrics. + int NUM_MATS = 3 +> +struct Gmem_tile_qkv { + + // The size of each LDG. + enum { BYTES_PER_LDG = 16 }; + // The size of a row in bytes. + enum { BYTES_PER_ROW = COLS * BITS_PER_ELEMENT / 8 }; + + // The number of threads to load a "row" of the matrix. + enum { THREADS_PER_ROW = BYTES_PER_ROW / BYTES_PER_LDG }; + + // The number of "rows" loaded per LDG. + enum { ROWS_PER_LDG = Cta_tile::THREADS_PER_CTA / THREADS_PER_ROW }; + // The number of LDGs needed to load a chunk of the Q matrix. + enum { LDGS = fmha::Div_up::VALUE }; + + // Ctor. + template< typename Params, typename BInfo > + inline __device__ Gmem_tile_qkv(const Params ¶ms, const int qkv_offset, const BInfo &binfo, const int tidx) + : params_qkv_stride_in_bytes_(params.qkv_stride_in_bytes) + , actual_seqlen(binfo.actual_seqlen) + , qkv_ptr_(reinterpret_cast(params.qkv_ptr)) { + + // Compute the position in the sequence (within the CTA for the moment). + int row = tidx / THREADS_PER_ROW; + // Compute the position of the thread in the row. + int col = tidx % THREADS_PER_ROW; + + // Store the row as we need it to disable the loads. + row_ = row; + + // The row offset in the batched GEMM. For each seq element, we store QKV in that order. + int64_t row_offset = (int64_t)row * params.qkv_stride_in_bytes; + // Add the block index. + row_offset += (int64_t)((binfo.sum_s * NUM_MATS + qkv_offset) * binfo.h + binfo.bidh) * BYTES_PER_ROW; + + // Assemble the final pointer. + qkv_ptr_ += row_offset + col * BYTES_PER_LDG; + } + + // Store data to shared memory. + template< typename Smem_tile > + inline __device__ void commit(Smem_tile &smem_tile) { + smem_tile.store(fetch_); + } + + // Load data from memory. + template< typename Smem_tile > + inline __device__ void load(Smem_tile &smem_tile) { + const void *ptrs[LDGS]; + uint32_t preds[LDGS]; + #pragma unroll + for( int ii = 0; ii < LDGS; ++ii ) { + ptrs[ii] = qkv_ptr_ + (int64_t)ii * ROWS_PER_LDG * params_qkv_stride_in_bytes_; + preds[ii] = ((row_ + ii * ROWS_PER_LDG) < min(ROWS, actual_seqlen)); + fetch_[ii] = make_uint4(0, 0, 0, 0); + } + + // not packing predicates removes restrictions (e.g. FP16 384, 4 warps) + Ldg_functor fct(fetch_, ptrs); + #pragma unroll + for( int ii = 0; ii < LDGS; ++ii ) { + fct.load(ii, preds[ii]); + } + } + + // Store data to memory. + inline __device__ void store(const uint4 (&data)[LDGS]) { + #pragma unroll + for( int ii = 0; ii < LDGS; ++ii ) { + char *ptr = qkv_ptr_ + (int64_t)ii * ROWS_PER_LDG * params_qkv_stride_in_bytes_; + if( (row_ + ii * ROWS_PER_LDG) < min(ROWS, actual_seqlen) ) { + fmha::stg(ptr, data[ii]); + } + } + } + + // Move the pointer to the next location. + inline __device__ void move() { + qkv_ptr_ += (int64_t)ROWS * params_qkv_stride_in_bytes_; + actual_seqlen -= ROWS; + } + + inline __device__ void move(int steps) { + qkv_ptr_ += (int64_t)ROWS * params_qkv_stride_in_bytes_ * steps; + actual_seqlen -= ROWS * steps; + } + + // The stride between rows for the QKV matrice. + int64_t params_qkv_stride_in_bytes_; + // The pointer. + char *qkv_ptr_; + // The fetch registers. + uint4 fetch_[LDGS]; + // Keep track of the row the thread is processing as we move the tile. + int row_; + // The length of the sequence loaded by that memory tile. + int actual_seqlen; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Cta_tile > +struct Gmem_tile_o { + + // The mma tile. + using Mma_tile = fmha::Hmma_tile; + + // The size of each element. + enum { BYTES_PER_ELEMENT = 2 }; + // The size of a row in bytes. + enum { BYTES_PER_ROW = Cta_tile::N * BYTES_PER_ELEMENT }; + + // The number of threads to store a "row" of the matrix. + enum { THREADS_PER_ROW = 16 }; + // The size of each STG. + enum { BYTES_PER_STG = BYTES_PER_ROW / THREADS_PER_ROW }; + + // The number of "rows" stored per iteration of the loop. The output of 1 MMA. + enum { ROWS = Cta_tile::M }; + // The number of "rows" stored per iteration of the loop. The output of 1 MMA. + enum { ROWS_PER_LOOP = ROWS <= 64 ? ROWS : (int)Mma_tile::M_PER_MMA_PER_CTA }; + // The number of outter loop for the stores. + enum { LOOPS = ROWS / ROWS_PER_LOOP }; + + // The number of "rows" stored per STG. + enum { ROWS_PER_STG = Cta_tile::THREADS_PER_CTA / THREADS_PER_ROW }; + // Do we have to guard against partial writes/reads. + enum { HAS_INCOMPLETE_STG = Cta_tile::M % ROWS_PER_STG != 0 }; + // The number of STGs needed to store a chunk of the Q matrix. + enum { STGS_PER_LOOP = fmha::Div_up::VALUE }; + // The number of STGs needed to store a chunk of the Q matrix in total. + enum { STGS = STGS_PER_LOOP * LOOPS }; + + // Ctor. + template + inline __device__ Gmem_tile_o(const Params ¶ms, const BInfo &binfo, int tidx) + : params_o_stride_in_bytes_(params.o_stride_in_bytes) + , actual_seqlen_(binfo.actual_seqlen) + , o_ptr_(reinterpret_cast(params.o_ptr)) { + + // Compute the position in the sequence (within the CTA for the moment). + int row = tidx / THREADS_PER_ROW; + // Compute the position of the thread in the row. + int col = tidx % THREADS_PER_ROW; + + // Store the row as we need it to disable loads. + row_ = row; + + // The row offset in the batched GEMM. + int64_t row_offset = (int64_t)row * params.o_stride_in_bytes + binfo.bidx * BYTES_PER_ROW; + // Assemble the final pointer. + o_ptr_ += row_offset + col * BYTES_PER_STG; + + // Is that thread active on the last STG? + if( HAS_INCOMPLETE_STG ) { + is_active_for_last_stg_ = row + (STGS - 1) * ROWS_PER_STG < Cta_tile::M; + } + } + + // Store data to global memory. + inline __device__ void store(const uint4 (&src)[STGS_PER_LOOP], int mi) { + + #pragma unroll + for( int ii = 0; ii < STGS_PER_LOOP; ++ii ) { + int jj = mi * STGS_PER_LOOP + ii; + if( this->row_ + jj * ROWS_PER_STG >= this->actual_seqlen_ ) { + break; + } + + float x = reinterpret_cast(src[ii].x); + float y = reinterpret_cast(src[ii].y); + float z = reinterpret_cast(src[ii].z); + float w = reinterpret_cast(src[ii].w); + uint2 out = float4_to_half4(x, y, z, w); + if( !HAS_INCOMPLETE_STG || (jj < STGS - 1 || this->is_active_for_last_stg_) ) { + fmha::stg(this->o_ptr_ + jj * ROWS_PER_STG * this->params_o_stride_in_bytes_, out); + } + } + } + + // Move the pointer to the next location. + inline __device__ void move() { + row_ += ROWS; + o_ptr_ += (int64_t)ROWS * params_o_stride_in_bytes_; + } + + inline __device__ void move(const int steps) { + row_ += ROWS * steps; + o_ptr_ += (int64_t)ROWS * params_o_stride_in_bytes_ * steps; + } + + // The stride between rows for the QKV matrice. + int64_t params_o_stride_in_bytes_; + // The pointer. + char *o_ptr_; + // Is the thread active for the last STG? + int is_active_for_last_stg_; + // Keep track of the row to disable loads. + int row_; + // The length of the sequence loaded by that memory tile. + int actual_seqlen_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Cta_tile, int BYTES_PER_ELEMENT > +struct Gmem_tile_mma_sd { + + // The mma tile. + using Mma_tile = fmha::Hmma_tile; + + // Each STG stores 8 elements. + enum { BYTES_PER_STG = BYTES_PER_ELEMENT * 8 }; + // The number of MMAs in the M dimension. + enum { MMAS_M = Mma_tile::MMAS_M }; + // The number of MMAs in the N dimension. + enum { MMAS_N = Mma_tile::MMAS_N }; + // The number of rows computed per MMA per thread block. + enum { M_PER_MMA_PER_CTA = Mma_tile::M_PER_MMA_PER_CTA }; + // The number of cols computed per MMA per thread block. + enum { N_PER_MMA_PER_CTA = Mma_tile::N_PER_MMA_PER_CTA }; + // The number of threads per block. + enum { THREADS_PER_CTA = Cta_tile::THREADS_PER_CTA }; + // The size of each row in bytes. I.e. how many bytes are stored per STG. + enum { BYTES_PER_ROW = THREADS_PER_CTA * BYTES_PER_STG }; + // The fixed sequence length. + enum { SEQLEN = Cta_tile::N }; + // The distance between two blocks (in bytes). + enum { BLOCK_STRIDE_BYTES = SEQLEN * SEQLEN * BYTES_PER_ELEMENT }; + // The distance between elements stored per loop (in bytes). + enum { LOOP_STRIDE_BYTES = MMAS_M * MMAS_N * BYTES_PER_ROW }; + + // The type of elements stored per STG. + using Type = typename fmha::Uint_from_size_in_bytes::Type; + + // Ctor. + template + inline __device__ Gmem_tile_mma_sd(void *ptr, const Params ¶ms, const int bidb, const int bidh, const int tidx) + : ptr_(static_cast(ptr)) { + + // The block index. + size_t bidx = bidb * params.h + bidh; + + // Set store location for each thread at the beginning of the loop + ptr_ += bidx * BLOCK_STRIDE_BYTES + tidx * BYTES_PER_STG; + } + + // Store to global memory. + inline __device__ void store(const Type &data, const int mi, const int ni) { + size_t offset = (mi * MMAS_N + ni) * BYTES_PER_ROW; + fmha::stg(ptr_ + offset, data); + } + + // Load from global memory. + inline __device__ void load(Type &data, const int mi, const int ni) { + size_t offset = (mi * MMAS_N + ni) * BYTES_PER_ROW; + fmha::ldg(data, ptr_ + offset); + } + + // Move to the next tile. + inline __device__ void move() { + ptr_ += LOOP_STRIDE_BYTES; + } + inline __device__ void move(const int steps) { + ptr_ += LOOP_STRIDE_BYTES * steps; + } + + // The pointer in global memory. + char *ptr_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Cta_tile, typename Base = Gmem_tile_mma_sd > +struct Gmem_tile_mma_s : public Base { + + // The number of mmas in the vertical dimension. + enum { M = Base::MMAS_M }; + // The number of mmas in the horizontal dimension. + enum { N = Base::MMAS_N }; + // The type of the vectors stored by each STG. + using Type = typename Base::Type; + + // Ctor. + template< typename Params, typename Block_info > + inline __device__ Gmem_tile_mma_s(const Params ¶ms, const Block_info& binfo, const int tidx) + : Base(params.s_ptr, params, binfo.bidb, binfo.bidh, tidx) { + } + + // Store to global memory. + template + inline __device__ void store(const float (&softmax)[2 * M][4 * N], const Mask &mask) { + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + + float tmp00 = softmax[2 * mi + 0][4 * ni + 0]; + float tmp01 = softmax[2 * mi + 0][4 * ni + 1]; + float tmp02 = softmax[2 * mi + 0][4 * ni + 2]; + float tmp03 = softmax[2 * mi + 0][4 * ni + 3]; + + float tmp10 = softmax[2 * mi + 1][4 * ni + 0]; + float tmp11 = softmax[2 * mi + 1][4 * ni + 1]; + float tmp12 = softmax[2 * mi + 1][4 * ni + 2]; + float tmp13 = softmax[2 * mi + 1][4 * ni + 3]; + + uint4 dst; + dst.x = fmha::float2_to_half2(tmp00, tmp01); + dst.y = fmha::float2_to_half2(tmp02, tmp03); + dst.z = fmha::float2_to_half2(tmp10, tmp11); + dst.w = fmha::float2_to_half2(tmp12, tmp13); + if( mask.is_valid(mi, ni, 0, 0) ) { + Base::store(dst, mi, ni); + } + } + } + } + + // Store to global memory. + template + inline __device__ void store(const Fragment (&frag)[N][M], const Mask& mask){ + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + uint4 dst; + dst.x = frag[ni][mi].reg(0); + dst.y = frag[ni][mi].reg(2); + dst.z = frag[ni][mi].reg(1); + dst.w = frag[ni][mi].reg(3); + if( mask.any_valid(mi, ni) ) { + Base::store(dst, mi, ni); + } + } + } + } + + // Load from global memory. + template + inline __device__ void load(uint4 (®s)[M][N], const Mask &mask) { + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + regs[mi][ni] = make_uint4(0, 0, 0, 0); + if( mask.any_valid(mi, ni) ) { + Base::load(regs[mi][ni], mi, ni); + } + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The base class. + typename Base = fmha::Gmem_tile_qkv +> +struct Gmem_tile_dout : public Base { + + // Ctor. + template + inline __device__ Gmem_tile_dout(const Params ¶ms, const BInfo &binfo, int tidx) + : Base(params, 0, binfo, tidx) { + + this->qkv_ptr_ = reinterpret_cast(params.o_ptr); + this->params_qkv_stride_in_bytes_ = params.o_stride_in_bytes; // needed for move + + // Compute the position of the thread in the row. + int col = tidx % Base::THREADS_PER_ROW; + + // The row offset in the batched GEMM. For each seq element, we store O in that order. + int64_t row_offset = (int64_t)this->row_ * params.o_stride_in_bytes + binfo.bidx * Base::BYTES_PER_ROW; + + // Assemble the final pointer. + this->qkv_ptr_ += row_offset + col * Base::BYTES_PER_LDG; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Cta_tile, typename Base = fmha::Gmem_tile_o > +struct Gmem_tile_dq : public Base { + + // Ctor. + template + inline __device__ Gmem_tile_dq(const Params ¶ms, const BInfo &binfo, int tidx) + : Base(params, binfo, tidx) { + this->o_ptr_ = reinterpret_cast(params.dqkv_ptr); + this->params_o_stride_in_bytes_ = params.qkv_stride_in_bytes; // needed for move + + // Compute the position of the thread in the row. + int col = tidx % Base::THREADS_PER_ROW; + + // The row offset in the batched GEMM. For each seq element, we store O in that order. + int64_t row_offset = (int64_t)this->row_ * params.qkv_stride_in_bytes + + (binfo.sum_s * 3 * binfo.h + binfo.bidh) * Base::BYTES_PER_ROW; + + // Assemble the final pointer. + this->o_ptr_ += row_offset + col * Base::BYTES_PER_STG; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha + diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/kernel_traits.h b/apex/apex/contrib/csrc/fmha/src/fmha/kernel_traits.h new file mode 100644 index 00000000..d51b47c5 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/kernel_traits.h @@ -0,0 +1,97 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct FMHA_kernel_traits { + + // The CTA description for the 1st GEMM. + using Cta_tile_p = fmha::Cta_tile_extd; + // The CTA description for the 2nd GEMM. + using Cta_tile_o = fmha::Cta_tile_extd; + + // Do we use one buffer for K and V. + enum { SHARE_SMEM_FOR_K_AND_V = (FLAGS & 0x08u) != 0u }; + // Do we keep K in registers. + enum { K_IN_REGS = (FLAGS & 0x10u) == 0u }; + + // The global memory tile to load Q. + using Gmem_tile_q = fmha::Gmem_tile_qkv; + + // The shared memory tile to swizzle Q. + using Smem_tile_q = fmha::Smem_tile_a; + + // The global memory tile to load K. + using Gmem_tile_k = fmha::Gmem_tile_qkv; + // The shared memory tile to swizzle K. + using Smem_tile_k = fmha::Smem_tile_b; + + // The global memory tile to load V. + using Gmem_tile_v = fmha::Gmem_tile_qkv; + // The shared memory tile to swizzle V. + using Smem_tile_v = fmha::Smem_tile_v; + + // The global memory tile to store O. + using Gmem_tile_o = fmha::Gmem_tile_o; + // The shared memory tile for O. + using Smem_tile_o = fmha::Smem_tile_o; + + // The global memory tile to load/store S. + using Gmem_tile_s = fmha::Gmem_tile_mma_s; + + // The shared memory tile to transpose S. + using Smem_tile_st = fmha::Smem_tile_mma_transposed; + + using Gmem_tile_do = fmha::Gmem_tile_dout; + + // Make sure the number of threads match. + static_assert((int)Gmem_tile_o::THREADS_PER_ROW == (int)Smem_tile_o::THREADS_PER_ROW, ""); + + // The number of threads. + enum { THREADS = Cta_tile_p::THREADS_PER_CTA }; + // Make sure the number of threads matches both CTAs. + static_assert((int)THREADS == (int)Cta_tile_o::THREADS_PER_CTA, ""); + + // The amount of shared memory needed to load Q and K. + enum { BYTES_PER_SMEM_QK = Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE }; + // The extra amount of shared memory needed to load V. + enum { BYTES_PER_SMEM_V = SHARE_SMEM_FOR_K_AND_V ? 0u : Smem_tile_v::BYTES_PER_TILE }; + // The amount of shared memory needed for Q, K and V.. + enum { BYTES_PER_SMEM_QKV = BYTES_PER_SMEM_QK + BYTES_PER_SMEM_V }; + // The amount of shared memory needed to load Q and store O. + enum { BYTES_PER_SMEM_QO = Smem_tile_q::BYTES_PER_TILE + Smem_tile_o::BYTES_PER_TILE }; + + // The amount of shared memory needed for Q, K, V and O. + enum { BYTES_PER_SMEM = fmha::Max::VALUE }; + // Make sure we have enough shared memory. + static_assert(Smem_tile_q::BYTES_PER_TILE + Smem_tile_o::BYTES_PER_TILE <= BYTES_PER_SMEM, ""); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/mask.h b/apex/apex/contrib/csrc/fmha/src/fmha/mask.h new file mode 100644 index 00000000..020258a0 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/mask.h @@ -0,0 +1,81 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +namespace fmha { + + +template +struct Mask { + using Mma_tile = fmha::Hmma_tile; + + template + __device__ Mask(const Params ¶ms, const BInfo &blockInfo, int tidx) { + + actual_seqlen = blockInfo.actual_seqlen; + + const int warp = tidx / Cta_tile::THREADS_PER_WARP; + const int lane = tidx % Cta_tile::THREADS_PER_WARP; + + static_assert(Cta_tile::WARPS_K == 1, ""); + + // find the warp in the Cta tile + const int warp_n = (warp / Cta_tile::WARPS_M); + const int warp_m = (warp % Cta_tile::WARPS_M); + // decompose warp into 8x4 tile + const int quad = lane / 4; + const int tid = (lane % 4) * 2; + row = warp_m * 16 + quad; + col = warp_n * 16 + tid; + } + + inline __device__ bool is_valid(const int mi, const int ni, const int ii, const int jj) const { + + // ii and jj iterate over the 2x4 fragment + const bool col_valid = (ni * Mma_tile::N_PER_MMA_PER_CTA + col + (jj & 2) * 4 + (jj & 1)) < actual_seqlen; + //&& (row + mi * Mma_tile::M_PER_MMA_PER_CTA + ii * 8) < actual_seqlen; + return col_valid; + // return row_valid && col_valid; + } + + //BERT Mask: if upper left is invalid, none are valid + inline __device__ bool any_valid(int mi, int ni) const { + return is_valid(mi, ni, 0, 0); + } + + inline __device__ void load(int it) { + row_offset = it * Cta_tile::M + row; + } + int row_offset; + + int row; + int col; + int actual_seqlen; +}; + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/smem_tile.h b/apex/apex/contrib/csrc/fmha/src/fmha/smem_tile.h new file mode 100644 index 00000000..80879140 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/smem_tile.h @@ -0,0 +1,1286 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include +#include + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The description of the tile computed by this CTA. + typename Cta_tile, + // The number of rows in the 2D shared memory buffer. + int M_, + // The number of cols. + int N_, + // The size in bits of each element. + int BITS_PER_ELEMENT_, + // The number of bytes per STS. + int BYTES_PER_STS_ = 16, + // The number of buffers. (Used in multistage and double buffer cases.) + int BUFFERS_PER_TILE_ = 1, + // Do we enable the fast path for LDS.128 and friends. + int ENABLE_LDS_FAST_PATH_ = 0, + // The number of rows that are used for the XOR swizzling to allow fast STS/LDS. + int ROWS_PER_XOR_PATTERN_ = 8, + // The number of cols that are used for the XOR swizzling to allow fast STS/LDS. + int COLS_PER_XOR_PATTERN_ = 1, + // Use or not predicates + bool USE_PREDICATES_ = true +> +struct Smem_tile_without_skews { + + // The size in bits of each element. + enum { BITS_PER_ELEMENT = BITS_PER_ELEMENT_ }; + // The size in bytes of a single STS. + enum { BYTES_PER_STS = BYTES_PER_STS_ }; + // The number of elements per STS. + enum { ELEMENTS_PER_STS = BYTES_PER_STS * 8 / BITS_PER_ELEMENT }; + // To support arbitrary N, we pad some values to a power-of-2. + enum { N_WITH_PADDING = Next_power_of_two::VALUE }; + // The number of bytes per row without packing of rows. + enum { BYTES_PER_ROW_BEFORE_PACKING = N_WITH_PADDING * BITS_PER_ELEMENT / 8 }; + // The number of bytes per row -- we want at least 128B per row. + enum { BYTES_PER_ROW = Max::VALUE }; + // The number of rows in shared memory (two rows may be packed into a single one). + enum { ROWS = M_ * BYTES_PER_ROW_BEFORE_PACKING / BYTES_PER_ROW }; + + // The number of threads per row. + enum { THREADS_PER_ROW_UNBOUNDED = BYTES_PER_ROW / BYTES_PER_STS }; + // The number of threads per row. + enum { THREADS_PER_ROW = Min::VALUE }; + + // The number of STS per row. + enum { STS_PER_ROW = BYTES_PER_ROW / THREADS_PER_ROW / BYTES_PER_STS }; + // It must be at least one. + static_assert(STS_PER_ROW >= 1, ""); + // The number of rows written with a single STS. + enum { ROWS_PER_STS = Cta_tile::THREADS_PER_CTA / THREADS_PER_ROW }; + // Make sure we write to at least one row per STS. Thanks Dr. Obvious ;) + static_assert(ROWS_PER_STS >= 1, ""); + // The number of STS needed to store all rows. + enum { STS_PER_COL = Div_up::VALUE }; + // The number of STS in total. + enum { STS = STS_PER_COL * STS_PER_ROW }; + + // The size of one buffer in bytes in shared memory. + enum { BYTES_PER_BUFFER = STS * BYTES_PER_STS * Cta_tile::THREADS_PER_CTA }; + // The number of buffers. + enum { BUFFERS_PER_TILE = BUFFERS_PER_TILE_ }; + // The size in bytes of total buffers. + enum { BYTES_PER_TILE = BYTES_PER_BUFFER * BUFFERS_PER_TILE }; + // The boundary for smem_read_offset and smem_write_offset increment. + enum { BYTES_PER_TILE_INC_BOUNDARY = BYTES_PER_TILE - BYTES_PER_BUFFER }; + + // Do we enable the LDS.128 fast path? + enum { ENABLE_LDS_FAST_PATH = ENABLE_LDS_FAST_PATH_ }; + static_assert(ENABLE_LDS_FAST_PATH == 0); + // The number of rows that are used for the XOR swizzling to allow fast STS/LDS. + enum { ROWS_PER_XOR_PATTERN = ROWS_PER_XOR_PATTERN_ }; + // The number of cols that are used for the XOR swizzling to allow fast STS/LDS. + enum { COLS_PER_XOR_PATTERN = COLS_PER_XOR_PATTERN_ * 16 / BYTES_PER_STS }; + // Use or not predicates + enum { USE_PREDICATES = USE_PREDICATES_ }; + + // The type of elements that are stored in shared memory by each thread. + using Store_type = typename Uint_from_size_in_bytes::Type; + + // Ctor. + inline __device__ Smem_tile_without_skews(void *smem, int tidx) + : smem_(__nvvm_get_smem_pointer(smem)) { + + // The row written by a thread. See doc/mma_smem_layout.xlsx. + int smem_write_row = tidx / THREADS_PER_ROW; + + // The XOR pattern. + int smem_write_xor = smem_write_row % ROWS_PER_XOR_PATTERN * COLS_PER_XOR_PATTERN; + // Compute the column and apply the XOR pattern. + int smem_write_col = (tidx % THREADS_PER_ROW) ^ smem_write_xor; + + // The offset. + this->smem_write_offset_ = smem_write_row*BYTES_PER_ROW + smem_write_col*BYTES_PER_STS; + + // TODO: Why not merge it with the read offset? + this->smem_read_buffer_ = __shfl_sync(0xffffffff, 0, 0); + this->smem_write_buffer_ = __shfl_sync(0xffffffff, 0, 0); + } + + // Compute the store pointers. + template< int N > + inline __device__ void compute_store_pointers(uint32_t (&ptrs)[N]) { + #pragma unroll + for( int ii = 0; ii < N; ++ii ) { + // Decompose the STS into row/col. + int row = ii / STS_PER_ROW; + int col = ii % STS_PER_ROW; + + // Assemble the offset. + int offset = smem_write_offset_ + row*ROWS_PER_STS*BYTES_PER_ROW; + + // Take the column into account. + if( STS_PER_ROW > 1 ) { + offset += col*THREADS_PER_ROW*BYTES_PER_STS; + } + + // Apply the XOR pattern if needed. + if( ROWS_PER_STS < ROWS_PER_XOR_PATTERN ) { + const int m = row * ROWS_PER_STS % ROWS_PER_XOR_PATTERN; + offset ^= m * COLS_PER_XOR_PATTERN * BYTES_PER_STS; + } + + // Assemble the final pointer :) + ptrs[ii] = smem_ + offset + smem_write_buffer_; + } + } + + inline __device__ void debug_reset() { + for( int buffer = 0; buffer < BYTES_PER_TILE; buffer += BYTES_PER_BUFFER) { + for( int row = 0; row < ROWS; ++row ) { + for( int col = 0; col < BYTES_PER_ROW; col += 4 ) { + if( threadIdx.x == 0 ) { + uint32_t val = 0x0; + sts(val, smem_ + row*BYTES_PER_ROW + col + buffer); + } + } + } + } + } + + // Print the content of the tile (only for debug ;)). + inline __device__ void debug_print() const { + for( int buffer = 0; buffer < BYTES_PER_TILE; buffer += BYTES_PER_BUFFER) { + for( int row = 0; row < ROWS; ++row ) { + for( int col = 0; col < BYTES_PER_ROW; col += 4 ) { + if( threadIdx.x == 0 ) { + uint32_t val; + lds(val, smem_ + row*BYTES_PER_ROW + col + buffer); + printf("block=(x=%2d, y=%2d, z=%2d) (smem_=%2d, buffer=%2d, row=%2d, byte=%4d)=0x%08x\n", + blockIdx.x, + blockIdx.y, + blockIdx.z, + smem_, + buffer, + row, + col, + val); + } + } + } + } + } + + // Move the read offset to next buffer. + inline __device__ void move_to_next_read_buffer() { + if( BUFFERS_PER_TILE > 1 && smem_read_buffer_ >= BYTES_PER_TILE_INC_BOUNDARY ) { + this->smem_read_buffer_ -= BYTES_PER_TILE_INC_BOUNDARY; + } else if( BUFFERS_PER_TILE > 1 ) { + this->smem_read_buffer_ += BYTES_PER_BUFFER; + } + } + + // Move the read offset to next buffer. TODO: Remove this member function!!! + inline __device__ void move_next_read_buffer() { + this->move_to_next_read_buffer(); + } + + // Move the read offset to next N buffer (circular-buffer). + inline __device__ void move_to_next_read_buffer(int N) { + if( BUFFERS_PER_TILE > 1 ) { + this->smem_read_buffer_ += N * BYTES_PER_BUFFER; + this->smem_read_buffer_ -= smem_read_buffer_ >= BYTES_PER_TILE ? BYTES_PER_TILE : 0; + } + } + + // Move the read offset to next N buffer (circular-buffer). TODO: Remove this member function!!! + inline __device__ void move_next_read_buffer(int N) { + this->move_to_next_read_buffer(N); + } + + // Move the write offset to next buffer. + inline __device__ void move_to_next_write_buffer() { + if( BUFFERS_PER_TILE > 1 && smem_write_buffer_ >= BYTES_PER_TILE_INC_BOUNDARY ) { + this->smem_write_buffer_ -= BYTES_PER_TILE_INC_BOUNDARY; + } else if( BUFFERS_PER_TILE > 1 ) { + this->smem_write_buffer_ += BYTES_PER_BUFFER; + } + } + + // Move the write offset to next buffer. TODO: Remove that member function! + inline __device__ void move_next_write_buffer() { + this->move_to_next_write_buffer(); + } + + // Move the read offset. + inline __device__ void move_read_offset(int delta) { + this->smem_read_offset_ += delta; + } + + // Move the write offset. + inline __device__ void move_write_offset(int delta) { + this->smem_write_offset_ += delta; + } + + // Store to the tile in shared memory. + template< int N > + inline __device__ void store(const Store_type (&data)[N], uint64_t = 0) { + uint32_t smem_ptrs[N]; + this->compute_store_pointers(smem_ptrs); + sts(smem_ptrs, data); + } + + // Store to the tile in shared memory. + template< int N, int M > + inline __device__ void store(const Store_type (&data)[N], uint32_t (&preds)[M], uint64_t = 0) { + uint32_t smem_ptrs[N]; + this->compute_store_pointers(smem_ptrs); + sts(smem_ptrs, data, preds); + } + + // Store to the tile in shared memory. + template< int N > + inline __device__ void store(const Store_type (&data)[N], uint32_t preds, uint64_t = 0) { + this->store(data, preds); + } + + // Store to the tile in shared memory. + template< int N > + inline __device__ void store(const void* (&gmem_ptrs)[N], uint32_t preds, uint64_t = 0) { + uint32_t tmp[1] = { preds }; + this->store(gmem_ptrs, tmp); + } + + // The shared memory pointer. + uint32_t smem_; + // The read offset. Reserve 4 offsets if needed. + int smem_read_offset_; + // The write offset. + int smem_write_offset_; + // The buffer base offset for read. + int smem_read_buffer_; + // The buffer base offset for write. + int smem_write_buffer_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The layout of the tile. + typename Layout, + // The size of the STS. + int BYTES_PER_STS = 16, + // The number of buffers per tile. + int BUFFERS_PER_TILE = 1, + // Use or not predicates + bool USE_PREDICATES = true +> +struct Smem_tile_a { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int MMAS_K, int MMAS_K_WITH_PADDING > +struct Compute_reset_mask { + // The potential mask. + enum { HALF = MMAS_K_WITH_PADDING / 2 }; + // The remainder. + enum { MOD = MMAS_K % HALF }; + // The final value. + enum { VALUE = (MMAS_K == MOD ? 0 : HALF) | Compute_reset_mask::VALUE }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int MMAS_K_WITH_PADDING > +struct Compute_reset_mask<0, MMAS_K_WITH_PADDING> { + enum { VALUE = 0 }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int MMAS_K > +struct Compute_reset_mask { + enum { VALUE = MMAS_K - 1 }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +struct Rows_per_xor_pattern_a { + // The size in bits. + enum { N_IN_BITS = N * fmha::BITS_PER_ELEMENT_A }; + // The number of rows. + enum { VALUE = N_IN_BITS <= 256 ? 2 : (N_IN_BITS <= 512 ? 4 : 8) }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +struct Rows_per_xor_pattern_row_a : public Rows_per_xor_pattern_a { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The size of the STS. + int BYTES_PER_STS, + // The number of buffers per tile. + int BUFFERS_PER_TILE, + // How many rows to use for the XOR pattern to avoid bank conflicts? + int ROWS_PER_XOR_PATTERN_ = Rows_per_xor_pattern_row_a::VALUE +> +struct Smem_tile_row_a : public Smem_tile_without_skews { + // The MMA tile. + using Mma_tile = fmha::Hmma_tile; + // The base class. + using Base = Smem_tile_without_skews; + // The fragment. + using Fragment = Fragment_a; + + // When we use padding to reach a power of two, special care has to be taken. + using Cta_tile_with_padding = Cta_tile_with_k_with_padding; + // The number of MMAs. + using Mma_tile_with_padding = fmha::Hmma_tile; + + // The size of a single LDS in bytes. + enum { BYTES_PER_LDS = 16 }; + + // Ctor. + inline __device__ Smem_tile_row_a(void *smem, int tidx) : Base(smem, tidx) { + + // For documentation on the layout, see doc/mma_smem_layout.xlsx. + + // The number of warps. + const int WARPS_M = Cta_tile::WARPS_M; + const int WARPS_N = Cta_tile::WARPS_N; + const int WARPS_K = Cta_tile::WARPS_K; + + static_assert(WARPS_M == 1); + static_assert(WARPS_N == 4 || WARPS_N == 8); + static_assert(WARPS_K == 1); + static_assert(Base::ROWS_PER_XOR_PATTERN == 8); + + // The row and column read by the thread. + int smem_read_row = (tidx & 0x0f); + int smem_read_col = (tidx & 0x07); + smem_read_col ^= (tidx & 0x10) / 16; + + // The shared memory offset. + this->smem_read_offset_ = smem_read_row*Base::BYTES_PER_ROW + smem_read_col*BYTES_PER_LDS; + } + + // Rewind smem_read_offset for last LDS phase in main loop. + inline __device__ void reverse_smem_read_offset(int ki = 0) { + // Undo the pointer increment for the next ni. + // Should match the load function below for ki = 0. + if( Mma_tile_with_padding::MMAS_K >= 2 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * 2; + } + } + + // Load from shared memory. + inline __device__ void load(Fragment (&a)[Mma_tile::MMAS_M], int ki) { + #pragma unroll + for( int mi = 0; mi < Mma_tile::MMAS_M; ++mi ) { + // Jump by as many matrix rows as needed (a row in smem may pack multiple matrix rows). + int offset = mi * Mma_tile::M_PER_MMA_PER_CTA * Base::BYTES_PER_ROW_BEFORE_PACKING; + + // Load using LDSM.M88.4. + uint4 tmp; + ldsm(tmp, this->smem_ + this->smem_read_offset_ + this->smem_read_buffer_ + offset); + + // Store the value into the fragment. + a[mi].reg(0) = tmp.x; + a[mi].reg(1) = tmp.y; + a[mi].reg(2) = tmp.z; + a[mi].reg(3) = tmp.w; + } + + // Move the offset to the next possition. See doc/mma_smem_layout.xlsx. + static_assert(Mma_tile_with_padding::MMAS_K < 64, "Not implemented"); + if( Mma_tile_with_padding::MMAS_K >= 32 && ki % 16 == 15 ) { + this->smem_read_offset_ ^= 31 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 16 && ki % 8 == 7 ) { + this->smem_read_offset_ ^= 15 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 8 && ki % 4 == 3 ) { + this->smem_read_offset_ ^= 7 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 4 && ki % 2 == 1 ) { + this->smem_read_offset_ ^= 3 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 2 ) { + this->smem_read_offset_ ^= 1 * BYTES_PER_LDS * 2; + } + } + + // Reset the read offset. + inline __device__ void reset_read_offset() { + // The number of MMAs in the K dimension. + enum { MMAS_K = Mma_tile::MMAS_K }; + // The number of MMAs in the K dimension when we include padding. + enum { MMAS_K_WITH_PADDING = Mma_tile_with_padding::MMAS_K }; + // Assemble the mask. + enum { MASK = Compute_reset_mask::VALUE }; + + // Reset the read offset. + this->smem_read_offset_ ^= MASK * BYTES_PER_LDS * 2; + } + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The size of the STS. + int BYTES_PER_STS, + // The number of buffers per tile. + int BUFFERS_PER_TILE +> +struct Smem_tile_a + : public Smem_tile_row_a { + // The base class. + using Base = Smem_tile_row_a; + + // Ctor. + inline __device__ Smem_tile_a(void *smem, int tidx) : Base(smem, tidx) { + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The layout of the tile. + typename Layout, + // The size of the STS. + int BYTES_PER_STS = 16, + // The number of buffers per tile. + int BUFFERS_PER_TILE = 1, + // Use or not predicates + bool USE_PREDICATES = true +> +struct Smem_tile_b { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +struct Rows_per_xor_pattern_b { + // The size in bits. + enum { N_IN_BITS = N * fmha::BITS_PER_ELEMENT_B }; + // The number of rows. + enum { VALUE = N_IN_BITS <= 256 ? 2 : (N_IN_BITS <= 512 ? 4 : 8) }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +struct Rows_per_xor_pattern_col_b : public Rows_per_xor_pattern_b { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The size of the STS. + int BYTES_PER_STS, + // The number of buffers per tile. + int BUFFERS_PER_TILE, + // How many rows to use for the XOR pattern to avoid bank conflicts? + int ROWS_PER_XOR_PATTERN_ = Rows_per_xor_pattern_col_b::VALUE +> +struct Smem_tile_col_b : public Smem_tile_without_skews { + // The MMA tile. + using Mma_tile = fmha::Hmma_tile; + // The base class. + using Base = Smem_tile_without_skews; + // The fragment. + using Fragment = Fragment_b< Col>; + + // When we use padding to reach a power of two, special care has to be taken. + using Cta_tile_with_padding = Cta_tile_with_k_with_padding< Cta_tile>; + // The number of MMAs. + using Mma_tile_with_padding = fmha::Hmma_tile; + + // The size of a single LDS in bytes. + enum { BYTES_PER_LDS = 16 }; + + // The number of STS per thread + enum { STS_PER_THREAD_ = Base::ROWS * Base::THREADS_PER_ROW / Cta_tile::THREADS_PER_CTA }; + // The number of STS per thread must be at least 1. + enum { STS_PER_THREAD = Max<1, STS_PER_THREAD_>::VALUE }; + + // Ctor. + inline __device__ Smem_tile_col_b(void *smem, int tidx) : Base(smem, tidx) { + + // For documentation on the layout, see doc/mma_smem_layout.xlsx. + + // The number of warps. + const int WARPS_M = Cta_tile::WARPS_M; + const int WARPS_N = Cta_tile::WARPS_N; + const int WARPS_K = Cta_tile::WARPS_K; + static_assert(Base::ROWS_PER_XOR_PATTERN == 8); + static_assert(WARPS_M == 1); + static_assert(WARPS_N == 4 || WARPS_N == 8); + static_assert(WARPS_K == 1); + + // The masks to select the warps. + const int WARP_MASK_N = Warp_masks::N; + + // The divisor for the warps. + const int WARP_DIV_N = WARPS_M * 1 * Cta_tile::THREADS_PER_WARP; + + // The row and column read by the thread. + int smem_read_row = (tidx & WARP_MASK_N) / WARP_DIV_N * Mma_tile::N_PER_MMA + + (tidx & 0x07) + + (tidx & 0x10) / 2; + int smem_read_col = (tidx & 0x07); + smem_read_col ^= (tidx & 0x08) / 8; + // The shared memory offset. + this->smem_read_offset_ = smem_read_row*Base::BYTES_PER_ROW + smem_read_col*BYTES_PER_LDS; + } + + // Rewind smem_read_offset for last LDS phase in main loop. + inline __device__ void reverse_smem_read_offset(int ki = 0) { + // Undo the pointer increment for the next ni. + // Should match the load function below for ki = 0. + if( Mma_tile_with_padding::MMAS_K >= 2 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * 2; + } + } + + // Load from shared memory. + inline __device__ void load(Fragment (&b)[Mma_tile::MMAS_N], int ki) { + #pragma unroll + for( int ni = 0; ni < Mma_tile::MMAS_N; ++ni ) { + // Jump by as many matrix rows as needed (a row in smem may pack multiple matrix rows). + int offset = ni * Mma_tile::N_PER_MMA_PER_CTA * Base::BYTES_PER_ROW_BEFORE_PACKING; + + // Load using LDSM.M88.4. + uint4 tmp; + ldsm(tmp, this->smem_ + this->smem_read_offset_ + this->smem_read_buffer_ + offset); + + // Store the value into the fragment. + b[ni].reg(0) = tmp.x; + b[ni].reg(1) = tmp.y; + b[ni].reg(2) = tmp.z; + b[ni].reg(3) = tmp.w; + } + + // Move the offset to the next possition. See doc/mma_smem_layout.xlsx. + static_assert(Mma_tile_with_padding::MMAS_K < 64, "Not implemented"); + if( Mma_tile_with_padding::MMAS_K >= 32 && ki % 16 == 15 ) { + this->smem_read_offset_ ^= 31 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 16 && ki % 8 == 7 ) { + this->smem_read_offset_ ^= 15 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 8 && ki % 4 == 3 ) { + this->smem_read_offset_ ^= 7 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 4 && ki % 2 == 1 ) { + this->smem_read_offset_ ^= 3 * BYTES_PER_LDS * 2; + } else if( Mma_tile_with_padding::MMAS_K >= 2 ) { + this->smem_read_offset_ ^= 1 * BYTES_PER_LDS * 2; + } + } + + // Reset the read offset. + inline __device__ void reset_read_offset() { + // The number of MMAs in the K dimension. + enum { MMAS_K = Mma_tile::MMAS_K }; + // The number of MMAs in the K dimension when we include padding. + enum { MMAS_K_WITH_PADDING = Mma_tile_with_padding::MMAS_K }; + // Assemble the mask. + enum { MASK = Compute_reset_mask::VALUE }; + + // Reset the read offset. + this->smem_read_offset_ ^= MASK * BYTES_PER_LDS * 2; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The size of the STS. + int BYTES_PER_STS, + // The number of buffers per tile. + int BUFFERS_PER_TILE +> +struct Smem_tile_b< Cta_tile, Col, BYTES_PER_STS, BUFFERS_PER_TILE > + : public Smem_tile_col_b { + + // The base class. + using Base = Smem_tile_col_b< Cta_tile, BYTES_PER_STS, BUFFERS_PER_TILE>; + + // Ctor. + inline __device__ Smem_tile_b(void *smem, int tidx) : Base(smem, tidx) { + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +struct Rows_per_xor_pattern_row_b : public Rows_per_xor_pattern_b< N> { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The size of the STS. + int BYTES_PER_STS, + // The number of buffers per tile. + int BUFFERS_PER_TILE, + // How many rows to use for the XOR pattern to avoid bank conflicts? + int ROWS_PER_XOR_PATTERN_ = Rows_per_xor_pattern_row_b::VALUE, + // How many cols to use for the XOR pattern to avoid bank conflicts? + int COLS_PER_XOR_PATTERN_ = 1 +> +struct Smem_tile_row_b : public Smem_tile_without_skews { + + // The MMA tile. + using Mma_tile = fmha::Hmma_tile; + // The base class. + using Base = Smem_tile_without_skews; + // The fragment. + using Fragment = Fragment_b; + + // Can we use LDSM? No if the data type is 32-bit large. + enum { USE_LDSMT = fmha::BITS_PER_ELEMENT_B == 16 }; + // The size of a single LDS in bytes. + enum { BYTES_PER_LDS = USE_LDSMT ? 16 : 4 }; + // The number of elements per LDS. + enum { ELEMENTS_PER_LDS = BYTES_PER_LDS * 8 / fmha::BITS_PER_ELEMENT_B }; + + // The number of STS per thread + enum { STS_PER_THREAD_ = Base::ROWS * Base::THREADS_PER_ROW / Cta_tile::THREADS_PER_CTA }; + // The number of STS per thread must be at least 1. + enum { STS_PER_THREAD = Max<1, STS_PER_THREAD_>::VALUE }; + + // Ctor. + inline __device__ Smem_tile_row_b(void *smem, int tidx) : Base(smem, tidx) { + + // The number of warps. + const int WARPS_M = Cta_tile::WARPS_M; + const int WARPS_N = Cta_tile::WARPS_N; + const int WARPS_K = Cta_tile::WARPS_K; + static_assert(WARPS_K == 1); + static_assert(WARPS_M == 4 || WARPS_M == 8); + static_assert(WARPS_N == 1); + + // The masks to select the warps. + const int WARP_MASK_N = Warp_masks::N; + const int WARP_MASK_K = Warp_masks::K; + + // The divisor for the warps. + const int WARP_DIV_N = WARPS_M * 1 * Cta_tile::THREADS_PER_WARP; + const int WARP_DIV_K = WARPS_M * WARPS_N * Cta_tile::THREADS_PER_WARP; + + // The row/col read by the thread. + int smem_read_row, smem_read_col; + + static_assert(USE_LDSMT); + static_assert(Base::ROWS_PER_XOR_PATTERN == 8); + + smem_read_row = (tidx & WARP_MASK_K) / WARP_DIV_K * Mma_tile::MMAS_K * 16 + + (tidx & 0x07) + (tidx & 0x08); + smem_read_col = (tidx & 0x07); + smem_read_col ^= (tidx & WARP_MASK_N) / WARP_DIV_N * 2 + (tidx & 0x10) / 16; + + // The shared memory offset. + this->smem_read_offset_ = smem_read_row*Base::BYTES_PER_ROW + smem_read_col*BYTES_PER_LDS; + + // Fill zeroes for group conv + } + + // Rewind smem_read_offset for last LDS phase in main loop. + inline __device__ void reverse_smem_read_offset(int ki = 0) { + // The size of each element in bits. + const int BITS_PER_ELT = fmha::BITS_PER_ELEMENT_B; + // The size in bytes of the data needed to compute an MMA per CTA. + const int BYTES_PER_MMA_PER_CTA = Mma_tile::N_PER_MMA_PER_CTA * BITS_PER_ELT / 8; + + #pragma unroll + for( int ni = 0; ni < Mma_tile::MMAS_N; ++ni ) { + // Undo the pointer increment for the next ni. + // Should match the load function below for ki = 0. + if( BYTES_PER_MMA_PER_CTA >= 128 ) { + // Nothing to do! + } else if( BYTES_PER_MMA_PER_CTA == 64 && Mma_tile::MMAS_N > 1 ) { + this->smem_read_offset_ ^= BYTES_PER_MMA_PER_CTA; + } else if( BYTES_PER_MMA_PER_CTA == 64 ) { + // Nothing to do! + } else if( BYTES_PER_MMA_PER_CTA == 32 && Mma_tile::MMAS_N == 4 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * (ni % 2 == 0 ? 2 : 6); + } else if( BYTES_PER_MMA_PER_CTA == 32 && Mma_tile::MMAS_N == 2 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * 2; + } + } + + // Reset smem_read_offset for odd MMAS_N > 1 (npo2 kernels) + if( BYTES_PER_MMA_PER_CTA == 64 && Mma_tile::MMAS_N > 1 && + Mma_tile::MMAS_N % 2 == 1 ) { + this->smem_read_offset_ ^= BYTES_PER_MMA_PER_CTA; + } + } + + // Load from shared memory. + inline __device__ void load(Fragment (&b)[Mma_tile::MMAS_N], int ki) { + // The size of each element in bits. + const int BITS_PER_ELT = fmha::BITS_PER_ELEMENT_B; + // The size in bytes of the data needed to compute an MMA per CTA. + const int BYTES_PER_MMA_PER_CTA = Mma_tile::N_PER_MMA_PER_CTA * BITS_PER_ELT / 8; + + #pragma unroll + for( int ni = 0; ni < Mma_tile::MMAS_N; ++ni ) { + // Prepare the offset. + int offset = ki * Base::ROWS_PER_XOR_PATTERN * 2 * Base::BYTES_PER_ROW; + if ( BYTES_PER_MMA_PER_CTA == 32 ) { + offset += this->smem_read_offset_; + } else if ( BYTES_PER_MMA_PER_CTA == 64 ) { + offset += this->smem_read_offset_ + (ni/2) * BYTES_PER_MMA_PER_CTA * 2; + } else { + offset += this->smem_read_offset_ + (ni ) * BYTES_PER_MMA_PER_CTA; + } + + // Load the data using LDSM.MT88.2. + uint32_t ptr = this->smem_ + this->smem_read_buffer_ + offset; + uint4 tmp; + if( USE_LDSMT ) { + ldsmt(tmp, ptr); + } else { + lds(tmp.x, (ptr ) + 0*Base::BYTES_PER_ROW); + lds(tmp.y, (ptr ) + 4*Base::BYTES_PER_ROW); + lds(tmp.z, (ptr ^ 32) + 0*Base::BYTES_PER_ROW); + lds(tmp.w, (ptr ^ 32) + 4*Base::BYTES_PER_ROW); + } + + // Store those values in the fragment. + b[ni].reg(0) = tmp.x; + b[ni].reg(1) = tmp.y; + b[ni].reg(2) = tmp.z; + b[ni].reg(3) = tmp.w; + + // Move the pointer for the next ni. I expect the compiler to not recompute those. + if( BYTES_PER_MMA_PER_CTA >= 128 ) { + // Nothing to do! + } else if( BYTES_PER_MMA_PER_CTA == 64 && Mma_tile::MMAS_N > 1 ) { + this->smem_read_offset_ ^= BYTES_PER_MMA_PER_CTA; + } else if( BYTES_PER_MMA_PER_CTA == 64 ) { + // Nothing to do! + } else if( BYTES_PER_MMA_PER_CTA == 32 && Mma_tile::MMAS_N == 4 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * (ni % 2 == 0 ? 2 : 6); + } else if( BYTES_PER_MMA_PER_CTA == 32 && Mma_tile::MMAS_N == 2 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * 2; + } + } + + // Reset smem_read_offset for odd MMAS_N > 1 (npo2 kernels) + if( BYTES_PER_MMA_PER_CTA == 64 && Mma_tile::MMAS_N > 1 && + Mma_tile::MMAS_N % 2 == 1 ) { + this->smem_read_offset_ ^= BYTES_PER_MMA_PER_CTA; + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + // The dimensions of the tile computed by the CTA. + typename Cta_tile, + // The size of the STS. + int BYTES_PER_STS, + // The number of buffers per tile. + int BUFFERS_PER_TILE +> +struct Smem_tile_b + : public Smem_tile_row_b { + + // The base class. + using Base = Smem_tile_row_b; + + // Ctor. + inline __device__ Smem_tile_b(void *smem, int tidx) : Base(smem, tidx) { + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Smem_tile_v : public fmha::Smem_tile_without_skews { + + // The base class. + using Base = Smem_tile_without_skews; + // The MMA tile. + using Mma_tile = fmha::Hmma_tile; + // The fragment. + using Fragment = Fragment_b< fmha::Col>; + + // The size of a single LDS in bytes. + enum { BYTES_PER_LDS = 16 }; + + // Ctor. + inline __device__ Smem_tile_v(void *smem, int tidx) : Base(smem, tidx) { + + // The row/col read by the thread. + int read_row, read_col; + + static_assert(Cta_tile::WARPS_M == 1 && Cta_tile::WARPS_N == 1 && (Cta_tile::WARPS_K == 4 || Cta_tile::WARPS_K == 8)); + + read_row = (tidx & 0xe0) / 2 + (tidx & 0x0f); + read_col = (tidx & 0x07); + read_col ^= (tidx & 0x10) / 16; + + // The shared memory offset. + this->smem_read_offset_ = read_row * Base::BYTES_PER_ROW + read_col * BYTES_PER_LDS; + } + + // Load from shared memory. + inline __device__ void load(Fragment (&b)[Mma_tile::MMAS_N], int ki) { +#pragma unroll + for( int ni = 0; ni < Mma_tile::MMAS_N; ++ni ) { + // Jump by 16 * #warps row. + int row = ki * 16 * Cta_tile::WARPS_K; + + // Load the data using LDSM.MT88.2. + uint4 tmp; + fmha::ldsmt(tmp, this->smem_ + this->smem_read_offset_ + row * Base::BYTES_PER_ROW); + b[ni].reg(0) = tmp.x; + b[ni].reg(1) = tmp.y; + b[ni].reg(2) = tmp.z; + b[ni].reg(3) = tmp.w; + + // Move the pointer for the next ni. I expect the compiler to not recompute those. + if( Mma_tile::MMAS_N == 4 ) { + this->smem_read_offset_ ^= BYTES_PER_LDS * (ni % 2 == 0 ? 2 : 6); + } else { + assert(false); // Not implemented! + } + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Smem_tile_o { + + // The MMA tile. + using Mma_tile = fmha::Hmma_tile; + // The accumulators. + using Accumulator = fmha::Fragment_accumulator; + // The accumulators. + using Data_type = typename Accumulator::Data_type; + + // The size of each element. + enum { BYTES_PER_ELEMENT = sizeof(Data_type) }; + // The size of each STS. + enum { BYTES_PER_STS = 8 }; + // The size of each row in shared memory. + enum { BYTES_PER_ROW = Cta_tile::N * Cta_tile::WARPS_K * BYTES_PER_ELEMENT }; + + // The size of each LDS. + enum { BYTES_PER_LDS = 16 }; + enum { THREADS_PER_ROW = 16 }; + + // The number of rows. + enum { ROWS = Cta_tile::M }; + // The number of "rows" to process per loop iteration (in the "epilogue"). + enum { ROWS_PER_LOOP = ROWS <= 64 ? ROWS : (int)Mma_tile::M_PER_MMA_PER_CTA }; + // The number of outer loops. + enum { LOOPS = ROWS / ROWS_PER_LOOP }; + // Make sure it matches our expectations. + static_assert(LOOPS == 1 || LOOPS == (int)Mma_tile::MMAS_M, ""); + + // The number of rows loaded per LDS. + enum { ROWS_PER_LDS = Cta_tile::THREADS_PER_CTA / THREADS_PER_ROW }; + // Do we have to guard against partial writes/reads. + enum { HAS_INCOMPLETE_LDS = ROWS_PER_LOOP % ROWS_PER_LDS != 0 }; + // The total number of LDS per loop. + enum { LDS_PER_LOOP = fmha::Div_up::VALUE }; + + // The amount of shared memory. + enum { BYTES_PER_TILE = ROWS_PER_LOOP * BYTES_PER_ROW }; + + // The write pointer. + uint32_t smem_write_, smem_read_; + // Is the thread active for the last LDS of the series? + int is_active_for_last_lds_; + + static_assert(BYTES_PER_ROW == 64 * 4 * Cta_tile::WARPS_K); + static_assert(LOOPS == 1 || LOOPS == (int)Mma_tile::MMAS_M, ""); + + // Ctor. + inline __device__ Smem_tile_o(void *smem, int tidx) { + + // Get a 32-bit value for the shared memory address. + uint32_t smem_ = __nvvm_get_smem_pointer(smem); + + static_assert(Cta_tile::WARPS_M == 1 && Cta_tile::WARPS_N == 1 && (Cta_tile::WARPS_K == 4 || Cta_tile::WARPS_K == 8)); + + int write_row = (tidx & 0x1c) / 4; + int write_col = (tidx); + + // Assemble the write pointer. + smem_write_ = smem_ + write_row * BYTES_PER_ROW + write_col * BYTES_PER_STS; + + // The element read by each thread. + int read_row = tidx / THREADS_PER_ROW; + int read_col = tidx % THREADS_PER_ROW; + + // Take the XOR pattern into account for the column. + read_col ^= 2 * (read_row & 0x7); + + // Assemble the read pointer. + this->smem_read_ = smem_ + read_row * BYTES_PER_ROW + read_col * BYTES_PER_LDS; + + // Is that thread active on the last LDS? + if( HAS_INCOMPLETE_LDS ) { + this->is_active_for_last_lds_ = read_row + (LDS_PER_LOOP - 1) * ROWS_PER_LDS < Cta_tile::M; + } + } + + // Load the output fragments. + inline __device__ void load(uint4 (&out)[LDS_PER_LOOP]) const { + #pragma unroll + for( int ii = 0; ii < LDS_PER_LOOP; ++ii ) { + + // Load the elements before the reduction (split-K). + uint4 tmp[Cta_tile::WARPS_K]; + #pragma unroll + for( int jj = 0; jj < Cta_tile::WARPS_K; ++jj ) { + int imm = ii * ROWS_PER_LDS * BYTES_PER_ROW + jj * Cta_tile::N * BYTES_PER_ELEMENT; + if( !HAS_INCOMPLETE_LDS || (ii < LDS_PER_LOOP - 1 || this->is_active_for_last_lds_) ) { + fmha::lds(tmp[jj], this->smem_read_ + imm); + } + } + + // Perform the reduction. + out[ii] = tmp[0]; + #pragma unroll + for( int jj = 1; jj < Cta_tile::WARPS_K; ++jj ) { + out[ii] = fmha::fadd4(out[ii], tmp[jj]); + } + } + } + // Store the accumulators. + template + inline __device__ void store(const Accumulator (&acc)[M][N], int mi) { + enum { M_PER_MMA = Mma_tile::M_PER_MMA_PER_CTA }; + #pragma unroll + for( int ni = 0; ni < Mma_tile::MMAS_N; ++ni ) { + + // The number of MMAs that are stored per loop iteration. + enum { MMAS_M_PER_LOOP = Mma_tile::MMAS_M / LOOPS }; + + // Store 1st column of the different MMAs. + #pragma unroll + for( int mj = 0; mj < MMAS_M_PER_LOOP; ++mj ) { + // Precompute the immediates to jump between rows. + int row_0 = (mj * M_PER_MMA + 0) * BYTES_PER_ROW; + int row_1 = (mj * M_PER_MMA + 8) * BYTES_PER_ROW; + uint2 tmp0, tmp1; + tmp0.x = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(0); + tmp0.y = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(1); + + tmp1.x = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(2); + tmp1.y = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(3); + + // Store. + fmha::sts(this->smem_write_ + row_0, tmp0); + fmha::sts(this->smem_write_ + row_1, tmp1); + } + + // Swizzle the write pointer using a XOR of 16B. + this->smem_write_ ^= 32; + + // Store 2nd column of the different MMAs. + #pragma unroll + for( int mj = 0; mj < MMAS_M_PER_LOOP; ++mj ) { + // Precompute the immediates to jump between rows. + int row_0 = (mj * M_PER_MMA + 0) * BYTES_PER_ROW; + int row_1 = (mj * M_PER_MMA + 8) * BYTES_PER_ROW; + + uint2 tmp0, tmp1; + tmp0.x = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(4); + tmp0.y = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(5); + + tmp1.x = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(6); + tmp1.y = acc[mi * MMAS_M_PER_LOOP + mj][ni].reg(7); + // Store. + fmha::sts(this->smem_write_ + row_0, tmp0); + fmha::sts(this->smem_write_ + row_1, tmp1); + } + + // Cancel the previous XOR of 1 + swizzle the write pointer using a XOR of 32B or 64B. + this->smem_write_ ^= (ni & 1) ? 7 * 32 : 3 * 32; + } + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Smem_tile_mma { + + using Mma_tile = fmha::Hmma_tile; + using Fragment = fmha::Fragment_a; + + enum { COLS = Cta_tile::N }; + enum { BYTES_PER_ELT = 2 }; + enum { BYTES_PER_STS = 4 }; + enum { BYTES_PER_ROW = COLS * BYTES_PER_ELT }; // TODO + enum { BYTES_PER_TILE = Cta_tile::M * BYTES_PER_ROW }; + + enum { WARPS_M = Cta_tile::WARPS_M }; + enum { WARPS_N = Cta_tile::WARPS_N }; + enum { WARPS_K = Cta_tile::WARPS_K }; + + static_assert(WARPS_K == 1); + inline __device__ Smem_tile_mma(char *smem, int tidx) { + smem_ = __nvvm_get_smem_pointer(smem); + + int write_col, write_row; + static_assert(WARPS_M == 1 && (WARPS_N == 4 || WARPS_N == 8) || (WARPS_M == 4 || WARPS_N == 8) || WARPS_N == 1); + if( WARPS_M == 1 && (WARPS_N == 4 || WARPS_N == 8) ) { + write_row = (tidx & 0x1c) / 4; + write_col = (tidx & 0xe0) / 4 + (tidx & 0x03); + } else { + write_row = (tidx & 0xe0) / 2 + (tidx & 0x1c) / 4; + write_col = (tidx & 0x03); + } + write_col ^= (write_row & 0x07) * 4; + + write_offset_ = write_row * BYTES_PER_ROW + write_col * BYTES_PER_STS; + } + + template + inline __device__ void store(const uint4 (®s)[M][N]) { + static_assert(COLS == Cta_tile::N); + for( int mi = 0; mi < M; mi++ ) { + for( int ni = 0; ni < N; ni++ ) { + size_t offset = write_offset_ + mi * WARPS_M * 16 * BYTES_PER_ROW + ni * WARPS_N * 16 * BYTES_PER_ELT; + fmha::sts(smem_ + offset + 0 * BYTES_PER_ROW, regs[mi][ni].x); + fmha::sts(smem_ + offset + 8 * BYTES_PER_ROW, regs[mi][ni].z); + offset ^= 4 * BYTES_PER_STS; + fmha::sts(smem_ + offset + 0 * BYTES_PER_ROW, regs[mi][ni].y); + fmha::sts(smem_ + offset + 8 * BYTES_PER_ROW, regs[mi][ni].w); + } + } + } + + uint32_t smem_; + uint32_t write_offset_; + uint32_t warp_m; + uint32_t warp_n; + uint32_t lane; +}; + +template< typename Cta_tile, typename Base = Smem_tile_mma< Cta_tile>> +struct Smem_tile_mma_transposed : public Base { + enum { BYTES_PER_LDS = 16 }; + enum { BYTES_PER_ROW = Base::BYTES_PER_ROW }; + enum { BYTES_PER_ELT = Base::BYTES_PER_ELT }; + enum { WARPS_M = Base::WARPS_M }; + enum { WARPS_N = Base::WARPS_N }; + static_assert(WARPS_M == 1 && (WARPS_N == 4 || WARPS_N == 8)); + using Fragment = typename Base::Fragment; + inline __device__ Smem_tile_mma_transposed(char *smem, int tidx) : Base(smem, tidx) { + + static_assert(WARPS_M == 1 && (WARPS_N == 4 || WARPS_N == 8)); + int read_row, read_col; + read_row = (tidx & 0x0f); + read_col = (tidx & 0xe0) / 16 + (tidx & 0x1c) / 16; + + read_col ^= (read_row & 0x07); + read_offset_ = read_row * BYTES_PER_ROW + read_col * BYTES_PER_LDS; + } + + template + inline __device__ void load(Fragment (&frag)[M][N]) { + static_assert(Base::COLS == Cta_tile::N); + for( int mi = 0; mi < M; mi++ ) { + for( int ni = 0; ni < N; ni++ ) { + size_t offset = read_offset_ + mi * WARPS_M * 16 * BYTES_PER_ROW + ni * WARPS_N * 16 * BYTES_PER_ELT; + uint4 dst; + fmha::ldsmt(dst, this->smem_ + offset); + frag[mi][ni].reg(0) = dst.x; + frag[mi][ni].reg(1) = dst.z; // Fragment A regs col major! + frag[mi][ni].reg(2) = dst.y; + frag[mi][ni].reg(3) = dst.w; + } + } + } + + uint32_t read_offset_; +}; + +template< typename Cta_tile, typename Base = Smem_tile_mma< Cta_tile>> +struct Smem_tile_mma_epilogue : public Base { + enum { BYTES_PER_LDS = 16 }; + enum { BYTES_PER_ROW = Base::BYTES_PER_ROW }; + enum { BYTES_PER_ELT = Base::BYTES_PER_ELT }; + enum { THREADS_PER_ROW = BYTES_PER_ROW / BYTES_PER_LDS }; + static_assert(THREADS_PER_ROW * BYTES_PER_LDS == BYTES_PER_ROW); + enum { ROWS_PER_LDS = Cta_tile::THREADS_PER_CTA / THREADS_PER_ROW }; + enum { NUM_LDS = Cta_tile::M / ROWS_PER_LDS }; + static_assert(NUM_LDS * ROWS_PER_LDS == Cta_tile::M); + enum { WARPS_M = Base::WARPS_M }; + enum { WARPS_N = Base::WARPS_N }; + static_assert((WARPS_M == 4 || WARPS_N == 8) || WARPS_N == 1); + + using Acc = fmha::Fragment_accumulator; + + inline __device__ Smem_tile_mma_epilogue(char *smem, int tidx) : Base(smem, tidx) { + const int read_row = tidx / THREADS_PER_ROW; + int read_col = tidx % THREADS_PER_ROW; + read_col ^= (read_row & 0x07); + read_offset_ = read_row * BYTES_PER_ROW + read_col * BYTES_PER_LDS; + } + + inline __device__ void load(uint4 (&data)[NUM_LDS]) { + for( int ii = 0; ii < NUM_LDS; ii++ ) { + size_t offset = read_offset_ + ii * ROWS_PER_LDS * BYTES_PER_ROW; + fmha::lds(data[ii], this->smem_ + offset); + } + } + + template + inline __device__ void store(const Acc (&acc)[M][N]){ + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + // 1st row - 4 elements per row. + float tmp00 = acc[mi][ni].elt(0); + float tmp01 = acc[mi][ni].elt(1); + float tmp02 = acc[mi][ni].elt(4); + float tmp03 = acc[mi][ni].elt(5); + // 2nd row - 4 elements per row. + float tmp10 = acc[mi][ni].elt(2); + float tmp11 = acc[mi][ni].elt(3); + float tmp12 = acc[mi][ni].elt(6); + float tmp13 = acc[mi][ni].elt(7); + + uint32_t x = fmha::float2_to_half2(tmp00, tmp01); + uint32_t y = fmha::float2_to_half2(tmp02, tmp03); + uint32_t z = fmha::float2_to_half2(tmp10, tmp11); + uint32_t w = fmha::float2_to_half2(tmp12, tmp13); + + size_t offset = (this->write_offset_ ^ (ni * 32)) + mi * WARPS_M * 16 * BYTES_PER_ROW; + fmha::sts(this->smem_ + offset + 0 * BYTES_PER_ROW, x); + fmha::sts(this->smem_ + offset + 8 * BYTES_PER_ROW, z); + offset ^= 4 * Base::BYTES_PER_STS; + fmha::sts(this->smem_ + offset + 0 * BYTES_PER_ROW, y); + fmha::sts(this->smem_ + offset + 8 * BYTES_PER_ROW, w); + } + } + } + + template + inline __device__ void store(const uint4 (®s)[M][N]) { + for( int mi = 0; mi < M; mi++ ) { + for( int ni = 0; ni < N; ni++ ) { + size_t offset = (this->write_offset_ ^ (ni * 32)) + mi * WARPS_M * 16 * BYTES_PER_ROW; + fmha::sts(this->smem_ + offset + 0 * BYTES_PER_ROW, regs[mi][ni].x); + fmha::sts(this->smem_ + offset + 8 * BYTES_PER_ROW, regs[mi][ni].z); + offset ^= 4 * Base::BYTES_PER_STS; + fmha::sts(this->smem_ + offset + 0 * BYTES_PER_ROW, regs[mi][ni].y); + fmha::sts(this->smem_ + offset + 8 * BYTES_PER_ROW, regs[mi][ni].w); + } + } + } + + uint32_t read_offset_; +}; + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/softmax.h b/apex/apex/contrib/csrc/fmha/src/fmha/softmax.h new file mode 100644 index 00000000..153f42d5 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/softmax.h @@ -0,0 +1,395 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Sum_ { + enum { IS_SUM = 1 }; + static inline __device__ float apply(float x, float y) { + return x + y; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Max_ { + enum { IS_SUM = 0 }; + static inline __device__ float apply(float x, float y) { + return x > y ? x : y; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ float apply_exp_(float x, float max) { + return __expf(x - max); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template struct ReadType {}; +template<> struct ReadType<4> { using T = float;}; +template<> struct ReadType<8> { using T = float2;}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Smem_tile_reduce { + // Helper class to distribute MMA tiles reduced over rows per warp over quads. + + // The Mma tile. + using Mma_tile = fmha::Hmma_tile; + + // The number of MMAs in M/N dimensions. + enum { MMAS_M = Mma_tile::MMAS_M }; + enum { MMAS_N = Mma_tile::MMAS_N }; + + enum { WARPS_M = Cta_tile::WARPS_M }; + enum { WARPS_N = Cta_tile::WARPS_N }; + + + static constexpr int ROWS = WARPS_M * MMAS_M * 16; + static constexpr int COLS = WARPS_N; + static_assert(COLS == 4 || COLS == 8); + static constexpr int ROWS_PER_XOR_PATTERN = (COLS == 8) ? 4 : 8; + static constexpr int BYTES_PER_TILE = ROWS * COLS * sizeof(float); + static constexpr int ELTS_PER_TILE = ROWS * COLS; + + static constexpr int THREADS_PER_GROUP = Kernel_traits::Gmem_tile_o::THREADS_PER_ROW; + static_assert(THREADS_PER_GROUP == 16); // DEBUG + static constexpr int ROWS_PER_WARP = 32 / THREADS_PER_GROUP; + static constexpr int LOOPS = Kernel_traits::Gmem_tile_o::LOOPS; + static_assert(LOOPS == 1); + + using read_t = typename ReadType::T; + + __device__ inline Smem_tile_reduce(float *smem_, const int tidx) { + + int lane = tidx % 32; + int warp = tidx / 32; + + int warp_m = warp % WARPS_M; + int warp_n = warp / WARPS_M; + + qid_ = lane % 4; + int qp = lane / 4; + + // Swizzle the column to avoid 2-fold bank conflicts when we have 8 warps. + // This won't affect reading as we assume commutative reduction ops. + const int col = warp_n ^ (qp / ROWS_PER_XOR_PATTERN); + smem_write_ = &smem_[warp_m * 16 * MMAS_M * WARPS_N + qp * WARPS_N + col]; + smem_read_ = &reinterpret_cast(smem_)[warp_m * 16 * MMAS_M * 4 + qp * 4 + qid_]; + + } + + __device__ inline void store(float (&frag)[2 * MMAS_M]) { + if( qid_ == 0 ) { + #pragma unroll + for( int mi = 0; mi < MMAS_M; mi++ ) { + int offset = mi * 16 * WARPS_N; + smem_write_[offset + 0 * 8 * WARPS_N] = frag[mi * 2 + 0]; + smem_write_[offset + 1 * 8 * WARPS_N] = frag[mi * 2 + 1]; + } + } + } + + __device__ inline void load(read_t (&frag)[2 * MMAS_M]) { + #pragma unroll + for( int mi = 0; mi < MMAS_M; mi++ ) { + int offset = mi * 16 * 4; + frag[mi * 2 + 0] = smem_read_[offset + 0 * 8 * 4]; + frag[mi * 2 + 1] = smem_read_[offset + 1 * 8 * 4]; + } + } + + int qid_; + float *smem_write_; + read_t *smem_read_; + +}; + + +template +struct Softmax_base { + + // The Mma tile. + using Mma_tile = fmha::Hmma_tile; + + // The number of MMAs in M/N dimensions. + enum { MMAS_M = Mma_tile::MMAS_M }; + enum { MMAS_N = Mma_tile::MMAS_N }; + + // The number of groups of warp such that we have at most 4 warps writing consecutive elements. + enum { GROUPS = fmha::Div_up::VALUE }; + // The number of elements that we are going to store per row. + enum { ELEMENTS_PER_ROW = Cta_tile::WARPS_N / GROUPS }; + // The number of rows. + enum { ROWS = Cta_tile::M * GROUPS }; + // The total number of elements. + enum { ELEMENTS = ROWS * ELEMENTS_PER_ROW }; + + // Ctor. + template + inline __device__ Softmax_base(const Params ¶ms, void *smem, int bidb, int tidx) + : // packed_mask_ptr_(reinterpret_cast(params.packed_mask_ptr)), + smem_(reinterpret_cast(smem)), tidx_(tidx) { + + // Move to the 1st mask loaded by the thread+ tidx; + // packed_mask_ptr_ += bidb * params.packed_mask_stride_in_bytes + tidx * sizeof(uint32_t); + + // Extract the position in the warp. + int warp = tidx / Cta_tile::THREADS_PER_WARP; + int lane = tidx % Cta_tile::THREADS_PER_WARP; + + // Decompose the warp index into M and N. + int warp_m = warp % Cta_tile::WARPS_M; + int warp_n = warp / Cta_tile::WARPS_M; + + // Decompose the warp-n index into group/position-inside-the-group. + int warp_g = warp_n / ELEMENTS_PER_ROW; + int warp_i = warp_n % ELEMENTS_PER_ROW; + + // The location written by the threads. + int write_row = warp_g * (ROWS / GROUPS) + warp_m * Mma_tile::M_PER_MMA + lane / 4; + int write_col = warp_i; + + // Assemble the write pointer. + smem_write_ = &smem_[write_row * ELEMENTS_PER_ROW + write_col]; + + // Assemble the read pointer. + smem_read_ = &smem_[warp_m * Mma_tile::M_PER_MMA + lane / 4]; + } + + template + inline __device__ void apply_mask(const Mask &mask) { + #pragma unroll + for( int mi = 0; mi < MMAS_M; ++mi ) { + #pragma unroll + for( int ii = 0; ii < 2; ++ii ) { + #pragma unroll + for( int ni = 0; ni < MMAS_N; ++ni ) { + #pragma unroll + for( int jj = 0; jj < 4; ++jj ) { + if( !mask.is_valid(mi, ni, ii, jj) ) { + elt_[2 * mi + ii][4 * ni + jj] = -INFINITY; + } + } + } + } + } + } + + // Apply the exp to all the elements. + inline __device__ void apply_exp(const float (&max)[MMAS_M * 2]) { + #pragma unroll + for( int mi = 0; mi < MMAS_M * 2; ++mi ) { + #pragma unroll + for( int ni = 0; ni < MMAS_N * 4; ++ni ) { + elt_[mi][ni] = apply_exp_(elt_[mi][ni], max[mi]); + } + } + } + + // Scale all the elements. + inline __device__ void scale(const float (&sum)[MMAS_M * 2]) { + // Precompute the inverse sum to normalize. Without -use_fast_math, it makes a huge deal. + float inv_sum[MMAS_M * 2]; + #pragma unroll + for( int mi = 0; mi < MMAS_M * 2; ++mi ) { + inv_sum[mi] = (sum[mi] == 0.f || sum[mi] != sum[mi]) ? 1.f : 1.f / sum[mi]; + } + + // Update the values. + #pragma unroll + for( int mi = 0; mi < MMAS_M * 2; ++mi ) { + #pragma unroll + for( int ni = 0; ni < MMAS_N * 4; ++ni ) { + elt_[mi][ni] *= inv_sum[mi]; + } + } + } + + // The pointer to the mask. + const char *packed_mask_ptr_; + // Shared memory for the CTA-wide reduction. + float *smem_, *smem_write_, *smem_read_; + // The current thread index. + int tidx_; + // The elements. + float elt_[MMAS_M * 2][MMAS_N * 4]; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Softmax : public Softmax_base { + + // The base class. + using Base = Softmax_base; + // The fragment. + using Fragment_a = fmha::Fragment_a; + + static_assert(Fragment_a::NUM_REGS == 4); + + enum { WARPS_M = Cta_tile::WARPS_M }; + enum { WARPS_N = Cta_tile::WARPS_N }; + // The MMAs. + enum { MMAS_M = Base::MMAS_M }; + enum { MMAS_N = Base::MMAS_N }; + + // The accumulators. + using Accumulator = fmha::Fragment_accumulator; + using Accumulator_out = Fragment; + static_assert(Accumulator_out::NUM_REGS == 4); + + static_assert(std::is_same::value); + + using Smem_tile_red = Smem_tile_reduce; + static_assert(Smem_tile_red::ELTS_PER_TILE == Cta_tile::M * WARPS_N); + // Ctor. + template + inline __device__ Softmax(const Params ¶ms, void *smem, int bidb, int tidx) + : Base(params, smem, bidb, tidx) + , params_scale_bmm1_(params.scale_bmm1) + , smem_sum_(static_cast(smem), tidx) + , smem_max_(static_cast(smem) + Smem_tile_red::ELTS_PER_TILE, tidx) { + } + + // Pack the data to a fragment for the next GEMM. + template + inline __device__ void pack(Fragment_a (&dst)[K][M]) const { + #pragma unroll + for( int mi = 0; mi < M; ++mi ) { + #pragma unroll + for( int ki = 0; ki < K; ++ki ) { + + // 1st row - 4 elements per row. + float tmp_00 = this->elt_[2 * mi + 0][4 * ki + 0]; + float tmp_01 = this->elt_[2 * mi + 0][4 * ki + 1]; + float tmp_02 = this->elt_[2 * mi + 0][4 * ki + 2]; + float tmp_03 = this->elt_[2 * mi + 0][4 * ki + 3]; + + // 2nd row - 4 elements per row. + float tmp_10 = this->elt_[2 * mi + 1][4 * ki + 0]; + float tmp_11 = this->elt_[2 * mi + 1][4 * ki + 1]; + float tmp_12 = this->elt_[2 * mi + 1][4 * ki + 2]; + float tmp_13 = this->elt_[2 * mi + 1][4 * ki + 3]; + + // Pack to 4 registers. + dst[ki][mi].reg(0) = fmha::float2_to_half2(tmp_00, tmp_01); + dst[ki][mi].reg(1) = fmha::float2_to_half2(tmp_10, tmp_11); + dst[ki][mi].reg(2) = fmha::float2_to_half2(tmp_02, tmp_03); + dst[ki][mi].reg(3) = fmha::float2_to_half2(tmp_12, tmp_13); + } + } + } + + // Scale FP32 fragments + inline __device__ void unpack(const Accumulator (&acc)[MMAS_M][MMAS_N]) { + const float scalef = reinterpret_cast(this->params_scale_bmm1_); + + #pragma unroll + for( int mi = 0; mi < MMAS_M; ++mi ) { + #pragma unroll + for( int ni = 0; ni < MMAS_N; ++ni ) { + // 1st row - 4 elements per row. + this->elt_[2 * mi + 0][4 * ni + 0] = acc[mi][ni].elt(0) * scalef; + this->elt_[2 * mi + 0][4 * ni + 1] = acc[mi][ni].elt(1) * scalef; + this->elt_[2 * mi + 0][4 * ni + 2] = acc[mi][ni].elt(4) * scalef; + this->elt_[2 * mi + 0][4 * ni + 3] = acc[mi][ni].elt(5) * scalef; + // 2nd row - 4 elements per row. + this->elt_[2 * mi + 1][4 * ni + 0] = acc[mi][ni].elt(2) * scalef; + this->elt_[2 * mi + 1][4 * ni + 1] = acc[mi][ni].elt(3) * scalef; + this->elt_[2 * mi + 1][4 * ni + 2] = acc[mi][ni].elt(6) * scalef; + this->elt_[2 * mi + 1][4 * ni + 3] = acc[mi][ni].elt(7) * scalef; + } + } + } + // Scale FP32 fragments + inline __device__ void unpack_noscale(const Accumulator (&acc)[MMAS_M][MMAS_N]) { + + #pragma unroll + for( int mi = 0; mi < MMAS_M; ++mi ) { + #pragma unroll + for( int ni = 0; ni < MMAS_N; ++ni ) { + // 1st row - 4 elements per row. + this->elt_[2 * mi + 0][4 * ni + 0] = acc[mi][ni].elt(0); + this->elt_[2 * mi + 0][4 * ni + 1] = acc[mi][ni].elt(1); + this->elt_[2 * mi + 0][4 * ni + 2] = acc[mi][ni].elt(4); + this->elt_[2 * mi + 0][4 * ni + 3] = acc[mi][ni].elt(5); + // 2nd row - 4 elements per row. + this->elt_[2 * mi + 1][4 * ni + 0] = acc[mi][ni].elt(2); + this->elt_[2 * mi + 1][4 * ni + 1] = acc[mi][ni].elt(3); + this->elt_[2 * mi + 1][4 * ni + 2] = acc[mi][ni].elt(6); + this->elt_[2 * mi + 1][4 * ni + 3] = acc[mi][ni].elt(7); + } + } + } + + + + template + __device__ inline void reduce_(float (&frag)[2 * MMAS_M], Operator &op, Smem_tile_red & smem_red) { + for( int mi = 0; mi < 2 * MMAS_M; mi++ ) { + frag[mi] = this->elt_[mi][0]; + for( int ni = 1; ni < 4 * MMAS_N; ni++ ) { + frag[mi] = op(frag[mi], this->elt_[mi][ni]); + } + } + quad_reduce(frag, frag, op); + + smem_red.store(frag); + __syncthreads(); + typename Smem_tile_red::read_t tmp[2 * MMAS_M]; + smem_red.load(tmp); + + quad_allreduce(frag, tmp, op); + } + + __device__ inline void reduce_max(float (&frag)[2 * MMAS_M]){ + MaxOp max; + reduce_(frag, max, smem_max_); + } + + __device__ inline void reduce_sum(float (&frag)[2 * MMAS_M]){ + SumOp sum; + reduce_(frag, sum, smem_sum_); + } + + + const uint32_t params_scale_bmm1_; + Smem_tile_red smem_max_; + Smem_tile_red smem_sum_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha/utils.h b/apex/apex/contrib/csrc/fmha/src/fmha/utils.h new file mode 100644 index 00000000..bedba0ef --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha/utils.h @@ -0,0 +1,1038 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include + +extern "C" __device__ uint32_t __nvvm_get_smem_pointer(void *ptr); + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct Row {}; +struct Col {}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int M, bool = (M & (M-1)) == 0 > +struct Next_power_of_two { +}; + +template< int M > +struct Next_power_of_two< M, true > { enum { VALUE = M }; }; +template<> +struct Next_power_of_two< 3, false> { enum { VALUE = 4 }; }; +template<> +struct Next_power_of_two< 5, false> { enum { VALUE = 8 }; }; +template<> +struct Next_power_of_two< 6, false> { enum { VALUE = 8 }; }; +template<> +struct Next_power_of_two< 7, false> { enum { VALUE = 8 }; }; +template<> +struct Next_power_of_two< 9, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 10, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 11, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 12, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 13, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 14, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 15, false> { enum { VALUE = 16 }; }; +template<> +struct Next_power_of_two< 24, false> { enum { VALUE = 32 }; }; +template<> +struct Next_power_of_two< 48, false> { enum { VALUE = 64 }; }; +template<> +struct Next_power_of_two< 80, false> { enum { VALUE = 128 }; }; +template<> +struct Next_power_of_two< 96, false> { enum { VALUE = 128 }; }; +template<> +struct Next_power_of_two<112, false> { enum { VALUE = 128 }; }; +template<> +struct Next_power_of_two<144, false> { enum { VALUE = 256 }; }; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, bool = (N & (N-1)) == 0 > +struct Prev_power_of_two { +}; + +template< int N > +struct Prev_power_of_two< N, true > { enum { VALUE = N }; }; +template<> +struct Prev_power_of_two< 3, false> { enum { VALUE = 2 }; }; +template<> +struct Prev_power_of_two< 5, false> { enum { VALUE = 4 }; }; +template<> +struct Prev_power_of_two< 6, false> { enum { VALUE = 4 }; }; +template<> +struct Prev_power_of_two< 7, false> { enum { VALUE = 4 }; }; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int M, int N > +struct Div_up { + enum { VALUE = (M + N-1) / N }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int A, int B > +struct Max { + enum { VALUE = A >= B ? A : B }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int A, int B, int C > +struct Max_3 { + enum { VALUE = Max::VALUE, C>::VALUE }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int A, int B > +struct Min { + enum { VALUE = A <= B ? A : B }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int SIZE_IN_BYTES > +struct Uint_from_size_in_bytes { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Uint_from_size_in_bytes<1> { + using Type = uint8_t; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Uint_from_size_in_bytes<2> { + using Type = uint16_t; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Uint_from_size_in_bytes<4> { + using Type = uint32_t; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Uint_from_size_in_bytes<8> { + using Type = uint2; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Uint_from_size_in_bytes<16> { + using Type = uint4; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int WARPS_M, int WARPS_N, int WARPS_K > +struct Warp_masks { +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Warp_masks<8, 1, 1> { enum { M = 0xe0, N = 0x00, K = 0x00 }; }; +template<> +struct Warp_masks<4, 2, 1> { enum { M = 0x60, N = 0x80, K = 0x00 }; }; +template<> +struct Warp_masks<4, 1, 2> { enum { M = 0x60, N = 0x00, K = 0x80 }; }; +template<> +struct Warp_masks<4, 1, 1> { enum { M = 0x60, N = 0x00, K = 0x00 }; }; +template<> +struct Warp_masks<2, 4, 1> { enum { M = 0x20, N = 0xc0, K = 0x00 }; }; +template<> +struct Warp_masks<2, 2, 2> { enum { M = 0x20, N = 0x40, K = 0x80 }; }; +template<> +struct Warp_masks<2, 2, 1> { enum { M = 0x20, N = 0x40, K = 0x00 }; }; +template<> +struct Warp_masks<2, 1, 2> { enum { M = 0x20, N = 0x00, K = 0x40 }; }; +template<> +struct Warp_masks<2, 1, 1> { enum { M = 0x20, N = 0x00, K = 0x00 }; }; +template<> +struct Warp_masks<1, 8, 1> { enum { M = 0x00, N = 0xe0, K = 0x00 }; }; +template<> +struct Warp_masks<1, 4, 2> { enum { M = 0x00, N = 0x60, K = 0x80 }; }; +template<> +struct Warp_masks<1, 4, 1> { enum { M = 0x00, N = 0x60, K = 0x00 }; }; +template<> +struct Warp_masks<1, 2, 2> { enum { M = 0x00, N = 0x20, K = 0x40 }; }; +template<> +struct Warp_masks<1, 2, 1> { enum { M = 0x00, N = 0x20, K = 0x00 }; }; +template<> +struct Warp_masks<1, 1, 4> { enum { M = 0x00, N = 0x00, K = 0x60 }; }; +template<> +struct Warp_masks<1, 1, 2> { enum { M = 0x00, N = 0x00, K = 0x20 }; }; +template<> +struct Warp_masks<1, 1, 1> { enum { M = 0x00, N = 0x00, K = 0x00 }; }; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename T > +inline __device__ __host__ T div_up(T m, T n) { + return (m + n-1) / n; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline int clz(int x) { + for( int i = 31; i >= 0; --i ) { + if( (1 << i) & x ) { + return 31 - i; + } + } + return 32; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline int find_log_2(int x, bool round_up = false) { + int a = 31 - clz(x); + if( round_up ) { + a += (x & (x-1)) ? 1 : 0; + } + return a; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hadd2(uint32_t a, uint32_t b) { + uint32_t c; + asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b)); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hmin2(uint32_t a, uint32_t b) { + uint32_t c; + asm volatile("min.f16x2 %0, %1, %2;" : "=r"(c) : "r"(a), "r"(b)); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hmul2(uint32_t a, uint32_t b) { + uint32_t c; + asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b)); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint2 hmul4(uint2 a, uint2 b) { + uint2 c; + c.x = hmul2(a.x, b.x); + c.y = hmul2(a.y, b.y); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint4 hmul8(uint4 a, uint4 b) { + uint4 c; + c.x = hmul2(a.x, b.x); + c.y = hmul2(a.y, b.y); + c.z = hmul2(a.z, b.z); + c.w = hmul2(a.w, b.w); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint4 hmul8(uint32_t a, uint4 b) { + uint4 c; + c.x = hmul2(a, b.x); + c.y = hmul2(a, b.y); + c.z = hmul2(a, b.z); + c.w = hmul2(a, b.w); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hrelu2(uint32_t x, uint32_t lb = 0) { + uint32_t res; +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + asm volatile( "max.f16x2 %0, %1, %2;\n" : "=r"(res) : "r"(x), "r"(lb)); +#else + const uint32_t zero = 0u; + asm volatile( \ + "{\n" \ + "\t .reg .f16x2 sela;\n" \ + "\t set.gtu.u32.f16x2 sela, %1, %2;\n" \ + "\t and.b32 %0, sela, %1;\n" + "}\n" : "=r"(res) : "r"(x), "r"(zero)); +#endif + return res; +} +static inline __device__ uint32_t habs2(uint32_t x) { + uint32_t res; + asm volatile( "abs.f16x2 %0, %1;\n" : "=r"(res) : "r"(x)); + return res; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +template< typename T > +static inline __device__ T clamp(T x, T lb, T ub) { + return x < lb ? lb : (x > ub ? ub : x); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint16_t clamp_to_zero(uint16_t x) { + uint16_t mask; + asm volatile("set.gtu %0, %1, 0;" : "=h"(mask) : "h"(x)); + return mask & x; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint16_t float_to_half(float f) { + uint16_t h; + asm volatile("cvt.rn.f16.f32 %0, %1;" : "=h"(h) : "f"(f)); + return h; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t float2_to_half2(float a, float b) { + uint32_t c; +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(c) : "f"(b), "f"(a)); +#else + uint16_t lo = float_to_half(a); + uint16_t hi = float_to_half(b); + asm volatile("mov.b32 %0, {%1, %2};\n" : "=r"(c) : "h"(lo), "h"(hi)); +#endif + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t float_to_half2(float a) { + return float2_to_half2(a,a); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t float2_to_half2(const float2 &f) { + return float2_to_half2(f.x, f.y); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint2 float4_to_half4(float x, float y, float z, float w) { + uint2 d; + d.x = float2_to_half2(x, y); + d.y = float2_to_half2(z, w); + return d; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hfma2(uint32_t a, uint32_t b, uint32_t c) { + uint32_t d; + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c)); + return d; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hfma2_relu(uint32_t a, uint32_t b, uint32_t c) { + uint32_t d; +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + asm volatile("fma.rn.f16x2.relu %0, %1, %2, %3;" : "=r"(d) : "r"(a), "r"(b), "r"(c)); +#else + d = hrelu2(hfma2(a, b, c)); +#endif + return d; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t h0_h0(uint32_t x) { + uint32_t y; + asm volatile("{.reg .f16 lo, hi; mov.b32 {lo, hi}, %1; mov.b32 %0, {lo, lo};}\n" + : "=r"(y) : "r"(x)); + return y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ float h0_to_float(uint32_t h2) { + float f; + asm volatile("{\n" \ + ".reg .f16 lo, hi;\n" \ + "mov.b32 {lo, hi}, %1;\n" \ + "cvt.f32.f16 %0, lo;\n" \ + "}\n" : "=f"(f) : "r"(h2)); + return f; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t h1_h1(uint32_t x) { + uint32_t y; + asm volatile("{.reg .f16 lo, hi; mov.b32 {lo, hi}, %1; mov.b32 %0, {hi, hi};}\n" + : "=r"(y) : "r"(x)); + return y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint16_t hadd(uint16_t a, uint16_t b) { + uint16_t d; + asm volatile("add.f16 %0, %1, %2;" : "=h"(d) : "h"(a), "h"(b)); + return d; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint32_t hadd(uint32_t a, uint32_t b) { + return hadd2(a, b); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint2 hadd4(uint2 a, uint2 b) { + uint2 c; + c.x = hadd2(a.x, b.x); + c.y = hadd2(a.y, b.y); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint2 hadd(uint2 a, uint2 b) { + return hadd4(a, b); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint4 hadd8(uint4 a, uint4 b) { + uint4 c; + c.x = hadd2(a.x, b.x); + c.y = hadd2(a.y, b.y); + c.z = hadd2(a.z, b.z); + c.w = hadd2(a.w, b.w); + return c; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint4 fadd4(uint4 a, uint4 b) { + float4 c; + c.x = reinterpret_cast(a.x) + reinterpret_cast(b.x); + c.y = reinterpret_cast(a.y) + reinterpret_cast(b.y); + c.z = reinterpret_cast(a.z) + reinterpret_cast(b.z); + c.w = reinterpret_cast(a.w) + reinterpret_cast(b.w); + return reinterpret_cast(c); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint4 hadd(uint4 a, uint4 b) { + return hadd8(a, b); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ float half_to_float(uint16_t h) { + float f; + asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h)); + return f; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ float2 half2_to_float2(uint32_t x) { + uint16_t lo, hi; + asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(x)); + return make_float2(half_to_float(lo), half_to_float(hi)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ void half2_to_float2(float &x, float &y, uint32_t h) { + float2 tmp = half2_to_float2(h); + x = tmp.x; + y = tmp.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint16_t hfma(uint16_t a, uint16_t b, uint16_t c) { + uint16_t d; + asm volatile("fma.rn.f16 %0, %1, %2, %3;" : "=h"(d) : "h"(a), "h"(b), "h"(c)); + return d; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ uint16_t hmul(uint16_t a, uint16_t b) { + uint16_t d; + asm volatile("mul.f16 %0, %1, %2;" : "=h"(d) : "h"(a), "h"(b)); + return d; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline __device__ float sigmoid(float x) { + return 1.f / (1.f + expf(-x)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void clear(uint16_t &dst) { + dst = uint16_t(0); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void clear(uint32_t &dst) { + dst = 0u; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void clear(uint2 &dst) { + dst = make_uint2(0u, 0u); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void clear(uint4 &dst) { + dst = make_uint4(0u, 0u, 0u, 0u); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// P R E D I C A T E P A C K I N G +// +//////////////////////////////////////////////////////////////////////////////////////////////////// +enum { BYTES_PER_REG = 4, PREDS_PER_BYTE = 4, PREDS_PER_REG = BYTES_PER_REG * PREDS_PER_BYTE }; + + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// G E N E R I C P R E D I C A T E D L D G S T S +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, int M, typename Functor > +inline __device__ void load_(Functor &fct, const uint32_t (&preds)[M]) { + + // The number of complete bytes (where we use all the predicates in a byte). + enum { COMPLETE = N / PREDS_PER_BYTE }; + // Make sure we did allocate enough predicates. + static_assert(Div_up::VALUE <= M, ""); + // The remainder. + enum { REMAINDER = N - COMPLETE * PREDS_PER_BYTE }; + // Make sure we got the math right and the remainder is between 0 and 3. + static_assert(REMAINDER >= 0 && REMAINDER <= 3, ""); + // The mask to extract the predicates. + enum { COMPLETE_MASK = (1 << PREDS_PER_BYTE) - 1 }; + + // Clear the fetch registers. + #pragma unroll + for( int ii = 0; ii < N; ++ii ) { + fct.clear(ii); + } + + // Run complete steps. + bool p[PREDS_PER_BYTE]; + #pragma unroll + for( int ii = 0; ii < COMPLETE; ++ii ) { + + // The predicate. + uint32_t reg = preds[ii / BYTES_PER_REG]; + + // Extract the predicates. + #pragma unroll + for( int jj = 0; jj < PREDS_PER_BYTE; ++jj ) { + uint32_t mask = 1u << (ii % BYTES_PER_REG * 8 + jj); + p[jj] = (reg & mask) != 0u; + } + + // Issue the loads. + #pragma unroll + for( int jj = 0; jj < PREDS_PER_BYTE; ++jj ) { + fct.load(ii * PREDS_PER_BYTE + jj, p[jj]); + } + } + + // Skip the rest of the code if we do not have a remainder. + if( REMAINDER > 0 ) { + + // The mask to extract the predicates. + enum { REMAINDER_MASK = (1 << REMAINDER) - 1 }; + + // The predicate register. + uint32_t reg = preds[COMPLETE / BYTES_PER_REG]; + + // Extract the predicates. + #pragma unroll + for( int jj = 0; jj < PREDS_PER_BYTE; ++jj ) { + uint32_t mask = 1u << (COMPLETE % BYTES_PER_REG * 8 + jj); + p[jj] = (reg & mask) != 0u; + } + + // Issue the loads. + #pragma unroll + for( int ii = 0; ii < REMAINDER; ++ii ) { + fct.load(COMPLETE * PREDS_PER_BYTE + ii, p[ii]); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int M, typename Functor > +inline __device__ void load_(Functor &fct, uint32_t preds) { + uint32_t tmp[1] = { preds }; + load_(fct, tmp); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// L D G +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldg(uint8_t &dst, const void *ptr) { + dst = *reinterpret_cast(ptr); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldg(uint16_t &dst, const void *ptr) { + dst = *reinterpret_cast(ptr); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldg(uint32_t &dst, const void *ptr) { + dst = *reinterpret_cast(ptr); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldg(uint2 &dst, const void *ptr) { + dst = *reinterpret_cast(ptr); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldg(uint4 &dst, const void *ptr) { + dst = *reinterpret_cast(ptr); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Data_type, int N > +struct Ldg_functor { + // Ctor. + inline __device__ Ldg_functor(Data_type (&fetch)[N], const void* (&ptrs)[N]) + : fetch_(fetch), ptrs_(ptrs) { + } + + // Clear the element. + inline __device__ void clear(int ii) { + fmha::clear(fetch_[ii]); + } + + // Trigger the loads. + inline __device__ void load(int ii, bool p) { + if( p ) { + ldg(fetch_[ii], ptrs_[ii]); + } + } + + // The fetch registers. + Data_type (&fetch_)[N]; + // The pointers. + const void* (&ptrs_)[N]; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Data_type, int N, int M > +inline __device__ void ldg_(Data_type (&fetch)[N], const void* (&ptrs)[N], uint32_t (&preds)[M]) { + Ldg_functor fct(fetch, ptrs); + load_(fct, preds); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, int M > +inline __device__ void ldg(uint8_t (&fetch)[N], const void* (&ptrs)[N], uint32_t (&preds)[M]) { + ldg_(fetch, ptrs, preds); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, int M > +inline __device__ void ldg(uint16_t (&fetch)[N], const void* (&ptrs)[N], uint32_t (&preds)[M]) { + ldg_(fetch, ptrs, preds); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, int M > +inline __device__ void ldg(uint32_t (&fetch)[N], const void* (&ptrs)[N], uint32_t (&preds)[M]) { + ldg_(fetch, ptrs, preds); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, int M > +inline __device__ void ldg(uint2 (&fetch)[N], const void* (&ptrs)[N], uint32_t (&preds)[M]) { + ldg_(fetch, ptrs, preds); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N, int M > +inline __device__ void ldg(uint4 (&fetch)[N], const void* (&ptrs)[N], uint32_t (&preds)[M]) { + ldg_(fetch, ptrs, preds); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// L D S +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void lds(uint16_t &dst, uint32_t ptr) { + asm volatile("ld.shared.b16 %0, [%1];\n" : "=h"(dst) : "r"(ptr)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void lds(uint32_t &dst, uint32_t ptr) { + asm volatile("ld.shared.b32 %0, [%1];\n" : "=r"(dst) : "r"(ptr)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void lds(uint2 &dst, uint32_t ptr) { + asm volatile("ld.shared.v2.b32 {%0, %1}, [%2];\n" : "=r"(dst.x), "=r"(dst.y) : "r"(ptr)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void lds(uint4 &dst, uint32_t ptr) { + asm volatile("ld.shared.v4.b32 {%0, %1, %2, %3}, [%4];\n" + : "=r"(dst.x) + , "=r"(dst.y) + , "=r"(dst.z) + , "=r"(dst.w) + : "r"(ptr)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// L D S M +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldsm(uint32_t &dst, uint32_t ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 730 + asm volatile("ldmatrix.sync.aligned.m8n8.x1.shared.b16 {%0}, [%1];\n" + : "=r"(dst) : "r"(ptr)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldsmt(uint32_t &dst, uint32_t ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 730 + asm volatile("ldmatrix.sync.aligned.m8n8.x1.trans.shared.b16 {%0}, [%1];\n" + : "=r"(dst) : "r"(ptr)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldsm(uint2 &dst, uint32_t ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 730 + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0, %1}, [%2];\n" + : "=r"(dst.x), "=r"(dst.y) : "r"(ptr)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldsmt(uint2 &dst, uint32_t ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 730 + asm volatile("ldmatrix.sync.aligned.m8n8.x2.trans.shared.b16 {%0, %1}, [%2];\n" + : "=r"(dst.x), "=r"(dst.y) : "r"(ptr)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldsm(uint4 &dst, uint32_t ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 730 + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(dst.x), "=r"(dst.y), "=r"(dst.z), "=r"(dst.w) : "r"(ptr)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void ldsmt(uint4 &dst, uint32_t ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 730 + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(dst.x), "=r"(dst.y), "=r"(dst.z), "=r"(dst.w) : "r"(ptr)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// S T G +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void stg(void *ptr, uint8_t val) { + *reinterpret_cast(ptr) = val; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void stg(void *ptr, uint16_t val) { + *reinterpret_cast(ptr) = val; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void stg(void *ptr, uint32_t val) { + *reinterpret_cast(ptr) = val; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void stg(void *ptr, uint2 val) { + *reinterpret_cast(ptr) = val; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void stg(void *ptr, uint4 val) { + *reinterpret_cast(ptr) = val; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +// +// S T S +// +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void sts(uint32_t ptr, uint16_t val) { + asm volatile("st.shared.b16 [%0], %1;\n" : : "r"(ptr), "h"(val)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void sts(uint32_t ptr, uint32_t val) { + asm volatile("st.shared.b32 [%0], %1;\n" : : "r"(ptr), "r"(val)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void sts(uint32_t ptr, uint2 val) { + asm volatile("st.shared.v2.b32 [%0], {%1, %2};\n" + : + : "r"(ptr) + , "r"(val.x) + , "r"(val.y)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void sts(uint32_t ptr, uint4 val) { + asm volatile("st.shared.v4.b32 [%0], {%1, %2, %3, %4};\n" + : + : "r"(ptr) + , "r"(val.x) + , "r"(val.y) + , "r"(val.z) + , "r"(val.w)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename Data_type, int N > +inline __device__ void sts_(uint32_t (&ptrs)[N], const Data_type (&data)[N]) { + #pragma unroll + for( int ii = 0; ii < N; ++ii ) { + sts(ptrs[ii], data[ii]); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +inline __device__ void sts(uint32_t (&ptrs)[N], const uint16_t (&data)[N]) { + sts_(ptrs, data); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +inline __device__ void sts(uint32_t (&ptrs)[N], const uint32_t (&data)[N]) { + sts_(ptrs, data); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +inline __device__ void sts(uint32_t (&ptrs)[N], const uint2 (&data)[N]) { + sts_(ptrs, data); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +inline __device__ void sts(uint32_t (&ptrs)[N], const uint4 (&data)[N]) { + sts_(ptrs, data); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct MaxOp { +__device__ inline T operator()(T const & x, T const & y) { return x > y ? x : y; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct SumOp { +__device__ inline T operator()(T const & x, T const & y) { return x + y; } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Allreduce { + static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4); + template + static __device__ inline T run(T x, Operator &op) { + constexpr int OFFSET = THREADS / 2; + x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET)); + return Allreduce::run(x, op); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct Allreduce<2> { +template +static __device__ inline T run(T x, Operator &op) { + x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1)); + return x; +} +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +__device__ inline void quad_reduce(float (&dst)[M], float (&src)[M], Operator &op) { + #pragma unroll + for(int mi=0; mi < M; mi++){ + dst[mi] = src[mi]; + dst[mi] = op(dst[mi], __shfl_down_sync(uint32_t(-1), dst[mi], 2)); + dst[mi] = op(dst[mi], __shfl_down_sync(uint32_t(-1), dst[mi], 1)); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +__device__ inline void quad_reduce(float (&dst)[M], float2 (&src)[M], Operator &op) { + float tmp[M]; + #pragma unroll + for(int mi=0; mi < M; mi++){ + tmp[mi] = op(src[mi].x, src[mi].y); + } + quad_reduce(dst, tmp, op); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +__device__ inline void quad_allreduce(float (&dst)[M], float (&src)[M], Operator &op) { + #pragma unroll + for(int mi=0; mi < M; mi++){ + dst[mi] = src[mi]; + dst[mi] = Allreduce<4>::run(dst[mi], op); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +__device__ inline void quad_allreduce(float (&dst)[M], float2 (&src)[M], Operator &op) { + float tmp[M]; + #pragma unroll + for(int mi=0; mi < M; mi++){ + tmp[mi] = op(src[mi].x, src[mi].y); + } + quad_allreduce(dst, tmp, op); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_128_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_128_64_kernel.sm80.cu new file mode 100644 index 00000000..517a5b75 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_128_64_kernel.sm80.cu @@ -0,0 +1,60 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_dgrad_kernel_1xN_reload.h" + +using Kernel_traits = FMHA_kernel_traits<128, 64, 16, 1, 4, 0x08u>; + +extern "C" __global__ void fmha_dgrad_fp16_128_64_sm80_kernel(Fused_multihead_attention_fprop_params params) { + fmha::compute_dv_1xN(params); + fmha::compute_dq_dk_1xN(params); +} + +void run_fmha_dgrad_fp16_128_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream) { + + constexpr int smem_size_softmax = Kernel_traits::Cta_tile_p::M * Kernel_traits::Cta_tile_p::WARPS_N * sizeof(float); + constexpr int smem_size_q = Kernel_traits::Smem_tile_q::BYTES_PER_TILE; + constexpr int smem_size_v = Kernel_traits::Smem_tile_v::BYTES_PER_TILE; + constexpr int smem_size_o = Kernel_traits::Smem_tile_o::BYTES_PER_TILE; + + using Smem_tile_s = fmha::Smem_tile_mma_transposed< Kernel_traits::Cta_tile_p>; + constexpr int smem_size_s = Smem_tile_s::BYTES_PER_TILE; + static_assert(smem_size_s == 16 * 128 * 2); + static_assert(smem_size_o == 16 * 64 * 4 * Kernel_traits::Cta_tile_p::WARPS_N); + + constexpr int smem_size_dv = smem_size_s + 2 * smem_size_q + smem_size_v + smem_size_softmax; + constexpr int smem_size_dq_dk = smem_size_s + smem_size_o + smem_size_q + smem_size_v; + constexpr int smem_size = std::max(smem_size_dv, smem_size_dq_dk); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute( + fmha_dgrad_fp16_128_64_sm80_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + dim3 grid(params.h, params.b); + fmha_dgrad_fp16_128_64_sm80_kernel<<>>(params); +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_256_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_256_64_kernel.sm80.cu new file mode 100644 index 00000000..ac22a162 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_256_64_kernel.sm80.cu @@ -0,0 +1,60 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_dgrad_kernel_1xN_reload.h" + +using Kernel_traits = FMHA_kernel_traits<256, 64, 16, 1, 4, 0x08u>; + +extern "C" __global__ void fmha_dgrad_fp16_256_64_sm80_kernel(Fused_multihead_attention_fprop_params params) { + fmha::compute_dv_1xN(params); + fmha::compute_dq_dk_1xN(params); +} + +void run_fmha_dgrad_fp16_256_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream) { + + constexpr int smem_size_softmax = Kernel_traits::Cta_tile_p::M * Kernel_traits::Cta_tile_p::WARPS_N * sizeof(float); + constexpr int smem_size_q = Kernel_traits::Smem_tile_q::BYTES_PER_TILE; + constexpr int smem_size_v = Kernel_traits::Smem_tile_v::BYTES_PER_TILE; + constexpr int smem_size_o = Kernel_traits::Smem_tile_o::BYTES_PER_TILE; + + using Smem_tile_s = fmha::Smem_tile_mma_transposed< Kernel_traits::Cta_tile_p>; + constexpr int smem_size_s = Smem_tile_s::BYTES_PER_TILE; + static_assert(smem_size_s == 16 * 256 * 2); + static_assert(smem_size_o == 16 * 64 * 4 * Kernel_traits::Cta_tile_p::WARPS_N); + + constexpr int smem_size_dv = smem_size_s + 2 * smem_size_q + smem_size_v + smem_size_softmax; + constexpr int smem_size_dq_dk = smem_size_s + smem_size_o + smem_size_q + smem_size_v; + constexpr int smem_size = std::max(smem_size_dv, smem_size_dq_dk); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute( + fmha_dgrad_fp16_256_64_sm80_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + dim3 grid(params.h, params.b); + fmha_dgrad_fp16_256_64_sm80_kernel<<>>(params); +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_384_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_384_64_kernel.sm80.cu new file mode 100644 index 00000000..7081438e --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_384_64_kernel.sm80.cu @@ -0,0 +1,60 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_dgrad_kernel_1xN_reload.h" + +using Kernel_traits = FMHA_kernel_traits<384, 64, 16, 1, 8, 0x08u>; + +extern "C" __global__ void fmha_dgrad_fp16_384_64_sm80_kernel(Fused_multihead_attention_fprop_params params) { + fmha::compute_dv_1xN(params); + fmha::compute_dq_dk_1xN(params); +} + +void run_fmha_dgrad_fp16_384_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream) { + + constexpr int smem_size_softmax = Kernel_traits::Cta_tile_p::M * Kernel_traits::Cta_tile_p::WARPS_N * sizeof(float); + constexpr int smem_size_q = Kernel_traits::Smem_tile_q::BYTES_PER_TILE; + constexpr int smem_size_v = Kernel_traits::Smem_tile_v::BYTES_PER_TILE; + constexpr int smem_size_o = Kernel_traits::Smem_tile_o::BYTES_PER_TILE; + + using Smem_tile_s = fmha::Smem_tile_mma_transposed< Kernel_traits::Cta_tile_p>; + constexpr int smem_size_s = Smem_tile_s::BYTES_PER_TILE; + static_assert(smem_size_s == 16 * 384 * 2); + static_assert(smem_size_o == 16 * 64 * 4 * Kernel_traits::Cta_tile_p::WARPS_N); + + constexpr int smem_size_dv = smem_size_s + 2 * smem_size_q + smem_size_v + smem_size_softmax; + constexpr int smem_size_dq_dk = smem_size_s + smem_size_o + smem_size_q + smem_size_v; + constexpr int smem_size = std::max(smem_size_dv, smem_size_dq_dk); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute( + fmha_dgrad_fp16_384_64_sm80_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + dim3 grid(params.h, params.b); + fmha_dgrad_fp16_384_64_sm80_kernel<<>>(params); +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_512_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_512_64_kernel.sm80.cu new file mode 100644 index 00000000..735006cc --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_512_64_kernel.sm80.cu @@ -0,0 +1,105 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_dgrad_kernel_1xN_reload.h" +#include "fmha_dgrad_kernel_1xN_reload_nl.h" + +using Kernel_traits = FMHA_kernel_traits<512, 64, 16, 1, 8, 0x08u>; + +extern "C" __global__ void fmha_dgrad_fp16_512_64_sm80_kernel(Fused_multihead_attention_fprop_params params) { + fmha::compute_dv_1xN(params); + fmha::compute_dq_dk_1xN(params); +} + +template +__global__ +void fmha_dgrad_fp16_512_64_sm80_nl_kernel(Fused_multihead_attention_fprop_params params){ + fmha::compute_dv_1xN_nl(params); + fmha::compute_dq_dk_1xN_nl(params); +} + +void run_fmha_dgrad_fp16_512_64_sm80(const Fused_multihead_attention_fprop_params ¶ms, cudaStream_t stream) { + + constexpr int smem_size_softmax = Kernel_traits::Cta_tile_p::M * Kernel_traits::Cta_tile_p::WARPS_N * sizeof(float); + constexpr int smem_size_q = Kernel_traits::Smem_tile_q::BYTES_PER_TILE; + constexpr int smem_size_v = Kernel_traits::Smem_tile_v::BYTES_PER_TILE; + constexpr int smem_size_o = Kernel_traits::Smem_tile_o::BYTES_PER_TILE; + + using Smem_tile_s = fmha::Smem_tile_mma_transposed< Kernel_traits::Cta_tile_p>; + constexpr int smem_size_s = Smem_tile_s::BYTES_PER_TILE; + static_assert(smem_size_s == 16 * 512 * 2); + static_assert(smem_size_o == 16 * 64 * 4 * Kernel_traits::Cta_tile_p::WARPS_N); + + constexpr int smem_size_dv = smem_size_s + 2 * smem_size_q + smem_size_v + smem_size_softmax; + constexpr int smem_size_dq_dk = smem_size_s + smem_size_o + smem_size_q + smem_size_v; + constexpr int smem_size = std::max(smem_size_dv, smem_size_dq_dk); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute( + fmha_dgrad_fp16_512_64_sm80_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + dim3 grid(params.h, params.b); + fmha_dgrad_fp16_512_64_sm80_kernel<<>>(params); +} + +void run_fmha_dgrad_fp16_512_64_sm80_nl(const Fused_multihead_attention_fprop_params ¶ms, const int num_chunks, cudaStream_t stream) { + + constexpr int smem_size_softmax = Kernel_traits::Cta_tile_p::M * Kernel_traits::Cta_tile_p::WARPS_N * sizeof(float); + constexpr int smem_size_q = Kernel_traits::Smem_tile_q::BYTES_PER_TILE; + constexpr int smem_size_v = Kernel_traits::Smem_tile_v::BYTES_PER_TILE; + constexpr int smem_size_o = Kernel_traits::Smem_tile_o::BYTES_PER_TILE; + + using Smem_tile_s = fmha::Smem_tile_mma_transposed; + constexpr int smem_size_s = Smem_tile_s::BYTES_PER_TILE; + static_assert(smem_size_s == 16 * 512 * 2); + static_assert(smem_size_o == 16 * 64 * 4 * Kernel_traits::Cta_tile_p::WARPS_N); + + constexpr int smem_size_dv = smem_size_s + 2 * smem_size_q + smem_size_v + smem_size_softmax; + constexpr int smem_size_dq_dk = smem_size_s + smem_size_o + smem_size_q + smem_size_v; + constexpr int smem_size = std::max(smem_size_dv, smem_size_dq_dk); + + auto kernel = fmha_dgrad_fp16_512_64_sm80_nl_kernel<2>; + + if( num_chunks == 2 ) { + kernel = fmha_dgrad_fp16_512_64_sm80_nl_kernel<2>; + }else if( num_chunks == 3 ) { + kernel = fmha_dgrad_fp16_512_64_sm80_nl_kernel<3>; + } else { + assert(false && "Unsupperted number of chunks"); + } + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + + dim3 grid(params.h, params.b, num_chunks); + + kernel<<>>(params); + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload.h b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload.h new file mode 100644 index 00000000..3c4b8174 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload.h @@ -0,0 +1,558 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include "fmha_kernel.h" +#include +#include + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ void compute_dv_1xN(const Params ¶ms) { + + // The description of the CTA tile for the 1st batched GEMM. + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + // The description of the CTA tile for the 2nd batched GEMM. + using Cta_tile_dv = + fmha::Cta_tile_extd; + + static_assert(Cta_tile_dv::M == 512 || Cta_tile_dv::M == 384 || Cta_tile_dv::M == 256 || Cta_tile_dv::M == 128); + static_assert(Cta_tile_dv::N == 64); + static_assert(Cta_tile_dv::K == 16); + + // The MMA tile for the 1st GEMM. + using Mma_tile_p = fmha::Hmma_tile; + // The MMA tile for the 2nd GEMM. + using Mma_tile_dv = fmha::Hmma_tile; + + // The global memory tile to load Q. + using Gmem_tile_q = typename Kernel_traits::Gmem_tile_q; + // The shared memory tile to swizzle Q. + // using Smem_tile_q = typename Kernel_traits::Smem_tile_q; + using Smem_tile_q = fmha::Smem_tile_a; + // The shared memory tile to reload Q as fragment b. + using Smem_tile_qt = fmha::Smem_tile_b; + + // The global memory tile to load K. + using Gmem_tile_k = typename Kernel_traits::Gmem_tile_k; + // The shared memory tile to swizzle K. + using Smem_tile_k = typename Kernel_traits::Smem_tile_k; + + // The global memory tile to load V. + using Gmem_tile_v = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle V. + using Smem_tile_v = typename Kernel_traits::Smem_tile_v; + + // The global memory tile to store O. + using Gmem_tile_o = typename Kernel_traits::Gmem_tile_o; + // The shared memory tile to swizzle O. + using Smem_tile_o = typename Kernel_traits::Smem_tile_o; + + // The global memory tile to store dV. + using Gmem_tile_dv = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle dV. + using Smem_tile_dv = fmha::Smem_tile_mma_epilogue; + static_assert(Smem_tile_dv::NUM_LDS == Gmem_tile_dv::LDGS); + static_assert(Smem_tile_dv::THREADS_PER_ROW == Gmem_tile_dv::THREADS_PER_ROW); + + using Gmem_tile_s = typename Kernel_traits::Gmem_tile_s; + using Smem_tile_st = typename Kernel_traits::Smem_tile_st; + using Gmem_tile_do = typename Kernel_traits::Gmem_tile_do; + + // Shared memory. + extern __shared__ char smem_[]; + + // The block index for the batch. + const int bidb = blockIdx.y; + // The block index for the head. + const int bidh = blockIdx.x; + // The thread index. + const int tidx = threadIdx.x; + + const BlockInfoPadded binfo(params, bidb, bidh, tidx); + if( binfo.stop_early() ) + return; + Mask mask(params, binfo, tidx); + + // Allocate the global memory tile loader for Q. + Gmem_tile_do gmem_q(params, binfo, tidx); // treating dout as Q + // Allocate the shared memory tile loader for Q. + Smem_tile_q smem_q(&smem_[0], tidx); + Smem_tile_qt smem_qt(&smem_[0], tidx); + Smem_tile_st smem_s(&smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE], tidx); + + // Allocate the global memory tile loader for K. + Gmem_tile_k gmem_k(params, 2, binfo, tidx); // treating V as K + // Allocate the shared memory tile loader for K. + Smem_tile_k smem_k(&smem_[Smem_tile_q::BYTES_PER_TILE], tidx); + + // Trigger the loads for Q. + gmem_q.load(smem_q); + // Trigger the loads for K. + gmem_k.load(smem_k); + + // Commit the data for Q and K to shared memory. + gmem_q.commit(smem_q); + gmem_k.commit(smem_k); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Load the fragments for Q. + typename Smem_tile_q::Fragment frag_q[2][Mma_tile_p::MMAS_M]; + smem_q.load(frag_q[0], 0); + + typename Smem_tile_qt::Fragment frag_qt[2][Mma_tile_dv::MMAS_N]; + static_assert(Smem_tile_qt::Fragment::NUM_REGS == 4); + static_assert(Mma_tile_dv::MMAS_K == 1); + smem_qt.load(frag_qt[0], 0); + + // Load the fragments for K. We keep the data in registers during the entire kernel. + typename Smem_tile_k::Fragment frag_k[2][Mma_tile_p::MMAS_N]; + smem_k.load(frag_k[0], 0); + + enum { BITS_PER_ELT_S = sizeof(fmha::A_type) * 8 }; + + Gmem_tile_s gmem_s(params, binfo, tidx); + + // Create the object to do the softmax. + using Softmax = fmha::Softmax; + Softmax softmax( + params, &smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_st::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE], bidb, tidx); + + enum { THREADS_PER_ROW = 32 }; + enum { M = Mma_tile_p::MMAS_M }; + enum { N = Mma_tile_p::MMAS_N }; + + // Declare the accumulators for the 2nd gemm. + fmha::Fragment_accumulator acc_dv[Mma_tile_dv::MMAS_M][Mma_tile_dv::MMAS_N]; + fmha::Clear_accumulator::apply(acc_dv); + + enum { STEPS = Cta_tile_p::N / Cta_tile_p::M }; + // Load over the entire sequence length. + for( int l = 0; l < STEPS; l++ ) { + const int loop = l * Cta_tile_p::M; + if( loop >= binfo.actual_seqlen ) + break; + + // Load S + uint4 s_regs[M][N]; + gmem_s.load(s_regs, mask); + fmha::Fragment_accumulator acc_p[Mma_tile_p::MMAS_M][Mma_tile_p::MMAS_N]; + fmha::Clear_accumulator::apply(acc_p); + // Do this part of P^T = (Q * K^T)^T. + #pragma unroll + for( int ki = 1; ki < Mma_tile_p::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_q.load(frag_q[ki & 1], ki); + smem_k.load(frag_k[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_p, frag_q[(ki - 1) & 1], frag_k[(ki - 1) & 1]); + } + + // Store s * dmask to smem for transpose + smem_s.store(s_regs); + + // Declare the accumulators for the 1st gemm. + // Do the final stage of math. + { + int ki = Mma_tile_p::MMAS_K; + fmha::gemm(acc_p, frag_q[(ki - 1) & 1], frag_k[(ki - 1) & 1]); + } + // Trigger the load for the next Q values. We're using double buffering, so reading qt is safe + if( l < STEPS - 1) { + smem_q.move_to_next_write_buffer(); + gmem_q.move(); + gmem_q.load(smem_q); + } + + + // Convert from the accumulator type to FP32 for Softmax. + softmax.unpack(acc_p); + + float s_mat[2 * M][4 * N]; + + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + uint4 &dst = s_regs[mi][ni]; + fmha::half2_to_float2(s_mat[2 * mi + 0][4 * ni + 0], s_mat[2 * mi + 0][4 * ni + 1], dst.x); + fmha::half2_to_float2(s_mat[2 * mi + 0][4 * ni + 2], s_mat[2 * mi + 0][4 * ni + 3], dst.y); + fmha::half2_to_float2(s_mat[2 * mi + 1][4 * ni + 0], s_mat[2 * mi + 1][4 * ni + 1], dst.z); + fmha::half2_to_float2(s_mat[2 * mi + 1][4 * ni + 2], s_mat[2 * mi + 1][4 * ni + 3], dst.w); + } + } + + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ii = 0; ii < 2; ii++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + #pragma unroll + for( int jj = 0; jj < 4; jj++ ) { + float & s_dmask = s_mat[2 * mi + ii][4 * ni + jj]; + const bool drop = reinterpret_cast(s_dmask) & 0x80000000; + const float d_s = drop ? 0.f : softmax.elt_[2 * mi + ii][4 * ni + jj] * params.rp_dropout; + s_dmask = fabsf(s_dmask); + softmax.elt_[2 * mi + ii][4 * ni + jj] = d_s * fabsf(s_dmask); + } + } + } + } + + float p_sum[2 * M]; + softmax.reduce_sum(p_sum); + + const float scalef = reinterpret_cast(params.scale_softmax); + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ii = 0; ii < 2; ii++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + #pragma unroll + for( int jj = 0; jj < 4; jj++ ) { + softmax.elt_[2 * mi + ii][4 * ni + jj] -= p_sum[2 * mi + ii] * (s_mat[2 * mi + ii][4 * ni + jj]) ; + softmax.elt_[2 * mi + ii][4 * ni + jj] *= scalef; + } + } + } + } + typename Smem_tile_st::Fragment frag_s[Mma_tile_dv::MMAS_K][Mma_tile_dv::MMAS_M]; + smem_s.load(frag_s); + for( int ki = 0; ki < Mma_tile_dv::MMAS_K; ki++ ) { + for( int mi = 0; mi < Mma_tile_dv::MMAS_M; mi++ ) { + for( int ii = 0; ii < Smem_tile_st::Fragment::NUM_REGS; ii++ ) { + frag_s[ki][mi].reg(ii) = fmha::hmul2(frag_s[ki][mi].reg(ii), params.scale_dropout); + frag_s[ki][mi].reg(ii) = fmha::hrelu2(frag_s[ki][mi].reg(ii)); + } + } + } + + gmem_s.store(softmax.elt_, mask); + gmem_s.move(); + + #pragma unroll + for( int ki = 1; ki < Mma_tile_dv::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_qt.load(frag_qt[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_dv, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + + // Do the final stage of math. + { + int ki = Mma_tile_dv::MMAS_K; + fmha::gemm(acc_dv, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + // Commit the values for Q into shared memory. + if(l < STEPS - 1) { + gmem_q.commit(smem_q); + } + + // Make sure we are reading from the correct buffer. + smem_q.move_to_next_read_buffer(); + smem_qt.move_to_next_read_buffer(); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Trigger the loads for the values of Q for the next iteration. + smem_q.load(frag_q[0], 0); + smem_k.load(frag_k[0], 0); + smem_qt.load(frag_qt[0], 0); + + } // Outer loop over the sequence length. + + // Epilogue swizzle for dV + Smem_tile_dv smem_dv(&smem_[Kernel_traits::Smem_tile_q::BYTES_PER_TILE], tidx); + smem_dv.store(acc_dv); + + __syncthreads(); + uint4 dv_out[Smem_tile_dv::NUM_LDS]; + smem_dv.load(dv_out); + Qkv_params dv_params; + dv_params.qkv_ptr = params.dqkv_ptr; + dv_params.qkv_stride_in_bytes = params.qkv_stride_in_bytes; + dv_params.h = params.h; + Gmem_tile_dv gmem_dv(dv_params, 2, binfo, tidx); + gmem_dv.store(dv_out); +} + +template +inline __device__ void compute_dq_dk_1xN(const Params ¶ms) { + + // The description of the CTA tile for the 1st batched GEMM. + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + using Cta_tile_o = typename Kernel_traits::Cta_tile_o; + // The description of the CTA tile for the 2nd batched GEMM. + using Cta_tile_dk = + fmha::Cta_tile_extd; + static_assert(Cta_tile_dk::M == 512 || Cta_tile_dk::M == 384 || Cta_tile_dk::M == 256 || Cta_tile_dk::M == 128); + static_assert(Cta_tile_dk::N == 64); + static_assert(Cta_tile_dk::K == 16); + + // The MMA tile for the 1st GEMM. + using Mma_tile_p = fmha::Hmma_tile; + using Mma_tile_o = fmha::Hmma_tile; + // The MMA tile for the 2nd GEMM. + using Mma_tile_dk = fmha::Hmma_tile; + + // The global memory tile to load Q. + using Gmem_tile_q = typename Kernel_traits::Gmem_tile_q; + // The shared memory tile to swizzle Q. + using Smem_tile_q = typename Kernel_traits::Smem_tile_q; + + // The global memory tile to load K. + using Gmem_tile_k = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle K. + using Smem_tile_k = typename Kernel_traits::Smem_tile_v; // K is used like V in fprop + + // The global memory tile to load V. + using Gmem_tile_v = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle V. + using Smem_tile_v = typename Kernel_traits::Smem_tile_v; + + // The global memory tile to store O. + // using Gmem_tile_o = typename Kernel_traits::Gmem_tile_o; + using Gmem_tile_o = fmha::Gmem_tile_dq; + // The shared memory tile to swizzle O. + using Smem_tile_o = typename Kernel_traits::Smem_tile_o; + + // The global memory tile to store dK. + using Gmem_tile_dk = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle dK. + using Smem_tile_dk = fmha::Smem_tile_mma_epilogue; + static_assert(Smem_tile_dk::NUM_LDS == Gmem_tile_dk::LDGS); + static_assert(Smem_tile_dk::THREADS_PER_ROW == Gmem_tile_dk::THREADS_PER_ROW); + + // The shared memory tile to reload Q transposed. + using Smem_tile_qt = fmha::Smem_tile_b; + + using Gmem_tile_s = typename Kernel_traits::Gmem_tile_s; + + using Smem_tile_st = typename Kernel_traits::Smem_tile_st; + + + enum { M = Mma_tile_p::MMAS_M }; + enum { N = Mma_tile_p::MMAS_N }; + static_assert(M == Mma_tile_o::MMAS_M); + static_assert(N == Mma_tile_o::MMAS_K); + // Shared memory. + extern __shared__ char smem_[]; + + // The block index for the batch. + const int bidb = blockIdx.y; + // The block index for the head. + const int bidh = blockIdx.x; + // The thread index. + const int tidx = threadIdx.x; + + const BlockInfoPadded binfo(params, bidb, bidh, tidx); + if( binfo.stop_early() ) + return; + + Mask mask(params, binfo, tidx); + + // Allocate the global memory tile loader for Q. + Gmem_tile_q gmem_q(params, 0, binfo, tidx); + // Allocate the shared memory tile loader for Q. + Smem_tile_q smem_q(&smem_[0], tidx); + Smem_tile_qt smem_qt(&smem_[0], tidx); + Smem_tile_st smem_s(&smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE + Smem_tile_o::BYTES_PER_TILE], tidx); + + // Allocate the global memory tile loader for K. + Gmem_tile_k gmem_k(params, 1, binfo, tidx); + // Allocate the shared memory tile loader for K. + Smem_tile_k smem_k(&smem_[Smem_tile_q::BYTES_PER_TILE], tidx); + + // Allocate the global memory tile loader for O. + Gmem_tile_o gmem_o(params, binfo, tidx); + // Allocate the shared memory tile loader for O. We use the same as K so be careful!!! + Smem_tile_o smem_o(&smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE], tidx); + + // Trigger the loads for Q. + gmem_q.load(smem_q); + // Trigger the loads for K. + gmem_k.load(smem_k); + + Gmem_tile_s gmem_s(params, binfo, tidx); + // Load dP + uint4 s_regs[M][N]; + gmem_s.load(s_regs, mask); + gmem_s.move(); + + // Commit the data for Q and K to shared memory. + gmem_q.commit(smem_q); + gmem_k.commit(smem_k); + + // Make sure the data is in shared memory. + __syncthreads(); + + typename Smem_tile_qt::Fragment frag_qt[2][Mma_tile_dk::MMAS_N]; + smem_qt.load(frag_qt[0], 0); + typename Smem_tile_k::Fragment frag_k[2][Mma_tile_o::MMAS_N]; + smem_k.load(frag_k[0], 0); + + enum { BITS_PER_ELT_S = sizeof(fmha::A_type) * 8 }; + + enum { THREADS_PER_ROW = 32 }; + enum { STEPS = Cta_tile_p::N / Cta_tile_p::M }; + + // Declare the accumulators for the 2nd gemm. + fmha::Fragment_accumulator acc_dk[Mma_tile_dk::MMAS_M][Mma_tile_dk::MMAS_N]; + fmha::Clear_accumulator::apply(acc_dk); + + // Load over the entire sequence length. + for( int l=0;l= binfo.actual_seqlen ) + break; + + // Pack dP as Fragment_a + fmha::Fragment_a frag_p[Mma_tile_o::MMAS_K][Mma_tile_o::MMAS_M]; + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + uint4 &dst = s_regs[mi][ni]; + frag_p[ni][mi].reg(0) = dst.x; // row 0, cols 0,1 + frag_p[ni][mi].reg(1) = dst.z; // row 8, cols 0,1 + frag_p[ni][mi].reg(2) = dst.y; // row 0, cols 8,9 + frag_p[ni][mi].reg(3) = dst.w; // row 8, cols 8,9 + } + } + + // Declare the accumulators for the 1st gemm. + fmha::Fragment_accumulator acc_o[Mma_tile_o::MMAS_M][Mma_tile_o::MMAS_N]; + fmha::Clear_accumulator::apply(acc_o); + + // Do this part of O = P^T * V^T. dQ = dP x dK + #pragma unroll + for( int ki = 1; ki < Mma_tile_o::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_k.load(frag_k[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_o, frag_p[ki - 1], frag_k[(ki - 1) & 1]); + } + + // Do the final stage of math. + { + int ki = Mma_tile_o::MMAS_K; + fmha::gemm(acc_o, frag_p[ki - 1], frag_k[(ki - 1) & 1]); + } + + // Store dP to smem for transpose + smem_s.store(s_regs); + if(l < STEPS - 1) { + // Load next part of S + gmem_s.load(s_regs, mask); + gmem_s.move(); + smem_q.move_to_next_write_buffer(); + gmem_q.move(); + gmem_q.load(smem_q); + } + // Loop over MMAS_M. + #pragma unroll + for( int ii = 0; ii < Gmem_tile_o::LOOPS; ++ii ) { + + // Swizzle the elements and do the final reduction. + smem_o.store(acc_o, ii); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Load from shared memory. + uint4 out[Gmem_tile_o::STGS_PER_LOOP]; + smem_o.load(out); + + // Make sure the data was read from shared memory. + if( ii < Gmem_tile_o::LOOPS - 1 ) { + __syncthreads(); + } + + // Output the values. + gmem_o.store(out, ii); + } + + // Move to the next part of the output. + gmem_o.move(); + + typename Smem_tile_st::Fragment frag_s[Mma_tile_dk::MMAS_K][Mma_tile_dk::MMAS_M]; + smem_s.load(frag_s); + + #pragma unroll + for( int ki = 1; ki < Mma_tile_dk::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_qt.load(frag_qt[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_dk, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + + // Do the final stage of math. + { + int ki = Mma_tile_dk::MMAS_K; + fmha::gemm(acc_dk, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + + // Commit the values for Q into shared memory. + if( l < STEPS - 1) { + gmem_q.commit(smem_q); + } + + // Make sure the data is in shared memory. + __syncthreads(); + + // Trigger the loads for the values of Q for the next iteration. + smem_qt.load(frag_qt[0], 0); + smem_k.load(frag_k[0], 0); + + } // Outer loop over the sequence length. + + // Epilogue swizzle for dK + Smem_tile_dk smem_dk(&smem_[0], tidx); + smem_dk.store(acc_dk); + __syncthreads(); + uint4 dk_out[Smem_tile_dk::NUM_LDS]; + smem_dk.load(dk_out); + Qkv_params dk_params; + dk_params.qkv_ptr = params.dqkv_ptr; + dk_params.qkv_stride_in_bytes = params.qkv_stride_in_bytes; + dk_params.h = params.h; + Gmem_tile_dk gmem_dk(dk_params, 1, binfo, tidx); + gmem_dk.store(dk_out); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload_nl.h b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload_nl.h new file mode 100644 index 00000000..26776d48 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_dgrad_kernel_1xN_reload_nl.h @@ -0,0 +1,569 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include "fmha_kernel.h" +#include +#include + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ void compute_dv_1xN_nl(const Params ¶ms) { + + // The description of the CTA tile for the 1st batched GEMM. + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + // The description of the CTA tile for the 2nd batched GEMM. + using Cta_tile_dv = fmha::Cta_tile_extd; + + static_assert(Cta_tile_dv::M == 512 || Cta_tile_dv::M == 384 || Cta_tile_dv::M == 256 || Cta_tile_dv::M == 128); + static_assert(Cta_tile_dv::N == 64); + static_assert(Cta_tile_dv::K == 16); + + // The MMA tile for the 1st GEMM. + using Mma_tile_p = fmha::Hmma_tile; + // The MMA tile for the 2nd GEMM. + using Mma_tile_dv = fmha::Hmma_tile; + + // The global memory tile to load Q. + using Gmem_tile_q = typename Kernel_traits::Gmem_tile_q; + // The shared memory tile to swizzle Q. + using Smem_tile_q = fmha::Smem_tile_a; + // The shared memory tile to reload Q as fragment b. + using Smem_tile_qt = fmha::Smem_tile_b; + + // The global memory tile to load K. + using Gmem_tile_k = typename Kernel_traits::Gmem_tile_k; + // The shared memory tile to swizzle K. + using Smem_tile_k = typename Kernel_traits::Smem_tile_k; + + // The global memory tile to load V. + using Gmem_tile_v = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle V. + using Smem_tile_v = typename Kernel_traits::Smem_tile_v; + + // The global memory tile to store dV. + using Gmem_tile_dv = fmha::Gmem_tile_qkv; + + // The shared memory tile to swizzle dV. + using Smem_tile_dv = fmha::Smem_tile_mma_epilogue; + static_assert(Smem_tile_dv::NUM_LDS == Gmem_tile_dv::LDGS); + static_assert(Smem_tile_dv::THREADS_PER_ROW == Gmem_tile_dv::THREADS_PER_ROW); + + using Gmem_tile_s = typename Kernel_traits::Gmem_tile_s; + using Smem_tile_st = typename Kernel_traits::Smem_tile_st; + using Gmem_tile_do = typename Kernel_traits::Gmem_tile_do; + + // Shared memory. + extern __shared__ char smem_[]; + + // The block index for the chunk. + const int bidc = blockIdx.z; + // The block index for the batch. + const int bidb = blockIdx.y; + // The block index for the head. + const int bidh = blockIdx.x; + // The thread index. + const int tidx = threadIdx.x; + + const BlockInfoPadded binfo(params, bidb, bidh, tidx); + if( binfo.stop_early() ) + return; + fmha::Mask mask(params, binfo, tidx); + + // Allocate the global memory tile loader for Q. + Gmem_tile_do gmem_q(params, binfo, tidx); // treating dout as Q + // Allocate the shared memory tile loader for Q. + Smem_tile_q smem_q(&smem_[0], tidx); + Smem_tile_qt smem_qt(&smem_[0], tidx); + Smem_tile_st smem_s(&smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE], tidx); + + // Allocate the global memory tile loader for K. + Gmem_tile_k gmem_k(params, 2, binfo, tidx); // treating V as K + // Allocate the shared memory tile loader for K. + Smem_tile_k smem_k(&smem_[Smem_tile_q::BYTES_PER_TILE], tidx); + + Gmem_tile_s gmem_s(params, binfo, tidx); + + using Noloop = Noloop_traits; + + Noloop nl_traits(bidc, binfo); + nl_traits.move_all(gmem_q, gmem_s); + + // Trigger the loads for Q. + gmem_q.load(smem_q); + // Trigger the loads for K. + gmem_k.load(smem_k); + + // Commit the data for Q and K to shared memory. + gmem_q.commit(smem_q); + gmem_k.commit(smem_k); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Load the fragments for Q. + typename Smem_tile_q::Fragment frag_q[2][Mma_tile_p::MMAS_M]; + smem_q.load(frag_q[0], 0); + + typename Smem_tile_qt::Fragment frag_qt[2][Mma_tile_dv::MMAS_N]; + static_assert(Smem_tile_qt::Fragment::NUM_REGS == 4); + static_assert(Mma_tile_dv::MMAS_K == 1); + smem_qt.load(frag_qt[0], 0); + + // Load the fragments for K. We keep the data in registers during the entire kernel. + typename Smem_tile_k::Fragment frag_k[2][Mma_tile_p::MMAS_N]; + smem_k.load(frag_k[0], 0); + + enum { BITS_PER_ELT_S = sizeof(fmha::A_type) * 8 }; + + // Create the object to do the softmax. + using Softmax = fmha::Softmax; + Softmax softmax( + params, &smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_st::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE], bidb, tidx); + + enum { THREADS_PER_ROW = 32 }; + enum { M = Mma_tile_p::MMAS_M }; + enum { N = Mma_tile_p::MMAS_N }; + + // Declare the accumulators for the 2nd gemm. + fmha::Fragment_accumulator acc_dv[Mma_tile_dv::MMAS_M][Mma_tile_dv::MMAS_N]; + fmha::Clear_accumulator::apply(acc_dv); + + // Load over the entire sequence length. + for(int l = 0; l < nl_traits.num_steps_;l++) { + + uint4 s_regs[M][N]; + gmem_s.load(s_regs, mask); + fmha::Fragment_accumulator acc_p[Mma_tile_p::MMAS_M][Mma_tile_p::MMAS_N]; + fmha::Clear_accumulator::apply(acc_p); + // Do this part of P^T = (Q * K^T)^T. + #pragma unroll + for( int ki = 1; ki < Mma_tile_p::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_q.load(frag_q[ki & 1], ki); + smem_k.load(frag_k[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_p, frag_q[(ki - 1) & 1], frag_k[(ki - 1) & 1]); + } + + smem_s.store(s_regs); + + // Declare the accumulators for the 1st gemm. + // Do the final stage of math. + { + int ki = Mma_tile_p::MMAS_K; + fmha::gemm(acc_p, frag_q[(ki - 1) & 1], frag_k[(ki - 1) & 1]); + } + // Trigger the load for the next Q values. We're using double buffering, so reading qt is safe + if(l < nl_traits.num_steps_ - 1) { + smem_q.move_to_next_write_buffer(); + gmem_q.move(); + gmem_q.load(smem_q); + } + // Convert from the accumulator type to FP32 for Softmax. + softmax.unpack(acc_p); + + float s_mat[2 * M][4 * N]; + + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + uint4 &dst = s_regs[mi][ni]; + fmha::half2_to_float2(s_mat[2 * mi + 0][4 * ni + 0], s_mat[2 * mi + 0][4 * ni + 1], dst.x); + fmha::half2_to_float2(s_mat[2 * mi + 0][4 * ni + 2], s_mat[2 * mi + 0][4 * ni + 3], dst.y); + fmha::half2_to_float2(s_mat[2 * mi + 1][4 * ni + 0], s_mat[2 * mi + 1][4 * ni + 1], dst.z); + fmha::half2_to_float2(s_mat[2 * mi + 1][4 * ni + 2], s_mat[2 * mi + 1][4 * ni + 3], dst.w); + } + } + + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ii = 0; ii < 2; ii++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + #pragma unroll + for( int jj = 0; jj < 4; jj++ ) { + float & s_dmask = s_mat[2 * mi + ii][4 * ni + jj]; + const bool drop = reinterpret_cast(s_dmask) & 0x80000000; + const float d_s= drop ? 0.f : softmax.elt_[2 * mi + ii][4 * ni + jj] * params.rp_dropout; + s_dmask = fabsf(s_dmask); + softmax.elt_[2 * mi + ii][4 * ni + jj] = d_s * (s_dmask); + } + } + } + } + + float p_sum[2 * M]; + softmax.reduce_sum(p_sum); + + const float scalef = reinterpret_cast(params.scale_softmax); + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ii = 0; ii < 2; ii++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + #pragma unroll + for( int jj = 0; jj < 4; jj++ ) { + softmax.elt_[2 * mi + ii][4 * ni + jj] -= p_sum[2 * mi + ii] * (s_mat[2 * mi + ii][4 * ni + jj]) ; + softmax.elt_[2 * mi + ii][4 * ni + jj] *= scalef; + } + } + } + } + + typename Smem_tile_st::Fragment frag_s[Mma_tile_dv::MMAS_K][Mma_tile_dv::MMAS_M]; + smem_s.load(frag_s); + for( int ki = 0; ki < Mma_tile_dv::MMAS_K; ki++ ) { + for( int mi = 0; mi < Mma_tile_dv::MMAS_M; mi++ ) { + for( int ii = 0; ii < Smem_tile_st::Fragment::NUM_REGS; ii++ ) { + frag_s[ki][mi].reg(ii) = fmha::hmul2(frag_s[ki][mi].reg(ii), params.scale_dropout); + frag_s[ki][mi].reg(ii) = fmha::hrelu2(frag_s[ki][mi].reg(ii)); + } + } + } + + gmem_s.store(softmax.elt_, mask); + gmem_s.move(); + + static_assert(Mma_tile_dv::MMAS_K == 1); // DEBUG + #pragma unroll + for( int ki = 1; ki < Mma_tile_dv::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_qt.load(frag_qt[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_dv, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + + // Do the final stage of math. + { + int ki = Mma_tile_dv::MMAS_K; + fmha::gemm(acc_dv, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + // Commit the values for Q into shared memory. + if(l < nl_traits.num_steps_ - 1) { + gmem_q.commit(smem_q); + } + + // Make sure we are reading from the correct buffer. + smem_q.move_to_next_read_buffer(); + smem_qt.move_to_next_read_buffer(); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Trigger the loads for the values of Q for the next iteration. + smem_q.load(frag_q[0], 0); + smem_k.load(frag_k[0], 0); + smem_qt.load(frag_qt[0], 0); + + } // Outer loop over the sequence length. + + // Epilogue for dV = (S * D)' * dout'. We're fully exposed to this! + + // Epilogue swizzle for dV + Smem_tile_dv smem_dv(&smem_[Kernel_traits::Smem_tile_q::BYTES_PER_TILE], tidx); + smem_dv.store(acc_dv); + + __syncthreads(); + + uint4 dv_out[Smem_tile_dv::NUM_LDS]; + smem_dv.load(dv_out); + Qkv_params dv_params; + dv_params.qkv_ptr = params.dkv_ptr; + dv_params.qkv_stride_in_bytes = params.h * 2 * CHUNKS * params.d * sizeof(half); + dv_params.h = params.h; + Gmem_tile_dv gmem_dv(dv_params, nl_traits.get_idx_dv(), binfo, tidx); + gmem_dv.store(dv_out); +} + +template +inline __device__ void compute_dq_dk_1xN_nl(const Params ¶ms) { + + // The description of the CTA tile for the 1st batched GEMM. + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + using Cta_tile_o = typename Kernel_traits::Cta_tile_o; + // The description of the CTA tile for the 2nd batched GEMM. + using Cta_tile_dk = fmha::Cta_tile_extd; + + static_assert(Cta_tile_dk::M == 512 || Cta_tile_dk::M == 384 || Cta_tile_dk::M == 256 || Cta_tile_dk::M == 128); + static_assert(Cta_tile_dk::N == 64); + static_assert(Cta_tile_dk::K == 16); + + // The MMA tile for the 1st GEMM. + using Mma_tile_p = fmha::Hmma_tile; + using Mma_tile_o = fmha::Hmma_tile; + // The MMA tile for the 2nd GEMM. + using Mma_tile_dk = fmha::Hmma_tile; + + // The global memory tile to load Q. + using Gmem_tile_q = typename Kernel_traits::Gmem_tile_q; + // The shared memory tile to swizzle Q. + using Smem_tile_q = typename Kernel_traits::Smem_tile_q; + + // The global memory tile to load K. + using Gmem_tile_k = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle K. + using Smem_tile_k = typename Kernel_traits::Smem_tile_v; // K is used like V in fprop + + // The global memory tile to load V. + using Gmem_tile_v = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle V. + using Smem_tile_v = typename Kernel_traits::Smem_tile_v; + + // The global memory tile to store O. + using Gmem_tile_o = Gmem_tile_dq; + // The shared memory tile to swizzle O. + using Smem_tile_o = typename Kernel_traits::Smem_tile_o; + + // The global memory tile to store dK. + using Gmem_tile_dk = fmha::Gmem_tile_qkv; + + // The shared memory tile to swizzle dK. + using Smem_tile_dk = fmha::Smem_tile_mma_epilogue; + static_assert(Smem_tile_dk::NUM_LDS == Gmem_tile_dk::LDGS); + static_assert(Smem_tile_dk::THREADS_PER_ROW == Gmem_tile_dk::THREADS_PER_ROW); + + // The shared memory tile to reload Q transposed. + using Smem_tile_qt = fmha::Smem_tile_b; + + // The global memory tile to load dP, stored in S + using Gmem_tile_s = Gmem_tile_mma_s; + // The shared memory tile to transpose dP. + using Smem_tile_st = Smem_tile_mma_transposed; + + using Noloop = Noloop_traits; + + enum { M = Mma_tile_p::MMAS_M }; + enum { N = Mma_tile_p::MMAS_N }; + static_assert(M == Mma_tile_o::MMAS_M); + static_assert(N == Mma_tile_o::MMAS_K); + // Shared memory. + extern __shared__ char smem_[]; + + const int bidc = blockIdx.z; + // The block index for the batch. + const int bidb = blockIdx.y; + // The block index for the head. + const int bidh = blockIdx.x; + // The thread index. + const int tidx = threadIdx.x; + + const BlockInfoPadded binfo(params, bidb, bidh, tidx); + if( binfo.stop_early() ) + return; + + fmha::Mask mask(params, binfo, tidx); + + // Allocate the global memory tile loader for Q. + Gmem_tile_q gmem_q(params, 0, binfo, tidx); + // Allocate the shared memory tile loader for Q (as B). + Smem_tile_qt smem_qt(&smem_[0], tidx); + // Allocate the global memory tile loader for dP. + Gmem_tile_s gmem_s(params, binfo, tidx); + // Allocate the shared memory tile loader for dP. + Smem_tile_st smem_s(&smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE + Smem_tile_o::BYTES_PER_TILE], tidx); + + // Allocate the global memory tile loader for K. + Gmem_tile_k gmem_k(params, 1, binfo, tidx); + // Allocate the shared memory tile loader for K. + Smem_tile_k smem_k(&smem_[Smem_tile_q::BYTES_PER_TILE], tidx); + + // Allocate the global memory tile loader for O. + Gmem_tile_o gmem_o(params, binfo, tidx); + // Allocate the shared memory tile loader for O. We use the same as K so be careful!!! + Smem_tile_o smem_o(&smem_[Smem_tile_q::BYTES_PER_TILE + Smem_tile_k::BYTES_PER_TILE], tidx); + + Noloop nl_traits(bidc, binfo); + + nl_traits.move_all(gmem_q, gmem_o, gmem_s); + + // Trigger the loads for Q. + gmem_q.load(smem_qt); + // Trigger the loads for K. + gmem_k.load(smem_k); + + uint4 s_regs[M][N]; + gmem_s.load(s_regs, mask); + + // Commit the data for Q and K to shared memory. + gmem_q.commit(smem_qt); + gmem_k.commit(smem_k); + + // Make sure the data is in shared memory. + __syncthreads(); + + typename Smem_tile_qt::Fragment frag_qt[2][Mma_tile_dk::MMAS_N]; + smem_qt.load(frag_qt[0], 0); + typename Smem_tile_k::Fragment frag_k[2][Mma_tile_o::MMAS_N]; + smem_k.load(frag_k[0], 0); + + enum { BITS_PER_ELT_S = sizeof(fmha::A_type) * 8 }; + + enum { THREADS_PER_ROW = 32 }; + + // Declare the accumulators for the 2nd gemm. + fmha::Fragment_accumulator acc_dk[Mma_tile_dk::MMAS_M][Mma_tile_dk::MMAS_N]; + fmha::Clear_accumulator::apply(acc_dk); + + // Load over the entire sequence length. + for(int l=0;l < nl_traits.num_steps_; l++) { + + // Pack dP as Fragment_a + fmha::Fragment_a frag_p[Mma_tile_o::MMAS_K][Mma_tile_o::MMAS_M]; + #pragma unroll + for( int mi = 0; mi < M; mi++ ) { + #pragma unroll + for( int ni = 0; ni < N; ni++ ) { + uint4 &dst = s_regs[mi][ni]; + frag_p[ni][mi].reg(0) = dst.x; + frag_p[ni][mi].reg(1) = dst.z; + frag_p[ni][mi].reg(2) = dst.y; + frag_p[ni][mi].reg(3) = dst.w; + } + } + smem_s.store(s_regs); + if(l < nl_traits.num_steps_- 1) { + // Load next part of S + gmem_s.move(); + gmem_s.load(s_regs, mask); + // Trigger the load for the next Q values. + smem_qt.move_to_next_write_buffer(); + gmem_q.move(); + gmem_q.load(smem_qt); + } + // Declare the accumulators for the 1st gemm. + fmha::Fragment_accumulator acc_o[Mma_tile_o::MMAS_M][Mma_tile_o::MMAS_N]; + fmha::Clear_accumulator::apply(acc_o); + + // Do this part of O = P^T * V^T. dQ = dP x dK + #pragma unroll + for( int ki = 1; ki < Mma_tile_o::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_k.load(frag_k[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_o, frag_p[ki - 1], frag_k[(ki - 1) & 1]); + } + + // Do the final stage of math. + { + int ki = Mma_tile_o::MMAS_K; + fmha::gemm(acc_o, frag_p[ki - 1], frag_k[(ki - 1) & 1]); + } + + static_assert(Gmem_tile_o::LOOPS == 1); //DEBUG + // Loop over MMAS_M. + #pragma unroll + for( int ii = 0; ii < Gmem_tile_o::LOOPS; ++ii ) { + + // Swizzle the elements and do the final reduction. + smem_o.store(acc_o, ii); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Load from shared memory. + uint4 out[Gmem_tile_o::STGS_PER_LOOP]; + smem_o.load(out); + + // Make sure the data was read from shared memory. + if( ii < Gmem_tile_o::LOOPS - 1 ) { + __syncthreads(); + } + + // Output the values. + gmem_o.store(out, ii); + } + + // Move to the next part of the output. + gmem_o.move(); + + typename Smem_tile_st::Fragment frag_s[Mma_tile_dk::MMAS_K][Mma_tile_dk::MMAS_M]; + smem_s.load(frag_s); + + static_assert(Mma_tile_dk::MMAS_K == 1); // DEBUG + + #pragma unroll + for( int ki = 1; ki < Mma_tile_dk::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + smem_qt.load(frag_qt[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_dk, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + + // Do the final stage of math. + { + int ki = Mma_tile_dk::MMAS_K; + fmha::gemm(acc_dk, frag_s[(ki - 1)], frag_qt[(ki - 1) & 1]); + } + + // Commit the values for Q into shared memory. + if(l < nl_traits.num_steps_- 1) { + gmem_q.commit(smem_qt); + __syncthreads(); + // Trigger the loads for the values of Q for the next iteration. + smem_qt.load(frag_qt[0], 0); + smem_k.load(frag_k[0], 0); + } + + } // Outer loop over the sequence length. + + // Epilogue for dK = dP' * dq. We're fully exposed to this! + + // Epilogue swizzle for dK + Smem_tile_dk smem_dk(&smem_[0], tidx); + smem_dk.store(acc_dk); + + __syncthreads(); + + uint4 dk_out[Smem_tile_dk::NUM_LDS]; + smem_dk.load(dk_out); + Qkv_params dk_params; + dk_params.qkv_ptr = params.dkv_ptr; + dk_params.qkv_stride_in_bytes = params.h * 2 * CHUNKS * params.d * sizeof(half); + dk_params.h = params.h; + Gmem_tile_dk gmem_dk(dk_params, nl_traits.get_idx_dk(), binfo, tidx); + gmem_dk.store(dk_out); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_fill.cu b/apex/apex/contrib/csrc/fmha/src/fmha_fill.cu new file mode 100644 index 00000000..0ee2adbe --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_fill.cu @@ -0,0 +1,69 @@ +/****************************************************************************** + * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include +#include +#include + +constexpr int block_size = 512; +constexpr int ctas_per_sm = 4; + +template +__global__ void +__launch_bounds__(block_size) +mha_fill_kernel(scalar_t* out_tensor, + const int32_t* const start_row, + const size_t num_rows) { + size_t row_stride = gridDim.y * blockDim.x; + size_t row_index = blockIdx.x + (size_t)start_row[0]; + size_t col_index = blockIdx.y * blockDim.x + threadIdx.x; + while (row_index < num_rows) { + out_tensor[row_index*row_stride + col_index] = 0; + row_index += gridDim.x; + } +} + +at::Tensor & mha_fill(at::Tensor &self, const at::Tensor &start_index) { + auto max_tokens = self.size(0); + auto self_2d = self.view({max_tokens, -1}); + auto fcd_size = self_2d.size(1); + TORCH_CHECK (self.is_contiguous(), "input not contiguous"); + TORCH_CHECK (fcd_size % block_size == 0, "input size not aligned to block size"); + const int num_mp = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + uint64_t num_blk_y = (uint64_t)(fcd_size / block_size); + uint64_t num_blk_x = (uint64_t)std::ceil(num_mp * ctas_per_sm / num_blk_y); + dim3 dim_grid(num_blk_x, num_blk_y); + dim3 dim_block(block_size); + + AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, self_2d.scalar_type(), "mha_padding_fill_", [&]() { + mha_fill_kernel<<>>( + self_2d.data_ptr(), start_index.data_ptr(), max_tokens); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + }); + return self; +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_128_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_128_64_kernel.sm80.cu new file mode 100644 index 00000000..4664e46a --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_128_64_kernel.sm80.cu @@ -0,0 +1,79 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_fprop_kernel_1xN.h" + +using Kernel_traits = FMHA_kernel_traits<128, 64, 16, 1, 4, 0x08u>; + +template +__global__ +void fmha_fprop_fp16_128_64_sm80_kernel(Fused_multihead_attention_fprop_params params, + const int total_heads) { + + fmha::device_1xN(params, total_heads); +} + +void run_fmha_fp16_128_64_sm80(Launch_params &launch_params, const bool configure) { + + auto kernel = launch_params.is_training ? &fmha_fprop_fp16_128_64_sm80_kernel : &fmha_fprop_fp16_128_64_sm80_kernel; + + constexpr int smem_size = fmha::get_dynamic_smem_size(); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + + const int sm_count = launch_params.props->multiProcessorCount; + int ctas_per_sm; + FMHA_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, Kernel_traits::THREADS, smem_size)); + int total_ctas = sm_count * ctas_per_sm; + + const int heads_total = launch_params.params.b * launch_params.params.h; + if(configure) { + + using Mma_tile_p = fmha::Hmma_tile; + constexpr size_t STEPS = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + constexpr size_t MMAS_M = Mma_tile_p::MMAS_M; + constexpr size_t MMAS_N = Mma_tile_p::MMAS_N; + + size_t heads_per_cta = ((heads_total + total_ctas - 1) / total_ctas); + size_t elts_per_head = STEPS * MMAS_M * MMAS_N * 8; + launch_params.elts_per_thread = heads_per_cta * elts_per_head; + return; + } + + dim3 grid(total_ctas); + kernel<<>>( + launch_params.params, + heads_total); + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); + +} + + diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_256_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_256_64_kernel.sm80.cu new file mode 100644 index 00000000..34df228e --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_256_64_kernel.sm80.cu @@ -0,0 +1,79 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_fprop_kernel_1xN.h" + +using Kernel_traits = FMHA_kernel_traits<256, 64, 16, 1, 4, 0x08u>; + +template +__global__ +void fmha_fprop_fp16_256_64_sm80_kernel(Fused_multihead_attention_fprop_params params, + const int total_heads) { + + fmha::device_1xN(params, total_heads); +} + +void run_fmha_fp16_256_64_sm80(Launch_params &launch_params, const bool configure) { + + auto kernel = launch_params.is_training ? &fmha_fprop_fp16_256_64_sm80_kernel : &fmha_fprop_fp16_256_64_sm80_kernel; + + constexpr int smem_size = fmha::get_dynamic_smem_size(); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + + const int sm_count = launch_params.props->multiProcessorCount; + int ctas_per_sm; + FMHA_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, Kernel_traits::THREADS, smem_size)); + int total_ctas = sm_count * ctas_per_sm; + + const int heads_total = launch_params.params.b * launch_params.params.h; + if(configure) { + + using Mma_tile_p = fmha::Hmma_tile; + constexpr size_t STEPS = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + constexpr size_t MMAS_M = Mma_tile_p::MMAS_M; + constexpr size_t MMAS_N = Mma_tile_p::MMAS_N; + + size_t heads_per_cta = ((heads_total + total_ctas - 1) / total_ctas); + size_t elts_per_head = STEPS * MMAS_M * MMAS_N * 8; + launch_params.elts_per_thread = heads_per_cta * elts_per_head; + return; + } + + dim3 grid(total_ctas); + kernel<<>>( + launch_params.params, + heads_total); + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); + +} + + diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_384_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_384_64_kernel.sm80.cu new file mode 100644 index 00000000..5e30452a --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_384_64_kernel.sm80.cu @@ -0,0 +1,79 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_fprop_kernel_1xN.h" + +using Kernel_traits = FMHA_kernel_traits<384, 64, 16, 1, 4, 0x18u>; + +template +__global__ +void fmha_fprop_fp16_384_64_sm80_kernel(Fused_multihead_attention_fprop_params params, + const int total_heads) { + + fmha::device_1xN(params, total_heads); +} + +void run_fmha_fp16_384_64_sm80(Launch_params &launch_params, const bool configure) { + + auto kernel = launch_params.is_training ? &fmha_fprop_fp16_384_64_sm80_kernel : &fmha_fprop_fp16_384_64_sm80_kernel; + + constexpr int smem_size = fmha::get_dynamic_smem_size(); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + + const int sm_count = launch_params.props->multiProcessorCount; + int ctas_per_sm; + FMHA_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, Kernel_traits::THREADS, smem_size)); + int total_ctas = sm_count * ctas_per_sm; + + const int heads_total = launch_params.params.b * launch_params.params.h; + if(configure) { + + using Mma_tile_p = fmha::Hmma_tile; + constexpr size_t STEPS = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + constexpr size_t MMAS_M = Mma_tile_p::MMAS_M; + constexpr size_t MMAS_N = Mma_tile_p::MMAS_N; + + size_t heads_per_cta = ((heads_total + total_ctas - 1) / total_ctas); + size_t elts_per_head = STEPS * MMAS_M * MMAS_N * 8; + launch_params.elts_per_thread = heads_per_cta * elts_per_head; + return; + } + + dim3 grid(total_ctas); + kernel<<>>( + launch_params.params, + heads_total); + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); + +} + + diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_512_64_kernel.sm80.cu b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_512_64_kernel.sm80.cu new file mode 100644 index 00000000..e37689e8 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_fp16_512_64_kernel.sm80.cu @@ -0,0 +1,137 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" +#include "fmha_fprop_kernel_1xN.h" + +using Kernel_traits = FMHA_kernel_traits<512, 64, 16, 1, 8, 0x00u>; + +template +__global__ +void fmha_fprop_fp16_512_64_sm80_kernel(Fused_multihead_attention_fprop_params params, + const int total_heads) { + + fmha::device_1xN(params, total_heads); +} + +template +__global__ +void fmha_fprop_fp16_512_64_sm80_kernel_nl(Fused_multihead_attention_fprop_params params, + const int num_full_heads, + const int num_main_groups, + const int main_group_size, + const int main_steps, + const int rest_steps) { + + fmha::device_1xN( + params, num_full_heads, num_main_groups, main_group_size, main_steps, rest_steps); +} + +void run_fmha_fp16_512_64_sm80_(Launch_params &launch_params, const bool configure) { + + auto kernel = launch_params.is_training ? &fmha_fprop_fp16_512_64_sm80_kernel : &fmha_fprop_fp16_512_64_sm80_kernel; + + constexpr int smem_size = fmha::get_dynamic_smem_size(); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + + const int sm_count = launch_params.props->multiProcessorCount; + int ctas_per_sm; + FMHA_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, Kernel_traits::THREADS, smem_size)); + int total_ctas = sm_count * ctas_per_sm; + + const int heads_total = launch_params.params.b * launch_params.params.h; + if(configure) { + + using Mma_tile_p = fmha::Hmma_tile; + constexpr size_t STEPS = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + constexpr size_t MMAS_M = Mma_tile_p::MMAS_M; + constexpr size_t MMAS_N = Mma_tile_p::MMAS_N; + + size_t heads_per_cta = ((heads_total + total_ctas - 1) / total_ctas); + size_t elts_per_head = STEPS * MMAS_M * MMAS_N * 8; + launch_params.elts_per_thread = heads_per_cta * elts_per_head; + return; + } + + dim3 grid(total_ctas); + kernel<<>>( + launch_params.params, + heads_total); + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); + +} + +void run_fmha_fp16_512_64_sm80_nl_(Launch_params &launch_params, const bool configure) { + + auto kernel = launch_params.is_training ? &fmha_fprop_fp16_512_64_sm80_kernel_nl : &fmha_fprop_fp16_512_64_sm80_kernel_nl; + + constexpr int smem_size = fmha::get_dynamic_smem_size(); + + if( smem_size >= 48 * 1024 ) { + FMHA_CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + } + + const int sm_count = launch_params.props->multiProcessorCount; + int ctas_per_sm; + FMHA_CHECK_CUDA(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&ctas_per_sm, kernel, Kernel_traits::THREADS, smem_size)); + int total_ctas = sm_count * ctas_per_sm; + + if(configure) { + const int heads_total = launch_params.params.b * launch_params.params.h; + std::tie(launch_params.num_full_heads, + launch_params.num_main_groups, + launch_params.heads_last_wave, + launch_params.main_steps, + launch_params.rest_steps, + launch_params.elts_per_thread) = fmha::work_dist(total_ctas, heads_total); + return; + } + + dim3 grid(total_ctas); + kernel<<>>( + launch_params.params, + launch_params.num_full_heads, + launch_params.num_main_groups, + launch_params.heads_last_wave, + launch_params.main_steps, + launch_params.rest_steps); + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); + +} + +void run_fmha_fp16_512_64_sm80(Launch_params &launch_params, const bool configure) { + if( launch_params.is_nl ) { + run_fmha_fp16_512_64_sm80_nl_(launch_params, configure); + } else { + run_fmha_fp16_512_64_sm80_(launch_params, configure); + } +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_fprop_kernel_1xN.h b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_kernel_1xN.h new file mode 100644 index 00000000..3c3e8ba1 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_fprop_kernel_1xN.h @@ -0,0 +1,531 @@ +/*************************************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include "fmha_kernel.h" +#include +#include + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Gemm_Q_K_base { + using Smem_tile_o = typename Kernel_traits::Smem_tile_o; + using Smem_tile_q = typename Kernel_traits::Smem_tile_q; + using Smem_tile_k = typename Kernel_traits::Smem_tile_k; + using Fragment_q = typename Smem_tile_q::Fragment; + using Fragment_k = typename Smem_tile_k::Fragment; + + // The description of the CTA tile for the 1st batched GEMM. + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + + // The MMA tile for the 1st GEMM. + using Mma_tile_p = fmha::Hmma_tile; + + static constexpr int SMEM_BYTES_SOFTMAX = Cta_tile_p::M * Cta_tile_p::WARPS_N * sizeof(float) * 2; + + __device__ inline Gemm_Q_K_base(char * smem_ptr_q, char * smem_ptr_k, const int tidx) + : smem_q(smem_ptr_q, tidx) + , smem_k(smem_ptr_k, tidx) { + + } + + __device__ inline void load_q() { + smem_q.load(frag_q[0], 0); + } + + __device__ inline void reload_q() { + smem_q.load(frag_q[0], 0); + } + + Fragment_q frag_q[2][Mma_tile_p::MMAS_M]; + Smem_tile_q smem_q; + Smem_tile_k smem_k; +}; + +template +struct Gemm_Q_K : public Gemm_Q_K_base { + + using Base = Gemm_Q_K_base; + using Smem_tile_o = typename Base::Smem_tile_o; + using Smem_tile_q = typename Base::Smem_tile_q; + using Smem_tile_k = typename Base::Smem_tile_k; + using Fragment_k = typename Base::Fragment_k; + using Mma_tile_p = typename Base::Mma_tile_p; + + enum { SHARE_SMEM_FOR_K_AND_V = Kernel_traits::SHARE_SMEM_FOR_K_AND_V }; + + enum { SMEM_OFFSET_O = Smem_tile_q::BYTES_PER_TILE }; + enum { SMEM_OFFSET_V = Smem_tile_q::BYTES_PER_TILE + (SHARE_SMEM_FOR_K_AND_V ? 0 : Smem_tile_k::BYTES_PER_TILE) }; + + // Q | K / V + // | O | SOFTMAX + static constexpr int SMEM_BYTES = Smem_tile_q::BYTES_PER_TILE + + std::max((SHARE_SMEM_FOR_K_AND_V ? 1 : 2) * Smem_tile_k::BYTES_PER_TILE, + Smem_tile_o::BYTES_PER_TILE + Base::SMEM_BYTES_SOFTMAX); + + __device__ inline Gemm_Q_K(char * smem_, const int tidx) + : Base(smem_, smem_ + Smem_tile_q::BYTES_PER_TILE, tidx) { + } + + __device__ inline void load_k(){ + #pragma unroll + for( int ki = 0; ki < Mma_tile_p::MMAS_K; ++ki ) { + Base::smem_k.load(frag_k[ki], ki); + } + } + + template + __device__ inline void operator()(Acc (&acc_p)[M][N]){ + // Do this part of P^T = (Q * K^T)^T. + #pragma unroll + for( int ki = 1; ki < Mma_tile_p::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + Base::smem_q.load(Base::frag_q[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_p, Base::frag_q[(ki - 1) & 1], frag_k[(ki - 1)]); + } + // Do the final stage of math. + { + int ki = Mma_tile_p::MMAS_K; + fmha::gemm(acc_p, Base::frag_q[(ki - 1) & 1], frag_k[(ki - 1)]); + } + } + + __device__ inline void reload_k(){ + // Noop. + } + + Fragment_k frag_k[Mma_tile_p::MMAS_K][Mma_tile_p::MMAS_N]; +}; + + +template +struct Gemm_Q_K : public Gemm_Q_K_base { + using Base = Gemm_Q_K_base; + using Smem_tile_o = typename Base::Smem_tile_o; + using Smem_tile_q = typename Base::Smem_tile_q; + using Smem_tile_k = typename Base::Smem_tile_k; + using Smem_tile_v = typename Kernel_traits::Smem_tile_v; + using Fragment_k = typename Base::Fragment_k; + using Mma_tile_p = typename Base::Mma_tile_p; + Fragment_k frag_k[2][Mma_tile_p::MMAS_N]; + + enum { SHARE_SMEM_FOR_K_AND_V = Kernel_traits::SHARE_SMEM_FOR_K_AND_V }; + + enum { SMEM_OFFSET_V = Smem_tile_q::BYTES_PER_TILE + (SHARE_SMEM_FOR_K_AND_V ? 0 : Smem_tile_k::BYTES_PER_TILE) }; + static_assert(Smem_tile_v::BYTES_PER_TILE == (int) Smem_tile_k::BYTES_PER_TILE); + enum { SMEM_OFFSET_O = SMEM_OFFSET_V + Smem_tile_v::BYTES_PER_TILE }; + + // Q | K/V + O + SOFTMAX + static constexpr int SMEM_BYTES = Smem_tile_q::BYTES_PER_TILE + + (SHARE_SMEM_FOR_K_AND_V ? 1 : 2) * Smem_tile_k::BYTES_PER_TILE + + Smem_tile_o::BYTES_PER_TILE + Base::SMEM_BYTES_SOFTMAX; + + __device__ inline Gemm_Q_K(char * smem_, const int tidx) + : Base(smem_, smem_ + Smem_tile_q::BYTES_PER_TILE, tidx) { + } + + __device__ inline void load_k(){ + Base::smem_k.load(frag_k[0], 0); + } + + template + __device__ inline void operator()(Acc (&acc_p)[M][N]){ + // Do this part of P^T = (Q * K^T)^T. + #pragma unroll + for( int ki = 1; ki < Mma_tile_p::MMAS_K; ++ki ) { + // Trigger the load from shared memory for the next series of Q values. + Base::smem_q.load(Base::frag_q[ki & 1], ki); + Base::smem_k.load(frag_k[ki & 1], ki); + // Do the math for the values already in registers. + fmha::gemm(acc_p, Base::frag_q[(ki - 1) & 1], frag_k[(ki - 1) & 1]); + } + // Do the final stage of math. + { + int ki = Mma_tile_p::MMAS_K; + fmha::gemm(acc_p, Base::frag_q[(ki - 1) & 1], frag_k[(ki - 1) & 1]); + } + } + + __device__ inline void reload_k(){ + Base::smem_k.load(frag_k[0], 0); + } +}; + +template +constexpr size_t get_dynamic_smem_size(){ + return Gemm_Q_K::SMEM_BYTES; +} + +template +inline __device__ void device_1xN_(const Params ¶ms, const int bidb, const int bidh, const int begin, const int steps, Prng & ph) { + + + // The description of the CTA tile for the 1st batched GEMM. + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + // The description of the CTA tile for the 2nd batched GEMM. + using Cta_tile_o = typename Kernel_traits::Cta_tile_o; + + // The MMA tile for the 1st GEMM. + using Mma_tile_p = fmha::Hmma_tile; + // The MMA tile for the 2nd GEMM. + using Mma_tile_o = fmha::Hmma_tile; + + // The global memory tile to load Q. + using Gmem_tile_q = typename Kernel_traits::Gmem_tile_q; + + // The global memory tile to load K. + using Gmem_tile_k = typename Kernel_traits::Gmem_tile_k; + + // The global memory tile to load V. + using Gmem_tile_v = typename Kernel_traits::Gmem_tile_v; + // The shared memory tile to swizzle V. + using Smem_tile_v = typename Kernel_traits::Smem_tile_v; + + // The global memory tile to store O. + using Gmem_tile_o = typename Kernel_traits::Gmem_tile_o; + // The shared memory tile to swizzle O. + using Smem_tile_o = typename Kernel_traits::Smem_tile_o; + + using Gmem_tile_s = typename Kernel_traits::Gmem_tile_s; + + using Gemm1 = Gemm_Q_K; + + using Softmax = fmha::Softmax; + + + // The number of threads per row. + enum { THREADS_PER_ROW = 32 }; + + enum { BITS_PER_ELT_S = sizeof(fmha::A_type) * 8 }; + + // Shared memory. + extern __shared__ char smem_[]; + + // The thread index. + const int tidx = threadIdx.x; + + const BlockInfoPadded binfo(params, bidb, bidh, tidx); + if( binfo.stop_early() ) return; + + Gemm1 gemm_q_k(smem_, tidx); + // Allocate the global memory tile loader for Q. + Gmem_tile_q gmem_q(params, 0, binfo, tidx); + // Allocate the global memory tile loader for O. + Gmem_tile_o gmem_o(params, binfo, tidx); + // Allocate the global memory tile loader for S. + Gmem_tile_s gmem_s(params, binfo, tidx); + // Wind gmem tiles to the correct position. + for( int it = 0; it < begin; it++ ) { + gmem_q.move(); + gmem_s.move(); + gmem_o.move(); + } + + fmha::Mask mask(params, binfo, tidx); + + // Allocate the global memory tile loader for K. + Gmem_tile_k gmem_k(params, 1, binfo, tidx); + // Allocate the global memory tile loader for V. + Gmem_tile_v gmem_v(params, 2, binfo, tidx); + // The base pointer of smem_v; + char *smem_v_ = &smem_[Gemm1::SMEM_OFFSET_V]; + + // Allocate the shared memory tile loader for V. We use the same as K so be careful!!! + Smem_tile_v smem_v(smem_v_, tidx); + + // Allocate the shared memory tile loader for O. We use the same as K so be careful!!! + Smem_tile_o smem_o(&smem_[Gemm1::SMEM_OFFSET_O], tidx); + + // Trigger the loads for K. + gmem_k.load(gemm_q_k.smem_k); + // Trigger the loads for Q. + gmem_q.load(gemm_q_k.smem_q); + // Trigger the loads for V. + gmem_v.load(smem_v); + + const uint32_t scale_bmm1 = reinterpret_cast(params.scale_bmm1); + #pragma unroll + for(int it=0;it < Gmem_tile_k::LDGS;it++){ + gmem_k.fetch_[it] = fmha::hmul8(scale_bmm1, gmem_k.fetch_[it]); + } + + + + // Commit the data for Q and V to shared memory. + gmem_q.commit(gemm_q_k.smem_q); + gmem_v.commit(smem_v); + + // Commit the data for K to shared memory. + if( !Kernel_traits::SHARE_SMEM_FOR_K_AND_V ) { + gmem_k.commit(gemm_q_k.smem_k); + } + + __syncthreads(); + + // Load the fragments for Q. + gemm_q_k.load_q(); + + // Load the fragments for V. We keep the data in registers during the entire kernel. + typename Smem_tile_v::Fragment frag_v[Mma_tile_o::MMAS_K][Mma_tile_o::MMAS_N]; + #pragma unroll + for( int ki = 0; ki < Mma_tile_o::MMAS_K; ++ki ) { + smem_v.load(frag_v[ki], ki); + } + + // Commit the data for V to shared memory if it has not been done already. + if( Kernel_traits::SHARE_SMEM_FOR_K_AND_V ) { + // Make sure we are done loading the fragments for K. + __syncthreads(); + + // Commit the data to shared memory for V. + gmem_k.commit(gemm_q_k.smem_k); + + // Make sure the data is in shared memory. + __syncthreads(); + } + + // Load the fragments for K. + gemm_q_k.load_k(); + uint32_t p_scaled = (uint32_t) 256.0 * params.p_dropout; + + // Create the object to do the softmax. + Softmax softmax(params, &smem_[Gemm1::SMEM_OFFSET_O + Smem_tile_o::BYTES_PER_TILE], bidb, tidx); + + // Load over the entire sequence length. + for( int l = 0; l < steps; l++ ) { + if(begin + l * Cta_tile_p::M >= binfo.actual_seqlen) break; + + // Declare the accumulators for the 1st gemm. + fmha::Fragment_accumulator acc_p[Mma_tile_p::MMAS_M][Mma_tile_p::MMAS_N]; + fmha::Clear_accumulator::apply(acc_p); + + // Do this part of P^T = (Q * K^T)^T. + gemm_q_k(acc_p); + + // Trigger the load for the next Q values. + if( l < steps - 1) { + gemm_q_k.smem_q.move_to_next_write_buffer(); + gmem_q.move(); + gmem_q.load(gemm_q_k.smem_q); + } + + // Load the mask for that iteration. + mask.load(begin + l); + + // Convert from the accumulator type to FP32 for Softmax. + softmax.unpack_noscale(acc_p); + + // Apply the mask. + softmax.apply_mask(mask); + + if( Kernel_traits::SHARE_SMEM_FOR_K_AND_V && l == 0 ) { + // if we share K and V, it could be that V was not fully read yet but we write into smem for reduction + __syncthreads(); + } + // Compute the max. + float p_max[Mma_tile_p::MMAS_M * 2]; + //softmax.template reduce(p_max); + softmax.reduce_max(p_max); + + // Compute the exponential value. + softmax.apply_exp(p_max); + + // Compute the sum. + float p_sum[Mma_tile_p::MMAS_M * 2]; + softmax.reduce_sum(p_sum); + + // Finalize softmax on the accumulators of P^T. + softmax.scale(p_sum); + + using Frag_p = fmha::Fragment_a; + Frag_p frag_p[Mma_tile_o::MMAS_K][Mma_tile_o::MMAS_M]; + if( Is_training ) { + auto encode_dropout = [](bool keep, float val) { return keep ? val : -val; }; + #pragma unroll + for( int mi = 0; mi < Mma_tile_p::MMAS_M; mi++ ) { + #pragma unroll + for( int ii = 0; ii < 2; ii++ ) { + #pragma unroll + for( int ni = 0; ni < Mma_tile_p::MMAS_N/4; ni++ ) { + uint8_t * rand_arr = (uint8_t*) &ph(); + // We encode the dropout pattern in the sign bit of the non-negative softmax to distinguish from pre-existing zeros + for (int ind=0; ind<16; ind++) + { + softmax.elt_[2 * mi + ii][16 * ni + ind] = + encode_dropout(rand_arr[ind] <= p_scaled, softmax.elt_[2 * mi + ii][16 * ni + ind]); + } + } + } + } + softmax.pack(frag_p); + gmem_s.store(frag_p, mask); + gmem_s.move(); + } else { + softmax.pack(frag_p); + } + + // Commit the values for Q into shared memory. + if(l < steps - 1) { + gmem_q.commit(gemm_q_k.smem_q); + } + + if( Is_training ) { + #pragma unroll + for( int ki = 0; ki < Mma_tile_o::MMAS_K; ki++ ) { + #pragma unroll + for( int mi = 0; mi < Mma_tile_o::MMAS_M; mi++ ) { + #pragma unroll + for( int ii = 0; ii < Frag_p::NUM_REGS; ii++ ) { + //"Apply" the dropout. + frag_p[ki][mi].reg(ii) = fmha::hmul2(frag_p[ki][mi].reg(ii), params.scale_dropout); + frag_p[ki][mi].reg(ii) = fmha::hrelu2(frag_p[ki][mi].reg(ii)); + } + } + } + } + + // Declare the accumulators for the 1st gemm. + fmha::Fragment_accumulator acc_o[Mma_tile_o::MMAS_M][Mma_tile_o::MMAS_N]; + fmha::Clear_accumulator::apply(acc_o); + + // Do this part of O = P^T * V^T. + #pragma unroll + for( int ki = 0; ki < Mma_tile_o::MMAS_K; ++ki ) { + fmha::gemm(acc_o, frag_p[ki], frag_v[ki]); + } + + // Loop over MMAS_M. + #pragma unroll + for( int ii = 0; ii < Gmem_tile_o::LOOPS; ++ii ) { + + // Swizzle the elements and do the final reduction. + smem_o.store(acc_o, ii); + + // Make sure the data is in shared memory. + __syncthreads(); + + // Load from shared memory. + uint4 out[Gmem_tile_o::STGS_PER_LOOP]; + smem_o.load(out); + + // Make sure the data was read from shared memory. + if( ii < Gmem_tile_o::LOOPS - 1 ) { + __syncthreads(); + } + + // Output the values. + gmem_o.store(out, ii); + } + + // Move to the next part of the output. + gmem_o.move(); + gemm_q_k.reload_k(); + + // Commit the values for Q into shared memory. + if(l < steps - 1) { + gemm_q_k.reload_q(); + } + + } // Outer loop over the sequence length. +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ void device_1xN(const Params ¶ms, + const int num_full_heads, + const int num_main_groups, + const int main_group_size, + const int main_steps, + const int rest_steps) { + + constexpr int STEPS = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + const int tidx_global = blockIdx.x * gridDim.x + threadIdx.x; + auto seeds = at::cuda::philox::unpack(params.philox_args); + Philox ph(std::get<0>(seeds), tidx_global, std::get<1>(seeds)); + for( int it = 0; it < num_full_heads; it++ ) { + const int bidx = it * gridDim.x + blockIdx.x; + const int bidh = bidx % params.h; + const int bidb = bidx / params.h; + fmha::device_1xN_(params, bidb, bidh, 0, STEPS, ph); + __syncthreads(); + } + if( main_group_size == 0 ) + return; + const int head_offset = num_full_heads * gridDim.x; + + if( blockIdx.x < main_group_size * num_main_groups ) { + // process within heads + const int group = blockIdx.x % num_main_groups; + const int bidx = blockIdx.x / num_main_groups; + const int bidh = (head_offset + bidx) % params.h; + const int bidb = (head_offset + bidx) / params.h; + const int offset = group * main_steps; + fmha::device_1xN_(params, bidb, bidh, offset, main_steps, ph); + } else { + if(rest_steps == 0 ) return; + // process across heads + const int bidx = blockIdx.x - main_group_size * num_main_groups; + const int offset = num_main_groups * main_steps; + const int total_heads = params.b * params.h; + const int rest_ctas = gridDim.x - main_group_size * num_main_groups; + for( int it = head_offset + bidx; it < total_heads; it += rest_ctas ) { + const int bidh = it % params.h; + const int bidb = it / params.h; + fmha::device_1xN_(params, bidb, bidh, offset, rest_steps, ph); + __syncthreads(); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ void device_1xN(const Params ¶ms, const int total_heads) { + + const int tidx_global = blockIdx.x * gridDim.x + threadIdx.x; + auto seeds = at::cuda::philox::unpack(params.philox_args); + Philox ph(std::get<0>(seeds), tidx_global, std::get<1>(seeds)); + constexpr int STEPS = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + + for(int bidx = blockIdx.x; bidx < total_heads; bidx += gridDim.x){ + const int bidh = bidx % params.h; + const int bidb = bidx / params.h; + fmha::device_1xN_(params, bidb, bidh, 0, STEPS, ph); + __syncthreads(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha + diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_kernel.h b/apex/apex/contrib/csrc/fmha/src/fmha_kernel.h new file mode 100644 index 00000000..63180b08 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_kernel.h @@ -0,0 +1,179 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace fmha { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct BlockInfoPadded { + + template + __device__ BlockInfoPadded(const Params ¶ms, + const int bidb, + const int bidh, + const int tidx) + : bidb(bidb), bidh(bidh), h(params.h) { + + // The block index. + sum_s = params.cu_seqlens[bidb]; + actual_seqlen = params.cu_seqlens[bidb + 1] - sum_s; + bidx = sum_s * params.h + bidh; + + tidx_global = (bidb * params.h + bidh) * THREADS_PER_CTA + tidx; + } + + __device__ bool stop_early() const { + return actual_seqlen == 0; + } + + int actual_seqlen; + int bidx; + int sum_s; + int bidh; + int bidb; + int tidx_global; + int h; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Noloop_traits{ + // Interpretation of Cta_tile dims, i.e. Cta_tile_p: + enum{ STEP = Cta_tile::M }; + enum{ SEQLEN = Cta_tile::N }; + + template + inline __device__ Noloop_traits(const int bidc, const Block_info& binfo) + : bidc_(bidc) { + const int seqlen = binfo.actual_seqlen; + const int steps = (seqlen + STEP - 1) / STEP; + const int steps_per_chunk = (steps + CHUNKS - 1) / CHUNKS; + + const int step_begin = bidc_ * steps_per_chunk; + const int step_end = min(steps, (bidc_ + 1) * steps_per_chunk); + const int actual_steps = max(0, step_end - step_begin); + loop_offset_ = step_begin; + num_steps_ = actual_steps; + + } + + template + inline __device__ void move_all(Tiles & ... tiles) const { + using expand_type = int[]; + for( int s = 0; s < loop_offset_; s++ ) { + expand_type{ (tiles.move(), 0)... }; + } + } + + inline __device__ int get_idx_dk() const { + //return bidc_; + return bidc_ * 2 + 0; + } + + inline __device__ int get_idx_dv() const { + //return CHUNKS + bidc_; + return bidc_ * 2 + 1; + } + + inline __device__ int offset_loop_count(const int l) { + // convert loop counter to position in the outer sequence + return (loop_offset_ + l) * STEP; + } + + const uint32_t bidc_; + int loop_offset_; + int num_steps_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +std::tuple work_dist(const int total_ctas, const int heads_total) { + + constexpr int STEPS_PER_HEAD = Kernel_traits::Cta_tile_p::N / Kernel_traits::Cta_tile_p::M; + + const int num_full_heads = heads_total / total_ctas; + const int heads_last_wave = heads_total % total_ctas; + + int num_main_groups = 0; + int main_steps = 0; + int rest_steps = 0; + if( heads_last_wave > 0 ) { + // Number of CTA groups that process within heads. + num_main_groups = total_ctas / heads_last_wave; + // Remaining CTAs that process between heads. + const int rest_ctas = total_ctas - (heads_last_wave * num_main_groups); + if(rest_ctas == 0) { + // We have exactly "num_main_groups" CTAs to process each of the remaining heads. + main_steps = (STEPS_PER_HEAD + num_main_groups - 1) / num_main_groups; + num_main_groups = STEPS_PER_HEAD / main_steps; // Here: main_step > 0 + rest_steps = STEPS_PER_HEAD % main_steps; + + } else { + // Ideal number of steps if we could load-balance as evenly as possible. + const int steps_ideal = (heads_last_wave * STEPS_PER_HEAD + total_ctas - 1) / total_ctas; + // Iterations that a "rest" CTA has to do at most. + const int max_rest_iters = (heads_last_wave + rest_ctas - 1) / rest_ctas; + // Find the first step distribution, s.t. the maximum work of the "rest" CTAs is less than the work of the main CTAs. + main_steps = steps_ideal; + rest_steps = STEPS_PER_HEAD - main_steps * num_main_groups; + for( ; main_steps * num_main_groups < STEPS_PER_HEAD; main_steps++ ) { + rest_steps = STEPS_PER_HEAD - main_steps * num_main_groups; + const int max_rest_total_steps = rest_steps * max_rest_iters; + if( max_rest_total_steps < main_steps ) + break; + } + rest_steps = STEPS_PER_HEAD - main_steps * num_main_groups; + } + } + + using Cta_tile_p = typename Kernel_traits::Cta_tile_p; + using Mma_tile_p = fmha::Hmma_tile; + + const int max_steps = STEPS_PER_HEAD * num_full_heads + std::max(main_steps, rest_steps); + const int elts_per_thread_per_step = Mma_tile_p::MMAS_M * Mma_tile_p::MMAS_N * 8; + const int elts_per_thread = max_steps * elts_per_thread_per_step; + + return {num_full_heads, num_main_groups, heads_last_wave, main_steps, rest_steps, elts_per_thread}; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace fmha diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_noloop_reduce.cu b/apex/apex/contrib/csrc/fmha/src/fmha_noloop_reduce.cu new file mode 100644 index 00000000..8e4b9efc --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_noloop_reduce.cu @@ -0,0 +1,177 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#include "fmha.h" + +inline __device__ float4 ldg128(const void *ptr) { + return *static_cast(ptr); +} + +inline __device__ void stg128(void *ptr, const float4 &data) { + *static_cast(ptr) = data; +} + +template +__global__ __launch_bounds__(THREADS) void fmha_noloop_reduce_kernel(void *__restrict__ out, + const void *__restrict__ in, + const int *__restrict__ cu_seqlens, + const int batch_size) { + + enum { BYTES_PER_LDG = 16 }; + enum { NUM_ELTS = BYTES_PER_LDG / sizeof(T) }; + + // One CTA hidden vector for K and V + enum { BYTES_PER_ROW = HIDDEN_SIZE * sizeof(T) * 2 }; + // The stride in bytes in dQKV + enum { OUT_STRIDE_BYTES = 3 * HIDDEN_SIZE * sizeof(T) }; + // The offset in bytes in dQKV to the dKV part for non-interleaved heads + enum { OUT_OFFSET_KV_BYTES = HIDDEN_SIZE * sizeof(T) }; + + static_assert(BYTES_PER_ROW == HIDDEN_SIZE * 2 * sizeof(T)); + + // Size in bytes of the input tile + enum { BYTES_PER_TILE = CHUNKS * BYTES_PER_ROW }; + + enum { BYTES_PER_CTA = THREADS * BYTES_PER_LDG }; + + enum { LDGS = BYTES_PER_ROW / BYTES_PER_CTA }; + static_assert(BYTES_PER_CTA * LDGS == BYTES_PER_ROW); + + union Vec_t { + float4 raw; + T elt[NUM_ELTS]; + }; + + // ZERO-OUT invalid positions in dQKV + const int total = cu_seqlens[batch_size]; + if(blockIdx.x >= total){ + enum { BYTES_PER_QKV_ROW = 3 * HIDDEN_SIZE * sizeof(T) }; + enum { STGS = BYTES_PER_QKV_ROW / BYTES_PER_LDG }; + + const float4 zeros = make_float4(0.f, 0.f, 0.f, 0.f); + + char *base_ptr = static_cast(out) + blockIdx.x * OUT_STRIDE_BYTES; + + for(int tidx = threadIdx.x; tidx < STGS; tidx += THREADS){ + stg128(base_ptr + tidx * BYTES_PER_LDG, zeros); + } + + return; + } + + // SETUP + const int offset_in = blockIdx.x * BYTES_PER_TILE + threadIdx.x * BYTES_PER_LDG; + const char *ptr_in = static_cast(in) + offset_in; + + const int offset_out = blockIdx.x * OUT_STRIDE_BYTES + threadIdx.x * BYTES_PER_LDG; + char *ptr_out = static_cast(out) + OUT_OFFSET_KV_BYTES + offset_out; + + // LOAD + + Vec_t local_in[CHUNKS][LDGS]; + + #pragma unroll + for( int c = 0; c < CHUNKS; c++ ) { + #pragma unroll + for( int l = 0; l < LDGS; l++ ) { + int offset = c * BYTES_PER_ROW + l * BYTES_PER_CTA; + local_in[c][l].raw = ldg128(ptr_in + offset); + } + } + + // UNPACK + float acc[LDGS][NUM_ELTS]; + + #pragma unroll + for( int l = 0; l < LDGS; l++ ) { + #pragma unroll + for( int e = 0; e < NUM_ELTS; e++ ) { + acc[l][e] = float(local_in[0][l].elt[e]); + } + } + + // COMPUTE + #pragma unroll + for( int c = 1; c < CHUNKS; c++ ) { + #pragma unroll + for( int l = 0; l < LDGS; l++ ) { + #pragma unroll + for( int e = 0; e < NUM_ELTS; e++ ) { + acc[l][e] += float(local_in[c][l].elt[e]); + } + } + } + + // PACK + Vec_t local_out[LDGS]; + + #pragma unroll + for( int l = 0; l < LDGS; l++ ) { + #pragma unroll + for( int e = 0; e < NUM_ELTS; e++ ) { + local_out[l].elt[e] = T(acc[l][e]); + } + } + + // STORE + #pragma unroll + for( int l = 0; l < LDGS; l++ ) { + const int offset = l * BYTES_PER_CTA; + stg128(ptr_out + offset, local_out[l].raw); + } +} + +void fmha_run_noloop_reduce(void *out, + const void *in, + const int *cu_seqlens, + const int hidden_size, + const int batch_size, + const int total, + const int num_chunks, + cudaStream_t stream) { + + const int blocks = total; + + if(hidden_size == 1024){ + + constexpr int HIDDEN_SIZE = 1024; + constexpr int THREADS = 256; + + if( num_chunks == 2 ) { + fmha_noloop_reduce_kernel<<>>(out, in, cu_seqlens, batch_size); + } else if( num_chunks == 3 ) { + fmha_noloop_reduce_kernel<<>>(out, in, cu_seqlens, batch_size); + } else { + assert(false && "Unsupported num_chunks"); + } + + }else{ + assert(false && "Unsupported hidden_size"); + } + + FMHA_CHECK_CUDA(cudaPeekAtLastError()); +} diff --git a/apex/apex/contrib/csrc/fmha/src/fmha_utils.h b/apex/apex/contrib/csrc/fmha/src/fmha_utils.h new file mode 100644 index 00000000..de07cc78 --- /dev/null +++ b/apex/apex/contrib/csrc/fmha/src/fmha_utils.h @@ -0,0 +1,92 @@ +/****************************************************************************** + * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define FMHA_CHECK_CUDA( call ) \ + do { \ + cudaError_t status_ = call; \ + if( status_ != cudaSuccess ) { \ + fprintf( stderr, \ + "CUDA error (%s:%d): %s\n", \ + __FILE__, \ + __LINE__, \ + cudaGetErrorString( status_ ) ); \ + exit( 1 ); \ + } \ + } while( 0 ) + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +enum Data_type { DATA_TYPE_FP16, DATA_TYPE_FP32, DATA_TYPE_INT32, DATA_TYPE_INT8 }; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline void set_alpha( uint32_t &alpha, float norm, Data_type dtype ) { + if( dtype == DATA_TYPE_FP16 ) { + half x = __float2half_rn( norm ); + uint16_t h = reinterpret_cast( x ); + ushort2 h2 = { h, h }; + alpha = reinterpret_cast( h2 ); + } else if( dtype == DATA_TYPE_FP32 ) { + alpha = reinterpret_cast( norm ); + } else if( dtype == DATA_TYPE_INT32 ) { + int32_t inorm = static_cast( norm ); + alpha = reinterpret_cast( inorm ); + } else { + assert( false ); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline size_t get_size_in_bytes( size_t n, Data_type dtype ) { + switch( dtype ) { + case DATA_TYPE_FP32: + return n * 4; + case DATA_TYPE_FP16: + return n * 2; + case DATA_TYPE_INT32: + return n * 4; + case DATA_TYPE_INT8: + return n; + default: + assert( false ); + return 0; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + diff --git a/apex/apex/contrib/csrc/focal_loss/focal_loss_cuda.cpp b/apex/apex/contrib/csrc/focal_loss/focal_loss_cuda.cpp new file mode 100644 index 00000000..8f39e18f --- /dev/null +++ b/apex/apex/contrib/csrc/focal_loss/focal_loss_cuda.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include + +// CUDA forward declarations + +std::vector focal_loss_forward_cuda( + const at::Tensor &cls_output, + const at::Tensor &cls_targets_at_level, + const at::Tensor &num_positives_sum, + const int64_t num_real_classes, + const float alpha, + const float gamma, + const float smoothing_factor); + +at::Tensor focal_loss_backward_cuda( + const at::Tensor &grad_output, + const at::Tensor &partial_grad, + const at::Tensor &num_positives_sum); + +// C++ interface + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) + +std::vector focal_loss_forward( + const at::Tensor &cls_output, + const at::Tensor &cls_targets_at_level, + const at::Tensor &num_positives_sum, + const int64_t num_real_classes, + const float alpha, + const float gamma, + const float smoothing_factor +) { + CHECK_INPUT(cls_output); + CHECK_INPUT(cls_targets_at_level); + CHECK_INPUT(num_positives_sum); + + return focal_loss_forward_cuda( + cls_output, + cls_targets_at_level, + num_positives_sum, + num_real_classes, + alpha, + gamma, + smoothing_factor); +} + +at::Tensor focal_loss_backward( + const at::Tensor &grad_output, + const at::Tensor &partial_grad, + const at::Tensor &num_positives_sum +) { + CHECK_INPUT(grad_output); + CHECK_INPUT(partial_grad); + + return focal_loss_backward_cuda(grad_output, partial_grad, num_positives_sum); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &focal_loss_forward, + "Focal loss calculation forward (CUDA)"); + m.def("backward", &focal_loss_backward, + "Focal loss calculation backward (CUDA)"); +} diff --git a/apex/apex/contrib/csrc/focal_loss/focal_loss_cuda_kernel.cu b/apex/apex/contrib/csrc/focal_loss/focal_loss_cuda_kernel.cu new file mode 100644 index 00000000..bda4f889 --- /dev/null +++ b/apex/apex/contrib/csrc/focal_loss/focal_loss_cuda_kernel.cu @@ -0,0 +1,267 @@ +#include +#include +#include + + +#define ASSERT_UINT4_ALIGNED(PTR) \ + TORCH_INTERNAL_ASSERT(is_aligned(PTR), "Tensor " #PTR " is not uint4 aligned") + +template bool is_aligned(const void *ptr) noexcept { + auto iptr = reinterpret_cast(ptr); + return !(iptr % alignof(T)); +} + +template +__global__ void focal_loss_forward_cuda_kernel( + outscalar_t *loss, scalar_t *partial_grad, + const scalar_t *__restrict__ cls_output, + const labelscalar_t *__restrict__ cls_targets_at_level, + const float *__restrict__ num_positives_sum, const int64_t num_examples, + const int64_t num_classes, const int64_t num_real_classes, + const float alpha, const float gamma, const float smoothing_factor) { + extern __shared__ unsigned char shm[]; + accscalar_t *loss_shm = reinterpret_cast(shm); + loss_shm[threadIdx.x] = 0; + accscalar_t loss_acc = 0; + + accscalar_t one = accscalar_t(1.0); + accscalar_t K = accscalar_t(2.0); + accscalar_t normalizer = one / static_cast(num_positives_sum[0]); + accscalar_t nn_norm, np_norm, pn_norm, pp_norm; + + // *_norm is used for label smoothing only + if (SMOOTHING) { + nn_norm = one - smoothing_factor / K; + np_norm = smoothing_factor / K; + pn_norm = smoothing_factor - smoothing_factor / K; + pp_norm = one - smoothing_factor + smoothing_factor / K; + } + + uint4 p_vec, grad_vec; + + // Accumulate loss on each thread + for (int64_t i = (blockIdx.x * blockDim.x + threadIdx.x) * ILP; + i < num_examples * num_classes; i += gridDim.x * blockDim.x * ILP) { + int64_t idy = i / num_classes; + labelscalar_t y = cls_targets_at_level[idy]; + int64_t base_yid = i % num_classes; + + int64_t pos_idx = idy * num_classes + y; + p_vec = *(uint4 *)&cls_output[i]; + + // Skip ignored matches + if (y == -2) { +#pragma unroll + for (int j = 0; j < ILP; j++) { + *((scalar_t *)(&grad_vec) + j) = 0; + } + *(uint4 *)&partial_grad[i] = grad_vec; + continue; + } + +#pragma unroll + for (int j = 0; j < ILP; j++) { + // Skip the pad classes + if (base_yid + j >= num_real_classes) { + *((scalar_t *)(&grad_vec) + j) = 0; + continue; + } + + accscalar_t p = static_cast(*((scalar_t *)(&p_vec) + j)); + accscalar_t exp_np = ::exp(-p); + accscalar_t exp_pp = ::exp(p); + accscalar_t sigma = one / (one + exp_np); + accscalar_t logee = (p >= 0) ? exp_np : exp_pp; + accscalar_t addee = (p >= 0) ? 0 : -p; + accscalar_t off_a = addee + ::log(one + logee); + + // Negative matches + accscalar_t base = SMOOTHING ? nn_norm * p : p; + accscalar_t off_b = (SMOOTHING ? np_norm : 0) - sigma; + accscalar_t coeff_f1 = one - alpha; + accscalar_t coeff_f2 = sigma; + accscalar_t coeff_b1 = gamma; + accscalar_t coeff_b2 = one - sigma; + + // Positive matches + if (y >= 0 && (i + j == pos_idx)) { + base = SMOOTHING ? pn_norm * p : 0; + off_b = (SMOOTHING ? pp_norm : one) - sigma; + coeff_f1 = alpha; + coeff_f2 = one - sigma; + coeff_b1 = -gamma; + coeff_b2 = sigma; + } + + accscalar_t coeff_f = coeff_f1 * ::pow(coeff_f2, gamma); + accscalar_t coeff_b = coeff_b1 * coeff_b2; + + accscalar_t loss_t = coeff_f * (base + off_a); + accscalar_t grad = coeff_f * (coeff_b * (base + off_a) - off_b); + + // Delay the normalize of partial gradient by num_positives_sum to back + // propagation because scalar_t reduces precision. Focal loss is very + // sensitive to the small gradient. No worry on overflow here since + // gradient has relative smaller range than input. + loss_acc += loss_t; + *((scalar_t *)(&grad_vec) + j) = static_cast(grad); + } + + // This can't ensure to generate stg.128 and may be two stg.64. + *(uint4 *)&partial_grad[i] = grad_vec; + } + loss_shm[threadIdx.x] = loss_acc; + + // Intra-CTA reduction + __syncthreads(); + for (unsigned int s = blockDim.x / 2; s > 0; s >>= 1) { + if (threadIdx.x < s) { + loss_shm[threadIdx.x] += loss_shm[threadIdx.x + s]; + } + __syncthreads(); + } + + // Inter-CTA reduction + if (threadIdx.x == 0) { + loss_acc = loss_shm[0] * normalizer; + atomicAdd(loss, loss_acc); + } +} + +template +__global__ void focal_loss_backward_cuda_kernel( + scalar_t *partial_grad, const outscalar_t *__restrict__ grad_output, + const float *__restrict__ num_positives_sum, const uint64_t numel) { + int64_t idx = (blockIdx.x * blockDim.x + threadIdx.x) * ILP; + + accscalar_t normalizer = static_cast(grad_output[0]) / + static_cast(num_positives_sum[0]); + + // The input is enforced to pad to use vector load, thus there's no need to + // check whether the last element of ILP can out of bound. + if (idx >= numel) + return; + + uint4 grad_vec; + grad_vec = *(uint4 *)&partial_grad[idx]; +#pragma unroll(ILP) + for (int i = 0; i < ILP; i++) { + auto grad = static_cast(*((scalar_t *)(&grad_vec) + i)); + grad *= normalizer; + *((scalar_t *)(&grad_vec) + i) = static_cast(grad); + } + *(uint4 *)&partial_grad[idx] = grad_vec; +} + +std::vector focal_loss_forward_cuda( + const at::Tensor &cls_output, const at::Tensor &cls_targets_at_level, + const at::Tensor &num_positives_sum, const int64_t num_real_classes, + const float alpha, const float gamma, const float smoothing_factor) { + // Checks required for correctness + TORCH_INTERNAL_ASSERT(cls_output.size(-1) >= num_real_classes, + "Incorrect number of real classes."); + TORCH_INTERNAL_ASSERT(cls_targets_at_level.scalar_type() == at::kLong, + "Invalid label type."); + TORCH_INTERNAL_ASSERT( + (num_positives_sum.numel() == 1) && + (num_positives_sum.scalar_type() == at::kFloat), + "Expect num_positives_sum to be a float32 tensor with only one element."); + TORCH_INTERNAL_ASSERT(cls_output.dim() == cls_targets_at_level.dim() + 1, + "Mis-matched dimensions between class output and label."); + for (int64_t i = 0; i < cls_targets_at_level.dim(); i++) + TORCH_INTERNAL_ASSERT(cls_output.size(i) == cls_targets_at_level.size(i), + "Mis-matched shape between class output and label."); + + // Checks required for better performance + const int ILP = sizeof(uint4) / cls_output.element_size(); + ASSERT_UINT4_ALIGNED(cls_output.data_ptr()); + TORCH_INTERNAL_ASSERT(cls_output.size(-1) % ILP == 0, + "Pad number of classes first to take advantage of 128 bit load."); + TORCH_INTERNAL_ASSERT(num_real_classes >= ILP, "Too few classes."); + + int64_t num_classes = cls_output.size(-1); + int64_t num_examples = cls_output.numel() / num_classes; + at::Tensor loss = at::zeros({}, cls_output.options().dtype(at::kFloat)); + + // Compute the incompelete gradient during fprop since most of the heavy + // functions of bprop are the same as fprop, thus trade memory for compute + // helps with focal loss. + at::Tensor partial_grad = at::empty_like(cls_output); + + // The grid contains 2 CTA per SM, each CTA loop on input with stride till the + // last item. + cudaDeviceProp props; + cudaGetDeviceProperties(&props, at::cuda::current_device()); + dim3 block(512); + dim3 grid(2 * props.multiProcessorCount); + + // Specialize on label smoothing or not to reduce redundant operations + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + if (smoothing_factor == 0.0f) { + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + cls_output.scalar_type(), "focal_loss_fprop", [&] { + using accscalar_t = at::acc_type; + using labelscalar_t = int64_t; + using outscalar_t = float; + const int ILP = sizeof(uint4) / sizeof(scalar_t); + focal_loss_forward_cuda_kernel + <<>>( + loss.data_ptr(), + partial_grad.data_ptr(), + cls_output.data_ptr(), + cls_targets_at_level.data_ptr(), + num_positives_sum.data_ptr(), num_examples, + num_classes, num_real_classes, alpha, gamma, + smoothing_factor); + }); + } else { + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + cls_output.scalar_type(), "focal_loss_fprop", [&] { + using accscalar_t = at::acc_type; + using labelscalar_t = int64_t; + using outscalar_t = float; + const int ILP = sizeof(uint4) / sizeof(scalar_t); + focal_loss_forward_cuda_kernel + <<>>( + loss.data_ptr(), + partial_grad.data_ptr(), + cls_output.data_ptr(), + cls_targets_at_level.data_ptr(), + num_positives_sum.data_ptr(), num_examples, + num_classes, num_real_classes, alpha, gamma, + smoothing_factor); + }); + } + + AT_CUDA_CHECK(cudaGetLastError()); + return {loss, partial_grad}; +} + +at::Tensor focal_loss_backward_cuda(const at::Tensor &grad_output, + const at::Tensor &partial_grad, + const at::Tensor &num_positives_sum) { + // Each thread process ILP elements + const int ILP = sizeof(uint4) / partial_grad.element_size(); + dim3 block(512); + dim3 grid((partial_grad.numel() + block.x * ILP - 1) / (block.x * ILP)); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + partial_grad.scalar_type(), "focal_loss_bprop", [&] { + using accscalar_t = at::acc_type; + using outscalar_t = float; + const int ILP = sizeof(uint4) / sizeof(scalar_t); + focal_loss_backward_cuda_kernel + <<>>(partial_grad.data_ptr(), + grad_output.data_ptr(), + num_positives_sum.data_ptr(), + partial_grad.numel()); + }); + + AT_CUDA_CHECK(cudaGetLastError()); + return partial_grad; +} diff --git a/apex/apex/contrib/csrc/groupbn/batch_norm.cu b/apex/apex/contrib/csrc/groupbn/batch_norm.cu new file mode 100644 index 00000000..1ec98eee --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/batch_norm.cu @@ -0,0 +1,328 @@ +#include +#include +#include + +#include "batch_norm.h" + +#include + +#include "compat.h" + +#define cudaCheckErrors(msg) \ + do { \ + cudaError_t __err = cudaGetLastError(); \ + if (__err != cudaSuccess) { \ + fprintf(stderr, "Fatal error: %s (%s at %s:%d)\n", \ + msg, cudaGetErrorString(__err), \ + __FILE__, __LINE__); \ + fprintf(stderr, "*** FAILED - ABORTING\n"); \ + exit(1); \ + } \ + } while (0) + +static size_t round_up_to_multiple(size_t x, int multiple) { + return ((x + multiple - 1) / multiple) * multiple; +} + +struct Workspace { + Workspace(size_t size) : size(size), data(NULL) { + auto& allocator = *::c10::cuda::CUDACachingAllocator::get(); + dataPtr = allocator.allocate(size); + data = dataPtr.get(); + } + Workspace(const Workspace&) = delete; + Workspace(Workspace&&) = default; + Workspace& operator=(Workspace&&) = default; + ~Workspace() = default; + + size_t size; + void* data; + c10::DataPtr dataPtr; +}; + +// Return {y} +at::Tensor nhwc_bn_fwd_train( + const at::Tensor& x, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + const bool fuse_relu, + void * my_data, + void * pair_data, + void * pair_data2, + void * pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop) { + + const int N = x.size(0); + const int H = x.size(1); + const int W = x.size(2); + const int C = x.size(3); + + // generating new magic number and use that for sync + int* magic = magic_tensor.DATA_PTR(); + *magic = (*magic + 1) & 0xff; + + // Allocate output tensor + at::Tensor y = at::empty({N, H, W, C}, x.options()); + + // Create wrapper + NhwcBatchNorm *bn = new NhwcBatchNorm(); + + bn->setInputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W, bn_group); + bn->setOutputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W); + + bn->setConstants(momentum, epsilon); + + // set pointers within the wrapper + bn->setInputOutputPointers(x.DATA_PTR(), + nullptr, + y.DATA_PTR(), + nullptr); + + bn->setWeightPointers({scale.DATA_PTR(), bias.DATA_PTR()}, {nullptr, nullptr}); + bn->setParameterPointers({running_mean.DATA_PTR(), running_inv_var.DATA_PTR()}); + + // deal with workspace(s) + auto workspace_bytes = bn->numWorkspaceBytes(); + // We'll create explicit tensors for the first 2 workspace ptrs, then allocate & offset + // an allocated workspace for the others + size_t total_workspace_bytes = 0; + std::vector workspace_offsets; + + for (auto index = 3; index < workspace_bytes.size(); ++index) { + total_workspace_bytes = round_up_to_multiple(total_workspace_bytes, 512); + workspace_offsets.push_back(total_workspace_bytes); + + auto alloc_bytes = workspace_bytes[index]; + total_workspace_bytes += alloc_bytes; + } + + // Allocate the workspace + Workspace ws(total_workspace_bytes); + + std::vector workspace; + workspace.push_back(minibatch_mean.DATA_PTR()); + workspace.push_back(minibatch_inv_var.DATA_PTR()); + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const int retired_cta_bytes = workspace_bytes[2]; + void* retired_ctas = ret_cta.DATA_PTR(); + assert(ret_cta.size(0)>=retired_cta_bytes); + workspace.push_back(retired_ctas); + + for (auto index = 3; index < workspace_bytes.size(); ++index) { + void *ptr = reinterpret_cast(ws.data) + workspace_offsets[index-3]; + workspace.push_back(ptr); + } + + bn->setWorkspacePointers(workspace, workspace_bytes); + + // Don't fuse in ReLU for now at least + bn->fwd(stream, fuse_relu, my_data, pair_data, pair_data2, pair_data3, bn_group, *magic, occupancy, grid_dim_x, coop); + + return y; +} + +at::Tensor nhwc_bn_fwd_eval( + const at::Tensor& x, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& ret_cta, + const int bn_group, + const float momentum, + const float epsilon, + const bool fuse_relu) { + + const int N = x.size(0); + const int H = x.size(1); + const int W = x.size(2); + const int C = x.size(3); + + // Allocate output tensor + at::Tensor y = at::empty({N, H, W, C}, x.options()); + + // Create wrapper + NhwcBatchNorm *bn = new NhwcBatchNorm(); + + bn->setInputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W, bn_group); + bn->setOutputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W); + + bn->setConstants(momentum, epsilon); + + // set pointers within the wrapper + bn->setInputOutputPointers(x.DATA_PTR(), + nullptr, + y.DATA_PTR(), + nullptr); + + bn->setWeightPointers({scale.DATA_PTR(), bias.DATA_PTR()}, {nullptr, nullptr}); + bn->setParameterPointers({running_mean.DATA_PTR(), running_inv_var.DATA_PTR()}); + + // deal with workspace(s) + auto workspace_bytes = bn->numWorkspaceBytes(); + // We'll create explicit tensors for the first 2 workspace ptrs, then allocate & offset + // an allocated workspace for the others + size_t total_workspace_bytes = 0; + std::vector workspace_offsets; + + for (auto index = 3; index < workspace_bytes.size(); ++index) { + total_workspace_bytes = round_up_to_multiple(total_workspace_bytes, 512); + workspace_offsets.push_back(total_workspace_bytes); + + auto alloc_bytes = workspace_bytes[index]; + total_workspace_bytes += alloc_bytes; + } + + // Allocate the workspace + Workspace ws(total_workspace_bytes); + + std::vector workspace; + workspace.push_back(nullptr); + workspace.push_back(nullptr); + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const int retired_cta_bytes = workspace_bytes[2]; + void* retired_ctas = ret_cta.DATA_PTR(); + assert(ret_cta.size(0)>=retired_cta_bytes); + workspace.push_back(retired_ctas); + + for (auto index = 3; index < workspace_bytes.size(); ++index) { + void *ptr = reinterpret_cast(ws.data) + workspace_offsets[index-3]; + workspace.push_back(ptr); + } + + bn->setWorkspacePointers(workspace, workspace_bytes); + + // Don't fuse in ReLU for now at least + bn->fwdInference(stream, fuse_relu); + + return y; + +} + +std::vector nhwc_bn_bwd( + const at::Tensor& x, + const at::Tensor& dy, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + const bool fuse_relu, + void * my_data, + void * pair_data, + void * pair_data2, + void * pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop) { + // shape + const int N = x.size(0); + const int H = x.size(1); + const int W = x.size(2); + const int C = x.size(3); + + // generating new magic number and use that for sync + int* magic = magic_tensor.DATA_PTR(); + *magic = (*magic + 1) & 0xff; + + // outputs + at::Tensor x_grad, scale_grad, bias_grad; + + // Allocate outputs + x_grad = at::empty_like(x); + scale_grad = at::empty_like(scale); + bias_grad = at::empty_like(bias); + + // Create wrapper + NhwcBatchNorm *bn = new NhwcBatchNorm(); + + bn->setInputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W, bn_group); + bn->setOutputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W); + + bn->setConstants(momentum, epsilon); + + // set pointers within the wrapper + bn->setInputOutputPointers(x.DATA_PTR(), + x_grad.DATA_PTR(), + nullptr, + dy.DATA_PTR()); + + bn->setWeightPointers({scale.DATA_PTR(), bias.DATA_PTR()}, {scale_grad.DATA_PTR(), bias_grad.DATA_PTR()}); + bn->setParameterPointers({running_mean.DATA_PTR(), running_inv_var.DATA_PTR()}); + + // deal with workspace(s) + auto workspace_bytes = bn->numWorkspaceBytes(); + // We'll create explicit tensors for the first 2 workspace ptrs, then allocate & offset + // an allocated workspace for the others + size_t total_workspace_bytes = 0; + std::vector workspace_offsets; + + for (auto index = 3; index < workspace_bytes.size(); ++index) { + total_workspace_bytes = round_up_to_multiple(total_workspace_bytes, 512); + workspace_offsets.push_back(total_workspace_bytes); + + auto alloc_bytes = workspace_bytes[index]; + total_workspace_bytes += alloc_bytes; + } + + // Allocate the workspace + Workspace ws(total_workspace_bytes); + + std::vector workspace; + workspace.push_back(minibatch_mean.DATA_PTR()); + workspace.push_back(minibatch_inv_var.DATA_PTR()); + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const int retired_cta_bytes = workspace_bytes[2]; + void* retired_ctas = ret_cta.DATA_PTR(); + assert(ret_cta.size(0)>=retired_cta_bytes); + workspace.push_back(retired_ctas); + + for (auto index = 3; index < workspace_bytes.size(); ++index) { + void *ptr = reinterpret_cast(ws.data) + workspace_offsets[index-3]; + workspace.push_back(ptr); + } + + bn->setWorkspacePointers(workspace, workspace_bytes); + + bn->dgrad(stream, fuse_relu, my_data, pair_data, pair_data2, pair_data3, bn_group, *magic, occupancy, grid_dim_x, coop); + + return std::vector{x_grad, scale_grad, bias_grad}; +} + +int nhwc_bn_fwd_occupancy() { + int device_id=-1; + cudaGetDevice(&device_id); + + //max occupancy supported by the code is 2 + return NhwcBatchNorm::smem_driven_fwd_occupancy(device_id, 2); +} + +int nhwc_bn_bwd_occupancy() { + int device_id=-1; + cudaGetDevice(&device_id); + + //max occupancy supported by the code is 2 + return NhwcBatchNorm::smem_driven_bwd_occupancy(device_id, 2); +} + + diff --git a/apex/apex/contrib/csrc/groupbn/batch_norm.h b/apex/apex/contrib/csrc/groupbn/batch_norm.h new file mode 100644 index 00000000..bb79d675 --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/batch_norm.h @@ -0,0 +1,735 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2018 by Contributors + * \file nhwc_batch_norm.h + * \brief CUDA NHWC Batch Normalization code + * \author Shankara Rao Thejaswi Nanditale, Dick Carter, Evgeni Krimer +*/ +#ifndef MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_H_ +#define MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_H_ + +#include + +#include +#include +#include +#include + +#include "nhwc_batch_norm_kernel.h" +#include "cuda_utils.h" + + +#define VERBOSE_DEFAULT false + +class NhwcBatchNorm { + public: + NhwcBatchNorm() { + name_ = "nhwc_batchnorm"; + createTensorDescriptor(&X_tensor_desc_); + createTensorDescriptor(&Y_tensor_desc_); + } + + ~NhwcBatchNorm() { + destroyTensorDescriptor(X_tensor_desc_); + destroyTensorDescriptor(Y_tensor_desc_); + } + + void die() { + std::cerr << "batchnorm not initialized" << std::endl; + exit(-1); + } + + void fwd(cudaStream_t stream, bool use_relu, void* my_data, void* pair_data, void* pair_data2, void* pair_data3, const int bn_group, const int magic, const int occupancy, const int grid_dim_x, const bool coop); + void dgrad(cudaStream_t stream, bool use_relu, void* my_data, void* pair_data, void* pair_data2, void* pair_data3, const int bn_group, const int magic, const int occupancy, const int grid_dim_x, const bool coop); + void fwdInference(cudaStream_t stream, bool use_relu); + dim3 calc_fwd_grid(int *loop, const int grid_dim_x); + dim3 calc_bwd_grid(int *loop, const int grid_dim_x); + + void setInputDescriptor(const cudnnTensorFormat_t format, + const cudnnDataType_t data_type, + int n, int c, int h, int w, int bn_group) { + m_ = n * h * w; + int m_bn_adjusted = m_ * bn_group; + c_ = c; + // factor to scale sum of squared errors to get saved variance. Must be 1/nhw. + svar_inv_count_ = 1.f / m_bn_adjusted; + // factor to scale sum of squared errors to get running variance. Should be 1/(nhw-1). + int divisor = m_bn_adjusted - 1; + // nhw == 1 is unlikely, but by setting the rvar_inv_count_ == 1.f, we avoid running var infs. + rvar_inv_count_ = divisor == 0 ? 1.f : 1.f / divisor; + setTensorDescriptor(X_tensor_desc_, format, data_type, n, c, h, w); + } + + void setOutputDescriptor(const cudnnTensorFormat_t format, + const cudnnDataType_t data_type, + int n, int c, int h, int w) { + setTensorDescriptor(Y_tensor_desc_, format, data_type, n, c, h, w); + } + + const std::vector numWorkspaceBytes() const; + + void setWorkspacePointers( + const std::vector& workspace, + const std::vector& num_workspace_bytes); + + void setInputOutputPointers(void* X, void* dX, void* Y, void *dY) { + X_ = X; + dX_ = dX; + Y_ = Y; + dY_ = dY; + } + + // Sets the pointers for the scale and weight (in that order) data and derivative buffers. + void setWeightPointers(const std::vector& weight_pointers, + const std::vector& deriv_pointers) { + assert(weight_pointers.size() == 2); + assert(deriv_pointers.size() == 2); + scale_ = static_cast(weight_pointers[0]); + bias_ = static_cast(weight_pointers[1]); + dscale_ = static_cast(deriv_pointers[0]); + dbias_ = static_cast(deriv_pointers[1]); + } + + // Sets the pointers for the population mean and variance buffers, in that order. + void setParameterPointers(const std::vector& param_pointers) { + assert(param_pointers.size() == 2); + population_mean_ = static_cast(param_pointers[0]); + population_variance_ = static_cast(param_pointers[1]); + } + + void setConstants(const double exp_avg_factor, const double eps) { + exp_avg_factor_ = exp_avg_factor; + eps_ = eps; + } + + void processCudnnStatus(const cudnnStatus_t& status, + const std::string& string = std::string(), + bool verbose = VERBOSE_DEFAULT) { + if (status != CUDNN_STATUS_SUCCESS) + LOG(FATAL) << string << " " << cudnnGetErrorString(status); + else if (verbose) + LOG(INFO) << string << " " << cudnnGetErrorString(status); + } + + void checkCudaStatus(const std::string& string = std::string(), + bool verbose = VERBOSE_DEFAULT) { + cudaError_t status = cudaGetLastError(); + if (status != cudaSuccess) + LOG(FATAL) << string << " " << cudaGetErrorString(status); + else if (verbose) + LOG(INFO) << string << " " << cudaGetErrorString(status); + } + + size_t size_retired_ctas(int grid_y) const { + // Note that the value of max_grid_y to handle known GPUs is about 160. + const int max_grid_y = 1024; + if (grid_y > max_grid_y) + LOG(INFO) << "GPU capabilities exceeds assumptions."; + const int retired_cta_bytes = max_grid_y * 2 * sizeof(int); + // Since the region will be initialized once and used for many kernels, + // the idea is to return an ample size that will cover all uses. + return retired_cta_bytes; + } + + cudnnTensorDescriptor_t X_tensor_desc_ = nullptr; + cudnnTensorDescriptor_t Y_tensor_desc_ = nullptr; + + void* X_ = nullptr; + void* dX_ = nullptr; + void* Y_ = nullptr; + void* dY_ = nullptr; + + // Learned scale and bias weights. + float* scale_ = nullptr; + float* dscale_ = nullptr; + float* bias_ = nullptr; + float* dbias_ = nullptr; + + // Computed population mean and variance parameters. + float* population_mean_ = nullptr; + float* population_variance_ = nullptr; + + // Workspace buffers for minibatch mean and variance (computed in fwd, needed by bwd). + float* minibatch_mean_ = nullptr; + float* minibatch_variance_ = nullptr; + + int m_ = 0; // Number of values per channel that BN is normalizing. + int c_ = 0; // Number of channels over which BN is normalizing. + + float svar_inv_count_ = 0.f; // factor to scale sum of squared errors to get saved variance + float rvar_inv_count_ = 0.f; // factor to scale sum of squared errors to get running variance + + double exp_avg_factor_ = 0.; + double eps_ = 0.; + std::string name_; + + private: + void setTensorDescriptor(cudnnTensorDescriptor_t descriptor, + cudnnTensorFormat_t format, + cudnnDataType_t data_type, + int n, int c, int h, int w) { + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; + status = cudnnSetTensor4dDescriptor(descriptor, format, data_type, n, c, h, w); + processCudnnStatus(status, "set tensor descriptor"); + } + + void createTensorDescriptor(cudnnTensorDescriptor_t *descriptor) { + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; + status = cudnnCreateTensorDescriptor(descriptor); + processCudnnStatus(status, "create tensor_descriptor"); + } + + void destroyTensorDescriptor(cudnnTensorDescriptor_t descriptor) { + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; + status = cudnnDestroyTensorDescriptor(descriptor); + processCudnnStatus(status, "destroy tensor_descriptor"); + } + + protected: + float *partial_sums_ = nullptr; + int *partial_counts_ = nullptr; + int *retired_ctas_ = nullptr; + + void _setFwdParams(NhwcBatchNormFwdParams *params) const; + void _setFwdInferenceParams(NhwcBatchNormFwdInferenceParams *params) const; + void _setBwdParams(NhwcBatchNormBwdParams *params) const; + + // @todo: ability to configure these? + // Kernel params + static const int USE_ONLINE_APPROACH = 1; + static const int THREADS_PER_CTA = 512; + static const int THREADS_PER_PIXEL = 16; + static const int C_ELEMENTS_PER_CTA = 64; + static const int ELEMENTS_PER_LDG = C_ELEMENTS_PER_CTA / THREADS_PER_PIXEL; + static const int MAX_SMEM_WITHOUT_OPT_IN = 48 * 1024; + + typedef uint16_t StorageType; + //typedef float StorageType; + // increasing this to 6 causes spills in fwd kernel! + static const int PIXELS_PER_THREAD_IN_REGISTERS_FWD = 5; + static const int PIXELS_PER_THREAD_IN_REGISTERS_BWD = 3; + static const int PIXELS_PER_THREAD_IN_SMEM_FWD = 10; + static const int PIXELS_PER_THREAD_IN_SMEM_BWD = 5; + + static const int PIXELS_PER_THREAD_FWD = PIXELS_PER_THREAD_IN_REGISTERS_FWD + \ + PIXELS_PER_THREAD_IN_SMEM_FWD; + static const int PIXELS_PER_THREAD_BWD = PIXELS_PER_THREAD_IN_REGISTERS_BWD + \ + PIXELS_PER_THREAD_IN_SMEM_BWD; + static const int PIXELS_PER_THREAD_FWD_INFERENCE = 4; + + // Derived params + static const size_t SMEM_SIZE_FWD = PIXELS_PER_THREAD_IN_SMEM_FWD*THREADS_PER_CTA*\ + ELEMENTS_PER_LDG*sizeof(StorageType); + static const size_t SMEM_SIZE_BWD = PIXELS_PER_THREAD_IN_SMEM_BWD*THREADS_PER_CTA*\ + ELEMENTS_PER_LDG*2*sizeof(StorageType); + static const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + static const int PIXELS_PER_CTA_FWD = THREADS_PER_CTA/THREADS_PER_PIXEL * \ + PIXELS_PER_THREAD_FWD; + static const int PIXELS_PER_CTA_BWD = THREADS_PER_CTA/THREADS_PER_PIXEL * \ + PIXELS_PER_THREAD_BWD; + static const int PIXELS_PER_CTA_FWD_INFERENCE = THREADS_PER_CTA/THREADS_PER_PIXEL * \ + PIXELS_PER_THREAD_FWD_INFERENCE; + + // max grid.y in case of group bn is limited by exchange buffer size + static const int MAX_GBN_BLOCK_Y = 256; + + // Helper function to launch the forward kernel. + + // We calculate (based on smem usage) the achievable occupancy and make sure we run a kernel + // version that was compiled with that occupancy in its launch bounds. This way, we avoid + // needless register spills. + void _fwdKernelLauncher(cudaStream_t stream, NhwcBatchNormFwdParams params, + dim3 grid_dim, int outer_loops, bool use_relu, const int occupancy, const bool coop) { + +#define LAUNCH_FWD_KERNEL(OUTER_LOOPS, USE_RELU, USE_ADD_RELU, COMPILED_FOR_OCCUPANCY, COOP) \ + do { \ + CHECK(SMEM_SIZE_FWD <= MAX_SMEM_WITHOUT_OPT_IN) << "Nhwc batchnorm kernel smem too big."; \ + auto fwd_func = nhwc_batch_norm_fwd< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_FWD, \ + PIXELS_PER_THREAD_IN_SMEM_FWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + USE_RELU, \ + USE_ADD_RELU, \ + COMPILED_FOR_OCCUPANCY>; \ + if (COMPILED_FOR_OCCUPANCY > 1) { \ + cudaFuncSetAttribute(fwd_func, cudaFuncAttributePreferredSharedMemoryCarveout, 100); \ + checkCudaStatus(name_ + " fwd ser coop kernel (cudaFuncSetAttribute carveout)"); \ + } \ + void *params_ptr = static_cast(¶ms); \ + using FWD_FUNC = decltype(nhwc_batch_norm_fwd< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_FWD, \ + PIXELS_PER_THREAD_IN_SMEM_FWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + USE_RELU, \ + USE_ADD_RELU, \ + COMPILED_FOR_OCCUPANCY>); \ + if (COOP) { \ + cudaLaunchCooperativeKernel(fwd_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_FWD, \ + stream); \ + } else { \ + cudaLaunchKernel(fwd_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_FWD, \ + stream); \ + } \ + checkCudaStatus(name_ + " fwd ser coop kernel"); \ + } while (0) + + // Don't try for an occupancy > 2 as this will squeeze register use and create spills. + if (outer_loops == 1 && use_relu) { + if (occupancy >= 2) + LAUNCH_FWD_KERNEL(1, true, false, 2, coop); + else + LAUNCH_FWD_KERNEL(1, true, false, 1, coop); + } else if (outer_loops == 1 && !use_relu) { + if (occupancy >= 2) + LAUNCH_FWD_KERNEL(1, false, false, 2, coop); + else + LAUNCH_FWD_KERNEL(1, false, false, 1, coop); + } else if (use_relu) { + if (occupancy >= 2) + LAUNCH_FWD_KERNEL(0, true, false, 2, coop); + else + LAUNCH_FWD_KERNEL(0, true, false, 1, coop); + } else { + if (occupancy >= 2) + LAUNCH_FWD_KERNEL(0, false, false, 2, coop); + else + LAUNCH_FWD_KERNEL(0, false, false, 1, coop); + } +#undef LAUNCH_FWD_KERNEL + } + + // Helper function to launch the backward kernel. + + void _bwdKernelLauncher(cudaStream_t stream, NhwcBatchNormBwdParams params, + dim3 grid_dim, int outer_loops, bool use_relu, const int occupancy, const bool coop) { +#define LAUNCH_BWD_KERNEL(OUTER_LOOPS, COMPILED_FOR_OCCUPANCY, COOP) \ + do { \ + CHECK(SMEM_SIZE_BWD <= MAX_SMEM_WITHOUT_OPT_IN) << "Nhwc batchnorm kernel smem too big."; \ + auto bwd_func = nhwc_batch_norm_bwd< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_BWD, \ + PIXELS_PER_THREAD_IN_SMEM_BWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + COMPILED_FOR_OCCUPANCY>; \ + if (COMPILED_FOR_OCCUPANCY > 1) { \ + cudaFuncSetAttribute(bwd_func, cudaFuncAttributePreferredSharedMemoryCarveout, 100); \ + checkCudaStatus(name_ + " bwd coop serial kernel (cudaFuncSetAttribute carveout)"); \ + } \ + void *params_ptr = static_cast(¶ms); \ + using BWD_FUNC = decltype(nhwc_batch_norm_bwd< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_BWD, \ + PIXELS_PER_THREAD_IN_SMEM_BWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + COMPILED_FOR_OCCUPANCY>); \ + if (COOP) { \ + cudaLaunchCooperativeKernel(bwd_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_BWD, \ + stream); \ + } else { \ + cudaLaunchKernel(bwd_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_BWD, \ + stream); \ + } \ + checkCudaStatus(name_ + " bwd coop serial kernel"); \ + } while (0) + +#define LAUNCH_BWD_RELU_KERNEL(OUTER_LOOPS, COMPILED_FOR_OCCUPANCY, COOP) \ + do { \ + CHECK(SMEM_SIZE_BWD <= MAX_SMEM_WITHOUT_OPT_IN) << "Nhwc batchnorm kernel smem too big."; \ + auto bwd_relu_func = nhwc_batch_norm_bwd_relu< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_BWD, \ + PIXELS_PER_THREAD_IN_SMEM_BWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + COMPILED_FOR_OCCUPANCY>; \ + if (COMPILED_FOR_OCCUPANCY > 1) { \ + cudaFuncSetAttribute(bwd_relu_func, cudaFuncAttributePreferredSharedMemoryCarveout, 100); \ + checkCudaStatus(name_ + " bwd-relu coop serial kernel (cudaFuncSetAttribute carveout)"); \ + } \ + void *params_ptr = static_cast(¶ms); \ + using BWD_RELU_FUNC = decltype(nhwc_batch_norm_bwd_relu< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_BWD, \ + PIXELS_PER_THREAD_IN_SMEM_BWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + COMPILED_FOR_OCCUPANCY>); \ + if (COOP) { \ + cudaLaunchCooperativeKernel(bwd_relu_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_BWD, \ + stream); \ + } else { \ + cudaLaunchKernel(bwd_relu_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_BWD, \ + stream); \ + } \ + checkCudaStatus(name_ + " bwd-relu coop serial kernel"); \ + } while (0) + + // Don't try for an occupancy > 2 as this will squeeze register use and create spills. + if (outer_loops == 1 && use_relu) { + if (occupancy >= 2) + LAUNCH_BWD_RELU_KERNEL(1, 2, coop); + else + LAUNCH_BWD_RELU_KERNEL(1, 1, coop); + } else if (outer_loops == 1 && !use_relu) { + if (occupancy >= 2) + LAUNCH_BWD_KERNEL(1, 2, coop); + else + LAUNCH_BWD_KERNEL(1, 1, coop); + } else if (use_relu) { + if (occupancy >= 2) + LAUNCH_BWD_RELU_KERNEL(0, 2, coop); + else + LAUNCH_BWD_RELU_KERNEL(0, 1, coop); + } else { + if (occupancy >= 2) + LAUNCH_BWD_KERNEL(0, 2, coop); + else + LAUNCH_BWD_KERNEL(0, 1, coop); + } +#undef LAUNCH_BWD_KERNEL + } + + public: + + // Calculate the expected fwd kernel occupancy, as dictated by shared memory usage. + static int smem_driven_fwd_occupancy(int device_id, const int max_cta_per_sm) { + using namespace at::cuda::utils; + int fwd_reduction_bytes = THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG*sizeof(float); + int fwd_smem_bytes = SMEM_SIZE_FWD + fwd_reduction_bytes; + int occupancy = MaxSharedMemoryPerMultiprocessor(device_id) / fwd_smem_bytes; + return std::min(max_cta_per_sm, occupancy); + } + + // Calculate the expected bwd kernel occupancy, as dictated by shared memory usage. + static int smem_driven_bwd_occupancy(int device_id, const int max_cta_per_sm) { + using namespace at::cuda::utils; + int bwd_reduction_bytes = THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG*sizeof(float); + int bwd_smem_bytes = SMEM_SIZE_BWD + bwd_reduction_bytes; + int occupancy = MaxSharedMemoryPerMultiprocessor(device_id) / bwd_smem_bytes; + return std::min(max_cta_per_sm, occupancy); + } +}; + +const std::vector NhwcBatchNorm::numWorkspaceBytes() const { + assert(c_ > 0); + + // choose the max memory required between fwd/bwd passes + int grid_x_fwd = div_up(m_, PIXELS_PER_CTA_FWD); + int grid_x_bwd = div_up(m_, PIXELS_PER_CTA_BWD); + int grid_x = max(grid_x_fwd, grid_x_bwd); + int grid_y = div_up(c_, C_ELEMENTS_PER_CTA); + + const size_t num_mean_bytes = c_ * sizeof(float); + const size_t num_variance_bytes = num_mean_bytes; + const size_t size_sums = grid_y*grid_x*THREADS_PER_PIXEL*\ + ELEMENTS_PER_LDG*2*sizeof(float); + const size_t size_counts = grid_y*grid_x*sizeof(int); + + return {num_mean_bytes, num_variance_bytes, + size_retired_ctas(grid_y), size_sums, size_counts}; +} + +void NhwcBatchNorm::setWorkspacePointers( + const std::vector& workspace, + const std::vector& num_workspace_bytes) { + assert(workspace.size() == 5); + assert(num_workspace_bytes.size() == 5); + + minibatch_mean_ = static_cast(workspace[0]); + minibatch_variance_ = static_cast(workspace[1]); + retired_ctas_ = static_cast(workspace[2]); + partial_sums_ = static_cast(workspace[3]); + partial_counts_ = static_cast(workspace[4]); +} + +void NhwcBatchNorm::_setFwdParams(NhwcBatchNormFwdParams *params) const { + params->gmem_src = static_cast(X_); + params->gmem_dst = static_cast(Y_); + params->gmem_src1 = nullptr; + params->gmem_bias = bias_; + params->gmem_scale = scale_; + params->gmem_running_mean = population_mean_; + params->gmem_running_var = population_variance_; + params->gmem_saved_mean = minibatch_mean_; + params->gmem_saved_var = minibatch_variance_; + params->gmem_relu_bitmask = nullptr; + params->nhw = m_; + params->c = c_; + params->svar_inv_count = svar_inv_count_; + params->rvar_inv_count = rvar_inv_count_; + params->gmem_sums = partial_sums_; + params->gmem_counts = partial_counts_; + params->gmem_retired_ctas = retired_ctas_; + params->var_eps = eps_; + params->outer_loops = 0; + params->exp_avg_factor = static_cast(exp_avg_factor_); + params->c_blks = div_up(c_, C_ELEMENTS_PER_CTA); +} + +void NhwcBatchNorm::_setFwdInferenceParams(NhwcBatchNormFwdInferenceParams + *params) const { + params->gmem_src = static_cast(X_); + params->gmem_dst = static_cast(Y_); + params->gmem_src1 = nullptr; + params->gmem_bias = bias_; + params->gmem_scale = scale_; + params->gmem_mean = population_mean_; + params->gmem_var = population_variance_; + params->nhw = m_; + params->c = c_; + params->var_eps = eps_; +} + +void NhwcBatchNorm::_setBwdParams(NhwcBatchNormBwdParams *params) const { + params->gmem_src = static_cast(X_); + params->gmem_dy = static_cast(dY_); + params->gmem_dst = static_cast(dX_); + params->gmem_dst1 = nullptr; + params->gmem_relu_bitmask = nullptr; + params->gmem_dscale = dscale_; + params->gmem_dbias = dbias_; + params->gmem_scale = scale_; + params->gmem_bias = bias_; + params->gmem_saved_mean = minibatch_mean_; + params->gmem_saved_var = minibatch_variance_; + params->nhw = m_; + params->c = c_; + params->svar_inv_count = svar_inv_count_; + params->gmem_sums = partial_sums_; + params->gmem_retired_ctas = retired_ctas_; + params->outer_loops = 0; + params->c_blks = div_up(c_, C_ELEMENTS_PER_CTA); +} + +void NhwcBatchNorm::fwdInference(cudaStream_t stream, bool use_relu) { + bool ptrs_are_set = + X_tensor_desc_ != nullptr + && Y_tensor_desc_ != nullptr + && scale_ != nullptr + && bias_ != nullptr + // && minibatch_mean_ != nullptr + // && minibatch_variance_ != nullptr + && population_mean_ != nullptr + && population_variance_ != nullptr + && X_ != nullptr + // && dX_ != nullptr + && Y_ != nullptr + // && dY_ != nullptr + // && dscale_ != nullptr + // && dbias_ != nullptr + && partial_sums_ != nullptr + && partial_counts_ != nullptr; + + if (!ptrs_are_set) + die(); + + dim3 grid_dim; + grid_dim.x = div_up(m_, PIXELS_PER_CTA_FWD_INFERENCE); + grid_dim.y = div_up(c_, C_ELEMENTS_PER_CTA); + + // @todo: maybe just move this inside initialize routine? + NhwcBatchNormFwdInferenceParams params; + _setFwdInferenceParams(¶ms); + + if (use_relu) { + nhwc_batch_norm_fwd_inference + + <<>>(params); + checkCudaStatus(name_ + " fwd_inference-relu kernel"); + } else { + nhwc_batch_norm_fwd_inference + + <<>>(params); + checkCudaStatus(name_ + " fwd_inference kernel"); + } +} + +dim3 NhwcBatchNorm::calc_fwd_grid(int *loop, const int grid_dim_x) { + dim3 grid_dim; + grid_dim.x = div_up(m_, PIXELS_PER_CTA_FWD); + int c_blks = div_up(c_, C_ELEMENTS_PER_CTA); + unsigned int max_grid_x = grid_dim_x; + if (grid_dim.x <= max_grid_x) { + *loop = 1; + if (max_grid_x / grid_dim.x > 1) { + grid_dim.y = std::min(c_blks, static_cast(max_grid_x / grid_dim.x)); + assert(grid_dim.y 1) { + grid_dim.y = std::min(c_blks, static_cast(max_grid_x / grid_dim.x)); + assert(grid_dim.y> 1); + + dim3 grid_dim = calc_fwd_grid(¶ms.outer_loops, grid_dim_x); + _fwdKernelLauncher(stream, params, grid_dim, params.outer_loops, use_relu, occupancy, coop); +} + +void NhwcBatchNorm::dgrad(cudaStream_t stream, bool use_relu, void* my_data, void* pair_data, void* pair_data2, void* pair_data3, + const int bn_group, const int magic, const int occupancy, const int grid_dim_x, const bool coop) { + bool ptrs_are_set = + X_tensor_desc_ != nullptr + && Y_tensor_desc_ != nullptr + && scale_ != nullptr + && (bias_ != nullptr || !use_relu) + && minibatch_mean_ != nullptr + && minibatch_variance_ != nullptr + // && population_mean_ != nullptr + // && population_variance_ != nullptr + && X_ != nullptr + && dX_ != nullptr + // && Y_ != nullptr + && dY_ != nullptr + && dscale_ != nullptr + && dbias_ != nullptr; + + if (!ptrs_are_set) + die(); + + // reset of retired_cta_count no longer needed + + NhwcBatchNormBwdParams params; + _setBwdParams(¶ms); + params.my_data = my_data; + params.pair_datas[0] = pair_data; + params.pair_datas[1] = pair_data2; + params.pair_datas[2] = pair_data3; + params.magic = magic; + params.sync_iters = (bn_group==8)?3:(bn_group >> 1); + params.wgrad_coeff = 1.0 / bn_group; + + dim3 grid_dim = calc_bwd_grid(¶ms.outer_loops, grid_dim_x); + _bwdKernelLauncher(stream, params, grid_dim, params.outer_loops, use_relu, occupancy, coop); +} + +#endif // MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_H_ diff --git a/apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.cu b/apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.cu new file mode 100644 index 00000000..e383fb80 --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.cu @@ -0,0 +1,340 @@ +#include +#include +#include + +#include "batch_norm_add_relu.h" + +#include + +#include "compat.h" + +//FIXME move the common stuff to common h file +#define cudaCheckErrors(msg) \ + do { \ + cudaError_t __err = cudaGetLastError(); \ + if (__err != cudaSuccess) { \ + fprintf(stderr, "Fatal error: %s (%s at %s:%d)\n", \ + msg, cudaGetErrorString(__err), \ + __FILE__, __LINE__); \ + fprintf(stderr, "*** FAILED - ABORTING\n"); \ + exit(1); \ + } \ + } while (0) + +static size_t round_up_to_multiple(size_t x, int multiple) { + return ((x + multiple - 1) / multiple) * multiple; +} + +struct Workspace { + Workspace(size_t size) : size(size), data(NULL) { + auto& allocator = *::c10::cuda::CUDACachingAllocator::get(); + dataPtr = allocator.allocate(size); + data = dataPtr.get(); + } + Workspace(const Workspace&) = delete; + Workspace(Workspace&&) = default; + Workspace& operator=(Workspace&&) = default; + ~Workspace() = default; + + size_t size; + void* data; + c10::DataPtr dataPtr; +}; + +// Return {y} +at::Tensor nhwc_bn_addrelu_fwd_train( + const at::Tensor& x, + const at::Tensor& z, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& bitmask, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + void * my_data, + void * pair_data, + void * pair_data2, + void * pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop) { + + const int N = x.size(0); + const int H = x.size(1); + const int W = x.size(2); + const int C = x.size(3); + + // generating new magic number and use that for sync + int* magic = magic_tensor.DATA_PTR(); + *magic = (*magic + 1) & 0xff; + + // Allocate output tensor + at::Tensor y = at::empty({N, H, W, C}, x.options()); + + // Create wrapper + NhwcBatchNormAddRelu *bn = new NhwcBatchNormAddRelu(); + + bn->setInputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W, bn_group); + bn->setOutputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W); + + bn->setConstants(momentum, epsilon); + + // set pointers within the wrapper + bn->setInputOutputPointers(x.DATA_PTR(), + nullptr, + y.DATA_PTR(), + nullptr, + z.DATA_PTR(), + nullptr); + + bn->setWeightPointers({scale.DATA_PTR(), bias.DATA_PTR()}, {nullptr, nullptr}); + bn->setParameterPointers({running_mean.DATA_PTR(), running_inv_var.DATA_PTR()}); + + // deal with workspace(s) + auto workspace_bytes = bn->numWorkspaceBytes(); + // We'll create explicit tensors for the first 2 workspace ptrs, then allocate & offset + // an allocated workspace for the others + size_t total_workspace_bytes = 0; + std::vector workspace_offsets; + + for (auto index = 4; index < workspace_bytes.size(); ++index) { + total_workspace_bytes = round_up_to_multiple(total_workspace_bytes, 512); + workspace_offsets.push_back(total_workspace_bytes); + + auto alloc_bytes = workspace_bytes[index]; + total_workspace_bytes += alloc_bytes; + } + + // Allocate the workspace + Workspace ws(total_workspace_bytes); + + std::vector workspace; + workspace.push_back(minibatch_mean.DATA_PTR()); + workspace.push_back(minibatch_inv_var.DATA_PTR()); + workspace.push_back(bitmask.DATA_PTR()); + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const int retired_cta_bytes = workspace_bytes[3]; + void* retired_ctas = ret_cta.DATA_PTR(); + assert(ret_cta.size(0)>=retired_cta_bytes); + + workspace.push_back(retired_ctas); + + for (auto index = 4; index < workspace_bytes.size(); ++index) { + void *ptr = reinterpret_cast(ws.data) + workspace_offsets[index-4]; + workspace.push_back(ptr); + } + + bn->setWorkspacePointers(workspace, workspace_bytes); + + // Don't fuse in ReLU for now at least + bn->fwd(stream, my_data, pair_data, pair_data2, pair_data3, bn_group, *magic, occupancy, grid_dim_x, coop); + + return y; +} + +at::Tensor nhwc_bn_addrelu_fwd_eval( + const at::Tensor& x, + const at::Tensor& z, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& ret_cta, + const int bn_group, + const float momentum, + const float epsilon) { + + const int N = x.size(0); + const int H = x.size(1); + const int W = x.size(2); + const int C = x.size(3); + + // Allocate output tensor + at::Tensor y = at::empty({N, H, W, C}, x.options()); + + // Create wrapper + NhwcBatchNormAddRelu *bn = new NhwcBatchNormAddRelu(); + + bn->setInputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W, bn_group); + bn->setOutputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W); + + bn->setConstants(momentum, epsilon); + + // set pointers within the wrapper + bn->setInputOutputPointers(x.DATA_PTR(), + nullptr, + y.DATA_PTR(), + nullptr, + z.DATA_PTR(), + nullptr); + + bn->setWeightPointers({scale.DATA_PTR(), bias.DATA_PTR()}, {nullptr, nullptr}); + bn->setParameterPointers({running_mean.DATA_PTR(), running_inv_var.DATA_PTR()}); + + // deal with workspace(s) + auto workspace_bytes = bn->numWorkspaceBytes(); + // We'll create explicit tensors for the first 2 workspace ptrs, then allocate & offset + // an allocated workspace for the others + size_t total_workspace_bytes = 0; + std::vector workspace_offsets; + + for (auto index = 4; index < workspace_bytes.size(); ++index) { + total_workspace_bytes = round_up_to_multiple(total_workspace_bytes, 512); + workspace_offsets.push_back(total_workspace_bytes); + + auto alloc_bytes = workspace_bytes[index]; + total_workspace_bytes += alloc_bytes; + } + + // Allocate the workspace + Workspace ws(total_workspace_bytes); + + std::vector workspace; + workspace.push_back(nullptr); + workspace.push_back(nullptr); + workspace.push_back(nullptr); + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const int retired_cta_bytes = workspace_bytes[3]; + void* retired_ctas = ret_cta.DATA_PTR(); + assert(ret_cta.size(0)>=retired_cta_bytes); + workspace.push_back(retired_ctas); + + for (auto index = 4; index < workspace_bytes.size(); ++index) { + void *ptr = reinterpret_cast(ws.data) + workspace_offsets[index-4]; + workspace.push_back(ptr); + } + + bn->setWorkspacePointers(workspace, workspace_bytes); + + // Don't fuse in ReLU for now at least + bn->fwdInference(stream); + + return y; + +} + +std::vector nhwc_bn_addrelu_bwd( + const at::Tensor& x, + const at::Tensor& dy, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& bitmask, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + void * my_data, + void * pair_data, + void * pair_data2, + void * pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop) { + // shape + const int N = x.size(0); + const int H = x.size(1); + const int W = x.size(2); + const int C = x.size(3); + + // generating new magic number and use that for sync + int* magic = magic_tensor.DATA_PTR(); + *magic = (*magic + 1) & 0xff; + + // outputs + at::Tensor x_grad, z_grad, scale_grad, bias_grad; + + // Allocate outputs + x_grad = at::empty_like(x); + z_grad = at::empty_like(x); + scale_grad = at::empty_like(scale); + bias_grad = at::empty_like(bias); + + // Create wrapper + NhwcBatchNormAddRelu *bn = new NhwcBatchNormAddRelu(); + + bn->setInputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W, bn_group); + bn->setOutputDescriptor(CUDNN_TENSOR_NHWC, CUDNN_DATA_HALF, N, C, H, W); + + bn->setConstants(momentum, epsilon); + + // set pointers within the wrapper + bn->setInputOutputPointers(x.DATA_PTR(), + x_grad.DATA_PTR(), + nullptr, + dy.DATA_PTR(), + nullptr, + z_grad.DATA_PTR()); + + bn->setWeightPointers({scale.DATA_PTR(), bias.DATA_PTR()}, {scale_grad.DATA_PTR(), bias_grad.DATA_PTR()}); + bn->setParameterPointers({running_mean.DATA_PTR(), running_inv_var.DATA_PTR()}); + + // deal with workspace(s) + auto workspace_bytes = bn->numWorkspaceBytes(); + // We'll create explicit tensors for the first 2 workspace ptrs, then allocate & offset + // an allocated workspace for the others + size_t total_workspace_bytes = 0; + std::vector workspace_offsets; + + for (auto index = 4; index < workspace_bytes.size(); ++index) { + total_workspace_bytes = round_up_to_multiple(total_workspace_bytes, 512); + workspace_offsets.push_back(total_workspace_bytes); + + auto alloc_bytes = workspace_bytes[index]; + total_workspace_bytes += alloc_bytes; + } + + // Allocate the workspace + Workspace ws(total_workspace_bytes); + + std::vector workspace; + workspace.push_back(minibatch_mean.DATA_PTR()); + workspace.push_back(minibatch_inv_var.DATA_PTR()); + workspace.push_back(bitmask.DATA_PTR()); + + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const int retired_cta_bytes = workspace_bytes[3]; + void* retired_ctas = ret_cta.DATA_PTR(); + assert(ret_cta.size(0)>=retired_cta_bytes); + workspace.push_back(retired_ctas); + + for (auto index = 4; index < workspace_bytes.size(); ++index) { + void *ptr = reinterpret_cast(ws.data) + workspace_offsets[index-4]; + workspace.push_back(ptr); + } + + bn->setWorkspacePointers(workspace, workspace_bytes); + + bn->dgrad(stream, my_data, pair_data, pair_data2, pair_data3, bn_group, *magic, occupancy, grid_dim_x, coop); + + return std::vector{x_grad, z_grad, scale_grad, bias_grad}; +} + +int nhwc_bn_addrelu_fwd_occupancy() { + int device_id=-1; + cudaGetDevice(&device_id); + + //max occupancy supported by the code is 2 + return NhwcBatchNormAddRelu::smem_driven_fwd_occupancy(device_id, 2); +} + +int nhwc_bn_addrelu_bwd_occupancy() { + int device_id=-1; + cudaGetDevice(&device_id); + + //max occupancy supported by the code is 2 + return NhwcBatchNormAddRelu::smem_driven_bwd_occupancy(device_id, 2); +} + diff --git a/apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.h b/apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.h new file mode 100644 index 00000000..3dfe7b26 --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/batch_norm_add_relu.h @@ -0,0 +1,682 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2018 by Contributors + * \file nhwc_batch_norm_add_relu.h + * \brief CUDA NHWC Batch Normalization code with fused addition + * \author Shankara Rao Thejaswi Nanditale, Dick Carter, Maxim Milakov, Evgeni Krimer +*/ +#ifndef MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_ADD_RELU_H_ +#define MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_ADD_RELU_H_ + +#include + +#include +#include +#include +#include + +#include "nhwc_batch_norm_kernel.h" +#include "cuda_utils.h" + + +#define VERBOSE_DEFAULT false + +class NhwcBatchNormAddRelu { + public: + NhwcBatchNormAddRelu() { + name_ = "nhwc_batchnormaddrelu"; + createTensorDescriptor(&X_tensor_desc_); + createTensorDescriptor(&Y_tensor_desc_); + } + + ~NhwcBatchNormAddRelu() { + destroyTensorDescriptor(X_tensor_desc_); + destroyTensorDescriptor(Y_tensor_desc_); + } + + void die() { + std::cerr << "batchnormaddrelu not initialized" << std::endl; + exit(-1); + } + + void fwd(cudaStream_t stream, void* my_data, void* pair_data, void* pair_data2, void* pair_data3, const int bn_group, const int magic, const int occupancy, const int grid_dim_x, const bool coop); + void dgrad(cudaStream_t stream, void* my_data, void* pair_data, void* pair_data2, void* pair_data3, const int bn_group, const int magic, const int occupancy, const int grid_dim_x, const bool coop); + void fwdInference(cudaStream_t stream); + dim3 calc_fwd_grid(int *loop, const int grid_dim_x); + dim3 calc_bwd_grid(int *loop, const int grid_dim_x); + + void setInputDescriptor(const cudnnTensorFormat_t format, + const cudnnDataType_t data_type, + int n, int c, int h, int w, int bn_group) { + m_ = n * h * w; + int m_bn_adjusted = m_ * bn_group; + c_ = c; + // factor to scale sum of squared errors to get saved variance. Must be 1/nhw. + svar_inv_count_ = 1.f / m_bn_adjusted; + // factor to scale sum of squared errors to get running variance. Should be 1/(nhw-1). + int divisor = m_bn_adjusted - 1; + // nhw == 1 is unlikely, but by setting the rvar_inv_count_ == 1.f, we avoid running var infs. + rvar_inv_count_ = divisor == 0 ? 1.f : 1.f / divisor; + setTensorDescriptor(X_tensor_desc_, format, data_type, n, c, h, w); + } + + void setOutputDescriptor(const cudnnTensorFormat_t format, + const cudnnDataType_t data_type, + int n, int c, int h, int w) { + setTensorDescriptor(Y_tensor_desc_, format, data_type, n, c, h, w); + } + + const std::vector numWorkspaceBytes() const; + + void setWorkspacePointers( + const std::vector& workspace, + const std::vector& num_workspace_bytes); + + void setInputOutputPointers(void* X, void* dX, void* Y, void *dY, void* addend, void* dAddend) { + X_ = X; + dX_ = dX; + Y_ = Y; + dY_ = dY; + addend_ = addend; + dAddend_ = dAddend; + } + + // Sets the pointers for the scale and weight (in that order) data and derivative buffers. + void setWeightPointers(const std::vector& weight_pointers, + const std::vector& deriv_pointers) { + assert(weight_pointers.size() == 2); + assert(deriv_pointers.size() == 2); + scale_ = static_cast(weight_pointers[0]); + bias_ = static_cast(weight_pointers[1]); + dscale_ = static_cast(deriv_pointers[0]); + dbias_ = static_cast(deriv_pointers[1]); + } + + // Sets the pointers for the population mean and variance buffers, in that order. + void setParameterPointers(const std::vector& param_pointers) { + assert(param_pointers.size() == 2); + population_mean_ = static_cast(param_pointers[0]); + population_variance_ = static_cast(param_pointers[1]); + } + + void setConstants(const double exp_avg_factor, const double eps) { + exp_avg_factor_ = exp_avg_factor; + eps_ = eps; + } + + void processCudnnStatus(const cudnnStatus_t& status, + const std::string& string = std::string(), + bool verbose = VERBOSE_DEFAULT) { + if (status != CUDNN_STATUS_SUCCESS) + LOG(FATAL) << string << " " << cudnnGetErrorString(status); + else if (verbose) + LOG(INFO) << string << " " << cudnnGetErrorString(status); + } + + void checkCudaStatus(const std::string& string = std::string(), + bool verbose = VERBOSE_DEFAULT) { + cudaError_t status = cudaGetLastError(); + if (status != cudaSuccess) + LOG(FATAL) << string << " " << cudaGetErrorString(status); + else if (verbose) + LOG(INFO) << string << " " << cudaGetErrorString(status); + } + + size_t size_retired_ctas(int grid_y) const { + // Note that the value of max_grid_y to handle known GPUs is about 160. + const int max_grid_y = 1024; + if (grid_y > max_grid_y) + LOG(INFO) << "GPU capabilities exceeds assumptions."; + const int retired_cta_bytes = max_grid_y * 2 * sizeof(int); + // Since the region will be initialized once and used for many kernels, + // the idea is to return an ample size that will cover all uses. + return retired_cta_bytes; + } + + cudnnTensorDescriptor_t X_tensor_desc_ = nullptr; + cudnnTensorDescriptor_t Y_tensor_desc_ = nullptr; + + void* X_ = nullptr; + void* dX_ = nullptr; + void* Y_ = nullptr; + void* dY_ = nullptr; + void* addend_ = nullptr; + void* dAddend_ = nullptr; + + // Learned scale and bias weights. + float* scale_ = nullptr; + float* dscale_ = nullptr; + float* bias_ = nullptr; + float* dbias_ = nullptr; + + // Computed population mean and variance parameters. + float* population_mean_ = nullptr; + float* population_variance_ = nullptr; + + // Workspace buffers for minibatch mean and variance (computed in fwd, needed by bwd). + float* minibatch_mean_ = nullptr; + float* minibatch_variance_ = nullptr; + + int m_ = 0; // Number of values per channel that BN is normalizing. + int c_ = 0; // Number of channels over which BN is normalizing. + + float svar_inv_count_ = 0.f; // factor to scale sum of squared errors to get saved variance + float rvar_inv_count_ = 0.f; // factor to scale sum of squared errors to get running variance + + double exp_avg_factor_ = 0.; + double eps_ = 0.; + std::string name_; + + private: + void setTensorDescriptor(cudnnTensorDescriptor_t descriptor, + cudnnTensorFormat_t format, + cudnnDataType_t data_type, + int n, int c, int h, int w) { + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; + status = cudnnSetTensor4dDescriptor(descriptor, format, data_type, n, c, h, w); + processCudnnStatus(status, "set tensor descriptor"); + } + + void createTensorDescriptor(cudnnTensorDescriptor_t *descriptor) { + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; + status = cudnnCreateTensorDescriptor(descriptor); + processCudnnStatus(status, "create tensor_descriptor"); + } + + void destroyTensorDescriptor(cudnnTensorDescriptor_t descriptor) { + cudnnStatus_t status = CUDNN_STATUS_SUCCESS; + status = cudnnDestroyTensorDescriptor(descriptor); + processCudnnStatus(status, "destroy tensor_descriptor"); + } + + protected: + float *partial_sums_ = nullptr; + int *partial_counts_ = nullptr; + int *retired_ctas_ = nullptr; + unsigned int *relu_bitmask_ = nullptr; + + void _setFwdParams(NhwcBatchNormFwdParams *params) const; + void _setFwdInferenceParams(NhwcBatchNormFwdInferenceParams *params) const; + void _setBwdParams(NhwcBatchNormBwdParams *params) const; + + // @todo: ability to configure these? + // Kernel params + static const int USE_ONLINE_APPROACH = 1; + static const int THREADS_PER_CTA = 512; + static const int THREADS_PER_PIXEL = 16; + static const int C_ELEMENTS_PER_CTA = 64; + static const int ELEMENTS_PER_LDG = C_ELEMENTS_PER_CTA / THREADS_PER_PIXEL; + static const int MAX_SMEM_WITHOUT_OPT_IN = 48 * 1024; + + typedef uint16_t StorageType; + // increasing this to 6 causes spills in fwd kernel! + static const int PIXELS_PER_THREAD_IN_REGISTERS_FWD = 5; + static const int PIXELS_PER_THREAD_IN_REGISTERS_BWD = 3; + static const int PIXELS_PER_THREAD_IN_SMEM_FWD = 10; + static const int PIXELS_PER_THREAD_IN_SMEM_BWD = 5; + + static const int PIXELS_PER_THREAD_FWD = PIXELS_PER_THREAD_IN_REGISTERS_FWD + \ + PIXELS_PER_THREAD_IN_SMEM_FWD; + static const int PIXELS_PER_THREAD_BWD = PIXELS_PER_THREAD_IN_REGISTERS_BWD + \ + PIXELS_PER_THREAD_IN_SMEM_BWD; + static const int PIXELS_PER_THREAD_FWD_INFERENCE = 4; + + // Derived params + static const size_t SMEM_SIZE_FWD = PIXELS_PER_THREAD_IN_SMEM_FWD*THREADS_PER_CTA*\ + ELEMENTS_PER_LDG*sizeof(StorageType); + static const size_t SMEM_SIZE_BWD = PIXELS_PER_THREAD_IN_SMEM_BWD*THREADS_PER_CTA*\ + ELEMENTS_PER_LDG*2*sizeof(StorageType); + static const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + static const int PIXELS_PER_CTA_FWD = THREADS_PER_CTA/THREADS_PER_PIXEL * \ + PIXELS_PER_THREAD_FWD; + static const int PIXELS_PER_CTA_BWD = THREADS_PER_CTA/THREADS_PER_PIXEL * \ + PIXELS_PER_THREAD_BWD; + static const int PIXELS_PER_CTA_FWD_INFERENCE = THREADS_PER_CTA/THREADS_PER_PIXEL * \ + PIXELS_PER_THREAD_FWD_INFERENCE; + + // max grid.y in case of group bn is limited by exchange buffer size + static const int MAX_GBN_BLOCK_Y = 256; + + // Helper function to launch the forward kernel. + + // We calculate (based on smem usage) the achievable occupancy and make sure we run a kernel + // version that was compiled with that occupancy in its launch bounds. This way, we avoid + // needless register spills. + void _fwdKernelLauncher(cudaStream_t stream, NhwcBatchNormFwdParams params, + dim3 grid_dim, int outer_loops, const int occupancy, const bool coop) { +#define LAUNCH_FWD_KERNEL(OUTER_LOOPS, USE_RELU, USE_ADD_RELU, COMPILED_FOR_OCCUPANCY, COOP) \ + do { \ + CHECK(SMEM_SIZE_FWD <= MAX_SMEM_WITHOUT_OPT_IN) << \ + "Nhwc batchnormaddrelu kernel smem too big."; \ + auto fwd_func = nhwc_batch_norm_fwd< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_FWD, \ + PIXELS_PER_THREAD_IN_SMEM_FWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + USE_RELU, \ + USE_ADD_RELU, \ + COMPILED_FOR_OCCUPANCY>; \ + if (COMPILED_FOR_OCCUPANCY > 1) { \ + cudaFuncSetAttribute(fwd_func, cudaFuncAttributePreferredSharedMemoryCarveout, 100); \ + checkCudaStatus(name_ + " fwd ser coop kernel (cudaFuncSetAttribute carveout)"); \ + } \ + void *params_ptr = static_cast(¶ms); \ + using FWD_FUNC = decltype(nhwc_batch_norm_fwd< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_FWD, \ + PIXELS_PER_THREAD_IN_SMEM_FWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + USE_RELU, \ + USE_ADD_RELU, \ + COMPILED_FOR_OCCUPANCY>); \ + if (COOP) { \ + cudaLaunchCooperativeKernel(fwd_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_FWD, \ + stream); \ + } else { \ + cudaLaunchKernel(fwd_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_FWD, \ + stream); \ + } \ + checkCudaStatus(name_ + " fwd ser coop kernel"); \ + } while (0) + + // Don't try for an occupancy > 2 as this will squeeze register use and create spills. + if (outer_loops == 1) { + if (occupancy >= 2) + LAUNCH_FWD_KERNEL(1, false, true, 2, coop); + else + LAUNCH_FWD_KERNEL(1, false, true, 1, coop); + } else { + if (occupancy >= 2) + LAUNCH_FWD_KERNEL(0, false, true, 2, coop); + else + LAUNCH_FWD_KERNEL(0, false, true, 1, coop); + } +#undef LAUNCH_FWD_KERNEL + } + + // Helper function to launch the backward kernel. + + void _bwdKernelLauncher(cudaStream_t stream, NhwcBatchNormBwdParams params, + dim3 grid_dim, int outer_loops, const int occupancy, const bool coop) { +#define LAUNCH_BWD_ADD_RELU_KERNEL(OUTER_LOOPS, COMPILED_FOR_OCCUPANCY, COOP) \ + do { \ + CHECK(SMEM_SIZE_BWD <= MAX_SMEM_WITHOUT_OPT_IN) << \ + "Nhwc batchnormaddrelu kernel smem too big."; \ + auto bwd_add_relu_func = nhwc_batch_norm_bwd_add_relu< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_BWD, \ + PIXELS_PER_THREAD_IN_SMEM_BWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + COMPILED_FOR_OCCUPANCY>; \ + if (COMPILED_FOR_OCCUPANCY > 1) { \ + cudaFuncSetAttribute(bwd_add_relu_func, \ + cudaFuncAttributePreferredSharedMemoryCarveout, 100); \ + checkCudaStatus(name_ + \ + " bwd-add-relu coop serial kernel (cudaFuncSetAttribute carveout)"); \ + } \ + void *params_ptr = static_cast(¶ms); \ + using BWD_ADD_RELU_FUNC = decltype(nhwc_batch_norm_bwd_add_relu< \ + StorageType, \ + THREADS_PER_CTA, \ + THREADS_PER_PIXEL, \ + PIXELS_PER_THREAD_IN_REGISTERS_BWD, \ + PIXELS_PER_THREAD_IN_SMEM_BWD, \ + ELEMENTS_PER_LDG, \ + USE_ONLINE_APPROACH, \ + OUTER_LOOPS, \ + COMPILED_FOR_OCCUPANCY>); \ + if (COOP) { \ + cudaLaunchCooperativeKernel(bwd_add_relu_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_BWD, \ + stream); \ + } else { \ + cudaLaunchKernel(bwd_add_relu_func, \ + grid_dim, \ + THREADS_PER_CTA, \ + ¶ms_ptr, \ + SMEM_SIZE_BWD, \ + stream); \ + } \ + checkCudaStatus(name_ + " bwd-add-relu coop serial kernel"); \ + } while (0) + + // Don't try for an occupancy > 2 as this will squeeze register use and create spills. + if (outer_loops == 1) { + if (occupancy >= 2) + LAUNCH_BWD_ADD_RELU_KERNEL(1, 2, coop); + else + LAUNCH_BWD_ADD_RELU_KERNEL(1, 1, coop); + } else { + if (occupancy >= 2) + LAUNCH_BWD_ADD_RELU_KERNEL(0, 2, coop); + else + LAUNCH_BWD_ADD_RELU_KERNEL(0, 1, coop); + } +#undef LAUNCH_BWD_KERNEL + } + + public: + // Calculate the expected fwd kernel occupancy, as dictated by shared memory usage. + static int smem_driven_fwd_occupancy(int device_id, const int max_cta_per_sm) { + using namespace at::cuda::utils; + int fwd_reduction_bytes = THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG*sizeof(float); + int fwd_smem_bytes = SMEM_SIZE_FWD + fwd_reduction_bytes; + int occupancy = MaxSharedMemoryPerMultiprocessor(device_id) / fwd_smem_bytes; + return std::min(max_cta_per_sm, occupancy); + } + + // Calculate the expected bwd kernel occupancy, as dictated by shared memory usage. + static int smem_driven_bwd_occupancy(int device_id, const int max_cta_per_sm) { + using namespace at::cuda::utils; + int bwd_reduction_bytes = THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG*sizeof(float); + int bwd_smem_bytes = SMEM_SIZE_BWD + bwd_reduction_bytes; + int occupancy = MaxSharedMemoryPerMultiprocessor(device_id) / bwd_smem_bytes; + return std::min(max_cta_per_sm, occupancy); + } +}; + +const std::vector NhwcBatchNormAddRelu::numWorkspaceBytes() const { + assert(c_ > 0); + + // choose the max memory required between fwd/bwd passes + int grid_x_fwd = div_up(m_, PIXELS_PER_CTA_FWD); + int grid_x_bwd = div_up(m_, PIXELS_PER_CTA_BWD); + int grid_x = max(grid_x_fwd, grid_x_bwd); + int grid_y = div_up(c_, C_ELEMENTS_PER_CTA); + + const size_t num_mean_bytes = c_ * sizeof(float); + const size_t num_variance_bytes = num_mean_bytes; + + int elems_per_group = ((m_ + 31) & ~31) * 2; + int group_count = div_up(c_, C_ELEMENTS_PER_CTA); + const size_t bitmask_bytes = elems_per_group * group_count * sizeof(unsigned int); + + const size_t size_sums = grid_y*grid_x*THREADS_PER_PIXEL*\ + ELEMENTS_PER_LDG*2*sizeof(float); + const size_t size_counts = grid_y*grid_x*sizeof(int); + + return {num_mean_bytes, num_variance_bytes, bitmask_bytes, + size_retired_ctas(grid_y), size_sums, size_counts}; +} + +void NhwcBatchNormAddRelu::setWorkspacePointers( + const std::vector& workspace, + const std::vector& num_workspace_bytes) { + assert(workspace.size() == 6); + assert(num_workspace_bytes.size() == 6); + + minibatch_mean_ = static_cast(workspace[0]); + minibatch_variance_ = static_cast(workspace[1]); + relu_bitmask_ = static_cast(workspace[2]); + retired_ctas_ = static_cast(workspace[3]); + partial_sums_ = static_cast(workspace[4]); + partial_counts_ = static_cast(workspace[5]); +} + +void NhwcBatchNormAddRelu::_setFwdParams(NhwcBatchNormFwdParams *params) const { + params->gmem_src = static_cast(X_); + params->gmem_dst = static_cast(Y_); + params->gmem_src1 = static_cast(addend_); + params->gmem_bias = bias_; + params->gmem_scale = scale_; + params->gmem_running_mean = population_mean_; + params->gmem_running_var = population_variance_; + params->gmem_saved_mean = minibatch_mean_; + params->gmem_saved_var = minibatch_variance_; + params->gmem_relu_bitmask = relu_bitmask_; + params->nhw = m_; + params->c = c_; + params->svar_inv_count = svar_inv_count_; + params->rvar_inv_count = rvar_inv_count_; + params->gmem_sums = partial_sums_; + params->gmem_counts = partial_counts_; + params->gmem_retired_ctas = retired_ctas_; + params->var_eps = eps_; + params->outer_loops = 0; + params->exp_avg_factor = static_cast(exp_avg_factor_); + params->c_blks = div_up(c_, C_ELEMENTS_PER_CTA); +} + +void NhwcBatchNormAddRelu::_setFwdInferenceParams(NhwcBatchNormFwdInferenceParams + *params) const { + params->gmem_src = static_cast(X_); + params->gmem_dst = static_cast(Y_); + params->gmem_src1 = static_cast(addend_); + params->gmem_bias = bias_; + params->gmem_scale = scale_; + params->gmem_mean = population_mean_; + params->gmem_var = population_variance_; + params->nhw = m_; + params->c = c_; + params->var_eps = eps_; +} + +void NhwcBatchNormAddRelu::_setBwdParams(NhwcBatchNormBwdParams *params) const { + params->gmem_src = static_cast(X_); + params->gmem_dy = static_cast(dY_); + params->gmem_dst = static_cast(dX_); + params->gmem_dst1 = static_cast(dAddend_); + params->gmem_relu_bitmask = relu_bitmask_; + params->gmem_dscale = dscale_; + params->gmem_dbias = dbias_; + params->gmem_scale = scale_; + params->gmem_bias = bias_; + params->gmem_saved_mean = minibatch_mean_; + params->gmem_saved_var = minibatch_variance_; + params->nhw = m_; + params->c = c_; + params->svar_inv_count = svar_inv_count_; + params->gmem_sums = partial_sums_; + params->gmem_retired_ctas = retired_ctas_; + params->outer_loops = 0; + params->c_blks = div_up(c_, C_ELEMENTS_PER_CTA); +} + +void NhwcBatchNormAddRelu::fwdInference(cudaStream_t stream) { + bool ptrs_are_set = + X_tensor_desc_ != nullptr + && Y_tensor_desc_ != nullptr + && scale_ != nullptr + && bias_ != nullptr + // && minibatch_mean_ != nullptr + // && minibatch_variance_ != nullptr + && population_mean_ != nullptr + && population_variance_ != nullptr + && X_ != nullptr + // && dX_ != nullptr + && Y_ != nullptr + && addend_ != nullptr + // && dY_ != nullptr + // && dscale_ != nullptr + // && dbias_ != nullptr + && partial_sums_ != nullptr + && partial_counts_ != nullptr; + + if (!ptrs_are_set) + die(); + + dim3 grid_dim; + grid_dim.x = div_up(m_, PIXELS_PER_CTA_FWD_INFERENCE); + grid_dim.y = div_up(c_, C_ELEMENTS_PER_CTA); + + // @todo: maybe just move this inside initialize routine? + NhwcBatchNormFwdInferenceParams params; + _setFwdInferenceParams(¶ms); + + nhwc_batch_norm_fwd_inference + + <<>>(params); + checkCudaStatus(name_ + " fwd_inference-relu kernel"); +} + +dim3 NhwcBatchNormAddRelu::calc_fwd_grid(int *loop, const int grid_dim_x) { + dim3 grid_dim; + grid_dim.x = div_up(m_, PIXELS_PER_CTA_FWD); + int c_blks = div_up(c_, C_ELEMENTS_PER_CTA); + unsigned int max_grid_x = grid_dim_x; + if (grid_dim.x <= max_grid_x) { + *loop = 1; + if (max_grid_x / grid_dim.x > 1) { + grid_dim.y = std::min(c_blks, static_cast(max_grid_x / grid_dim.x)); + assert(grid_dim.y 1) { + grid_dim.y = std::min(c_blks, static_cast(max_grid_x / grid_dim.x)); + assert(grid_dim.y> 1); + + dim3 grid_dim = calc_fwd_grid(¶ms.outer_loops, grid_dim_x); + _fwdKernelLauncher(stream, params, grid_dim, params.outer_loops, occupancy, coop); +} + +void NhwcBatchNormAddRelu::dgrad(cudaStream_t stream, void* my_data, void* pair_data, void* pair_data2, void* pair_data3, + const int bn_group, const int magic, const int occupancy, const int grid_dim_x, const bool coop) { + bool ptrs_are_set = + X_tensor_desc_ != nullptr + && Y_tensor_desc_ != nullptr + && scale_ != nullptr + && bias_ != nullptr + && minibatch_mean_ != nullptr + && minibatch_variance_ != nullptr + && relu_bitmask_ != nullptr + // && population_mean_ != nullptr + // && population_variance_ != nullptr + && X_ != nullptr + && dX_ != nullptr + // && Y_ != nullptr + && dY_ != nullptr + && dAddend_ != nullptr + && dscale_ != nullptr + && dbias_ != nullptr + && retired_ctas_ != nullptr; + + if (!ptrs_are_set) + die(); + + // reset of retired_cta_count no longer needed + + NhwcBatchNormBwdParams params; + _setBwdParams(¶ms); + + params.my_data = my_data; + params.pair_datas[0] = pair_data; + params.pair_datas[1] = pair_data2; + params.pair_datas[2] = pair_data3; + params.magic = magic; + params.sync_iters = (bn_group==8)?3:(bn_group >> 1); + params.wgrad_coeff = 1.0 / bn_group; + + dim3 grid_dim = calc_bwd_grid(¶ms.outer_loops, grid_dim_x); + _bwdKernelLauncher(stream, params, grid_dim, params.outer_loops, occupancy, coop); +} + +#endif // MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_ADD_RELU_H_ diff --git a/apex/apex/contrib/csrc/groupbn/cuda_utils.h b/apex/apex/contrib/csrc/groupbn/cuda_utils.h new file mode 100644 index 00000000..9f003840 --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/cuda_utils.h @@ -0,0 +1,20 @@ +#include +#ifndef CUDA_UTILS_H +#define CUDA_UTILS_H + +namespace at { +namespace cuda { + +namespace utils { + +static inline int MaxSharedMemoryPerMultiprocessor(int device_id) { + return getDeviceProperties(device_id)->sharedMemPerMultiprocessor; +} + + +} +} +} + + +#endif diff --git a/apex/apex/contrib/csrc/groupbn/interface.cpp b/apex/apex/contrib/csrc/groupbn/interface.cpp new file mode 100644 index 00000000..8cea5f9d --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/interface.cpp @@ -0,0 +1,175 @@ +#include +#include +#include + +#include +#include +#include +#include +#include "ATen/Scalar.h" +#ifndef VERSION_GE_1_1 +#include "ATen/Type.h" +#endif +#include "ATen/Tensor.h" +#include "ATen/Storage.h" +#include "ATen/Generator.h" + + +namespace py = pybind11; + +int64_t get_buffer_size( + const int bn_sync_steps); + +void* get_data_ptr( + const at::Tensor& data); + +void* get_remote_data_ptr( + const at::Tensor& handle, + const int64_t offset); + +void close_remote_data( + const at::Tensor& handle); + +at::Tensor nhwc_bn_fwd_train( + const at::Tensor& x, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + const bool fuse_relu, + void* my_data, + void* pair_data, + void* pair_data2, + void* pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop); + +at::Tensor nhwc_bn_fwd_eval( + const at::Tensor& x, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& ret_cta, + const int bn_group, + const float momentum, + const float epsilon, + const bool fuse_relu); + +std::vector nhwc_bn_bwd( + const at::Tensor& x, + const at::Tensor& dy, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + const bool fuse_relu, + void* my_data, + void* pair_data, + void* pair_data2, + void* pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop); + +at::Tensor nhwc_bn_addrelu_fwd_train( + const at::Tensor& x, + const at::Tensor& z, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& bitmask, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + void* my_data, + void* pair_data, + void* pair_data2, + void* pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop); + +at::Tensor nhwc_bn_addrelu_fwd_eval( + const at::Tensor& x, + const at::Tensor& z, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& ret_cta, + const int bn_group, + const float momentum, + const float epsilon); + +std::vector nhwc_bn_addrelu_bwd( + const at::Tensor& x, + const at::Tensor& dy, + const at::Tensor& scale, + const at::Tensor& bias, + const at::Tensor& running_mean, + const at::Tensor& running_inv_var, + const at::Tensor& minibatch_mean, + const at::Tensor& minibatch_inv_var, + const at::Tensor& bitmask, + const at::Tensor& ret_cta, + const float momentum, + const float epsilon, + void* my_data, + void* pair_data, + void* pair_data2, + void* pair_data3, + const int bn_group, + const at::Tensor& magic_tensor, + const int occupancy, + const int grid_dim_x, + const bool coop); + +int nhwc_bn_fwd_occupancy(); +int nhwc_bn_bwd_occupancy(); + +int nhwc_bn_addrelu_fwd_occupancy(); +int nhwc_bn_addrelu_bwd_occupancy(); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + + m.def("get_buffer_size", &get_buffer_size, "get_buffer_size"); + m.def("get_data_ptr", &get_data_ptr, "get_data_ptr"); + m.def("get_remote_data_ptr", &get_remote_data_ptr, "get_remote_data_ptr"); + m.def("close_remote_data", &close_remote_data, "close_remote_data"); + + m.def("bn_fwd_nhwc", &nhwc_bn_fwd_train, "bn_fwd_nhwc"); + m.def("bn_fwd_eval_nhwc", &nhwc_bn_fwd_eval, "bn_fwd_eval_nhwc"); + m.def("bn_bwd_nhwc", &nhwc_bn_bwd, "bn_bwd_nhwc"); + + m.def("bn_fwd_nhwc_occupancy", &nhwc_bn_fwd_occupancy, "bn_fwd_nhwc_occupancy"); + m.def("bn_bwd_nhwc_occupancy", &nhwc_bn_bwd_occupancy, "bn_bwd_nhwc_occupancy"); + + m.def("bn_addrelu_fwd_nhwc", &nhwc_bn_addrelu_fwd_train, "bn_addrelu_fwd_nhwc"); + m.def("bn_addrelu_fwd_eval_nhwc", &nhwc_bn_addrelu_fwd_eval, "bn_addrelu_fwd_eval_nhwc"); + m.def("bn_addrelu_bwd_nhwc", &nhwc_bn_addrelu_bwd, "bn_addrelu_bwd_nhwc"); + + m.def("bn_addrelu_fwd_nhwc_occupancy", &nhwc_bn_addrelu_fwd_occupancy, "bn_addrelu_fwd_nhwc_occupancy"); + m.def("bn_addrelu_bwd_nhwc_occupancy", &nhwc_bn_addrelu_bwd_occupancy, "bn_addrelu_bwd_nhwc_occupancy"); +} + diff --git a/apex/apex/contrib/csrc/groupbn/ipc.cu b/apex/apex/contrib/csrc/groupbn/ipc.cu new file mode 100644 index 00000000..6b152a0d --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/ipc.cu @@ -0,0 +1,129 @@ +#include +#include + +#include + +#include "compat.h" + + +#define cudaCheckErrors(msg) \ + do { \ + cudaError_t __err = cudaGetLastError(); \ + if (__err != cudaSuccess) { \ + fprintf(stderr, "Fatal error: %s (%s at %s:%d)\n", \ + msg, cudaGetErrorString(__err), \ + __FILE__, __LINE__); \ + fprintf(stderr, "*** FAILED - ABORTING\n"); \ + exit(1); \ + } \ + } while (0) + +template<> +struct std::hash { + size_t operator() (const cudaIpcMemHandle_t& handle) const { + size_t hash = 0; + uint8_t* ptr = (uint8_t*)&handle; + assert(sizeof(uint8_t) == 1); + for (int i=0; i +struct std::equal_to { + bool operator() (const cudaIpcMemHandle_t &lhs, + const cudaIpcMemHandle_t &rhs) const { + return (std::memcmp((void*) &lhs, + (void*) &rhs, + sizeof(cudaIpcMemHandle_t)) == 0); + } +}; + +namespace { + +namespace gpuipc { +//from: src/operator/nn/cudnn/nhwc_batch_norm_kernel.h +// The number of threads per pixel. +const int THREADS_PER_PIXEL = 16; +// The number of elements per ldg. +const int ELEMENTS_PER_LDG = 4; +// The number of reducing ops, each uses its own space : mean, var, dscale, dbias +const int REDUCE_OPS = 4; +// Maximum block.y supported - limited due to buffer allocation +const int MAX_BLOCK_Y = 256; +const int MAX_OFFSET = REDUCE_OPS*MAX_BLOCK_Y; +const int BYTES_PER_ELEM = 4; +// Buffer size per sync step +const int SINGLE_SYNC_BUFFER_BYTES = MAX_OFFSET*THREADS_PER_PIXEL*2*ELEMENTS_PER_LDG*BYTES_PER_ELEM; +}; + +class IpcMemHandleRegistry { +public: + void* getPtr(const cudaIpcMemHandle_t& handle, int64_t offset) { + if (registry_.count(handle) == 0) { + registry_.insert(std::make_pair(handle, RegistryEntry())); + registry_[handle].dev_ptr = ipcOpenMem(handle); + } + registry_[handle].ref_count++; + return (((uint8_t*)registry_[handle].dev_ptr) + offset); + } + + void releasePtr(const cudaIpcMemHandle_t& handle) { + if (registry_.count(handle) == 0) { + } + if (--registry_[handle].ref_count == 0) { + ipcCloseMem(registry_[handle].dev_ptr); + registry_.erase(handle); + } + } + + struct RegistryEntry { + void* dev_ptr; + int ref_count; + RegistryEntry() : dev_ptr(NULL) , ref_count(0) {} + }; + +protected: + std::unordered_map registry_; + + void* ipcOpenMem(const cudaIpcMemHandle_t& handle) { + void *data; + cudaIpcOpenMemHandle(&data, handle, cudaIpcMemLazyEnablePeerAccess); + cudaCheckErrors("ipc init"); + return data; + } + + void ipcCloseMem(void* dev_ptr) { + cudaIpcCloseMemHandle(dev_ptr); + cudaCheckErrors("ipc close"); + } + +}; + +} + +static IpcMemHandleRegistry ipc_mem_registry; + +int64_t get_buffer_size(const int bn_sync_steps) { + return bn_sync_steps * gpuipc::SINGLE_SYNC_BUFFER_BYTES; +} + +void* get_remote_data_ptr(const at::Tensor& handle, const int64_t offset) { + cudaIpcMemHandle_t my_handle; + memcpy((unsigned char *)(&my_handle), handle.DATA_PTR(), sizeof(my_handle)); + return ipc_mem_registry.getPtr(my_handle, offset); +} + +void close_remote_data(const at::Tensor& handle) { + cudaIpcMemHandle_t my_handle; + memcpy((unsigned char *)(&my_handle), handle.DATA_PTR(), sizeof(my_handle)); + ipc_mem_registry.releasePtr(my_handle); +} + +void* get_data_ptr( + const at::Tensor& data) { + return data.DATA_PTR(); +} diff --git a/apex/apex/contrib/csrc/groupbn/nhwc_batch_norm_kernel.h b/apex/apex/contrib/csrc/groupbn/nhwc_batch_norm_kernel.h new file mode 100644 index 00000000..8430f309 --- /dev/null +++ b/apex/apex/contrib/csrc/groupbn/nhwc_batch_norm_kernel.h @@ -0,0 +1,2685 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2018 by Contributors + * \file nhwc_batch_norm_kernel.h + * \brief CUDA NHWC Batch Normalization code + * \author Shankara Rao Thejaswi Nanditale, Dick Carter, Maxim Milakov, Evgeni Krimer +*/ +#ifndef MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_KERNEL_H_ +#define MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_KERNEL_H_ + +#include +#include + +#define DEVICE_FUNCTION static inline __device__ + +// CTA margin used by cooperative launch. Can be overridden by env var NHWC_BATCHNORM_LAUNCH_MARGIN. +#define NHWC_BATCHNORM_LAUNCH_MARGIN_MIN 3 +#define NHWC_BATCHNORM_LAUNCH_MARGIN_DEFAULT NHWC_BATCHNORM_LAUNCH_MARGIN_MIN + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< typename T, int ELEMENTS_PER_LDG > +struct PackedStorage { + enum { PACKED_ELEMENTS_PER_LDG = ELEMENTS_PER_LDG }; + typedef T Type; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int ELEMENTS_PER_LDG > +struct PackedStorage { + enum { PACKED_ELEMENTS_PER_LDG = ELEMENTS_PER_LDG/2 }; + typedef int Type; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void from_float(int (&dst)[N], const float (&src)[2*N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + uint16_t lo, hi; + asm volatile("cvt.rn.f16.f32 %0, %1;" : "=h"(lo) : "f"(src[2*i+0])); + asm volatile("cvt.rn.f16.f32 %0, %1;" : "=h"(hi) : "f"(src[2*i+1])); + asm volatile("mov.b32 %0, {%1, %2};" : "=r"(dst[i]) : "h"(lo), "h"(hi)); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void from_float(float (&dst)[N], const float (&src)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + dst[i] = src[i]; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void to_float(float (&dst)[2*N], int (&src)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + uint16_t lo, hi; + asm volatile("mov.b32 {%0, %1}, %2;" : "=h"(lo), "=h"(hi) : "r"(src[i])); + asm volatile("cvt.f32.f16 %0, %1;" : "=f"(dst[2*i+0]) : "h"(lo)); + asm volatile("cvt.f32.f16 %0, %1;" : "=f"(dst[2*i+1]) : "h"(hi)); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void to_float(float (&dst)[N], float (&src)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + dst[i] = src[i]; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void ldg(int (&dst)[1], const uint16_t *gmem) { + dst[0] = __ldg((const int*) gmem); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void ldg_stream(int (&dst)[1], const uint16_t *gmem) { + unsigned int tmp; + asm volatile ("ld.global.cs.nc.s32 %0, [%1];" : "=r"(tmp) : "l" ((const uint *)gmem)); + dst[0] = tmp; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void ldg(int (&dst)[2], const uint16_t *gmem) { + int2 tmp = __ldg((const int2*) gmem); + dst[0] = tmp.x; + dst[1] = tmp.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void ldg_stream(int (&dst)[2], const uint16_t *gmem) { + int2 tmp; + asm volatile ("ld.global.cs.nc.v2.s32 {%0,%1}, [%2];" + : "=r"(tmp.x), "=r"(tmp.y) : "l"((const int2 *)gmem)); + dst[0] = tmp.x; + dst[1] = tmp.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void ldg(float (&dst)[N], const uint16_t *gmem) { + int tmp[N/2]; + ldg(tmp, gmem); + to_float(dst, tmp); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void ldg_stream(float (&dst)[N], const uint16_t *gmem) { + int tmp[N/2]; + ldg_stream(tmp, gmem); + to_float(dst, tmp); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void stg(uint16_t *gmem, int (&src)[1]) { + reinterpret_cast(gmem)[0] = src[0]; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void stg_stream(uint16_t *gmem, int (&src)[1]) { + unsigned int tmp = src[0]; + asm volatile ("st.global.cs.s32 [%0], %1;" + :: "l"((uint *)gmem) , "r"(tmp)); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void stg(uint16_t *gmem, int (&src)[2]) { + reinterpret_cast(gmem)[0] = make_int2(src[0], src[1]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void stg_stream(uint16_t *gmem, int (&src)[2]) { + asm volatile ("st.global.cs.v2.s32 [%0], {%1,%2};" + :: "l"((uint *)gmem) , "r"(src[0]), "r"( src[1])); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void stg(uint16_t *gmem, float (&src)[N]) { + int tmp[N/2]; + from_float(tmp, src); + stg(gmem, tmp); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void stg_stream(uint16_t *gmem, float (&src)[N]) { + int tmp[N/2]; + from_float(tmp, src); + stg_stream(gmem, tmp); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void read_from_gmem(float (&dst)[2], const float *gmem, int idx) { + float2 tmp = __ldg(reinterpret_cast(&gmem[2*idx])); + dst[0] = tmp.x; + dst[1] = tmp.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void read_from_gmem(float (&dst)[4], const float *gmem, int idx) { + float4 tmp = __ldg(reinterpret_cast(&gmem[4*idx])); + dst[0] = tmp.x; + dst[1] = tmp.y; + dst[2] = tmp.z; + dst[3] = tmp.w; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void read_from_smem(float (&x)[2], const float *smem, int idx) { + float2 tmp = *(const float2*) &smem[2*idx]; + x[0] = tmp.x; + x[1] = tmp.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void read_from_smem(int (&x)[1], const int *smem, int idx) { + x[0] = smem[idx]; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void read_from_smem(float (&x)[4], const float *smem, int idx) { + float4 tmp = *(const float4*) &smem[4*idx]; + x[0] = tmp.x; + x[1] = tmp.y; + x[2] = tmp.z; + x[3] = tmp.w; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void read_from_smem(int (&x)[2], const int *smem, int idx) { + int2 tmp = *(const int2*) &smem[2*idx]; + x[0] = tmp.x; + x[1] = tmp.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void write_to_gmem(float *gmem, int idx, const float (&src)[2]) { + reinterpret_cast(&gmem[2*idx])[0] = make_float2(src[0], src[1]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void write_to_gmem(float *gmem, int idx, const float (&src)[4]) { + reinterpret_cast(&gmem[4*idx])[0] = make_float4(src[0], src[1], src[2], src[3]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void scaled_write_to_gmem(float *gmem, int idx, const float (&src)[4], const float coeff) { + reinterpret_cast(&gmem[4*idx])[0] = make_float4(src[0]*coeff, src[1]*coeff, src[2]*coeff, src[3]*coeff); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void write_to_smem(float *smem, int idx, const float (&x)[2]) { + reinterpret_cast(&smem[2*idx])[0] = make_float2(x[0], x[1]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void write_to_smem(int *smem, int idx, const int (&x)[1]) { + smem[idx] = x[0]; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void write_to_smem(float *smem, int idx, const float (&x)[4]) { + reinterpret_cast(&smem[4*idx])[0] = make_float4(x[0], x[1], x[2], x[3]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +DEVICE_FUNCTION void write_to_smem(int *smem, int idx, const int (&x)[2]) { + reinterpret_cast(&smem[2*idx])[0] = make_int2(x[0], x[1]); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void zero_array(int (&dst)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + dst[i] = 0; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int N > +DEVICE_FUNCTION void zero_array(float (&dst)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + dst[i] = 0.f; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void add(float (&x)[N], const float (&y)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + x[i] += y[i]; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void multiply(float (&x)[N], const float (&y)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + x[i] *= y[i]; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void scale_(float (&x)[N], float scalar) { + #pragma unroll + for (int i = 0; i < N; ++i) { + x[i] *= scalar; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void normalize(float (&x)[N], const float (&bias)[N], + const float (&scale)[N], const float (&m1)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + x[i] = bias[i] + scale[i] * (x[i] - m1[i]); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION Storage relu(Storage in) { + Storage zero = (Storage)0.f; + return (in < zero)? zero : in; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void relu_activation(float (&x)[N]) { + #pragma unroll + for (int i = 0; i < N; ++i) { + x[i] = relu(x[i]); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// +template< int THREADS_PER_CTA > +DEVICE_FUNCTION void parallel_sums_16x2(float *smem, float (&x)[4], int nhw, + void* params_my_data, void** params_pair_datas, int off, + const int magic, + const int sync_iters) { + // The size of a warp. + const int THREADS_PER_WARP = 32; + // The number of warps in a CTA. + const int WARPS_PER_CTA = THREADS_PER_CTA / THREADS_PER_WARP; + // The number of threads per pixel. + const int THREADS_PER_PIXEL = 16; + // The number of elements per ldg. + const int ELEMENTS_PER_LDG = 4; + // The number of reducing ops, each uses its own space : mean, var, dscale, dbias + const int REDUCE_OPS = 4; + // Maximum block.y supported - limited due to buffer allocation + const int MAX_BLOCK_Y = 256; + const int MAX_OFFSET = REDUCE_OPS*MAX_BLOCK_Y; + // The warp decomposition. + const int warp_id = threadIdx.x / THREADS_PER_WARP; + const int lane_id = threadIdx.x % THREADS_PER_WARP; + // total size of data per sync iter + const int data_total = MAX_OFFSET*THREADS_PER_PIXEL*ELEMENTS_PER_LDG*2; + + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + x[i] += __shfl_sync(0xffffffffU, x[i], THREADS_PER_PIXEL+lane_id); + } + + // The warp leaders, write to SMEM. + if (lane_id < THREADS_PER_PIXEL) { + write_to_smem(smem, warp_id*THREADS_PER_PIXEL + lane_id, x); + } + + // The data is in SMEM. Do the final reduction. + __syncthreads(); + + // The 1st warp does all the work. + // We do the final reduction each half-warp sequentially reduces the final values. + if (warp_id == 0) { + read_from_smem(x, smem, threadIdx.x); + + #pragma unroll + for (int offset = 1; + offset < WARPS_PER_CTA/(THREADS_PER_WARP / THREADS_PER_PIXEL); ++offset) { + float y[ELEMENTS_PER_LDG]; + // Read the mean and variance from the other pixel. + read_from_smem(y, smem, threadIdx.x + offset*THREADS_PER_WARP); + // Compute the updated sum. + add(x, y); + } + + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + x[i] += __shfl_sync(0xffffffffU, x[i], THREADS_PER_PIXEL+lane_id); + } + + // Make sure the data was read from SMEM. + __syncwarp(); + + // Store the final values. + if (threadIdx.x < THREADS_PER_PIXEL) { + // probably could do it earlier, before sync + + for (int sync_iter=0; sync_iter < sync_iters; ++sync_iter) { + //float* params_pair_data = (reinterpret_cast(params_pair_datas))[sync_iter]; + void* params_pair_data = params_pair_datas[sync_iter]; + + // skip the space consumed by previous sync iterations + const int xbuf_offset = sync_iter*data_total; + // data starts after flags, but have to skip previous + const int data_offset = xbuf_offset + + off*ELEMENTS_PER_LDG*THREADS_PER_PIXEL*2 + + ELEMENTS_PER_LDG*threadIdx.x*2; + + // after sums for this GPU were computed, let CTA0 broadcast the sum to over GPU + if (blockIdx.x == 0) { + volatile float * write_data = + &((reinterpret_cast(params_pair_data))[data_offset]); + + // write the data to memory region to be reflected to other GPU + asm volatile ("st.global.wt.v4.b32 [%0], {%1,%2,%3,%4};" + :: "l"(write_data) , "f"(x[0]), "r"(magic), "f"(x[2]), "r"(magic)); + + asm volatile ("st.global.wt.v4.b32 [%0], {%1,%2,%3,%4};" + :: "l"(write_data+4) , "f"(x[1]), "r"(magic), "f"(x[3]), "r"(magic)); + } + + // now each CTA (on each GPU) reads the data written by CTA 0 of the other GPU + volatile float * read_data = + &((reinterpret_cast(params_my_data))[data_offset]); + + float other[4]; + uint32_t other_flag_a, other_flag_b; + do { + asm volatile ("ld.volatile.global.v4.b32 {%0, %1, %2, %3}, [%4];" + : "=f"(other[0]), "=r"(other_flag_a), "=f"(other[2]), "=r"(other_flag_b) : "l"(read_data)); + } while ((other_flag_a != magic) || (other_flag_b != magic)); + + do { + asm volatile ("ld.volatile.global.v4.b32 {%0, %1, %2, %3}, [%4];" + : "=f"(other[1]), "=r"(other_flag_a), "=f"(other[3]), "=r"(other_flag_b) : "l"(read_data+4)); + } while ((other_flag_a != magic) || (other_flag_b != magic)); + + add(x, other); + } + // finally, after syncing up and accounting for partial sums from + // other GPUs as required, write the result + + + write_to_smem(smem, threadIdx.x, x); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int THREADS_PER_CTA > +DEVICE_FUNCTION void parallel_sums_8x4(float *smem, float (&x)[4], int nhw) { + // The size of a warp. + const int THREADS_PER_WARP = 32; + // The number of warps in a CTA. + const int WARPS_PER_CTA = THREADS_PER_CTA / THREADS_PER_WARP; + // The number of threads per pixel. + const int THREADS_PER_PIXEL = 8; + // The number of elements per ldg. + const int ELEMENTS_PER_LDG = 4; + // The warp decomposition. + const int warp_id = threadIdx.x / THREADS_PER_WARP; + const int lane_id = threadIdx.x % THREADS_PER_WARP; + + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + x[i] += __shfl_sync(0xffffffffU, x[i], THREADS_PER_PIXEL+lane_id); + x[i] += __shfl_sync(0xffffffffU, x[i], THREADS_PER_PIXEL*2+lane_id); + } + + // The warp leaders, write to SMEM. + if (lane_id < THREADS_PER_PIXEL) { + write_to_smem(smem, warp_id*THREADS_PER_PIXEL + lane_id, x); + } + + // The data is in SMEM. Do the final reduction. + __syncthreads(); + + // The 1st warp does all the work. + // We do the final reduction each half-warp sequentially reduces the final values. + if (warp_id == 0) { + read_from_smem(x, smem, threadIdx.x); + + #pragma unroll + for (int offset = 1; + offset < WARPS_PER_CTA/(THREADS_PER_WARP / THREADS_PER_PIXEL); ++offset) { + float y[ELEMENTS_PER_LDG]; + // Read the mean and variance from the other pixel. + read_from_smem(y, smem, threadIdx.x + offset*THREADS_PER_WARP); + // Compute the updated sum. + add(x, y); + } + + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + x[i] += __shfl_sync(0xffffffffU, x[i], THREADS_PER_PIXEL+lane_id); + x[i] += __shfl_sync(0xffffffffU, x[i], THREADS_PER_PIXEL*2+lane_id); + } + + // Make sure the data was read from SMEM. + __syncwarp(); + + // Store the final values. + if (threadIdx.x < THREADS_PER_PIXEL) { + write_to_smem(smem, threadIdx.x, x); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int THREADS_PER_CTA, int THREADS_PER_PIXEL, int ELEMENTS_PER_LDG > +DEVICE_FUNCTION void parallel_sums(float *smem, float (&x)[ELEMENTS_PER_LDG], int nhw) { + // The size of a warp. + const int THREADS_PER_WARP = 32; + // The number of warps in a CTA. + const int WARPS_PER_CTA = THREADS_PER_CTA / THREADS_PER_WARP; + // The number of pixels computed by a single warp. + const int PIXELS_PER_WARP = THREADS_PER_WARP / THREADS_PER_PIXEL; + + // The position in the warp. + const int nhw_in_warp = nhw % PIXELS_PER_WARP; + // The C in the warp. + const int c_in_warp = threadIdx.x % THREADS_PER_PIXEL; + + // Store the values to shared memory. + write_to_smem(smem, threadIdx.x, x); + + // Compute the parallel sums. + for (int offset = PIXELS_PER_WARP/2; offset > 0; offset /= 2) { + // NOP. + __syncwarp(); + + // Read the running sum from the other thread. + float y[ELEMENTS_PER_LDG]; + if (nhw_in_warp < offset) { + read_from_smem(y, smem, threadIdx.x + offset*THREADS_PER_PIXEL); + } + + // Compute the updated sum. + add(x, y); + + // NOP. + __syncwarp(); + + // Update the sum in SMEM. + if (offset > 1 && nhw_in_warp < offset) { + write_to_smem(smem, threadIdx.x, x); + } + } + + // The warps are done. Do the final reduction at the CTA level. + __syncthreads(); + + // The warp leaders, write to SMEM. + const int idx = (threadIdx.x/THREADS_PER_WARP)*THREADS_PER_PIXEL + c_in_warp; + if (nhw_in_warp == 0) { + write_to_smem(smem, idx, x); + } + + // The data is in SMEM. Do the final reduction. + __syncthreads(); + + // Read the 1st element to prepare the work. + if (nhw < WARPS_PER_CTA/2) { + read_from_smem(x, smem, threadIdx.x); + } + + // We have the running mean and running m2. Let's build the mean/var of the CTA. + for (int offset = WARPS_PER_CTA/2; offset > 0; offset /= 2) { + // NOP. + __syncwarp(); + + // Read the mean and variance from the other pixel. + float y[ELEMENTS_PER_LDG]; + if (nhw < offset) { + read_from_smem(y, smem, threadIdx.x + offset*THREADS_PER_PIXEL); + } + + // Compute the updated sum. + add(x, y); + + // NOP. + __syncwarp(); + + // Store the mean/var for the different pixels. + if (nhw < offset) { + write_to_smem(smem, threadIdx.x, x); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< int THREADS_PER_PIXEL, int ELEMENTS_PER_LDG > +struct ParallelSums { + template< int THREADS_PER_CTA > + DEVICE_FUNCTION void dispatch(float *smem, float (&x)[ELEMENTS_PER_LDG], int nhw) { + parallel_sums(smem, x, nhw); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template<> +struct ParallelSums<16, 4> { + template< int THREADS_PER_CTA > + DEVICE_FUNCTION void dispatch(float *smem, float (&x)[4], int nhw) { + parallel_sums_16x2(smem, x, nhw, 0, 0, 0, 0, 0); + } + + template< int THREADS_PER_CTA > + DEVICE_FUNCTION void dispatchX(float *smem, float (&x)[4], int nhw, void* params_my_data, void** params_pair_datas, int off, const int magic, const unsigned int& sync_iters) { + parallel_sums_16x2(smem, x, nhw, params_my_data, params_pair_datas, off, magic, sync_iters); + } +}; + +template<> +struct ParallelSums<8, 4> { + template< int THREADS_PER_CTA > + DEVICE_FUNCTION void dispatch(float *smem, float (&x)[4], int nhw) { + parallel_sums_8x4(smem, x, nhw); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +static inline int div_up(int m, int n) { + return (m + n - 1) / n; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// It is expected that all threads in the CTA enter this function! +DEVICE_FUNCTION void inter_block_sync(int* gmem_retired_ctas, int expected_count, bool master) { + + // Register the CTA. + if (threadIdx.x == 0) { + // Issue the membar. + __threadfence(); + // Notify that the CTA is done. + int val_to_add = 1; + if (master) { + val_to_add = -(expected_count - 1); + } + atomicAdd(gmem_retired_ctas, val_to_add); + } + + // Are all CTAs done? + if (threadIdx.x == 0) { + int retired_ctas = -1; + do { + __threadfence(); + asm volatile ("ld.global.cg.b32 %0, [%1];" + : "=r"(retired_ctas) : "l"(gmem_retired_ctas)); + } while (retired_ctas != 0); + } + __syncthreads(); + +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct NhwcBatchNormFwdInferenceParams { + // The input/output tensors. + uint16_t *gmem_src, *gmem_dst, *gmem_src1; + // the final mean and variance as calculated during the training process + float *gmem_mean, *gmem_var; + // The bias/scale. + float *gmem_bias, *gmem_scale; + // The dimensions. + int nhw, c; + // epsilon + float var_eps; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +// No DESIRED_OCCUPANCY launch bounds needed, as this is not launched cooperatively +template< + typename Storage, + int THREADS_PER_CTA, + int THREADS_PER_PIXEL, + int ELEMENTS_PER_LDG, + bool USE_RELU, + bool USE_ADD_RELU +> +__global__ __launch_bounds__(THREADS_PER_CTA) + void nhwc_batch_norm_fwd_inference(NhwcBatchNormFwdInferenceParams params) { + // The number of pixels loaded in a single LDG. + const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + // The number of C elements per CTA. + const int C_ELEMENTS_PER_CTA = THREADS_PER_PIXEL*ELEMENTS_PER_LDG; + + // The start position in the NHW dimension where the CTA starts. + const int cta_nhw_stride = gridDim.x * PIXELS_PER_LDG; + // Compute the NHW coordinate of the thread in the CTA. + const int thread_in_cta_nhw = threadIdx.x / THREADS_PER_PIXEL; + // thread's starting point in NHW + const int thread_nhw = thread_in_cta_nhw + blockIdx.x * PIXELS_PER_LDG; + + // The position in the C dimension where the CTA starts. + const int cta_c = blockIdx.y * C_ELEMENTS_PER_CTA; + // Compute the C coordinate of the thread in the CTA. + const int thread_in_cta_c = threadIdx.x % THREADS_PER_PIXEL; + // Compute the C coordinate of the thread. + const int thread_c = cta_c + thread_in_cta_c*ELEMENTS_PER_LDG; + + // Is the thread working on a valid C dimension? + const int is_valid_c = thread_c < params.c; + + float mean[ELEMENTS_PER_LDG], var[ELEMENTS_PER_LDG]; + float scale[ELEMENTS_PER_LDG], bias[ELEMENTS_PER_LDG]; + zero_array(mean); + zero_array(var); + zero_array(scale); + zero_array(bias); + if (is_valid_c) { + read_from_gmem(var, ¶ms.gmem_var[cta_c], thread_in_cta_c); + read_from_gmem(scale, ¶ms.gmem_scale[cta_c], thread_in_cta_c); + read_from_gmem(mean, ¶ms.gmem_mean[cta_c], thread_in_cta_c); + read_from_gmem(bias, ¶ms.gmem_bias[cta_c], thread_in_cta_c); + } + + // Update the scale with the stddev and eps. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + scale[i] *= rsqrtf(var[i] + params.var_eps); + } + + // The base pointers for reading/writing + uint16_t *const gmem_src = ¶ms.gmem_src[thread_c]; + uint16_t *const gmem_dst = ¶ms.gmem_dst[thread_c]; + const uint16_t *gmem_src1 = nullptr; + if (USE_ADD_RELU) { + gmem_src1 = ¶ms.gmem_src1[thread_c]; + } + + // apply BN + for (int nhw = thread_nhw; nhw < params.nhw; nhw += cta_nhw_stride) { + float x_math[ELEMENTS_PER_LDG]; + zero_array(x_math); + if (is_valid_c) { + ldg(x_math, &gmem_src[nhw*params.c]); + } + + // Normalize and apply activation function + normalize(x_math, bias, scale, mean); + if (USE_ADD_RELU) { + float x1_math[ELEMENTS_PER_LDG]; + ldg(x1_math, &gmem_src1[nhw*params.c]); + add(x_math, x1_math); + relu_activation(x_math); + } else if (USE_RELU) { + relu_activation(x_math); + } + + if (is_valid_c) { + stg(&gmem_dst[nhw*params.c], x_math); + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct NhwcBatchNormFwdParams { + // The input/output tensors. + uint16_t *gmem_src, *gmem_dst, *gmem_src1; + // The bias/scale. + float *gmem_bias, *gmem_scale; + // running mean/var (refer BN API from cudnn doc) + float *gmem_running_mean, *gmem_running_var; + // saved mean/var (refer BN API from cudnn doc) + float *gmem_saved_mean, *gmem_saved_var; + // ReLU bitmask + unsigned int *gmem_relu_bitmask; + // The dimensions. + int nhw, c; + // factor to scale sum of squared errors to get saved variance. Must be 1/nhw. + float svar_inv_count; + // factor to scale sum of squared errors to get running variance. Should be 1/nhw or 1/(nhw-1). + float rvar_inv_count; + // The buffer to do the reduction for mean, stddev and count. + float *gmem_sums; + // The buffer to count items in the different CTAs. + int *gmem_counts; + // The counters of retired CTAs. + int *gmem_retired_ctas; + // The epsilon to apply to the computation of the variance. + float var_eps; + // outer loop count + int outer_loops; + // exponential average factor + float exp_avg_factor; + // number of CTAs along .x dimension + int c_blks; + + void* my_data; + void* pair_datas[4]; + int magic; + int sync_iters; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + typename Storage, + int THREADS_PER_CTA, + int THREADS_PER_PIXEL, + int PIXELS_PER_THREAD_IN_REGISTERS, + int PIXELS_PER_THREAD_IN_SMEM, + int ELEMENTS_PER_LDG, + int USE_ONLINE_APPROACH, + int OUTER_LOOPS_, + bool USE_RELU, + bool USE_ADD_RELU, + int DESIRED_OCCUPANCY +> +__global__ __launch_bounds__(THREADS_PER_CTA, DESIRED_OCCUPANCY) + void nhwc_batch_norm_fwd(NhwcBatchNormFwdParams params) { + // The number of pixels loaded in a single LDG. + const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + // The number of pixels computed per CTA stored in registers. + const int PIXELS_PER_CTA_IN_REGISTERS = PIXELS_PER_THREAD_IN_REGISTERS * PIXELS_PER_LDG; + // The number of pixels computed per CTA stored in SMEM. + const int PIXELS_PER_CTA_IN_SMEM = PIXELS_PER_THREAD_IN_SMEM*PIXELS_PER_LDG; + // The number of C elements per CTA. + const int C_ELEMENTS_PER_CTA = THREADS_PER_PIXEL*ELEMENTS_PER_LDG; + + // Shared memory to do CTA-wide parallel sums. + __shared__ float smem[THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG]; + + // Compute the NHW coordinate of the thread in the CTA. + const int thread_in_cta_nhw = threadIdx.x / THREADS_PER_PIXEL; + + // The adapter for the storage. + typedef PackedStorage PackedStorage_; + // The data type for packed storage in SMEM. + typedef typename PackedStorage_::Type PackedStorageType; + // The number of elements in the packed storage. + const int PACKED_ELEMENTS_PER_LDG = PackedStorage_::PACKED_ELEMENTS_PER_LDG; + // Registers to keep the data live for the persistent approach. + PackedStorageType x_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + + // Shared memory buffer to store the extra pixels. + extern __shared__ PackedStorageType smem_storage_packed[]; + + for (int c_blk_index = blockIdx.y; c_blk_index < params.c_blks; c_blk_index += gridDim.y) { + // The position in the NHW dimension where the CTA starts. + int cta_nhw_regs = blockIdx.x * PIXELS_PER_CTA_IN_REGISTERS; + // The position in the NHW dimension where the CTA starts for the portion in SMEM. + int cta_nhw_smem = blockIdx.x * PIXELS_PER_CTA_IN_SMEM; + + // The position in the C dimension where the CTA starts. + const int cta_c = c_blk_index * C_ELEMENTS_PER_CTA; + // Compute the C coordinate of the thread in the CTA. + const int thread_in_cta_c = threadIdx.x % THREADS_PER_PIXEL; + // Compute the C coordinate of the thread. + int thread_c = cta_c + thread_in_cta_c*ELEMENTS_PER_LDG; + + // Is the thread working on a valid C dimension? + const int is_valid_c = thread_c < params.c; + + // Clamp thread_c so that we load from valid locations even if we don't use the value + if (!is_valid_c) + thread_c = params.c - 4; + + // Single pass numerically stable algorithm, see: + // https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm + // + // n = 0, mean = 0.0, M2 = 0.0 + // + // for x in data: + // n += 1 + // delta = x - mean + // mean += delta/n + // delta2 = x - mean + // M2 += delta*delta2 + // + // if n < 2: + // return float('nan') + // else: + // return M2 / (n - 1) + + // Register to store the number of elements read so far. + float count = 0.f, mean[ELEMENTS_PER_LDG], m2[ELEMENTS_PER_LDG]; + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + mean[i] = 0.f; + m2[i] = 0.f; + } + + // The number of elements loaded by this CTA. + int cta_count = 0; + // The base pointer to load from. + const uint16_t *gmem_src = ¶ms.gmem_src[thread_c]; + + // outer loops + int OUTER_LOOPS = OUTER_LOOPS_ == 1? 1 : params.outer_loops; + // Load the batch of elements. Compute the mean/var across those elements. + const int pixels_per_iteration = PIXELS_PER_CTA_IN_REGISTERS*gridDim.x; + + if (OUTER_LOOPS_ != 1) { + // We cannot load everything to store persistently, so let's makes sure registers and + // smem are fully utilized, offset is evenly divisible by 32 + int offset = (pixels_per_iteration * OUTER_LOOPS + + PIXELS_PER_CTA_IN_SMEM * gridDim.x - params.nhw) & ~31; + cta_nhw_regs -= offset; + cta_nhw_smem -= offset; + } + + #pragma unroll 1 + for (int loop_i = 0; loop_i < OUTER_LOOPS; ++loop_i) { + // The nhw position. + int nhw_regs = cta_nhw_regs + loop_i*pixels_per_iteration; + // Update the number of elements loaded by this CTA. TODO: Skip if <= 0!!! + cta_count += max(min(nhw_regs + PIXELS_PER_CTA_IN_REGISTERS, params.nhw) - + max(nhw_regs, 0), 0); + + // Load the data and compute the local mean/sum and the variance. + if (USE_ONLINE_APPROACH) { + // Read the elements from memory. + float is_valid[PIXELS_PER_THREAD_IN_REGISTERS]; + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = nhw_regs + thread_in_cta_nhw + i*PIXELS_PER_LDG; + zero_array(x_storage[i]); + is_valid[i] = 0.f; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + if (loop_i == OUTER_LOOPS - 1) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + } else { + ldg(x_storage[i], &gmem_src[idx*params.c]); + } + is_valid[i] = 1.f; + } + } + + // Do the math. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float. + float x_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + + // Update the count. + count += is_valid[i]; + // Invert the count. + float inv_count = is_valid[i] ? 1.f / count : 0.f; + + // Update the mean and m2 using deltas. + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + float delta0 = x_math[j] - mean[j]; + mean[j] += delta0 * inv_count; + float delta1 = x_math[j] - mean[j]; + m2[j] += delta0 * delta1 * is_valid[i]; + } + } + } else { + // Read the elements from memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = nhw_regs + thread_in_cta_nhw + i*PIXELS_PER_LDG; + zero_array(x_storage[i]); + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + if (loop_i == OUTER_LOOPS - 1) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + } else { + ldg(x_storage[i], &gmem_src[idx*params.c]); + } + count += 1.f; + } + } + + // Sum the elements in registers. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float. + float x_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + + // Update the mean and m2 using deltas. + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + mean[j] += x_math[j]; + } + } + + // Compute the mean. + float inv_count = 1.f / count; + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + mean[j] *= inv_count; + } + + // Compute the variance. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float. + float x_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + + // Is it a valid pixel? + float is_valid = i < static_cast(count) ? 1.f : 0.f; + // Update the mean and m2 using deltas. + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + m2[j] += (x_math[j] - mean[j]) * (x_math[j] - mean[j]) * is_valid; + } + } + } + } + + // The elements to load and store in SMEM. + int smem_nhw = OUTER_LOOPS*pixels_per_iteration + cta_nhw_smem; + // Load elements from SMEM, update the CTA count. + int pixels_in_smem = min(smem_nhw + PIXELS_PER_CTA_IN_SMEM, params.nhw) - max(smem_nhw, 0); + if (pixels_in_smem > 0) { + cta_count += pixels_in_smem; + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + float is_pixel_valid = (((unsigned int)idx < + (unsigned int)params.nhw) && is_valid_c) ? 1.f : 0.f; + + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG]; + ldg_stream(x_storage_local, &gmem_src[(is_pixel_valid ? idx : 0)*params.c]); + + // The offset to store in SMEM. + const int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + // Store in SMEM. + write_to_smem(&smem_storage_packed[offset], threadIdx.x, x_storage_local); + // Update the count. + count += is_pixel_valid; + // Invert the count. + float inv_count = is_pixel_valid ? 1.f / count : 0.f; + + float x_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + // Update the mean and m2 using deltas. + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + float delta0 = x_math[j] - mean[j]; + mean[j] += delta0 * inv_count; + float delta1 = x_math[j] - mean[j]; + m2[j] += delta0 * delta1 * is_pixel_valid; + } + } + } + + // We scale the mean by the number of elements. It brings more stability. + float m1[ELEMENTS_PER_LDG]; + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + m1[i] = mean[i] * count; + } + + // Run the parallel sum accross the CTA to get the local sum. + ParallelSums::dispatch( + smem, m1, thread_in_cta_nhw); + __syncthreads(); + + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(m1, smem, thread_in_cta_c); + __syncthreads(); + + // Adjust the variance. + float inv_cta_count = 1.f / static_cast(cta_count); + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + float mean_diff = m1[i]*inv_cta_count - mean[i]; + m2[i] = m2[i] + mean_diff * mean_diff * count; + } + + // Run the parallel sum accross the CTA to get the local adjusted variance. + ParallelSums::dispatch( + smem, m2, thread_in_cta_nhw); + + // The workspace in global memory is distributed across the different CTA. + int gmem_sums_offset = c_blk_index*gridDim.x*C_ELEMENTS_PER_CTA*2; + + // Write the data for the CTA to global memory. + float *gmem_sums = ¶ms.gmem_sums[gmem_sums_offset]; + if (threadIdx.x < THREADS_PER_PIXEL) { + const int idx = blockIdx.x*THREADS_PER_PIXEL + threadIdx.x; + write_to_gmem(&gmem_sums[ 0], idx, m1); + write_to_gmem(&gmem_sums[C_ELEMENTS_PER_CTA*gridDim.x], idx, m2); + } + + // The memory location to store the number of pixels per CTA. + int *gmem_counts = ¶ms.gmem_counts[c_blk_index*gridDim.x]; + if (threadIdx.x == 0) { + gmem_counts[blockIdx.x] = cta_count; + } + + // Read the bias and scale. + float bias[ELEMENTS_PER_LDG], scale[ELEMENTS_PER_LDG]; + if (is_valid_c) { + read_from_gmem(bias, ¶ms.gmem_bias[cta_c], thread_in_cta_c); + read_from_gmem(scale, ¶ms.gmem_scale[cta_c], thread_in_cta_c); + } + + // The counters to count how many CTAs have retired at this point. + // A given cta uses the same counter every other time through the outer loop. + int *gmem_retired_ctas = ¶ms.gmem_retired_ctas[c_blk_index % (2 * gridDim.y)]; + inter_block_sync(gmem_retired_ctas, gridDim.x, blockIdx.x == 0); + + // Reset the mean to compute the global mean. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + m1[i] = 0.f; + } + + // Build the global mean. + #pragma unroll 1 + for (int idx = threadIdx.x; idx < THREADS_PER_PIXEL*gridDim.x; idx += THREADS_PER_CTA) { + float tmp[ELEMENTS_PER_LDG]; + read_from_gmem(tmp, gmem_sums, idx); + add(m1, tmp); + } + + if (params.sync_iters>0) + { + ParallelSums::dispatchX( + smem, m1, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+3, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, m1, thread_in_cta_nhw); + } + __syncthreads(); + + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(m1, smem, thread_in_cta_c); + __syncthreads(); + + // Normalize the mean. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + m1[i] = m1[i] * params.svar_inv_count; + } + + // Reset the variance. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + m2[i] = 0.f; + } + + // for add+relu fusion + const uint16_t *gmem_src1 = nullptr; + if (USE_ADD_RELU) { + gmem_src1 = ¶ms.gmem_src1[thread_c]; + } + + // Build the global variance. + #pragma unroll 1 + for (int idx = threadIdx.x; idx < THREADS_PER_PIXEL*gridDim.x; idx += THREADS_PER_CTA) { + // Read the means computed by different CTAs (again). Reuse tmp if we have 1 iteration. + float tmp_mean[ELEMENTS_PER_LDG], tmp_var[ELEMENTS_PER_LDG]; + read_from_gmem(tmp_mean, &gmem_sums[ 0], idx); + read_from_gmem(tmp_var, &gmem_sums[C_ELEMENTS_PER_CTA*gridDim.x], idx); + + // Read the number of pixels visited by a given CTA. + cta_count = __ldg(&gmem_counts[idx / THREADS_PER_PIXEL]); + + // Compute the diff to update the variance. + float mean_diff[ELEMENTS_PER_LDG], inv_cta_count = 1.f / static_cast(cta_count); + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + mean_diff[i] = m1[i] - tmp_mean[i]*inv_cta_count; + } + + // Update the variance. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + m2[i] += tmp_var[i] + mean_diff[i]*mean_diff[i]*static_cast(cta_count); + } + } + + if (params.sync_iters>0) + { + ParallelSums::dispatchX( + smem, m2, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+2, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, m2, thread_in_cta_nhw); + } + __syncthreads(); + + read_from_smem(m2, smem, thread_in_cta_c); + + // Finalize the stddev. + // becasue saved var and running var may have different denominator, we don't do it here + // scale_(m2, inv_count); + + // store the saved mean/var + float svarinv[ELEMENTS_PER_LDG]; + bool is_valid_for_saving = is_valid_c && blockIdx.x == 0 && thread_in_cta_nhw == 0; + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + svarinv[i] = rsqrtf(m2[i] * params.svar_inv_count + params.var_eps); + } + if (is_valid_for_saving) { + write_to_gmem(params.gmem_saved_mean, thread_c/ELEMENTS_PER_LDG, m1); + write_to_gmem(params.gmem_saved_var, thread_c/ELEMENTS_PER_LDG, svarinv); + } + + // store the running mean/var + float rmean[ELEMENTS_PER_LDG], rvar[ELEMENTS_PER_LDG]; + zero_array(rmean); + zero_array(rvar); + if (params.exp_avg_factor != 1.f && is_valid_for_saving) { + read_from_gmem(rmean, params.gmem_running_mean, thread_c/ELEMENTS_PER_LDG); + read_from_gmem(rvar, params.gmem_running_var, thread_c/ELEMENTS_PER_LDG); + } + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + rmean[i] = (1.f - params.exp_avg_factor) * rmean[i] + \ + params.exp_avg_factor * m1[i]; + rvar[i] = (1.f - params.exp_avg_factor) * rvar[i] + \ + params.exp_avg_factor * (m2[i] * params.rvar_inv_count); + } + if (is_valid_for_saving) { + write_to_gmem(params.gmem_running_mean, thread_c/ELEMENTS_PER_LDG, rmean); + write_to_gmem(params.gmem_running_var, thread_c/ELEMENTS_PER_LDG, rvar); + } + + // Update the scale with the stddev and eps. + multiply(scale, svarinv); + + // The base pointer to write to. + uint16_t *const gmem_dst = ¶ms.gmem_dst[thread_c]; + + unsigned int *const gmem_relu_bitmask = params.gmem_relu_bitmask + + ((params.nhw + 31) & ~31) * 2 * c_blk_index; + + // Store the elements in registers. + #pragma unroll 1 + for (int loop_i = OUTER_LOOPS-1; loop_i >= 0; --loop_i) { + // The value for nhw. + int out_nhw = cta_nhw_regs + loop_i*pixels_per_iteration; + + // Normalize the elements and write to memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_valid_nhw = + static_cast(idx) < static_cast(params.nhw); + const bool is_valid = is_valid_nhw && is_valid_c; + // Convert to float. + float x_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + + // Normalize and apply activation function + normalize(x_math, bias, scale, m1); + if (USE_ADD_RELU) { + float x1_math[ELEMENTS_PER_LDG]; + ldg_stream(x1_math, &gmem_src1[(is_valid ? idx : 0)*params.c]); + add(x_math, x1_math); + unsigned int relu_mask; + int lane_id = threadIdx.x & 31; + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + bool rectified = x_math[i] < 0.0F; + unsigned int local_relu_mask = __ballot_sync(0xFFFFFFFFU, rectified); + if (lane_id == i) { + // Thread 0 remembers the relu_mask from the first time through this + // loop, Thread 1 the next, Thread 2 the next, and Thread 3 the last. + relu_mask = local_relu_mask; + } + if (rectified) { + x_math[i] = 0.0F; + } + } + if (is_valid_nhw && (lane_id < ELEMENTS_PER_LDG)) { + gmem_relu_bitmask[idx * 2 + lane_id] = relu_mask; + } + } else if (USE_RELU) { + relu_activation(x_math); + } + + // Write back. + if (is_valid) { + stg_stream(&gmem_dst[idx*params.c], x_math); + } + } + + // The next value of nhw. + out_nhw -= pixels_per_iteration; + + // Read the next elements from memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + } + } + } + + // Normalize the elements from SMEM and write them out. + if (pixels_in_smem > 0) { + #pragma unroll 2 + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_valid_nhw = + static_cast(idx) < static_cast(params.nhw); + const bool is_valid = is_valid_nhw && is_valid_c; + + // Read from SMEM. + const int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG]; + read_from_smem(x_storage_local, &smem_storage_packed[offset], threadIdx.x); + float x_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + + // Normalize and apply activation function + normalize(x_math, bias, scale, m1); + if (USE_ADD_RELU) { + float x1_math[ELEMENTS_PER_LDG]; + ldg_stream(x1_math, &gmem_src1[(is_valid ? idx : 0)*params.c]); + add(x_math, x1_math); + unsigned int relu_mask; + int lane_id = threadIdx.x & 31; + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + bool rectified = x_math[i] < 0.0F; + unsigned int local_relu_mask = __ballot_sync(0xFFFFFFFFU, rectified); + if (lane_id == i) { + relu_mask = local_relu_mask; + } + if (rectified) { + x_math[i] = 0.0F; + } + } + if (is_valid_nhw && (lane_id < ELEMENTS_PER_LDG)) { + gmem_relu_bitmask[idx * 2 + lane_id] = relu_mask; + } + } else if (USE_RELU) { + relu_activation(x_math); + } + + // Write back. + if (is_valid) { + stg_stream(&gmem_dst[idx*params.c], x_math); + } + } + } + // We're about to start on the next c-blk. Needed? + __syncthreads(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct NhwcBatchNormBwdParams { + // The input/output tensors. + uint16_t *gmem_src, *gmem_dy, *gmem_dst, *gmem_dst1; + // dscale/dbias + float *gmem_dscale, *gmem_dbias; + // The scale and bias. + float *gmem_scale, *gmem_bias; + // The mean/inv-var saved from fwd pass + float *gmem_saved_mean, *gmem_saved_var; + // ReLU bitmask + unsigned int *gmem_relu_bitmask; + // The dimensions. + int nhw, c; + // factor to scale sum of squared errors to get saved variance. Must be 1/nhw. + float svar_inv_count; + // The buffer to do the reduction for dscale and dbias + float *gmem_sums; + // The counters of retired CTAs. + int *gmem_retired_ctas; + // outer loop count + int outer_loops; + // number of CTAs along .x dimension + int c_blks; + + void* my_data; + void* pair_datas[4]; + int magic; + int sync_iters; + float wgrad_coeff; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void relu_bwd(float (&dy)[N], const float (&x)[N], + const float (&mean_var_scale_bias)[N], + const float (&var_scale)[N], bool valid_data) { + #pragma unroll + for (int j = 0; j < N; ++j) { + float y = (x[j] * var_scale[j]) + mean_var_scale_bias[j]; + if ((y <= 0.f) && valid_data) { + dy[j] = 0.f; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void relu_bwd(float (&dy)[N], const float (&y)[N], bool valid_data) { + #pragma unroll + for (int j = 0; j < N; ++j) { + if ((y[j] <= 0.f) && valid_data) { + dy[j] = 0.f; + } + } +} + +template +DEVICE_FUNCTION void relu_bwd(float (&dy)[N], const bool (&rectified)[N], bool valid_data) { + #pragma unroll + for (int j = 0; j < N; ++j) { + if (rectified[j] && valid_data) { + dy[j] = 0.f; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void relu_bwd_for_dx(float (&dy)[N], + const float (&x)[N], + const float (&mean_var_scale_bias)[N], + const float (&var_scale)[N]) { + #pragma unroll + for (int j = 0; j < N; ++j) { + float y = (x[j] * var_scale[j]) + mean_var_scale_bias[j]; + if (y <= 0.f) { + dy[j] = 0.f; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void relu_bwd_for_dx(float (&dy)[N], const float (&y)[N]) { + #pragma unroll + for (int j = 0; j < N; ++j) { + if (y[j] <= 0.f) { + dy[j] = 0.f; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void bwd_update(float (&dscale)[N], float (&dbias)[N], + const float (&dy)[N], const float (&x)[N], + const float (&mean)[N], float inv_count) { + #pragma unroll + for (int j = 0; j < N; ++j) { + float delta0 = dy[j] - dbias[j]; + dbias[j] += delta0 * inv_count; + delta0 = (dy[j] * (x[j] - mean[j])) - dscale[j]; + dscale[j] += delta0 * inv_count; + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +DEVICE_FUNCTION void bwd_dx(float (&dx)[N], const float (&dy)[N], + const float (&var)[N], const float (&x)[N], const float (&mean)[N], + const float (&dscale)[N], const float (&dbias)[N], float inv_count) { + #pragma unroll + for (int j = 0; j < N; ++j) { + float tmp1 = dy[j] - (dbias[j]* inv_count); + float tmp2 = dscale[j] * inv_count; + float tmp3 = x[j] - mean[j]; + dx[j] = var[j] * (tmp1 - (tmp2 * tmp3)); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + typename Storage, + int THREADS_PER_CTA, + int THREADS_PER_PIXEL, + int PIXELS_PER_THREAD_IN_REGISTERS, + int PIXELS_PER_THREAD_IN_SMEM, + int ELEMENTS_PER_LDG, + int USE_ONLINE_APPROACH, + int OUTER_LOOPS_, + int DESIRED_OCCUPANCY +> +__global__ __launch_bounds__(THREADS_PER_CTA, DESIRED_OCCUPANCY) + void nhwc_batch_norm_bwd(NhwcBatchNormBwdParams params) { + // The number of pixels loaded in a single LDG. + const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + // The number of pixels computed per CTA stored in registers. + const int PIXELS_PER_CTA_IN_REGISTERS = PIXELS_PER_THREAD_IN_REGISTERS * PIXELS_PER_LDG; + // The number of pixels computed per CTA stored in SMEM. + const int PIXELS_PER_CTA_IN_SMEM = PIXELS_PER_THREAD_IN_SMEM*PIXELS_PER_LDG; + // The number of C elements per CTA. + const int C_ELEMENTS_PER_CTA = THREADS_PER_PIXEL*ELEMENTS_PER_LDG; + + // Shared memory to do CTA-wide parallel sums. + __shared__ float smem[THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG]; + + // The adapter for the storage. + typedef PackedStorage PackedStorage_; + // The data type for packed storage in SMEM. + typedef typename PackedStorage_::Type PackedStorageType; + // The number of elements in the packed storage. + const int PACKED_ELEMENTS_PER_LDG = PackedStorage_::PACKED_ELEMENTS_PER_LDG; + // Registers to keep the data live for the persistent approach. + PackedStorageType x_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + PackedStorageType dy_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + + // Shared memory buffer to store the extra pixels. + extern __shared__ PackedStorageType smem_storage_packed[]; + + for (int c_blk_index = blockIdx.y; c_blk_index < params.c_blks; c_blk_index += gridDim.y) { + // The position in the NHW dimension where the CTA starts. + int cta_nhw_regs = blockIdx.x * PIXELS_PER_CTA_IN_REGISTERS; + // The position in the NHW dimension where the CTA starts for the portion in SMEM. + int cta_nhw_smem = blockIdx.x * PIXELS_PER_CTA_IN_SMEM; + // Compute the NHW coordinate of the thread in the CTA. + const int thread_in_cta_nhw = threadIdx.x / THREADS_PER_PIXEL; + + // The position in the C dimension where the CTA starts. + const int cta_c = c_blk_index * C_ELEMENTS_PER_CTA; + // Compute the C coordinate of the thread in the CTA. + const int thread_in_cta_c = threadIdx.x % THREADS_PER_PIXEL; + // Compute the C coordinate of the thread. + const int thread_c = cta_c + thread_in_cta_c*ELEMENTS_PER_LDG; + + // Is the thread working on a valid C dimension? + const int is_valid_c = thread_c < params.c; + + // Registers to store the mean used for entire duration + float mean[ELEMENTS_PER_LDG]; + zero_array(mean); + if (is_valid_c) { + read_from_gmem(mean, params.gmem_saved_mean, thread_c/ELEMENTS_PER_LDG); + } + + // accumulation related registers + float count = 0.f, dscale[ELEMENTS_PER_LDG], dbias[ELEMENTS_PER_LDG]; + zero_array(dscale); + zero_array(dbias); + + // The number of elements loaded by this CTA. + int cta_count = 0; + // The base pointers to load from. + const uint16_t *gmem_src = ¶ms.gmem_src[thread_c]; + const uint16_t *gmem_dy = ¶ms.gmem_dy[thread_c]; + + // outer loops + int OUTER_LOOPS = OUTER_LOOPS_ == 1? 1 : params.outer_loops; + // Load the batch of elements. Compute sum across them + const int pixels_per_iteration = PIXELS_PER_CTA_IN_REGISTERS*gridDim.x; + + if (OUTER_LOOPS_ != 1) { + // We cannot load everything to store persistently, so let's makes sure registers and + // smem are fully utilized + int offset = params.nhw - pixels_per_iteration * OUTER_LOOPS - + PIXELS_PER_CTA_IN_SMEM * gridDim.x; + cta_nhw_regs += offset; + cta_nhw_smem += offset; + } + + #pragma unroll 1 + for (int loop_i = 0; loop_i < OUTER_LOOPS; ++loop_i) { + // The nhw position. + int nhw_regs = cta_nhw_regs + loop_i*pixels_per_iteration; + // Update the number of elements loaded by this CTA. TODO: Skip if <= 0!!! + cta_count += max(0, min(PIXELS_PER_CTA_IN_REGISTERS, params.nhw-nhw_regs)); + + // Read the elements from memory. + float is_valid[PIXELS_PER_THREAD_IN_REGISTERS]; + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = nhw_regs + thread_in_cta_nhw + i*PIXELS_PER_LDG; + zero_array(x_storage[i]); + zero_array(dy_storage[i]); + is_valid[i] = 0.f; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + if (loop_i == OUTER_LOOPS - 1) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + ldg_stream(dy_storage[i], &gmem_dy[idx*params.c]); + } else { + ldg(x_storage[i], &gmem_src[idx*params.c]); + ldg(dy_storage[i], &gmem_dy[idx*params.c]); + } + is_valid[i] = 1.f; + } + } + + // Do the math. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float and update + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + to_float(dy_math, dy_storage[i]); + + // Update the count. + count += is_valid[i]; + // Invert the count. + float inv_count = is_valid[i] ? 1.f / count : 0.f; + + bwd_update(dscale, dbias, dy_math, x_math, mean, inv_count); + } + } + + // The elements to load and store in SMEM. + int smem_nhw = OUTER_LOOPS*pixels_per_iteration + cta_nhw_smem; + // Load elements from SMEM, update the CTA count. + int pixels_in_smem = min(PIXELS_PER_CTA_IN_SMEM, params.nhw-smem_nhw); + if (pixels_in_smem > 0) { + cta_count += pixels_in_smem; + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + bool is_pixel_valid = (((unsigned int)idx < + (unsigned int)params.nhw) && is_valid_c); + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG], + dy_storage_local[PACKED_ELEMENTS_PER_LDG]; + zero_array(x_storage_local); + zero_array(dy_storage_local); + if (is_pixel_valid) { + ldg_stream(x_storage_local, &gmem_src[idx*params.c]); + ldg_stream(dy_storage_local, &gmem_dy[idx*params.c]); + } + + // The offset to store in SMEM. + int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + // Store in SMEM. + write_to_smem(&smem_storage_packed[offset], threadIdx.x, x_storage_local); + offset += PIXELS_PER_THREAD_IN_SMEM*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + write_to_smem(&smem_storage_packed[offset], threadIdx.x, dy_storage_local); + // Update the count. + count += is_pixel_valid; + // Invert the count. + float inv_count = is_pixel_valid ? 1.f / count : 0.f; + + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + to_float(dy_math, dy_storage_local); + + bwd_update(dscale, dbias, dy_math, x_math, mean, inv_count); + } + } + + // We scale the mean by the number of elements. It brings more stability. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + dbias[i] *= count; + dscale[i] *= count; + } + + // dscale parallel sum + ParallelSums::dispatch( + smem, dscale, thread_in_cta_nhw); + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dscale, smem, thread_in_cta_c); + __syncthreads(); + + // dbias parallel sum + ParallelSums::dispatch( + smem, dbias, thread_in_cta_nhw); + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dbias, smem, thread_in_cta_c); + __syncthreads(); + + // The workspace in global memory is distributed across the different CTA. + int gmem_sums_offset = c_blk_index*gridDim.x*C_ELEMENTS_PER_CTA*2; + // Write the data for the CTA to global memory. + float *gmem_sums = ¶ms.gmem_sums[gmem_sums_offset]; + if (threadIdx.x < THREADS_PER_PIXEL) { + const int idx = blockIdx.x*THREADS_PER_PIXEL + threadIdx.x; + write_to_gmem(&gmem_sums[ 0], idx, dscale); + write_to_gmem(&gmem_sums[C_ELEMENTS_PER_CTA*gridDim.x], idx, dbias); + } + + // The counters to count how many CTAs have retired at this point. + // A given cta uses the same counter every other time through the outer loop. + int *gmem_retired_ctas = ¶ms.gmem_retired_ctas[c_blk_index % (2 * gridDim.y)]; + inter_block_sync(gmem_retired_ctas, gridDim.x, blockIdx.x == 0); + + // Reset the accumulators for global summation + zero_array(dscale); + zero_array(dbias); + + // Build the global accumulation + #pragma unroll 1 + for (int idx = threadIdx.x; idx < THREADS_PER_PIXEL*gridDim.x; idx += THREADS_PER_CTA) { + float tmp1[ELEMENTS_PER_LDG], tmp2[ELEMENTS_PER_LDG]; + read_from_gmem(tmp1, gmem_sums, idx); + read_from_gmem(tmp2, gmem_sums+C_ELEMENTS_PER_CTA*gridDim.x, idx); + + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + dscale[i] += tmp1[i]; + dbias[i] += tmp2[i]; + } + } + + // dscale parallel sum + if (params.sync_iters>0) { + ParallelSums::dispatchX( + smem, dscale, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+1, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, dscale, thread_in_cta_nhw); + } + + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dscale, smem, thread_in_cta_c); + __syncthreads(); + + // dbias parallel sum + if (params.sync_iters>0) { + ParallelSums::dispatchX( + smem, dbias, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+0, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, dbias, thread_in_cta_nhw); + } + + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dbias, smem, thread_in_cta_c); + + // inv-var + float var[ELEMENTS_PER_LDG]; + zero_array(var); + if (is_valid_c) { + read_from_gmem(var, params.gmem_saved_var, thread_c/ELEMENTS_PER_LDG); + } + + // Normalize the dscale. + multiply(dscale, var); + + // store dscale/dbias + bool is_valid_for_saving = is_valid_c && blockIdx.x == 0 && thread_in_cta_nhw == 0; + if (is_valid_for_saving) { + if (params.sync_iters>0) + { + scaled_write_to_gmem(params.gmem_dscale, thread_c/ELEMENTS_PER_LDG, dscale, params.wgrad_coeff); + scaled_write_to_gmem(params.gmem_dbias, thread_c/ELEMENTS_PER_LDG, dbias, params.wgrad_coeff); + } else { + write_to_gmem(params.gmem_dscale, thread_c/ELEMENTS_PER_LDG, dscale); + write_to_gmem(params.gmem_dbias, thread_c/ELEMENTS_PER_LDG, dbias); + } + } + + // scale + float scale[ELEMENTS_PER_LDG]; + zero_array(scale); + if (is_valid_c) { + read_from_gmem(scale, params.gmem_scale, thread_c/ELEMENTS_PER_LDG); + } + + // Further normalize the dscale to be used in dx calculation + multiply(dscale, var); + // scale the inv-var as well, afterwards + multiply(var, scale); + + // inverse count + float inv_count = params.svar_inv_count; + + // The base pointer to write to. + uint16_t *const gmem_dst = ¶ms.gmem_dst[thread_c]; + + // Store the elements in registers. + #pragma unroll 1 + for (int loop_i = OUTER_LOOPS-1; loop_i >= 0; --loop_i) { + // The value for nhw. + int out_nhw = cta_nhw_regs + loop_i*pixels_per_iteration; + + // Normalize the elements and write to memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float. + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + to_float(dy_math, dy_storage[i]); + + float dx[ELEMENTS_PER_LDG]; + bwd_dx(dx, dy_math, var, x_math, mean, dscale, dbias, inv_count); + + // Write back. + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + stg_stream(&gmem_dst[idx*params.c], dx); + } + } + + // The next value of nhw. + out_nhw -= pixels_per_iteration; + + // Read the next elements from memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + ldg_stream(dy_storage[i], &gmem_dy[idx*params.c]); + } + } + } + + // Normalize the elements from SMEM and write them out. + if (pixels_in_smem > 0) { + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_valid = ((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c; + if (is_valid) { + // Read from SMEM. + int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG], + dy_storage_local[PACKED_ELEMENTS_PER_LDG]; + read_from_smem(x_storage_local, &smem_storage_packed[offset], threadIdx.x); + offset += PIXELS_PER_THREAD_IN_SMEM*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + read_from_smem(dy_storage_local, &smem_storage_packed[offset], threadIdx.x); + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + to_float(dy_math, dy_storage_local); + + float dx[ELEMENTS_PER_LDG]; + bwd_dx(dx, dy_math, var, x_math, mean, dscale, dbias, inv_count); + + // Write back. + stg_stream(&gmem_dst[idx*params.c], dx); + } + } + } + // We're about to start on the next c-blk. Needed? + __syncthreads(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + typename Storage, + int THREADS_PER_CTA, + int THREADS_PER_PIXEL, + int PIXELS_PER_THREAD_IN_REGISTERS, + int PIXELS_PER_THREAD_IN_SMEM, + int ELEMENTS_PER_LDG, + int USE_ONLINE_APPROACH, + int OUTER_LOOPS_, + int DESIRED_OCCUPANCY +> +__global__ __launch_bounds__(THREADS_PER_CTA, DESIRED_OCCUPANCY) + void nhwc_batch_norm_bwd_relu(NhwcBatchNormBwdParams params) { + // The number of pixels loaded in a single LDG. + const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + // The number of pixels computed per CTA stored in registers. + const int PIXELS_PER_CTA_IN_REGISTERS = PIXELS_PER_THREAD_IN_REGISTERS * PIXELS_PER_LDG; + // The number of pixels computed per CTA stored in SMEM. + const int PIXELS_PER_CTA_IN_SMEM = PIXELS_PER_THREAD_IN_SMEM*PIXELS_PER_LDG; + // The number of C elements per CTA. + const int C_ELEMENTS_PER_CTA = THREADS_PER_PIXEL*ELEMENTS_PER_LDG; + + // Shared memory to do CTA-wide parallel sums. + __shared__ float smem[THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG]; + + // The adapter for the storage. + typedef PackedStorage PackedStorage_; + // The data type for packed storage in SMEM. + typedef typename PackedStorage_::Type PackedStorageType; + // The number of elements in the packed storage. + const int PACKED_ELEMENTS_PER_LDG = PackedStorage_::PACKED_ELEMENTS_PER_LDG; + // Registers to keep the data live for the persistent approach. + PackedStorageType x_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + PackedStorageType dy_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + + // Shared memory buffer to store the extra pixels. + extern __shared__ PackedStorageType smem_storage_packed[]; + + for (int c_blk_index = blockIdx.y; c_blk_index < params.c_blks; c_blk_index += gridDim.y) { + // The position in the NHW dimension where the CTA starts. + int cta_nhw_regs = blockIdx.x * PIXELS_PER_CTA_IN_REGISTERS; + // The position in the NHW dimension where the CTA starts for the portion in SMEM. + int cta_nhw_smem = blockIdx.x * PIXELS_PER_CTA_IN_SMEM; + // Compute the NHW coordinate of the thread in the CTA. + const int thread_in_cta_nhw = threadIdx.x / THREADS_PER_PIXEL; + + // The position in the C dimension where the CTA starts. + const int cta_c = c_blk_index * C_ELEMENTS_PER_CTA; + // Compute the C coordinate of the thread in the CTA. + const int thread_in_cta_c = threadIdx.x % THREADS_PER_PIXEL; + // Compute the C coordinate of the thread. + const int thread_c = cta_c + thread_in_cta_c*ELEMENTS_PER_LDG; + + // Is the thread working on a valid C dimension? + const int is_valid_c = thread_c < params.c; + + + // Registers to store the mean/var/scale/bias used for the entire duration + // Register usage optimizations: + // 1. Can combine bias - (mean * var * scale) into a single register + // 2. Can combine var * scale into a single register + float varscale[ELEMENTS_PER_LDG]; + zero_array(varscale); + if (is_valid_c) { + read_from_gmem(varscale, params.gmem_saved_var, thread_c/ELEMENTS_PER_LDG); + } + float tmp[ELEMENTS_PER_LDG]; + zero_array(tmp); + if (is_valid_c) { + read_from_gmem(tmp, params.gmem_scale, thread_c/ELEMENTS_PER_LDG); + } + multiply(varscale, tmp); + float mean[ELEMENTS_PER_LDG]; + zero_array(mean); + if (is_valid_c) { + read_from_gmem(mean, params.gmem_saved_mean, thread_c/ELEMENTS_PER_LDG); + } + zero_array(tmp); + if (is_valid_c) { + read_from_gmem(tmp, params.gmem_bias, thread_c/ELEMENTS_PER_LDG); + } + float mean_var_scale_bias[ELEMENTS_PER_LDG]; + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + mean_var_scale_bias[i] = tmp[i] - (mean[i] * varscale[i]); + } + + // accumulation related registers + float count = 0.f, dscale[ELEMENTS_PER_LDG], dbias[ELEMENTS_PER_LDG]; + zero_array(dscale); + zero_array(dbias); + + // The number of elements loaded by this CTA. + int cta_count = 0; + // The base pointers to load from. + const uint16_t *gmem_src = ¶ms.gmem_src[thread_c]; + const uint16_t *gmem_dy = ¶ms.gmem_dy[thread_c]; + + // outer loops + int OUTER_LOOPS = OUTER_LOOPS_ == 1? 1 : params.outer_loops; + // Load the batch of elements. Compute sum across them + const int pixels_per_iteration = PIXELS_PER_CTA_IN_REGISTERS*gridDim.x; + + if (OUTER_LOOPS_ != 1) { + // We cannot load everything to store persistently, so let's makes sure registers and + // smem are fully utilized + int offset = params.nhw - pixels_per_iteration * OUTER_LOOPS - + PIXELS_PER_CTA_IN_SMEM * gridDim.x; + cta_nhw_regs += offset; + cta_nhw_smem += offset; + } + + #pragma unroll 1 + for (int loop_i = 0; loop_i < OUTER_LOOPS; ++loop_i) { + // The nhw position. + int nhw_regs = cta_nhw_regs + loop_i*pixels_per_iteration; + // Update the number of elements loaded by this CTA. TODO: Skip if <= 0!!! + cta_count += max(0, min(PIXELS_PER_CTA_IN_REGISTERS, params.nhw-nhw_regs)); + + // Read the elements from memory. + float is_valid[PIXELS_PER_THREAD_IN_REGISTERS]; + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = nhw_regs + thread_in_cta_nhw + i*PIXELS_PER_LDG; + zero_array(x_storage[i]); + zero_array(dy_storage[i]); + is_valid[i] = 0.f; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + if (loop_i == OUTER_LOOPS - 1) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + ldg_stream(dy_storage[i], &gmem_dy[idx*params.c]); + } else { + ldg(x_storage[i], &gmem_src[idx*params.c]); + ldg(dy_storage[i], &gmem_dy[idx*params.c]); + } + is_valid[i] = 1.f; + } + } + + // Do the math. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float and update + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + to_float(dy_math, dy_storage[i]); + + // Update the count. + count += is_valid[i]; + // Invert the count. + float inv_count = is_valid[i] ? 1.f / count : 0.f; + + relu_bwd(dy_math, x_math, mean_var_scale_bias, varscale, is_valid[i]); + bwd_update(dscale, dbias, dy_math, x_math, mean, inv_count); + } + } + + // The elements to load and store in SMEM. + int smem_nhw = OUTER_LOOPS*pixels_per_iteration + cta_nhw_smem; + // Load elements from SMEM, update the CTA count. + int pixels_in_smem = min(PIXELS_PER_CTA_IN_SMEM, params.nhw-smem_nhw); + if (pixels_in_smem > 0) { + cta_count += pixels_in_smem; + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + bool is_pixel_valid = (((unsigned int)idx < + (unsigned int)params.nhw) && is_valid_c); + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG], + dy_storage_local[PACKED_ELEMENTS_PER_LDG]; + zero_array(x_storage_local); + zero_array(dy_storage_local); + if (is_pixel_valid) { + ldg_stream(x_storage_local, &gmem_src[idx*params.c]); + ldg_stream(dy_storage_local, &gmem_dy[idx*params.c]); + } + + // The offset to store in SMEM. + int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + // Store in SMEM. + write_to_smem(&smem_storage_packed[offset], threadIdx.x, x_storage_local); + offset += PIXELS_PER_THREAD_IN_SMEM*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + write_to_smem(&smem_storage_packed[offset], threadIdx.x, dy_storage_local); + // Update the count. + count += is_pixel_valid; + // Invert the count. + float inv_count = is_pixel_valid ? 1.f / count : 0.f; + + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + to_float(dy_math, dy_storage_local); + + relu_bwd(dy_math, x_math, mean_var_scale_bias, varscale, is_pixel_valid); + bwd_update(dscale, dbias, dy_math, x_math, mean, inv_count); + } + } + + // We scale the mean by the number of elements. It brings more stability. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + dbias[i] *= count; + dscale[i] *= count; + } + + // dscale parallel sum + ParallelSums::dispatch( + smem, dscale, thread_in_cta_nhw); + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dscale, smem, thread_in_cta_c); + __syncthreads(); + + // dbias parallel sum + ParallelSums::dispatch( + smem, dbias, thread_in_cta_nhw); + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dbias, smem, thread_in_cta_c); + __syncthreads(); + + // The workspace in global memory is distributed across the different CTA. + int gmem_sums_offset = c_blk_index*gridDim.x*C_ELEMENTS_PER_CTA*2; + // Write the data for the CTA to global memory. + float *gmem_sums = ¶ms.gmem_sums[gmem_sums_offset]; + if (threadIdx.x < THREADS_PER_PIXEL) { + const int idx = blockIdx.x*THREADS_PER_PIXEL + threadIdx.x; + write_to_gmem(&gmem_sums[ 0], idx, dscale); + write_to_gmem(&gmem_sums[C_ELEMENTS_PER_CTA*gridDim.x], idx, dbias); + } + + // The counters to count how many CTAs have retired at this point. + // A given cta uses the same counter every other time through the outer loop. + int *gmem_retired_ctas = ¶ms.gmem_retired_ctas[c_blk_index % (2 * gridDim.y)]; + inter_block_sync(gmem_retired_ctas, gridDim.x, blockIdx.x == 0); + + // Reset the accumulators for global summation + zero_array(dscale); + zero_array(dbias); + + // Build the global accumulation + #pragma unroll 1 + for (int idx = threadIdx.x; idx < THREADS_PER_PIXEL*gridDim.x; idx += THREADS_PER_CTA) { + float tmp1[ELEMENTS_PER_LDG], tmp2[ELEMENTS_PER_LDG]; + read_from_gmem(tmp1, gmem_sums, idx); + read_from_gmem(tmp2, gmem_sums+C_ELEMENTS_PER_CTA*gridDim.x, idx); + + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + dscale[i] += tmp1[i]; + dbias[i] += tmp2[i]; + } + } + + // dscale parallel sum + if (params.sync_iters>0) { + ParallelSums::dispatchX( + smem, dscale, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+1, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, dscale, thread_in_cta_nhw); + } + + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dscale, smem, thread_in_cta_c); + __syncthreads(); + + // dbias parallel sum + if (params.sync_iters>0) { + ParallelSums::dispatchX( + smem, dbias, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+0, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, dbias, thread_in_cta_nhw); + } + + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dbias, smem, thread_in_cta_c); + + // Normalize the dscale. + float var[ELEMENTS_PER_LDG]; + zero_array(var); + if (is_valid_c) { + read_from_gmem(var, params.gmem_saved_var, thread_c/ELEMENTS_PER_LDG); + } + multiply(dscale, var); + + // store dscale/dbias + bool is_valid_for_saving = is_valid_c && blockIdx.x == 0 && thread_in_cta_nhw == 0; + if (is_valid_for_saving) { + if (params.sync_iters>0) + { + scaled_write_to_gmem(params.gmem_dscale, thread_c/ELEMENTS_PER_LDG, dscale, params.wgrad_coeff); + scaled_write_to_gmem(params.gmem_dbias, thread_c/ELEMENTS_PER_LDG, dbias, params.wgrad_coeff); + } else { + write_to_gmem(params.gmem_dscale, thread_c/ELEMENTS_PER_LDG, dscale); + write_to_gmem(params.gmem_dbias, thread_c/ELEMENTS_PER_LDG, dbias); + } + } + + // Further normalize the dscale to be used in dx calculation + float scale[ELEMENTS_PER_LDG]; + zero_array(scale); + if (is_valid_c) { + read_from_gmem(scale, params.gmem_scale, thread_c/ELEMENTS_PER_LDG); + } + multiply(dscale, var); + // scale the inv-var as well, afterwards + multiply(var, scale); + + // inverse count + float inv_count = params.svar_inv_count; + + // The base pointer to write to. + uint16_t *const gmem_dst = ¶ms.gmem_dst[thread_c]; + + // Store the elements in registers. + #pragma unroll 1 + for (int loop_i = OUTER_LOOPS-1; loop_i >= 0; --loop_i) { + // The value for nhw. + int out_nhw = cta_nhw_regs + loop_i*pixels_per_iteration; + + // Normalize the elements and write to memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + // Convert to float. + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + to_float(dy_math, dy_storage[i]); + relu_bwd_for_dx(dy_math, x_math, mean_var_scale_bias, var); + + float dx[ELEMENTS_PER_LDG]; + bwd_dx(dx, dy_math, var, x_math, mean, dscale, dbias, inv_count); + + // Write back. + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + stg_stream(&gmem_dst[idx*params.c], dx); + } + } + + // The next value of nhw. + out_nhw -= pixels_per_iteration; + + // Read the next elements from memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + ldg_stream(dy_storage[i], &gmem_dy[idx*params.c]); + } + } + } + + // Normalize the elements from SMEM and write them out. + if (pixels_in_smem > 0) { + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_valid = ((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c; + if (is_valid) { + // Read from SMEM. + int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG], + dy_storage_local[PACKED_ELEMENTS_PER_LDG]; + read_from_smem(x_storage_local, &smem_storage_packed[offset], threadIdx.x); + offset += PIXELS_PER_THREAD_IN_SMEM*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + read_from_smem(dy_storage_local, &smem_storage_packed[offset], threadIdx.x); + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + to_float(dy_math, dy_storage_local); + relu_bwd_for_dx(dy_math, x_math, mean_var_scale_bias, var); + + float dx[ELEMENTS_PER_LDG]; + bwd_dx(dx, dy_math, var, x_math, mean, dscale, dbias, inv_count); + + // Write back. + stg_stream(&gmem_dst[idx*params.c], dx); + } + } + } + // We're about to start on the next c-blk. Needed? + __syncthreads(); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + typename Storage, + int THREADS_PER_CTA, + int THREADS_PER_PIXEL, + int PIXELS_PER_THREAD_IN_REGISTERS, + int PIXELS_PER_THREAD_IN_SMEM, + int ELEMENTS_PER_LDG, + int USE_ONLINE_APPROACH, + int OUTER_LOOPS_, + int DESIRED_OCCUPANCY +> +__global__ __launch_bounds__(THREADS_PER_CTA, DESIRED_OCCUPANCY) + void nhwc_batch_norm_bwd_add_relu(NhwcBatchNormBwdParams params) { + // The number of pixels loaded in a single LDG. + const int PIXELS_PER_LDG = THREADS_PER_CTA / THREADS_PER_PIXEL; + // The number of pixels computed per CTA stored in registers. + const int PIXELS_PER_CTA_IN_REGISTERS = PIXELS_PER_THREAD_IN_REGISTERS * PIXELS_PER_LDG; + // The number of pixels computed per CTA stored in SMEM. + const int PIXELS_PER_CTA_IN_SMEM = PIXELS_PER_THREAD_IN_SMEM*PIXELS_PER_LDG; + // The number of C elements per CTA. + const int C_ELEMENTS_PER_CTA = THREADS_PER_PIXEL*ELEMENTS_PER_LDG; + + // Shared memory to do CTA-wide parallel sums. + __shared__ float smem[THREADS_PER_PIXEL*(THREADS_PER_CTA/32)*ELEMENTS_PER_LDG]; + + // The adapter for the storage. + typedef PackedStorage PackedStorage_; + // The data type for packed storage in SMEM. + typedef typename PackedStorage_::Type PackedStorageType; + // The number of elements in the packed storage. + const int PACKED_ELEMENTS_PER_LDG = PackedStorage_::PACKED_ELEMENTS_PER_LDG; + // Registers to keep the data live for the persistent approach. + PackedStorageType x_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + PackedStorageType dy_storage[PIXELS_PER_THREAD_IN_REGISTERS][PACKED_ELEMENTS_PER_LDG]; + + // Shared memory buffer to store the extra pixels. + extern __shared__ PackedStorageType smem_storage_packed[]; + + for (int c_blk_index = blockIdx.y; c_blk_index < params.c_blks; c_blk_index += gridDim.y) { + // The position in the NHW dimension where the CTA starts. + int cta_nhw_regs = blockIdx.x * PIXELS_PER_CTA_IN_REGISTERS; + // The position in the NHW dimension where the CTA starts for the portion in SMEM. + int cta_nhw_smem = blockIdx.x * PIXELS_PER_CTA_IN_SMEM; + // Compute the NHW coordinate of the thread in the CTA. + const int thread_in_cta_nhw = threadIdx.x / THREADS_PER_PIXEL; + + // The position in the C dimension where the CTA starts. + const int cta_c = c_blk_index * C_ELEMENTS_PER_CTA; + // Compute the C coordinate of the thread in the CTA. + const int thread_in_cta_c = threadIdx.x % THREADS_PER_PIXEL; + // Compute the C coordinate of the thread. + const int thread_c = cta_c + thread_in_cta_c*ELEMENTS_PER_LDG; + + // Is the thread working on a valid C dimension? + const int is_valid_c = thread_c < params.c; + + float mean[ELEMENTS_PER_LDG]; + zero_array(mean); + if (is_valid_c) { + read_from_gmem(mean, params.gmem_saved_mean, thread_c/ELEMENTS_PER_LDG); + } + + // accumulation related registers + float count = 0.f, dscale[ELEMENTS_PER_LDG], dbias[ELEMENTS_PER_LDG]; + zero_array(dscale); + zero_array(dbias); + + // The number of elements loaded by this CTA. + int cta_count = 0; + // The base pointers to load from. + const uint16_t *gmem_src = ¶ms.gmem_src[thread_c]; + const uint16_t *gmem_dy = ¶ms.gmem_dy[thread_c]; + uint16_t *gmem_dst1 = ¶ms.gmem_dst1[thread_c]; + + // outer loops + int OUTER_LOOPS = OUTER_LOOPS_ == 1? 1 : params.outer_loops; + // Load the batch of elements. Compute sum across them + const int pixels_per_iteration = PIXELS_PER_CTA_IN_REGISTERS*gridDim.x; + + if (OUTER_LOOPS_ != 1) { + // We cannot load everything to store persistently, so let's makes sure registers and + // smem are fully utilized, offset is evenly divisible by 32 + int offset = (pixels_per_iteration * OUTER_LOOPS + PIXELS_PER_CTA_IN_SMEM * gridDim.x - + params.nhw) & ~31; + cta_nhw_regs -= offset; + cta_nhw_smem -= offset; + } + + const unsigned int *const gmem_relu_bitmask = params.gmem_relu_bitmask + + ((params.nhw + 31) & ~31) * 2 * c_blk_index; + + #pragma unroll 1 + for (int loop_i = 0; loop_i < OUTER_LOOPS; ++loop_i) { + // The nhw position. + int nhw_regs = cta_nhw_regs + loop_i*pixels_per_iteration; + // Update the number of elements loaded by this CTA. TODO: Skip if <= 0!!! + cta_count += max(0, min(PIXELS_PER_CTA_IN_REGISTERS, params.nhw-nhw_regs)); + + int lane_id = threadIdx.x & 31; + + // Read the elements from memory. + float is_valid[PIXELS_PER_THREAD_IN_REGISTERS]; + unsigned int relu_mask[PIXELS_PER_THREAD_IN_REGISTERS]; + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = nhw_regs + thread_in_cta_nhw + i*PIXELS_PER_LDG; + zero_array(x_storage[i]); + zero_array(dy_storage[i]); + is_valid[i] = 0.f; + const bool is_valid_nhw = + static_cast(idx) < static_cast(params.nhw); + if (is_valid_nhw) { + if (is_valid_c) { + if (loop_i == OUTER_LOOPS - 1) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + ldg_stream(dy_storage[i], &gmem_dy[idx*params.c]); + } else { + ldg(x_storage[i], &gmem_src[idx*params.c]); + ldg(dy_storage[i], &gmem_dy[idx*params.c]); + } + is_valid[i] = 1.f; + } + + if (lane_id < ELEMENTS_PER_LDG) { + relu_mask[i] = gmem_relu_bitmask[idx * 2 + lane_id]; + } + } + } + + // Do the math. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = nhw_regs + thread_in_cta_nhw + i*PIXELS_PER_LDG; + // Convert to float and update + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + bool rectified[ELEMENTS_PER_LDG]; + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + rectified[j] = ((__shfl_sync(0xFFFFFFFFU, relu_mask[i], j) & + (1U << lane_id)) != 0); + } + to_float(x_math, x_storage[i]); + to_float(dy_math, dy_storage[i]); + + // Update the count. + count += is_valid[i]; + // Invert the count. + float inv_count = is_valid[i] ? 1.f / count : 0.f; + + relu_bwd(dy_math, rectified, is_valid[i]); + bwd_update(dscale, dbias, dy_math, x_math, mean, inv_count); + + // Lastly we need 'dy' only for BN, so store the 'relu-dgrad'ed version + from_float(dy_storage[i], dy_math); + + // dZ for elementwise add + if (is_valid[i]) { + if (loop_i == OUTER_LOOPS - 1) { + stg_stream(&gmem_dst1[idx*params.c], dy_storage[i]); + } else { + stg(&gmem_dst1[idx*params.c], dy_storage[i]); + } + } + } + } + + // The elements to load and store in SMEM. + int smem_nhw = OUTER_LOOPS*pixels_per_iteration + cta_nhw_smem; + // Load elements from SMEM, update the CTA count. + int pixels_in_smem = min(PIXELS_PER_CTA_IN_SMEM, params.nhw-smem_nhw); + if (pixels_in_smem > 0) { + cta_count += pixels_in_smem; + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_pixel_valid_nhw = + static_cast(idx) < static_cast(params.nhw); + const bool is_pixel_valid = is_pixel_valid_nhw && is_valid_c; + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG], + dy_storage_local[PACKED_ELEMENTS_PER_LDG]; + unsigned int relu_mask; + int lane_id = threadIdx.x & 31; + zero_array(x_storage_local); + zero_array(dy_storage_local); + if (is_pixel_valid_nhw) { + if (is_valid_c) { + ldg_stream(x_storage_local, &gmem_src[idx*params.c]); + ldg_stream(dy_storage_local, &gmem_dy[idx*params.c]); + } + if (lane_id < ELEMENTS_PER_LDG) { + relu_mask = gmem_relu_bitmask[idx * 2 + lane_id]; + } + } + bool rectified[ELEMENTS_PER_LDG]; + #pragma unroll + for (int j = 0; j < ELEMENTS_PER_LDG; ++j) { + rectified[j] = ((__shfl_sync(0xFFFFFFFFU, relu_mask, j) & + (1U << lane_id)) != 0); + } + + // The offset to store in SMEM. + int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + // Store in SMEM. + write_to_smem(&smem_storage_packed[offset], threadIdx.x, x_storage_local); + offset += PIXELS_PER_THREAD_IN_SMEM*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + // Update the count. + count += is_pixel_valid; + // Invert the count. + float inv_count = is_pixel_valid ? 1.f / count : 0.f; + + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + to_float(dy_math, dy_storage_local); + + relu_bwd(dy_math, rectified, is_pixel_valid); + bwd_update(dscale, dbias, dy_math, x_math, mean, inv_count); + + from_float(dy_storage_local, dy_math); + // dZ for elementwise add + if (is_pixel_valid) { + stg_stream(&gmem_dst1[idx*params.c], dy_storage_local); + } + // only store the 'relu-dgrad'ed version! + write_to_smem(&smem_storage_packed[offset], threadIdx.x, dy_storage_local); + } + } + + // We scale the mean by the number of elements. It brings more stability. + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + dbias[i] *= count; + dscale[i] *= count; + } + + // dscale parallel sum + ParallelSums::dispatch( + smem, dscale, thread_in_cta_nhw); + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dscale, smem, thread_in_cta_c); + __syncthreads(); + + // dbias parallel sum + ParallelSums::dispatch( + smem, dbias, thread_in_cta_nhw); + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dbias, smem, thread_in_cta_c); + __syncthreads(); + + // The workspace in global memory is distributed across the different CTA. + int gmem_sums_offset = c_blk_index*gridDim.x*C_ELEMENTS_PER_CTA*2; + // Write the data for the CTA to global memory. + float *gmem_sums = ¶ms.gmem_sums[gmem_sums_offset]; + if (threadIdx.x < THREADS_PER_PIXEL) { + const int idx = blockIdx.x*THREADS_PER_PIXEL + threadIdx.x; + write_to_gmem(&gmem_sums[ 0], idx, dscale); + write_to_gmem(&gmem_sums[C_ELEMENTS_PER_CTA*gridDim.x], idx, dbias); + } + + // The counters to count how many CTAs have retired at this point. + // A given cta uses the same counter every other time through the outer loop. + int *gmem_retired_ctas = ¶ms.gmem_retired_ctas[c_blk_index % (2 * gridDim.y)]; + inter_block_sync(gmem_retired_ctas, gridDim.x, blockIdx.x == 0); + + // Reset the accumulators for global summation + zero_array(dscale); + zero_array(dbias); + + // Build the global accumulation + #pragma unroll 1 + for (int idx = threadIdx.x; idx < THREADS_PER_PIXEL*gridDim.x; idx += THREADS_PER_CTA) { + float tmp1[ELEMENTS_PER_LDG], tmp2[ELEMENTS_PER_LDG]; + read_from_gmem(tmp1, gmem_sums, idx); + read_from_gmem(tmp2, gmem_sums+C_ELEMENTS_PER_CTA*gridDim.x, idx); + + #pragma unroll + for (int i = 0; i < ELEMENTS_PER_LDG; ++i) { + dscale[i] += tmp1[i]; + dbias[i] += tmp2[i]; + } + } + + // dscale parallel sum + if (params.sync_iters>0) { + ParallelSums::dispatchX( + smem, dscale, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+1, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, dscale, thread_in_cta_nhw); + } + + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dscale, smem, thread_in_cta_c); + __syncthreads(); + + // dbias parallel sum + if (params.sync_iters>0) { + ParallelSums::dispatchX( + smem, dbias, thread_in_cta_nhw, params.my_data, params.pair_datas, 4*c_blk_index+0, params.magic, params.sync_iters); + } else { + ParallelSums::dispatch( + smem, dbias, thread_in_cta_nhw); + } + + __syncthreads(); + // The values in shared memory correspond to the CTA-wide sums. + read_from_smem(dbias, smem, thread_in_cta_c); + + // Normalize the dscale. + float var[ELEMENTS_PER_LDG]; + zero_array(var); + if (is_valid_c) { + read_from_gmem(var, params.gmem_saved_var, thread_c/ELEMENTS_PER_LDG); + } + multiply(dscale, var); + + // store dscale/dbias + bool is_valid_for_saving = is_valid_c && blockIdx.x == 0 && thread_in_cta_nhw == 0; + if (is_valid_for_saving) { + if (params.sync_iters>0) + { + scaled_write_to_gmem(params.gmem_dscale, thread_c/ELEMENTS_PER_LDG, dscale, params.wgrad_coeff); + scaled_write_to_gmem(params.gmem_dbias, thread_c/ELEMENTS_PER_LDG, dbias, params.wgrad_coeff); + } else { + write_to_gmem(params.gmem_dscale, thread_c/ELEMENTS_PER_LDG, dscale); + write_to_gmem(params.gmem_dbias, thread_c/ELEMENTS_PER_LDG, dbias); + } + } + + // Further normalize the dscale to be used in dx calculation + float scale[ELEMENTS_PER_LDG]; + zero_array(scale); + if (is_valid_c) { + read_from_gmem(scale, params.gmem_scale, thread_c/ELEMENTS_PER_LDG); + } + multiply(dscale, var); + // scale the inv-var as well, afterwards + multiply(var, scale); + + // inverse count + float inv_count = params.svar_inv_count; + + // The base pointer to write to. + uint16_t *const gmem_dst = ¶ms.gmem_dst[thread_c]; + + // Store the elements in registers. + #pragma unroll 1 + for (int loop_i = OUTER_LOOPS-1; loop_i >= 0; --loop_i) { + // The value for nhw. + int out_nhw = cta_nhw_regs + loop_i*pixels_per_iteration; + + // Normalize the elements and write to memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_valid = ((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c; + // Convert to float. + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage[i]); + to_float(dy_math, dy_storage[i]); + + float dx[ELEMENTS_PER_LDG]; + bwd_dx(dx, dy_math, var, x_math, mean, dscale, dbias, inv_count); + + // Write back. + if (is_valid) { + stg_stream(&gmem_dst[idx*params.c], dx); + } + } + + // The next value of nhw. + out_nhw -= pixels_per_iteration; + + // Read the next elements from memory. + #pragma unroll + for (int i = 0; i < PIXELS_PER_THREAD_IN_REGISTERS; ++i) { + const int idx = out_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + float y[ELEMENTS_PER_LDG]; + zero_array(y); + if (((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c) { + ldg_stream(x_storage[i], &gmem_src[idx*params.c]); + ldg_stream(dy_storage[i], &gmem_dst1[idx*params.c]); + } + } + } + + // Normalize the elements from SMEM and write them out. + if (pixels_in_smem > 0) { + for (int i = 0; i < PIXELS_PER_THREAD_IN_SMEM; ++i) { + const int idx = smem_nhw + thread_in_cta_nhw + i*PIXELS_PER_LDG; + const bool is_valid = ((unsigned int)idx < (unsigned int)params.nhw) && is_valid_c; + if (is_valid) { + // Read from SMEM. + int offset = i*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + PackedStorageType x_storage_local[PACKED_ELEMENTS_PER_LDG], + dy_storage_local[PACKED_ELEMENTS_PER_LDG]; + read_from_smem(x_storage_local, &smem_storage_packed[offset], threadIdx.x); + offset += PIXELS_PER_THREAD_IN_SMEM*THREADS_PER_CTA*PACKED_ELEMENTS_PER_LDG; + read_from_smem(dy_storage_local, &smem_storage_packed[offset], threadIdx.x); + float x_math[ELEMENTS_PER_LDG], dy_math[ELEMENTS_PER_LDG]; + to_float(x_math, x_storage_local); + to_float(dy_math, dy_storage_local); + + float dx[ELEMENTS_PER_LDG]; + bwd_dx(dx, dy_math, var, x_math, mean, dscale, dbias, inv_count); + + // Write back. + stg_stream(&gmem_dst[idx*params.c], dx); + } + } + } + // We're about to start on the next c-blk. Needed? + __syncthreads(); + } +} + +#endif // MXNET_OPERATOR_NN_CUDNN_NHWC_BATCH_NORM_KERNEL_H_ diff --git a/apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda.cpp b/apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda.cpp new file mode 100644 index 00000000..c74094ff --- /dev/null +++ b/apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda.cpp @@ -0,0 +1,139 @@ +#include + +#include +#include + +void index_mul_2d_float_foward_cuda(at::Tensor &out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1); + +void index_mul_2d_float_backward_cuda(at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1); + +void index_mul_2d_float_backward_backward_cuda(at::Tensor &grad_grad_out, + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &grad_grad_in1, + const at::Tensor &grad_grad_in2, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1); + +void index_mul_2d_half_foward_cuda(at::Tensor &out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1); + +void index_mul_2d_half_backward_cuda(at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1); + +void index_mul_2d_half_backward_backward_cuda(at::Tensor &grad_grad_out, + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &grad_grad_in1, + const at::Tensor &grad_grad_in2, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1); + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) + +void index_mul_2d_float_forward( + at::Tensor &out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) +{ + return index_mul_2d_float_foward_cuda(out, in1, in2, idx1); +} + +void index_mul_2d_float_backward( + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) +{ + return index_mul_2d_float_backward_cuda(grad_in1, grad_in2, grad_out, in1, in2, idx1); +} + +void index_mul_2d_float_backwrad_backward( + at::Tensor &grad_grad_out, + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &grad_grad_in1, + const at::Tensor &grad_grad_in2, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) +{ + return index_mul_2d_float_backward_backward_cuda(grad_grad_out, grad_in1, grad_in2, grad_out, grad_grad_in1, grad_grad_in2, in1, in2, idx1); +} + +void index_mul_2d_half_forward( + at::Tensor &out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) +{ + return index_mul_2d_half_foward_cuda(out, in1, in2, idx1); +} + +void index_mul_2d_half_backward( + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) +{ + return index_mul_2d_half_backward_cuda(grad_in1, grad_in2, grad_out, in1, in2, idx1); +} + +void index_mul_2d_half_backwrad_backward( + at::Tensor &grad_grad_out, + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &grad_grad_in1, + const at::Tensor &grad_grad_in2, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) +{ + return index_mul_2d_half_backward_backward_cuda(grad_grad_out, grad_in1, grad_in2, grad_out, grad_grad_in1, grad_grad_in2, in1, in2, idx1); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("float_forward", &index_mul_2d_float_forward, + "index mul float calculation forward (CUDA)"); + m.def("float_backward", &index_mul_2d_float_backward, + "index mul float calculation backward (CUDA)"); + m.def("float_backward_backward", &index_mul_2d_float_backwrad_backward, + "index mul float calculation backward backward (CUDA)"); + m.def("half_forward", &index_mul_2d_half_forward, + "index mul half calculation forward (CUDA)"); + m.def("half_backward", &index_mul_2d_half_backward, + "index mul half calculation backward (CUDA)"); + m.def("half_backward_backward", &index_mul_2d_half_backwrad_backward, + "index mul half calculation backward backward (CUDA)"); +} + diff --git a/apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda_kernel.cu b/apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda_kernel.cu new file mode 100644 index 00000000..5181170a --- /dev/null +++ b/apex/apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda_kernel.cu @@ -0,0 +1,479 @@ +#include +#include +#include +#include + + +__global__ void index_mul_2d_float_dim64( + float *out, + const float *in1, + const float *in2, + const int64_t *idx1, + const int64_t size) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + constexpr int fea_dim = 64; + + if (start_idx < size) { + int64_t vec_idx1 = (idx1[start_idx] * fea_dim) / 4 + tidx; + int64_t vec_idx2 = (start_idx * fea_dim) / 4 + tidx; + + float4 res, src1, src2; + src1 = reinterpret_cast(in1)[vec_idx1]; + src2 = reinterpret_cast(in2)[vec_idx2]; + res.x = src1.x * src2.x; + res.y = src1.y * src2.y; + res.z = src1.z * src2.z; + res.w = src1.w * src2.w; + reinterpret_cast(out)[vec_idx2] = res; + } +} + +__global__ void index_mul_2d_float( + float *out, + const float *in1, + const float *in2, + const int64_t *idx1, + const int64_t size, + const int64_t fea_dim) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + const int stride = blockDim.x; + + if (start_idx < size) { + int64_t vec_idx1 = (idx1[start_idx] * fea_dim); + int64_t vec_idx2 = (start_idx * fea_dim); + + for (int i = tidx; i < fea_dim; i += stride) { + out[vec_idx2 + i] = in1[vec_idx1 + i] * in2[vec_idx2 + i]; + } + } +} + +__global__ void index_mul_2d_half( + at::Half *out, + const at::Half *in1, + const at::Half *in2, + const int64_t *idx1, + const int64_t size, + const int64_t fea_dim) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + const int stride = blockDim.x; + + if (start_idx < size) { + int64_t vec_idx1 = (idx1[start_idx] * fea_dim); + int64_t vec_idx2 = (start_idx * fea_dim); + + for (int i = tidx; i < fea_dim; i += stride) { + out[vec_idx2 + i] = at::Half(static_cast(in1[vec_idx1 + i]) * static_cast(in2[vec_idx2 + i])); + } + } +} + +__global__ void index_mul_2d_grad_float_dim64( + float *grad_in1, + float *grad_in2, + const float *grad_out, + const float *in1, + const float *in2, + const int64_t *idx1, + const int64_t size) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + constexpr int fea_dim = 64; + + if (start_idx < size) { + int64_t vec_idx1 = (idx1[start_idx] * fea_dim) / 4 + tidx; + int64_t vec_idx2 = (start_idx * fea_dim) / 4 + tidx; + + float4 src_in1, src_in2, src_grad_out, dst_grad_in2; + src_grad_out = reinterpret_cast(grad_out)[vec_idx2]; + src_in1 = reinterpret_cast(in1)[vec_idx1]; + src_in2 = reinterpret_cast(in2)[vec_idx2]; + int64_t grad_in1_base_idx = idx1[start_idx] * fea_dim + tidx * 4; + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 0, src_grad_out.x * src_in2.x); + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 1, src_grad_out.y * src_in2.y); + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 2, src_grad_out.z * src_in2.z); + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 3, src_grad_out.w * src_in2.w); + dst_grad_in2.x = src_grad_out.x * src_in1.x; + dst_grad_in2.y = src_grad_out.y * src_in1.y; + dst_grad_in2.z = src_grad_out.z * src_in1.z; + dst_grad_in2.w = src_grad_out.w * src_in1.w; + reinterpret_cast(grad_in2)[vec_idx2] = dst_grad_in2; + } +} + +__global__ void index_mul_2d_grad_float( + float *grad_in1, + float *grad_in2, + const float *grad_out, + const float *in1, + const float *in2, + const int64_t *idx1, + const int64_t size, + const int64_t fea_dim) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + const int stride = blockDim.x; + + if (start_idx < size) { + int64_t vec_idx1 = idx1[start_idx] * fea_dim; + int64_t vec_idx2 = start_idx * fea_dim; + + for (int i = tidx; i < fea_dim; i += stride) { + float src_in1 = in1[vec_idx1 + i]; + float src_in2 = in2[vec_idx2 + i]; + float src_grad_out = grad_out[vec_idx2 + i]; + grad_in2[vec_idx2 + i] = src_grad_out * src_in1; + gpuAtomicAdd(grad_in1 + vec_idx1 + i, src_grad_out * src_in2); + } + } +} + +__global__ void index_mul_2d_grad_half( + at::Half *grad_in1, + at::Half *grad_in2, + const at::Half *grad_out, + const at::Half *in1, + const at::Half *in2, + const int64_t *idx1, + const int64_t size, + const int64_t fea_dim) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + const int stride = blockDim.x; + + if (start_idx < size) { + int64_t vec_idx1 = idx1[start_idx] * fea_dim; + int64_t vec_idx2 = start_idx * fea_dim; + + for (int i = tidx; i < fea_dim; i += stride) { + float src_in1 = static_cast(in1[vec_idx1 + i]); + float src_in2 = static_cast(in2[vec_idx2 + i]); + float src_grad_out = static_cast(grad_out[vec_idx2 + i]); + grad_in2[vec_idx2 + i] = at::Half(src_grad_out * src_in1); + gpuAtomicAdd(grad_in1 + vec_idx1 + i, at::Half(src_grad_out * src_in2)); + } + } +} + +__global__ void index_mul_2d_grad_grad_float_dim64( + float *grad_grad_out, + float *grad_in1, + float *grad_in2, + const float *grad_out, + const float *grad_grad_in1, + const float *grad_grad_in2, + const float *in1, + const float *in2, + const int64_t *idx1, + const int64_t size) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + constexpr int fea_dim = 64; + + if (start_idx < size) { + int64_t vec_idx1 = (idx1[start_idx] * fea_dim) / 4 + tidx; + int64_t vec_idx2 = (start_idx * fea_dim) / 4 + tidx; + + float4 src_grad_grad_in1, src_in1, src_grad_grad_in2, src_in2, src_grad_out; + float4 dst_grad_grad_out, dst_grad_in2; + src_grad_grad_in1 = reinterpret_cast(grad_grad_in1)[vec_idx1]; + src_in1 = reinterpret_cast(in1)[vec_idx1]; + src_grad_grad_in2 = reinterpret_cast(grad_grad_in2)[vec_idx2]; + src_in2 = reinterpret_cast(in2)[vec_idx2]; + dst_grad_grad_out.x = src_grad_grad_in1.x * src_in2.x + src_grad_grad_in2.x * src_in1.x; + dst_grad_grad_out.y = src_grad_grad_in1.y * src_in2.y + src_grad_grad_in2.y * src_in1.y; + dst_grad_grad_out.z = src_grad_grad_in1.z * src_in2.z + src_grad_grad_in2.z * src_in1.z; + dst_grad_grad_out.w = src_grad_grad_in1.w * src_in2.w + src_grad_grad_in2.w * src_in1.w; + reinterpret_cast(grad_grad_out)[vec_idx2] = dst_grad_grad_out; + src_grad_out = reinterpret_cast(grad_out)[vec_idx2]; + int64_t grad_in1_base_idx = idx1[start_idx] * fea_dim + tidx * 4; + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 0, src_grad_grad_in2.x * src_grad_out.x); + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 1, src_grad_grad_in2.y * src_grad_out.y); + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 2, src_grad_grad_in2.z * src_grad_out.z); + gpuAtomicAdd(grad_in1 + grad_in1_base_idx + 3, src_grad_grad_in2.w * src_grad_out.w); + dst_grad_in2.x = src_grad_grad_in1.x * src_grad_out.x; + dst_grad_in2.y = src_grad_grad_in1.y * src_grad_out.y; + dst_grad_in2.z = src_grad_grad_in1.z * src_grad_out.z; + dst_grad_in2.w = src_grad_grad_in1.w * src_grad_out.w; + reinterpret_cast(grad_in2)[vec_idx2] = dst_grad_in2; + } +} + +__global__ void index_mul_2d_grad_grad_float( + float *grad_grad_out, + float *grad_in1, + float *grad_in2, + const float *grad_out, + const float *grad_grad_in1, + const float *grad_grad_in2, + const float *in1, + const float *in2, + const int64_t *idx1, + const int64_t size, + const int64_t fea_dim) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + const int stride = blockDim.x; + + if (start_idx < size) { + int64_t vec_idx1 = idx1[start_idx] * fea_dim; + int64_t vec_idx2 = start_idx * fea_dim; + + for (int i = tidx; i < fea_dim; i += stride) { + float src_grad_grad_in1 = grad_grad_in1[vec_idx1 + i]; + float src_grad_grad_in2 = grad_grad_in2[vec_idx2 + i]; + float src_in1 = in1[vec_idx1 + i]; + float src_in2 = in2[vec_idx2 + i]; + float src_grad_out = grad_out[vec_idx2 + i]; + grad_grad_out[vec_idx2 + i] = src_grad_grad_in1 * src_in2 + src_grad_grad_in2 * src_in1; + grad_in2[vec_idx2 + i] = src_grad_grad_in1 * src_grad_out; + gpuAtomicAdd(grad_in1 + vec_idx1 + i, src_grad_grad_in2 * src_grad_out); + } + } +} + +__global__ void index_mul_2d_grad_grad_half( + at::Half *grad_grad_out, + at::Half *grad_in1, + at::Half *grad_in2, + const at::Half *grad_out, + const at::Half *grad_grad_in1, + const at::Half *grad_grad_in2, + const at::Half *in1, + const at::Half *in2, + const int64_t *idx1, + const int64_t size, + const int64_t fea_dim) +{ + const int tidx = threadIdx.x; + const int tidy = threadIdx.y; + const int bidx = blockIdx.x; + const int start_idx = bidx * blockDim.y + tidy; + const int stride = blockDim.x; + + if (start_idx < size) { + int64_t vec_idx1 = idx1[start_idx] * fea_dim; + int64_t vec_idx2 = start_idx * fea_dim; + + for (int i = tidx; i < fea_dim; i += stride) { + float src_grad_grad_in1 = static_cast(grad_grad_in1[vec_idx1 + i]); + float src_grad_grad_in2 = static_cast(grad_grad_in2[vec_idx2 + i]); + float src_in1 = static_cast(in1[vec_idx1 + i]); + float src_in2 = static_cast(in2[vec_idx2 + i]); + float src_grad_out = static_cast(grad_out[vec_idx2 + i]); + grad_grad_out[vec_idx2 + i] = at::Half(src_grad_grad_in1 * src_in2 + src_grad_grad_in2 * src_in1); + grad_in2[vec_idx2 + i] = at::Half(src_grad_grad_in1 * src_grad_out); + gpuAtomicAdd(grad_in1 + vec_idx1 + i, at::Half(src_grad_grad_in2 * src_grad_out)); + } + } +} + +void index_mul_2d_float_foward_cuda(at::Tensor &out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) { + const int64_t size = in2.size(0); + const int64_t fea_dim = in2.size(1); + if (size < 0){ + return; + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (fea_dim == 64) { + const int BLOCK_THREADS_DIMX = 16; + const int BLOCK_THREADS_DIMY = 16; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_float_dim64<<>>( + out.data_ptr(), in1.data_ptr(), in2.data_ptr(), + idx1.data_ptr(), size); + } else { + const int BLOCK_THREADS_DIMX = 32; + const int BLOCK_THREADS_DIMY = 8; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_float<<>>( + out.data_ptr(), in1.data_ptr(), in2.data_ptr(), + idx1.data_ptr(), size, fea_dim); + } + + AT_CUDA_CHECK(cudaGetLastError()); +} + +void index_mul_2d_float_backward_cuda(at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) { + const int64_t size = in2.size(0); + const int64_t fea_dim = in2.size(1); + if (size < 0){ + return; + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (fea_dim == 64) { + const int BLOCK_THREADS_DIMX = 16; + const int BLOCK_THREADS_DIMY = 16; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_grad_float_dim64<<>>( + grad_in1.data_ptr(), grad_in2.data_ptr(), grad_out.data_ptr(), + in1.data_ptr(), in2.data_ptr(), idx1.data_ptr(), size); + + AT_CUDA_CHECK(cudaGetLastError()); + } else { + const int BLOCK_THREADS_DIMX = 32; + const int BLOCK_THREADS_DIMY = 8; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_grad_float<<>>( + grad_in1.data_ptr(), grad_in2.data_ptr(), grad_out.data_ptr(), + in1.data_ptr(), in2.data_ptr(), idx1.data_ptr(), size, fea_dim); + } +} + +void index_mul_2d_float_backward_backward_cuda(at::Tensor &grad_grad_out, + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &grad_grad_in1, + const at::Tensor &grad_grad_in2, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) { + const int64_t size = in2.size(0); + const int64_t fea_dim = in2.size(1); + if (size < 0){ + return; + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (fea_dim == 64) { + const int BLOCK_THREADS_DIMX = 16; + const int BLOCK_THREADS_DIMY = 16; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_grad_grad_float_dim64<<>>( + grad_grad_out.data_ptr(), grad_in1.data_ptr(), grad_in2.data_ptr(), + grad_out.data_ptr(), grad_grad_in1.data_ptr(), grad_grad_in2.data_ptr(), + in1.data_ptr(), in2.data_ptr(), idx1.data_ptr(), size); + } else { + const int BLOCK_THREADS_DIMX = 32; + const int BLOCK_THREADS_DIMY = 8; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_grad_grad_float<<>>( + grad_grad_out.data_ptr(), grad_in1.data_ptr(), grad_in2.data_ptr(), + grad_out.data_ptr(), grad_grad_in1.data_ptr(), grad_grad_in2.data_ptr(), + in1.data_ptr(), in2.data_ptr(), idx1.data_ptr(), size, fea_dim); + } + + AT_CUDA_CHECK(cudaGetLastError()); +} + +void index_mul_2d_half_foward_cuda(at::Tensor &out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) { + const int64_t size = in2.size(0); + const int64_t fea_dim = in2.size(1); + if (size < 0){ + return; + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int BLOCK_THREADS_DIMX = 32; + const int BLOCK_THREADS_DIMY = 8; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_half<<>>( + out.data_ptr(), in1.data_ptr(), in2.data_ptr(), + idx1.data_ptr(), size, fea_dim); + + AT_CUDA_CHECK(cudaGetLastError()); +} + +void index_mul_2d_half_backward_cuda(at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) { + const int64_t size = in2.size(0); + const int64_t fea_dim = in2.size(1); + if (size < 0){ + return; + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int BLOCK_THREADS_DIMX = 32; + const int BLOCK_THREADS_DIMY = 8; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_grad_half<<>>( + grad_in1.data_ptr(), grad_in2.data_ptr(), grad_out.data_ptr(), + in1.data_ptr(), in2.data_ptr(), idx1.data_ptr(), size, fea_dim); +} + +void index_mul_2d_half_backward_backward_cuda(at::Tensor &grad_grad_out, + at::Tensor &grad_in1, + at::Tensor &grad_in2, + const at::Tensor &grad_out, + const at::Tensor &grad_grad_in1, + const at::Tensor &grad_grad_in2, + const at::Tensor &in1, + const at::Tensor &in2, + const at::Tensor &idx1) { + const int64_t size = in2.size(0); + const int64_t fea_dim = in2.size(1); + if (size < 0){ + return; + } + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + const int BLOCK_THREADS_DIMX = 32; + const int BLOCK_THREADS_DIMY = 8; + const int BLOCK_NUMS = (size + BLOCK_THREADS_DIMY - 1) / BLOCK_THREADS_DIMY; + + index_mul_2d_grad_grad_half<<>>( + grad_grad_out.data_ptr(), grad_in1.data_ptr(), grad_in2.data_ptr(), + grad_out.data_ptr(), grad_grad_in1.data_ptr(), grad_grad_in2.data_ptr(), + in1.data_ptr(), in2.data_ptr(), idx1.data_ptr(), size, fea_dim); + + AT_CUDA_CHECK(cudaGetLastError()); +} \ No newline at end of file diff --git a/apex/apex/contrib/csrc/layer_norm/ln.h b/apex/apex/contrib/csrc/layer_norm/ln.h new file mode 100644 index 00000000..6ab709b0 --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln.h @@ -0,0 +1,202 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace layer_norm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct LaunchParams{ + + size_t workspace_bytes; + size_t barrier_size; + + cudaDeviceProp * props; + + cudaStream_t stream; + + Params params; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct ParamsBase { + ParamsBase() + : ctas_per_col(0) + , rows(0) + , cols(0) + , x(nullptr) + , mu(nullptr) + , rs(nullptr) + , gamma(nullptr) + , workspace(nullptr) + , barrier(nullptr) + { + } + + // For Multi-CTA, number of different CTA groups. Otherwise same as gridDim.x. + int ctas_per_col; + + // Input is interpreted as matrix. We normalize across columns. + int rows; + int cols; + + // Common data pointers. + void *x; + void *mu; + void *rs; + void *gamma; + + // Multi-CTA workspace in gmem. + void *workspace; + + // Multi-CTA sync barriers in gmem. + int *barrier; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct FwdParams : public ParamsBase { + FwdParams() + : ParamsBase() + , z(nullptr) + , beta(nullptr) + , epsilon(0.f) + { + } + + // Output of LN FWD. + void *z; + void *beta; + float epsilon; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct BwdParams : public ParamsBase { + BwdParams() + : ParamsBase() + , dz(nullptr) + , dbeta_part(nullptr) + , dgamma_part(nullptr) + , dx(nullptr) + , dbeta(nullptr) + , dgamma(nullptr) + { + } + + // Input: gradient wrt. LN FWD output. + void *dz; + + // Workspace for Wgrad pre-reduction. + void *dbeta_part; + void *dgamma_part; + + // Output: Dgrad. + void *dx; + // Output: Wgrad. + void *dbeta; + void *dgamma; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +using FwdFunction = std::function&, const bool)>; +using BwdFunction = std::function&, const bool)>; +using FunctionKey = uint64_t; +using FwdRegistry = std::unordered_map; +using BwdRegistry = std::unordered_map; + +extern FwdRegistry FWD_FUNCS; +extern BwdRegistry BWD_FUNCS; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +using fp32 = float; +using fp16 = half; +using bf16 = nv_bfloat16; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct TypeId{}; + +template<> +struct TypeId{ + constexpr static uint32_t Value = 0; +}; + +template<> +struct TypeId{ + constexpr static uint32_t Value = 1; +}; + +template<> +struct TypeId{ + constexpr static uint32_t Value = 2; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Type2Key{ + constexpr static uint32_t Value = TypeId::Value << S; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct WeightType2Key : public Type2Key{}; + +template +struct InputType2Key : public Type2Key{}; + +template +struct OutputType2Key : public Type2Key{}; + +template +struct ComputeType2Key : public Type2Key{}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Types2Key{ + constexpr static uint32_t Value = WeightType2Key::Value | InputType2Key::Value | OutputType2Key::Value | ComputeType2Key::Value; + constexpr static inline uint64_t get(const uint64_t hidden_size){ + constexpr uint64_t type_key = Value; + return (type_key << 32) | hidden_size; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct FwdRegistrar{ + FwdRegistrar(FwdFunction f){ + uint64_t key = Types2Key::get(HIDDEN_SIZE); + FWD_FUNCS.insert({ key, f }); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct BwdRegistrar{ + BwdRegistrar(BwdFunction f){ + uint64_t key = Types2Key::get(HIDDEN_SIZE); + BWD_FUNCS.insert({ key, f }); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace layer_norm diff --git a/apex/apex/contrib/csrc/layer_norm/ln_api.cpp b/apex/apex/contrib/csrc/layer_norm/ln_api.cpp new file mode 100644 index 00000000..30e4a5fe --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_api.cpp @@ -0,0 +1,246 @@ +#include +#include "ATen/cuda/CUDAContext.h" + +#include "ln.h" + +/* + +Supported Type combinations: + +input compute weights output +======================================= +fp32 fp32 fp32 fp32 +fp16 fp32 fp16 fp16 +bf16 fp32 bf16 bf16 +fp32 fp32 fp16 fp16 +fp32 fp32 bf16 bf16 + +Remarks: +Output type = Weight type +Compute always in FP32 + +*/ + +namespace layer_norm { + +// Create registries and provide runtime versions of config hash functions. + +FwdRegistry FWD_FUNCS; +BwdRegistry BWD_FUNCS; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +uint32_t get_type_id(torch::Dtype dtype){ + if( dtype == torch::kFloat16 ) { + return TypeId::Value; + } else if( dtype == torch::kBFloat16 ) { + return TypeId::Value; + } else if( dtype == torch::kFloat32 ) { + return TypeId::Value; + } else { + TORCH_CHECK(false, "Type not supported: ", dtype); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +uint64_t get_key(torch::Dtype wtype, torch::Dtype itype, torch::Dtype otype, torch::Dtype ctype, uint64_t hidden_size) { + using namespace layer_norm; + uint64_t type_key = get_type_id(wtype) | (get_type_id(itype) << 2) | (get_type_id(otype) << 4) | (get_type_id(ctype) << 6); + uint64_t launcher_key = (type_key << 32) | hidden_size; + return launcher_key; +} + +} // namespace layer_norm + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +layer_norm::FwdFunction & get_fwd_launcher(torch::Dtype wtype, torch::Dtype itype, torch::Dtype otype, torch::Dtype ctype, uint32_t hidden_size) { + auto iter = layer_norm::FWD_FUNCS.find(layer_norm::get_key(wtype, itype, otype, ctype, hidden_size)); + if( iter != layer_norm::FWD_FUNCS.end() ) { + return iter->second; + } else { + TORCH_CHECK(false, "FWD: Unsupported hidden_size or types: ", hidden_size, wtype, itype, otype, ctype); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +layer_norm::BwdFunction & get_bwd_launcher(torch::Dtype wtype, torch::Dtype itype, torch::Dtype otype, torch::Dtype ctype, uint32_t hidden_size) { + auto iter = layer_norm::BWD_FUNCS.find(layer_norm::get_key(wtype, itype, otype, ctype, hidden_size)); + if( iter != layer_norm::BWD_FUNCS.end() ) { + return iter->second; + } else { + TORCH_CHECK(false, "BWD: Unsupported hidden_size or types: ", hidden_size, wtype, itype, otype, ctype); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +std::vector ln_fwd(const at::Tensor &x, // BxSxhidden_size + const at::Tensor &gamma, // hidden_size + const at::Tensor &beta, // hidden_size + const float epsilon +) { + auto itype = x.scalar_type(); + auto wtype = gamma.scalar_type(); + auto otype = wtype; + auto ctype = torch::kFloat32; + + TORCH_CHECK(beta.scalar_type() == wtype); + + TORCH_CHECK(x.is_cuda()) + TORCH_CHECK(gamma.is_cuda()) + TORCH_CHECK(beta.is_cuda()) + + TORCH_CHECK(x.is_contiguous()); + auto sizes = x.sizes(); + TORCH_CHECK(sizes.size() == 2); + + const int rows = sizes[0]; + const int cols = sizes[1]; + auto hidden_size = gamma.numel(); + + TORCH_CHECK(gamma.sizes() == beta.sizes()); + TORCH_CHECK(hidden_size == cols); + + TORCH_CHECK(epsilon >= 0.f); + + auto opts = x.options(); + + auto z = torch::empty(sizes, opts.dtype(otype)); + + auto mu = torch::empty({ rows }, opts.dtype(ctype)); + auto rsigma = torch::empty({ rows }, opts.dtype(ctype)); + + layer_norm::LaunchParams launch_params; + + launch_params.props = at::cuda::getCurrentDeviceProperties(); + launch_params.stream = at::cuda::getCurrentCUDAStream().stream(); + + // Request the kernel launcher. + auto launcher = get_fwd_launcher(wtype, itype, otype, ctype, hidden_size); + + // Query the kernel-specific launch parameters. + launcher(launch_params, true); + + at::Tensor workspace, barrier; + + // Set the kernel runtime parameters. + layer_norm::FwdParams ¶ms = launch_params.params; + params.rows = rows; + params.cols = cols; + params.x = x.data_ptr(); + params.mu = mu.data_ptr(); + params.rs = rsigma.data_ptr(); + params.gamma = gamma.data_ptr(); + params.beta = beta.data_ptr(); + params.z = z.data_ptr(); + params.epsilon = epsilon; + + if( launch_params.barrier_size > 0 ) { + auto options = x.options(); + barrier = torch::zeros(launch_params.barrier_size, options.dtype(torch::kInt32)); + workspace = torch::empty(launch_params.workspace_bytes, options.dtype(torch::kChar)); + params.workspace = workspace.data_ptr(); + params.barrier = barrier.data_ptr(); + } + + // Launch the kernel. + launcher(launch_params, false); + + return { z, mu, rsigma }; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +std::vector ln_bwd(const at::Tensor &dz, // BxSxhidden_size + const at::Tensor &x, // BxSxhidden_size + const at::Tensor &mu, // BxS, FP32! + const at::Tensor &rsigma, // BxS, FP32! + const at::Tensor &gamma // hidden_size +) { + + auto itype = x.scalar_type(); + auto wtype = gamma.scalar_type(); + auto otype = wtype; + auto ctype = torch::kFloat32; + + TORCH_CHECK(dz.dtype() == otype); + TORCH_CHECK(mu.dtype() == ctype); + TORCH_CHECK(rsigma.dtype() == ctype); + + TORCH_CHECK(x.is_cuda()); + TORCH_CHECK(dz.is_cuda()); + TORCH_CHECK(mu.is_cuda()); + TORCH_CHECK(rsigma.is_cuda()); + TORCH_CHECK(gamma.is_cuda()); + + TORCH_CHECK(x.is_contiguous()); + TORCH_CHECK(dz.is_contiguous()); + + auto sizes = x.sizes(); + TORCH_CHECK(sizes.size() == 2); + TORCH_CHECK(dz.sizes() == sizes); + auto rows = sizes[0]; + auto cols = sizes[1]; + + auto hidden_size = gamma.numel(); + + TORCH_CHECK(mu.numel() == rows); + TORCH_CHECK(mu.sizes() == rsigma.sizes()); + + TORCH_CHECK(gamma.numel() == cols); + + auto options = x.options(); + + auto dx = torch::empty_like(x); + auto dgamma = torch::empty_like(gamma); + auto dbeta = torch::empty_like(gamma); + + layer_norm::LaunchParams launch_params; + launch_params.stream = at::cuda::getCurrentCUDAStream().stream(); + launch_params.props = at::cuda::getCurrentDeviceProperties(); + + auto launcher = get_bwd_launcher(wtype, itype, otype, ctype, hidden_size); + + launcher(launch_params, true); + + auto dgamma_part = torch::empty({ launch_params.params.ctas_per_col, hidden_size }, options.dtype(ctype)); + auto dbeta_part = torch::empty({ launch_params.params.ctas_per_col, hidden_size }, options.dtype(ctype)); + at::Tensor workspace, barrier; + + layer_norm::BwdParams ¶ms = launch_params.params; + params.rows = rows; + params.cols = cols; + params.x = x.data_ptr(); + params.mu = mu.data_ptr(); + params.rs = rsigma.data_ptr(); + params.gamma = gamma.data_ptr(); + params.dz = dz.data_ptr(); + params.dx = dx.data_ptr(); + params.dbeta = dbeta.data_ptr(); + params.dgamma = dgamma.data_ptr(); + params.dbeta_part = dbeta_part.data_ptr(); + params.dgamma_part = dgamma_part.data_ptr(); + + if( launch_params.barrier_size > 0 ) { + // TODO Any way to avoid this? + barrier = torch::zeros(launch_params.barrier_size, options.dtype(torch::kInt32)); + workspace = torch::empty(launch_params.workspace_bytes, options.dtype(torch::kChar)); + params.workspace = workspace.data_ptr(); + params.barrier = barrier.data_ptr(); + } + + launcher(launch_params, false); + + return { dx, dgamma, dbeta, dgamma_part, dbeta_part }; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.doc() = "CUDA LayerNorm"; + m.def("ln_fwd", &ln_fwd, "Run LayerNorm forward kernel"); + m.def("ln_bwd", &ln_bwd, "Run LayerNorm backward kernel"); +} diff --git a/apex/apex/contrib/csrc/layer_norm/ln_bwd_kernels.cuh b/apex/apex/contrib/csrc/layer_norm/ln_bwd_kernels.cuh new file mode 100644 index 00000000..8595f5ed --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_bwd_kernels.cuh @@ -0,0 +1,315 @@ +#pragma once + +namespace layer_norm { + +template +__global__ __launch_bounds__(Ktraits::THREADS_PER_CTA) +void ln_bwd_kernel(layer_norm::BwdParams params) { + + enum { ROWS_PER_CTA = Ktraits::ROWS_PER_CTA }; + enum { WARPS_M = Ktraits::WARPS_M }; + enum { WARPS_N = Ktraits::WARPS_N }; + enum { THREADS_PER_ROW = Ktraits::THREADS_PER_ROW }; + enum { COLS = Ktraits::COLS }; + enum { BYTES_PER_ROW = Ktraits::BYTES_PER_ROW }; + enum { LDGS = Ktraits::LDGS }; + enum { NUM_ELTS = Ktraits::ELTS_PER_LDG }; + enum { THREADS_PER_WARP = Ktraits::THREADS_PER_WARP }; + enum { CTAS_PER_ROW = Ktraits::CTAS_PER_ROW }; + + using compute_t = typename Ktraits::compute_t; + using index_t = typename Ktraits::index_t; + using Ivec = typename Ktraits::Ivec; + using Ovec = typename Ktraits::Ovec; + using Wvec = typename Ktraits::Wvec; + using Cvec = typename Ktraits::Cvec; + using Reducer = typename Ktraits::Reducer; + using reduce_t = typename Reducer::Type; + + extern __shared__ char smem_[]; + + const index_t tidx = threadIdx.x; + const index_t bidn = blockIdx.x % CTAS_PER_ROW; + const index_t bidm = blockIdx.x / CTAS_PER_ROW; + const index_t lane = tidx % THREADS_PER_WARP; + const index_t warp = tidx / THREADS_PER_WARP; + const index_t warp_m = warp / Ktraits::WARPS_N; + const index_t warp_n = warp % Ktraits::WARPS_N; + const index_t tid_r = warp_n * THREADS_PER_WARP + lane; + + const index_t r = bidm * Ktraits::ROWS_PER_CTA + warp_m; + const index_t c = bidn * THREADS_PER_ROW + warp_n * THREADS_PER_WARP + lane; + + static_assert(COLS == THREADS_PER_ROW * LDGS * NUM_ELTS * CTAS_PER_ROW); + + Cvec dzy_sum[LDGS]; + Cvec dz_sum[LDGS]; + + memset(dzy_sum, 0, sizeof(dzy_sum)); + memset(dz_sum, 0, sizeof(dz_sum)); + + compute_t * smem_wgrad = reinterpret_cast(smem_); + char *smem_dgrad = smem_ + Ktraits::SMEM_BYTES_WGRAD; + + Reducer reducer(params, bidm, bidn, warp_m, warp_n, lane, smem_dgrad); + + Sum sum; + + constexpr float rn = 1.f / float(COLS); + Wvec gamma[LDGS]; + index_t idx = c; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + gamma[it].load_from(params.gamma, idx); + idx += Ktraits::VEC_COLS_PER_LDG; + } + // TODO if ROWS_PER_CTA does not divide rows, we might get divergence in the + // last blocks with syncthreads! + // grid stride over rows + #pragma unroll 1 + for( int row = r; row < params.rows; row += params.ctas_per_col * ROWS_PER_CTA ) { + const compute_t mu_r = static_cast(params.mu)[row]; + const compute_t rs_r = static_cast(params.rs)[row]; + Ivec x[LDGS]; + Ovec dz[LDGS]; + index_t idx = row * Ktraits::VEC_COLS + c; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + dz[it].load_from(params.dz, idx); + x[it].load_from(params.x, idx); + idx += Ktraits::VEC_COLS_PER_LDG; + } + + compute_t dy[LDGS * NUM_ELTS]; + compute_t y[LDGS * NUM_ELTS]; + + compute_t mdy_local = 0.f; + compute_t mdyy_local = 0.f; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + #pragma unroll + for( int jt = 0; jt < NUM_ELTS; jt++ ) { + compute_t x_tmp = x[it].data.elt[jt]; + compute_t y_tmp = rs_r * (x_tmp - mu_r); + compute_t dy_tmp = compute_t(gamma[it].data.elt[jt]); + dy_tmp *= compute_t(dz[it].data.elt[jt]); + compute_t dz_tmp = dz[it].data.elt[jt]; + + mdy_local += dy_tmp; + mdyy_local += dy_tmp * y_tmp; + + dy[it * NUM_ELTS + jt] = dy_tmp; + y[it * NUM_ELTS + jt] = y_tmp; + + dzy_sum[it].data.elt[jt] += dz_tmp * y_tmp; + dz_sum[it].data.elt[jt] += dz_tmp; + } + } + + reduce_t result = reducer.allreduce({mdy_local, mdyy_local}, sum); + mdy_local = layer_norm::Get<0>::of(result) * rn; + mdyy_local = layer_norm::Get<1>::of(result) * rn; + + Ivec dx[LDGS]; + idx = row * Ktraits::VEC_COLS + c; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + #pragma unroll + for( int jt = 0; jt < NUM_ELTS; jt++ ) { + compute_t dy_tmp = dy[it * NUM_ELTS + jt]; + compute_t y_tmp = y[it * NUM_ELTS + jt]; + compute_t dx_tmp = rs_r * (dy_tmp - (mdyy_local * y_tmp + mdy_local)); + dx[it].data.elt[jt] = dx_tmp; + } + dx[it].store_to(params.dx, idx); + idx += Ktraits::VEC_COLS_PER_LDG; + } + + } // end: grid stride loop + + if( WARPS_M == 1 ) { + idx = r * Ktraits::VEC_COLS + c; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + dz_sum[it].store_to(params.dbeta_part, idx); + dzy_sum[it].store_to(params.dgamma_part, idx); + idx += Ktraits::VEC_COLS_PER_LDG; + } + } else { + static_assert(WARPS_M == 1 || Ktraits::CTAS_PER_ROW == 1, "Multiple rows per CTA not supported for Multi-CTA."); + // Finalize reduction of part dgamma and dbeta for this CTA + // by reducing over the rows held across the WARPS_M warps + + // Assumption: blockSize divides hidden size. + enum { NUM_RES = COLS / Ktraits::THREADS_PER_CTA }; + static_assert(NUM_RES * Ktraits::THREADS_PER_CTA == COLS, ""); + + idx = warp_m * Ktraits::VEC_COLS + tid_r; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + dz_sum[it].store_to(smem_wgrad, idx); + idx += THREADS_PER_ROW; + } + __syncthreads(); + compute_t cta_dz_sum[NUM_RES]; + memset(cta_dz_sum, 0, sizeof(compute_t) * NUM_RES); + for( int it = 0; it < ROWS_PER_CTA; it++ ) { + for( int jt = 0; jt < NUM_RES; jt++ ) { + cta_dz_sum[jt] += smem_wgrad[it * COLS + tidx + jt * Ktraits::THREADS_PER_CTA]; + } + } + __syncthreads(); + + idx = warp_m * Ktraits::VEC_COLS + tid_r; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + dzy_sum[it].store_to(smem_wgrad, idx); + idx += THREADS_PER_ROW; + } + __syncthreads(); + compute_t cta_dzy_sum[NUM_RES]; + memset(cta_dzy_sum, 0, sizeof(compute_t) * NUM_RES); + for( int it = 0; it < ROWS_PER_CTA; it++ ) { + for( int jt = 0; jt < NUM_RES; jt++ ) { + cta_dzy_sum[jt] += smem_wgrad[it * COLS + tidx + jt * Ktraits::THREADS_PER_CTA]; + } + } + + compute_t *dgamma_part = static_cast(params.dgamma_part) + bidm * COLS + tidx; + for( int jt = 0; jt < NUM_RES; jt++ ) { + *dgamma_part = cta_dzy_sum[jt]; + dgamma_part += Ktraits::THREADS_PER_CTA; + } + + compute_t *dbeta_part = static_cast(params.dbeta_part) + bidm * COLS + tidx; + for( int jt = 0; jt < NUM_RES; jt++ ) { + *dbeta_part = cta_dz_sum[jt]; + dbeta_part += Ktraits::THREADS_PER_CTA; + } + } +} + +template +__global__ __launch_bounds__(Kernel_traits::THREADS_PER_CTA) +void ln_bwd_finalize_kernel(BwdParams params) +{ + + using compute_t = typename Kernel_traits::compute_t; + using weight_t = typename Kernel_traits::weight_t; + using index_t = typename Kernel_traits::index_t; + using Reducer = typename Kernel_traits::Reducer; + using reduce_t = typename Reducer::Type; + + Sum sum; + enum { NUM_ELT = Kernel_traits::ELTS_PER_LDG }; + enum { THREADS_PER_WARP = Kernel_traits::THREADS_PER_WARP }; + + __shared__ char smem_[Kernel_traits::SMEM_BYTES_PER_CTA]; + + constexpr uint32_t bidm = 0; + + const uint32_t bidn = blockIdx.x; + const uint32_t tidx = threadIdx.x; + const uint32_t warp = tidx / THREADS_PER_WARP; + const uint32_t lane = tidx % THREADS_PER_WARP; + + Reducer reducer(params, bidm, bidn, 0, 0, lane, smem_); + + const uint32_t c = bidn * THREADS_PER_WARP + lane; + const uint32_t c_out = bidn * THREADS_PER_WARP / 2 + lane; + constexpr uint32_t COL_STRIDE = Kernel_traits::CTAS * THREADS_PER_WARP; + for( uint32_t col = c, col_out = c_out; col < Kernel_traits::COLS; col += COL_STRIDE, col_out += COL_STRIDE / 2 ) { + // Each thread sums over NUM_ELT columns. + Vec dbeta_local, dgamma_local; + memset(&dgamma_local, 0, sizeof(dgamma_local)); + memset(&dbeta_local, 0, sizeof(dbeta_local)); + for( uint32_t row = warp; row < params.ctas_per_col; row += Kernel_traits::ROWS_PER_CTA ) { + index_t idx = row * Kernel_traits::COLS + col; + + Vec dbeta_part, dgamma_part; + dbeta_part.load_from(params.dbeta_part, idx); + dgamma_part.load_from(params.dgamma_part, idx); + #pragma unroll + for( int it = 0; it < NUM_ELT; it++ ) { + dgamma_local.data.elt[it] += dgamma_part.data.elt[it]; + dbeta_local.data.elt[it] += dbeta_part.data.elt[it]; + } + } + + void * smem_gamma = smem_; + void * smem_beta = &smem_[Kernel_traits::SMEM_BYTES_TRANSPOSE]; + + const int write_row = warp; + const int write_col = lane ^ write_row; + const int write_idx = write_row * THREADS_PER_WARP + write_col; + + dgamma_local.store_to(smem_gamma, write_idx); + dbeta_local.store_to(smem_beta, write_idx); + + __syncthreads(); + + // It would be probably safe to reuse the first row of smem_beta and smem_gamma + void * smem_gamma_out = &smem_[2 * Kernel_traits::SMEM_BYTES_TRANSPOSE]; + void * smem_beta_out = &smem_[2 * Kernel_traits::SMEM_BYTES_TRANSPOSE + Kernel_traits::SMEM_BYTES_OUTPUT]; + + + // More than one iter iff ROWS_PER_CTA < 32. + for( int w = warp; w < THREADS_PER_WARP; w += Kernel_traits::ROWS_PER_CTA ) { + const int read_row = lane; + const int read_col = w ^ read_row; + const int read_idx = read_row * THREADS_PER_WARP + read_col; + + memset(&dbeta_local, 0, sizeof(dbeta_local)); + memset(&dgamma_local, 0, sizeof(dgamma_local)); + + // Load beta and gamma transposed + if(read_row < Kernel_traits::ROWS_PER_CTA){ + dbeta_local.load_from(smem_beta, read_idx); + dgamma_local.load_from(smem_gamma, read_idx); + } + + // Call reducer on the loaded value(s) and convert. + #pragma unroll + for( int it = 0; it < NUM_ELT; it++ ) { + compute_t b_i = dbeta_local.data.elt[it]; + compute_t g_i = dgamma_local.data.elt[it]; + b_i = reducer.allreduce(b_i, sum); + g_i = reducer.allreduce(g_i, sum); + + dgamma_local.data.elt[it] = g_i; + dbeta_local.data.elt[it] = b_i; + } + + // Leader stores the result at the current column. + if(lane == 0){ + dgamma_local.store_to(smem_gamma_out, w); + dbeta_local.store_to(smem_beta_out, w); + } + + } + + // All writes done. + __syncthreads(); + + // Pack and store: 2-wide stores with half the threads. + if( warp == Kernel_traits::ROWS_PER_CTA - 1 && lane < THREADS_PER_WARP / 2 ) { + + using src_t = typename TypeToVec2::Type; + using dst_t = typename TypeToVec2::Type; + Vec dbeta_vec2, dgamma_vec2; + Vec dbeta_out2, dgamma_out2; + + dgamma_vec2.load_from(smem_gamma_out, lane); + dbeta_vec2.load_from(smem_beta_out, lane); + #pragma unroll + for( int it = 0; it < NUM_ELT; it++ ) { + dgamma_out2.data.elt[it] = Converter::convert(dgamma_vec2.data.elt[it]); + dbeta_out2.data.elt[it] = Converter::convert(dbeta_vec2.data.elt[it]); + } + dgamma_out2.store_to(params.dgamma, col_out); + dbeta_out2.store_to(params.dbeta, col_out); + + } + } +} +} // namespace layer_norm diff --git a/apex/apex/contrib/csrc/layer_norm/ln_bwd_semi_cuda_kernel.cu b/apex/apex/contrib/csrc/layer_norm/ln_bwd_semi_cuda_kernel.cu new file mode 100644 index 00000000..3893d4e0 --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_bwd_semi_cuda_kernel.cu @@ -0,0 +1,240 @@ +#include "ln.h" +#include "ln_utils.cuh" +#include "ln_kernel_traits.h" +#include "ln_bwd_kernels.cuh" + +using namespace layer_norm; + +template< + typename weight_t, + typename input_t, + typename output_t, + typename compute_t, + typename index_t, + int HIDDEN_SIZE, + int CTAS_PER_ROW, + int WARPS_M, + int WARPS_N, + int BYTES_PER_LDG_MAIN, + int BYTES_PER_LDG_FINAL +> +void launch_(LaunchParams &launch_params, const bool configure_params){ + + using Kernel_traits = Kernel_traits; + auto kernel = &ln_bwd_kernel; + + if( configure_params ) { + int ctas_per_sm; + cudaError status_ = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &ctas_per_sm, kernel, Kernel_traits::THREADS_PER_CTA, Kernel_traits::SMEM_BYTES); + launch_params.params.ctas_per_col = launch_params.props->multiProcessorCount * ctas_per_sm / Kernel_traits::CTAS_PER_ROW; + launch_params.barrier_size = 0; + launch_params.workspace_bytes = 0; + if(Kernel_traits::CTAS_PER_ROW > 1) { + launch_params.barrier_size = 2 * launch_params.params.ctas_per_col; + launch_params.workspace_bytes = launch_params.params.ctas_per_col + * Kernel_traits::WARPS_M + * Kernel_traits::CTAS_PER_ROW + * sizeof(typename Kernel_traits::reduce_t) + * 2; + } + return; + } + + if( Kernel_traits::SMEM_BYTES >= 48 * 1024 ) { + CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, Kernel_traits::SMEM_BYTES)); + } + auto stream = launch_params.stream; + auto ctas_per_col = launch_params.params.ctas_per_col; + + if( Kernel_traits::CTAS_PER_ROW == 1 ) { + kernel<<>>(launch_params.params); + } else { + dim3 grid(Kernel_traits::CTAS_PER_ROW * ctas_per_col); + dim3 block(Kernel_traits::THREADS_PER_CTA); + void *params_ = (void *)&launch_params.params; + cudaLaunchCooperativeKernel((void *)kernel, grid, block, (void **)¶ms_, Kernel_traits::SMEM_BYTES, stream); + } + + using Kernel_traits_f = layer_norm::Kernel_traits_finalize; + + auto kernel_f = &layer_norm::ln_bwd_finalize_kernel; + kernel_f<<>>(launch_params.params); +} + +// Create backward launch function and register. Macro signature: +// HIDDEN_SIZE, WTYPE, ITYPE, OTYPE, CTYPE, CTAS_PER_ROW, WARPS_M, WARPS_N, BYTES_PER_LDG, BYTES_PER_LDG_FINAL + +REGISTER_BWD_LAUNCHER( 768, fp32, fp32, fp32, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 768, fp16, fp16, fp16, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 768, fp16, fp32, fp16, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 768, bf16, bf16, bf16, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 768, bf16, fp32, bf16, fp32, 1, 4, 1, 16, 4); + +REGISTER_BWD_LAUNCHER( 1024, fp32, fp32, fp32, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 1024, fp16, fp16, fp16, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 1024, fp16, fp32, fp16, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 1024, bf16, bf16, bf16, fp32, 1, 4, 1, 16, 4); +REGISTER_BWD_LAUNCHER( 1024, bf16, fp32, bf16, fp32, 1, 4, 1, 16, 4); + +REGISTER_BWD_LAUNCHER( 1536, fp32, fp32, fp32, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 1536, fp16, fp16, fp16, fp32, 1, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER( 1536, fp16, fp32, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 1536, bf16, bf16, bf16, fp32, 1, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER( 1536, bf16, fp32, bf16, fp32, 1, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER( 2048, fp32, fp32, fp32, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 2048, fp16, fp16, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 2048, fp16, fp32, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 2048, bf16, bf16, bf16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 2048, bf16, fp32, bf16, fp32, 1, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER( 2304, fp32, fp32, fp32, fp32, 1, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER( 2304, fp16, fp16, fp16, fp32, 1, 1, 4, 4, 4); +REGISTER_BWD_LAUNCHER( 2304, fp16, fp32, fp16, fp32, 1, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER( 2304, bf16, bf16, bf16, fp32, 1, 1, 4, 4, 4); +REGISTER_BWD_LAUNCHER( 2304, bf16, fp32, bf16, fp32, 1, 1, 4, 8, 4); + +REGISTER_BWD_LAUNCHER( 3072, fp32, fp32, fp32, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 3072, fp16, fp16, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 3072, fp16, fp32, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 3072, bf16, bf16, bf16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 3072, bf16, fp32, bf16, fp32, 1, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER( 3840, fp32, fp32, fp32, fp32, 1, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER( 3840, fp16, fp16, fp16, fp32, 1, 1, 4, 4, 4); +REGISTER_BWD_LAUNCHER( 3840, fp16, fp32, fp16, fp32, 1, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER( 3840, bf16, bf16, bf16, fp32, 1, 1, 4, 4, 4); +REGISTER_BWD_LAUNCHER( 3840, bf16, fp32, bf16, fp32, 1, 1, 4, 8, 4); + +REGISTER_BWD_LAUNCHER( 4096, fp32, fp32, fp32, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 4096, fp16, fp16, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 4096, fp16, fp32, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 4096, bf16, bf16, bf16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 4096, bf16, fp32, bf16, fp32, 1, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER( 5120, fp32, fp32, fp32, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 5120, fp16, fp16, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 5120, fp16, fp32, fp16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 5120, bf16, bf16, bf16, fp32, 1, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 5120, bf16, fp32, bf16, fp32, 1, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER( 6144, fp32, fp32, fp32, fp32, 1, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER( 6144, fp16, fp16, fp16, fp32, 1, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER( 6144, fp16, fp32, fp16, fp32, 1, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER( 6144, bf16, bf16, bf16, fp32, 1, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER( 6144, bf16, fp32, bf16, fp32, 1, 1, 8, 16, 4); + +REGISTER_BWD_LAUNCHER( 8192, fp32, fp32, fp32, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 8192, fp16, fp16, fp16, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 8192, fp16, fp32, fp16, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 8192, bf16, bf16, bf16, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER( 8192, bf16, fp32, bf16, fp32, 2, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(10240, fp32, fp32, fp32, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(10240, fp16, fp16, fp16, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(10240, fp16, fp32, fp16, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(10240, bf16, bf16, bf16, fp32, 2, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(10240, bf16, fp32, bf16, fp32, 2, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(12288, fp32, fp32, fp32, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(12288, fp16, fp16, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(12288, fp16, fp32, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(12288, bf16, bf16, bf16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(12288, bf16, fp32, bf16, fp32, 4, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(12800, fp32, fp32, fp32, fp32, 5, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(12800, fp16, fp16, fp16, fp32, 5, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(12800, fp16, fp32, fp16, fp32, 5, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(12800, bf16, bf16, bf16, fp32, 5, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(12800, bf16, fp32, bf16, fp32, 5, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(14336, fp32, fp32, fp32, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(14336, fp16, fp16, fp16, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(14336, fp16, fp32, fp16, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(14336, bf16, bf16, bf16, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(14336, bf16, fp32, bf16, fp32, 4, 1, 4, 8, 4); + +REGISTER_BWD_LAUNCHER(15360, fp32, fp32, fp32, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(15360, fp16, fp16, fp16, fp32, 4, 1, 4, 4, 4); +REGISTER_BWD_LAUNCHER(15360, fp16, fp32, fp16, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(15360, bf16, bf16, bf16, fp32, 4, 1, 4, 4, 4); +REGISTER_BWD_LAUNCHER(15360, bf16, fp32, bf16, fp32, 4, 1, 4, 8, 4); + +REGISTER_BWD_LAUNCHER(16384, fp32, fp32, fp32, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(16384, fp16, fp16, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(16384, fp16, fp32, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(16384, bf16, bf16, bf16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(16384, bf16, fp32, bf16, fp32, 4, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(18432, fp32, fp32, fp32, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(18432, fp16, fp16, fp16, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(18432, fp16, fp32, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(18432, bf16, bf16, bf16, fp32, 4, 1, 4, 8, 4); +REGISTER_BWD_LAUNCHER(18432, bf16, fp32, bf16, fp32, 4, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(20480, fp32, fp32, fp32, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(20480, fp16, fp16, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(20480, fp16, fp32, fp16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(20480, bf16, bf16, bf16, fp32, 4, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(20480, bf16, fp32, bf16, fp32, 4, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(24576, fp32, fp32, fp32, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(24576, fp16, fp16, fp16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(24576, fp16, fp32, fp16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(24576, bf16, bf16, bf16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(24576, bf16, fp32, bf16, fp32, 4, 1, 8, 16, 4); + +REGISTER_BWD_LAUNCHER(25600, fp32, fp32, fp32, fp32, 5, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(25600, fp16, fp16, fp16, fp32, 5, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(25600, fp16, fp32, fp16, fp32, 5, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(25600, bf16, bf16, bf16, fp32, 5, 1, 4, 16, 4); +REGISTER_BWD_LAUNCHER(25600, bf16, fp32, bf16, fp32, 5, 1, 4, 16, 4); + +REGISTER_BWD_LAUNCHER(30720, fp32, fp32, fp32, fp32, 4, 1, 8, 8, 4); +REGISTER_BWD_LAUNCHER(30720, fp16, fp16, fp16, fp32, 4, 1, 8, 4, 4); +REGISTER_BWD_LAUNCHER(30720, fp16, fp32, fp16, fp32, 4, 1, 8, 8, 4); +REGISTER_BWD_LAUNCHER(30720, bf16, bf16, bf16, fp32, 4, 1, 8, 4, 4); +REGISTER_BWD_LAUNCHER(30720, bf16, fp32, bf16, fp32, 4, 1, 8, 8, 4); + +REGISTER_BWD_LAUNCHER(32768, fp32, fp32, fp32, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(32768, fp16, fp16, fp16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(32768, fp16, fp32, fp16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(32768, bf16, bf16, bf16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(32768, bf16, fp32, bf16, fp32, 4, 1, 8, 16, 4); + +REGISTER_BWD_LAUNCHER(40960, fp32, fp32, fp32, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(40960, fp16, fp16, fp16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(40960, fp16, fp32, fp16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(40960, bf16, bf16, bf16, fp32, 4, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(40960, bf16, fp32, bf16, fp32, 4, 1, 8, 16, 4); + +REGISTER_BWD_LAUNCHER(49152, fp32, fp32, fp32, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(49152, fp16, fp16, fp16, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(49152, fp16, fp32, fp16, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(49152, bf16, bf16, bf16, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(49152, bf16, fp32, bf16, fp32, 8, 1, 8, 16, 4); + +REGISTER_BWD_LAUNCHER(65536, fp32, fp32, fp32, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(65536, fp16, fp16, fp16, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(65536, fp16, fp32, fp16, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(65536, bf16, bf16, bf16, fp32, 8, 1, 8, 16, 4); +REGISTER_BWD_LAUNCHER(65536, bf16, fp32, bf16, fp32, 8, 1, 8, 16, 4); + diff --git a/apex/apex/contrib/csrc/layer_norm/ln_fwd_cuda_kernel.cu b/apex/apex/contrib/csrc/layer_norm/ln_fwd_cuda_kernel.cu new file mode 100644 index 00000000..dc4e89cf --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_fwd_cuda_kernel.cu @@ -0,0 +1,228 @@ +#include "ln.h" +#include "ln_utils.cuh" +#include "ln_kernel_traits.h" +#include "ln_fwd_kernels.cuh" + +using namespace layer_norm; + +template< + typename weight_t, + typename input_t, + typename output_t, + typename compute_t, + typename index_t, + int HIDDEN_SIZE, + int CTAS_PER_ROW, + int WARPS_M, + int WARPS_N, + int BYTES_PER_LDG +> +void launch_(LaunchParams &launch_params, const bool configure_params){ + + using Kernel_traits = Kernel_traits; + auto kernel = &ln_fwd_kernel; + + if( configure_params ) { + int ctas_per_sm; + cudaError status_ = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &ctas_per_sm, kernel, Kernel_traits::THREADS_PER_CTA, Kernel_traits::SMEM_BYTES_FWD); + launch_params.params.ctas_per_col = launch_params.props->multiProcessorCount * ctas_per_sm / Kernel_traits::CTAS_PER_ROW; + launch_params.barrier_size = 0; + launch_params.workspace_bytes = 0; + if(Kernel_traits::CTAS_PER_ROW > 1) { + launch_params.barrier_size = 2 * launch_params.params.ctas_per_col; + launch_params.workspace_bytes = launch_params.params.ctas_per_col + * Kernel_traits::WARPS_M + * Kernel_traits::CTAS_PER_ROW + * sizeof(typename Kernel_traits::Stats::stats_t) + * 2; + } + return; + } + + if( Kernel_traits::SMEM_BYTES_FWD >= 48 * 1024 ) { + CHECK_CUDA(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, Kernel_traits::SMEM_BYTES_FWD)); + } + auto stream = launch_params.stream; + auto ctas_per_col = launch_params.params.ctas_per_col; + + if( Kernel_traits::CTAS_PER_ROW == 1 ) { + kernel<<>>(launch_params.params); + } else { + dim3 grid(Kernel_traits::CTAS_PER_ROW * ctas_per_col); + dim3 block(Kernel_traits::THREADS_PER_CTA); + void *params_ = (void *)&launch_params.params; + cudaLaunchCooperativeKernel((void *)kernel, grid, block, (void **)¶ms_, Kernel_traits::SMEM_BYTES_FWD, stream); + } + +} + +// Create forward launch function and register. Macro signature: +// HIDDEN_SIZE, WTYPE, ITYPE, OTYPE, CTYPE, CTAS_PER_ROW, WARPS_M, WARPS_N, BYTES_PER_LDG + +REGISTER_FWD_LAUNCHER( 768, fp32, fp32, fp32, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 768, fp16, fp16, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 768, fp16, fp32, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 768, bf16, bf16, bf16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 768, bf16, fp32, bf16, fp32, 1, 4, 1, 16); + +REGISTER_FWD_LAUNCHER( 1024, fp32, fp32, fp32, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1024, fp16, fp16, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1024, fp16, fp32, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1024, bf16, bf16, bf16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1024, bf16, fp32, bf16, fp32, 1, 4, 1, 16); + +REGISTER_FWD_LAUNCHER( 1536, fp32, fp32, fp32, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1536, fp16, fp16, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1536, fp16, fp32, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1536, bf16, bf16, bf16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 1536, bf16, fp32, bf16, fp32, 1, 4, 1, 16); + +REGISTER_FWD_LAUNCHER( 2048, fp32, fp32, fp32, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2048, fp16, fp16, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2048, fp16, fp32, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2048, bf16, bf16, bf16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2048, bf16, fp32, bf16, fp32, 1, 4, 1, 16); + +REGISTER_FWD_LAUNCHER( 2304, fp32, fp32, fp32, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2304, fp16, fp16, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2304, fp16, fp32, fp16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2304, bf16, bf16, bf16, fp32, 1, 4, 1, 16); +REGISTER_FWD_LAUNCHER( 2304, bf16, fp32, bf16, fp32, 1, 4, 1, 16); + +REGISTER_FWD_LAUNCHER( 3072, fp32, fp32, fp32, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 3072, fp16, fp16, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 3072, fp16, fp32, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 3072, bf16, bf16, bf16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 3072, bf16, fp32, bf16, fp32, 1, 1, 4, 16); + +REGISTER_FWD_LAUNCHER( 3840, fp32, fp32, fp32, fp32, 1, 1, 4, 4); +REGISTER_FWD_LAUNCHER( 3840, fp16, fp16, fp16, fp32, 1, 1, 4, 4); +REGISTER_FWD_LAUNCHER( 3840, fp16, fp32, fp16, fp32, 1, 1, 4, 4); +REGISTER_FWD_LAUNCHER( 3840, bf16, bf16, bf16, fp32, 1, 1, 4, 4); +REGISTER_FWD_LAUNCHER( 3840, bf16, fp32, bf16, fp32, 1, 1, 4, 4); + +REGISTER_FWD_LAUNCHER( 4096, fp32, fp32, fp32, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 4096, fp16, fp16, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 4096, fp16, fp32, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 4096, bf16, bf16, bf16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 4096, bf16, fp32, bf16, fp32, 1, 1, 4, 16); + +REGISTER_FWD_LAUNCHER( 5120, fp32, fp32, fp32, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 5120, fp16, fp16, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 5120, fp16, fp32, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 5120, bf16, bf16, bf16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 5120, bf16, fp32, bf16, fp32, 1, 1, 4, 16); + +REGISTER_FWD_LAUNCHER( 6144, fp32, fp32, fp32, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 6144, fp16, fp16, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 6144, fp16, fp32, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 6144, bf16, bf16, bf16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 6144, bf16, fp32, bf16, fp32, 1, 1, 4, 16); + +REGISTER_FWD_LAUNCHER( 8192, fp32, fp32, fp32, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 8192, fp16, fp16, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 8192, fp16, fp32, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 8192, bf16, bf16, bf16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER( 8192, bf16, fp32, bf16, fp32, 1, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(10240, fp32, fp32, fp32, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(10240, fp16, fp16, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER(10240, fp16, fp32, fp16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER(10240, bf16, bf16, bf16, fp32, 1, 1, 4, 16); +REGISTER_FWD_LAUNCHER(10240, bf16, fp32, bf16, fp32, 1, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(12288, fp32, fp32, fp32, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(12288, fp16, fp16, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(12288, fp16, fp32, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(12288, bf16, bf16, bf16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(12288, bf16, fp32, bf16, fp32, 2, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(12800, fp32, fp32, fp32, fp32, 2, 1, 4, 4); +REGISTER_FWD_LAUNCHER(12800, fp16, fp16, fp16, fp32, 2, 1, 4, 4); +REGISTER_FWD_LAUNCHER(12800, fp16, fp32, fp16, fp32, 2, 1, 4, 4); +REGISTER_FWD_LAUNCHER(12800, bf16, bf16, bf16, fp32, 2, 1, 4, 4); +REGISTER_FWD_LAUNCHER(12800, bf16, fp32, bf16, fp32, 2, 1, 4, 4); + +REGISTER_FWD_LAUNCHER(14336, fp32, fp32, fp32, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(14336, fp16, fp16, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(14336, fp16, fp32, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(14336, bf16, bf16, bf16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(14336, bf16, fp32, bf16, fp32, 2, 1, 4, 8); + +REGISTER_FWD_LAUNCHER(15360, fp32, fp32, fp32, fp32, 2, 1, 4, 8); +REGISTER_FWD_LAUNCHER(15360, fp16, fp16, fp16, fp32, 2, 1, 4, 8); +REGISTER_FWD_LAUNCHER(15360, fp16, fp32, fp16, fp32, 2, 1, 4, 8); +REGISTER_FWD_LAUNCHER(15360, bf16, bf16, bf16, fp32, 2, 1, 4, 8); +REGISTER_FWD_LAUNCHER(15360, bf16, fp32, bf16, fp32, 2, 1, 4, 8); + +REGISTER_FWD_LAUNCHER(16384, fp32, fp32, fp32, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(16384, fp16, fp16, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(16384, fp16, fp32, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(16384, bf16, bf16, bf16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(16384, bf16, fp32, bf16, fp32, 2, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(18432, fp32, fp32, fp32, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(18432, fp16, fp16, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(18432, fp16, fp32, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(18432, bf16, bf16, bf16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(18432, bf16, fp32, bf16, fp32, 4, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(20480, fp32, fp32, fp32, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(20480, fp16, fp16, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(20480, fp16, fp32, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(20480, bf16, bf16, bf16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(20480, bf16, fp32, bf16, fp32, 2, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(24576, fp32, fp32, fp32, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(24576, fp16, fp16, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(24576, fp16, fp32, fp16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(24576, bf16, bf16, bf16, fp32, 2, 1, 4, 16); +REGISTER_FWD_LAUNCHER(24576, bf16, fp32, bf16, fp32, 2, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(25600, fp32, fp32, fp32, fp32, 4, 1, 4, 4); +REGISTER_FWD_LAUNCHER(25600, fp16, fp16, fp16, fp32, 2, 1, 4, 8); +REGISTER_FWD_LAUNCHER(25600, fp16, fp32, fp16, fp32, 4, 1, 4, 4); +REGISTER_FWD_LAUNCHER(25600, bf16, bf16, bf16, fp32, 2, 1, 4, 8); +REGISTER_FWD_LAUNCHER(25600, bf16, fp32, bf16, fp32, 4, 1, 4, 4); + +REGISTER_FWD_LAUNCHER(30720, fp32, fp32, fp32, fp32, 4, 1, 4, 4); +REGISTER_FWD_LAUNCHER(30720, fp16, fp16, fp16, fp32, 4, 1, 4, 4); +REGISTER_FWD_LAUNCHER(30720, fp16, fp32, fp16, fp32, 4, 1, 4, 4); +REGISTER_FWD_LAUNCHER(30720, bf16, bf16, bf16, fp32, 4, 1, 4, 4); +REGISTER_FWD_LAUNCHER(30720, bf16, fp32, bf16, fp32, 4, 1, 4, 4); + +REGISTER_FWD_LAUNCHER(32768, fp32, fp32, fp32, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(32768, fp16, fp16, fp16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(32768, fp16, fp32, fp16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(32768, bf16, bf16, bf16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(32768, bf16, fp32, bf16, fp32, 4, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(40960, fp32, fp32, fp32, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(40960, fp16, fp16, fp16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(40960, fp16, fp32, fp16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(40960, bf16, bf16, bf16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(40960, bf16, fp32, bf16, fp32, 4, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(49152, fp32, fp32, fp32, fp32, 8, 1, 4, 16); +REGISTER_FWD_LAUNCHER(49152, fp16, fp16, fp16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(49152, fp16, fp32, fp16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(49152, bf16, bf16, bf16, fp32, 4, 1, 4, 16); +REGISTER_FWD_LAUNCHER(49152, bf16, fp32, bf16, fp32, 4, 1, 4, 16); + +REGISTER_FWD_LAUNCHER(65536, fp32, fp32, fp32, fp32, 8, 1, 4, 16); +REGISTER_FWD_LAUNCHER(65536, fp16, fp16, fp16, fp32, 8, 1, 4, 16); +REGISTER_FWD_LAUNCHER(65536, fp16, fp32, fp16, fp32, 8, 1, 4, 16); +REGISTER_FWD_LAUNCHER(65536, bf16, bf16, bf16, fp32, 8, 1, 4, 16); +REGISTER_FWD_LAUNCHER(65536, bf16, fp32, bf16, fp32, 8, 1, 4, 16); + diff --git a/apex/apex/contrib/csrc/layer_norm/ln_fwd_kernels.cuh b/apex/apex/contrib/csrc/layer_norm/ln_fwd_kernels.cuh new file mode 100644 index 00000000..64e72974 --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_fwd_kernels.cuh @@ -0,0 +1,110 @@ +#pragma once + +#include "ln.h" + +namespace layer_norm { + +template +__global__ __launch_bounds__(Ktraits::THREADS_PER_CTA) +void ln_fwd_kernel(FwdParams params) { + + enum { ROWS_PER_CTA = Ktraits::ROWS_PER_CTA }; + enum { WARPS_N = Ktraits::WARPS_N }; + enum { WARPS_M = Ktraits::WARPS_M }; + enum { THREADS_PER_ROW = Ktraits::THREADS_PER_ROW }; + enum { VEC_COLS_PER_LDG = Ktraits::VEC_COLS_PER_LDG }; + enum { BYTES_PER_ROW = Ktraits::BYTES_PER_ROW }; + enum { LDGS = Ktraits::LDGS }; + enum { NUM_ELTS = Ktraits::NUM_ELTS }; + enum { CTAS_PER_ROW = Ktraits::CTAS_PER_ROW }; + + using output_t = typename Ktraits::output_t; + using index_t = typename Ktraits::index_t; + using compute_t = typename Ktraits::compute_t; + using Ivec = typename Ktraits::Ivec; + using Ovec = typename Ktraits::Ovec; + using Wvec = typename Ktraits::Wvec; + using Cvec = typename Ktraits::Cvec; + + using Stats = typename Ktraits::Stats; + using stats_t = typename Stats::stats_t; + + extern __shared__ char smem_[]; + + const index_t tidx = threadIdx.x; + const index_t bidn = blockIdx.x % CTAS_PER_ROW; + const index_t bidm = blockIdx.x / CTAS_PER_ROW; + const index_t lane = tidx % THREADS_PER_WARP; + const index_t warp = tidx / THREADS_PER_WARP; + const index_t warp_m = warp / WARPS_N; + const index_t warp_n = warp % WARPS_N; + + const index_t r = bidm * ROWS_PER_CTA + warp_m; + const index_t c = bidn * THREADS_PER_ROW + warp_n * THREADS_PER_WARP + lane; + + Stats stats(params, bidm, bidn, warp_m, warp_n, lane, smem_); + + compute_t *mu_ptr = static_cast(params.mu); + compute_t *rs_ptr = static_cast(params.rs); + + Wvec gamma[LDGS]; + Wvec beta[LDGS]; + index_t idx = c; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + gamma[it].load_from(params.gamma, idx); + beta[it].load_from(params.beta, idx); + idx += VEC_COLS_PER_LDG; + } + + constexpr compute_t rn = 1.f / compute_t(Ktraits::COLS); + + for( int row = r; row < params.rows; row += params.ctas_per_col * ROWS_PER_CTA ) { + Ivec x[LDGS]; + index_t idx = row * Ktraits::VEC_COLS + c; + compute_t xf[LDGS * NUM_ELTS]; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + x[it].load_from(params.x, idx); + #pragma unroll + for( int jt = 0; jt < NUM_ELTS; jt++ ) { + compute_t x_ij = compute_t(x[it].data.elt[jt]); + xf[it * NUM_ELTS + jt] = x_ij; + } + idx += VEC_COLS_PER_LDG; + } + + stats_t s = stats.compute(xf, rn); + + compute_t mu = layer_norm::Get<0>::of(s); + compute_t m2 = layer_norm::Get<1>::of(s); + + if( bidn == 0 && warp_n == 0 && lane == 0 ) { + mu_ptr[row] = mu; + } + + compute_t rs = rsqrtf(rn * m2 + params.epsilon); + + if( bidn == 0 && warp_n == 0 && lane == 0 ) { + rs_ptr[row] = rs; + } + + Ovec z[LDGS]; + idx = row * Ktraits::VEC_COLS + c; + #pragma unroll + for( int it = 0; it < LDGS; it++ ) { + #pragma unroll + for( int jt = 0; jt < NUM_ELTS; jt++ ) { + output_t y_ij = output_t(rs * (xf[it * NUM_ELTS + jt] - mu)); + output_t g_ij = gamma[it].data.elt[jt]; + output_t b_ij = beta[it].data.elt[jt]; + z[it].data.elt[jt] = (g_ij * y_ij + b_ij); + } + z[it].store_to(params.z, idx); + idx += VEC_COLS_PER_LDG; + } + + } +} + +} // namespace layer_norm diff --git a/apex/apex/contrib/csrc/layer_norm/ln_kernel_traits.h b/apex/apex/contrib/csrc/layer_norm/ln_kernel_traits.h new file mode 100644 index 00000000..ed745c5e --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_kernel_traits.h @@ -0,0 +1,159 @@ +#pragma once + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace layer_norm { +template< + uint32_t HIDDEN_SIZE_, + typename weight_t_, + typename input_t_, + typename output_t_, + typename compute_t_, + typename index_t_, + uint32_t THREADS_PER_CTA_ +> +struct Kernel_traits_base { + + using weight_t = weight_t_; + using input_t = input_t_; + using output_t = output_t_; + using compute_t = compute_t_; + using index_t = index_t_; + + enum { HIDDEN_SIZE = HIDDEN_SIZE_ }; + enum { THREADS_PER_CTA = THREADS_PER_CTA_ }; + enum { THREADS_PER_WARP = 32 }; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template< + uint32_t HIDDEN_SIZE_, + typename weight_t_, + typename input_t_, + typename output_t_, + typename compute_t_, + typename index_t_, + uint32_t THREADS_PER_CTA_, + uint32_t BYTES_PER_LDG_, + typename Base = Kernel_traits_base +> +struct Kernel_traits_finalize : public Base { + enum { ROWS_PER_CTA = Base::THREADS_PER_CTA / Base::THREADS_PER_WARP }; + static_assert((int) ROWS_PER_CTA <= (int) Base::THREADS_PER_WARP); + // Bytes per global load from the input. + enum { BYTES_PER_LDG = BYTES_PER_LDG_ }; + // Number of elements fetched by a global load. + enum { ELTS_PER_LDG = BYTES_PER_LDG / sizeof(compute_t_) }; + // Bytes per global store of the weights. + enum { BYTES_PER_STG = ELTS_PER_LDG * sizeof(weight_t_) }; + static_assert(sizeof(BYTES_PER_LDG) == 4, "Conflict-free smem transpose only implemented for 4B compute type!"); + static_assert(Base::THREADS_PER_CTA == ROWS_PER_CTA * Base::THREADS_PER_WARP, "We assume one warp per row!"); + // The total number of BYTES_PER_LDG-wide words in a hidden vector. + enum { COLS = HIDDEN_SIZE_ * sizeof(compute_t_) / BYTES_PER_LDG }; + static_assert(COLS * BYTES_PER_LDG == HIDDEN_SIZE_ * sizeof(compute_t_)); + + // Shared memory size to transpose the CTA result. + enum { SMEM_BYTES_TRANSPOSE = Base::THREADS_PER_CTA * BYTES_PER_LDG }; + // Shared memory size to coalsece the CTA result. + enum { SMEM_BYTES_OUTPUT = Base::THREADS_PER_WARP * BYTES_PER_LDG }; + // Shared memory requirement per CTA. + enum { SMEM_BYTES_PER_CTA = 2 * SMEM_BYTES_TRANSPOSE + 2 * SMEM_BYTES_OUTPUT }; + + // The type of the reducer. + using Reducer = layer_norm::Reducer; + + // Condition for the whole CTA to participate in syncthreads. + static_assert(COLS % Base::THREADS_PER_WARP == 0); + enum { CTAS = COLS / Base::THREADS_PER_WARP }; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + + +template< + typename weight_t_, + typename input_t_, + typename output_t_, + typename compute_t_, + typename index_t_, + uint32_t HIDDEN_SIZE_, + uint32_t CTAS_PER_ROW_, + uint32_t WARPS_M_, + uint32_t WARPS_N_, + uint32_t BYTES_PER_LDG_ = 16, + typename Base = Kernel_traits_base< + HIDDEN_SIZE_, + weight_t_, + input_t_, + output_t_, + compute_t_, + index_t_, + WARPS_M_*WARPS_N_*THREADS_PER_WARP + > +> +struct Kernel_traits : public Base { + + using input_t = typename Base::input_t; + using weight_t = typename Base::weight_t; + using compute_t = typename Base::compute_t; + using output_t = typename Base::output_t; + using index_t = typename Base::index_t; + + enum { CTAS_PER_ROW = CTAS_PER_ROW_ }; + enum { WARPS_M = WARPS_M_ }; + enum { WARPS_N = WARPS_N_ }; + enum { COLS = HIDDEN_SIZE_ }; + enum { HIDDEN_SIZE = HIDDEN_SIZE_ }; + enum { BYTES_PER_LDG = BYTES_PER_LDG_ }; + enum { NUM_ELTS = BYTES_PER_LDG / sizeof(input_t) }; + + enum { THREADS_PER_ROW = WARPS_N * THREADS_PER_WARP }; + enum { THREADS_PER_CTA = WARPS_M * THREADS_PER_ROW }; + enum { ROWS_PER_CTA = WARPS_M }; + + enum { BYTES_PER_ROW = COLS * sizeof(input_t) }; + enum { BYTES_PER_ROW_PER_CTA = THREADS_PER_ROW * BYTES_PER_LDG }; + // Multi-row per CTA not supported for multi-CTA => no smem for WGRAD needed + enum { SMEM_BYTES_WGRAD = CTAS_PER_ROW > 1 ? 0 : ROWS_PER_CTA * COLS * sizeof(compute_t) }; + static_assert(WARPS_M == 1 || CTAS_PER_ROW == 1); + + using reduce_t = typename layer_norm::TypeToVec2::Type; + using Reducer = layer_norm::Reducer; + + enum { SMEM_BYTES_DGRAD = Reducer::SMEM_BYTES }; + enum { SMEM_BYTES = SMEM_BYTES_DGRAD + SMEM_BYTES_WGRAD }; + + using Ivec = layer_norm::Vec; + using Ovec = layer_norm::Vec; + using Wvec = layer_norm::Vec; + using Cvec = layer_norm::Vec; + enum { ELTS_PER_LDG = BYTES_PER_LDG / sizeof(input_t) }; + + // Assume that each thread can handle the same number of elements in the output and weights as in the input. + static_assert(sizeof(input_t) >= sizeof(output_t)); + static_assert(sizeof(input_t) >= sizeof(weight_t)); + // The number of columns fetched per load from input: one per thread. + enum { VEC_COLS_PER_LDG = CTAS_PER_ROW * THREADS_PER_ROW }; + // The total number of vectorized loads/stores per hidden vector. + enum { VEC_COLS = COLS / ELTS_PER_LDG }; + // The number of loads per thread for the input. + enum { LDGS = VEC_COLS / VEC_COLS_PER_LDG }; + static_assert(LDGS * VEC_COLS_PER_LDG == VEC_COLS); + //static_assert(LDGS * BYTES_PER_ROW_PER_CTA * CTAS_PER_ROW == BYTES_PER_ROW, ""); + + using Stats = layer_norm::Stats; + enum { SMEM_BYTES_FWD = Stats::SMEM_BYTES }; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace layer_norm diff --git a/apex/apex/contrib/csrc/layer_norm/ln_utils.cuh b/apex/apex/contrib/csrc/layer_norm/ln_utils.cuh new file mode 100644 index 00000000..e18d36de --- /dev/null +++ b/apex/apex/contrib/csrc/layer_norm/ln_utils.cuh @@ -0,0 +1,733 @@ +#pragma once + +#include + +#include +#include + +#include "ln.h" + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +constexpr uint32_t THREADS_PER_WARP = 32; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline void check_cuda_(cudaError_t status, const char *file, int line) { + if( status != cudaSuccess ) { + fprintf(stderr, "CUDA Error: %s %s %d\n", cudaGetErrorString(status), file, line); + exit(status); + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define CHECK_CUDA(ans) \ + { check_cuda_((ans), __FILE__, __LINE__); } + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define DIVUP(x, y) (((x) + ((y)-1)) / (y)) + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define REGISTER_FWD_LAUNCHER(HIDDEN_SIZE, WTYPE, ITYPE, OTYPE, CTYPE, CTAS_PER_ROW, WARPS_M, WARPS_N, BYTES_PER_LDG) \ + void ln_fwd_##HIDDEN_SIZE##_##WTYPE##_##ITYPE##_##OTYPE##_##CTYPE(LaunchParams &launch_params, \ + const bool configure_params) { \ + launch_( \ + launch_params, configure_params); \ + } \ + static FwdRegistrar reg_##HIDDEN_SIZE##_##WTYPE##_##ITYPE##_##OTYPE##_##CTYPE( \ + ln_fwd_##HIDDEN_SIZE##_##WTYPE##_##ITYPE##_##OTYPE##_##CTYPE) + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define REGISTER_BWD_LAUNCHER( \ + HIDDEN_SIZE, WTYPE, ITYPE, OTYPE, CTYPE, CTAS_PER_ROW, WARPS_M, WARPS_N, BYTES_PER_LDG, BYTES_PER_LDG_FINALIZE) \ + void ln_bwd_##HIDDEN_SIZE##_##WTYPE##_##ITYPE##_##OTYPE##_##CTYPE(LaunchParams &launch_params, \ + const bool configure_params) { \ + launch_(launch_params, configure_params); \ + } \ + static BwdRegistrar reg_##HIDDEN_SIZE##_##WTYPE##_##ITYPE##_##OTYPE##_##CTYPE( \ + ln_bwd_##HIDDEN_SIZE##_##WTYPE##_##ITYPE##_##OTYPE##_##CTYPE) + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ float2 operator+(const float2 & a, const float2 & b){ + return {a.x + b.x, a.y + b.y}; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +inline __device__ void operator+=(float2 & a, const float2 & b){ + a.x += b.x; + a.y += b.y; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Sum { + inline __device__ Sum(){} + inline __device__ T operator()(const T &a, const T &b){ + return a + b; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ T warp_shuffle_xor(const T & x, uint32_t idx){ + return __shfl_xor_sync(uint32_t(-1), x, idx); +} + +template<> +inline __device__ float2 warp_shuffle_xor(const float2 & x, uint32_t idx){ + return { warp_shuffle_xor(x.x, idx), warp_shuffle_xor(x.y, idx) }; +} + +template +inline __device__ T warp_shuffle_down(const T & x, uint32_t idx){ + return __shfl_down_sync(uint32_t(-1), x, idx); +} + +template<> +inline __device__ float2 warp_shuffle_down(const float2 & x, uint32_t idx){ + return { warp_shuffle_down(x.x, idx), warp_shuffle_down(x.y, idx) }; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace layer_norm { + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct uint16 { + uint4 u; + uint4 v; + uint4 s; + uint4 t; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct uint8 { + uint4 u; + uint4 v; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct BytesToType {}; + +template<> +struct BytesToType<64> { + using Type = uint16; + static_assert(sizeof(Type) == 64); +}; + +template<> +struct BytesToType<32> { + using Type = uint8; + static_assert(sizeof(Type) == 32); +}; + +template<> +struct BytesToType<16> { + using Type = uint4; + static_assert(sizeof(Type) == 16); +}; + +template<> +struct BytesToType<8> { + using Type = uint64_t; + static_assert(sizeof(Type) == 8); +}; + +template<> +struct BytesToType<4> { + using Type = uint32_t; + static_assert(sizeof(Type) == 4); +}; + +template<> +struct BytesToType<2> { + using Type = uint16_t; + static_assert(sizeof(Type) == 2); +}; + +template<> +struct BytesToType<1> { + using Type = uint8_t; + static_assert(sizeof(Type) == 1); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct TypeToVec2 {}; + +template<> +struct TypeToVec2 { + using Type = float2; +}; + +template<> +struct TypeToVec2 { + using Type = half2; +}; + +template<> +struct TypeToVec2 { + using Type = nv_bfloat162; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Get { + template + static inline __device__ R of(const T &vec); +}; + +template<> +template +inline __device__ R Get<0>::of(const T &vec) { + return vec.x; +} + +template<> +template +inline __device__ R Get<1>::of(const T &vec) { + return vec.y; +} + +template<> +template +inline __device__ R Get<2>::of(const T &vec) { + return vec.z; +} + +template<> +template +inline __device__ R Get<3>::of(const T &vec) { + return vec.w; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Converter{ + static inline __device__ Dst convert(const Src &from) { + return Dst(from); + } +}; + +template<> +struct Converter{ + static inline __device__ half2 convert(const float2 &x) { + return __float22half2_rn(x); + } +}; + +template<> +struct Converter{ + static inline __device__ nv_bfloat162 convert(const float2 &x) { +#if __CUDA_ARCH__ >= 800 + return __float22bfloat162_rn(x); +#else + union { + nv_bfloat162 raw; + nv_bfloat16 x; + nv_bfloat16 y; + } tmp; + tmp.x = __float2bfloat16_rn(x.x); + tmp.y = __float2bfloat16_rn(x.y); + return tmp.raw; +#endif + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Zeros{ + static inline __device__ T get() { + return T(0.f); + } +}; + +template<> +struct Zeros{ + static inline __device__ float2 get() { + return make_float2(0.f, 0.f); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Vec { + + enum { BYTES = NUM_ELT * sizeof(Elt_type) }; + + using Vec_type = typename BytesToType::Type; + + using Alias_type = union { + Vec_type vec; + Elt_type elt[NUM_ELT]; + }; + + Alias_type data; + + template + inline __device__ void to(Vec &other) { + #pragma unroll + for( int it = 0; it < NUM_ELT; it++ ) { + other.data.elt[it] = S(this->data.elt[it]); + } + } + + template + inline __device__ void assign(const Op &op) { + #pragma unroll + for( int it = 0; it < NUM_ELT; it++ ) { + this->data.elt[it] = op(it); + } + } + + inline __device__ void load_from(const void *base_ptr, const size_t idx) { + this->data.vec = static_cast(base_ptr)[idx]; + } + + inline __device__ void store_to(void *base_ptr, const size_t idx) { + static_cast(base_ptr)[idx] = this->data.vec; + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct InterCTASync { + + template + inline __device__ InterCTASync(Params & params, uint32_t bidm, uint32_t bidn) + : phase_counter_(0) + , b0_(params.barrier + bidm) // The barrier for this group of CTAs. + , b1_(params.barrier + bidm + params.ctas_per_col) // The barrier for this group of CTAs. + { + // BARRIERS ARE ASSUMED TO BE INITIALIZED TO 0! + } + + inline __device__ void spin_wait_(int *barrier, int step, int expected) { + asm volatile("red.release.gpu.global.add.s32 [%0], %1;" ::"l"(barrier), "r"(step)); + for( int found = -1; found != expected; ) { + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];" : "=r"(found) : "l"(barrier)); + } + } + + inline __device__ void sync(){ + // ALL THREADS MUST ENTER! + + // We switch barrier every iteration. + int *barrier = phase_counter_ & 0x1 ? b1_ : b0_; + // We decrement every other iteration. + bool dec = phase_counter_ & 0x2; + int step = dec ? -1 : 1; + int expected = dec ? 0 : CTAS_PER_ROW; + // There are only 4 phases: up/down for b0/b1. + phase_counter_ = (phase_counter_ + 1) & 0x3; + + if( threadIdx.x == 0 ) { + spin_wait_(barrier, step, expected); + } + // CTA waits for thread 0 + __syncthreads(); + } + + int phase_counter_; + int * b0_; + int * b1_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Reducer : public Reducer { + + using InterCTASync = InterCTASync; + using Base = Reducer; + using Type = typename Base::Type; + + enum { SMEM_BYTES = Base::SMEM_BYTES }; + + enum { WS_BARRIER_BYTES = 2 * sizeof(int) }; + enum { WS_DATA_BYTES = WARPS_M * CTAS_PER_ROW * sizeof(T) }; + + // size of the barriers + temporary result per CTA (multiply with CTAS_PER_ROW to get total) + enum { WORKSPACE_BYTES_PER_GROUP = Base::WORKSPACE_BYTES_PER_GROUP + WS_BARRIER_BYTES + WS_DATA_BYTES }; + + template + inline __device__ Reducer(Params & params, uint32_t bidm, uint32_t bidn, uint32_t warp_m, uint32_t warp_n, uint32_t lane, void * smem) + : Base(params, bidm, bidn, warp_m, warp_n, lane, smem) + , inter_cta_(params, bidm, bidn) + , bidn_(bidn) // CTA id within the group. + , w0_(static_cast(params.workspace) + (bidm * WARPS_M + warp_m) * CTAS_PER_ROW) + , w1_(w0_ + params.ctas_per_col * WARPS_M * CTAS_PER_ROW) + { + } + + template + inline __device__ T allreduce(T data, Op &op) { + data = Base::reduce(data, op); + // We switch workspace every iteration. + T *workspace = inter_cta_.phase_counter_ & 0x1 ? w1_ : w0_; + + // Warp leaders 0 hold the CTA-local results. + if( this->warp_n_ == 0 && this->lane_ == 0 ) { + workspace[bidn_] = data; + } + inter_cta_.sync(); + static_assert(CTAS_PER_ROW <= 32); + T total = Zeros::get(); + if(this->lane_ < CTAS_PER_ROW){ + total = workspace[this->lane_]; + } + total = Reducer::allreduce_(total, op); + + return total; + } + + InterCTASync inter_cta_; + + T *w0_; + T *w1_; + int bidn_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Reducer { + + using Type = T; + enum { SMEM_BYTES = 0 }; + enum { WORKSPACE_BYTES_PER_GROUP = 0 }; + + enum { THREADS_PER_WARP = 32 }; + + template + inline __device__ Reducer(Params & params, uint32_t bidm, uint32_t bidn, uint32_t warp_m, uint32_t warp_n, uint32_t lane, void * smem) + : warp_n_(warp_n) + , lane_(lane) + { + } + + template + static inline __device__ T allreduce_(T data, Op &op) { + #pragma unroll + for( int it = 1; it < THREADS_PER_WARP; it *= 2 ) { + data = op(data, warp_shuffle_xor(data, it)); + } + return data; + } + + template + inline __device__ T allreduce(T data, Op &op) { + return allreduce_(data, op); + } + + template + inline __device__ T reduce(T data, Op &op){ + // only lane 0 holds the result! + #pragma unroll + for( int it = THREADS_PER_WARP / 2; it > 0; it /= 2 ) { + data = op(data, warp_shuffle_down(data, it)); + } + return data; + } + int warp_n_; + int lane_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Reducer : public Reducer { + + using Base = Reducer; + + using Type = T; + + enum { SMEM_BYTES = Base::SMEM_BYTES + WARPS_M * WARPS_N * sizeof(T) * 2 }; + enum { WORKSPACE_BYTES_PER_GROUP = 0 }; + + enum { THREADS_PER_WARP = 32 }; + + template + inline __device__ Reducer(Params & params, uint32_t bidm, uint32_t bidn, uint32_t warp_m, uint32_t warp_n, uint32_t lane, void * smem) + : Base(params, bidm, bidn, warp_m, warp_n, lane, smem) + , use0_(true) + { + smem0_ = &static_cast(smem)[warp_m * WARPS_N]; + smem1_ = smem0_ + WARPS_M * WARPS_N; + } + + template + inline __device__ T allreduce(T data, Op & op) { + T * smem = use0_ ? smem0_ : smem1_; + use0_ = !use0_; + data = Base::reduce(data, op); + if( this->lane_ == 0 ) { + smem[this->warp_n_] = data; + } + __syncthreads(); + T out = Zeros::get(); + #pragma unroll + for( int it = 0; it < WARPS_N; it++ ) { + out = op(out, smem[it]); + } + return out; + } + + template + inline __device__ T reduce(T data, Op &op) { + T * smem = use0_ ? smem0_ : smem1_; + use0_ = !use0_; + // only intra-CTA group leader holds the result! + data = Base::reduce(data, op); + if( this->lane_ == 0 ) { + smem[this->warp_n_] = data; + } + __syncthreads(); + T out = Zeros::get(); + if( this->warp_n_ == 0 && this->lane_ == 0 ) { + #pragma unroll + for( int it = 0; it < WARPS_N; it++ ) { + out = op(out, smem[it]); + } + } + return out; + } + + T * smem0_; + T * smem1_; + bool use0_; + +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +inline __device__ void warp_chan_upd_dynamic(T &m_a, T &m2_a, T &n_a, int num_active){ + //Assume at least leftmost is valid and init: step = next_pow2(num_active) / 2 (might get NaN otherwise) + int highest_bit_set = (8 * sizeof(num_active)) - __clz(num_active - 1); + + #pragma unroll + for( int step = (1 << (highest_bit_set - 1)); step > 0; step /= 2 ) { + // Exchange + T n_b = warp_shuffle_down(n_a, step); + T m_b = warp_shuffle_down(m_a, step); + T m2_b = warp_shuffle_down(m2_a, step); + + // Update + const T n_ab = n_a + n_b; // We can handle one of them being 0, not both. + const T rn_ab = 1.f / n_ab; // Might have different n per thread, otherwise this would simplify :( + const T delta = m_a - m_b; + const float m2_ab = m2_a + m2_b + delta * delta * n_a * n_b * rn_ab; + const float m_ab = (n_a * m_a + n_b * m_b) * rn_ab; + + n_a = n_ab; + m_a = m_ab; + m2_a = m2_ab; + } + // Intra-warp broadcast (only lane 0 has valid stats). + m_a = __shfl_sync(uint32_t(-1), m_a, 0); + m2_a = __shfl_sync(uint32_t(-1), m2_a, 0); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Stats { + // This could be done generically with the Reducer. But then we would have to exchange 3 instead of 2 fields. + + using InterCTASync = InterCTASync; + using BlockStats = Stats; + using stats_t = typename BlockStats::stats_t; + + enum { SMEM_BYTES = BlockStats::SMEM_BYTES }; + + template + inline __device__ Stats(Params & params, uint32_t bidm, uint32_t bidn, uint32_t warp_m, uint32_t warp_n, uint32_t lane, void * smem) + : inter_cta_(params, bidm, bidn) + , block_stats_(params, bidm, bidn, warp_m, warp_n, lane, smem) + , bidn_(bidn) // CTA id within the group. + , w0_(static_cast(params.workspace) + (bidm * WARPS_M + warp_m) * CTAS_PER_ROW) + , w1_(w0_ + params.ctas_per_col * WARPS_M * CTAS_PER_ROW) + , warp_n_(warp_n) + , lane_(lane) + { + } + + template + inline __device__ stats_t compute(const T (&elts)[N], const T rn) { + constexpr T ELTS_PER_ROW_PER_CTA = N * WARPS_N * THREADS_PER_WARP; + // TODO rn is not really needed here.. + constexpr T block_rn = 1.f / T(ELTS_PER_ROW_PER_CTA); + stats_t block_stats = block_stats_.compute(elts, block_rn); + + stats_t *workspace = inter_cta_.phase_counter_ & 0x1 ? w1_ : w0_; + + if( warp_n_ == 0 && lane_ == 0 ) { + workspace[bidn_] = block_stats; + } + + // Wait for all CTAS_PER_ROW CTAS in the group to have written their result. + inter_cta_.sync(); + + T n = Zeros::get(); + T m = Zeros::get(); + T m2 = Zeros::get(); + + // Assume CTA group size in N less than 32, such that we can finalize with a single warp. + static_assert(CTAS_PER_ROW <= 32); + + // Every warp does the final reduction locally. + if( lane_ < CTAS_PER_ROW ) { + stats_t result = workspace[lane_]; + n = ELTS_PER_ROW_PER_CTA; + m = layer_norm::Get<0>::of(result); + m2 = layer_norm::Get<1>::of(result); + } + + warp_chan_upd_dynamic(m, m2, n, CTAS_PER_ROW); + + return { m, m2 }; + } + + InterCTASync inter_cta_; + BlockStats block_stats_; + + stats_t *w0_; + stats_t *w1_; + int bidn_; + int warp_n_; + int lane_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Stats { + + using WarpStats = Stats; + using stats_t = typename WarpStats::stats_t; + + enum { SMEM_BYTES = WARPS_M * WARPS_N * sizeof(stats_t) * 2 }; + + template + inline __device__ Stats(Params & params, uint32_t bidm, uint32_t bidn, uint32_t warp_m, uint32_t warp_n, uint32_t lane, void * smem) + : warp_stats_(params, bidm, bidn, warp_m, warp_n, lane, smem) + , use0_(true) + { + smem0_ = static_cast(smem) + warp_m * WARPS_N; + smem1_ = smem0_ + WARPS_M * WARPS_N; + } + + template + inline __device__ stats_t compute(const T (&elts)[N], const T rn) { + stats_t * smem = use0_ ? smem0_ : smem1_; + use0_ = !use0_; + // Compute warp local for all WARPS_N + constexpr T warp_rn = 1.f / T(N * THREADS_PER_WARP); + stats_t warp_stats = warp_stats_.compute(elts, warp_rn); + + //Each warp warp leader stores its stats + const auto warp_n = warp_stats_.reducer_.warp_n_; + const auto lane = warp_stats_.reducer_.lane_; + if( lane == 0 ) { + smem[warp_n] = warp_stats; + } + __syncthreads(); + + T n = Zeros::get(); + T m = Zeros::get(); + T m2 = Zeros::get(); + + // Assume that there are less than 32 warps, such that we can finalize with a single warp + static_assert(WARPS_N <= 32); + if(lane < WARPS_N){ + stats_t result = smem[lane]; + n = N * THREADS_PER_WARP; + m = layer_norm::Get<0>::of(result); + m2 = layer_norm::Get<1>::of(result); + } + + warp_chan_upd_dynamic(m, m2, n, WARPS_N); + + return { m, m2 }; + } + WarpStats warp_stats_; + stats_t * smem0_; + stats_t * smem1_; + bool use0_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct Stats { + + using stats_t = typename TypeToVec2::Type; + // The simple Warp reducer. + using Reducer = Reducer; + + enum { SMEM_BYTES = 0 }; + + template + inline __device__ Stats(Params & params, uint32_t bidm, uint32_t bidn, uint32_t warp_m, uint32_t warp_n, uint32_t lane, void * smem) + : reducer_(params, bidm, bidn, warp_m, warp_n, lane, smem) + { + } + + template + inline __device__ stats_t compute(const T (&elts)[N], const T rn) { + + auto sum = Sum(); + + T m = Zeros::get(); + #pragma unroll + for( int it = 0; it < N; it++ ) { + m += elts[it]; + } + m = reducer_.allreduce(m, sum) * rn; + + T m2 = Zeros::get(); + #pragma unroll + for( int it = 0; it < N; it++ ) { + T diff = (elts[it] - m); + m2 += diff * diff; + } + m2 = reducer_.allreduce(m2, sum); + + return {m, m2}; + } + + Reducer reducer_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace layer_norm diff --git a/apex/apex/contrib/csrc/multihead_attn/additive_masked_softmax_dropout_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/additive_masked_softmax_dropout_cuda.cu new file mode 100644 index 00000000..373a217f --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/additive_masked_softmax_dropout_cuda.cu @@ -0,0 +1,113 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "softmax.cuh" + +// symbol to be automatically resolved by PyTorch libs + +namespace multihead_attn { +namespace fused_softmax { +namespace additive_mask_softmax_dropout { + +std::vector fwd_cuda(bool is_training, int heads, + torch::Tensor const &input, + const half *pad_mask, float dropout_prob) { + const int attn_batches = input.size(0); + const int sequences = attn_batches / heads; + const int q_seq_len = input.size(1); + const int k_seq_len = q_seq_len; + // const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = input.options().requires_grad(false); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *input_ptr = static_cast(input.data_ptr()); + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + // Padded Softmax + [[maybe_unused]] bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), k_seq_len, k_seq_len, + attn_batches * q_seq_len); + } else { + softmax_success = dispatch_additive_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), pad_mask, k_seq_len, + k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + + if (is_training) { + // use at:: function so that C++ version generates the same random mask as + // python version + auto dropout_tuple = + at::_fused_dropout(softmax_results, 1.0f - dropout_prob); + dropout_results = std::get<0>(dropout_tuple); + dropout_mask = std::get<1>(dropout_tuple); + } + + // Matmul2 + + return {dropout_results, dropout_mask, softmax_results}; +} + +torch::Tensor bwd_cuda(int heads, torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + torch::Tensor const &dropout_mask, float dropout_prob) { + const int attn_batches = output_grads.size(0); + const int q_seq_len = output_grads.size(1); + const int k_seq_len = q_seq_len; + // const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + // torch::Tensor input_grads = torch::empty_like(output_grads); + + // Apply Dropout Mask and Scale by Dropout Probability + // Softmax Grad + dispatch_masked_scale_softmax_backward_stream( + static_cast(output_grads.data_ptr()), + static_cast(output_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), + 1.0 / (1.0 - dropout_prob), k_seq_len, k_seq_len, + attn_batches * q_seq_len, stream); + // backward pass is completely in-place + return output_grads; +} +} // namespace additive_mask_softmax_dropout +} // namespace fused_softmax +} // namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/dropout.cuh b/apex/apex/contrib/csrc/multihead_attn/dropout.cuh new file mode 100644 index 00000000..6f3922a6 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/dropout.cuh @@ -0,0 +1,272 @@ +#pragma once +#include + +#ifdef OLD_GENERATOR_PATH +#include +#else +#include +#endif + +#include +#include + +namespace { +constexpr int UNROLL = 4; +} // namespace + +template +__global__ void +apex_fused_dropout_kernel(scalar_t const *inputs, scalar_t *outputs, + uint8_t *mask, IndexType totalElements, accscalar_t p, + std::pair seeds) { + accscalar_t pinv = accscalar_t(1) / p; + IndexType idx = blockIdx.x * blockDim.x + threadIdx.x; + + curandStatePhilox4_32_10_t state; + curand_init(seeds.first, idx, seeds.second, &state); + + IndexType rounded_size = + ((totalElements - 1) / (blockDim.x * gridDim.x * UNROLL) + 1) * + blockDim.x * gridDim.x * UNROLL; + for (IndexType linearIndex = idx; linearIndex < rounded_size; + linearIndex += gridDim.x * blockDim.x * UNROLL) { + float4 rand = curand_uniform4(&state); + scalar_t src[UNROLL]; + rand.x = rand.x <= p; + rand.y = rand.y <= p; + rand.z = rand.z <= p; + rand.w = rand.w <= p; + + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + src[ii] = inputs[li]; + } + } + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + outputs[li] = src[ii] * (&rand.x)[ii] * pinv; + mask[li] = (uint8_t)(&rand.x)[ii]; + } + } + __syncthreads(); + } +} + +template +__global__ void apex_dropout_add_kernel(scalar_t const *inputs, + scalar_t const *add_inputs, + scalar_t *outputs, uint8_t *mask, + IndexType totalElements, accscalar_t p, + std::pair seeds) { + accscalar_t pinv = accscalar_t(1) / p; + IndexType idx = blockIdx.x * blockDim.x + threadIdx.x; + + curandStatePhilox4_32_10_t state; + curand_init(seeds.first, idx, seeds.second, &state); + + IndexType rounded_size = + ((totalElements - 1) / (blockDim.x * gridDim.x * UNROLL) + 1) * + blockDim.x * gridDim.x * UNROLL; + for (IndexType linearIndex = idx; linearIndex < rounded_size; + linearIndex += gridDim.x * blockDim.x * UNROLL) { + float4 rand = curand_uniform4(&state); + scalar_t src[UNROLL]; + scalar_t add_src[UNROLL]; + rand.x = rand.x <= p; + rand.y = rand.y <= p; + rand.z = rand.z <= p; + rand.w = rand.w <= p; + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + src[ii] = inputs[li]; + add_src[ii] = add_inputs[li]; + } + } + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + accscalar_t int1 = src[ii] * (&rand.x)[ii] * pinv; + outputs[li] = + static_cast(static_cast(add_src[ii]) + int1); + mask[li] = (uint8_t)(&rand.x)[ii]; + } + } + __syncthreads(); + } +} + +template +__global__ void apex_add_kernel(scalar_t const *inputs, + scalar_t const *add_inputs, scalar_t *outputs, + IndexType totalElements) { + IndexType idx = blockIdx.x * blockDim.x + threadIdx.x; + IndexType rounded_size = + ((totalElements - 1) / (blockDim.x * gridDim.x * UNROLL) + 1) * + blockDim.x * gridDim.x * UNROLL; + for (IndexType linearIndex = idx; linearIndex < rounded_size; + linearIndex += gridDim.x * blockDim.x * UNROLL) { + scalar_t src[UNROLL]; + scalar_t add_src[UNROLL]; + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + src[ii] = inputs[li]; + add_src[ii] = add_inputs[li]; + } + } + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + outputs[li] = src[ii] + add_src[ii]; + } + } + __syncthreads(); + } +} + +template +__global__ void apex_masked_scale_kernel(scalar_t const *inputs, + scalar_t *outputs, uint8_t const *mask, + IndexType totalElements, + accscalar_t scale) { + IndexType idx = blockIdx.x * blockDim.x + threadIdx.x; + IndexType rounded_size = + ((totalElements - 1) / (blockDim.x * gridDim.x * UNROLL) + 1) * + blockDim.x * gridDim.x * UNROLL; + for (IndexType linearIndex = idx; linearIndex < rounded_size; + linearIndex += gridDim.x * blockDim.x * UNROLL) { + scalar_t src[UNROLL]; + scalar_t msk[UNROLL]; + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + src[ii] = static_cast(inputs[li]); + msk[ii] = static_cast(mask[li]); + } + } + for (int ii = 0; ii < UNROLL; ii++) { + IndexType li = linearIndex + blockDim.x * gridDim.x * ii; + if (li < totalElements) { + outputs[li] = static_cast(src[ii]) * scale * + static_cast(msk[ii]); + } + } + } +} + +template +void apex_fused_dropout_cuda(scalar_t const *inputs, scalar_t *outputs, + uint8_t *mask, IndexType totalElements, + accscalar_t p) { + auto gen = at::cuda::detail::getDefaultCUDAGenerator(); + + int block_size = 256; + dim3 dim_block(block_size); + dim3 grid((totalElements + block_size - 1) / block_size); + unsigned int blocks_per_sm = + at::cuda::getCurrentDeviceProperties()->maxThreadsPerMultiProcessor / + block_size; + grid.x = std::min((unsigned int)at::cuda::getCurrentDeviceProperties() + ->multiProcessorCount * + blocks_per_sm, + grid.x); + + // number of times random will be generated per thread, to offset philox + // counter in the random state + int64_t counter_offset = + ((totalElements - 1) / (block_size * grid.x * UNROLL) + 1) * UNROLL; + std::pair rng_engine_inputs; + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + rng_engine_inputs = + at::check_generator(gen)->philox_engine_inputs( + counter_offset); + } + + apex_fused_dropout_kernel + <<>>( + inputs, outputs, mask, totalElements, p, rng_engine_inputs); + C10_CUDA_CHECK(cudaGetLastError()); +} + +template +void apex_dropout_add_cuda(scalar_t const *inputs, scalar_t const *add_inputs, + scalar_t *outputs, uint8_t *mask, + IndexType totalElements, accscalar_t p) { + auto gen = at::cuda::detail::getDefaultCUDAGenerator(); + + int block_size = 256; + dim3 dim_block(block_size); + dim3 grid((totalElements + block_size - 1) / block_size); + unsigned int blocks_per_sm = + at::cuda::getCurrentDeviceProperties()->maxThreadsPerMultiProcessor / + block_size; + grid.x = std::min((unsigned int)at::cuda::getCurrentDeviceProperties() + ->multiProcessorCount * + blocks_per_sm, + grid.x); + + // number of times random will be generated per thread, to offset philox + // counter in the random state + int64_t counter_offset = + ((totalElements - 1) / (block_size * grid.x * UNROLL) + 1) * UNROLL; + std::pair rng_engine_inputs; + { + // See Note [Acquire lock when using random generators] + std::lock_guard lock(gen.mutex()); + rng_engine_inputs = + at::check_generator(gen)->philox_engine_inputs( + counter_offset); + } + + apex_dropout_add_kernel + <<>>( + inputs, add_inputs, outputs, mask, totalElements, p, + rng_engine_inputs); + C10_CUDA_CHECK(cudaGetLastError()); +} + +template +void apex_add_cuda(scalar_t const *inputs, scalar_t const *add_inputs, + scalar_t *outputs, IndexType totalElements) { + int block_size = 256; + dim3 dim_block(block_size); + dim3 grid((totalElements + block_size - 1) / block_size); + unsigned int blocks_per_sm = + at::cuda::getCurrentDeviceProperties()->maxThreadsPerMultiProcessor / + block_size; + grid.x = std::min((unsigned int)at::cuda::getCurrentDeviceProperties() + ->multiProcessorCount * + blocks_per_sm, + grid.x); + + apex_add_kernel + <<>>( + inputs, add_inputs, outputs, totalElements); + C10_CUDA_CHECK(cudaGetLastError()); +} + +template +void apex_masked_scale_cuda(scalar_t const *inputs, scalar_t *outputs, + uint8_t const *mask, IndexType totalElements, + accscalar_t scale) { + int block_size = 256; + dim3 dim_block(block_size); + dim3 grid((totalElements + block_size - 1) / block_size); + unsigned int blocks_per_sm = + at::cuda::getCurrentDeviceProperties()->maxThreadsPerMultiProcessor / + block_size; + grid.x = std::min((unsigned int)at::cuda::getCurrentDeviceProperties() + ->multiProcessorCount * + blocks_per_sm, + grid.x); + + apex_masked_scale_kernel + <<>>( + inputs, outputs, mask, totalElements, scale); + C10_CUDA_CHECK(cudaGetLastError()); +} diff --git a/apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu new file mode 100644 index 00000000..d56b8076 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu @@ -0,0 +1,368 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "softmax.cuh" +#include "strided_batched_gemm.cuh" + +namespace multihead_attn { +namespace encdec { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, + torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob) { + const int embed_dim = inputs_q.size(2); + const int sequences = inputs_q.size(1); + const int q_seq_len = inputs_q.size(0); + const int k_seq_len = inputs_kv.size(0); + const int batches_q = sequences * q_seq_len; + const int batches_kv = sequences * k_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_q_dim = embed_dim; + const int output_lin_kv_dim = 2 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim_q = attn_batches * head_dim; + const int lead_dim_kv = attn_batches * 2 * head_dim; + const int batch_stride_q = head_dim; + const int batch_stride_kv = 2 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = inputs_q.options().requires_grad(false); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor input_lin_q_results = + torch::empty({q_seq_len, sequences, output_lin_q_dim}, act_options); + torch::Tensor input_lin_kv_results = + torch::empty({k_seq_len, sequences, output_lin_kv_dim}, act_options); + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + torch::Tensor matmul2_results = + torch::empty({q_seq_len, attn_batches, head_dim}, act_options); + torch::Tensor outputs = torch::empty_like(inputs_q, act_options); + + // Input Linear Results Pointers to Q, K, and V of interviewed activations + void *q_lin_results_ptr = static_cast(input_lin_q_results.data_ptr()); + void *k_lin_results_ptr = + static_cast(input_lin_kv_results.data_ptr()); + void *v_lin_results_ptr = static_cast( + static_cast(input_lin_kv_results.data_ptr()) + head_dim); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + char a_layout_t{'t'}; + char a_layout_n{'n'}; + char b_layout_n{'n'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + // Input Linear Q Fwd + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_q_dim, batches_q, embed_dim, + static_cast(&alpha), + static_cast(input_weights_q.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(inputs_q.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), q_lin_results_ptr, + CUDA_R_16F, output_lin_q_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear KV Fwd + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_kv_dim, batches_kv, + embed_dim, static_cast(&alpha), + static_cast(input_weights_kv.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(inputs_kv.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), k_lin_results_ptr, + CUDA_R_16F, output_lin_kv_dim, CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul1 of Dot-Product Attention Plus scaling by 1/Sqrt(head size) + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, scale, + static_cast(k_lin_results_ptr), lead_dim_kv, + batch_stride_kv, static_cast(q_lin_results_ptr), lead_dim_q, + batch_stride_q, beta, static_cast(softmax_results_ptr), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Padded Softmax + bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + } else { + if (use_time_mask) { + softmax_success = dispatch_time_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, q_seq_len); + } else { + softmax_success = dispatch_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + } + assert(softmax_success); + + if (is_training) { + apex_fused_dropout_cuda( + static_cast(softmax_results.data_ptr()), + static_cast(dropout_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0f - dropout_prob)); + } + + // Matmul2 + gemm_switch_fp32accum( + a_layout_n, b_layout_n, head_dim, q_seq_len, k_seq_len, alpha, + static_cast(v_lin_results_ptr), lead_dim_kv, + batch_stride_kv, + (is_training) ? static_cast(dropout_results.data_ptr()) + : static_cast(softmax_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, + static_cast(matmul2_results.data_ptr()), head_dim * attn_batches, + head_dim, attn_batches); + + // Output Linear + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, embed_dim, batches_q, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(matmul2_results.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(outputs.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO1_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_lin_q_results, + input_lin_kv_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + outputs}; +} + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_q_results, + torch::Tensor const &input_lin_kv_results, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + const int embed_dim = inputs_q.size(2); + const int sequences = inputs_q.size(1); + const int q_seq_len = inputs_q.size(0); + const int k_seq_len = inputs_kv.size(0); + const int batches_q = sequences * q_seq_len; + const int batches_kv = sequences * k_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_q_dim = embed_dim; + const int output_lin_kv_dim = 2 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim_q = attn_batches * head_dim; + const int lead_dim_kv = attn_batches * 2 * head_dim; + const int batch_stride_q = head_dim; + const int batch_stride_kv = 2 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + torch::Tensor input_q_grads = torch::empty_like(inputs_q); + torch::Tensor input_kv_grads = torch::empty_like(inputs_kv); + torch::Tensor input_weight_q_grads = torch::empty_like(input_weights_q); + torch::Tensor input_weight_kv_grads = torch::empty_like(input_weights_kv); + torch::Tensor output_weight_grads = torch::empty_like(output_weights); + // Intermediate Tensor Allocations + at::Tensor output_lin_grads = torch::empty_like(matmul2_results); + at::Tensor matmul2_grads = torch::empty_like(dropout_results); + at::Tensor input_lin_q_output_grads = torch::empty_like(input_lin_q_results); + at::Tensor input_lin_kv_output_grads = + torch::empty_like(input_lin_kv_results); + + auto q_lin_results_ptr = static_cast(input_lin_q_results.data_ptr()); + auto k_lin_results_ptr = static_cast(input_lin_kv_results.data_ptr()); + auto v_lin_results_ptr = + static_cast(input_lin_kv_results.data_ptr()) + head_dim; + + auto q_lin_grads_ptr = + static_cast(input_lin_q_output_grads.data_ptr()); + auto k_lin_grads_ptr = + static_cast(input_lin_kv_output_grads.data_ptr()); + auto v_lin_grads_ptr = + static_cast(input_lin_kv_output_grads.data_ptr()) + head_dim; + + char a_layout_n{'n'}; + char a_layout_t{'t'}; + char b_layout_n{'n'}; + char b_layout_t{'t'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + + // Output Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches_q, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Output Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, embed_dim, batches_q, + static_cast(&alpha), + static_cast(matmul2_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_weight_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul2 Dgrad1 + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, alpha, + static_cast(v_lin_results_ptr), lead_dim_kv, + batch_stride_kv, static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, beta, + static_cast(matmul2_grads.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Matmul2 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, alpha, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, + static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, v_lin_grads_ptr, + lead_dim_kv, batch_stride_kv, attn_batches); + + // Apply Dropout Mask and Scale by Dropout Probability + apex_masked_scale_cuda( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0 / (1.0 - dropout_prob))); + + // Softmax Grad + bool softmax_success = false; + softmax_success = dispatch_softmax_backward( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + assert(softmax_success); + + // Matmul1 Dgrad1 + gemm_switch_fp32accum(a_layout_n, b_layout_n, head_dim, q_seq_len, + k_seq_len, scale, k_lin_results_ptr, lead_dim_kv, + batch_stride_kv, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, q_lin_grads_ptr, + lead_dim_q, batch_stride_q, attn_batches); + + // Matmul1 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, scale, q_lin_results_ptr, lead_dim_q, + batch_stride_q, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, k_lin_grads_ptr, + lead_dim_kv, batch_stride_kv, attn_batches); + + // Input Linear Q Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches_q, output_lin_q_dim, + static_cast(&alpha), + static_cast(input_weights_q.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(q_lin_grads_ptr), CUDA_R_16F, + output_lin_q_dim, static_cast(&beta), + static_cast(input_q_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear Q Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_q_dim, batches_q, + static_cast(&alpha), + static_cast(inputs_q.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(q_lin_grads_ptr), CUDA_R_16F, output_lin_q_dim, + static_cast(&beta), + static_cast(input_weight_q_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear KV Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches_kv, + output_lin_kv_dim, static_cast(&alpha), + static_cast(input_weights_kv.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(k_lin_grads_ptr), CUDA_R_16F, + output_lin_kv_dim, static_cast(&beta), + static_cast(input_kv_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear KV Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_kv_dim, + batches_kv, static_cast(&alpha), + static_cast(inputs_kv.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(k_lin_grads_ptr), CUDA_R_16F, output_lin_kv_dim, + static_cast(&beta), + static_cast(input_weight_kv_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_q_grads, input_kv_grads, input_weight_q_grads, + input_weight_kv_grads, output_weight_grads}; +} + +} // end namespace cublas_gemmex +} // end namespace encdec +} // end namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu new file mode 100644 index 00000000..10c9b8ce --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu @@ -0,0 +1,445 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "layer_norm.cuh" +#include "softmax.cuh" +#include "strided_batched_gemm.cuh" + +namespace multihead_attn { +namespace encdec_norm_add { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob) { + const int embed_dim = inputs_q.size(2); + const int sequences = inputs_q.size(1); + const int q_seq_len = inputs_q.size(0); + const int k_seq_len = inputs_kv.size(0); + const int batches_q = sequences * q_seq_len; + const int batches_kv = sequences * k_seq_len; + const int total_tokens_q = batches_q * embed_dim; + const int head_dim = embed_dim / heads; + const int output_lin_q_dim = embed_dim; + const int output_lin_kv_dim = 2 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim_q = attn_batches * head_dim; + const int lead_dim_kv = attn_batches * 2 * head_dim; + const int batch_stride_q = head_dim; + const int batch_stride_kv = 2 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = inputs_q.options().requires_grad(false); + auto lyr_nrm_options = act_options.dtype(torch::kFloat32); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor lyr_nrm_mean = torch::empty({batches_q}, lyr_nrm_options); + torch::Tensor lyr_nrm_invvar = torch::empty({batches_q}, lyr_nrm_options); + torch::Tensor lyr_nrm_results = torch::empty_like(inputs_q, act_options); + + torch::Tensor input_lin_q_results = + torch::empty({q_seq_len, sequences, output_lin_q_dim}, act_options); + torch::Tensor input_lin_kv_results = + torch::empty({k_seq_len, sequences, output_lin_kv_dim}, act_options); + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + torch::Tensor matmul2_results = + torch::empty({q_seq_len, attn_batches, head_dim}, act_options); + torch::Tensor output_lin_results = torch::empty_like(inputs_q, act_options); + torch::Tensor dropout_add_mask = torch::empty_like(inputs_q, mask_options); + torch::Tensor outputs = torch::empty_like(inputs_q, act_options); + + // Input Linear Results Pointers to Q, K, and V of interviewed activations + void *q_lin_results_ptr = static_cast(input_lin_q_results.data_ptr()); + void *k_lin_results_ptr = + static_cast(input_lin_kv_results.data_ptr()); + void *v_lin_results_ptr = static_cast( + static_cast(input_lin_kv_results.data_ptr()) + head_dim); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + char a_layout_t{'t'}; + char a_layout_n{'n'}; + char b_layout_n{'n'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + // Layer Norm + HostApplyLayerNorm( + static_cast(lyr_nrm_results.data_ptr()), + static_cast(lyr_nrm_mean.data_ptr()), + static_cast(lyr_nrm_invvar.data_ptr()), + static_cast(inputs_q.data_ptr()), + static_cast(batches_q), // n1 + static_cast(embed_dim), // n2 + 1.0e-5, static_cast(lyr_nrm_gamma_weights.data_ptr()), + static_cast(lyr_nrm_beta_weights.data_ptr())); + + // Input Linear Q Fwd + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_q_dim, batches_q, embed_dim, + static_cast(&alpha), + static_cast(input_weights_q.data_ptr()), CUDA_R_16F, + embed_dim, + // static_cast(inputs_q.data_ptr()), + static_cast(lyr_nrm_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), q_lin_results_ptr, + CUDA_R_16F, output_lin_q_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear KV Fwd + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_kv_dim, batches_kv, + embed_dim, static_cast(&alpha), + static_cast(input_weights_kv.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(inputs_kv.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), k_lin_results_ptr, + CUDA_R_16F, output_lin_kv_dim, CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul1 of Dot-Product Attention Plus scaling by 1/Sqrt(head size) + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, scale, + static_cast(k_lin_results_ptr), lead_dim_kv, + batch_stride_kv, static_cast(q_lin_results_ptr), lead_dim_q, + batch_stride_q, beta, static_cast(softmax_results_ptr), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Padded Softmax + bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + } else { + if (use_time_mask) { + softmax_success = dispatch_time_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, q_seq_len); + } else { + softmax_success = dispatch_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + } + assert(softmax_success); + + if (is_training) { + apex_fused_dropout_cuda( + static_cast(softmax_results.data_ptr()), + static_cast(dropout_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0f - dropout_prob)); + } + + // Matmul2 + gemm_switch_fp32accum( + a_layout_n, b_layout_n, head_dim, q_seq_len, k_seq_len, alpha, + static_cast(v_lin_results_ptr), lead_dim_kv, + batch_stride_kv, + (is_training) ? static_cast(dropout_results.data_ptr()) + : static_cast(softmax_results.data_ptr()), + // static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, + static_cast(matmul2_results.data_ptr()), head_dim * attn_batches, + head_dim, attn_batches); + + // Output Linear + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, embed_dim, batches_q, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(matmul2_results.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(output_lin_results.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO1_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // End-of-block Dropout-Add + if (is_training) { + apex_dropout_add_cuda( + static_cast(output_lin_results.data_ptr()), + static_cast(inputs_q.data_ptr()), + static_cast(outputs.data_ptr()), + static_cast(dropout_add_mask.data_ptr()), total_tokens_q, + (1.0f - dropout_prob)); + } else { + apex_add_cuda( + static_cast(output_lin_results.data_ptr()), + static_cast(inputs_q.data_ptr()), + static_cast(outputs.data_ptr()), total_tokens_q); + } + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + input_lin_q_results, + input_lin_kv_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + dropout_add_mask, + outputs}; +} + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_q_results, + torch::Tensor const &input_lin_kv_results, + torch::Tensor const &lyr_nrm_results, torch::Tensor const &lyr_nrm_mean, + torch::Tensor const &lyr_nrm_invvar, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights_q, torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, torch::Tensor const &dropout_mask, + torch::Tensor const &dropout_add_mask, float dropout_prob) { + const int embed_dim = inputs_q.size(2); + const int sequences = inputs_q.size(1); + const int q_seq_len = inputs_q.size(0); + const int k_seq_len = inputs_kv.size(0); + const int batches_q = sequences * q_seq_len; + const int batches_kv = sequences * k_seq_len; + const int total_tokens_q = batches_q * embed_dim; + const int head_dim = embed_dim / heads; + const int output_lin_q_dim = embed_dim; + const int output_lin_kv_dim = 2 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim_q = attn_batches * head_dim; + const int lead_dim_kv = attn_batches * 2 * head_dim; + const int batch_stride_q = head_dim; + const int batch_stride_kv = 2 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + torch::Tensor input_q_grads = torch::empty_like(inputs_q); + torch::Tensor input_kv_grads = torch::empty_like(inputs_kv); + torch::Tensor lyr_nrm_gamma_grads = torch::empty_like(lyr_nrm_gamma_weights); + torch::Tensor lyr_nrm_beta_grads = torch::empty_like(lyr_nrm_beta_weights); + torch::Tensor input_weight_q_grads = torch::empty_like(input_weights_q); + torch::Tensor input_weight_kv_grads = torch::empty_like(input_weights_kv); + torch::Tensor output_weight_grads = torch::empty_like(output_weights); + // Intermediate Tensor Allocations + at::Tensor dropout_add_grads = torch::empty_like(output_grads); + at::Tensor output_lin_grads = torch::empty_like(matmul2_results); + at::Tensor matmul2_grads = torch::empty_like(dropout_results); + at::Tensor input_lin_q_output_grads = torch::empty_like(input_lin_q_results); + at::Tensor input_lin_kv_output_grads = + torch::empty_like(input_lin_kv_results); + at::Tensor input_lin_q_grads = torch::empty_like(inputs_q); + + auto q_lin_results_ptr = static_cast(input_lin_q_results.data_ptr()); + auto k_lin_results_ptr = static_cast(input_lin_kv_results.data_ptr()); + auto v_lin_results_ptr = + static_cast(input_lin_kv_results.data_ptr()) + head_dim; + + auto q_lin_grads_ptr = + static_cast(input_lin_q_output_grads.data_ptr()); + auto k_lin_grads_ptr = + static_cast(input_lin_kv_output_grads.data_ptr()); + auto v_lin_grads_ptr = + static_cast(input_lin_kv_output_grads.data_ptr()) + head_dim; + + char a_layout_n{'n'}; + char a_layout_t{'t'}; + char b_layout_n{'n'}; + char b_layout_t{'t'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + + // Dropout Add Backward + apex_masked_scale_cuda( + static_cast(output_grads.data_ptr()), + static_cast(dropout_add_grads.data_ptr()), + static_cast(dropout_add_mask.data_ptr()), total_tokens_q, + (1.0 / (1.0 - dropout_prob))); + + // Output Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches_q, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(dropout_add_grads.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(output_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Output Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, embed_dim, batches_q, + static_cast(&alpha), + static_cast(matmul2_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(dropout_add_grads.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(output_weight_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul2 Dgrad1 + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, alpha, + static_cast(v_lin_results_ptr), lead_dim_kv, + batch_stride_kv, static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, beta, + static_cast(matmul2_grads.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Matmul2 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, alpha, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, + static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, v_lin_grads_ptr, + lead_dim_kv, batch_stride_kv, attn_batches); + + // Apply Dropout Mask and Scale by Dropout Probability + apex_masked_scale_cuda( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0 / (1.0 - dropout_prob))); + + // Softmax Grad + bool softmax_success = false; + softmax_success = dispatch_softmax_backward( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + assert(softmax_success); + + // Matmul1 Dgrad1 + gemm_switch_fp32accum(a_layout_n, b_layout_n, head_dim, q_seq_len, + k_seq_len, scale, k_lin_results_ptr, lead_dim_kv, + batch_stride_kv, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, q_lin_grads_ptr, + lead_dim_q, batch_stride_q, attn_batches); + + // Matmul1 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, scale, q_lin_results_ptr, lead_dim_q, + batch_stride_q, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, k_lin_grads_ptr, + lead_dim_kv, batch_stride_kv, attn_batches); + + // Input Linear Q Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches_q, output_lin_q_dim, + static_cast(&alpha), + static_cast(input_weights_q.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(q_lin_grads_ptr), CUDA_R_16F, + output_lin_q_dim, static_cast(&beta), + // static_cast(input_q_grads.data_ptr()), + static_cast(input_lin_q_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear Q Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_q_dim, batches_q, + static_cast(&alpha), + static_cast(inputs_q.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(q_lin_grads_ptr), CUDA_R_16F, output_lin_q_dim, + static_cast(&beta), + static_cast(input_weight_q_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear KV Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches_kv, + output_lin_kv_dim, static_cast(&alpha), + static_cast(input_weights_kv.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(k_lin_grads_ptr), CUDA_R_16F, + output_lin_kv_dim, static_cast(&beta), + static_cast(input_kv_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear KV Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_kv_dim, + batches_kv, static_cast(&alpha), + static_cast(inputs_kv.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(k_lin_grads_ptr), CUDA_R_16F, output_lin_kv_dim, + static_cast(&beta), + static_cast(input_weight_kv_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Fused Layer Norm Bwd with Residual Add + HostLayerNormGradient( + static_cast(input_lin_q_grads.data_ptr()), + static_cast(output_grads.data_ptr()), + static_cast(lyr_nrm_mean.data_ptr()), + static_cast(lyr_nrm_invvar.data_ptr()), inputs_q, + static_cast(batches_q), // n1 + static_cast(embed_dim), // n2 + static_cast(lyr_nrm_gamma_weights.data_ptr()), + static_cast(lyr_nrm_beta_weights.data_ptr()), 1.0e-5, + static_cast(input_q_grads.data_ptr()), + static_cast(lyr_nrm_gamma_grads.data_ptr()), + static_cast(lyr_nrm_beta_grads.data_ptr())); + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_q_grads, input_kv_grads, lyr_nrm_gamma_grads, + lyr_nrm_beta_grads, input_weight_q_grads, input_weight_kv_grads, + output_weight_grads}; +} + +} // end namespace cublas_gemmex +} // end namespace encdec_norm_add +} // end namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/layer_norm.cuh b/apex/apex/contrib/csrc/multihead_attn/layer_norm.cuh new file mode 100644 index 00000000..16c1eeef --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/layer_norm.cuh @@ -0,0 +1,636 @@ +#pragma once +#include +#include +#include +#include + +namespace { +template +__device__ void cuWelfordOnlineSum(const U curr, U &mu, U &sigma2, U &count) { + count = count + U(1); + U delta = curr - mu; + U lmean = mu + delta / count; + mu = lmean; + U delta2 = curr - lmean; + sigma2 = sigma2 + delta * delta2; +} + +template +__device__ void cuChanOnlineSum(const U muB, const U sigma2B, const U countB, + U &mu, U &sigma2, U &count) { + U delta = muB - mu; + U nA = count; + U nB = countB; + count = count + countB; + U nX = count; + if (nX > U(0)) { + nA = nA / nX; + nB = nB / nX; + mu = nA * mu + nB * muB; + sigma2 = sigma2 + sigma2B + delta * delta * nA * nB * nX; + } else { + mu = U(0); + sigma2 = U(0); + } +} + +template +__device__ void cuWelfordMuSigma2(const T *__restrict__ vals, const int n1, + const int n2, const int i1, U &mu, U &sigma2, + U *buf) { + // Assumptions: + // 1) blockDim.x == warpSize + // 2) Tensor is contiguous + // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available. + // + // compute variance and mean over n2 + U count = U(0); + mu = U(0); + sigma2 = U(0); + if (i1 < n1) { + // one warp normalizes one n1 index, + // synchronization is implicit + // initialize with standard Welford algorithm + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + const T *lvals = vals + i1 * n2; + int l = 4 * thrx; + for (; l + 3 < n2; l += 4 * numx) { + for (int k = 0; k < 4; ++k) { + U curr = static_cast(lvals[l + k]); + cuWelfordOnlineSum(curr, mu, sigma2, count); + } + } + for (; l < n2; ++l) { + U curr = static_cast(lvals[l]); + cuWelfordOnlineSum(curr, mu, sigma2, count); + } + // intra-warp reductions + for (int l = 0; l <= 4; ++l) { + int srcLaneB = (threadIdx.x + (1 << l)) & 31; + U muB = WARP_SHFL(mu, srcLaneB); + U countB = WARP_SHFL(count, srcLaneB); + U sigma2B = WARP_SHFL(sigma2, srcLaneB); + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } + // threadIdx.x == 0 has correct values for each warp + // inter-warp reductions + if (blockDim.y > 1) { + U *ubuf = (U *)buf; + U *ibuf = (U *)(ubuf + blockDim.y); + for (int offset = blockDim.y / 2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.x == 0 && threadIdx.y >= offset && + threadIdx.y < 2 * offset) { + const int wrt_y = threadIdx.y - offset; + ubuf[2 * wrt_y] = mu; + ubuf[2 * wrt_y + 1] = sigma2; + ibuf[wrt_y] = count; + } + __syncthreads(); + // lower half merges + if (threadIdx.x == 0 && threadIdx.y < offset) { + U muB = ubuf[2 * threadIdx.y]; + U sigma2B = ubuf[2 * threadIdx.y + 1]; + U countB = ibuf[threadIdx.y]; + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } + __syncthreads(); + } + // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct values + if (threadIdx.x == 0 && threadIdx.y == 0) { + ubuf[0] = mu; + ubuf[1] = sigma2; + } + __syncthreads(); + mu = ubuf[0]; + sigma2 = ubuf[1] / U(n2); + // don't care about final value of count, we know count == n2 + } else { + mu = WARP_SHFL(mu, 0); + sigma2 = WARP_SHFL(sigma2 / U(n2), 0); + } + } +} + +template <> +__device__ void cuWelfordMuSigma2(const at::Half *__restrict__ vals, + const int n1, const int n2, const int i1, + float &mu, float &sigma2, float *buf) { + // Assumptions: + // 1) blockDim.x == warpSize + // 2) Tensor is contiguous + // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available. + // + // compute variance and mean over n2 + float count = 0.0f; + mu = float(0); + sigma2 = float(0); + + if (i1 < n1) { + // one warp normalizes one n1 index, + // synchronization is implicit + // initialize with standard Welford algorithm + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + const at::Half *lvals = vals + i1 * n2; + int l = 8 * thrx; + if ((((size_t)lvals) & 3) != 0) { + // 16 bit alignment + // first thread consumes first point + if (thrx == 0) { + float curr = static_cast(lvals[0]); + cuWelfordOnlineSum(curr, mu, sigma2, count); + } + ++l; + } + // at this point, lvals[l] are 32 bit aligned for all threads. + for (; l + 7 < n2; l += 8 * numx) { + for (int k = 0; k < 8; k += 2) { + float2 curr = __half22float2(*((__half2 *)(lvals + l + k))); + cuWelfordOnlineSum(curr.x, mu, sigma2, count); + cuWelfordOnlineSum(curr.y, mu, sigma2, count); + } + } + for (; l < n2; ++l) { + float curr = static_cast(lvals[l]); + cuWelfordOnlineSum(curr, mu, sigma2, count); + } + // intra-warp reductions + for (int l = 0; l <= 4; ++l) { + int srcLaneB = (threadIdx.x + (1 << l)) & 31; + float muB = WARP_SHFL(mu, srcLaneB); + float countB = WARP_SHFL(count, srcLaneB); + float sigma2B = WARP_SHFL(sigma2, srcLaneB); + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } + // threadIdx.x == 0 has correct values for each warp + // inter-warp reductions + if (blockDim.y > 1) { + float *ubuf = (float *)buf; + float *ibuf = (float *)(ubuf + blockDim.y); + for (int offset = blockDim.y / 2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.x == 0 && threadIdx.y >= offset && + threadIdx.y < 2 * offset) { + const int wrt_y = threadIdx.y - offset; + ubuf[2 * wrt_y] = mu; + ubuf[2 * wrt_y + 1] = sigma2; + ibuf[wrt_y] = count; + } + __syncthreads(); + // lower half merges + if (threadIdx.x == 0 && threadIdx.y < offset) { + float muB = ubuf[2 * threadIdx.y]; + float sigma2B = ubuf[2 * threadIdx.y + 1]; + float countB = ibuf[threadIdx.y]; + cuChanOnlineSum(muB, sigma2B, countB, mu, sigma2, count); + } + __syncthreads(); + } + // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct values + if (threadIdx.x == 0 && threadIdx.y == 0) { + ubuf[0] = mu; + ubuf[1] = sigma2; + } + __syncthreads(); + mu = ubuf[0]; + sigma2 = ubuf[1] / float(n2); + // don't care about final value of count, we know count == n2 + } else { + mu = WARP_SHFL(mu, 0); + sigma2 = WARP_SHFL(sigma2 / float(n2), 0); + } + } +} + +template __device__ U rsqrt(U v) { return U(1) / sqrt(v); } +template <> __device__ float rsqrt(float v) { return rsqrtf(v); } +template <> __device__ double rsqrt(double v) { return rsqrt(v); } + +// This is the un-specialized struct. Note that we prevent instantiation of +// this struct by putting an undefined symbol in the function body so it won't +// compile. +// template +// struct SharedMemory +// { +// // Ensure that we won't compile any un-specialized types +// __device__ T *getPointer() +// { +// extern __device__ void error(void); +// error(); +// return NULL; +// } +// }; +// https://github.com/NVIDIA/apex/issues/246 +template struct SharedMemory; +template <> struct SharedMemory { + __device__ float *getPointer() { + extern __shared__ float s_float[]; + return s_float; + } +}; + +template <> struct SharedMemory { + __device__ double *getPointer() { + extern __shared__ double s_double[]; + return s_double; + } +}; + +template +__global__ void +cuApplyLayerNorm(T *__restrict__ output_vals, U *__restrict__ mean, + U *__restrict__ invvar, const T *__restrict__ vals, + const int n1, const int n2, const U epsilon, + const T *__restrict__ gamma, const T *__restrict__ beta) { + // Assumptions: + // 1) blockDim.x == warpSize + // 2) Tensors are contiguous + // + for (auto i1 = blockIdx.y; i1 < n1; i1 += gridDim.y) { + SharedMemory shared; + U *buf = shared.getPointer(); + U mu, sigma2; + cuWelfordMuSigma2(vals, n1, n2, i1, mu, sigma2, buf); + const T *lvals = vals + i1 * n2; + T *ovals = output_vals + i1 * n2; + U c_invvar = rsqrt(sigma2 + epsilon); + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + if (gamma != NULL && beta != NULL) { + for (int i = thrx; i < n2; i += numx) { + U curr = static_cast(lvals[i]); + ovals[i] = gamma[i] * static_cast(c_invvar * (curr - mu)) + beta[i]; + } + } else { + for (int i = thrx; i < n2; i += numx) { + U curr = static_cast(lvals[i]); + ovals[i] = static_cast(c_invvar * (curr - mu)); + } + } + if (threadIdx.x == 0 && threadIdx.y == 0) { + mean[i1] = mu; + invvar[i1] = c_invvar; + } + } +} + +template +__device__ void cuLoadWriteStridedInputs( + const int i1_block, const int thr_load_row_off, const int thr_load_col_off, + const int i2_off, const int row_stride, U *warp_buf1, U *warp_buf2, + const T *input, const T *dout, const int i1_end, const int n2, + const U *__restrict__ mean, const U *__restrict__ invvar) { + int i1 = i1_block + thr_load_row_off; + if (i1 < i1_end) { + U curr_mean = mean[i1]; + U curr_invvar = invvar[i1]; + for (int k = 0; k < blockDim.y; ++k) { + int i2 = i2_off + k; + int load_idx = i1 * n2 + i2; + int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; + if (i2 < n2) { + U curr_input = static_cast(input[load_idx]); + U curr_dout = static_cast(dout[load_idx]); + warp_buf1[write_idx] = curr_dout; + warp_buf2[write_idx] = + curr_dout * (curr_input - curr_mean) * curr_invvar; + } else { + warp_buf1[write_idx] = U(0); + warp_buf2[write_idx] = U(0); + } + } + } else { + for (int k = 0; k < blockDim.y; ++k) { + int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; + warp_buf1[write_idx] = U(0); + warp_buf2[write_idx] = U(0); + } + } +} + +template +__device__ void cuLoadAddStridedInputs( + const int i1_block, const int thr_load_row_off, const int thr_load_col_off, + const int i2_off, const int row_stride, U *warp_buf1, U *warp_buf2, + const T *input, const T *dout, const int i1_end, const int n2, + const U *__restrict__ mean, const U *__restrict__ invvar) { + int i1 = i1_block + thr_load_row_off; + if (i1 < i1_end) { + U curr_mean = mean[i1]; + U curr_invvar = invvar[i1]; + for (int k = 0; k < blockDim.y; ++k) { + int i2 = i2_off + k; + int load_idx = i1 * n2 + i2; + int write_idx = thr_load_row_off * row_stride + thr_load_col_off + k; + if (i2 < n2) { + U curr_input = static_cast(input[load_idx]); + U curr_dout = static_cast(dout[load_idx]); + warp_buf1[write_idx] += curr_dout; + warp_buf2[write_idx] += + curr_dout * (curr_input - curr_mean) * curr_invvar; + } + } + } +} + +template +__global__ void cuComputePartGradGammaBeta( + const T *__restrict__ dout, const T *__restrict__ input, const int n1, + const int n2, const U *__restrict__ mean, const U *__restrict__ invvar, + U epsilon, U *part_grad_gamma, U *part_grad_beta) { + const int numsegs_n1 = + (n1 + blockDim.y * blockDim.y - 1) / (blockDim.y * blockDim.y); + const int segs_per_block = (numsegs_n1 + gridDim.y - 1) / gridDim.y; + const int i1_beg = blockIdx.y * segs_per_block * blockDim.y * blockDim.y; + const int i1_beg_plus_one = + (blockIdx.y + 1) * segs_per_block * blockDim.y * blockDim.y; + const int i1_end = i1_beg_plus_one < n1 ? i1_beg_plus_one : n1; + const int row_stride = blockDim.x + 1; + const int thr_load_col_off = (threadIdx.x * blockDim.y) & (blockDim.x - 1); + const int thr_load_row_off = + (threadIdx.x * blockDim.y) / blockDim.x + threadIdx.y * blockDim.y; + const int i2_off = blockIdx.x * blockDim.x + thr_load_col_off; + SharedMemory shared; + U *buf = shared.getPointer(); // buf has at least blockDim.x * blockDim.y * + // blockDim.y + (blockDim.y - + // 1)*(blockDim.x/blockDim.y) elements + U *warp_buf1 = (U *)buf; + U *warp_buf2 = warp_buf1 + blockDim.y * blockDim.y * row_stride; + // compute partial sums from strided inputs + // do this to increase number of loads in flight + cuLoadWriteStridedInputs(i1_beg, thr_load_row_off, thr_load_col_off, i2_off, + row_stride, warp_buf1, warp_buf2, input, dout, + i1_end, n2, mean, invvar); + for (int i1_block = i1_beg + blockDim.y * blockDim.y; i1_block < i1_end; + i1_block += blockDim.y * blockDim.y) { + cuLoadAddStridedInputs(i1_block, thr_load_row_off, thr_load_col_off, i2_off, + row_stride, warp_buf1, warp_buf2, input, dout, + i1_end, n2, mean, invvar); + } + __syncthreads(); + // inter-warp reductions + // sum within each warp + U acc1 = U(0); + U acc2 = U(0); + for (int k = 0; k < blockDim.y; ++k) { + int row1 = threadIdx.y + k * blockDim.y; + int idx1 = row1 * row_stride + threadIdx.x; + acc1 += warp_buf1[idx1]; + acc2 += warp_buf2[idx1]; + } + warp_buf1[threadIdx.y * row_stride + threadIdx.x] = acc1; + warp_buf2[threadIdx.y * row_stride + threadIdx.x] = acc2; + __syncthreads(); + // sum all warps + for (int offset = blockDim.y / 2; offset > 1; offset /= 2) { + if (threadIdx.y < offset) { + int row1 = threadIdx.y; + int row2 = threadIdx.y + offset; + int idx1 = row1 * row_stride + threadIdx.x; + int idx2 = row2 * row_stride + threadIdx.x; + warp_buf1[idx1] += warp_buf1[idx2]; + warp_buf2[idx1] += warp_buf2[idx2]; + } + __syncthreads(); + } + int i2 = blockIdx.x * blockDim.x + threadIdx.x; + if (threadIdx.y == 0 && i2 < n2) { + int row1 = threadIdx.y; + int row2 = threadIdx.y + 1; + int idx1 = row1 * row_stride + threadIdx.x; + int idx2 = row2 * row_stride + threadIdx.x; + part_grad_beta[blockIdx.y * n2 + i2] = warp_buf1[idx1] + warp_buf1[idx2]; + part_grad_gamma[blockIdx.y * n2 + i2] = warp_buf2[idx1] + warp_buf2[idx2]; + } +} + +template +__global__ void +cuComputeGradGammaBeta(const U *part_grad_gamma, const U *part_grad_beta, + const int part_size, const int n1, const int n2, + T *grad_gamma, T *grad_beta) { + // sum partial gradients for gamma and beta + SharedMemory shared; + U *buf = shared.getPointer(); + int i2 = blockIdx.x * blockDim.x + threadIdx.x; + if (i2 < n2) { + // each warp does sequential reductions until reduced part_size is num_warps + int num_warp_reductions = part_size / blockDim.y; + U sum_gamma = U(0); + U sum_beta = U(0); + const U *part_grad_gamma_ptr = + part_grad_gamma + threadIdx.y * num_warp_reductions * n2 + i2; + const U *part_grad_beta_ptr = + part_grad_beta + threadIdx.y * num_warp_reductions * n2 + i2; + for (int warp_offset = 0; warp_offset < num_warp_reductions; + ++warp_offset) { + sum_gamma += part_grad_gamma_ptr[warp_offset * n2]; + sum_beta += part_grad_beta_ptr[warp_offset * n2]; + } + // inter-warp reductions + const int nbsize3 = blockDim.x * blockDim.y / 2; + for (int offset = blockDim.y / 2; offset >= 1; offset /= 2) { + // top half write to shared memory + if (threadIdx.y >= offset && threadIdx.y < 2 * offset) { + const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x; + buf[write_idx] = sum_gamma; + buf[write_idx + nbsize3] = sum_beta; + } + __syncthreads(); + // bottom half sums + if (threadIdx.y < offset) { + const int read_idx = threadIdx.y * blockDim.x + threadIdx.x; + sum_gamma += buf[read_idx]; + sum_beta += buf[read_idx + nbsize3]; + } + __syncthreads(); + } + // write out fully summed gradients + if (threadIdx.y == 0) { + grad_gamma[i2] = sum_gamma; + grad_beta[i2] = sum_beta; + } + } +} + + +template +__global__ void +cuComputeGradInput(const T *__restrict__ dout, const T *__restrict__ dout_resid, + const T *__restrict__ input, const int n1, const int n2, + const U *__restrict__ mean, const U *__restrict__ invvar, + U epsilon, const T *gamma, T *grad_input) { + for (auto i1 = blockIdx.y; i1 < n1; i1 += gridDim.y) { + U sum_loss1 = U(0); + U sum_loss2 = U(0); + const U c_mean = mean[i1]; + const U c_invvar = invvar[i1]; + const T *k_input = input + i1 * n2; + const T *k_dout = dout + i1 * n2; + const T *k_dout_resid = dout_resid + i1 * n2; + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + if (gamma != NULL) { + int l = 4 * thrx; + for (; l + 3 < n2; l += 4 * numx) { + for (int k = 0; k < 4; ++k) { + const U c_h = static_cast(k_input[l + k]); + const U c_loss = static_cast(k_dout[l + k]); + sum_loss1 += c_loss * static_cast(gamma[l + k]); + sum_loss2 += + c_loss * static_cast(gamma[l + k]) * (c_h - c_mean) * c_invvar; + } + } + for (; l < n2; ++l) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + sum_loss1 += c_loss * static_cast(gamma[l]); + sum_loss2 += + c_loss * static_cast(gamma[l]) * (c_h - c_mean) * c_invvar; + } + } else { + int l = 4 * thrx; + for (; l + 3 < n2; l += 4 * numx) { + for (int k = 0; k < 4; ++k) { + const U c_h = static_cast(k_input[l + k]); + const U c_loss = static_cast(k_dout[l + k]); + sum_loss1 += c_loss; + sum_loss2 += c_loss * (c_h - c_mean) * c_invvar; + } + } + for (; l < n2; ++l) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + sum_loss1 += c_loss; + sum_loss2 += c_loss * (c_h - c_mean) * c_invvar; + } + } + // intra-warp reductions + for (int mask = blockDim.x / 2; mask > 0; mask /= 2) { + sum_loss1 += WARP_SHFL_XOR(sum_loss1, mask); + sum_loss2 += WARP_SHFL_XOR(sum_loss2, mask); + } + // inter-warp reductions + if (blockDim.y > 1) { + SharedMemory shared; + U *buf = shared.getPointer(); + for (int offset = blockDim.y / 2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.y >= offset && threadIdx.y < 2 * offset) { + const int wrt_i = (threadIdx.y - offset) * blockDim.x + threadIdx.x; + buf[2 * wrt_i] = sum_loss1; + buf[2 * wrt_i + 1] = sum_loss2; + } + __syncthreads(); + // lower half merges + if (threadIdx.y < offset) { + const int read_i = threadIdx.y * blockDim.x + threadIdx.x; + sum_loss1 += buf[2 * read_i]; + sum_loss2 += buf[2 * read_i + 1]; + } + __syncthreads(); + } + if (threadIdx.y == 0) { + buf[2 * threadIdx.x] = sum_loss1; + buf[2 * threadIdx.x + 1] = sum_loss2; + } + __syncthreads(); + if (threadIdx.y != 0) { + sum_loss1 = buf[2 * threadIdx.x]; + sum_loss2 = buf[2 * threadIdx.x + 1]; + } + } + // all threads now have the two sums over l + U fH = (U)n2; + U term1 = (U(1) / fH) * c_invvar; + T *k_grad_input = grad_input + i1 * n2; + if (gamma != NULL) { + for (int l = thrx; l < n2; l += numx) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + const T c_resid = static_cast(k_dout_resid[l]); + U f_grad_input = fH * c_loss * static_cast(gamma[l]); + f_grad_input -= sum_loss1; + f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2; + f_grad_input *= term1; + k_grad_input[l] = static_cast(f_grad_input) + c_resid; + } + } else { + for (int l = thrx; l < n2; l += numx) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + const T c_resid = static_cast(k_dout_resid[l]); + U f_grad_input = fH * c_loss; + f_grad_input -= sum_loss1; + f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2; + f_grad_input *= term1; + k_grad_input[l] = static_cast(f_grad_input) + c_resid; + } + } + } +} + +template +void HostApplyLayerNorm(T *output, U *mean, U *invvar, const T *input, int n1, + int n2, double epsilon, const T *gamma, const T *beta) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const dim3 threads(32, 4, 1); + const uint64_t maxGridY = + at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + const dim3 blocks(1, std::min((uint64_t)n1, maxGridY), 1); + int nshared = + threads.y > 1 ? threads.y * sizeof(U) + (threads.y / 2) * sizeof(U) : 0; + cuApplyLayerNorm<<>>( + output, mean, invvar, input, n1, n2, U(epsilon), gamma, beta); +} + +template +void HostLayerNormGradient(const T *dout, const T *dout_resid, const U *mean, + const U *invvar, const at::Tensor &input, int n1, + int n2, const T *gamma, const T *beta, + double epsilon, T *grad_input, T *grad_gamma, + T *grad_beta) { + auto stream = at::cuda::getCurrentCUDAStream().stream(); + + if (gamma != NULL && beta != NULL) { + // compute grad_gamma(j) and grad_beta(j) + const int part_size = 16; + const dim3 threads2(32, 4, 1); + const dim3 blocks2((n2 + threads2.x - 1) / threads2.x, part_size, 1); + const int nshared2_a = + 2 * sizeof(U) * threads2.y * threads2.y * (threads2.x + 1); + const int nshared2_b = threads2.x * threads2.y * sizeof(U); + const int nshared2 = nshared2_a > nshared2_b ? nshared2_a : nshared2_b; + at::Tensor part_grad_gamma = at::empty( + {part_size, n2}, + input.options().dtype(input.scalar_type() == at::ScalarType::Half + ? at::ScalarType::Float + : input.scalar_type())); + at::Tensor part_grad_beta = at::empty_like(part_grad_gamma); + cuComputePartGradGammaBeta<<>>( + dout, static_cast(input.data_ptr()), n1, n2, mean, invvar, + U(epsilon), static_cast(part_grad_gamma.data_ptr()), + static_cast(part_grad_beta.data_ptr())); + + const dim3 threads3(32, 8, 1); + const dim3 blocks3((n2 + threads2.x - 1) / threads2.x, 1, 1); + const int nshared3 = threads3.x * threads3.y * sizeof(U); + cuComputeGradGammaBeta<<>>( + static_cast(part_grad_gamma.data_ptr()), + static_cast(part_grad_beta.data_ptr()), part_size, n1, n2, + grad_gamma, grad_beta); + } + + // compute grad_input + const uint64_t maxGridY = + at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + const dim3 blocks1(1, std::min((uint64_t)n1, maxGridY), 1); + const dim3 threads1(32, 4, 1); + int nshared = threads1.y > 1 ? threads1.y * threads1.x * sizeof(U) : 0; + cuComputeGradInput<<>>( + dout, dout_resid, static_cast(input.data_ptr()), n1, n2, mean, + invvar, U(epsilon), gamma, grad_input); +} +} // namespace diff --git a/apex/apex/contrib/csrc/multihead_attn/masked_softmax_dropout_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/masked_softmax_dropout_cuda.cu new file mode 100644 index 00000000..f9a031d5 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/masked_softmax_dropout_cuda.cu @@ -0,0 +1,124 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "softmax.cuh" + +namespace multihead_attn { +namespace fused_softmax { +namespace mask_softmax_dropout { + +std::vector fwd_cuda(bool is_training, int heads, + torch::Tensor const &input, + const uint8_t *pad_mask, + float dropout_prob) { + const int attn_batches = input.size(0); + const int sequences = attn_batches / heads; + const int q_seq_len = input.size(1); + const int k_seq_len = q_seq_len; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = input.options().requires_grad(false); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *input_ptr = static_cast(input.data_ptr()); + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + // Padded Softmax + bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), k_seq_len, k_seq_len, + attn_batches * q_seq_len); + } else { + softmax_success = dispatch_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), pad_mask, k_seq_len, + k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + + if (is_training) { + // use at:: function so that C++ version generates the same random mask as + // python version + auto dropout_tuple = + at::_fused_dropout(softmax_results, 1.0f - dropout_prob); + dropout_results = std::get<0>(dropout_tuple); + dropout_mask = std::get<1>(dropout_tuple); + } + + // Matmul2 + + return {dropout_results, dropout_mask, softmax_results}; +} + +torch::Tensor bwd_cuda(int heads, torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + torch::Tensor const &dropout_mask, + const uint8_t *padding_mask, float dropout_prob) { + const int attn_batches = output_grads.size(0); + const int q_seq_len = output_grads.size(1); + const int k_seq_len = q_seq_len; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + // torch::Tensor input_grads = torch::empty_like(output_grads); + + // Apply Dropout Mask and Scale by Dropout Probability + // Softmax Grad + if (padding_mask == nullptr) { + dispatch_masked_scale_softmax_backward_stream( + static_cast(output_grads.data_ptr()), + static_cast(output_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), + 1.0 / (1.0 - dropout_prob), k_seq_len, k_seq_len, + attn_batches * q_seq_len, stream); + } else { + dispatch_masked_scale_softmax_backward_masked_out_stream( + static_cast(output_grads.data_ptr()), + static_cast(output_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), + static_cast(padding_mask), 1.0 / (1.0 - dropout_prob), + k_seq_len, k_seq_len, attn_batches * q_seq_len, heads, stream); + } + // backward pass is completely in-place + return output_grads; +} +} // namespace mask_softmax_dropout +} // namespace fused_softmax +} // namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/multihead_attn_frontend.cpp b/apex/apex/contrib/csrc/multihead_attn/multihead_attn_frontend.cpp new file mode 100644 index 00000000..99c8e6fb --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/multihead_attn_frontend.cpp @@ -0,0 +1,835 @@ +#include + +#include +#include + + +#define CHECK_CUDA(x) \ + TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) \ + TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) + +namespace multihead_attn { +namespace fused_softmax { +namespace additive_mask_softmax_dropout { + +std::vector fwd_cuda(bool is_training, int heads, + torch::Tensor const &input, + const half *pad_mask, float dropout_prob); + +torch::Tensor bwd_cuda(int heads, torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + torch::Tensor const &dropout_mask, float dropout_prob); + +std::vector fwd(bool use_mask, bool is_training, int heads, + torch::Tensor const &input, + torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(input.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Half, + "Only BYTE is supported"); + } + + return fwd_cuda(is_training, heads, input, + use_mask ? static_cast(pad_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +torch::Tensor bwd(bool use_mask, int heads, torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + torch::Tensor const &dropout_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + // TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + // "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, softmax_results, dropout_mask, + dropout_prob); +} + +} // namespace additive_mask_softmax_dropout +namespace mask_softmax_dropout { + +std::vector fwd_cuda(bool is_training, int heads, + torch::Tensor const &input, + const uint8_t *pad_mask, + float dropout_prob); + +torch::Tensor bwd_cuda(int heads, torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + torch::Tensor const &dropout_mask, + const uint8_t *padding_mask, float dropout_prob); + +std::vector fwd(bool use_mask, bool is_training, int heads, + torch::Tensor const &input, + torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(input.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + } + + return fwd_cuda(is_training, heads, input, + use_mask ? static_cast(pad_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +torch::Tensor bwd(bool use_mask, int heads, torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + torch::Tensor const &dropout_mask, + torch::Tensor const &padding_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + // TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + // "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, softmax_results, dropout_mask, + use_mask + ? static_cast(padding_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +} // end namespace mask_softmax_dropout +} // end namespace fused_softmax + +namespace encdec { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, + torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob); +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_q_results, + torch::Tensor const &input_lin_kv_results, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob); + +std::vector +fwd(bool use_mask, bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs_q, torch::Tensor const &inputs_kv, + torch::Tensor const &input_weights_q, torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(inputs_q.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs_kv.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights_q.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(input_weights_kv.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + + TORCH_CHECK(inputs_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + } + + return fwd_cuda(use_time_mask, is_training, heads, inputs_q, inputs_kv, + input_weights_q, input_weights_kv, output_weights, + use_mask ? static_cast(pad_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +std::vector +bwd(int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_q_results, + torch::Tensor const &input_lin_kv_results, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(matmul2_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_q_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_kv_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs_q.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs_kv.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights_q.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(input_weights_kv.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(matmul2_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_q_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_kv_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, matmul2_results, dropout_results, + softmax_results, input_lin_q_results, input_lin_kv_results, + inputs_q, inputs_kv, input_weights_q, input_weights_kv, + output_weights, dropout_mask, dropout_prob); +} + +} // end namespace cublas_gemmex +} // end namespace encdec + +namespace encdec_norm_add { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights_q, + torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob); + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_q_results, + torch::Tensor const &input_lin_kv_results, + torch::Tensor const &lyr_nrm_results, torch::Tensor const &lyr_nrm_mean, + torch::Tensor const &lyr_nrm_invvar, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights_q, torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, torch::Tensor const &dropout_mask, + torch::Tensor const &dropout_add_mask, float dropout_prob); + +std::vector +fwd(bool use_mask, bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs_q, torch::Tensor const &inputs_kv, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights_q, torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(inputs_q.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs_kv.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_gamma_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(lyr_nrm_beta_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(input_weights_q.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(input_weights_kv.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + + TORCH_CHECK(inputs_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_gamma_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_beta_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + } + + return fwd_cuda(use_time_mask, is_training, heads, inputs_q, inputs_kv, + lyr_nrm_gamma_weights, lyr_nrm_beta_weights, input_weights_q, + input_weights_kv, output_weights, + use_mask ? static_cast(pad_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +std::vector +bwd(int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_q_results, + torch::Tensor const &input_lin_kv_results, + torch::Tensor const &lyr_nrm_results, torch::Tensor const &lyr_nrm_mean, + torch::Tensor const &lyr_nrm_invvar, torch::Tensor const &inputs_q, + torch::Tensor const &inputs_kv, torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights_q, torch::Tensor const &input_weights_kv, + torch::Tensor const &output_weights, torch::Tensor const &dropout_mask, + torch::Tensor const &dropout_add_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(matmul2_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_q_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_kv_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_mean.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(lyr_nrm_invvar.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(inputs_q.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs_kv.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_gamma_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(lyr_nrm_beta_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(input_weights_q.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(input_weights_kv.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_add_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(matmul2_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_q_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_kv_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_mean.scalar_type() == at::ScalarType::Float, + "Only FLOAT is supported"); + TORCH_CHECK(lyr_nrm_invvar.scalar_type() == at::ScalarType::Float, + "Only FLOAT is supported"); + TORCH_CHECK(inputs_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_gamma_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_beta_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_q.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights_kv.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + TORCH_CHECK(dropout_add_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, matmul2_results, dropout_results, + softmax_results, input_lin_q_results, input_lin_kv_results, + lyr_nrm_results, lyr_nrm_mean, lyr_nrm_invvar, inputs_q, + inputs_kv, lyr_nrm_gamma_weights, lyr_nrm_beta_weights, + input_weights_q, input_weights_kv, output_weights, + dropout_mask, dropout_add_mask, dropout_prob); +} + +} // end namespace cublas_gemmex +} // end namespace encdec_norm_add + +namespace self { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs, + torch::Tensor const &input_weights, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob); + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob); + +std::vector +fwd(bool use_mask, bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs, torch::Tensor const &input_weights, + torch::Tensor const &output_weights, torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + } + + return fwd_cuda( + use_time_mask, is_training, heads, inputs, input_weights, output_weights, + use_mask ? static_cast(pad_mask.data_ptr()) : nullptr, + dropout_prob); +} + +std::vector +bwd(int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(matmul2_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(matmul2_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, matmul2_results, dropout_results, + softmax_results, input_lin_results, inputs, input_weights, + output_weights, dropout_mask, dropout_prob); +} + +} // end namespace cublas_gemmex +} // end namespace self +namespace self_bias { +namespace cublas_gemmex { + +std::vector +fwd_cuda(bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs, torch::Tensor const &input_weights, + torch::Tensor const &output_weights, torch::Tensor const &input_biases, + torch::Tensor const &output_biases, const uint8_t *pad_mask, + float dropout_prob); + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + // torch::Tensor const& input_biases, + // torch::Tensor const& output_biases, + torch::Tensor const &dropout_mask, float dropout_prob); + +std::vector +fwd(bool use_mask, bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs, torch::Tensor const &input_weights, + torch::Tensor const &output_weights, torch::Tensor const &input_biases, + torch::Tensor const &output_biases, torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + } + + return fwd_cuda(use_time_mask, is_training, heads, inputs, input_weights, + output_weights, input_biases, output_biases, + use_mask ? static_cast(pad_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +std::vector +bwd(int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(matmul2_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(matmul2_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, matmul2_results, dropout_results, + softmax_results, input_lin_results, inputs, input_weights, + output_weights, dropout_mask, dropout_prob); +} + +} // end namespace cublas_gemmex +} // namespace self_bias +namespace self_bias_additive_mask { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs, + torch::Tensor const &input_weights, + torch::Tensor const &output_weights, + torch::Tensor const &input_biases, + torch::Tensor const &output_biases, + const half *pad_mask, float dropout_prob); + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + // torch::Tensor const& softmax_results, + torch::Tensor const &bmm1_results, torch::Tensor const &pad_mask, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + // torch::Tensor const& input_biases, + // torch::Tensor const& output_biases, + torch::Tensor const &dropout_mask, float dropout_prob); + +std::vector +fwd(bool use_mask, bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs, torch::Tensor const &input_weights, + torch::Tensor const &output_weights, torch::Tensor const &input_biases, + torch::Tensor const &output_biases, torch::Tensor const &pad_mask, + float dropout_prob) { + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(use_mask, "no mask is not supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Half, + "Only Half is supported"); + } + + return fwd_cuda(use_time_mask, is_training, heads, inputs, input_weights, + output_weights, input_biases, output_biases, + use_mask ? static_cast(pad_mask.data_ptr()) + : nullptr, + dropout_prob); +} + +std::vector +bwd(int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &bmm1_results, torch::Tensor const &pad_mask, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(matmul2_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(matmul2_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, matmul2_results, dropout_results, + bmm1_results, pad_mask, input_lin_results, inputs, + input_weights, output_weights, dropout_mask, dropout_prob); +} + +} // end namespace cublas_gemmex +} // namespace self_bias_additive_mask + +namespace self_norm_add { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob); + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, + torch::Tensor const &lyr_nrm_results, torch::Tensor const &lyr_nrm_mean, + torch::Tensor const &lyr_nrm_invvar, torch::Tensor const &inputs, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, torch::Tensor const &dropout_add_mask, + float dropout_prob); + +std::vector +fwd(bool use_mask, bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs, torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &pad_mask, float dropout_prob) { + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_gamma_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(lyr_nrm_beta_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_gamma_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_beta_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + + if (use_mask) { + TORCH_CHECK(pad_mask.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(pad_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + } + + return fwd_cuda( + use_time_mask, is_training, heads, inputs, lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, input_weights, output_weights, + use_mask ? static_cast(pad_mask.data_ptr()) : nullptr, + dropout_prob); +} + +std::vector +bwd(int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, + torch::Tensor const &lyr_nrm_results, torch::Tensor const &lyr_nrm_mean, + torch::Tensor const &lyr_nrm_invvar, torch::Tensor const &inputs, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, torch::Tensor const &dropout_add_mask, + float dropout_prob) { + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(matmul2_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(input_lin_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_results.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_mean.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(lyr_nrm_invvar.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(inputs.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(lyr_nrm_gamma_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(lyr_nrm_beta_weights.dim() == 1, "expected 1D tensor"); + TORCH_CHECK(input_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(output_weights.dim() == 2, "expected 2D tensor"); + TORCH_CHECK(dropout_mask.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(dropout_add_mask.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK(output_grads.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(matmul2_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(softmax_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_lin_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_results.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_mean.scalar_type() == at::ScalarType::Float, + "Only FLOAT is supported"); + TORCH_CHECK(lyr_nrm_invvar.scalar_type() == at::ScalarType::Float, + "Only FLOAT is supported"); + TORCH_CHECK(inputs.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_gamma_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(lyr_nrm_beta_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(input_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(output_weights.scalar_type() == at::ScalarType::Half, + "Only HALF is supported"); + TORCH_CHECK(dropout_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + TORCH_CHECK(dropout_add_mask.scalar_type() == at::ScalarType::Byte, + "Only BYTE is supported"); + + return bwd_cuda(heads, output_grads, matmul2_results, dropout_results, + softmax_results, input_lin_results, lyr_nrm_results, + lyr_nrm_mean, lyr_nrm_invvar, inputs, lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, input_weights, output_weights, + dropout_mask, dropout_add_mask, dropout_prob); +} + +} // end namespace cublas_gemmex +} // end namespace self_norm_add +} // end namespace multihead_attn + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("additive_mask_softmax_dropout_forward", + &multihead_attn::fused_softmax::additive_mask_softmax_dropout::fwd, + "Self Multihead Attention masked softmax dropout -- Forward."); + m.def("additive_mask_softmax_dropout_backward", + &multihead_attn::fused_softmax::additive_mask_softmax_dropout::bwd, + "Self Multihead Attention masked softmax dropout -- Backward."); + m.def("mask_softmax_dropout_forward", &multihead_attn::fused_softmax::mask_softmax_dropout::fwd, + "Self Multihead Attention masked softmax dropout -- Forward."); + m.def("mask_softmax_dropout_backward", &multihead_attn::fused_softmax::mask_softmax_dropout::bwd, + "Self Multihead Attention masked softmax dropout -- Backward."); + m.def("encdec_multihead_attn_forward", &multihead_attn::encdec::cublas_gemmex::fwd, + "Encdec Multihead Attention Forward."); + m.def("encdec_multihead_attn_backward", &multihead_attn::encdec::cublas_gemmex::bwd, + "Encdec Multihead Attention Backward."); + m.def("encdec_multihead_attn_norm_add_forward", &multihead_attn::encdec_norm_add::cublas_gemmex::fwd, + "Encdec Multihead Attention Plus Layer Norm and Residual Add Forward."); + m.def( + "encdec_multihead_attn_norm_add_backward", &multihead_attn::encdec_norm_add::cublas_gemmex::bwd, + "Encdec Multihead Attention Plus Layer Norm and Residual Add Backward."); + m.def("self_attn_forward", &multihead_attn::self::cublas_gemmex::fwd, + "Self Multihead Attention Forward."); + m.def("self_attn_backward", &multihead_attn::self::cublas_gemmex::bwd, + "Self Multihead Attention Backward."); + m.def("self_attn_bias_forward", &multihead_attn::self_bias::cublas_gemmex::fwd, + "Self Multihead Attention with Bias -- Forward."); + m.def("self_attn_bias_backward", &multihead_attn::self_bias::cublas_gemmex::bwd, + "Self Multihead Attention with Bias -- Backward."); + m.def("self_attn_bias_additive_mask_forward", &multihead_attn::self_bias_additive_mask::cublas_gemmex::fwd, + "Self Multihead Attention with Bias -- Forward."); + m.def("self_attn_bias_additive_mask_backward", + &multihead_attn::self_bias_additive_mask::cublas_gemmex::bwd, + "Self Multihead Attention with Bias -- Backward."); + m.def("self_attn_norm_add_forward", &multihead_attn::self_norm_add::cublas_gemmex::fwd, + "Self Multihead Attention Plus Layer Norm and Residual Add Forward."); + m.def("self_attn_norm_add_backward", &multihead_attn::self_norm_add::cublas_gemmex::bwd, + "Self Multihead Attention Plus Layer Norm and Residual Add Backward."); +} + +#undef CHECK_CUDA +#undef CHECK_CONTIGUOUS +#undef CHECK_INPUT diff --git a/apex/apex/contrib/csrc/multihead_attn/philox.cuh b/apex/apex/contrib/csrc/multihead_attn/philox.cuh new file mode 100644 index 00000000..d2076ab5 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/philox.cuh @@ -0,0 +1,126 @@ +#pragma once +// Philox CUDA. + +namespace { + +class Philox { +public: + __device__ inline Philox(unsigned long long seed, + unsigned long long subsequence, + unsigned long long offset) : STATE(0) { + //key.x = (unsigned int)seed; + //key.y = (unsigned int)(seed >> 32); + //counter = make_uint4(0, 0, 0, 0); + //counter.z = (unsigned int)(subsequence); + //counter.w = (unsigned int)(subsequence >> 32); + //STATE = 0; + //incr_n(offset / 4); + + key = reinterpret_cast(seed); + ull2 * tmp = reinterpret_cast(&counter); + tmp->x = offset / 4; + tmp->y = subsequence; + } + __device__ inline uint4 operator()() { + if (STATE == 0) { + uint4 counter_ = counter; + uint2 key_ = key; + // 7-round philox + for (int i = 0; i < 6; i++) { + counter_ = single_round(counter_, key_); + key_.x += (kPhilox10A); + key_.y += (kPhilox10B); + } + output = single_round(counter_, key_); + incr(); + } + // return a float4 directly + // unsigned long ret; + // switch(STATE) { + // case 0: ret = output.x; break; + // case 1: ret = output.y; break; + // case 2: ret = output.z; break; + // case 3: ret = output.w; break; + //} + // STATE = (STATE + 1) % 4; + return output; + } + +private: + struct ull2 { + uint64_t x; + uint64_t y; + }; + uint4 counter; + uint4 output; + uint2 key; + unsigned int STATE; + __device__ inline void incr_n(unsigned long long n) { + unsigned int nlo = (unsigned int)(n); + unsigned int nhi = (unsigned int)(n >> 32); + counter.x += nlo; + if (counter.x < nlo) + nhi++; + counter.y += nhi; + if (nhi <= counter.y) + return; + if (++counter.z) + return; + ++counter.w; + } + + __device__ uint4 incr128 (uint4 ctr) + { + uint4 res; + asm ("add.cc.u32 %0, %4, %8;\n\t" + "addc.cc.u32 %1, %5, %9;\n\t" + "addc.cc.u32 %2, %6, %10;\n\t" + "addc.u32 %3, %7, %11;\n\t" + : "=r"(res.x), "=r"(res.y), "=r"(res.z), "=r"(res.w) + : "r"(ctr.x), "r"(ctr.y), "r"(ctr.z), "r"(ctr.w), + "n"(1), "n"(0), "n"(0), "n"(0)); + return res; + } + + __device__ inline void incr() { + counter = incr128(counter); + } + __device__ unsigned int mulhilo32(unsigned int a, unsigned int b, + unsigned int *result_high) { + *result_high = __umulhi(a, b); + return a * b; + } + __device__ uint2 mulhilo32_v2 (unsigned int a, unsigned int b) + { + uint2 *res; + unsigned long long tmp; + asm ("mul.wide.u32 %0, %1, %2;\n\t" + : "=l"(tmp) + : "r"(a), "r"(b)); + res = (uint2*)(&tmp); + return *res; + } + __device__ inline uint4 single_round(uint4 ctr, uint2 key) { + //unsigned int hi0; + //unsigned int hi1; + //unsigned int lo0 = mulhilo32(kPhiloxSA, ctr.x, &hi0); + //unsigned int lo1 = mulhilo32(kPhiloxSB, ctr.z, &hi1); + //uint4 ret = {hi1 ^ ctr.y ^ key.x, lo1, hi0 ^ ctr.w ^ key.y, lo0}; + uint2 res0 = mulhilo32_v2(kPhiloxSA, ctr.x); + uint2 res1 = mulhilo32_v2(kPhiloxSB, ctr.z); + uint4 ret = {res1.y ^ ctr.y ^ key.x, res1.x, res0.y ^ ctr.w ^ key.y, res0.x}; + return ret; + } + static const unsigned long kPhilox10A = 0x9E3779B9; + static const unsigned long kPhilox10B = 0xBB67AE85; + static const unsigned long kPhiloxSA = 0xD2511F53; + static const unsigned long kPhiloxSB = 0xCD9E8D57; +}; +// Inverse of 2^32. +constexpr float M_RAN_INVM32 = 2.3283064e-10f; +__device__ __inline__ float4 uniform4(uint4 x) { + return make_float4(x.x * M_RAN_INVM32, x.y * M_RAN_INVM32, x.z * M_RAN_INVM32, + x.w * M_RAN_INVM32); +} + +} // namespace diff --git a/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_additive_mask_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_additive_mask_cuda.cu new file mode 100644 index 00000000..c3a090c2 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_additive_mask_cuda.cu @@ -0,0 +1,306 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "softmax.cuh" +#include "strided_batched_gemm.cuh" + +namespace multihead_attn { +namespace self_bias_additive_mask { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs, + torch::Tensor const &input_weights, + torch::Tensor const &output_weights, + torch::Tensor const &input_biases, + torch::Tensor const &output_biases, + const half *pad_mask, float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + [[maybe_unused]] const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta_zero = 0.0; + const float beta_one = 1.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = inputs.options().requires_grad(false); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor input_lin_results = + torch::empty({q_seq_len, sequences, output_lin_dim}, act_options); + torch::Tensor bmm1_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + torch::Tensor matmul2_results = + torch::empty({q_seq_len, attn_batches, head_dim}, act_options); + torch::Tensor outputs = torch::empty_like(inputs, act_options); + + // Input Linear Results Pointers to Q, K, and V of interviewed activations + void *q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + void *k_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + head_dim); + void *v_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + 2 * head_dim); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *bmm1_results_ptr = static_cast(bmm1_results.data_ptr()); + void *dropout_results_ptr = static_cast(dropout_results.data_ptr()); + + char a_layout_t{'t'}; + char a_layout_n{'n'}; + char b_layout_n{'n'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + // Input Linear Fwd + input_lin_results.copy_(input_biases); + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(inputs.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta_one), q_lin_results_ptr, + CUDA_R_16F, output_lin_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul1 of Dot-Product Attention Plus scaling by 1/Sqrt(head size) + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, scale, + static_cast(k_lin_results_ptr), lead_dim, batch_stride, + static_cast(q_lin_results_ptr), lead_dim, batch_stride, + beta_zero, static_cast(bmm1_results_ptr), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + // Padded Softmax + [[maybe_unused]] bool softmax_success = false; + if (is_training) { + softmax_success = + dispatch_additive_masked_softmax_dropout( + reinterpret_cast(dropout_results_ptr), + (is_training) + ? reinterpret_cast(dropout_mask.data_ptr()) + : nullptr, + reinterpret_cast(bmm1_results_ptr), pad_mask, + attn_batches * q_seq_len * q_seq_len, k_seq_len, k_seq_len, + attn_batches * q_seq_len, attn_batches * q_seq_len / sequences, + 1.0f - dropout_prob, stream); + } else { + softmax_success = dispatch_additive_masked_softmax( + reinterpret_cast( + dropout_results_ptr), // this is actually softmax results, but + // making it consistent for the next function + reinterpret_cast(bmm1_results_ptr), pad_mask, k_seq_len, + k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + + // Matmul2 + gemm_switch_fp32accum( + a_layout_n, b_layout_n, head_dim, q_seq_len, k_seq_len, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + static_cast(dropout_results.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, beta_zero, + static_cast(matmul2_results.data_ptr()), head_dim * attn_batches, + head_dim, attn_batches); + + outputs.copy_(output_biases); + + // Output Linear + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(matmul2_results.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta_one), + static_cast(outputs.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO1_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_lin_results, bmm1_results, dropout_results, + dropout_mask, matmul2_results, outputs}; +} + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &bmm1_results, torch::Tensor const &pad_mask, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + [[maybe_unused]] const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + torch::Tensor input_grads = torch::empty_like(inputs); + torch::Tensor input_weight_grads = torch::empty_like(input_weights); + torch::Tensor output_weight_grads = torch::empty_like(output_weights); + // Intermediate Tensor Allocations + at::Tensor output_lin_grads = torch::empty_like(matmul2_results); + at::Tensor matmul2_grads = torch::empty_like(dropout_results); + at::Tensor input_lin_output_grads = torch::empty_like(input_lin_results); + + auto q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + auto k_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + head_dim; + auto v_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + 2 * head_dim; + + auto q_lin_grads_ptr = static_cast(input_lin_output_grads.data_ptr()); + auto k_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + head_dim; + auto v_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + 2 * head_dim; + + char a_layout_n{'n'}; + char a_layout_t{'t'}; + char b_layout_n{'n'}; + char b_layout_t{'t'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + + // Output Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + // Output Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, embed_dim, batches, + static_cast(&alpha), + static_cast(matmul2_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_weight_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + auto output_bias_grads = output_grads.view({-1, embed_dim}).sum(0, false); + // MatMul2 Dgrad1 + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, beta, + static_cast(matmul2_grads.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Matmul2 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, alpha, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, + static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, v_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Apply Dropout Mask and Scale by Dropout Probability + // Softmax Grad + dispatch_masked_scale_softmax_backward_recompute( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + reinterpret_cast(bmm1_results.data_ptr()), + reinterpret_cast(pad_mask.data_ptr()), + static_cast(dropout_mask.data_ptr()), + 1.0 / (1.0 - dropout_prob), k_seq_len, k_seq_len, + attn_batches * q_seq_len / sequences, attn_batches * q_seq_len, stream); + + // Matmul1 Dgrad1 + gemm_switch_fp32accum(a_layout_n, b_layout_n, head_dim, q_seq_len, + k_seq_len, scale, k_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, q_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Matmul1 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, scale, q_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, k_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + // Input Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, output_lin_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(input_lin_output_grads.data_ptr()), + // static_cast(q_lin_grads_ptr), + CUDA_R_16F, output_lin_dim, static_cast(&beta), + static_cast(input_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_dim, batches, + static_cast(&alpha), + static_cast(inputs.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(q_lin_grads_ptr), CUDA_R_16F, output_lin_dim, + static_cast(&beta), + static_cast(input_weight_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + auto input_bias_grads = + input_lin_output_grads.view({-1, output_lin_dim}).sum(0, false); + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_grads, input_weight_grads, output_weight_grads, + input_bias_grads, output_bias_grads}; +} + +} // end namespace cublas_gemmex +} // namespace self_bias_additive_mask +} // end namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_cuda.cu new file mode 100644 index 00000000..e9b3e9e9 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_cuda.cu @@ -0,0 +1,312 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "softmax.cuh" +#include "strided_batched_gemm.cuh" + +namespace multihead_attn { +namespace self_bias { +namespace cublas_gemmex { + +std::vector +fwd_cuda(bool use_time_mask, bool is_training, int heads, + torch::Tensor const &inputs, torch::Tensor const &input_weights, + torch::Tensor const &output_weights, torch::Tensor const &input_biases, + torch::Tensor const &output_biases, const uint8_t *pad_mask, + float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + [[maybe_unused]] const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta_zero = 0.0; + const float beta_one = 1.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = inputs.options().requires_grad(false); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor input_lin_results = + torch::empty({q_seq_len, sequences, output_lin_dim}, act_options); + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + torch::Tensor matmul2_results = + torch::empty({q_seq_len, attn_batches, head_dim}, act_options); + torch::Tensor outputs = torch::empty_like(inputs, act_options); + + // Input Linear Results Pointers to Q, K, and V of interviewed activations + void *q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + void *k_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + head_dim); + void *v_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + 2 * head_dim); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + char a_layout_t{'t'}; + char a_layout_n{'n'}; + char b_layout_n{'n'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + // Input Linear Fwd + input_lin_results.copy_(input_biases); + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(inputs.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta_one), q_lin_results_ptr, + CUDA_R_16F, output_lin_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul1 of Dot-Product Attention Plus scaling by 1/Sqrt(head size) + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, scale, + static_cast(k_lin_results_ptr), lead_dim, batch_stride, + static_cast(q_lin_results_ptr), lead_dim, batch_stride, + beta_zero, static_cast(softmax_results_ptr), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + // Padded Softmax + [[maybe_unused]] bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + } else { + if (use_time_mask) { + softmax_success = dispatch_time_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, q_seq_len); + } else { + softmax_success = dispatch_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + } + + if (is_training) { + // use at:: function so that C++ version generates the same random mask as + // python version + auto dropout_tuple = + at::_fused_dropout(softmax_results, 1.0f - dropout_prob); + dropout_results = std::get<0>(dropout_tuple); + dropout_mask = std::get<1>(dropout_tuple); + } + + // Matmul2 + gemm_switch_fp32accum( + a_layout_n, b_layout_n, head_dim, q_seq_len, k_seq_len, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + (is_training) ? static_cast(dropout_results.data_ptr()) + : static_cast(softmax_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta_zero, + static_cast(matmul2_results.data_ptr()), head_dim * attn_batches, + head_dim, attn_batches); + + outputs.copy_(output_biases); + + // Output Linear + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(matmul2_results.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta_one), + static_cast(outputs.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO1_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_lin_results, softmax_results, dropout_results, + dropout_mask, matmul2_results, outputs}; +} + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + [[maybe_unused]] const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + torch::Tensor input_grads = torch::empty_like(inputs); + torch::Tensor input_weight_grads = torch::empty_like(input_weights); + torch::Tensor output_weight_grads = torch::empty_like(output_weights); + // Intermediate Tensor Allocations + at::Tensor output_lin_grads = torch::empty_like(matmul2_results); + at::Tensor matmul2_grads = torch::empty_like(dropout_results); + at::Tensor input_lin_output_grads = torch::empty_like(input_lin_results); + + auto q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + auto k_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + head_dim; + auto v_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + 2 * head_dim; + + auto q_lin_grads_ptr = static_cast(input_lin_output_grads.data_ptr()); + auto k_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + head_dim; + auto v_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + 2 * head_dim; + + char a_layout_n{'n'}; + char a_layout_t{'t'}; + char b_layout_n{'n'}; + char b_layout_t{'t'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + + // Output Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + // Output Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, embed_dim, batches, + static_cast(&alpha), + static_cast(matmul2_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_weight_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + auto output_bias_grads = output_grads.view({-1, embed_dim}).sum(0, false); + // MatMul2 Dgrad1 + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, beta, + static_cast(matmul2_grads.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Matmul2 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, alpha, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, + static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, v_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Apply Dropout Mask and Scale by Dropout Probability + // Softmax Grad + dispatch_masked_scale_softmax_backward_stream( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), + 1.0 / (1.0 - dropout_prob), k_seq_len, k_seq_len, + attn_batches * q_seq_len, stream); + + // Matmul1 Dgrad1 + gemm_switch_fp32accum(a_layout_n, b_layout_n, head_dim, q_seq_len, + k_seq_len, scale, k_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, q_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Matmul1 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, scale, q_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, k_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + // Input Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, output_lin_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(input_lin_output_grads.data_ptr()), + // static_cast(q_lin_grads_ptr), + CUDA_R_16F, output_lin_dim, static_cast(&beta), + static_cast(input_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_dim, batches, + static_cast(&alpha), + static_cast(inputs.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(q_lin_grads_ptr), CUDA_R_16F, output_lin_dim, + static_cast(&beta), + static_cast(input_weight_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + auto input_bias_grads = + input_lin_output_grads.view({-1, output_lin_dim}).sum(0, false); + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_grads, input_weight_grads, output_weight_grads, + input_bias_grads, output_bias_grads}; +} + +} // end namespace cublas_gemmex +} // namespace self_bias +} // end namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu new file mode 100644 index 00000000..9701da7c --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu @@ -0,0 +1,307 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "softmax.cuh" +#include "strided_batched_gemm.cuh" + +namespace multihead_attn { +namespace self { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs, + torch::Tensor const &input_weights, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = inputs.options().requires_grad(false); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor input_lin_results = + torch::empty({q_seq_len, sequences, output_lin_dim}, act_options); + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + torch::Tensor matmul2_results = + torch::empty({q_seq_len, attn_batches, head_dim}, act_options); + torch::Tensor outputs = torch::empty_like(inputs, act_options); + + // Input Linear Results Pointers to Q, K, and V of interviewed activations + void *q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + void *k_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + head_dim); + void *v_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + 2 * head_dim); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + char a_layout_t{'t'}; + char a_layout_n{'n'}; + char b_layout_n{'n'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + // Input Linear Fwd + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(inputs.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), q_lin_results_ptr, + CUDA_R_16F, output_lin_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul1 of Dot-Product Attention Plus scaling by 1/Sqrt(head size) + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, scale, + static_cast(k_lin_results_ptr), lead_dim, batch_stride, + static_cast(q_lin_results_ptr), lead_dim, batch_stride, + beta, static_cast(softmax_results_ptr), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Padded Softmax + bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + } else { + if (use_time_mask) { + softmax_success = dispatch_time_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, q_seq_len); + } else { + softmax_success = dispatch_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + } + assert(softmax_success); + + if (is_training) { + apex_fused_dropout_cuda( + static_cast(softmax_results.data_ptr()), + static_cast(dropout_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0f - dropout_prob)); + } + + // Matmul2 + gemm_switch_fp32accum( + a_layout_n, b_layout_n, head_dim, q_seq_len, k_seq_len, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + (is_training) ? static_cast(dropout_results.data_ptr()) + : static_cast(softmax_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, + static_cast(matmul2_results.data_ptr()), head_dim * attn_batches, + head_dim, attn_batches); + + // Output Linear + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(matmul2_results.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(outputs.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_lin_results, softmax_results, dropout_results, + dropout_mask, matmul2_results, outputs}; +} + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, torch::Tensor const &inputs, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + torch::Tensor input_grads = torch::empty_like(inputs); + torch::Tensor input_weight_grads = torch::empty_like(input_weights); + torch::Tensor output_weight_grads = torch::empty_like(output_weights); + // Intermediate Tensor Allocations + at::Tensor output_lin_grads = torch::empty_like(matmul2_results); + at::Tensor matmul2_grads = torch::empty_like(dropout_results); + at::Tensor input_lin_output_grads = torch::empty_like(input_lin_results); + + auto q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + auto k_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + head_dim; + auto v_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + 2 * head_dim; + + auto q_lin_grads_ptr = static_cast(input_lin_output_grads.data_ptr()); + auto k_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + head_dim; + auto v_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + 2 * head_dim; + + char a_layout_n{'n'}; + char a_layout_t{'t'}; + char b_layout_n{'n'}; + char b_layout_t{'t'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + + // Output Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Output Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, embed_dim, batches, + static_cast(&alpha), + static_cast(matmul2_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(output_grads.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), + static_cast(output_weight_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul2 Dgrad1 + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, beta, + static_cast(matmul2_grads.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Matmul2 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, alpha, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, + static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, v_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Apply Dropout Mask and Scale by Dropout Probability + apex_masked_scale_cuda( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0 / (1.0 - dropout_prob))); + + // Softmax Grad + bool softmax_success = false; + softmax_success = dispatch_softmax_backward( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + assert(softmax_success); + + // Matmul1 Dgrad1 + gemm_switch_fp32accum(a_layout_n, b_layout_n, head_dim, q_seq_len, + k_seq_len, scale, k_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, q_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Matmul1 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, scale, q_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, k_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Input Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, output_lin_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(q_lin_grads_ptr), CUDA_R_16F, + output_lin_dim, static_cast(&beta), + static_cast(input_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_dim, batches, + static_cast(&alpha), + static_cast(inputs.data_ptr()), CUDA_R_16F, embed_dim, + static_cast(q_lin_grads_ptr), CUDA_R_16F, output_lin_dim, + static_cast(&beta), + static_cast(input_weight_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_grads, input_weight_grads, output_weight_grads}; +} + +} // end namespace cublas_gemmex +} // end namespace self +} // end namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu new file mode 100644 index 00000000..7a6ec15c --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu @@ -0,0 +1,385 @@ +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include "dropout.cuh" +#include "layer_norm.cuh" +#include "softmax.cuh" +#include "strided_batched_gemm.cuh" + +namespace multihead_attn { +namespace self_norm_add { +namespace cublas_gemmex { + +std::vector fwd_cuda(bool use_time_mask, bool is_training, + int heads, torch::Tensor const &inputs, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights, + torch::Tensor const &output_weights, + const uint8_t *pad_mask, + float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int total_tokens = batches * embed_dim; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // There is no reason to use more than one stream as every kernel is + // sequentially dependent + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // 3 Intermediate Results + Output (Note: dropout intermediates are generated + // by ATen library code) + auto act_options = inputs.options().requires_grad(false); + auto lyr_nrm_options = act_options.dtype(torch::kFloat32); + auto mask_options = act_options.dtype(torch::kUInt8); + + torch::Tensor lyr_nrm_mean = torch::empty({batches}, lyr_nrm_options); + torch::Tensor lyr_nrm_invvar = torch::empty({batches}, lyr_nrm_options); + torch::Tensor lyr_nrm_results = torch::empty_like(inputs, act_options); + + torch::Tensor input_lin_results = + torch::empty({q_seq_len, sequences, output_lin_dim}, act_options); + torch::Tensor softmax_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_results = + torch::empty({attn_batches, q_seq_len, k_seq_len}, act_options); + torch::Tensor dropout_mask = + torch::empty({attn_batches, q_seq_len, k_seq_len}, mask_options); + torch::Tensor matmul2_results = + torch::empty({q_seq_len, attn_batches, head_dim}, act_options); + torch::Tensor output_lin_results = torch::empty_like(inputs, act_options); + torch::Tensor dropout_add_mask = torch::empty_like(inputs, mask_options); + torch::Tensor outputs = torch::empty_like(inputs, act_options); + + // Input Linear Results Pointers to Q, K, and V of interviewed activations + void *q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + void *k_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + head_dim); + void *v_lin_results_ptr = static_cast( + static_cast(input_lin_results.data_ptr()) + 2 * head_dim); + + // Softmax Intermediate Result Ptr (used by Matmul1 -> Softmax) + void *softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + char a_layout_t{'t'}; + char a_layout_n{'n'}; + char b_layout_n{'n'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + // Layer Norm + HostApplyLayerNorm( + static_cast(lyr_nrm_results.data_ptr()), + static_cast(lyr_nrm_mean.data_ptr()), + static_cast(lyr_nrm_invvar.data_ptr()), + static_cast(inputs.data_ptr()), + static_cast(batches), // n1 + static_cast(embed_dim), // n2 + 1.0e-5, static_cast(lyr_nrm_gamma_weights.data_ptr()), + static_cast(lyr_nrm_beta_weights.data_ptr())); + + // Input Linear Fwd + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, output_lin_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, + // static_cast(inputs.data_ptr()), + static_cast(lyr_nrm_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(&beta), q_lin_results_ptr, + CUDA_R_16F, output_lin_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul1 of Dot-Product Attention Plus scaling by 1/Sqrt(head size) + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, scale, + static_cast(k_lin_results_ptr), lead_dim, batch_stride, + static_cast(q_lin_results_ptr), lead_dim, batch_stride, + beta, static_cast(softmax_results_ptr), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Padded Softmax + bool softmax_success = false; + if (pad_mask == nullptr) { + softmax_success = dispatch_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + } else { + if (use_time_mask) { + softmax_success = dispatch_time_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, q_seq_len); + } else { + softmax_success = dispatch_masked_softmax( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(softmax_results_ptr), pad_mask, + k_seq_len, k_seq_len, attn_batches * q_seq_len, + attn_batches * q_seq_len / sequences); + } + } + assert(softmax_success); + + if (is_training) { + apex_fused_dropout_cuda( + static_cast(softmax_results.data_ptr()), + static_cast(dropout_results.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0f - dropout_prob)); + } + + // Matmul2 + gemm_switch_fp32accum( + a_layout_n, b_layout_n, head_dim, q_seq_len, k_seq_len, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + (is_training) ? static_cast(dropout_results.data_ptr()) + : static_cast(softmax_results.data_ptr()), + // static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, + static_cast(matmul2_results.data_ptr()), head_dim * attn_batches, + head_dim, attn_batches); + + // Output Linear + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_T, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(matmul2_results.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(output_lin_results.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // End-of-block Dropout-Add + if (is_training) { + apex_dropout_add_cuda( + static_cast(output_lin_results.data_ptr()), + static_cast(inputs.data_ptr()), + static_cast(outputs.data_ptr()), + static_cast(dropout_add_mask.data_ptr()), total_tokens, + (1.0f - dropout_prob)); + } else { + apex_add_cuda( + static_cast(output_lin_results.data_ptr()), + static_cast(inputs.data_ptr()), + static_cast(outputs.data_ptr()), total_tokens); + } + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {lyr_nrm_results, lyr_nrm_mean, lyr_nrm_invvar, input_lin_results, + softmax_results, dropout_results, dropout_mask, matmul2_results, + dropout_add_mask, outputs}; +} + +std::vector bwd_cuda( + int heads, torch::Tensor const &output_grads, + torch::Tensor const &matmul2_results, torch::Tensor const &dropout_results, + torch::Tensor const &softmax_results, + torch::Tensor const &input_lin_results, + torch::Tensor const &lyr_nrm_results, torch::Tensor const &lyr_nrm_mean, + torch::Tensor const &lyr_nrm_invvar, torch::Tensor const &inputs, + torch::Tensor const &lyr_nrm_gamma_weights, + torch::Tensor const &lyr_nrm_beta_weights, + torch::Tensor const &input_weights, torch::Tensor const &output_weights, + torch::Tensor const &dropout_mask, torch::Tensor const &dropout_add_mask, + float dropout_prob) { + const int embed_dim = inputs.size(2); + const int sequences = inputs.size(1); + const int q_seq_len = inputs.size(0); + const int k_seq_len = q_seq_len; + const int batches = sequences * q_seq_len; + const int total_tokens = batches * embed_dim; + const int head_dim = embed_dim / heads; + const int output_lin_dim = 3 * embed_dim; + const int attn_batches = heads * sequences; + const int lead_dim = attn_batches * 3 * head_dim; + const int batch_stride = 3 * head_dim; + const int dropout_elems = attn_batches * q_seq_len * k_seq_len; + const float alpha = 1.0; + const float beta = 0.0; + const float scale = 1.0 / sqrt(static_cast(head_dim)); + + // TODO: Streams can be used in Backprop but I haven't added more than one + // in my first attempt to create the code + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + + // Output Tensor Allocations + torch::Tensor input_grads = torch::empty_like(inputs); + torch::Tensor lyr_nrm_gamma_grads = torch::empty_like(lyr_nrm_gamma_weights); + torch::Tensor lyr_nrm_beta_grads = torch::empty_like(lyr_nrm_beta_weights); + torch::Tensor input_weight_grads = torch::empty_like(input_weights); + torch::Tensor output_weight_grads = torch::empty_like(output_weights); + // Intermediate Tensor Allocations + torch::Tensor dropout_add_grads = torch::empty_like(output_grads); + torch::Tensor output_lin_grads = torch::empty_like(matmul2_results); + torch::Tensor matmul2_grads = torch::empty_like(dropout_results); + torch::Tensor input_lin_output_grads = torch::empty_like(input_lin_results); + torch::Tensor input_lin_grads = torch::empty_like(inputs); + + auto q_lin_results_ptr = static_cast(input_lin_results.data_ptr()); + auto k_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + head_dim; + auto v_lin_results_ptr = + static_cast(input_lin_results.data_ptr()) + 2 * head_dim; + + auto q_lin_grads_ptr = static_cast(input_lin_output_grads.data_ptr()); + auto k_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + head_dim; + auto v_lin_grads_ptr = + static_cast(input_lin_output_grads.data_ptr()) + 2 * head_dim; + + char a_layout_n{'n'}; + char a_layout_t{'t'}; + char b_layout_n{'n'}; + char b_layout_t{'t'}; + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_TENSOR_OP_MATH)); + + // Dropout Add Backward + apex_masked_scale_cuda( + static_cast(output_grads.data_ptr()), + static_cast(dropout_add_grads.data_ptr()), + static_cast(dropout_add_mask.data_ptr()), total_tokens, + (1.0 / (1.0 - dropout_prob))); + + // Output Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, embed_dim, + static_cast(&alpha), + static_cast(output_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(dropout_add_grads.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(output_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Output Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, embed_dim, batches, + static_cast(&alpha), + static_cast(matmul2_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(dropout_add_grads.data_ptr()), + CUDA_R_16F, embed_dim, static_cast(&beta), + static_cast(output_weight_grads.data_ptr()), CUDA_R_16F, + embed_dim, CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // MatMul2 Dgrad1 + gemm_switch_fp32accum( + a_layout_t, b_layout_n, k_seq_len, q_seq_len, head_dim, alpha, + static_cast(v_lin_results_ptr), lead_dim, batch_stride, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, beta, + static_cast(matmul2_grads.data_ptr()), k_seq_len, + k_seq_len * q_seq_len, attn_batches); + + // Matmul2 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, alpha, + static_cast(output_lin_grads.data_ptr()), + head_dim * attn_batches, head_dim, + static_cast(dropout_results.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, v_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Apply Dropout Mask and Scale by Dropout Probability + apex_masked_scale_cuda( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + static_cast(dropout_mask.data_ptr()), dropout_elems, + (1.0 / (1.0 - dropout_prob))); + + // Softmax Grad + bool softmax_success = false; + softmax_success = dispatch_softmax_backward( + static_cast(matmul2_grads.data_ptr()), + static_cast(matmul2_grads.data_ptr()), + reinterpret_cast(softmax_results.data_ptr()), k_seq_len, + k_seq_len, attn_batches * q_seq_len); + assert(softmax_success); + + // Matmul1 Dgrad1 + gemm_switch_fp32accum(a_layout_n, b_layout_n, head_dim, q_seq_len, + k_seq_len, scale, k_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, q_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Matmul1 Dgrad2 + gemm_switch_fp32accum(a_layout_n, b_layout_t, head_dim, k_seq_len, + q_seq_len, scale, q_lin_results_ptr, lead_dim, + batch_stride, + static_cast(matmul2_grads.data_ptr()), + k_seq_len, k_seq_len * q_seq_len, beta, k_lin_grads_ptr, + lead_dim, batch_stride, attn_batches); + + // Input Linear Dgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_N, embed_dim, batches, output_lin_dim, + static_cast(&alpha), + static_cast(input_weights.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(q_lin_grads_ptr), CUDA_R_16F, + output_lin_dim, static_cast(&beta), + // static_cast(input_grads.data_ptr()), + static_cast(input_lin_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, + // CUBLAS_GEMM_ALGO10_TENSOR_OP)); + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Input Linear Wgrad + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, CUBLAS_OP_N, CUBLAS_OP_T, embed_dim, output_lin_dim, batches, + static_cast(&alpha), + // static_cast(inputs.data_ptr()), + static_cast(lyr_nrm_results.data_ptr()), CUDA_R_16F, + embed_dim, static_cast(q_lin_grads_ptr), CUDA_R_16F, + output_lin_dim, static_cast(&beta), + static_cast(input_weight_grads.data_ptr()), CUDA_R_16F, embed_dim, + CUDA_R_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP)); + + // Fused Layer Norm Bwd with Residual Add + HostLayerNormGradient( + static_cast(input_lin_grads.data_ptr()), + static_cast(output_grads.data_ptr()), + static_cast(lyr_nrm_mean.data_ptr()), + static_cast(lyr_nrm_invvar.data_ptr()), inputs, + static_cast(batches), // n1 + static_cast(embed_dim), // n2 + static_cast(lyr_nrm_gamma_weights.data_ptr()), + static_cast(lyr_nrm_beta_weights.data_ptr()), 1.0e-5, + static_cast(input_grads.data_ptr()), + static_cast(lyr_nrm_gamma_grads.data_ptr()), + static_cast(lyr_nrm_beta_grads.data_ptr())); + + TORCH_CUDABLAS_CHECK(cublasSetMathMode(handle, CUBLAS_DEFAULT_MATH)); + + return {input_grads, lyr_nrm_gamma_grads, lyr_nrm_beta_grads, + input_weight_grads, output_weight_grads}; +} + +} // end namespace cublas_gemmex +} // end namespace self_norm_add +} // end namespace multihead_attn diff --git a/apex/apex/contrib/csrc/multihead_attn/softmax.cuh b/apex/apex/contrib/csrc/multihead_attn/softmax.cuh new file mode 100644 index 00000000..254e8421 --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/softmax.cuh @@ -0,0 +1,3142 @@ +#pragma once +#include "philox.cuh" +#include +#include + +#ifdef OLD_GENERATOR_PATH +#include +#else +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace { +template +__device__ __inline__ void copy_vector(Datatype *dst, const Datatype *src); + +template +__device__ __inline__ void apply_mask(Datatype *dst, Datatype value, + const uint8_t *src); + +template +__device__ __inline__ void apply_additive_mask(Datatype *dst, + const Datatype *additive_mask); + +template <> +__device__ __inline__ void copy_vector<__half, 1>(__half *dst, + const __half *src) { + *dst = *src; +} + +template <> +__device__ __inline__ void copy_vector(float *dst, const float *src) { + *dst = *src; +} + +template <> +__device__ __inline__ void copy_vector<__half, 4>(__half *dst, + const __half *src) { + *((float2 *)dst) = *((float2 *)src); +} +template <> +__device__ __inline__ void copy_vector(uint8_t *dst, + const uint8_t *src) { + *dst = *src; +} + +template <> +__device__ __inline__ void copy_vector(uint8_t *dst, + const uint8_t *src) { + *((half2 *)dst) = *((half2 *)src); +} + +template <> +__device__ __inline__ void apply_mask<__half, 1>(__half *dst, __half value, + const uint8_t *src) { + if (*src == 1) { + *dst = value; + } +} + +template <> +__device__ __inline__ void +apply_additive_mask<__half, 1>(__half *dst, const __half *additive_mask) { + *dst += *additive_mask; +} + +template <> +__device__ __inline__ void +apply_additive_mask<__half, 4>(__half *dst, const __half *additive_mask) { + *dst += *additive_mask; + *(dst + 1) += *(additive_mask + 1); + *(dst + 2) += *(additive_mask + 2); + *(dst + 3) += *(additive_mask + 3); +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Warp Softmax forward +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +__global__ void softmax_warp_forward(input_t *dst, const output_t *src, + int batch_size, int stride, + int element_count) { + assert(ELEMENTS_PER_LDG_STG == 1); + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + + src += first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + dst += first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + + // load data from global memory + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements_input[i][it + element] = + -std::numeric_limits::infinity(); + } + + if (element_index < batch_element_count) { + copy_vector( + &elements_input[i][it], src + i * element_count + it * WARP_SIZE); + } + } + } + + // convert input_t to acc_t + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + // elements[i][it] = expf(elements[i][it] - max_value[i]); + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // dst[i * element_count + it * WARP_SIZE] = elements[i][it] / sum[i]; + output_t out[ELEMENTS_PER_LDG_STG]; + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = elements[i][it + element] / sum[i]; + } + copy_vector( + dst + i * element_count + it * WARP_SIZE, out); + } else { + break; + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using softmax_forward_func = void (*)(input_t *dst, const output_t *src, + int batch_size, int stride, + int element_count); + +template +bool warp_softmax_kernel(int log2_elements, int &warp_size, + int &batches_per_warp, + softmax_forward_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + switch (log2_elements) { + case 0: // 1 + kernel = &softmax_warp_forward; + break; + case 1: // 2 + kernel = &softmax_warp_forward; + break; + case 2: // 4 + kernel = &softmax_warp_forward; + break; + case 3: // 8 + kernel = &softmax_warp_forward; + break; + case 4: // 16 + kernel = &softmax_warp_forward; + break; + case 5: // 32 + kernel = &softmax_warp_forward; + break; + case 6: // 64 + kernel = &softmax_warp_forward; + break; + case 7: // 128 + kernel = &softmax_warp_forward; + break; + case 8: // 256 + kernel = &softmax_warp_forward; + break; + case 9: // 512 + kernel = &softmax_warp_forward; + break; + case 10: // 1024 + kernel = &softmax_warp_forward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_softmax(output_t *dst, const input_t *src, int softmax_elements, + int softmax_elements_stride, int batch_count) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + softmax_forward_func kernel; + int warp_size, batches_per_warp; + if (!warp_softmax_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + dst, src, batch_count, softmax_elements_stride, softmax_elements); + return true; + } + return false; +} + +template +__global__ void additive_masked_softmax_dropout_warp_forward_vec4( + output_t *dst, uint8_t *dropout_mask, const input_t *src, + const input_t *pad_mask, int batch_size, int stride, int element_count, + int pad_batch_stride, at::PhiloxCudaState philox_args, float p) { + + assert(ELEMENTS_PER_LDG_STG == 4); + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + int tid = blockIdx.x * blockDim.x * blockDim.y + threadIdx.y * blockDim.x + + threadIdx.x; + acc_t pinv = acc_t(1) / p; + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + // vectorize if element_count is multiple of 4, else don't vectorize + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset; + dst += thread_offset; + dropout_mask += thread_offset; + + // load data from global memory + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + int pad_thread_offset = ((first_batch + i) / pad_batch_stride) * stride + + ELEMENTS_PER_LDG_STG * local_idx; + const half *curr_mask = pad_mask + pad_thread_offset; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + // masking_value is a large negative value + elements_input[i][it + element] = -10000; + } + + if (element_index < batch_element_count) { + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + copy_vector(&elements_input[i][it], + src + itr_idx); + apply_additive_mask( + &elements_input[i][it], + curr_mask + + itr_jmp); //(__half)-std::numeric_limits::infinity() + } + } + } + // convert input_t to acc_t + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + auto seeds = at::cuda::philox::unpack(philox_args); + Philox ph(std::get<0>(seeds), tid, std::get<1>(seeds)); + uint8_t rands[WARP_BATCH][WARP_ITERATIONS]; + float4 rand_num; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + rand_num = uniform4(ph()); + rands[i][it] = (rand_num.x <= p) > 0.5; + rands[i][it + 1] = (rand_num.y <= p) > 0.5; + rands[i][it + 2] = (rand_num.z <= p) > 0.5; + rands[i][it + 3] = (rand_num.w <= p) > 0.5; + copy_vector( + dropout_mask + i * element_count + it * WARP_SIZE, &rands[i][it]); + } + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + output_t out[ELEMENTS_PER_LDG_STG]; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = rands[i][it + element] * + (pinv * (elements[i][it + element] / sum[i])); + } + copy_vector( + dst + i * element_count + it * WARP_SIZE, out); + + } else { + break; + } + } + } +} + +template +__global__ void additive_masked_softmax_dropout_warp_forward( + output_t *dst, uint8_t *dropout_mask, const input_t *src, + const input_t *pad_mask, int batch_size, int stride, int element_count, + int pad_batch_stride, at::PhiloxCudaState philox_args, float p) { + assert(ELEMENTS_PER_LDG_STG == 1); + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + int tid = blockIdx.x * blockDim.x * blockDim.y + threadIdx.y * blockDim.x + + threadIdx.x; + acc_t pinv = acc_t(1) / p; + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + // vectorize if element_count is multiple of 4, else don't vectorize + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + + int thread_offset = first_batch * stride + local_idx; + src += thread_offset; + dst += thread_offset; + dropout_mask += thread_offset; + + // load data from global memory + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + int pad_thread_offset = + ((first_batch + i) / pad_batch_stride) * stride + local_idx; + const half *curr_mask = pad_mask + pad_thread_offset; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += 1) { + int element_index = local_idx + it * WARP_SIZE; +#pragma unroll + for (int element = 0; element < 1; ++element) { + // masking_value is a large negative value + elements_input[i][it + element] = -10000; + } + + if (element_index < batch_element_count) { + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + copy_vector(&elements_input[i][it], src + itr_idx); + apply_additive_mask(&elements_input[i][it], + curr_mask + itr_jmp); + } + } + } + // convert input_t to acc_t + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + curandStatePhilox4_32_10_t state; + auto seeds = at::cuda::philox::unpack(philox_args); + curand_init(std::get<0>(seeds), tid, std::get<1>(seeds), &state); + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += 1) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < element_count) { + output_t out[1]; + acc_t softmax_out[1]; + uint8_t dropout_mask_temp[1]; + // generate a vector of random numbers here + float rand = curand_uniform(&state); + float *rand_ptr = (float *)(&rand); +#pragma unroll + for (int element = 0; element < 1; ++element) { + softmax_out[element] = (elements[i][it + element] / sum[i]); + rand_ptr[element] = rand_ptr[element] <= p; + out[element] = rand_ptr[element] * pinv * softmax_out[element]; + dropout_mask_temp[element] = + rand_ptr[element] > 0.5; // just to distinguish 0.0f and 1.0f + } + copy_vector(dst + i * element_count + it * WARP_SIZE, out); + copy_vector(dropout_mask + i * element_count + + it * WARP_SIZE, + dropout_mask_temp); + + } else { + break; + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using additive_masked_softmax_dropout_forward_func = void (*)( + output_t *dst, uint8_t *dropout_mask, const input_t *src, + const input_t *pad_mask, int batch_size, int stride, int element_count, + int pad_batch_stride, at::PhiloxCudaState philox_args, float p); + +template +bool warp_additive_masked_softmax_dropout_kernel( + int element_count, int log2_elements, int &warp_size, int &batches_per_warp, + additive_masked_softmax_dropout_forward_func + &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + bool flag_vec4 = (element_count % 4 == 0); + switch (log2_elements) { + case 0: // 1 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 1: // 2 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 2: // 4 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 3: // 8 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 4: // 16 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 5: // 32 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 6: // 64 + kernel = &additive_masked_softmax_dropout_warp_forward; + break; + case 7: // 128 + if (flag_vec4) + kernel = &additive_masked_softmax_dropout_warp_forward_vec4< + input_t, output_t, acc_t, 2, 4, 32, 4>; + else + kernel = + &additive_masked_softmax_dropout_warp_forward; + break; + case 8: // 256 + if (flag_vec4) + kernel = &additive_masked_softmax_dropout_warp_forward_vec4< + input_t, output_t, acc_t, 1, 8, 32, 4>; + else + kernel = + &additive_masked_softmax_dropout_warp_forward; + break; + case 9: // 512 + if (flag_vec4) + kernel = &additive_masked_softmax_dropout_warp_forward_vec4< + input_t, output_t, acc_t, 1, 16, 32, 4>; + else + kernel = + &additive_masked_softmax_dropout_warp_forward; + break; + case 10: // 1024 + if (flag_vec4) + kernel = &additive_masked_softmax_dropout_warp_forward_vec4< + input_t, output_t, acc_t, 1, 32, 32, 4>; + else + kernel = + &additive_masked_softmax_dropout_warp_forward; + break; + case 11: // 2048 + if (flag_vec4) + kernel = &additive_masked_softmax_dropout_warp_forward_vec4< + input_t, output_t, acc_t, 1, 64, 32, 4>; + else + kernel = + &additive_masked_softmax_dropout_warp_forward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_additive_masked_softmax_dropout( + output_t *dst, uint8_t *dropout_mask, const input_t *src, + const input_t *pad_mask, int totalElements, int softmax_elements, + int softmax_elements_stride, int batch_count, int pad_batch_stride, float p, + cudaStream_t streamid) // p is the probability to keep, not drop +{ + + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 2048) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + additive_masked_softmax_dropout_forward_func + kernel; + int warp_size, batches_per_warp; + if (!warp_additive_masked_softmax_dropout_kernel( + softmax_elements, log2_elements, warp_size, batches_per_warp, + kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + c10::optional gen_; + auto gen = at::get_generator_or_default( + gen_, at::cuda::detail::getDefaultCUDAGenerator()); + int64_t counter_offset = (totalElements / (blocks * threads_per_block) + 1); + at::PhiloxCudaState rng_engine_inputs; + { + std::lock_guard lock(gen->mutex_); + rng_engine_inputs = gen->philox_cuda_state(counter_offset); + } + + // compute launch size + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + dst, dropout_mask, src, pad_mask, batch_count, softmax_elements_stride, + softmax_elements, pad_batch_stride, rng_engine_inputs, p); + return true; + } + return false; +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +__global__ void additive_masked_softmax_warp_forward( + input_t *dst, const output_t *src, const input_t *pad_mask, int batch_size, + int stride, int element_count, int pad_batch_stride) { + assert(ELEMENTS_PER_LDG_STG == 1); + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset; + dst += thread_offset; + + // load data from global memory + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + int pad_thread_offset = ((first_batch + i) / pad_batch_stride) * stride + + ELEMENTS_PER_LDG_STG * local_idx; + const half *curr_mask = pad_mask + pad_thread_offset; + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + // masking_value is a large negative value + elements_input[i][it + element] = -10000; + } + + if (element_index < batch_element_count) { + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + copy_vector(&elements_input[i][it], + src + itr_idx); + // apply_mask(&elements_input[i][it], + // (__half)-std::numeric_limits::infinity(), + // curr_mask + itr_jmp); + elements_input[i][it] += *(curr_mask + itr_jmp); + } + } + } + + // convert input_t to acc_t + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + // elements[i][it] = expf(elements[i][it] - max_value[i]); + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // dst[i * element_count + it * WARP_SIZE] = elements[i][it] / sum[i]; + output_t out[ELEMENTS_PER_LDG_STG]; + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = elements[i][it + element] / sum[i]; + } + copy_vector( + dst + i * element_count + it * WARP_SIZE, out); + } else { + break; + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using additive_masked_softmax_forward_func = void (*)( + input_t *dst, const output_t *src, const half *pad_mask, int batch_size, + int stride, int element_count, int pad_batch_stride); + +template +bool warp_additive_masked_softmax_kernel( + int log2_elements, int &warp_size, int &batches_per_warp, + additive_masked_softmax_forward_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + switch (log2_elements) { + case 0: // 1 + kernel = &additive_masked_softmax_warp_forward; + break; + case 1: // 2 + kernel = &additive_masked_softmax_warp_forward; + break; + case 2: // 4 + kernel = &additive_masked_softmax_warp_forward; + break; + case 3: // 8 + kernel = &additive_masked_softmax_warp_forward; + break; + case 4: // 16 + kernel = &additive_masked_softmax_warp_forward; + break; + case 5: // 32 + kernel = &additive_masked_softmax_warp_forward; + break; + case 6: // 64 + kernel = &additive_masked_softmax_warp_forward; + break; + case 7: // 128 + kernel = &additive_masked_softmax_warp_forward; + break; + case 8: // 256 + kernel = &additive_masked_softmax_warp_forward; + break; + case 9: // 512 + kernel = &additive_masked_softmax_warp_forward; + break; + case 10: // 1024 + kernel = &additive_masked_softmax_warp_forward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_additive_masked_softmax(output_t *dst, const input_t *src, + const input_t *pad_mask, + int softmax_elements, + int softmax_elements_stride, + int batch_count, int pad_batch_stride) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + additive_masked_softmax_forward_func kernel; + int warp_size, batches_per_warp; + if (!warp_additive_masked_softmax_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + dst, src, pad_mask, batch_count, softmax_elements_stride, + softmax_elements, pad_batch_stride); + return true; + } + return false; +} + +template +bool dispatch_additive_masked_softmax_stream( + output_t *dst, const input_t *src, const input_t *pad_mask, + int softmax_elements, int softmax_elements_stride, int batch_count, + int pad_batch_stride, cudaStream_t streamid) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + additive_masked_softmax_forward_func kernel; + int warp_size, batches_per_warp; + if (!warp_additive_masked_softmax_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // launch + kernel<<>>( + dst, src, pad_mask, batch_count, softmax_elements_stride, + softmax_elements, pad_batch_stride); + return true; + } + return false; +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +__global__ void +masked_softmax_warp_forward(input_t *dst, const output_t *src, + const uint8_t *pad_mask, int batch_size, int stride, + int element_count, int pad_batch_stride) { + assert(ELEMENTS_PER_LDG_STG == 1); + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset; + dst += thread_offset; + + // load data from global memory + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + int pad_thread_offset = ((first_batch + i) / pad_batch_stride) * stride + + ELEMENTS_PER_LDG_STG * local_idx; + const uint8_t *curr_mask = pad_mask + pad_thread_offset; + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements_input[i][it + element] = + -std::numeric_limits::infinity(); + } + + if (element_index < batch_element_count) { + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + copy_vector(&elements_input[i][it], + src + itr_idx); + apply_mask( + &elements_input[i][it], + __float2half(-std::numeric_limits::infinity()), + curr_mask + itr_jmp); + } + } + } + + // convert input_t to acc_t + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + // elements[i][it] = expf(elements[i][it] - max_value[i]); + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // dst[i * element_count + it * WARP_SIZE] = elements[i][it] / sum[i]; + output_t out[ELEMENTS_PER_LDG_STG]; + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = elements[i][it + element] / sum[i]; + } + copy_vector( + dst + i * element_count + it * WARP_SIZE, out); + } else { + break; + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using masked_softmax_forward_func = void (*)(input_t *dst, const output_t *src, + const uint8_t *pad_mask, + int batch_size, int stride, + int element_count, + int pad_batch_stride); + +template +bool warp_masked_softmax_kernel( + int log2_elements, int &warp_size, int &batches_per_warp, + masked_softmax_forward_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + switch (log2_elements) { + case 0: // 1 + kernel = &masked_softmax_warp_forward; + break; + case 1: // 2 + kernel = &masked_softmax_warp_forward; + break; + case 2: // 4 + kernel = &masked_softmax_warp_forward; + break; + case 3: // 8 + kernel = &masked_softmax_warp_forward; + break; + case 4: // 16 + kernel = + &masked_softmax_warp_forward; + break; + case 5: // 32 + kernel = + &masked_softmax_warp_forward; + break; + case 6: // 64 + kernel = + &masked_softmax_warp_forward; + break; + case 7: // 128 + kernel = + &masked_softmax_warp_forward; + break; + case 8: // 256 + kernel = + &masked_softmax_warp_forward; + break; + case 9: // 512 + kernel = + &masked_softmax_warp_forward; + break; + case 10: // 1024 + kernel = + &masked_softmax_warp_forward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_masked_softmax(output_t *dst, const input_t *src, + const uint8_t *pad_mask, int softmax_elements, + int softmax_elements_stride, int batch_count, + int pad_batch_stride) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + masked_softmax_forward_func kernel; + int warp_size, batches_per_warp; + if (!warp_masked_softmax_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + dst, src, pad_mask, batch_count, softmax_elements_stride, + softmax_elements, pad_batch_stride); + return true; + } + return false; +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +__global__ void time_masked_softmax_warp_forward( + input_t *dst, const output_t *src, const uint8_t *pad_mask, int batch_size, + int stride, int element_count, int mod_seq_len) { + assert(ELEMENTS_PER_LDG_STG == 1); + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset; + dst += thread_offset; + + // load data from global memory + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + int pad_thread_offset = ((first_batch + i) % mod_seq_len) * stride + + ELEMENTS_PER_LDG_STG * local_idx; + const uint8_t *curr_mask = pad_mask + pad_thread_offset; + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements_input[i][it + element] = + -std::numeric_limits::infinity(); + } + + if (element_index < batch_element_count) { + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + copy_vector(&elements_input[i][it], + src + itr_idx); + apply_mask( + &elements_input[i][it], + __float2half(-std::numeric_limits::infinity()), + curr_mask + itr_jmp); + } + } + } + + // convert input_t to acc_t + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + // elements[i][it] = expf(elements[i][it] - max_value[i]); + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // dst[i * element_count + it * WARP_SIZE] = elements[i][it] / sum[i]; + output_t out[ELEMENTS_PER_LDG_STG]; + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = elements[i][it + element] / sum[i]; + } + copy_vector( + dst + i * element_count + it * WARP_SIZE, out); + } else { + break; + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using time_masked_softmax_forward_func = + void (*)(input_t *dst, const output_t *src, const uint8_t *pad_mask, + int batch_size, int stride, int element_count, int mod_seq_len); + +template +bool warp_time_masked_softmax_kernel( + int log2_elements, int &warp_size, int &batches_per_warp, + time_masked_softmax_forward_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + switch (log2_elements) { + case 0: // 1 + kernel = + &time_masked_softmax_warp_forward; + break; + case 1: // 2 + kernel = + &time_masked_softmax_warp_forward; + break; + case 2: // 4 + kernel = + &time_masked_softmax_warp_forward; + break; + case 3: // 8 + kernel = + &time_masked_softmax_warp_forward; + break; + case 4: // 16 + kernel = &time_masked_softmax_warp_forward; + break; + case 5: // 32 + kernel = &time_masked_softmax_warp_forward; + break; + case 6: // 64 + kernel = &time_masked_softmax_warp_forward; + break; + case 7: // 128 + kernel = &time_masked_softmax_warp_forward; + break; + case 8: // 256 + kernel = &time_masked_softmax_warp_forward; + break; + case 9: // 512 + kernel = &time_masked_softmax_warp_forward; + break; + case 10: // 1024 + kernel = &time_masked_softmax_warp_forward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_time_masked_softmax(output_t *dst, const input_t *src, + const uint8_t *pad_mask, int softmax_elements, + int softmax_elements_stride, int batch_count, + int mod_seq_len) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + time_masked_softmax_forward_func kernel; + int warp_size, batches_per_warp; + if (!warp_time_masked_softmax_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + dst, src, pad_mask, batch_count, softmax_elements_stride, + softmax_elements, mod_seq_len); + return true; + } + return false; +} + +int log2_ceil_native(int value) { + int log2_value = 0; + while ((1 << log2_value) < value) + ++log2_value; + return log2_value; +} + +template +__device__ __forceinline__ T +WARP_SHFL_XOR_NATIVE(T value, int laneMask, int width = warpSize, + unsigned int mask = 0xffffffff) { +#if CUDA_VERSION >= 9000 + return __shfl_xor_sync(mask, value, laneMask, width); +#else + return __shfl_xor(value, laneMask, width); +#endif +} + +template +__device__ __forceinline__ void warp_reduce_sum(acc_t *sum) { +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + acc_t b = WARP_SHFL_XOR_NATIVE(sum[i], offset, WARP_SIZE); + sum[i] = sum[i] + b; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Warp softmax backward functions as fused variants of +// at::softmax_backward_data function +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// softmax backward data function is taken from native pytorch, elementwise mul +// is fused in the epolog, as well as masking and scaling for fusing dropout + +template +__global__ void masked_scale_softmax_warp_backward_masked_dgrad( + output_t *gradInput, const input_t *grad, const input_t *output, + const uint8_t *mask, const uint8_t *pad_mask, acc_t scale, int batch_size, + int stride, int element_count, int heads) { + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_backward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x % WARP_SIZE; + + // the first element to process by the current thread + int thread_offset = first_batch * stride + local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + mask += thread_offset; + + // The nested loops over WARP_BATCH and then WARP_ITERATIONS can be simplified + // to one loop, but I think doing so would obfuscate the logic of the + // algorithm, thus I chose to keep the nested loops. This should have no + // impact on performance because the loops are unrolled anyway. + + // load data from global memory + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS]; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + for (int it = 0; it < WARP_ITERATIONS; ++it) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + grad_reg[i][it] = + (input_t)((acc_t)mask[i * element_count + it * WARP_SIZE] * + (acc_t)grad[i * element_count + it * WARP_SIZE] * + (acc_t)scale) * + output[i * element_count + it * WARP_SIZE]; + output_reg[i][it] = output[i * element_count + it * WARP_SIZE]; + } else { + grad_reg[i][it] = acc_t(0); + output_reg[i][it] = acc_t(0); + } + } + } + + acc_t sum[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] = grad_reg[i][0]; +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + sum[i] += grad_reg[i][it]; + } + } + warp_reduce_sum(sum); + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + int total_ind = thread_offset + i * element_count + it * WARP_SIZE; + int pad_mask_ind = + element_count * + (total_ind / (heads * element_count * element_count)) + + total_ind % element_count; + uint8_t pad_mask_element = 1 - pad_mask[pad_mask_ind]; + if (pad_mask_element == 0) + gradInput[i * element_count + it * WARP_SIZE] = 0; + else { + if (is_log_softmax) { + gradInput[i * element_count + it * WARP_SIZE] = + (grad_reg[i][it] - std::exp(output_reg[i][it]) * sum[i]); + } else { + gradInput[i * element_count + it * WARP_SIZE] = + (grad_reg[i][it] - output_reg[i][it] * sum[i]); + } + } + } + } + } +} +template +void dispatch_masked_scale_softmax_backward_masked_out( + output_t *grad_input, const input_t *grad, const input_t *output, + const uint8_t *mask, const uint8_t *pad_mask, acc_t scale, + int softmax_elements, int softmax_elements_stride, int batch_count, + int heads) { + TORCH_INTERNAL_ASSERT(softmax_elements >= 0 && softmax_elements <= 1024); + if (softmax_elements == 0) { + return; + } else { + int log2_elements = log2_ceil_native(softmax_elements); + const int next_power_of_two = 1 << log2_elements; + + // This value must match the WARP_SIZE constexpr value computed inside + // softmax_warp_backward. + int warp_size = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside + // softmax_warp_backward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 1: // 2 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 2: // 4 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 3: // 8 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 4: // 16 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 5: // 32 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 6: // 64 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 7: // 128 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 8: // 256 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 9: // 512 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 10: // 1024 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + default: + break; + } + } +} + +template +void dispatch_masked_scale_softmax_backward_masked_out_stream( + output_t *grad_input, const input_t *grad, const input_t *output, + const uint8_t *mask, const uint8_t *pad_mask, acc_t scale, + int softmax_elements, int softmax_elements_stride, int batch_count, + int heads, cudaStream_t streamid) { + TORCH_INTERNAL_ASSERT(softmax_elements >= 0 && softmax_elements <= 1024); + if (softmax_elements == 0) { + return; + } else { + int log2_elements = log2_ceil_native(softmax_elements); + const int next_power_of_two = 1 << log2_elements; + // This value must match the WARP_SIZE constexpr value computed inside + // softmax_warp_backward. + int warp_size = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + // This value must match the WARP_BATCH constexpr value computed inside + // softmax_warp_backward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 1: // 2 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 2: // 4 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 3: // 8 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 4: // 16 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 5: // 32 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 6: // 64 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 7: // 128 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 8: // 256 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 9: // 512 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + case 10: // 1024 + masked_scale_softmax_warp_backward_masked_dgrad + <<>>( + grad_input, grad, output, mask, pad_mask, scale, batch_count, + softmax_elements_stride, softmax_elements, heads); + break; + default: + break; + } + } +} + +template +__global__ void +masked_scale_softmax_warp_backward(output_t *gradInput, const input_t *grad, + const input_t *output, const uint8_t *mask, + acc_t scale, int batch_size, int stride, + int element_count) { + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_backward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x % WARP_SIZE; + + // the first element to process by the current thread + int thread_offset = first_batch * stride + local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + mask += thread_offset; + + // The nested loops over WARP_BATCH and then WARP_ITERATIONS can be simplified + // to one loop, but I think doing so would obfuscate the logic of the + // algorithm, thus I chose to keep the nested loops. This should have no + // impact on performance because the loops are unrolled anyway. + + // load data from global memory + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS]; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + for (int it = 0; it < WARP_ITERATIONS; ++it) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + grad_reg[i][it] = + (input_t)((acc_t)mask[i * element_count + it * WARP_SIZE] * + (acc_t)grad[i * element_count + it * WARP_SIZE] * + (acc_t)scale) * + output[i * element_count + it * WARP_SIZE]; + output_reg[i][it] = output[i * element_count + it * WARP_SIZE]; + } else { + grad_reg[i][it] = acc_t(0); + output_reg[i][it] = acc_t(0); + } + } + } + + acc_t sum[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] = grad_reg[i][0]; +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + sum[i] += grad_reg[i][it]; + } + } + warp_reduce_sum(sum); + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + if (is_log_softmax) { + gradInput[i * element_count + it * WARP_SIZE] = + (grad_reg[i][it] - std::exp(output_reg[i][it]) * sum[i]); + } else { + gradInput[i * element_count + it * WARP_SIZE] = + (grad_reg[i][it] - output_reg[i][it] * sum[i]); + } + } + } + } +} + +template +__global__ void masked_scale_softmax_warp_backward_recompute( + output_t *gradInput, const input_t *grad, const input_t *softmax_input, + const input_t *pad_mask, const uint8_t *mask, acc_t scale, int batch_size, + int stride, int pad_batch_stride, int element_count) { + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x % WARP_SIZE; + // vectorize if a row length is multiple of 4 + int flag_vec4 = element_count & 3 == 0; + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS]; + input_t elements_input[WARP_BATCH][WARP_ITERATIONS]; + + // the first element to process by the current thread + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + + grad += thread_offset; + softmax_input += thread_offset; + gradInput += thread_offset; + mask += thread_offset; + + // The nested loops over WARP_BATCH and then WARP_ITERATIONS can be simplified + // to one loop, but I think doing so would obfuscate the logic of the + // algorithm, thus I chose to keep the nested loops. This should have no + // impact on performance because the loops are unrolled anyway. + + // load data from global memory + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + int pad_thread_offset = ((first_batch + i) / pad_batch_stride) * stride + + ELEMENTS_PER_LDG_STG * local_idx; + const input_t *curr_mask = pad_mask + pad_thread_offset; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + // masking_value is a large negative value + elements_input[i][it + element] = -10000; + grad_reg[i][it + element] = acc_t(0); + } + + if (element_index < batch_element_count) { + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + copy_vector(&elements_input[i][it], + softmax_input + itr_idx); + apply_additive_mask( + &elements_input[i][it], + curr_mask + + itr_jmp); //(__half)-std::numeric_limits::infinity() + uint8_t mask_temp[ELEMENTS_PER_LDG_STG]; + input_t grad_temp[ELEMENTS_PER_LDG_STG]; + copy_vector(&mask_temp[0], + mask + itr_idx); + copy_vector(&grad_temp[0], + grad + itr_idx); +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + grad_reg[i][it + element] = + ((acc_t)mask_temp[element] * (acc_t)grad_temp[element] * + (acc_t)scale); + } + } + } + } + // load data from global memory + + // convert input_t to acc_t + // TODO : remove this, input is already acc_t type in register + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = elements_input[i][it]; + } + } + + constexpr uint32_t FULL_MASK = 0xffffffff; + + // compute local max_value + + // take the max_value of the first element to avoid one max call + acc_t max_value[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + } + +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = + (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + +// reduction max_value +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + float val[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + val[i] = __shfl_xor_sync(FULL_MASK, max_value[i], offset, WARP_SIZE); + } +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = max_value[i] > val[i] ? max_value[i] : val[i]; + } + } + + // compute local sum + acc_t sum[WARP_BATCH]{0.0f}; + +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + // elements[i][it] = expf(elements[i][it] - max_value[i]); + elements[i][it] = std::exp(elements[i][it] - max_value[i]); + sum[i] += elements[i][it]; + } + } + +// reduction sum +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it++) { + elements[i][it] = elements[i][it] / sum[i]; + grad_reg[i][it] = grad_reg[i][it] * elements[i][it]; + } + } + + acc_t grad_sum[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + grad_sum[i] = grad_reg[i][0]; +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + grad_sum[i] += grad_reg[i][it]; + } + } + warp_reduce_sum(grad_sum); + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + output_t grad_input_reg[ELEMENTS_PER_LDG_STG]; +#pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; element++) { + if (is_log_softmax) { + grad_input_reg[element] = + (grad_reg[i][it + element] - + std::exp(elements[i][it + element]) * grad_sum[i]); + } else { + grad_input_reg[element] = (grad_reg[i][it + element] - + elements[i][it + element] * grad_sum[i]); + } + } + copy_vector( + gradInput + i * element_count + it * WARP_SIZE, grad_input_reg); + } + } + } +} + +template +using masked_scale_softmax_warp_backward_recompute_func = void (*)( + output_t *gradInput, const input_t *grad, const input_t *softmax_input, + const input_t *pad_mask, const uint8_t *mask, acc_t scale, int batch_size, + int stride, int pad_batch_stride, int element_count); + +template +bool masked_scale_softmax_warp_backward_recompute_kernel( + int element_count, int log2_elements, int &warp_size, int &batches_per_warp, + masked_scale_softmax_warp_backward_recompute_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + bool flag_vec4 = (element_count % 4 == 0); + switch (log2_elements) { + case 0: // 1 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 1, 1, 1, is_log_softmax>; + break; + case 1: // 2 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 1, 2, 1, is_log_softmax>; + break; + case 2: // 4 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 1, 4, 1, is_log_softmax>; + break; + case 3: // 8 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 1, 8, 1, is_log_softmax>; + break; + case 4: // 16 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 1, 16, 1, is_log_softmax>; + break; + case 5: // 32 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 1, 32, 1, is_log_softmax>; + break; + case 6: // 64 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 2, 32, 1, is_log_softmax>; + break; + case 7: // 128 + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 2, 4, 32, 1, is_log_softmax>; + break; + case 8: // 256 + if (flag_vec4) + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 8, 32, 4, is_log_softmax>; + else + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 8, 32, 1, is_log_softmax>; + break; + case 9: // 512 + if (flag_vec4) + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 16, 32, 4, is_log_softmax>; + else + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 16, 32, 1, is_log_softmax>; + break; + case 10: // 1024 + if (flag_vec4) + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 32, 32, 4, is_log_softmax>; + else + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 32, 32, 1, is_log_softmax>; + break; + case 11: // 2048 + if (flag_vec4) + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 64, 32, 4, is_log_softmax>; + else + kernel = &masked_scale_softmax_warp_backward_recompute< + input_t, output_t, acc_t, 1, 64, 32, 1, is_log_softmax>; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_masked_scale_softmax_backward_recompute( + output_t *grad_input, const input_t *grad, const input_t *softmax_input, + const input_t *pad_mask, const uint8_t *mask, acc_t scale, + int softmax_elements, int softmax_elements_stride, int pad_batch_stride, + int batch_count, cudaStream_t streamid) { + + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 2048) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + masked_scale_softmax_warp_backward_recompute_func + kernel; + int warp_size, batches_per_warp; + if (!masked_scale_softmax_warp_backward_recompute_kernel< + input_t, output_t, acc_t, is_log_softmax>( + softmax_elements, log2_elements, warp_size, batches_per_warp, + kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + + // compute launch size + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + grad_input, grad, softmax_input, pad_mask, mask, scale, batch_count, + softmax_elements_stride, pad_batch_stride, softmax_elements); + return true; + } + return false; +} + +template +void dispatch_masked_scale_softmax_backward_stream( + output_t *grad_input, const input_t *grad, const input_t *output, + const uint8_t *mask, acc_t scale, int softmax_elements, + int softmax_elements_stride, int batch_count, cudaStream_t streamid) { + TORCH_INTERNAL_ASSERT(softmax_elements >= 0 && softmax_elements <= 1024); + if (softmax_elements == 0) { + return; + } else { + int log2_elements = log2_ceil_native(softmax_elements); + const int next_power_of_two = 1 << log2_elements; + // This value must match the WARP_SIZE constexpr value computed inside + // softmax_warp_backward. + int warp_size = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + // This value must match the WARP_BATCH constexpr value computed inside + // softmax_warp_backward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 1: // 2 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 2: // 4 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 3: // 8 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 4: // 16 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 5: // 32 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 6: // 64 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 7: // 128 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 8: // 256 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 9: // 512 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + case 10: // 1024 + masked_scale_softmax_warp_backward + <<>>( + grad_input, grad, output, mask, scale, batch_count, + softmax_elements_stride, softmax_elements); + break; + default: + break; + } + } +} + +// elementwise multiplication called in at::softmax_backward_data is fused +// inside softmax dgrad kernel as a result of fusion, intermediate +// multiplication result is stored in fp32 in registers, instead of fp16 +template +__global__ void +softmax_warp_backward_fused_native(output_t *gradInput, const input_t *grad, + const input_t *output, int batch_size, + int stride, int element_count) { + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_backward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x % WARP_SIZE; + + // the first element to process by the current thread + int thread_offset = first_batch * stride + local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + + // The nested loops over WARP_BATCH and then WARP_ITERATIONS can be simplified + // to one loop, but I think doing so would obfuscate the logic of the + // algorithm, thus I chose to keep the nested loops. This should have no + // impact on performance because the loops are unrolled anyway. + + // load data from global memory + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS]; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS]; + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + for (int it = 0; it < WARP_ITERATIONS; ++it) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + grad_reg[i][it] = grad[i * element_count + it * WARP_SIZE] * + output[i * element_count + it * WARP_SIZE]; + output_reg[i][it] = output[i * element_count + it * WARP_SIZE]; + } else { + grad_reg[i][it] = acc_t(0); + output_reg[i][it] = acc_t(0); + } + } + } + + acc_t sum[WARP_BATCH]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] = grad_reg[i][0]; //* output_reg[i][0]; +#pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + sum[i] += grad_reg[i][it]; // * output_reg[i][it]; + } + } + warp_reduce_sum(sum); + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + int element_index = local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + if (is_log_softmax) { + gradInput[i * element_count + it * WARP_SIZE] = + (grad_reg[i][it] - std::exp(output_reg[i][it]) * sum[i]); + } else { + gradInput[i * element_count + it * WARP_SIZE] = + (grad_reg[i][it] - output_reg[i][it] * sum[i]); + } + } + } + } +} + +template +void dispatch_softmax_backward_fused_native( + output_t *grad_input, const input_t *grad, const input_t *output, + int softmax_elements, int softmax_elements_stride, int batch_count) { + TORCH_INTERNAL_ASSERT(softmax_elements >= 0 && softmax_elements <= 1024); + if (softmax_elements == 0) { + return; + } else { + int log2_elements = log2_ceil_native(softmax_elements); + const int next_power_of_two = 1 << log2_elements; + + // This value must match the WARP_SIZE constexpr value computed inside + // softmax_warp_backward. + int warp_size = + (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside + // softmax_warp_backward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 1: // 2 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 2: // 4 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 3: // 8 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 4: // 16 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 5: // 32 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 6: // 64 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 7: // 128 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 8: // 256 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 9: // 512 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + case 10: // 1024 + softmax_warp_backward_fused_native + <<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + break; + default: + break; + } + } +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Warp softmax backward +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +template +__global__ void softmax_warp_backward(__half *gradInput, const __half *grad, + const __half *output, int batch_size, + int stride, int element_count) { + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + + // the first element to process by the current thread + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + + // load data from global memory + input_t grad_reg_input[WARP_BATCH][WARP_ITERATIONS] = {0.0f}; + input_t output_reg_input[WARP_BATCH][WARP_ITERATIONS] = {0.0f}; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + copy_vector( + &grad_reg_input[i][it], grad + i * element_count + it * WARP_SIZE); + copy_vector(&output_reg_input[i][it], + output + i * element_count + + it * WARP_SIZE); + } + } + } + + // convert half to floating point + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS]; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + grad_reg[i][it] = grad_reg_input[i][it]; + output_reg[i][it] = output_reg_input[i][it]; + } + } + + // compute thread local sum + acc_t sum[WARP_BATCH] = {0}; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += grad_reg[i][it] * output_reg[i][it]; + } + } + + // reduction sum + constexpr uint32_t FULL_MASK = 0xffffffff; +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + output_t out[ELEMENTS_PER_LDG_STG]; + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = (output_reg[i][it + element] * + (grad_reg[i][it + element] - sum[i])); + } + // store them in global memory + copy_vector( + gradInput + i * element_count + it * WARP_SIZE, out); + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using softmax_backward_func = void (*)(output_t *gradInput, const input_t *grad, + const input_t *output, int batch_size, + int stride, int element_count); + +template +bool warp_softmax_backward_kernel( + int log2_elements, int &warp_size, int &batches_per_warp, + softmax_backward_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + switch (log2_elements) { + case 0: // 1 + kernel = &softmax_warp_backward; + break; + case 1: // 2 + kernel = &softmax_warp_backward; + break; + case 2: // 4 + kernel = &softmax_warp_backward; + break; + case 3: // 8 + kernel = &softmax_warp_backward; + break; + case 4: // 16 + kernel = &softmax_warp_backward; + break; + case 5: // 32 + kernel = &softmax_warp_backward; + break; + case 6: // 64 + kernel = &softmax_warp_backward; + break; + case 7: // 128 + kernel = &softmax_warp_backward; + break; + case 8: // 256 + kernel = &softmax_warp_backward; + break; + case 9: // 512 + kernel = &softmax_warp_backward; + break; + case 10: // 1024 + kernel = &softmax_warp_backward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_softmax_backward(output_t *grad_input, const input_t *grad, + const input_t *output, int softmax_elements, + int softmax_elements_stride, int batch_count) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + softmax_backward_func kernel; + int warp_size, batches_per_warp; + if (!warp_softmax_backward_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + return true; + } + return false; +} + +template +bool dispatch_softmax_backward_stream(output_t *grad_input, const input_t *grad, + const input_t *output, + int softmax_elements, + int softmax_elements_stride, + int batch_count, cudaStream_t streamid) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + softmax_backward_func kernel; + int warp_size, batches_per_warp; + if (!warp_softmax_backward_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // launch + kernel<<>>( + grad_input, grad, output, batch_count, softmax_elements_stride, + softmax_elements); + return true; + } + return false; +} + +template +__global__ void +masked_softmax_warp_backward(__half *gradInput, const __half *grad, + const __half *output, const uint8_t *pad_mask, + int batch_size, int stride, int element_count, + int pad_batch_stride) { + int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the + // batch + int local_idx = threadIdx.x; + + // the first element to process by the current thread + int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + + // load data from global memory + input_t grad_reg_input[WARP_BATCH][WARP_ITERATIONS] = {0.0f}; + input_t output_reg_input[WARP_BATCH][WARP_ITERATIONS] = {0.0f}; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + copy_vector( + &grad_reg_input[i][it], grad + i * element_count + it * WARP_SIZE); + copy_vector(&output_reg_input[i][it], + output + i * element_count + + it * WARP_SIZE); + } + } + } + + // convert half to floating point + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS]; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS]; +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + for (int it = 0; it < WARP_ITERATIONS; ++it) { + grad_reg[i][it] = grad_reg_input[i][it]; + output_reg[i][it] = output_reg_input[i][it]; + } + } + + // compute thread local sum + acc_t sum[WARP_BATCH] = {0}; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += grad_reg[i][it] * output_reg[i][it]; + } + } + + // reduction sum + constexpr uint32_t FULL_MASK = 0xffffffff; +#pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] += __shfl_xor_sync(FULL_MASK, sum[i], offset, WARP_SIZE); + } + } + +// store result +#pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; + int pad_thread_offset = ((first_batch + i) / pad_batch_stride) * stride + + ELEMENTS_PER_LDG_STG * local_idx; + const uint8_t *curr_mask = pad_mask + pad_thread_offset; +#pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it += ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + output_t out[ELEMENTS_PER_LDG_STG]; + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = (output_reg[i][it + element] * + (grad_reg[i][it + element] - sum[i])); + } + // store them in global memory + int itr_jmp = it * WARP_SIZE; + int itr_idx = i * element_count + itr_jmp; + // It is kind of unfortunate this has to be here to zero something out + // that is close to zero in the first place + apply_mask(&out[0], 0.0, + curr_mask + itr_jmp); + copy_vector(gradInput + itr_idx, out); + } + } + } +} + +// WARP_BATCH number of batches. +// WARP_ITERATOINS The number of iterations required for one warp to iterate +// over all data. WARP_SIZE number of elements working on a single batch, has to +// be a power of two. ELEMENTS_PER_LDG_STG has to be 1. +template +using masked_softmax_backward_func = + void (*)(output_t *gradInput, const input_t *grad, const input_t *output, + const uint8_t *pad_mask, int batch_size, int stride, + int element_count, int pad_batch_stride); + +template +bool warp_masked_softmax_backward_kernel( + int log2_elements, int &warp_size, int &batches_per_warp, + masked_softmax_backward_func &kernel) { + // determine size of a warp + const int next_power_of_two = 1 << log2_elements; + warp_size = (next_power_of_two < 32) ? next_power_of_two : 32; + + // determine how many batches a warp should process. + batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + switch (log2_elements) { + case 0: // 1 + kernel = + &masked_softmax_warp_backward; + break; + case 1: // 2 + kernel = + &masked_softmax_warp_backward; + break; + case 2: // 4 + kernel = + &masked_softmax_warp_backward; + break; + case 3: // 8 + kernel = + &masked_softmax_warp_backward; + break; + case 4: // 16 + kernel = + &masked_softmax_warp_backward; + break; + case 5: // 32 + kernel = + &masked_softmax_warp_backward; + break; + case 6: // 64 + kernel = + &masked_softmax_warp_backward; + break; + case 7: // 128 + kernel = + &masked_softmax_warp_backward; + break; + case 8: // 256 + kernel = + &masked_softmax_warp_backward; + break; + case 9: // 512 + kernel = + &masked_softmax_warp_backward; + break; + case 10: // 1024 + kernel = + &masked_softmax_warp_backward; + break; + default: + return false; + } + return true; +} + +template +bool dispatch_masked_softmax_backward(output_t *grad_input, const input_t *grad, + const input_t *output, + const uint8_t *pad_mask, + int softmax_elements, + int softmax_elements_stride, + int batch_count, int pad_batch_stride) { + if (softmax_elements == 0) { + return true; + } else if (softmax_elements <= 1024) { + // compute function index. there's a function for each power of two size up + // to 1024. + int log2_elements = 0; + while ((1 << log2_elements) < softmax_elements) + ++log2_elements; + + masked_softmax_backward_func kernel; + int warp_size, batches_per_warp; + if (!warp_masked_softmax_backward_kernel( + log2_elements, warp_size, batches_per_warp, kernel)) { + return false; + } + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // compute warps per block. + int warps_per_block = (threads_per_block / warp_size); + + // compute launch size + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = (batch_count + batches_per_block - 1) / batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + + // launch + kernel<<>>( + grad_input, grad, output, pad_mask, batch_count, + softmax_elements_stride, softmax_elements, pad_batch_stride); + return true; + } + return false; +} +} // namespace diff --git a/apex/apex/contrib/csrc/multihead_attn/strided_batched_gemm.cuh b/apex/apex/contrib/csrc/multihead_attn/strided_batched_gemm.cuh new file mode 100644 index 00000000..b207b17f --- /dev/null +++ b/apex/apex/contrib/csrc/multihead_attn/strided_batched_gemm.cuh @@ -0,0 +1,565 @@ +#pragma once +#include +#include + +#include +#include +#include +#include + +//#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { +cublasOperation_t convertTransToCublasOperation(char trans) { + if (trans == 't') + return CUBLAS_OP_T; + else if (trans == 'n') + return CUBLAS_OP_N; + else if (trans == 'c') + return CUBLAS_OP_C; + else { + TORCH_CHECK(false, "trans must be one of: t, n, c"); + return CUBLAS_OP_T; + } +} + +void CublasStridedBatchedGemm( + char transa, char transb, long m, long n, long k, + float alpha, const half *a, long lda, long strideA, const half *b, long ldb, + long strideB, float beta, half *c, long ldc, long strideC, long batchCount, + cublasGemmAlgo_t algo = CUBLAS_GEMM_DEFAULT_TENSOR_OP) { + cublasOperation_t opa = convertTransToCublasOperation(transa); + cublasOperation_t opb = convertTransToCublasOperation(transb); + + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + cublasSetStream(handle, stream); + float fAlpha = alpha; + float fBeta = beta; + TORCH_CUDABLAS_CHECK(cublasGemmStridedBatchedEx( + handle, opa, opb, (int)m, (int)n, (int)k, (void *)&fAlpha, a, CUDA_R_16F, + (int)lda, strideA, b, CUDA_R_16F, (int)ldb, strideB, (void *)&fBeta, c, + CUDA_R_16F, (int)ldc, strideC, (int)batchCount, CUDA_R_32F, algo)); +} + +} // namespace + +// TODO(mkozuki): Make use of the int template parameters or discard them. +template +void CutlassGemm_FP32Accum( + cudaStream_t stream, + long m, long n, long k, float alpha, + const half* a, long lda, long long int batch_stride_A, + const half* b, long ldb, long long int batch_stride_B, + float beta, + half* c, long ldc, long long int batch_stride_C, long batch_count +) { + using Gemm = cutlass::gemm::device::GemmBatched< + /* Element type of A matrix */half, /* Layout of A matrix */LayoutA, + /* Element type of B matrix */half, /* Layout of B matrix */LayoutB, + /* Element type of C matrix */half, /* Layout of C matrix */cutlass::layout::ColumnMajor, + /* Element Accumulator*/float + >; + Gemm gemm_op; + cutlass::Status status = gemm_op({ + {static_cast(m), static_cast(n), static_cast(k)}, + {a, lda}, batch_stride_A, + {b, ldb}, batch_stride_B, + {c, ldc}, batch_stride_C, + {c, ldc}, batch_stride_C, + {alpha, beta}, static_cast(batch_count) + }, nullptr, stream); + C10_CUDA_CHECK(status != cutlass::Status::kSuccess ? cudaErrorUnknown : cudaSuccess); +} + +namespace { +void gemm_switch_fp32accum(char transa, char transb, long m, + long n, long k, float alpha, const half *a, long lda, + long strideA, const half *b, long ldb, long strideB, + float beta, half *c, long ldc, long strideC, + long batchCount) { + auto stream = c10::cuda::getCurrentCUDAStream(); + // printf("GEMM -> %c%c M: %i N: %i K: %i Alpha: %f Beta: %f\n", (transa == + // 't' ? 'T' : 'N'), (transb =='t' ? 'T' : 'N'), m, n, k, alpha, beta); + if ((transa == 't') && (transb == 'n')) { + if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x7)) { + CublasStridedBatchedGemm(transa, transb, m, n, k, alpha, a, lda, + strideA, b, ldb, strideB, beta, c, ldc, strideC, + batchCount, CUBLAS_GEMM_ALGO0_TENSOR_OP); + } + else if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else { + CublasStridedBatchedGemm(transa, transb, m, n, k, alpha, a, lda, + strideA, b, ldb, strideB, beta, c, ldc, strideC, + batchCount); + } + } else if ((transa == 'n') && (transb == 'n')) { + if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x7)) { + CublasStridedBatchedGemm(transa, transb, m, n, k, alpha, a, lda, + strideA, b, ldb, strideB, beta, c, ldc, strideC, + batchCount, CUBLAS_GEMM_ALGO0_TENSOR_OP); + } + else if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else { + CublasStridedBatchedGemm(transa, transb, m, n, k, alpha, a, lda, + strideA, b, ldb, strideB, beta, c, ldc, strideC, + batchCount); + } + } else if ((transa == 'n') && (transb == 't')) { + if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x7)) { + CublasStridedBatchedGemm(transa, transb, m, n, k, alpha, a, lda, + strideA, b, ldb, strideB, beta, c, ldc, strideC, + batchCount, CUBLAS_GEMM_ALGO0_TENSOR_OP); + } + else if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x7) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x3) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x7) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x3) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x7)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x3)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else if (!(lda & 0x1) && !(ldb & 0x1) && !(ldc & 0x1)) { + CutlassGemm_FP32Accum( + stream, m, n, k, alpha, a, lda, strideA, b, ldb, strideB, beta, c, + ldc, strideC, batchCount); + } else { + CublasStridedBatchedGemm(transa, transb, m, n, k, alpha, a, lda, + strideA, b, ldb, strideB, beta, c, ldc, strideC, + batchCount); + } + } else { + TORCH_CHECK(false, "TransA and TransB are invalid"); + } +} + +void adjustLdLevel3(char transa, char transb, int64_t m, int64_t n, int64_t k, + int64_t *lda, int64_t *ldb, int64_t *ldc) { + int transa_ = ((transa == 't') || (transa == 'T')); + int transb_ = ((transb == 't') || (transb == 'T')); + + // Note: leading dimensions generally are checked that they are > 0 and at + // least as big the result requires (even if the value won't be used). + if (n <= 1) + *ldc = std::max(m, 1); + + if (transa_) { + if (m <= 1) + *lda = std::max(k, 1); + } else { + if (k <= 1) + *lda = std::max(m, 1); + } + + if (transb_) { + if (k <= 1) + *ldb = std::max(n, 1); + } else { + if (n <= 1) + *ldb = std::max(k, 1); + } +} + +void HgemmStridedBatched(char transa, char transb, long m, + long n, long k, float alpha, const half *a, long lda, + long strideA, const half *b, long ldb, long strideB, + float beta, half *c, long ldc, long strideC, + long batchCount) { + if ((m >= INT_MAX) || (n >= INT_MAX) || (k >= INT_MAX) || (lda >= INT_MAX) || + (ldb >= INT_MAX) || (ldc >= INT_MAX) || (batchCount >= INT_MAX)) + + { + TORCH_CHECK(false, "Cublas_SgemmStridedBatched only supports m, n, k, lda, ldb, ldc, " + "batchCount" + "with the bound [val] <= %d", + INT_MAX); + } + + adjustLdLevel3(transa, transb, m, n, k, &lda, &ldb, &ldc); + + gemm_switch_fp32accum(transa, transb, m, n, k, alpha, a, lda, strideA, + b, ldb, strideB, beta, c, ldc, strideC, batchCount); +} + +} // namespace diff --git a/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p.cpp b/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p.cpp new file mode 100644 index 00000000..2797cc19 --- /dev/null +++ b/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p.cpp @@ -0,0 +1,25 @@ +/** + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nccl_p2p_cuda.cuh" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("get_unique_nccl_id", &apex::contrib::nccl_p2p::get_unique_nccl_id, "get_unique_nccl_id"); + m.def("init_nccl_comm", &apex::contrib::nccl_p2p::init_nccl_comm, "init_nccl_comm"); + m.def("left_right_halo_exchange_inplace", &apex::contrib::nccl_p2p::left_right_halo_exchange_inplace, "left_right_halo_exchange_inplace"); + m.def("left_right_halo_exchange", &apex::contrib::nccl_p2p::left_right_halo_exchange, "left_right_halo_exchange"); + m.def("add_delay", &apex::contrib::nccl_p2p::add_delay, "add_delay"); +} diff --git a/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cu b/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cu new file mode 100644 index 00000000..c386dcfb --- /dev/null +++ b/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cu @@ -0,0 +1,211 @@ +#include +#include +#include +#include +#include +#include +#include +#include "nccl.h" + +/* + * This file implements a crude but effective mechanism for copying data between tenors owned by different ranks + * on the same machine using cudaMemcpyAsync peer-to-peer transfers. + */ + +namespace { + +__global__ void AddDelay_kernel(const int delay, int* counter) { + if (blockIdx.x == 0 && threadIdx.x == 0) { + // waste time while doing something compiler can't predict, thus preventing it from optimizing away this code. + int new_counter = 0; + double elapsed = 0; + clock_t start = clock(); + do { + clock_t now = clock(); + elapsed = (double)(now - start)*1e9 / CLOCKS_PER_SEC; + ++new_counter; + } while (elapsed < (double)delay); + *counter = new_counter; + } +} + +class NcclCommWrapper +{ + private: + ncclComm_t comm; + int rank, world_size; + + ncclDataType_t get_nccl_type(at::Tensor input) + { + switch (input.scalar_type()) + { + case at::ScalarType::Half: + return ncclFloat16; + case at::ScalarType::Float: + return ncclFloat32; + case at::ScalarType::Double: + return ncclFloat64; + case at::ScalarType::Byte: + return ncclUint8; + case at::ScalarType::Char: + return ncclInt8; + case at::ScalarType::Int: + return ncclInt32; + case at::ScalarType::Long: + return ncclInt64; + case at::ScalarType::BFloat16: + return ncclBfloat16; + default: + assert(false); + } + } + + public: + NcclCommWrapper() + { + memset(&comm, 0, sizeof(ncclComm_t)); + rank = 0; + world_size = 0; + } + NcclCommWrapper(ncclUniqueId id, int my_rank, int num_ranks) + { + ncclCommInitRank(&comm, num_ranks, id, my_rank); + rank = my_rank; + world_size = num_ranks; + } + + ~NcclCommWrapper() + { + printf("ncclCommDestroy()\n"); + ncclCommDestroy(comm); + } + + void left_right_halo_exchange_inplace(int left_rank, int right_rank, at::Tensor left_output_halo, at::Tensor right_output_halo, at::Tensor left_input_halo, at::Tensor right_input_halo) + { + auto stream = at::cuda::getCurrentCUDAStream(); + ncclGroupStart(); + ncclDataType_t ncclType = get_nccl_type(left_output_halo); + bool left_zero = (left_rank < 0); + bool right_zero = (right_rank < 0); + size_t left_n = torch::numel(left_output_halo); + size_t right_n = torch::numel(right_output_halo); + assert(left_n > 0 && left_n == right_n); + if (left_zero) { + left_input_halo.zero_(); + } else { + AT_DISPATCH_ALL_TYPES_AND3(at::ScalarType::Bool, at::ScalarType::BFloat16, at::ScalarType::Half, left_output_halo.scalar_type(), "left_halo_exch", [&]() { + // send left (to my_rank - 1) + ncclSend(left_output_halo.data_ptr(), left_n, ncclType, left_rank, comm, stream); + // receive left (from my_rank - 1) + ncclRecv(left_input_halo.data_ptr(), right_n, ncclType, left_rank, comm, stream); + }); + } + if (right_zero) { + right_input_halo.zero_(); + } else { + AT_DISPATCH_ALL_TYPES_AND3(at::ScalarType::Bool, at::ScalarType::BFloat16, at::ScalarType::Half, right_output_halo.scalar_type(), "right_halo_exch", [&]() { + // send right (to my_rank + 1 ) + ncclSend(right_output_halo.data_ptr(), right_n, ncclType, right_rank, comm, stream); + // receive right (from my_rank + 1) + ncclRecv(right_input_halo.data_ptr(), left_n, ncclType, right_rank, comm, stream); + }); + } + ncclGroupEnd(); + } + + std::vector left_right_halo_exchange(int left_rank, int right_rank, at::Tensor left_output_halo, at::Tensor right_output_halo) + { + // after halo exchange: + // left_output_halo of rank+1 ends up in right_input_halo of rank + // right_output_halo of rank-1 ends up in left_input_halo of rank + auto right_input_halo = torch::empty_like(left_output_halo); + auto left_input_halo = torch::empty_like(right_output_halo); + left_right_halo_exchange_inplace(left_rank, right_rank, left_output_halo, right_output_halo, left_input_halo, right_input_halo); + return {left_input_halo, right_input_halo}; + } +}; + +class ManagedObjects +{ + public: + ManagedObjects() + { + } + ~ManagedObjects() + { + for (auto it = _nccl_comms.begin(); it != _nccl_comms.end(); ++it) + { + delete *it; + } + } + + int add_comm(NcclCommWrapper* comm) + { + int handle = _nccl_comms.size(); + _nccl_comms.push_back(comm); + return handle; + } + + NcclCommWrapper& get_comm(int handle) + { + assert(handle >= 0 && handle < _nccl_comms.size()); + return *_nccl_comms[handle]; + } + + private: + std::vector _nccl_comms; +}; +class ManagedObjects mo; + +} // end anonymous namespace + +namespace apex { namespace contrib { namespace nccl_p2p { + +at::Tensor get_unique_nccl_id(int n) +{ + ncclUniqueId id; + ncclGetUniqueId(&id); + auto id_tensor = torch::empty({n,(int)sizeof(ncclUniqueId)}, torch::dtype(torch::kUInt8).device(torch::kCPU).requires_grad(false)); + auto id_ptr = id_tensor.data_ptr(); + size_t offset = 0; + for (int i = 0; i < n; ++i) + { + ncclUniqueId id; + ncclGetUniqueId(&id); + memcpy(id_ptr+offset, &id, sizeof(ncclUniqueId)); + offset += sizeof(ncclUniqueId); + } + return id_tensor; +} + +int init_nccl_comm(at::Tensor unique_nccl_id, int my_rank, int num_ranks) +{ + ncclUniqueId id; + auto unique_nccl_id_ptr = unique_nccl_id.data_ptr(); + memcpy(&id, unique_nccl_id_ptr, sizeof(ncclUniqueId)); + NcclCommWrapper* comm = new NcclCommWrapper(id, my_rank, num_ranks); + int handle = mo.add_comm(comm); + comm = 0L; + return handle; +} + +void left_right_halo_exchange_inplace(int handle, int left_rank, int right_rank, at::Tensor left_output_halo, at::Tensor right_output_halo, at::Tensor left_input_halo, at::Tensor right_input_halo) +{ + class NcclCommWrapper& communicator = mo.get_comm(handle); + return communicator.left_right_halo_exchange_inplace(left_rank, right_rank, left_output_halo, right_output_halo, left_input_halo, right_input_halo); +} + +std::vector left_right_halo_exchange(int handle, int left_rank, int right_rank, at::Tensor left_output_halo, at::Tensor right_output_halo) +{ + class NcclCommWrapper& communicator = mo.get_comm(handle); + return communicator.left_right_halo_exchange(left_rank, right_rank, left_output_halo, right_output_halo); +} + +void add_delay(int delay) +{ + auto stream = at::cuda::getCurrentCUDAStream(); + auto t = torch::empty({1}, torch::dtype(torch::kInt32).device(torch::kCUDA)); + AddDelay_kernel<<<1,1,0,stream>>>(delay, t.data_ptr()); +} + +}}} diff --git a/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cuh b/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cuh new file mode 100644 index 00000000..6d29420b --- /dev/null +++ b/apex/apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cuh @@ -0,0 +1,45 @@ +/** + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#ifndef _nccl_p2p_h_ +#define _nccl_p2p_h_ + +namespace apex { namespace contrib { namespace nccl_p2p { +at::Tensor get_unique_nccl_id(int n); +int init_nccl_comm( + at::Tensor unique_nccl_id, + int my_rank, + int num_ranks + ); +void left_right_halo_exchange_inplace( + int handle, + int left_rank, + int right_rank, + at::Tensor left_output_halo, + at::Tensor right_output_halo, + at::Tensor left_input_halo, + at::Tensor right_input_halo); +std::vector left_right_halo_exchange( + int handle, + int left_rank, + int right_rank, + at::Tensor left_output_halo, + at::Tensor right_output_halo); +void add_delay(int delay); +}}} +#endif diff --git a/apex/apex/contrib/csrc/nccl_p2p/nccl_version.cpp b/apex/apex/contrib/csrc/nccl_p2p/nccl_version.cpp new file mode 100644 index 00000000..ad03666e --- /dev/null +++ b/apex/apex/contrib/csrc/nccl_p2p/nccl_version.cpp @@ -0,0 +1,11 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +// This file is used to check the version of NCCL detected. +#include + +#include + +std::tuple get_nccl_version(); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("get_nccl_version", &get_nccl_version); +} diff --git a/apex/apex/contrib/csrc/nccl_p2p/nccl_version_check.cu b/apex/apex/contrib/csrc/nccl_p2p/nccl_version_check.cu new file mode 100644 index 00000000..37f9d459 --- /dev/null +++ b/apex/apex/contrib/csrc/nccl_p2p/nccl_version_check.cu @@ -0,0 +1,10 @@ +// Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + +// This file is used to check the version of NCCL detected. +#include +#include + + +std::tuple get_nccl_version() { + return { int(NCCL_MAJOR), int(NCCL_MINOR) }; +} diff --git a/apex/apex/contrib/csrc/optimizers/fused_adam_cuda.cpp b/apex/apex/contrib/csrc/optimizers/fused_adam_cuda.cpp new file mode 100644 index 00000000..1370cb20 --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/fused_adam_cuda.cpp @@ -0,0 +1,86 @@ +#include + +// CUDA forward declaration +void fused_strided_check_finite(at::Tensor & overflow_flag, at::Tensor & p_copy, int stride, int clear_overflow_first); + +void fused_adam_cuda(at::Tensor & p, at::Tensor & p_copy, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay); +void fused_reversible_adam_cuda(at::Tensor & p, at::Tensor & p_copy, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay); +void fused_maybe_adam_undo_cuda(at::Tensor & overflow_flag, at::Tensor & p, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay); + +void fused_adam_cuda_mt(int chunk_size, at::Tensor overflow_flag, std::vector> tensor_lists, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay); + +void maybe_cast_cuda(at::Tensor & overflow_flag, at::Tensor & p_in, at::Tensor & p_out); +void maybe_cast_cuda_mt(int chunk_size, at::Tensor overflow_flag, std::vector> tensor_lists); + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +// C++ interface +void strided_check_finite( + at::Tensor& overflow_flag, + at::Tensor& p_copy, + int stride, + int clear_overflow_first + ) { + CHECK_INPUT(p_copy); + fused_strided_check_finite(overflow_flag, p_copy, stride, clear_overflow_first); +} +void adam(at::Tensor & p, at::Tensor & p_copy, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay) { + CHECK_INPUT(p); + if (p_copy.numel() > 0) CHECK_INPUT(p_copy); + CHECK_INPUT(m); + CHECK_INPUT(v); + CHECK_INPUT(g); + int64_t num_elem = p.numel(); + TORCH_CHECK(m.numel() == num_elem, "number of elements in m and p tensors should be equal"); + TORCH_CHECK(v.numel() == num_elem, "number of elements in v and p tensors should be equal"); + TORCH_CHECK(g.numel() == num_elem, "number of elements in g and p tensors should be equal"); + TORCH_CHECK(p_copy.numel() == num_elem || p_copy.numel() == 0, "number of elements in p_copy and p tensors should be equal, or p_copy should be empty"); + + fused_adam_cuda(p, p_copy, m, v, g, lr, beta1, beta2, eps, grad_scale, step, mode, bias_correction, decay); +} +void reversible_adam(at::Tensor & p, at::Tensor & p_copy, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay) { + CHECK_INPUT(p); + if (p_copy.numel() > 0) CHECK_INPUT(p_copy); + CHECK_INPUT(m); + CHECK_INPUT(v); + CHECK_INPUT(g); + int64_t num_elem = p.numel(); + TORCH_CHECK(m.numel() == num_elem, "number of elements in m and p tensors should be equal"); + TORCH_CHECK(v.numel() == num_elem, "number of elements in v and p tensors should be equal"); + TORCH_CHECK(g.numel() == num_elem, "number of elements in g and p tensors should be equal"); + TORCH_CHECK(p_copy.numel() == num_elem || p_copy.numel() == 0, "number of elements in p_copy and p tensors should be equal, or p_copy should be empty"); + + fused_reversible_adam_cuda(p, p_copy, m, v, g, lr, beta1, beta2, eps, grad_scale, step, mode, bias_correction, decay); +} +void maybe_adam_undo(at::Tensor & overflow_flag, at::Tensor & p, at::Tensor & m, at::Tensor & v, at::Tensor & g, float lr, float beta1, float beta2, float eps, float grad_scale, int step, int mode, int bias_correction, float decay) { + CHECK_INPUT(p); + CHECK_INPUT(m); + CHECK_INPUT(v); + CHECK_INPUT(g); + int64_t num_elem = p.numel(); + TORCH_CHECK(m.numel() == num_elem, "number of elements in m and p tensors should be equal"); + TORCH_CHECK(v.numel() == num_elem, "number of elements in v and p tensors should be equal"); + TORCH_CHECK(g.numel() == num_elem, "number of elements in g and p tensors should be equal"); + + fused_maybe_adam_undo_cuda(overflow_flag, p, m, v, g, lr, beta1, beta2, eps, grad_scale, step, mode, bias_correction, decay); +} +void maybe_cast(at::Tensor & overflow_flag, at::Tensor & p_in, at::Tensor & p_out) { + CHECK_INPUT(p_in); + CHECK_INPUT(p_out); + int64_t num_elem = p_in.numel(); + TORCH_CHECK(p_out.numel() == num_elem, "number of elements in p_in and p_out should be equal"); + + maybe_cast_cuda(overflow_flag, p_in, p_out); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("strided_check_finite", &strided_check_finite, "Strided finite check."); + m.def("adam", &adam, "Adam optimized CUDA implementation."); + m.def("reversible_adam", &reversible_adam, "Reversible Adam optimized CUDA implementation."); + m.def("adam_mt", &fused_adam_cuda_mt, "Multi tensor Adam optimized CUDA implementation."); + m.def("maybe_adam_undo", &maybe_adam_undo, "Undo function for Adam optimized CUDA implementation."); + m.def("maybe_cast", &maybe_cast, "Unpack byte tensor containing e5m2 floats."); + m.def("maybe_cast_mt", &maybe_cast_cuda_mt, "Unpack byte tensor containing e5m2 floats."); +} diff --git a/apex/apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu b/apex/apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu new file mode 100644 index 00000000..9209df4b --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu @@ -0,0 +1,1037 @@ +#include +#include +#include +#include + +#include "ATen/ATen.h" +#include "ATen/cuda/CUDAContext.h" +#include "ATen/cuda/detail/IndexUtils.cuh" +#include "ATen/TensorUtils.h" +// #include "ATen/Type.h" +#include "ATen/AccumulateType.h" + +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +#include "type_shim.h" + +typedef enum{ + ADAM_MODE_0 =0, // eps under square root + ADAM_MODE_1 =1 // eps outside square root +} adamMode_t; + +template +__global__ void adam_cuda_kernel( + T* __restrict__ p, + GRAD_T* __restrict__ p_copy, // For mixed precision training, pass NULL if not needed + T* __restrict__ m, + T* __restrict__ v, + const GRAD_T * __restrict__ g, + const float b1, + const float b2, + const float eps, + const float grad_scale, + const float step_size, + const size_t tsize, + adamMode_t mode, + const float decay) +{ + //Assuming 2D grids and 2D blocks + const int blockId = gridDim.x * blockIdx.y + blockIdx.x; + const int threadsPerBlock = blockDim.x * blockDim.y; + const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x; + const int i = (blockId * threadsPerBlock + threadIdInBlock); + const int totThreads = gridDim.x*gridDim.y*threadsPerBlock; + + for (int j = i; j < tsize; j+=totThreads) { + T scaled_grad = g[j]/grad_scale; + m[j] = b1*m[j] + (1-b1)*scaled_grad; + v[j] = b2*v[j] + (1-b2)*scaled_grad*scaled_grad; + float denom; + if (mode == ADAM_MODE_0) + denom = sqrtf(v[j] + eps); + else // Mode 1 + denom = sqrtf(v[j]) + eps; + float update = (m[j]/denom) + (decay*p[j]); + p[j] = p[j] - (step_size*update); + if (p_copy != NULL) p_copy[j] = (GRAD_T) p[j]; + } +} + +template +struct AdamFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata& tl, + const float b1, + const float b2, + const float eps, + const float grad_scale, + const float step_size, + adamMode_t mode, + const float decay) + { + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + T* p = (T *)tl.addresses[0][tensor_loc]; + p += chunk_idx*chunk_size; + T* m = (T *)tl.addresses[1][tensor_loc]; + m += chunk_idx*chunk_size; + T* v = (T *)tl.addresses[2][tensor_loc]; + v += chunk_idx*chunk_size; + GRAD_T* g = (GRAD_T *)tl.addresses[3][tensor_loc]; + g += chunk_idx*chunk_size; + GRAD_T* p_copy = NULL; + if (DEPTH == 5) { + p_copy = (GRAD_T *)tl.addresses[4][tensor_loc]; + p_copy += chunk_idx*chunk_size; + } + + n -= chunk_idx*chunk_size; + + T incoming_p[ILP]; + T incoming_m[ILP]; + T incoming_v[ILP]; + T incoming_g[ILP]; + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && + chunk_size % ILP == 0 && + is_aligned(p) && + is_aligned(m) && + is_aligned(v) && + is_aligned(g) && + is_aligned(p_copy)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + GRAD_T tmp_g[ILP]; + load_store(incoming_p, p, 0, i_start); + load_store(incoming_m, m, 0, i_start); + load_store(incoming_v, v, 0, i_start); + load_store(tmp_g, g, 0, i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + incoming_g[ii] = static_cast(tmp_g[ii]); + T scaled_grad = incoming_g[ii]/grad_scale; + incoming_m[ii] = b1*incoming_m[ii] + (1-b1)*scaled_grad; + incoming_v[ii] = b2*incoming_v[ii] + (1-b2)*scaled_grad*scaled_grad; + float denom; + if (mode == ADAM_MODE_0) + denom = sqrtf(incoming_v[ii] + eps); + else // Mode 1 + denom = sqrtf(incoming_v[ii]) + eps; + float update = (incoming_m[ii]/denom) + (decay*incoming_p[ii]); + incoming_p[ii] = incoming_p[ii] - (step_size*update); + if (DEPTH == 5) tmp_g[ii] = static_cast(incoming_p[ii]); + } + load_store(p, incoming_p, i_start, 0); + load_store(m, incoming_m, i_start, 0); + load_store(v, incoming_v, i_start, 0); + if (DEPTH == 5) load_store(p_copy, tmp_g, i_start, 0); + } + } + else + { + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) { + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + incoming_p[ii] = 0; + incoming_m[ii] = 0; + incoming_v[ii] = 0; + incoming_g[ii] = 0; + + int i = i_start + threadIdx.x + ii*blockDim.x; + if (i < n && i < chunk_size) { + incoming_p[ii] = p[i]; + incoming_m[ii] = m[i]; + incoming_v[ii] = v[i]; + incoming_g[ii] = static_cast(g[i]); + } + } + + // note for clarification to future michael: + // From a pure memory dependency perspective, there's likely no point unrolling + // the write loop, since writes just fire off once their LDGs arrive. + // Put another way, the STGs are dependent on the LDGs, but not on each other. + // There is still compute ILP benefit from unrolling the loop though. +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int j = i_start + threadIdx.x + ii*blockDim.x; + + if(j < n && j < chunk_size) { + T scaled_grad = incoming_g[ii]/grad_scale; + m[j] = b1*incoming_m[ii] + (1-b1)*scaled_grad; + v[j] = b2*incoming_v[ii] + (1-b2)*scaled_grad*scaled_grad; + float denom; + if (mode == ADAM_MODE_0) + denom = sqrtf(v[j] + eps); + else // Mode 1 + denom = sqrtf(v[j]) + eps; + float update = (m[j]/denom) + (decay*incoming_p[ii]); + p[j] = incoming_p[ii] - (step_size*update); + if (DEPTH == 5) p_copy[j] = (GRAD_T) p[j]; + } + } + } + } + } +}; + +void fused_adam_cuda( + at::Tensor & p, + at::Tensor & p_copy, + at::Tensor & m, + at::Tensor & v, + at::Tensor & g, + float lr, + float beta1, + float beta2, + float eps, + float grad_scale, + int step, + int mode, + int bias_correction, + float decay) +{ +// using namespace at; + + //Get tensor size + int tsize = p.numel(); + //Determine #threads and #blocks + const int threadsPerBlock = 512; + const dim3 blocks((tsize+threadsPerBlock-1)/threadsPerBlock); + TORCH_CHECK(at::cuda::detail::canUse32BitIndexMath(p), "parameter tensor is too large to be indexed with int32"); + //Constants + float step_size = 0; + if (bias_correction == 1) { + const float bias_correction1 = 1 - std::pow(beta1, step); + const float bias_correction2 = 1 - std::pow(beta2, step); + step_size = lr * std::sqrt(bias_correction2)/bias_correction1; + } + else { + step_size = lr; + } + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (g.scalar_type() == at::ScalarType::Half) { +//all other values should be fp32 for half gradients + TORCH_CHECK(p.scalar_type() == at::ScalarType::Float, "expected parameter to be of float type"); +//dispatch is done on the gradient type + using namespace at; // prevents "toString is undefined" errors + DISPATCH_FLOAT_AND_HALF(g.scalar_type(), 0, "adam_cuda_kernel", + using accscalar_t = at::acc_type; + adam_cuda_kernel<<>>( + p.DATA_PTR(), + p_copy.numel() ? p_copy.DATA_PTR() : NULL, + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } else { + using namespace at; + DISPATCH_DOUBLE_AND_FLOAT(g.scalar_type(), 0, "adam_cuda_kernel", + adam_cuda_kernel<<>>( + p.DATA_PTR(), + NULL, //don't output p_copy for fp32, it's wasted write + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } + C10_CUDA_CHECK(cudaGetLastError()); + +} + +void fused_adam_cuda_mt( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, // p, m, v, g, p_copy + float lr, + float beta1, + float beta2, + float eps, + float grad_scale, + int step, + int mode, + int bias_correction, + float decay) { + + //Constants + float step_size = 0; + if (bias_correction == 1) { + const float bias_correction1 = 1 - std::pow(beta1, step); + const float bias_correction2 = 1 - std::pow(beta2, step); + step_size = lr * std::sqrt(bias_correction2)/bias_correction1; + } + else { + step_size = lr; + } + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + size_t tl_sz = tensor_lists.size(); + TORCH_CHECK(tl_sz == 4 || tl_sz == 5, "expected tensor lists of size 4 or 5"); + + if (tensor_lists[3][0].scalar_type() == at::ScalarType::Half) { +//alher values should be fp32 for half gradients + TORCH_CHECK(tensor_lists[0][0].scalar_type() == at::ScalarType::Float, "expected parameter to be of float type"); +//dich is done on the gradient type + if (tl_sz == 5) { + DISPATCH_FLOAT_AND_HALF(tensor_lists[3][0].scalar_type(), 0, "adam_cuda_mt_kernel", + using accscalar_t = at::acc_type; + multi_tensor_apply<5>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamFunctor<5, accscalar_t, scalar_t_0>(), + beta1, + beta2, + eps, + grad_scale, + step_size, + (adamMode_t) mode, + decay); + ); + } else { + DISPATCH_FLOAT_AND_HALF(tensor_lists[3][0].scalar_type(), 0, "adam_cuda_mt_kernel", + using accscalar_t = at::acc_type; + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamFunctor<4, accscalar_t, scalar_t_0>(), + beta1, + beta2, + eps, + grad_scale, + step_size, + (adamMode_t) mode, + decay); + ); + } + } else { + if (tl_sz == 5) { + DISPATCH_DOUBLE_AND_FLOAT(tensor_lists[3][0].scalar_type(), 0, "adam_cuda_mt_kernel", + multi_tensor_apply<5>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamFunctor<5, scalar_t_0, scalar_t_0>(), + beta1, + beta2, + eps, + grad_scale, + step_size, + (adamMode_t) mode, + decay); + ); + } else { + DISPATCH_DOUBLE_AND_FLOAT(tensor_lists[3][0].scalar_type(), 0, "adam_cuda_mt_kernel", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamFunctor<4, scalar_t_0, scalar_t_0>(), + beta1, + beta2, + eps, + grad_scale, + step_size, + (adamMode_t) mode, + decay); + ); + } + } + C10_CUDA_CHECK(cudaGetLastError()); +} + +template +__device__ void convert(const FROM_T vi, TO_T& vo) +{ + vo = static_cast(vi); +} + +template <> +__device__ void convert(const float vi, uint8_t& vo) +{ + union S + { + float as_float; + int as_int; + }; + S s; + s.as_float = vi; + s.as_int = s.as_int & 0xFF800000; + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_half = static_cast(vi + s.as_float / 8.0f); + vo = t.as_byte[1]; +} + +template <> +__device__ void convert(const uint8_t vi, float& vo) +{ + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_byte[0] = 0; + t.as_byte[1] = vi; + vo = static_cast(t.as_half); +} + +template <> +__device__ void convert(const at::Half vi, uint8_t& vo) +{ + union S + { + float as_float; + int as_int; + }; + S s; + s.as_float = static_cast(vi); + s.as_int = s.as_int & 0xFF800000; + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_half = static_cast(vi + s.as_float / 8.0f); + vo = t.as_byte[1]; +} + +template <> +__device__ void convert(const uint8_t vi, at::Half& vo) +{ + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_byte[0] = 0; + t.as_byte[1] = vi; + vo = t.as_half; +} + +template +__global__ void strided_check_finite_cuda_kernel( + volatile int* noop_gmem, + GRAD_T* __restrict__ p_copy, + const size_t tsize, + int stride, + int clear_overflow_first) +{ + //Assuming 2D grids and 2D blocks + const int blockId = gridDim.x * blockIdx.y + blockIdx.x; + const int threadsPerBlock = blockDim.x * blockDim.y; + const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x; + const int i = (blockId * threadsPerBlock + threadIdInBlock) * stride; + const int totThreads = gridDim.x*gridDim.y*threadsPerBlock*stride; + + if (clear_overflow_first) { + if (i == 0) { + *noop_gmem = 0; + } + __syncthreads(); + } + + for (int j = i; j < tsize; j+=totThreads) { + GRAD_T pi = p_copy[j]; + if (!isfinite(pi)) { + *noop_gmem = 1; + } + } +} +template <> +__global__ void strided_check_finite_cuda_kernel( + volatile int* noop_gmem, + uint8_t* __restrict__ p_copy, + const size_t tsize, + int stride, + int clear_overflow_first) +{ + //Assuming 2D grids and 2D blocks + const int blockId = gridDim.x * blockIdx.y + blockIdx.x; + const int threadsPerBlock = blockDim.x * blockDim.y; + const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x; + const int i = (blockId * threadsPerBlock + threadIdInBlock) * stride; + const int totThreads = gridDim.x*gridDim.y*threadsPerBlock*stride; + + if (clear_overflow_first) { + if (i == 0) { + *noop_gmem = 0; + } + __syncthreads(); + } + + for (int j = i; j < tsize; j+=totThreads) { + at::Half pi; + convert(p_copy[j], pi); + if (!isfinite(pi)) { + *noop_gmem = 1; + } + } +} + +template +__global__ void maybe_cast_kernel( + volatile int* overflow_flag, + const FROM_T* p_in, + TO_T* p_out, + const size_t tsize) +{ + if (overflow_flag && *overflow_flag != 0) return; + + //Assuming 2D grids and 2D blocks + const int blockId = gridDim.x * blockIdx.y + blockIdx.x; + const int threadsPerBlock = blockDim.x * blockDim.y; + const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x; + const int i = (blockId * threadsPerBlock + threadIdInBlock); + const int totThreads = gridDim.x*gridDim.y*threadsPerBlock; + + FROM_T pi[ILP]; + TO_T po[ILP]; + + for(int j_start = 0; j_start < tsize; j_start+=totThreads*ILP) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + pi[ii] = 0; + + int j = j_start + i + totThreads*ii; + if (j < tsize) { + pi[ii] = p_in[j]; + } + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + convert(pi[ii], po[ii]); + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int j = j_start + i + totThreads*ii; + if (j < tsize) { + p_out[j] = po[ii]; + } + } + } +} + +template +__global__ void reversible_adam_cuda_kernel( + T* __restrict__ p, + REDU_T* __restrict__ p_copy, // For mixed precision training, pass NULL if not needed + T* __restrict__ m, + T* __restrict__ v, + const GRAD_T * __restrict__ g, + const float b1, + const float b2, + const float eps, + const float grad_scale, + const float step_size, + const size_t tsize, + adamMode_t mode, + const float decay) +{ + //Assuming 2D grids and 2D blocks + const int blockId = gridDim.x * blockIdx.y + blockIdx.x; + const int threadsPerBlock = blockDim.x * blockDim.y; + const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x; + const int i = (blockId * threadsPerBlock + threadIdInBlock); + const int totThreads = gridDim.x*gridDim.y*threadsPerBlock; + + T mi[ILP]; + T vi[ILP]; + T pi[ILP]; + T gi[ILP]; + + bool overflow = false; + for(int j_start = 0; j_start < tsize; j_start+=totThreads*ILP) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + mi[ii] = T(0); + vi[ii] = T(0); + pi[ii] = T(0); + gi[ii] = GRAD_T(0); + + int j = j_start + i + totThreads*ii; + if (j < tsize) { + pi[ii] = p[j]; + mi[ii] = m[j]; + vi[ii] = v[j]; + gi[ii] = static_cast(g[j]); + } + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + T scaled_grad = gi[ii]/grad_scale; + if (isfinite(scaled_grad)) { + mi[ii] = b1*mi[ii] + (1-b1)*scaled_grad; + vi[ii] = b2*vi[ii] + (1-b2)*scaled_grad*scaled_grad; + float denom; + if (mode == ADAM_MODE_0) + denom = sqrtf(vi[ii] + eps); + else // Mode 1 + denom = sqrtf(vi[ii]) + eps; + float update = (mi[ii]/denom) + (decay*pi[ii]); + pi[ii] = pi[ii] - (step_size*update); + } else { + overflow = true; + } + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int j = j_start + i + totThreads*ii; + if (j < tsize) { + m[j] = mi[ii]; + v[j] = vi[ii]; + p[j] = pi[ii]; + if (p_copy != NULL) { + convert(pi[ii], p_copy[j]); + } + } + } + } + + if (p_copy != NULL) { + __syncthreads(); + if (overflow) { + convert(float(INFINITY), p_copy[0]); + } + } +} + +template +__global__ void maybe_adam_undo_cuda_kernel( + volatile int* overflow_flag, + T* __restrict__ p, + T* __restrict__ m, + T* __restrict__ v, + const GRAD_T * __restrict__ g, + const float b1, + const float b2, + const float eps, + const float grad_scale, + const float step_size, + const size_t tsize, + adamMode_t mode, + const float decay) +{ + // NB! Skip undo kernel when overflow flag is NOT set + if (overflow_flag && *overflow_flag == 0) return; + + //Assuming 2D grids and 2D blocks + const int blockId = gridDim.x * blockIdx.y + blockIdx.x; + const int threadsPerBlock = blockDim.x * blockDim.y; + const int threadIdInBlock = threadIdx.y * blockDim.x + threadIdx.x; + const int i = (blockId * threadsPerBlock + threadIdInBlock); + const int totThreads = gridDim.x*gridDim.y*threadsPerBlock; + + T mi[ILP]; + T vi[ILP]; + T pi[ILP]; + T gi[ILP]; + + for(int j_start = 0; j_start < tsize; j_start+=totThreads*ILP) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + mi[ii] = T(0); + vi[ii] = T(0); + pi[ii] = T(0); + gi[ii] = GRAD_T(0); + + int j = j_start + i*ILP; + if (j < tsize) { + pi[ii] = p[j]; + mi[ii] = m[j]; + vi[ii] = v[j]; + gi[ii] = static_cast(g[j]); + } + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + T scaled_grad = gi[ii]/grad_scale; + if (isfinite(scaled_grad)) { + float denom; + if (mode == ADAM_MODE_0) + denom = sqrtf(vi[ii] + eps); + else // Mode 1 + denom = sqrtf(vi[ii]) + eps; + pi[ii] = (pi[ii] + step_size*(mi[ii]/denom)) / (1.0f - step_size*decay); + mi[ii] = (mi[ii] - (1-b1)*scaled_grad) / b1; + vi[ii] = (vi[ii] - (1-b2)*scaled_grad*scaled_grad) / b2; + // Make sure round off errors don't create (small) negative value. + // This can happen if we have to revert the very first step. + vi[ii] = vi[ii] >= 0.0f ? vi[ii] : 0.0f; + } + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int j = j_start + i*ILP; + if (j < tsize) { + m[j] = mi[ii]; + v[j] = vi[ii]; + p[j] = pi[ii]; + } + } + } +} + +template +struct MaybeCastFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* overflow_flag, + TensorListMetadata& tl) + { + if (overflow_flag && *overflow_flag != 0) return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + FROM_T* p_in = (FROM_T *)tl.addresses[0][tensor_loc]; + p_in += chunk_idx*chunk_size; + TO_T* p_out = (TO_T *)tl.addresses[1][tensor_loc]; + p_out += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + int dim = chunk_size < n ? chunk_size : n; + + FROM_T pi[ILP]; + TO_T po[ILP]; + + for(int j_start = 0; j_start < dim; j_start+=blockDim.x*ILP) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + pi[ii] = FROM_T(0); + int j = j_start + threadIdx.x + ii*blockDim.x; + if (j < dim) { + pi[ii] = p_in[j]; + } + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + convert(pi[ii], po[ii]); + } + +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int j = j_start + threadIdx.x + ii*blockDim.x; + if (j < dim) { + p_out[j] = po[ii]; + } + } + } + } +}; + +void fused_strided_check_finite( + at::Tensor & overflow_flag, + at::Tensor & p_copy, + int stride, + int clear_overflow_first) +{ + //Get tensor size + int tsize = p_copy.numel(); + int niter = (tsize + stride - 1) / stride; + + //Determine #threads and #blocks + const int threadsPerBlock = 512; + //In order to avoid race condition, blocks must be 1 when clear_overflow_first flag is set. + const dim3 blocks(clear_overflow_first ? 1 : (niter+threadsPerBlock-1)/threadsPerBlock); + TORCH_CHECK(at::cuda::detail::canUse32BitIndexMath(p_copy), "parameter tensor is too large to be indexed with int32"); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + using namespace at; // prevents "toString is undefined" errors + DISPATCH_FLOAT_HALF_AND_BYTE(p_copy.scalar_type(), 0, "check_finite_cuda_kernel", + strided_check_finite_cuda_kernel<<>>( + overflow_flag.DATA_PTR(), + p_copy.DATA_PTR(), + tsize, + stride, + clear_overflow_first); + ); + C10_CUDA_CHECK(cudaGetLastError()); +} + +void fused_reversible_adam_cuda( + at::Tensor & p, + at::Tensor & p_copy, + at::Tensor & m, + at::Tensor & v, + at::Tensor & g, + float lr, + float beta1, + float beta2, + float eps, + float grad_scale, + int step, + int mode, + int bias_correction, + float decay) +{ +// using namespace at; + + //Get tensor size + int tsize = p.numel(); + //Determine #threads and #blocks + const int threadsPerBlock = 512; + const dim3 blocks((tsize+threadsPerBlock-1)/threadsPerBlock); + TORCH_CHECK(at::cuda::detail::canUse32BitIndexMath(p), "parameter tensor is too large to be indexed with int32"); + //Constants + float step_size = 0; + if (bias_correction == 1) { + const float bias_correction1 = 1 - std::pow(beta1, step); + const float bias_correction2 = 1 - std::pow(beta2, step); + step_size = lr * std::sqrt(bias_correction2)/bias_correction1; + } + else { + step_size = lr; + } + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (g.scalar_type() == at::ScalarType::Half) { + //all other values should be fp32 for half gradients + TORCH_CHECK(p.scalar_type() == at::ScalarType::Float, "expected parameter to be of float type"); + //dispatch is done on the gradient type + using namespace at; // prevents "toString is undefined" errors + if (p_copy.numel() == 0 || p_copy.scalar_type() == g.scalar_type()) { + DISPATCH_FLOAT_AND_HALF(g.scalar_type(), 0, "adam_cuda_kernel", + using accscalar_t = at::acc_type; + reversible_adam_cuda_kernel<<>>( + p.DATA_PTR(), + p_copy.numel() ? p_copy.DATA_PTR() : NULL, + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } else { + TORCH_CHECK(p_copy.scalar_type() == at::ScalarType::Byte, "expected parameter to be of byte type"); + DISPATCH_FLOAT_AND_HALF(g.scalar_type(), 0, "adam_cuda_e5m2_kernel", + using accscalar_t = at::acc_type; + reversible_adam_cuda_kernel<<>>( + p.DATA_PTR(), + p_copy.DATA_PTR(), + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } + } else { + using namespace at; + DISPATCH_DOUBLE_AND_FLOAT(g.scalar_type(), 0, "adam_cuda_kernel", + reversible_adam_cuda_kernel<<>>( + p.DATA_PTR(), + NULL, //don't output p_copy for fp32, it's wasted write + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } + C10_CUDA_CHECK(cudaGetLastError()); +} + +void maybe_cast_cuda( + at::Tensor & overflow_flag, + at::Tensor & p_in, + at::Tensor & p_out) +{ + //Get tensor size + int tsize = p_in.numel(); + TORCH_CHECK(tsize == p_out.numel(), "p_in.numel() must equal p_out.numel()"); + //Determine #threads and #blocks + const int threadsPerBlock = 512; + const dim3 blocks((tsize+threadsPerBlock-1)/threadsPerBlock); + TORCH_CHECK(at::cuda::detail::canUse32BitIndexMath(p_in), "parameter tensor is too large to be indexed with int32"); + //Constants + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + DISPATCH_FLOAT_HALF_AND_BYTE(p_in.scalar_type(), 0, "maybe_cast_cuda" + DISPATCH_FLOAT_HALF_AND_BYTE(p_out.scalar_type(), 1, "maybe_cast_cuda", + maybe_cast_kernel<<>>( + overflow_flag.numel() ? overflow_flag.DATA_PTR() : NULL, + p_in.DATA_PTR(), + p_out.DATA_PTR(), + tsize); )) + C10_CUDA_CHECK(cudaGetLastError()); +} + +void maybe_cast_cuda_mt( + int chunk_size, + at::Tensor overflow_flag, + std::vector> tensor_lists) // p_in, p_out +{ + //Constants + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + size_t tl_sz = tensor_lists.size(); + TORCH_CHECK(tl_sz == 2, "expected tensor lists of size 2"); + + DISPATCH_FLOAT_HALF_AND_BYTE(tensor_lists[0][0].scalar_type(), 0, "maybe_cast_cuda_mt_kernel", + DISPATCH_FLOAT_HALF_AND_BYTE(tensor_lists[1][0].scalar_type(), 1, "maybe_cast_cuda_mt_kernel", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + overflow_flag, + tensor_lists, + MaybeCastFunctor<2, scalar_t_0, scalar_t_1>()); )) + C10_CUDA_CHECK(cudaGetLastError()); +} + +void fused_maybe_adam_undo_cuda( + at::Tensor & overflow_flag, + at::Tensor & p, + at::Tensor & m, + at::Tensor & v, + at::Tensor & g, + float lr, + float beta1, + float beta2, + float eps, + float grad_scale, + int step, + int mode, + int bias_correction, + float decay) +{ + //Get tensor size + int tsize = p.numel(); + //Determine #threads and #blocks + const int threadsPerBlock = 512; + const dim3 blocks((tsize+threadsPerBlock-1)/threadsPerBlock); + TORCH_CHECK(at::cuda::detail::canUse32BitIndexMath(p), "parameter tensor is too large to be indexed with int32"); + //Constants + float step_size = 0; + if (bias_correction == 1) { + const float bias_correction1 = 1 - std::pow(beta1, step); + const float bias_correction2 = 1 - std::pow(beta2, step); + step_size = lr * std::sqrt(bias_correction2)/bias_correction1; + } + else { + step_size = lr; + } + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (g.scalar_type() == at::ScalarType::Half) { + //all other values should be fp32 for half gradients + TORCH_CHECK(p.scalar_type() == at::ScalarType::Float, "expected parameter to be of float type"); + //dispatch is done on the gradient type + using namespace at; // prevents "toString is undefined" errors + DISPATCH_FLOAT_AND_HALF(g.scalar_type(), 0, "adam_cuda_kernel", + using accscalar_t = at::acc_type; + maybe_adam_undo_cuda_kernel<<>>( + overflow_flag.numel() ? overflow_flag.DATA_PTR() : NULL, + p.DATA_PTR(), + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } else { + using namespace at; + DISPATCH_DOUBLE_AND_FLOAT(g.scalar_type(), 0, "adam_cuda_kernel", + maybe_adam_undo_cuda_kernel<<>>( + overflow_flag.numel() ? overflow_flag.DATA_PTR() : NULL, + p.DATA_PTR(), + m.DATA_PTR(), + v.DATA_PTR(), + g.DATA_PTR(), + beta1, + beta2, + eps, + grad_scale, + step_size, + tsize, + (adamMode_t) mode, + decay); + ); + } + C10_CUDA_CHECK(cudaGetLastError()); +} diff --git a/apex/apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp b/apex/apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp new file mode 100644 index 00000000..98a24115 --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp @@ -0,0 +1,21 @@ +#include + +void multi_tensor_lamb_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + const float global_grad_norm, + const float max_grad_norm); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("lamb", &multi_tensor_lamb_cuda, "Computes and apply update for LAMB optimizer"); +} diff --git a/apex/apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu b/apex/apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu new file mode 100644 index 00000000..3bb93b03 --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu @@ -0,0 +1,294 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +typedef enum{ + MOMENT_MODE_0 =0, // L2 regularization mode + MOMENT_MODE_1 =1 // Decoupled weight decay mode +} adamMode_t; + +std::tuple multi_tensor_l2norm_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python); + +using MATH_T = float; + +template +struct LAMBStage1Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<4>& tl, + const float beta1, + const float beta2, + const float beta3, + const float beta1_correction, + const float beta2_correction, + const float epsilon, + adamMode_t mode, + const float decay, + const float global_grad_norm, + const float max_global_grad_norm) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + float clipped_global_grad_norm = global_grad_norm > max_global_grad_norm ? global_grad_norm / max_global_grad_norm : 1.0f; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + T* m = (T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + T* v = (T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i]; + // special ?optimization? for lamb stage 1 + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = p[i]; + } + r_m[ii] = m[i]; + r_v[ii] = v[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + g[i] = r_p[ii]; + m[i] = r_m[ii]; + v[i] = r_v[ii]; + } + } + } + } +}; + +// Step 2 reads in 'update' value and per-tensor param_norm and update_norm. +// It computes new parameter value. +template +struct LAMBStage2Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<2>& tl, + const float* per_tensor_param_norm, + const float* per_tensor_update_norm, + const float learning_rate, + const float decay) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + MATH_T ratio = learning_rate; + // apply adaptive learning rate to parameters with non-zero weight decay + if (decay != 0.0) + { + float param_norm = per_tensor_param_norm[tensor_num]; + float update_norm = per_tensor_update_norm[tensor_num]; + ratio = (update_norm != 0.0f && param_norm != 0.0f) ? learning_rate * (param_norm / update_norm) : learning_rate; + } + + T* update = (T*)tl.addresses[0][tensor_loc]; + update += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_p[ILP]; + MATH_T r_update[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_p[ii] = p[i]; + r_update[ii] = update[i]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = r_p[ii] - (ratio * r_update[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + } + } + } + } +}; + + +void multi_tensor_lamb_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + const float global_grad_norm, + const float max_grad_norm) +{ + using namespace at; + // Master weight and 32bit momentum(potentially changing) is not handled by this + // So we assume every tensor are all in the same type + + // Handle bias correction mode + float bias_correction1 = 1.0f, bias_correction2 = 1.0f; + if (bias_correction == 1) { + bias_correction1 = 1 - std::pow(beta1, step); + bias_correction2 = 1 - std::pow(beta2, step); + } + + // Handle grad averaging mode + float beta3 = 1.0f; + if (grad_averaging == 1) beta3 = 1 - beta1; + + std::vector> grad_list(tensor_lists.begin(), tensor_lists.begin()+1); + std::vector> param_list(tensor_lists.begin()+1, tensor_lists.begin()+2); + + // Compute per tensor param norm + auto param_norm_tuple = multi_tensor_l2norm_cuda(chunk_size, noop_flag, param_list, true); + + // We now in-place modify grad to store update before compute its norm + // Generally this is not a issue since people modify grad in step() method all the time + // We can also grab list of empty tensor to avoid this, but I'd like to save space/cpu code + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_1", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + LAMBStage1Functor(), + beta1, + beta2, + beta3, // 1-beta1 or 1 depends on averaging mode + bias_correction1, + bias_correction2, + epsilon, + (adamMode_t) mode, + weight_decay, + global_grad_norm, + max_grad_norm); ) + + // Compute update norms + auto update_norm_tuple = multi_tensor_l2norm_cuda(chunk_size, noop_flag, grad_list, true); + + std::vector> grad_param_list(tensor_lists.begin(), tensor_lists.begin()+2); + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_2", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + noop_flag, + grad_param_list, + LAMBStage2Functor(), + std::get<1>(param_norm_tuple).DATA_PTR(), + std::get<1>(update_norm_tuple).DATA_PTR(), + lr, + weight_decay); ) + + AT_CUDA_CHECK(cudaGetLastError()); + +} diff --git a/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam.cpp b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam.cpp new file mode 100644 index 00000000..5be8b284 --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam.cpp @@ -0,0 +1,40 @@ +#include + +void multi_tensor_fused_adam_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor grad_scale, + float lr, + float beta1, + float beta2, + float eps, + int step, + int mode, + int bias_correction, + float weight_decay); + +void multi_tensor_fused_adam_with_param_remainders_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor grad_scale, + float lr, + float beta1, + float beta2, + float eps, + int step, + int mode, + int bias_correction, + float weight_decay); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("multi_tensor_fused_adam", + &multi_tensor_fused_adam_cuda, + "CUDA kernels for multi-tensor Adam, " + "with param copy"); + m.def("multi_tensor_fused_adam_with_param_remainders", + &multi_tensor_fused_adam_with_param_remainders_cuda, + "CUDA kernel for multi-tensor Adam, " + "with stored param remainders and param copy"); +} diff --git a/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam_kernel.cu b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam_kernel.cu new file mode 100644 index 00000000..f769e57c --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_adam_kernel.cu @@ -0,0 +1,430 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include +#include +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(const T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store( + T* dst, + const T* src, + int dst_offset = 0, + int src_offset = 0){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((const LT*)src)[src_offset]; +} + +// (1-t)*x + t*y +__device__ __forceinline__ float lerp(float t, float x, float y) { + // See https://developer.nvidia.com/blog/lerp-faster-cuda/ + return fma(t, y, fma(-t, x, x)); +} + +typedef enum{ + ADAM_MODE_0 =0, // L2 regularization mode + ADAM_MODE_1 =1 // Decoupled weight decay mode(AdamW) +} adamMode_t; + +/* Multi-tensor Adam + * + * Updates params in-place and outputs a copy with a desired datatype. + */ +template +struct DistAdamFunctor +{ + // Vectorized local compute + __device__ __forceinline__ static void local_step( + T p[ILP], + T m[ILP], + T v[ILP], + const GRAD_T g[ILP], + const float grad_scale, + const float beta1, + const float beta2, + const float beta1_correction, + const float beta2_correction, + const float eps, + const float lr, + adamMode_t mode, + const float weight_decay) { + if (mode == ADAM_MODE_0) { // L2 +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + float scaled_grad = (g[ii] * grad_scale) + (weight_decay * p[ii]); + float next_m = lerp(beta1, scaled_grad, m[ii]); + float next_v = lerp(beta2, scaled_grad*scaled_grad, v[ii]); + float next_m_unbiased = next_m / beta1_correction; + float next_v_unbiased = next_v / beta2_correction; + float denom = sqrtf(next_v_unbiased) + eps; + float update = next_m_unbiased / denom; + m[ii] = next_m; + v[ii] = next_v; + p[ii] -= lr * update; + } + } else { // weight decay +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + float scaled_grad = g[ii] * grad_scale; + float next_m = lerp(beta1, scaled_grad, m[ii]); + float next_v = lerp(beta2, scaled_grad*scaled_grad, v[ii]); + float next_m_unbiased = next_m / beta1_correction; + float next_v_unbiased = next_v / beta2_correction; + float denom = sqrtf(next_v_unbiased) + eps; + float update = (next_m_unbiased / denom) + (weight_decay * p[ii]); + m[ii] = next_m; + v[ii] = next_v; + p[ii] -= lr * update; + } + } + } + + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<5>& tl, + const float* grad_scale_ptr, + const float beta1, + const float beta2, + const float beta1_correction, + const float beta2_correction, + const float eps, + const float lr, + adamMode_t mode, + const float weight_decay) const + { + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + const float grad_scale = *grad_scale_ptr; + + T* p_in = (T *)tl.addresses[0][tensor_loc]; + p_in += chunk_idx*chunk_size; + T* m = (T *)tl.addresses[1][tensor_loc]; + m += chunk_idx*chunk_size; + T* v = (T *)tl.addresses[2][tensor_loc]; + v += chunk_idx*chunk_size; + const GRAD_T* g = (GRAD_T *)tl.addresses[3][tensor_loc]; + g += chunk_idx*chunk_size; + PARAM_OUT_T* p_out = (PARAM_OUT_T *)tl.addresses[4][tensor_loc]; + p_out += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + n = chunk_size < n ? chunk_size : n; + + const bool aligned = (n % ILP == 0 && + is_aligned(p_in) && + is_aligned(m) && + is_aligned(v) && + is_aligned(g) && + is_aligned(p_out)); + + for (int i_start = threadIdx.x*ILP; i_start < n; i_start += blockDim.x*ILP) { + T local_p[ILP]; + T local_m[ILP]; + T local_v[ILP]; + GRAD_T local_g[ILP]; + PARAM_OUT_T local_p_out[ILP]; + + // Load + if (aligned) { + load_store(local_p, p_in + i_start); + load_store(local_m, m + i_start); + load_store(local_v, v + i_start); + load_store(local_g, g + i_start); + } else { +#pragma unroll + for (int ii = 0, i = i_start; ii < ILP; ii++, i++) { + if (i < n) { + local_p[ii] = p_in[i]; + local_m[ii] = m[i]; + local_v[ii] = v[i]; + local_g[ii] = g[i]; + } else { + local_p[ii] = 0; + local_m[ii] = 0; + local_v[ii] = 0; + local_g[ii] = 0; + } + } + } + + // Local compute + local_step( + local_p, local_m, local_v, local_g, grad_scale, + beta1, beta2, beta1_correction, beta2_correction, + eps, lr, mode, weight_decay); +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + local_p_out[ii] = static_cast(local_p[ii]); + } + + // Store + if (aligned) { + load_store(p_in + i_start, local_p); + load_store(m + i_start, local_m); + load_store(v + i_start, local_v); + load_store(p_out + i_start, local_p_out); + } else { +#pragma unroll + for (int ii = 0, i = i_start; ii < ILP; ii++, i++) { + if (i < n) { + p_in[i] = local_p[ii]; + m[i] = local_m[ii]; + v[i] = local_v[ii]; + p_out[i] = local_p_out[ii]; + } + } + } + } + } +}; + +/* Functor for multi-tensor Adam with implicit main params + * + * If params are BF16 and optimizer state is FP32, it is not necessary + * to store FP32 main params. Instead, store 16-bit param remainder + * and combine with BF16 param to reconstruct the FP32 main param. + */ +template +struct DistAdamWithParamRemaindersFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<6>& tl, + const float* grad_scale_ptr, + const float beta1, + const float beta2, + const float beta1_correction, + const float beta2_correction, + const float eps, + const float lr, + adamMode_t mode, + const float weight_decay) const + { + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + const float grad_scale = *grad_scale_ptr; + + int16_t* p_in = (int16_t *)tl.addresses[0][tensor_loc]; + p_in += chunk_idx*chunk_size; + int16_t* p_rem = (int16_t *)tl.addresses[1][tensor_loc]; + p_rem += chunk_idx*chunk_size; + float* m = (float *)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + float* v = (float *)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + const GRAD_T* g = (GRAD_T *)tl.addresses[4][tensor_loc]; + g += chunk_idx*chunk_size; + int16_t* p_out = (int16_t *)tl.addresses[5][tensor_loc]; + p_out += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + n = chunk_size < n ? chunk_size : n; + + const bool aligned = (n % ILP == 0 && + is_aligned(p_in) && + is_aligned(p_rem) && + is_aligned(m) && + is_aligned(v) && + is_aligned(g) && + is_aligned(p_out)); + + for (int i_start = threadIdx.x*ILP; i_start < n; i_start += blockDim.x*ILP) { + union fp32_or_int162 { + float fp32; + int16_t int16[2]; + }; + fp32_or_int162 local_p[ILP]; + int16_t local_p_bf16[ILP]; + int16_t local_p_rem[ILP]; + float local_m[ILP]; + float local_v[ILP]; + GRAD_T local_g[ILP]; + + // Load + if (aligned) { + load_store(local_p_bf16, p_in + i_start); + load_store(local_p_rem, p_rem + i_start); + load_store(local_m, m + i_start); + load_store(local_v, v + i_start); + load_store(local_g, g + i_start); + } else { +#pragma unroll + for (int ii = 0, i = i_start; ii < ILP; ii++, i++) { + if (i < n) { + local_p_bf16[ii] = p_in[i]; + local_p_rem[ii] = p_rem[i]; + local_m[ii] = m[i]; + local_v[ii] = v[i]; + local_g[ii] = g[i]; + } else { + local_p_bf16[ii] = 0; + local_p_rem[ii] = 0; + local_m[ii] = 0; + local_v[ii] = 0; + local_g[ii] = 0; + } + } + } + + // Reconstruct FP32 params +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + if (local_p_rem[ii] < 0) + local_p_bf16[ii]--; // Undo rounding + local_p[ii].int16[1] = local_p_bf16[ii]; + local_p[ii].int16[0] = local_p_rem[ii]; + } + + // Local compute + using LocalFunctor = DistAdamFunctor; + LocalFunctor::local_step( + reinterpret_cast(local_p), local_m, local_v, local_g, grad_scale, + beta1, beta2, beta1_correction, beta2_correction, + eps, lr, mode, weight_decay); + + // Split into BF16 params (rounded-to-nearest) and remainders +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + local_p_bf16[ii] = local_p[ii].int16[1]; + local_p_rem[ii] = local_p[ii].int16[0]; + if (local_p_rem[ii] < 0) + local_p_bf16[ii]++; // Round up + } + + // Store + if (aligned) { + load_store(p_rem + i_start, local_p_rem); + load_store(m + i_start, local_m); + load_store(v + i_start, local_v); + load_store(p_out + i_start, local_p_bf16); + } else { +#pragma unroll + for (int ii = 0, i = i_start; ii < ILP; ii++, i++) { + if (i < n) { + p_rem[i] = local_p_rem[ii]; + m[i] = local_m[ii]; + v[i] = local_v[ii]; + p_out[i] = local_p_bf16[ii]; + } + } + } + } + } +}; + +void multi_tensor_fused_adam_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, // p_in, m, v, g, p_out + at::Tensor grad_scale, + float lr, + float beta1, + float beta2, + float eps, + int step, + int mode, + int bias_correction, + float weight_decay) +{ + using namespace at; + + // Expect p_in, m, v, g, p_out + size_t tl_sz = tensor_lists.size(); + TORCH_CHECK(tl_sz == 5, "expected tensor lists of size 5"); + const auto p_in_type = tensor_lists[0][0].scalar_type(); + const auto g_type = tensor_lists[3][0].scalar_type(); + const auto p_out_type = tensor_lists[4][0].scalar_type(); + + float beta1_correction = 1.0f, beta2_correction = 1.0f; + if (bias_correction == 1) { + beta1_correction = 1 - std::pow(beta1, step); + beta2_correction = 1 - std::pow(beta2, step); + } + + DISPATCH_FLOAT_HALF_AND_BFLOAT(p_in_type, 0, "dist_adam_cuda_kernel", + DISPATCH_FLOAT_HALF_AND_BFLOAT(g_type, 1, "dist_adam_cuda_kernel", + DISPATCH_FLOAT_HALF_AND_BFLOAT(p_out_type, 2, "dist_adam_cuda_kernel", + multi_tensor_apply<5>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + DistAdamFunctor(), + grad_scale.data_ptr(), + beta1, + beta2, + beta1_correction, + beta2_correction, + eps, + lr, + (adamMode_t) mode, + weight_decay); + ))); + C10_CUDA_CHECK(cudaGetLastError()); +} + +void multi_tensor_fused_adam_with_param_remainders_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, // p_in, p_rem, m, v, g, p_out + at::Tensor grad_scale, + float lr, + float beta1, + float beta2, + float eps, + int step, + int mode, + int bias_correction, + float weight_decay) +{ + using namespace at; + + // Expect p_in, p_rem, m, v, g, p_out + size_t tl_sz = tensor_lists.size(); + TORCH_CHECK(tl_sz == 6, "expected tensor lists of size 6"); + const auto g_type = tensor_lists[4][0].scalar_type(); + + float beta1_correction = 1.0f, beta2_correction = 1.0f; + if (bias_correction == 1) { + beta1_correction = 1 - std::pow(beta1, step); + beta2_correction = 1 - std::pow(beta2, step); + } + + DISPATCH_FLOAT_HALF_AND_BFLOAT(g_type, 0, "dist_adam_with_param_remainders_cuda_kernel", + multi_tensor_apply<6>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + DistAdamWithParamRemaindersFunctor(), + grad_scale.data_ptr(), + beta1, + beta2, + beta1_correction, + beta2_correction, + eps, + lr, + (adamMode_t) mode, + weight_decay); + ); + C10_CUDA_CHECK(cudaGetLastError()); +} diff --git a/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb.cpp b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb.cpp new file mode 100644 index 00000000..584b2a0e --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb.cpp @@ -0,0 +1,36 @@ +#include + +void multi_tensor_lamb_compute_update_term_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_beta1, + at::Tensor per_tensor_beta2, + at::Tensor per_tensor_beta3, + at::Tensor per_tensor_bias_correction, + at::Tensor step, + at::Tensor per_tensor_epsilon, + const int mode, + at::Tensor per_tensor_decay, + at::Tensor global_scale, + at::Tensor global_grad_norm, + const float max_grad_norm); + +void multi_tensor_lamb_update_weights_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_param_norm, + at::Tensor per_tensor_update_norm, + at::Tensor update_norm_offset, + at::Tensor learning_rate, + at::Tensor per_tensor_decay, + at::Tensor global_grad_norm, + bool use_nvlamb); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("multi_tensor_lamb_compute_update_term", &multi_tensor_lamb_compute_update_term_cuda, + "Computes update term for LAMB optimizer"); + m.def("multi_tensor_lamb_update_weights", &multi_tensor_lamb_update_weights_cuda, + "Applies update term for LAMB optimizer"); +} diff --git a/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb_kernel.cu b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb_kernel.cu new file mode 100644 index 00000000..95ee009b --- /dev/null +++ b/apex/apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb_kernel.cu @@ -0,0 +1,506 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +template +__device__ void convert(const FROM_T vi, TO_T& vo) +{ + vo = static_cast(vi); +} + +template <> +__device__ void convert(const float vi, uint8_t& vo) +{ + union S + { + float as_float; + int as_int; + }; + S s; + s.as_float = vi; + s.as_int = s.as_int & 0xFF800000; + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_half = static_cast(vi + s.as_float / 8.0f); + vo = t.as_byte[1]; +} + +template <> +__device__ void convert(const uint8_t vi, float& vo) +{ + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_byte[0] = 0; + t.as_byte[1] = vi; + vo = static_cast(t.as_half); +} + +template <> +__device__ void convert(const at::Half vi, uint8_t& vo) +{ + union S + { + float as_float; + int as_int; + }; + S s; + s.as_float = static_cast(vi); + s.as_int = s.as_int & 0xFF800000; + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_half = static_cast(vi + s.as_float / 8.0f); + vo = t.as_byte[1]; +} + +template <> +__device__ void convert(const uint8_t vi, at::Half& vo) +{ + union T + { + at::Half as_half; + uint8_t as_byte[2]; + }; + T t; + t.as_byte[0] = 0; + t.as_byte[1] = vi; + vo = t.as_half; +} + +typedef enum{ + MOMENT_MODE_0 =0, // L2 regularization mode + MOMENT_MODE_1 =1 // Decoupled weight decay mode +} adamMode_t; + +template +struct DistOptLAMBStage1Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<5>& tl, + const MATH_T* per_tensor_beta1, + const MATH_T* per_tensor_beta2, + const MATH_T* per_tensor_beta3, + const int* per_tensor_bias_correction, + const int* step, + const MATH_T* per_tensor_epsilon, + adamMode_t mode, + const MATH_T* per_tensor_decay, + const MATH_T* global_scale, + const MATH_T* global_grad_norm, + const float max_grad_norm) + { + // I'd like this kernel to propagate infs/nans. + if (*noop_gmem == 1) + return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + float combined_scale = *global_scale; + if (max_grad_norm > 0) { + combined_scale = max_grad_norm / (*global_grad_norm / *global_scale + 1e-6); + combined_scale = *global_scale / std::min((float) 1.0, combined_scale); + } + + MATH_T beta1 = per_tensor_beta1[tensor_num]; + MATH_T beta2 = per_tensor_beta2[tensor_num]; + MATH_T beta3 = 1 - beta1; + MATH_T beta1_correction, beta2_correction; + if (per_tensor_bias_correction[tensor_num] == 1) { + beta1_correction = 1 - pow(beta1, *step); + beta2_correction = 1 - pow(beta2, *step); + } else { + beta1_correction = (MATH_T) 1.0; + beta2_correction = (MATH_T) 1.0; + } + MATH_T epsilon = per_tensor_epsilon[tensor_num]; + MATH_T decay = per_tensor_decay[tensor_num]; + + GRAD_T* g = (GRAD_T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + T* m = (T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + T* v = (T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + MATH_T* u = (MATH_T*)tl.addresses[4][tensor_loc]; + u += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && + chunk_size % ILP == 0 && + is_aligned(g) && + is_aligned(p) && + is_aligned(m) && + is_aligned(v)) + { + GRAD_T l_g[ILP]; + T l_p[ILP]; + T l_m[ILP]; + T l_v[ILP]; + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(l_g, g, 0, i_start); + if (decay != 0) + load_store(l_p, p, 0, i_start); + load_store(l_m, m, 0, i_start); + load_store(l_v, v, 0, i_start); + // unpack +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_g[ii] = l_g[ii]; + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = l_p[ii]; + } + r_m[ii] = l_m[ii]; + r_v[ii] = l_v[ii]; + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / combined_scale; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / combined_scale; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + l_m[ii] = r_m[ii]; + l_v[ii] = r_v[ii]; + } + // store + load_store(u, r_p, i_start, 0); + load_store(m, l_m, i_start, 0); + load_store(v, l_v, i_start, 0); + } + } + else + { + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i]; + // special ?optimization? for lamb stage 1 + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = p[i]; + } + r_m[ii] = m[i]; + r_v[ii] = v[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / combined_scale; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / combined_scale; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + u[i] = r_p[ii]; + m[i] = r_m[ii]; + v[i] = r_v[ii]; + } + } + } + } + } +}; + +// Step 2 reads in 'update' value and per-tensor param_norm and update_norm. +// It computes new parameter value. +template +struct DistOptLAMBStage2Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<3>& tl, + const MATH_T* per_tensor_param_norm, + const MATH_T* per_tensor_update_norm, + const long* update_norm_offset, + const MATH_T* learning_rate, + const MATH_T* per_tensor_decay, + const MATH_T* global_grad_norm, + bool use_nvlamb) + { + // I'd like this kernel to propagate infs/nans. + if (*noop_gmem == 1) + return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + MATH_T decay = per_tensor_decay[tensor_num]; + + MATH_T ratio = *learning_rate; + // nvlamb: apply adaptive learning rate to all parameters + // otherwise, only apply to those with non-zero weight decay + if (use_nvlamb || (decay != (MATH_T) 0.0)) + { + MATH_T param_norm = per_tensor_param_norm[tensor_num]; + MATH_T update_norm = per_tensor_update_norm[update_norm_offset[tensor_num]]; + ratio = (update_norm != 0.0 && param_norm != 0.0) ? (*learning_rate) * (param_norm / update_norm) : (*learning_rate); + } + + MATH_T* update = (MATH_T*)tl.addresses[0][tensor_loc]; + update += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + GRAD_T* p_copy = (GRAD_T*)tl.addresses[2][tensor_loc]; + p_copy += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && + chunk_size % ILP == 0 && + is_aligned(p) && + is_aligned(update)) + { + T r_p[ILP]; + MATH_T r_update[ILP]; + GRAD_T r_p_copy[ILP]; + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_p, p, 0, i_start); + load_store(r_update, update, 0, i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = static_cast(r_p[ii]) - (ratio * r_update[ii]); + convert(r_p[ii], r_p_copy[ii]); + } + load_store(p, r_p, i_start, 0); + load_store(p_copy, r_p_copy, i_start, 0); + } + } + else + { + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_p[ILP]; + MATH_T r_update[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_p[ii] = p[i]; + r_update[ii] = update[i]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = r_p[ii] - (ratio * r_update[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + convert(r_p[ii], p_copy[i]); + } + } + } + } + } +}; + +void multi_tensor_lamb_compute_update_term_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_beta1, + at::Tensor per_tensor_beta2, + at::Tensor per_tensor_beta3, + at::Tensor per_tensor_bias_correction, + at::Tensor step, + at::Tensor per_tensor_epsilon, + const int mode, + at::Tensor per_tensor_decay, + at::Tensor global_scale, + at::Tensor global_grad_norm, + const float max_grad_norm) +{ + using namespace at; + + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 0, "lamb_stage_1", + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 1, "lamb_stage_1", + DISPATCH_FLOAT_AND_HALF(tensor_lists[4][0].scalar_type(), 2, "lamb_stage_1", + multi_tensor_apply<5>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + DistOptLAMBStage1Functor(), + per_tensor_beta1.DATA_PTR(), + per_tensor_beta2.DATA_PTR(), + per_tensor_beta3.DATA_PTR(), + per_tensor_bias_correction.DATA_PTR(), + step.DATA_PTR(), + per_tensor_epsilon.DATA_PTR(), + (adamMode_t) mode, + per_tensor_decay.DATA_PTR(), + global_scale.DATA_PTR(), + global_grad_norm.DATA_PTR(), + max_grad_norm); ))) + + AT_CUDA_CHECK(cudaGetLastError()); +} + +void multi_tensor_lamb_update_weights_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_param_norm, + at::Tensor per_tensor_update_norm, + at::Tensor update_norm_offset, + at::Tensor learning_rate, + at::Tensor per_tensor_decay, + at::Tensor global_grad_norm, + bool use_nvlamb) +{ + using namespace at; + + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 0, "lamb_stage_2", + DISPATCH_FLOAT_HALF_AND_BYTE(tensor_lists[2][0].scalar_type(), 1, "lamb_stage_2", + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 2, "lamb_stage_2", + multi_tensor_apply<3>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + DistOptLAMBStage2Functor(), + per_tensor_param_norm.DATA_PTR(), + per_tensor_update_norm.DATA_PTR(), + update_norm_offset.DATA_PTR(), + learning_rate.DATA_PTR(), + per_tensor_decay.DATA_PTR(), + global_grad_norm.DATA_PTR(), + use_nvlamb); ))) + + AT_CUDA_CHECK(cudaGetLastError()); +} diff --git a/apex/apex/contrib/csrc/peer_memory/peer_memory.cpp b/apex/apex/contrib/csrc/peer_memory/peer_memory.cpp new file mode 100644 index 00000000..2c4f7739 --- /dev/null +++ b/apex/apex/contrib/csrc/peer_memory/peer_memory.cpp @@ -0,0 +1,29 @@ +/** + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "peer_memory_cuda.cuh" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("allocate_raw", &apex::contrib::peer_memory::allocate_raw, "allocate_raw"); + m.def("free_raw", &apex::contrib::peer_memory::free_raw, "free_raw"); + m.def("zero", &apex::contrib::peer_memory::zero, "zero"); + m.def("get_raw_ipc_address", &apex::contrib::peer_memory::get_raw_ipc_address, "get_raw_ipc_address"); + m.def("get_raw_peers", &apex::contrib::peer_memory::get_raw_peers, "get_raw_peers"); + m.def("blob_view_half", &apex::contrib::peer_memory::blob_view_half, "blob_view_half"); + m.def("blob_view_float", &apex::contrib::peer_memory::blob_view_float, "blob_view_float"); + m.def("blob_view_int", &apex::contrib::peer_memory::blob_view_int, "blob_view_int"); + m.def("push_pull_halos_1d", &apex::contrib::peer_memory::push_pull_halos_1d, "push_pull_halos_1d"); +} diff --git a/apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cu b/apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cu new file mode 100644 index 00000000..67540585 --- /dev/null +++ b/apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cu @@ -0,0 +1,741 @@ +#include +#include +#include +#include +#include +#include +#include +#include "nccl.h" + +#define CUDACHECK(cmd) do { \ + cudaError_t err = cmd; \ + if( err != cudaSuccess ) { \ + char hostname[1024]; \ + gethostname(hostname, 1024); \ + printf("%s: CUDA failure %s:%d '%s'\n", \ + hostname, \ + __FILE__,__LINE__,cudaGetErrorString(err)); \ + } \ +} while(0) + +namespace { + +constexpr int THREADS_PER_CTA = 128; + +/* Basic deleter function for from_blob function. +void deleter(void* ptr) +{ + printf("deleter(ptr=%p)\n",ptr); + cudaFree(ptr); +} +*/ + +template +at::Tensor blob_view(T* raw_ptr, std::vector shape, const at::TensorOptions& options, bool channels_last) +{ + size_t size = 1; + std::vector strides(shape.size()); + if (channels_last) { + assert(shape.size() == 4); + strides[0] = shape[1]*shape[2]*shape[3]; + strides[1] = 1; + strides[2] = shape[1]*shape[3]; + strides[3] = shape[1]; + } else { + int idx = strides.size(); + for (auto it = shape.rbegin(); it != shape.rend(); ++it) + { + strides[--idx] = size; + size *= *it; + } + } + size *= sizeof(T); + // TODO: Implement dynamic reuse of pooled peer memory. + // We provide no deleter function because all peer memory allocations are static in this implementation. + return torch::from_blob((void*)raw_ptr, shape, strides, 0L, options); +} + +void tensor_shape(at::Tensor t, bool explicit_nhwc, int& N, int& C, int& H, int& W) +{ + if (t.dim() == 3) { + N = 1; + if (explicit_nhwc) { + C = t.size(2); + H = t.size(0); + W = t.size(1); + } else { + C = t.size(0); + H = t.size(1); + W = t.size(2); + } + } else if (t.dim() == 4) { + if (explicit_nhwc) { + N = t.size(0); + C = t.size(3); + H = t.size(1); + W = t.size(2); + } else { + N = t.size(0); + C = t.size(1); + H = t.size(2); + W = t.size(3); + } + } else { + printf("%s;%d - t.dim() must be either 3 or 4 (was %d)\n",__FILE__,__LINE__,int(t.dim())); + assert(t.dim() == 3 || t.dim() == 4); + } +} + +void tensor_strides(at::Tensor t, bool explicit_nhwc, int& stride_N, int& stride_C, int& stride_H, int& stride_W) +{ + if (t.dim() == 3) { + if (explicit_nhwc) { + stride_C = t.stride(2); + stride_H = t.stride(0); + stride_W = t.stride(1); + } else { + stride_C = t.stride(0); + stride_H = t.stride(1); + stride_W = t.stride(2); + } + stride_N = t.size(0)*t.size(1)*t.size(2); + } else if (t.dim() == 4) { + if (explicit_nhwc) { + stride_N = t.stride(0); + stride_C = t.stride(3); + stride_H = t.stride(1); + stride_W = t.stride(2); + } else { + stride_N = t.stride(0); + stride_C = t.stride(1); + stride_H = t.stride(2); + stride_W = t.stride(3); + } + } else { + printf("%s;%d - t.dim() must be either 3 or 4 (was %d)\n",__FILE__,__LINE__,t.dim()); + assert(t.dim() == 3 || t.dim() == 4); + } +} + +template +inline __device__ void __zero(T* dst) +{ + *dst = T(0); +} + +inline __device__ void __zero(int2* dst) +{ + *dst = {0, 0}; +} + +template +inline __device__ void zero_tensor( + const int dim0, + const int dim1, + const int dim2, + T* __restrict__ data, + const int data_stride0, + const int data_stride1, + const int data_stride2, + const int thread_id, + const int block_id, + const int num_blocks + ) +{ + const int global_id = thread_id + block_id * THREADS_PER_CTA; + const int num_threads = num_blocks * THREADS_PER_CTA; + const int count = dim0 * dim1 * dim2; + for (int i = global_id; i < count; i += num_threads) { + int offset; + if (contiguous) { + offset = i; + } else { + const int j2 = i % dim2; + const int k = i / dim2; + const int j1 = k % dim1; + const int j0 = k / dim1; + offset = j0 * data_stride0 + j1 * data_stride1 + j2 * data_stride2; + } + __zero(data + offset); + } +} + +template +inline __device__ void push_pull_tensor( + const int dim0, + const int dim1, + const int dim2, + const T* __restrict__ data_in, + const int data_in_stride0, + const int data_in_stride1, + const int data_in_stride2, + T* __restrict__ data_out, + const int data_out_stride0, + const int data_out_stride1, + const int data_out_stride2, + int4* local_peer, + int4* remote_peer, + const int thread_id, + const int block_id, + const int num_blocks + ) +{ + // 128b=16B NVLink flit + // Note: Use last 4B as a semaphore + static_assert(sizeof(T) <= 12); + union Flit { + T payload; + uint uints[4]; + }; + // Communication bit indicates whether flit has been received from + // a remote GPU + constexpr uint communication_mask = 1 << 0; + // Status bit is used to choose the active peer buffer in an + // alternating double buffer scheme. We use buffer 1 if the bits + // match, use buffer 2 if the bits differ, and invert the bit + // after finishing with a buffer. + constexpr uint status_mask = 1 << 1; + + // Split peer memory into two sets of buffers + // Note: Each block owns a THREADS_PER_CTA*2*16B chunk of peer + // memory + const int peer_offset1 = block_id * THREADS_PER_CTA * 2 + thread_id; + const int peer_offset2 = peer_offset1 + THREADS_PER_CTA; + volatile int* local_peer1 = reinterpret_cast(local_peer + peer_offset1); + volatile int* local_peer2 = reinterpret_cast(local_peer + peer_offset2); + volatile int* remote_peer1 = reinterpret_cast(remote_peer + peer_offset1); + volatile int* remote_peer2 = reinterpret_cast(remote_peer + peer_offset2); + + // Iterate through tensor entries + const int num_threads = num_blocks * THREADS_PER_CTA; + const int count = dim0 * dim1 * dim2; + for (int i0 = block_id * THREADS_PER_CTA; i0 < count; i0 += num_threads) { + const int i = i0 + thread_id; + const bool has_data = i < count; + + // Calculate buffer positions + int data_in_offset, data_out_offset; + if (contiguous) { + data_in_offset = i; + data_out_offset = i; + } else { + const int j2 = i % dim2; + const int k = i / dim2; + const int j1 = k % dim1; + const int j0 = k / dim1; + data_in_offset = j0 * data_in_stride0 + j1 * data_in_stride1 + j2 * data_in_stride2; + data_out_offset = j0 * data_out_stride0 + j1 * data_out_stride1 + j2 * data_out_stride2; + } + + // Determine which peer memory buffer to use + // Note: The status bit is not affected by asynchronous + // communication from the remote GPU. + Flit local_message1, local_message2; + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" : + "=r"(local_message1.uints[0]), + "=r"(local_message1.uints[1]), + "=r"(local_message1.uints[2]), + "=r"(local_message1.uints[3]) + : "l"(local_peer1) : "memory"); + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" : + "=r"(local_message2.uints[0]), + "=r"(local_message2.uints[1]), + "=r"(local_message2.uints[2]), + "=r"(local_message2.uints[3]) + : "l"(local_peer2) : "memory"); + const uint status1 = local_message1.uints[3] & status_mask; + const uint status2 = local_message2.uints[3] & status_mask; + const bool peer1_is_active = (status1 ^ status2) == 0; + volatile int* ox = peer1_is_active ? remote_peer1 : remote_peer2; + volatile int* ix = peer1_is_active ? local_peer1 : local_peer2; + const uint status = peer1_is_active ? status1 : status2; + Flit recv_message = peer1_is_active ? local_message1 : local_message2; + + // Send flit to remote GPU + // Note: Set communication bit and keep status bit + Flit send_message; + if (has_data) { + send_message.payload = data_in[data_in_offset]; + } + send_message.uints[3] = communication_mask | status; + asm volatile("st.volatile.global.v4.u32 [%0], {%1,%2,%3,%4};" :: + "l"(ox), + "r"(send_message.uints[0]), + "r"(send_message.uints[1]), + "r"(send_message.uints[2]), + "r"(send_message.uints[3]) + : "memory"); + + // Recieve flit from peer + while ((recv_message.uints[3] & communication_mask) == 0) { + asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" : + "=r"(recv_message.uints[0]), + "=r"(recv_message.uints[1]), + "=r"(recv_message.uints[2]), + "=r"(recv_message.uints[3]) + : "l"(ix) : "memory"); + } + if (has_data) { + data_out[data_out_offset] = recv_message.payload; + } + + // Reset semaphore + // Note: Clear communication bit and invert status bit + uint flag = ~status & status_mask; + asm volatile("st.volatile.global.v4.u32 [%0], {%1,%2,%3,%4};" :: + "l"(ix), + "n"(0), + "n"(0), + "n"(0), + "r"(flag) + : "memory"); + if (i0 + num_threads < count) { + __threadfence_system(); + } + } +} + +template +#if __CUDA_ARCH__ >= 700 +__launch_bounds__(THREADS_PER_CTA) +#endif +__global__ void push_pull_halos_1d_kernel( + // top halo, + T* toh, int toh_stride0, int toh_stride1, int toh_stride2, // top output halo (local) + const T* tih, int tih_stride0, int tih_stride1, int tih_stride2, // top input halo (local) + int4* tox, // top output transfer buffer (remote peer) + int4* tix, // top input transfer buffer (local peer) + // btm halo + T* boh, int boh_stride0, int boh_stride1, int boh_stride2, // btm output halo (local) + const T* bih, int bih_stride0, int bih_stride1, int bih_stride2, // btm input halo (local) + int4* box, // btm output transfer buffer (remote peer) + int4* bix, // btm input transfer buffer (local peer) + // dimensions + int dim0, int dim1, int dim2, + bool top_first // whether to launch communicate top halo first + ) +{ + const int num_blocks_side = gridDim.x / 2; + const int block_id_side = (blockIdx.x < num_blocks_side + ? blockIdx.x + : blockIdx.x - num_blocks_side); + const bool in_top_block = top_first == (blockIdx.x < num_blocks_side); + if (in_top_block) { + if (top_zero) { + zero_tensor( + dim0, dim1, dim2, + toh, toh_stride0, toh_stride1, toh_stride2, + threadIdx.x, block_id_side, num_blocks_side); + } else { + push_pull_tensor( + dim0, dim1, dim2, + tih, tih_stride0, tih_stride1, tih_stride2, + toh, toh_stride0, toh_stride1, toh_stride2, + tix, tox, + threadIdx.x, block_id_side, num_blocks_side); + } + } else { + if (btm_zero) { + zero_tensor( + dim0, dim1, dim2, + boh, boh_stride0, boh_stride1, boh_stride2, + threadIdx.x, block_id_side, num_blocks_side); + } else { + push_pull_tensor( + dim0, dim1, dim2, + bih, bih_stride0, bih_stride1, bih_stride2, + boh, boh_stride0, boh_stride1, boh_stride2, + bix, box, + threadIdx.x, block_id_side, num_blocks_side); + } + } +} + +__global__ void delay_kernel(int delay_nanoseconds, int* counter) +{ + if (blockIdx.x == 0 && threadIdx.x == 0) { + // waste time while doing something compiler can't predict, thus preventing it from optimizing away this code. + int new_counter = 0; + double elapsed = 0; + clock_t start = clock(); + do { + clock_t now = clock(); + elapsed = (double)(now - start)*1e9 / CLOCKS_PER_SEC; + ++new_counter; + } while (elapsed < (double)delay_nanoseconds); + *counter = new_counter; + } +} + +} + +namespace apex { namespace contrib { namespace peer_memory { + +int64_t allocate_raw(int64_t size) +{ + float* ptr = 0L; + cudaMalloc(&ptr, size); + cudaMemset(ptr, 0, size); + return (int64_t)ptr; +} + +void free_raw(int64_t raw) +{ + cudaFree((void*)raw); +} + +void zero(int64_t raw, int64_t size) +{ + cudaMemset((void*)raw, 0, size); +} + +at::Tensor get_raw_ipc_address(int64_t raw) +{ + cudaIpcMemHandle_t mem_handle; + CUDACHECK( cudaIpcGetMemHandle(&mem_handle, (void*)raw) ); + const int n = sizeof(cudaIpcMemHandle_t); + auto address_tensor = torch::empty({n}, torch::dtype(torch::kUInt8)); + auto address_tensor_p = address_tensor.data_ptr(); + memcpy(address_tensor_p, (uint8_t*)&mem_handle, n); + return address_tensor; +} + +std::vector get_raw_peers(at::Tensor ipc_addresses, int peer_rank, int64_t raw) +{ + int peer_group_size = ipc_addresses.size(0); + std::vector results(peer_group_size); + for (int i = 0; i < peer_group_size; ++i) { + if (i != peer_rank) { + cudaIpcMemHandle_t mem_handle; + memcpy(&mem_handle, ipc_addresses.index({i}).data_ptr(), sizeof(cudaIpcMemHandle_t)); + void* p = 0L; + CUDACHECK( cudaIpcOpenMemHandle((void**)&p, mem_handle, cudaIpcMemLazyEnablePeerAccess) ); + results[i] = (int64_t)p; + } else { + results[i] = (int64_t)raw; + } + } + return results; +} + +at::Tensor blob_view_half(int64_t raw, std::vector shape, bool channels_last) +{ + return blob_view((at::Half*)raw, shape, torch::dtype(torch::kFloat16).device(torch::kCUDA), channels_last); +} + +at::Tensor blob_view_float(int64_t raw, std::vector shape, bool channels_last) +{ + return blob_view((float*)raw, shape, torch::dtype(torch::kFloat32).device(torch::kCUDA), channels_last); +} + +at::Tensor blob_view_int(int64_t raw, std::vector shape, bool channels_last) +{ + return blob_view((int*)raw, shape, torch::dtype(torch::kInt32).device(torch::kCUDA), channels_last); +} + +void push_pull_halos_1d( + bool diagnostics, + bool explicit_nhwc, + int numSM, // number of SMs to use (zero corresponds to all SMs) + int rank, // rank in spatial parallel group + bool top_zero, // if top halo should be zeroed + at::Tensor top_in_halo, // top input halo buffer (in local device memory, sent to top neighbor) + at::Tensor top_in_transfer, // top input transfer buffer (in local peer memory) + at::Tensor top_out_transfer, // top output transfer buffer (in top neighbor peer memory) + at::Tensor top_out_halo, // top output halo buffer (in local device memory, received from top neighbor) + bool btm_zero, // if btm halo should be zeroed + at::Tensor btm_in_halo, // btm input halo buffer (in local device memory, sent to btm neighbor) + at::Tensor btm_in_transfer, // btm input transfer buffer (in local peer memory) + at::Tensor btm_out_transfer, // btm output transfer buffer (in btm neighbor peer memory) + at::Tensor btm_out_halo // btm output halo buffer (in local device memory, received from btm neighbor) + ) +{ + // basic checks of inputs + TORCH_CHECK(!(top_zero && btm_zero)); + TORCH_CHECK(top_in_halo.is_cuda()); + TORCH_CHECK(top_out_transfer.is_cuda()); + TORCH_CHECK(top_in_transfer.is_cuda()); + TORCH_CHECK(top_out_halo.is_cuda()); + TORCH_CHECK(btm_in_halo.is_cuda()); + TORCH_CHECK(btm_out_transfer.is_cuda()); + TORCH_CHECK(btm_in_transfer.is_cuda()); + TORCH_CHECK(btm_out_halo.is_cuda()); + + // tensor shapes + int tih_N, tih_C, tih_H, tih_W; + tensor_shape(top_in_halo, explicit_nhwc, tih_N, tih_C, tih_H, tih_W); + int toh_N, toh_C, toh_H, toh_W; + tensor_shape(top_out_halo, explicit_nhwc, toh_N, toh_C, toh_H, toh_W); + int bih_N, bih_C, bih_H, bih_W; + tensor_shape(btm_in_halo, explicit_nhwc, bih_N, bih_C, bih_H, bih_W); + int boh_N, boh_C, boh_H, boh_W; + tensor_shape(btm_out_halo, explicit_nhwc, boh_N, boh_C, boh_H, boh_W); + TORCH_CHECK(toh_N == tih_N && tih_N == boh_N && boh_N == bih_N && + toh_C == tih_C && tih_C == boh_C && boh_C == bih_C && + toh_H == tih_H && tih_H == boh_H && boh_H == bih_H && + toh_W == tih_W && tih_W == boh_W && boh_W == bih_W); + int NN=toh_N, NC=toh_C, NH=toh_H, NW=toh_W; + if (diagnostics) { + printf("rank %d: NN=%d, NC=%d, NH=%d, NW=%d\n", rank, NN, NC, NH, NW); + } + TORCH_CHECK(NN == 1); + + // tensor strides + int tih_stride_N, tih_stride_C, tih_stride_H, tih_stride_W; + tensor_strides(top_in_halo, explicit_nhwc, tih_stride_N, tih_stride_C, tih_stride_H, tih_stride_W); + int toh_stride_N, toh_stride_C, toh_stride_H, toh_stride_W; + tensor_strides(top_out_halo, explicit_nhwc, toh_stride_N, toh_stride_C, toh_stride_H, toh_stride_W); + int bih_stride_N, bih_stride_C, bih_stride_H, bih_stride_W; + tensor_strides(btm_in_halo, explicit_nhwc, bih_stride_N, bih_stride_C, bih_stride_H, bih_stride_W); + int boh_stride_N, boh_stride_C, boh_stride_H, boh_stride_W; + tensor_strides(btm_out_halo, explicit_nhwc, boh_stride_N, boh_stride_C, boh_stride_H, boh_stride_W); + if (diagnostics) { + printf("rank %d: tih_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, tih_stride_N, tih_stride_C, tih_stride_H, tih_stride_W); + printf("rank %d: toh_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, toh_stride_N, toh_stride_C, toh_stride_H, toh_stride_W); + printf("rank %d: bih_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, bih_stride_N, bih_stride_C, bih_stride_H, bih_stride_W); + printf("rank %d: boh_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, boh_stride_N, boh_stride_C, boh_stride_H, boh_stride_W); + } + + // determine if nhwc + bool is_nhwc = (toh_stride_C == 1); + if (diagnostics) { + printf("rank %d: is_nhwc = %s\n", rank, is_nhwc ? "true" : "false"); + } + + // determine if contiguous + bool contiguous = true; + if ((NN-1)*toh_stride_N + (NC-1)*toh_stride_C + + (NH-1)*toh_stride_H + (NW-1)*toh_stride_W + != NN*NC*NH*NW - 1) { + contiguous = false; + } + if ((NN-1)*boh_stride_N + (NC-1)*boh_stride_C + + (NH-1)*boh_stride_H + (NW-1)*boh_stride_W + != NN*NC*NH*NW - 1) { + contiguous = false; + } + if (!top_zero) { + if (toh_stride_N != tih_stride_N || toh_stride_C != tih_stride_C || + toh_stride_H != tih_stride_H || toh_stride_W != tih_stride_W) { + contiguous = false; + } + } + if (!btm_zero) { + if (boh_stride_N != bih_stride_N || boh_stride_C != bih_stride_C || + boh_stride_H != bih_stride_H || boh_stride_W != bih_stride_W) { + contiguous = false; + } + } + if (diagnostics) { + printf("rank %d: contiguous = %s\n", rank, contiguous ? "true" : "false"); + } + + // determine whether to communicate top halo first + bool top_first = rank % 2 != 0; + if (diagnostics) { + printf("rank %d: top_first = %s\n", rank, top_first ? "true" : "false"); + } + + // peer memory buffers + int tox_size = top_out_transfer.numel() * top_out_transfer.element_size(); + int tix_size = top_in_transfer.numel() * top_in_transfer.element_size(); + int box_size = btm_out_transfer.numel() * btm_out_transfer.element_size(); + int bix_size = btm_in_transfer.numel() * btm_in_transfer.element_size(); + if (!top_zero) { + TORCH_CHECK(top_out_transfer.is_contiguous()); + TORCH_CHECK(top_in_transfer.is_contiguous()); + TORCH_CHECK(tox_size == tix_size); + } + if (!btm_zero) { + TORCH_CHECK(btm_out_transfer.is_contiguous()); + TORCH_CHECK(btm_in_transfer.is_contiguous()); + TORCH_CHECK(box_size == bix_size); + } + + // figure out launch parameters + int device; + cudaGetDevice(&device); + cudaDeviceProp prop; + cudaGetDeviceProperties(&prop, device); + if (numSM <= 0 || numSM > prop.multiProcessorCount) { + numSM = prop.multiProcessorCount; + } + auto current_stream = at::cuda::getCurrentCUDAStream(); + dim3 block(THREADS_PER_CTA, 1, 1); + + // helper macros to launch templated kernel +#define LAUNCH_PUSH_PULL_HALO_KERNEL_BASE(T, CONTIGUOUS, TOP_ZERO, BTM_ZERO, KERNEL_ARGS, NUM_ELEMENTS) \ + do { \ + /* kernel configuration */ \ + int numBlocksPerSm; \ + cudaOccupancyMaxActiveBlocksPerMultiprocessor( \ + &numBlocksPerSm, \ + push_pull_halos_1d_kernel, \ + THREADS_PER_CTA, \ + 0); \ + dim3 grid(numSM*numBlocksPerSm,1,1); \ + if (grid.x % 2 != 0) { \ + /* require even number of blocks (half for top, half for bottom) */ \ + grid.x -= 1; \ + } \ + if ((grid.x / 2) * THREADS_PER_CTA > NUM_ELEMENTS) { \ + /* only need enough blocks to cover top and bottom halo elements */ \ + grid.x = 2 * ((NUM_ELEMENTS + THREADS_PER_CTA - 1) / THREADS_PER_CTA); \ + } \ + if (!TOP_ZERO) { \ + /* require 2*128b=32B peer memory per thread */ \ + if ((grid.x / 2) * THREADS_PER_CTA * 32 > tox_size) { \ + grid.x = 2 * (tox_size / (THREADS_PER_CTA * 32)); \ + } \ + } \ + if (!BTM_ZERO) { \ + /* require 2*128b=32B peer memory per thread */ \ + if ((grid.x / 2) * THREADS_PER_CTA * 32 > box_size) { \ + grid.x = 2 * (box_size / (THREADS_PER_CTA * 32)); \ + } \ + } \ + TORCH_CHECK(grid.x >= 2); \ + \ + /* launch kernel */ \ + cudaLaunchCooperativeKernel( \ + (void*)push_pull_halos_1d_kernel, \ + grid, \ + block, \ + KERNEL_ARGS, \ + 0, \ + current_stream); \ + } while (false) +#define LAUNCH_PUSH_PULL_HALO_KERNEL(T, CONTIGUOUS, KERNEL_ARGS, NUM_ELEMENTS) \ + do { \ + if (top_zero) { \ + LAUNCH_PUSH_PULL_HALO_KERNEL_BASE(T, CONTIGUOUS, true, false, KERNEL_ARGS, NUM_ELEMENTS); \ + } else if (btm_zero) { \ + LAUNCH_PUSH_PULL_HALO_KERNEL_BASE(T, CONTIGUOUS, false, true, KERNEL_ARGS, NUM_ELEMENTS); \ + } else { \ + LAUNCH_PUSH_PULL_HALO_KERNEL_BASE(T, CONTIGUOUS, false, false, KERNEL_ARGS, NUM_ELEMENTS); \ + } \ + } while (false) + + AT_DISPATCH_ALL_TYPES_AND(at::ScalarType::Half, top_out_halo.scalar_type(), "push_pull_halos_1d_kernel", [&]{ + if (diagnostics) { + printf("rank %d: size(scalar_t) = %ld\n", rank, sizeof(scalar_t)); + } + scalar_t* toh_p = top_out_halo.data_ptr(); + scalar_t* tih_p = top_in_halo.data_ptr(); + int4* tox_p = reinterpret_cast(top_out_transfer.data_ptr()); + int4* tix_p = reinterpret_cast(top_in_transfer.data_ptr()); + scalar_t* boh_p = btm_out_halo.data_ptr(); + scalar_t* bih_p = btm_in_halo.data_ptr(); + int4* box_p = reinterpret_cast(btm_out_transfer.data_ptr()); + int4* bix_p = reinterpret_cast(btm_in_transfer.data_ptr()); + if (diagnostics) printf("rank %d: choosing halo exchange kernel\n", rank); + + // do int2 vector loads if channel count permits + if (contiguous && + (NN*NH*NW*NC * sizeof(scalar_t)) % sizeof(int2) == 0) { + // can do contiguous int2 transfers + if (diagnostics) { + } + toh_stride_N = toh_stride_H = toh_stride_W = toh_stride_C = 1; + tih_stride_N = tih_stride_H = tih_stride_W = tih_stride_C = 1; + boh_stride_N = boh_stride_H = boh_stride_W = boh_stride_C = 1; + bih_stride_N = bih_stride_H = bih_stride_W = bih_stride_C = 1; + NC = (NN*NH*NW*NC * sizeof(scalar_t)) / sizeof(int2); + NN = NH = NW = 1; + if (diagnostics) { + printf("rank %d: launching contiguous int2 halo exchange kernel\n", + rank); + printf("rank %d: NC=%d, NH=%d, NW=%d\n", rank, NC, NH, NW); + } + void *kernel_args[] = { + (int2**)&toh_p, &toh_stride_H, &toh_stride_W, &toh_stride_C, + (int2**)&tih_p, &tih_stride_H, &tih_stride_W, &tih_stride_C, + &tox_p, &tix_p, + (int2**)&boh_p, &boh_stride_H, &boh_stride_W, &boh_stride_C, + (int2**)&bih_p, &bih_stride_H, &bih_stride_W, &bih_stride_C, + &box_p, &bix_p, + &NH, &NW, &NC, + &top_first + }; + int num_elem = NN*NH*NW*NC; + LAUNCH_PUSH_PULL_HALO_KERNEL(int2, true, kernel_args, num_elem); + } else if (is_nhwc && (NC * sizeof(scalar_t)) % sizeof(int2) == 0) { + // can do strided int2 transfers + int divisor = sizeof(int2) / sizeof(scalar_t); + if (diagnostics) { + printf("rank %d: launching strided int2 halo exchange kernel\n", + rank); + } + toh_stride_N /= divisor; toh_stride_H /= divisor; toh_stride_W /= divisor; + tih_stride_N /= divisor; tih_stride_H /= divisor; tih_stride_W /= divisor; + boh_stride_N /= divisor; boh_stride_H /= divisor; boh_stride_W /= divisor; + bih_stride_N /= divisor; bih_stride_H /= divisor; bih_stride_W /= divisor; + NC /= divisor; + if (diagnostics) { + printf("rank %d: divisor=%d\n", rank, divisor); + printf("rank %d: tih_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, tih_stride_N, tih_stride_C, tih_stride_H, tih_stride_W); + printf("rank %d: toh_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, toh_stride_N, toh_stride_C, toh_stride_H, toh_stride_W); + printf("rank %d: bih_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, bih_stride_N, bih_stride_C, bih_stride_H, bih_stride_W); + printf("rank %d: boh_stride :: N=%d, C=%d, H=%d, W=%d\n", + rank, boh_stride_N, boh_stride_C, boh_stride_H, boh_stride_W); + printf("rank %d: NC=%d, NH=%d, NW=%d\n", rank, NC, NH, NW); + } + void *kernel_args[] = { + (int2**)&toh_p, &toh_stride_H, &toh_stride_W, &toh_stride_C, + (int2**)&tih_p, &tih_stride_H, &tih_stride_W, &tih_stride_C, + &tox_p, &tix_p, + (int2**)&boh_p, &boh_stride_H, &boh_stride_W, &boh_stride_C, + (int2**)&bih_p, &bih_stride_H, &bih_stride_W, &bih_stride_C, + &box_p, &bix_p, + &NH, &NW, &NC, + &top_first + }; + int num_elem = NH*NW*NC; + LAUNCH_PUSH_PULL_HALO_KERNEL(int2, false, kernel_args, num_elem); + } else { + // cannot do int2 transfers + if (diagnostics) { + printf("rank %d: launching non-int2 halo exchange kernel\n", + rank); + } + int num_elem = NC*NH*NW; + if (is_nhwc) { + void *kernel_args[] = { + &toh_p, &toh_stride_H, &toh_stride_W, &toh_stride_C, + &tih_p, &tih_stride_H, &tih_stride_W, &tih_stride_C, + &tox_p, &tix_p, + &boh_p, &boh_stride_H, &boh_stride_W, &boh_stride_C, + &bih_p, &bih_stride_H, &bih_stride_W, &bih_stride_C, + &box_p, &bix_p, + &NH, &NW, &NC, + &top_first + }; + LAUNCH_PUSH_PULL_HALO_KERNEL(scalar_t, false, kernel_args, num_elem); + } else { + void *kernel_args[] = { + &toh_p, &toh_stride_C, &toh_stride_H, &toh_stride_W, + &tih_p, &tih_stride_C, &tih_stride_H, &tih_stride_W, + &tox_p, &tix_p, + &boh_p, &boh_stride_C, &boh_stride_H, &boh_stride_W, + &bih_p, &bih_stride_C, &bih_stride_H, &bih_stride_W, + &box_p, &bix_p, + &NC, &NH, &NW, + &top_first + }; + LAUNCH_PUSH_PULL_HALO_KERNEL(scalar_t, false, kernel_args, num_elem); + } + } + } ); + +#undef LAUNCH_PUSH_PULL_HALO_KERNEL_BASE +#undef LAUNCH_PUSH_PULL_HALO_KERNEL +} + +} } } diff --git a/apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cuh b/apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cuh new file mode 100644 index 00000000..83d11a2c --- /dev/null +++ b/apex/apex/contrib/csrc/peer_memory/peer_memory_cuda.cuh @@ -0,0 +1,48 @@ +/** + * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#ifndef _peer_memory_h_ +#define _peer_memory_h_ + +namespace apex { namespace contrib { namespace peer_memory { + int64_t allocate_raw(int64_t size); + void free_raw(int64_t raw); + void zero(int64_t raw, int64_t size); + at::Tensor get_raw_ipc_address(int64_t raw); + std::vector get_raw_peers(at::Tensor ipc_addresses, int peer_rank, int64_t raw); + at::Tensor blob_view_half(int64_t raw, std::vector shape, bool channels_last); + at::Tensor blob_view_float(int64_t raw, std::vector shape, bool channels_last); + at::Tensor blob_view_int(int64_t raw, std::vector shape, bool channels_last); + void push_pull_halos_1d( + bool diagnostics, + bool explicit_nhwc, + int numSM, // number of SMs to use + int peer_rank, // rank in spatial parallel group + bool top_zero, // if top halo should be zeroed + at::Tensor top_out_halo, // top output halo buffer (in local device memory, received from top neighbor) + at::Tensor top_inp_transfer, // top input transfer buffer (in local peer memory) + at::Tensor top_out_transfer, // top output transfer buffer (in top neighbor peer memory) + at::Tensor top_inp_halo, // top input halo buffer (in local device memory, sent to top neighbor) + bool btm_zero, // if btm halo should be zeroed + at::Tensor btm_out_halo, // btm output halo buffer (in local device memory, received from btm neighbor) + at::Tensor btm_inp_transfer, // btm input transfer buffer (in local peer memory) + at::Tensor btm_out_transfer, // btm output transfer buffer (in btm neighbor peer memory) + at::Tensor btm_inp_halo // btm input halo buffer (in local device memory, sent to btm neighbor) + ); +} } } +#endif diff --git a/apex/apex/contrib/csrc/transducer/transducer_joint.cpp b/apex/apex/contrib/csrc/transducer/transducer_joint.cpp new file mode 100755 index 00000000..351e7cab --- /dev/null +++ b/apex/apex/contrib/csrc/transducer/transducer_joint.cpp @@ -0,0 +1,98 @@ +#include +#include + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::vector transducer_joint_cuda_forward( + torch::Tensor f, + torch::Tensor g, + torch::Tensor fLen, + torch::Tensor gLen, + torch::Tensor batchOffset, + int64_t packedBatch, + int opt, + bool packOutput, + bool relu, + bool dropout, + float dropoutProb, + int tileSize); + + +std::vector transducer_joint_cuda_backward( + std::vector in, + torch::Tensor fLen, + torch::Tensor gLen, + torch::Tensor batchOffset, + int maxFLen, + int maxGLen, + bool packOutput, + float scale); + +std::vector transducer_joint_forward( + torch::Tensor f, + torch::Tensor g, + torch::Tensor fLen, + torch::Tensor gLen, + torch::Tensor batchOffset, + int64_t packedBatch, + int opt, + bool packOutput, + bool relu, + bool dropout, + float dropoutProb, + int tileSize) { + CHECK_INPUT(f); + CHECK_INPUT(g); + CHECK_INPUT(fLen); + CHECK_INPUT(gLen); + if (packOutput) + CHECK_INPUT(batchOffset); + return transducer_joint_cuda_forward( + f, + g, + fLen, + gLen, + batchOffset, + packedBatch, + opt, + packOutput, + relu, + dropout, + dropoutProb, + tileSize); +} + +std::vector transducer_joint_backward( + std::vector in, + torch::Tensor fLen, + torch::Tensor gLen, + torch::Tensor batchOffset, + int maxFLen, + int maxGLen, + bool packOutput, + float scale) { + for (auto t : in){ + CHECK_INPUT(t); + } + CHECK_INPUT(fLen); + CHECK_INPUT(gLen); + if (packOutput) + CHECK_INPUT(batchOffset); + return transducer_joint_cuda_backward( + in, + fLen, + gLen, + batchOffset, + maxFLen, + maxGLen, + packOutput, + scale); +} + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &transducer_joint_forward, "transducer joint forward (CUDA)"); + m.def("backward", &transducer_joint_backward, "transducer joint backward (CUDA)"); +} \ No newline at end of file diff --git a/apex/apex/contrib/csrc/transducer/transducer_joint_kernel.cu b/apex/apex/contrib/csrc/transducer/transducer_joint_kernel.cu new file mode 100755 index 00000000..1e6a465d --- /dev/null +++ b/apex/apex/contrib/csrc/transducer/transducer_joint_kernel.cu @@ -0,0 +1,979 @@ +#include +#include +#include + +#include +#include + +#ifdef OLD_GENERATOR_PATH +#include +#else +#include +#endif + +#include +#include +#include + +#include "philox.cuh" + +// Warp reduce kernels to reduce N groups of data into N numbers, where N = warpSize / width. +// width should be a power of 2 and should be less than warpSize. +template +__device__ __forceinline__ scalar_t warpReduce(scalar_t x, int width=C10_WARP_SIZE){ + for (unsigned offset = width/2; offset > 0; offset /= 2){ + x += __shfl_down_sync(0xffffffff, x, offset, width); + } + return x; +} + +inline int largestPowerOfTwo(int x){ + int y = 1; + while (y <= x) + y <<= 1; + return y >> 1; +} + +/* +Figure out vectorization type for masks. +Similar to how PyTorch figures out acc_t here: +aten/src/ATen/AccumulateType.h +*/ +template +struct MaskVecType { }; + +template <> struct MaskVecType<1> { using type = uint8_t; }; +template <> struct MaskVecType<2> { using type = uint16_t; }; +template <> struct MaskVecType<4> { using type = uint32_t; }; + +template +using mvec_type = typename MaskVecType::type; + +// Helper class to calculate pointer offset that can be shared by different flavors of kernels. +// For fwd, batch offset and stride are different for packing and non-packing mode. +struct OffsetCalFwd{ + __device__ __forceinline__ OffsetCalFwd( + int64_t batch, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t gLen, + int64_t hiddenSize, + bool packOutput) : + batch(batch), + batchOffset(batchOffset), + maxFLen(maxFLen), + maxGLen(maxGLen), + gLen(gLen), + hiddenSize(hiddenSize), + packOutput(packOutput) + {} + + int64_t batch; + const int64_t *batchOffset; + int64_t maxFLen; + int64_t maxGLen; + int64_t gLen; + int64_t hiddenSize; + bool packOutput; + + __device__ __forceinline__ int64_t getBatchOffset(){ + return packOutput ? ((batch==0) ? 0 : batchOffset[batch-1])*hiddenSize + : batch*maxFLen*maxGLen*hiddenSize; + } + + __device__ __forceinline__ int64_t getStrideF(){ + return packOutput ? gLen*hiddenSize : maxGLen*hiddenSize; + } + + +}; + +// Helper class to calculate pointer offset that can be shared by different flavors of kernels +// For bwd, batch offset and stride are different for packing and non-packing mode. +// The reducion is done for two input tensors. Therefore, generating two sets of offsets +// according to bwdFasterDim can lead to a unified implementation in the actual kernel. +struct OffsetCalBwd{ + __device__ __forceinline__ OffsetCalBwd( + int64_t batch, + const int64_t *batchOffset, + const int *fLen, + const int *gLen, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + bool packOutput, + bool bwdFasterDim) : + batch(batch), + batchOffset(batchOffset), + maxFLen(maxFLen), + maxGLen(maxGLen), + fLen(fLen), + gLen(gLen), + hiddenSize(hiddenSize), + packOutput(packOutput), + bwdFasterDim(bwdFasterDim) + {} + + int64_t batch; + const int64_t *batchOffset; + const int *fLen; + const int *gLen; + int64_t maxFLen; + int64_t maxGLen; + int64_t hiddenSize; + bool packOutput; + bool bwdFasterDim; // whether doing bwd on the faster moving dimension + + __device__ __forceinline__ int64_t getBatchOffset(){ + return packOutput ? ((batch==0) ? 0 : batchOffset[batch-1])*hiddenSize + : batch*maxFLen*maxGLen*hiddenSize; + } + + __device__ __forceinline__ int64_t getMaxXLen(){ + return bwdFasterDim ? maxGLen : maxFLen; + } + + __device__ __forceinline__ auto getMyXLen() -> decltype(gLen[batch]){ + return bwdFasterDim ? gLen[batch] : fLen[batch]; + } + + __device__ __forceinline__ auto getMyYLen() -> decltype(gLen[batch]){ + return bwdFasterDim ? fLen[batch] : gLen[batch]; + } + + __device__ __forceinline__ int64_t getStrideX(){ + return bwdFasterDim ? hiddenSize : ((packOutput ? gLen[batch] : maxGLen) * hiddenSize); + } + + __device__ __forceinline__ int64_t getStrideY(){ + return bwdFasterDim ? ((packOutput ? gLen[batch] : maxGLen) * hiddenSize) : hiddenSize; + } +}; + + +// Vanila transducer joint forward kernel +// Detail of this joint function can be found in: +// [1] Sequence Transduction with Recurrent Neural Networks. + +// f is a tensor of shape [batch, T, H] +// g is a tensor of shape [batch, U, H] +// the transducer joint does +// sum = f.unsqueeze(dim=2) + g.unsqueeze(dim=1) +// The resultant tensor is of shape [batch, T, U, H] +// Each thread block is working on one "batch" of data in the output tensor, [batch, t, u, :] + +// This joint function can optionally pack the output where the output tensor with a shape of +// [B, T, U, H] is packed into [B_packed, H]. +// Don't-care region (t > fLen) or (u > gLen) is removed. +// To enable packing, the starting offset for each batch need to be specified with batchOffset. +template +__global__ void transducer_joint_forward( + const scalar_t *f, + const scalar_t *g, + const int *fLen, + const int *gLen, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + bool packOutput, + scalar_t *sum) { + + + const int batch = blockIdx.z; + const int t = blockIdx.y; + const int u = blockIdx.x; + const auto myFLen = fLen[batch]; + const auto myGLen = gLen[batch]; + + OffsetCal offsetCal(batch, batchOffset, maxFLen, maxGLen, myGLen, hiddenSize, packOutput); + const auto myBatchOffset = offsetCal.getBatchOffset(); + const auto strideF = offsetCal.getStrideF(); + scalar_t const *myF = f + batch*maxFLen*hiddenSize + t*hiddenSize; + scalar_t const *myG = g + batch*maxGLen*hiddenSize + u*hiddenSize; + scalar_t *mySum = sum + myBatchOffset + t*strideF + u * hiddenSize; + + if (t < myFLen and u < myGLen){ + #pragma unroll + for (int h = threadIdx.x; h < hiddenSize; h += blockDim.x){ + if (h < hiddenSize){ + mySum[h] = myF[h] + myG[h]; + } + } + } + else if (packOutput == false and t < maxFLen and u < maxGLen){ + // Need to write finite data to don't-care region because we instantiate the result tensor + // with torch::empty for performance reasons. Even though it is don't-care region, the + // contents need to be finite, otherwise could lead to NaN in WGRAD. + // In packing mode, this write is no longer necessary as we remove the don't-care region + // from the output. + // Picking -1 (over 0) here for ease of testing. + #pragma unroll + for (int h = threadIdx.x; h < hiddenSize; h += blockDim.x){ + if (h < hiddenSize){ + mySum[h] = -1; + } + } + } +} + +/* +Tiled version of the joint forward kernel +Detail of this joint function can be found in: +[1] Sequence Transduction with Recurrent Neural Networks. + +f is a tensor of shape [batch, T, H] +g is a tensor of shape [batch, U, H] +the transducer joint does +sum = f.unsqueeze(dim=2) + g.unsqueeze(dim=1) +The resultant tensor is of shape [batch, T, U, H] +Each thread is working on a tile of the shape of tileF x tileG in the result tensor. +The input for the tile is first loaded in the register and is reused tileG and tileF times. + +This joint function can optionally pack the output where the output tensor with a shape of +[B, T, U, H] is packed into [B_packed, H]. +Don't-care region (t > fLen) or (u > gLen) is removed. +To enable packing, the starting offset for each batch need to be specified with batchOffset. + +Optionally this joint function performs ReLU and/or dropout on the joint output, which is +controlled by arguments relu and dropout, respectively. philoxArgs is argument used for generating +pseudorandom number. When at least one of operations in ReLU and dropout is activated, the joint +function is a masked operation, which is controlled by the template argument masked. In this case, +masks are saved to backward. +*/ +template +__global__ void transducer_joint_tiled_forward( + const scalar_t *f, + const scalar_t *g, + const int *fLen, + const int *gLen, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + int64_t hiddenPerBlock, + bool packOutput, + bool relu, + bool dropout, + float p, + at::PhiloxCudaState philoxArgs, + scalar_t *sum, + uint8_t *mask) { + + static_assert(U == 4, "U has to be 4, as random numbers are generated in batch of 4"); + + const int batch = blockIdx.z; + const int t = blockIdx.y * tileF; + const int hiddenBlock = (hiddenSize + hiddenPerBlock - 1) / hiddenPerBlock; + const int u = blockIdx.x / hiddenBlock * tileG; + const int hOffset = (blockIdx.x % hiddenBlock) * hiddenPerBlock; + const int h = threadIdx.x; + const auto myFLen = fLen[batch]; + const auto myGLen = gLen[batch]; + + OffsetCal offsetCal(batch, batchOffset, maxFLen, maxGLen, myGLen, hiddenSize, packOutput); + const auto myBatchOffset = offsetCal.getBatchOffset(); + const auto strideF = offsetCal.getStrideF(); + + scalar_t const *myF = f + batch*maxFLen*hiddenSize + t*hiddenSize + hOffset; + scalar_t const *myG = g + batch*maxGLen*hiddenSize + u*hiddenSize + hOffset; + scalar_t *mySum = sum + myBatchOffset + t*strideF + u*hiddenSize + hOffset; + uint8_t *myMask = mask + myBatchOffset + t*strideF + u*hiddenSize + hOffset; + + // The following code is only needed for dropout. We try to bypass them as much as possible. + auto seeds = masked ? at::cuda::philox::unpack(philoxArgs) + : std::make_tuple(static_cast(0), static_cast(0)); + uint64_t tid = masked ? (static_cast(blockIdx.z)*gridDim.y*gridDim.x + + blockIdx.y*gridDim.x + blockIdx.x) * blockDim.x + threadIdx.x + : 0; + Philox ph(std::get<0>(seeds), tid, std::get<1>(seeds)); + scalar_t scale = masked ? ((p == 0) ? 0 : 1 / p) : 0; + bool dropoutMask[U]; + + if (t < myFLen and u < myGLen and hOffset+h < hiddenSize){ + // register buffers for tiled input reuse + scalar_t fBuffer[tileF], gBuffer[tileG]; + for (int i = 0; i < tileF; ++i){ + if (t + i < myFLen) + fBuffer[i] = myF[i*hiddenSize + h]; + } + for (int j = 0; j < tileG; ++j){ + if (u + j < myGLen) + gBuffer[j] = myG[j*hiddenSize + h]; + } + #pragma unroll + for (int i = 0; i < tileF; ++i){ + if (t + i < myFLen){ + #pragma unroll + for (int j = 0; j < tileG; ++j){ + int idx = i*tileG + j; + if (masked and dropout and idx % U == 0){ + // For performance, generate 4 random numbers in one shot + // auto rand4 = curand_uniform4(&state); + auto rand4 = uniform4(ph()); + dropoutMask[0] = rand4.x < p; + dropoutMask[1] = rand4.y < p; + dropoutMask[2] = rand4.z < p; + dropoutMask[3] = rand4.w < p; + } + + if (u + j < myGLen){ + scalar_t out = fBuffer[i] + gBuffer[j]; + if (masked){ + // Apply ReLU here when relu is True + bool localMask = relu ? (out>0) : 1; + localMask = dropout ? localMask & dropoutMask[idx%U] : localMask; + out = dropout ? out*localMask*scale : out*localMask; + myMask[i*strideF + j*hiddenSize + h] = static_cast(localMask); + } + mySum[i*strideF + j*hiddenSize + h] = out; + } + else if (packOutput == false and u + j < maxGLen) + mySum[i*strideF + j*hiddenSize + h] = -1; + } + } + else if (packOutput == false and t + i < maxFLen){ + // Again need to write finite data to don't-care region + #pragma unroll + for (int j = 0; j < tileG; ++j){ + if (u + j < maxGLen) + mySum[i*strideF + j*hiddenSize + h] = -1; + } + } + } + } + else if (packOutput == false and t < maxFLen and u < maxGLen and hOffset+h < hiddenSize){ + // Only need to ensure the finity in normal mode + #pragma unroll + for (int i = 0; i < tileF; ++i){ + if (t + i < maxFLen){ + #pragma unroll + for (int j = 0; j < tileG; ++j){ + if (u + j < maxGLen) + mySum[i*strideF + j*hiddenSize + h] = -1; + } + } + } + } +} + +/* +Bwd operation (reduction) on one input tensor. Since the operation performed for the two input +tensors are exactly the same, only one kernel is needed, and the different indexing offsets +and strides are handled by OffsetCalBwd. + +When packing is enabled in the fwd op, unpacking is needed to restore the gradients in a +non-packed form. + +When ReLU and/or dropout are performed in the fwd pass, this operation becomes a masked operation, +and mask contains the mask information. +*/ +template +__device__ void transducer_joint_single_backward( + const scalar_t *grad, + const uint8_t *mask, + const int *fLen, + const int *gLen, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + bool packOutput, + bool bwdFasterDim, // whether bwd on the faster moving dimension (u) + float scale, + scalar_t *inGrad, + int yBlockOffset=0) { + + + const int batch = blockIdx.z; + // For the second input tensor, this offset need to be subtracted because the first yBlockOffset + // sets of thread blocks are for the first input tensor. + const int x = blockIdx.y-yBlockOffset; + const int hOffset = blockIdx.x*C10_WARP_SIZE; + const int wid = threadIdx.y; + const int lid = threadIdx.x; + const int numWarp = blockDim.y; + extern __shared__ char smem8[]; + auto smem = reinterpret_cast(smem8); + + OffsetCal offsetCal(batch, batchOffset, fLen, gLen, maxFLen, maxGLen, hiddenSize, packOutput, + bwdFasterDim); + const auto maxXLen = offsetCal.getMaxXLen(); + const auto myXLen = offsetCal.getMyXLen(); + const auto myYLen = offsetCal.getMyYLen(); + scalar_t *myInGrad = inGrad + batch*maxXLen*hiddenSize + x*hiddenSize + hOffset; + + if (x < myXLen){ + + const auto myBatchOffset = offsetCal.getBatchOffset(); + const auto strideX = offsetCal.getStrideX(); + const auto strideY = offsetCal.getStrideY(); + const scalar_t *myGrad = grad + myBatchOffset + x*strideX + hOffset; + const uint8_t *myMask = masked ? mask + myBatchOffset + x*strideX + hOffset : nullptr; + + // Each warp reduces numYPerWarp "y" first + acc_t warpSum = 0; + auto numYPerWarp = (myYLen+numWarp-1)/numWarp; + #pragma unroll + for (int warpY = 0; warpY < numYPerWarp; ++warpY){ + auto y = wid*numYPerWarp + warpY; + if (y < myYLen and (hOffset+lid) < hiddenSize) + if (masked) + warpSum += static_cast(myGrad[y*strideY + lid]) * myMask[y*strideY + lid] * scale; + else + warpSum += myGrad[y*strideY + lid]; + } + + // transpose partial sum in SMEM and reduce further using warpReduce + smem[lid*numWarp + wid] = warpSum; + __syncthreads(); + auto sum = smem[wid*C10_WARP_SIZE + lid]; + sum = warpReduce(sum, numWarp); + + // a a b b c c d d + // a a b b c c d d + // a a b b c c d d + // a a b b c c d d + // example of 4 warps (a, b, c, d) with 8 threads per warp + // Each warp need 8 / 4 = 2 threads to write the results. + if (hOffset+wid*C10_WARP_SIZE/numWarp+lid/numWarp < hiddenSize){ + if (lid % numWarp == 0){ + myInGrad[wid*C10_WARP_SIZE/numWarp + lid/numWarp] = sum; + } + } + } + else if (wid == 0 and hOffset + lid < hiddenSize){ + // Need to ensure the grad is zero for don't care region + myInGrad[lid] = 0; + } +} + +/* +Actual bwd (reduction) kernel get launched. +Call transducer_joint_single_backward twice on two input tensors. +The two bwd ops are launched together, the first op uses blockIdx.y < maxFLen, and the second op +uses the rest. +When ReLU and/or dropout are performed in the fwd pass, this operation becomes a masked operation, +and mask contains the mask information. +*/ +template +__global__ void transducer_joint_combined_backward( + const scalar_t *grad, + const uint8_t *mask, + const int *fLen, + const int *gLen, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + bool packOutput, + float scale, + scalar_t *fGrad, + scalar_t *gGrad) { + if (blockIdx.y < maxFLen){ + transducer_joint_single_backward( + grad, + mask, + fLen, + gLen, + batchOffset, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + false, + scale, + fGrad); + } + else{ + transducer_joint_single_backward( + grad, + mask, + fLen, + gLen, + batchOffset, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + true, + scale, + gGrad, + maxFLen); + } +} + +/* +Vectorized version of transducer_joint_single_backward +Doing exact same operation as transducer_joint_single_backward except the load and store are +vectorized. +When packing is enabled in the fwd op, unpacking is needed to restore the gradients in a +non-packed form. +When ReLU and/or dropout are performed in the fwd pass, this operation becomes a masked operation, +and mask contains the mask information. +*/ +template +__device__ void transducer_joint_single_vec_backward( + const scalar_t *grad, + const uint8_t *mask, + const int *fLen, + const int *gLen, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + bool packOutput, + bool bwdFasterDim, + float scale, + scalar_t *inGrad, + int yBlockOffset=0){ + + const int batch = blockIdx.z; + const int x = blockIdx.y - yBlockOffset; + const int hOffset = blockIdx.x*C10_WARP_SIZE*V; + const int wid = threadIdx.y; + const int lid = threadIdx.x; + const int numWarp = blockDim.y; + + // Figure out the vectorization type for mask + using mvec_t = mvec_type; + + OffsetCal offsetCal(batch, batchOffset, fLen, gLen, maxFLen, maxGLen, hiddenSize, packOutput, + bwdFasterDim); + const auto maxXLen = offsetCal.getMaxXLen(); + const auto myXLen = offsetCal.getMyXLen(); + const auto myYLen = offsetCal.getMyYLen(); + scalar_t *myInGrad = inGrad + batch*maxXLen*hiddenSize + x*hiddenSize + hOffset; + extern __shared__ char smem8[]; + auto smem = reinterpret_cast(smem8); + + acc_t warpSum[V]; + scalar_t inBuffer[V]; + uint8_t maskBuffer[V]; + scalar_t outBuffer[V]; + auto myInGradVec = reinterpret_cast(myInGrad); + auto outBufferVec = reinterpret_cast(outBuffer); + + if (x < myXLen){ + const auto myBatchOffset = offsetCal.getBatchOffset(); + const auto strideX = offsetCal.getStrideX(); + const auto strideY = offsetCal.getStrideY(); + const scalar_t *myGrad = grad + myBatchOffset + x*strideX + hOffset; + const uint8_t *myMask = masked ? mask + myBatchOffset + x*strideX + hOffset + :nullptr; + + for (int i = 0; i < V; ++i) + warpSum[i] = 0; + + // Each warp reduces numYPerWarp "y" first + auto numYPerWarp = (myYLen+numWarp-1)/numWarp; + for (int warpY = 0; warpY < numYPerWarp; ++warpY){ + auto y = wid*numYPerWarp + warpY; + auto myGradVec = reinterpret_cast(myGrad + y*strideY); + auto myMaskVec = masked ? reinterpret_cast(myMask + y*strideY) + : nullptr; + auto inBufferVec = reinterpret_cast(inBuffer); + auto maskBufferVec = reinterpret_cast(maskBuffer); + if (hOffset + lid*V < hiddenSize and y < myYLen){ + *inBufferVec = myGradVec[lid]; // vectorized load + if (masked){ + *maskBufferVec = myMaskVec[lid]; + #pragma unroll + for (int i = 0; i < V; ++i) + warpSum[i] += static_cast(inBuffer[i]) * maskBuffer[i] * scale; + } + else{ + #pragma unroll + for (int i = 0; i < V; ++i) + warpSum[i] += inBuffer[i]; + } + } + } + + // transpose partial sum in SMEM and reduce further using warpReduce + for (int i = 0; i < V; ++i){ + smem[lid*numWarp + wid] = warpSum[i]; + __syncthreads(); + auto sum = smem[wid*C10_WARP_SIZE + lid]; + + if (hOffset+(wid*C10_WARP_SIZE/numWarp)*V < hiddenSize){ + sum = warpReduce(sum, numWarp); + if (lid % numWarp == 0){ + outBuffer[i] = sum; + } + } + __syncthreads(); + } + + // a a b b c c d d + // a a b b c c d d + // a a b b c c d d + // a a b b c c d d + // example of 4 warps (a, b, c, d) with 8 threads per warp + // Each warp need 8 / 4 = 2 threads to write the results. + if (lid % numWarp == 0 and hOffset+(wid*C10_WARP_SIZE/numWarp + lid/numWarp)*V < hiddenSize) + myInGradVec[wid*C10_WARP_SIZE/numWarp + lid/numWarp] = *outBufferVec; + } + else if (wid == 0 and hOffset + lid*V < hiddenSize){ + // Need to ensure the grad is zero for don't care region + myInGradVec[lid] = 0; + } +} + +/* +Vecotrized version of transducer_joint_combined_backward +Call transducer_joint_single_vec_backward twice on two input tensors. +The two bwd ops are launched together, the first op uses blockIdx.y < maxFLen, and the second op +uses the rest. +When ReLU and/or dropout are performed in the fwd pass, this operation becomes a masked operation, +and mask contains the mask information. +*/ +template +__global__ void transducer_joint_combined_vec_backward( + const scalar_t *grad, + const uint8_t *mask, + const int *fLen, + const int *gLen, + const int64_t *batchOffset, + int64_t maxFLen, + int64_t maxGLen, + int64_t hiddenSize, + bool packOutput, + float scale, + scalar_t *fGrad, + scalar_t *gGrad) { + if (blockIdx.y < maxFLen){ + transducer_joint_single_vec_backward( + grad, + mask, + fLen, + gLen, + batchOffset, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + false, + scale, + fGrad); + } + else{ + transducer_joint_single_vec_backward( + grad, + mask, + fLen, + gLen, + batchOffset, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + true, + scale, + gGrad, + maxFLen); + } +} + + + + +std::vector transducer_joint_cuda_forward( + torch::Tensor f, + torch::Tensor g, + torch::Tensor fLen, + torch::Tensor gLen, + torch::Tensor batchOffset, + int64_t packedBatch, + int opt, + bool packOutput, + bool relu, + bool dropout, + float dropoutProb, + int tileSize){ + + + auto tensorOpt = f.options(); + auto dtype = f.scalar_type(); + const auto batchSize = f.size(0); + const auto maxFLen = f.size(1); + const auto maxGLen = g.size(1); + const auto hiddenSize = f.size(2); + bool masked = dropout or relu; + + int64_t *batchOffsetPtr = nullptr; + torch::Tensor sum, mask; + auto maskOpt = tensorOpt.dtype(torch::kUInt8); + if (!packOutput){ + sum = torch::empty({batchSize, maxFLen, maxGLen, hiddenSize}, tensorOpt); + batchOffsetPtr = nullptr; + if (masked) + mask = torch::empty({batchSize, maxFLen, maxGLen, hiddenSize}, maskOpt); + } + else{ + sum = torch::empty({packedBatch, hiddenSize}, tensorOpt); + batchOffsetPtr = batchOffset.data_ptr(); + if (masked) + mask = torch::empty({packedBatch, hiddenSize}, maskOpt); + } + uint8_t *maskPtr = masked ? mask.data_ptr() : nullptr; + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + TORCH_CHECK(opt == 0 or opt == 1, "Got an invalid optimization level ", opt); + // Simple heuristics + const int numThread = std::min(128, (static_cast(hiddenSize)+C10_WARP_SIZE-1) + / C10_WARP_SIZE * C10_WARP_SIZE); + + if (opt == 0){ + // vanilla kernel + const int threads = numThread; + const dim3 blocks(maxGLen, maxFLen, batchSize); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(dtype, "transducer_joint_forward", ([&] { + transducer_joint_forward + <<>>( + f.data_ptr(), + g.data_ptr(), + fLen.data_ptr(), + gLen.data_ptr(), + batchOffsetPtr, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + sum.data_ptr()); + })); + } + if (opt == 1){ + // tiled version. For simplicity, assume tileF == tileG, even though the kernel can + // support more general cases. + const int threads = numThread; + const int hiddenPerBlock = numThread; + const int hiddenBlock = (hiddenSize + hiddenPerBlock - 1) / hiddenPerBlock; + const dim3 blocks( (maxGLen+tileSize-1)/tileSize * hiddenBlock, + (maxFLen+tileSize-1)/tileSize, + batchSize); + + TORCH_CHECK(tileSize == 1 or tileSize == 2 or tileSize == 4, + "Expected tileSize to be in [1, 2, 4], but got ", tileSize); + + at::PhiloxCudaState rng_engine_inputs; + if (masked){ + // set up PRG when the input is masked. rng_engine_inputs will be used as a space filler + // for non-masked calls. + // Therefore no need to initialize. + c10::optional gen_; + auto gen = at::get_generator_or_default(gen_, + at::cuda::detail::getDefaultCUDAGenerator()); + // counterOffset records how many cuRAND calls each thread makes. For a tiled kernel, + // each thread processes tileF * tileG output elements. + int64_t counterOffset = tileSize * tileSize; + { + std::lock_guard lock(gen->mutex_); + rng_engine_inputs = gen->philox_cuda_state(counterOffset); + } + } + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(dtype, "transducer_joint_forward", ([&] { + void(*kernel)(const scalar_t*, const scalar_t*, const int*, const int*, const int64_t*, + int64_t, int64_t, int64_t, int64_t, bool, bool, bool, float, + at::PhiloxCudaState, scalar_t*, uint8_t*); + if (masked){ + switch (tileSize){ + case 2: + kernel = &transducer_joint_tiled_forward; + break; + case 4: + kernel = &transducer_joint_tiled_forward; + break; + } + } + else{ + switch (tileSize){ + case 1: + kernel = &transducer_joint_tiled_forward; + break; + case 2: + kernel = &transducer_joint_tiled_forward; + break; + case 4: + kernel = &transducer_joint_tiled_forward; + break; + } + } + + kernel<<>>( + f.data_ptr(), + g.data_ptr(), + fLen.data_ptr(), + gLen.data_ptr(), + batchOffsetPtr, + maxFLen, + maxGLen, + hiddenSize, + hiddenPerBlock, + packOutput, + relu, + dropout, + 1.0f - dropoutProb, + rng_engine_inputs, + sum.data_ptr(), + maskPtr); + })); + } + + C10_CUDA_CHECK(cudaGetLastError()); + if (masked) + return {sum, mask}; + else + return {sum}; +} + +std::vector transducer_joint_cuda_backward( + std::vector in, + torch::Tensor fLen, + torch::Tensor gLen, + torch::Tensor batchOffset, + int maxFLen, + int maxGLen, + bool packOutput, + float scale){ + + auto grad = in[0]; + bool masked = (in.size() == 2); + uint8_t *maskPtr = masked ? in[1].data_ptr() : nullptr; + + auto tensorOpt = grad.options(); + auto dtype = grad.scalar_type(); + const int batchSize = fLen.size(0); + const int hiddenSize = grad.size(-1); + + const auto deviceProperties = at::cuda::getCurrentDeviceProperties(); + const int maxNumWarp = deviceProperties->maxThreadsPerBlock / C10_WARP_SIZE; + + torch::Tensor fGrad = torch::empty({batchSize, maxFLen, hiddenSize}, tensorOpt); + torch::Tensor gGrad = torch::empty({batchSize, maxGLen, hiddenSize}, tensorOpt); + + int64_t *batchOffsetPtr = (!packOutput) ? nullptr : batchOffset.data_ptr(); + + // The number "y" I would like each thread to work on + const int workPerThread = 32; + // Since the bwd for f and g have the same thread block size, we need to use the max of the two. + int numWarp = largestPowerOfTwo((std::max(maxFLen, maxGLen) + workPerThread-1) / workPerThread); + // Would like to have at least 2 warps + numWarp = std::max(2, numWarp); + // cap on the maximum number of warps allowed + numWarp = std::min(maxNumWarp, numWarp); + + // Need smem for transposing the partial sum. The partial sum is in a matrix of the shape + // numWarp x warpSize + const int smemSize = numWarp * C10_WARP_SIZE; + const dim3 threads(C10_WARP_SIZE, numWarp, 1); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(dtype, "transducer_joint_cuda_backward_kernel", ([&] { + auto gradPtr = grad.data_ptr(); + auto fLenPtr = fLen.data_ptr(); + auto gLenPtr = gLen.data_ptr(); + auto fGradPtr = fGrad.data_ptr(); + auto gGradPtr = gGrad.data_ptr(); + + // resolve the acc_t type + using acc_t = at::acc_type; + using vec_t = uint64_t; + + constexpr int vectFactor = sizeof(vec_t) / sizeof(scalar_t); + constexpr int vecAlignment = std::alignment_of::value; + + // if all input and output tensors meet the alignment requirement + bool memAlign = (reinterpret_cast(gradPtr) % vecAlignment == 0) + and (reinterpret_cast(fGradPtr) % vecAlignment == 0) + and (reinterpret_cast(gGradPtr) % vecAlignment == 0); + + if (vectFactor > 1 and hiddenSize%vectFactor == 0 and memAlign){ + // If vectorization helps and the alignment requirement is met, use the vectorized + // kernel. For simplicity, hiddenSize needs to be a multiple vecFactor. + const dim3 blocks( (hiddenSize+C10_WARP_SIZE*vectFactor-1)/(C10_WARP_SIZE*vectFactor), + maxFLen+maxGLen, + batchSize); + if (masked){ + transducer_joint_combined_vec_backward + + <<>>( + gradPtr, + maskPtr, + fLenPtr, + gLenPtr, + batchOffsetPtr, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + scale, + fGradPtr, + gGradPtr); + } + else{ + transducer_joint_combined_vec_backward + + <<>>( + gradPtr, + maskPtr, + fLenPtr, + gLenPtr, + batchOffsetPtr, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + scale, + fGradPtr, + gGradPtr); + } + } + else{ + const dim3 blocks((hiddenSize+C10_WARP_SIZE-1)/C10_WARP_SIZE, + maxFLen + maxGLen, batchSize); + if (masked){ + transducer_joint_combined_backward + <<>>( + gradPtr, + maskPtr, + fLenPtr, + gLenPtr, + batchOffsetPtr, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + scale, + fGradPtr, + gGradPtr); + } + else{ + transducer_joint_combined_backward + <<>>( + gradPtr, + maskPtr, + fLenPtr, + gLenPtr, + batchOffsetPtr, + maxFLen, + maxGLen, + hiddenSize, + packOutput, + scale, + fGradPtr, + gGradPtr); + } + } + })); + + return {fGrad, gGrad}; +} diff --git a/apex/apex/contrib/csrc/transducer/transducer_loss.cpp b/apex/apex/contrib/csrc/transducer/transducer_loss.cpp new file mode 100644 index 00000000..f63a67f1 --- /dev/null +++ b/apex/apex/contrib/csrc/transducer/transducer_loss.cpp @@ -0,0 +1,109 @@ +#include +#include + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::vector transducer_loss_cuda_forward( + torch::Tensor x, + torch::Tensor label, + torch::Tensor audLen, + torch::Tensor txtLen, + torch::Tensor batchOffset, + int maxFLen, + int blankIdx, + int opt, + bool packedInput); + +torch::Tensor transducer_loss_cuda_backward( + torch::Tensor x, + torch::Tensor lossGrad, + torch::Tensor alpha, + torch::Tensor beta, + torch::Tensor audLen, + torch::Tensor txtLen, + torch::Tensor label, + torch::Tensor batchOffset, + int maxFLen, + int blankIdx, + int opt, + bool fuseSoftmaxBackward, + bool packedInput); + + +std::vector transducer_loss_forward( + torch::Tensor x, + torch::Tensor label, + torch::Tensor fLen, + torch::Tensor yLen, + torch::Tensor batchOffset, + int maxFLen, + int blankIdx, + int opt, + bool packedInput + ) { + + CHECK_INPUT(x); + CHECK_INPUT(label); + CHECK_INPUT(fLen); + CHECK_INPUT(yLen); + if (packedInput) + CHECK_INPUT(batchOffset); + return transducer_loss_cuda_forward( + x, + label, + fLen, + yLen, + batchOffset, + maxFLen, + blankIdx, + opt, + packedInput); +} + +torch::Tensor transducer_loss_backward( + torch::Tensor x, + torch::Tensor lossGrad, + torch::Tensor alpha, + torch::Tensor beta, + torch::Tensor fLen, + torch::Tensor yLen, + torch::Tensor label, + torch::Tensor batchOffset, + int maxFLen, + int blankIdx, + int opt, + bool fuseSoftmaxBackward, + bool packedInput){ + + CHECK_INPUT(x); + CHECK_INPUT(label); + CHECK_INPUT(lossGrad); + CHECK_INPUT(alpha); + CHECK_INPUT(beta); + CHECK_INPUT(fLen); + CHECK_INPUT(yLen); + if (packedInput) + CHECK_INPUT(batchOffset); + + return transducer_loss_cuda_backward( + x, + lossGrad, + alpha, + beta, + fLen, + yLen, + label, + batchOffset, + maxFLen, + blankIdx, + opt, + fuseSoftmaxBackward, + packedInput); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &transducer_loss_forward, "transducer loss forward (CUDA)"); + m.def("backward", &transducer_loss_backward, "transducer loss backward (CUDA)"); +} diff --git a/apex/apex/contrib/csrc/transducer/transducer_loss_kernel.cu b/apex/apex/contrib/csrc/transducer/transducer_loss_kernel.cu new file mode 100755 index 00000000..295e14b3 --- /dev/null +++ b/apex/apex/contrib/csrc/transducer/transducer_loss_kernel.cu @@ -0,0 +1,767 @@ +#include + +#include +#include + +#include +#include +#include +#include + +template +__device__ __forceinline__ scalar_t logSumExp(scalar_t a, scalar_t b) { + // standard log-sum-exp trick is used here to provide better numerical stability + return (a >= b) ? a + std::log1p(exp(b-a)) : b + std::log1p(exp(a-b)); +} + +// Vanilla transducer loss function (i.e. forward-backward algorithm) +// Detail of this loss function can be found in: +// [1] Sequence Transduction with Recurrent Neural Networks. + +// Forward (alpha) and backward (beta) path are launched together. Input is assumed to be converted +// into log scale by the preceding log_softmax layer +// Diagonal wavefront advancing usually used in dynamic programming is leveraged here. +// alpha and beta are of acc_t type, as they are essentially accumulators. + +// This loss function supports packed input where a tensor of shape [B, T, U, H] is packed into +// [B_packed, H]. +// Don't-care region (t > audLen) or (u > txtLen) is removed. +// To support the packed input, the starting offsets for each batch need to be specified with +// batchOffset. +template +__global__ void transducer_loss_forward( + const scalar_t* x, + const int* label, + const int* audLen, + const int* txtLen, + const int64_t* batchOffset, + int64_t dictSize, // 64-bit indexing for data tensor + int64_t blankIdx, + int64_t maxFLen, + int64_t maxGLen, + bool packedInput, + acc_t* alpha, + acc_t* beta, + scalar_t* loss) { + + const int batch = blockIdx.y; + const int tid = threadIdx.x; + const auto myFLen = audLen[batch]; + // Note that start of the sentence is added as 1 here + const auto myGLen = txtLen[batch] + 1; + const auto myLabel = label + batch * (maxGLen-1); + const int64_t myBatchOffset = packedInput ? (batch == 0 ? 0 : batchOffset[batch-1]) + : batch * maxFLen * maxGLen; + const int64_t myStrideT = packedInput ? myGLen : maxGLen; + const scalar_t* myX = x + myBatchOffset * dictSize; + int u = tid; + + if (blockIdx.x == 0){ + // alpha path + acc_t* myAlpha = alpha + batch*maxFLen*maxGLen; + if (u == 0) + myAlpha[0] = 0; + __syncthreads(); + + for (int64_t step = 1; step < myFLen+myGLen-1; ++step){ + // Move along the diagonal wavefront to leverage available parallelism + for (u = tid; u < myGLen; u += blockDim.x){ + int64_t t = step - u; + if (t >= 0 and t < myFLen and u >= 0 and u < myGLen){ + // Eq(16) in [1] + if (u == 0){ + // alpha(t, u) = alpha(t-1, u) * null(t-1, u) + myAlpha[t*maxGLen + u] = myAlpha[(t-1)*maxGLen] + + myX[((t-1)*myStrideT) * dictSize + blankIdx]; + } + else if (t == 0){ + // alpha(t, u-1) = alpha(t, u-1) * y(t, u-1) + myAlpha[u] = myAlpha[u - 1] + myX[(u - 1) * dictSize + myLabel[u - 1]]; + } + else{ + // alpha(t, u) = alpha(t-1, u) * null(t-1, u) + alpha(t, u-1) * y(t, u-1) + acc_t current = myAlpha[(t-1)*maxGLen + u] + + myX[((t-1)*myStrideT + u) * dictSize + blankIdx]; + acc_t next = myAlpha[t*maxGLen + u - 1] + + myX[(t*myStrideT + u - 1) * dictSize + myLabel[u - 1]]; + myAlpha[t*maxGLen + u] = logSumExp(next, current); + } + } + } + __syncthreads(); + } + } + else if (blockIdx.x == 1){ + // beta path + acc_t* myBeta = beta + batch*maxFLen*maxGLen; + if (u == 0){ + myBeta[(myFLen-1)*maxGLen + myGLen - 1] = myX[((myFLen-1)*myStrideT + + myGLen - 1) * dictSize + blankIdx]; + } + __syncthreads(); + + for (int64_t step = myFLen+myGLen - 3; step >= 0; --step){ + for (u = tid; u < myGLen; u += blockDim.x){ + int64_t t = step - u; + if (t >= 0 and t < myFLen and u >=0 and u < myGLen){ + // Eq(18) in [1] + if (u == myGLen - 1){ + // beta(t, u) = beta(t+1, u) * null(t, u) + myBeta[t*maxGLen + u] = myBeta[(t+1)*maxGLen + u] + + myX[(t*myStrideT + u) * dictSize + blankIdx]; + } + else if (t == myFLen - 1){ + // beta(t, u) = beta(t, u+1) * y(t, u) + myBeta[t*maxGLen + u] = myBeta[t*maxGLen + u + 1] + + myX[(t*myStrideT + u) * dictSize + myLabel[u]]; + } + else{ + // beta(t, u) = beta(t+1, u)*null(t, u) + beta(t, u+1)*y(t, u) + acc_t current = myBeta[(t+1)*maxGLen + u] + + myX[(t*myStrideT + u) * dictSize + blankIdx]; + acc_t next = myBeta[t*maxGLen + u + 1] + + myX[(t*myStrideT + u) * dictSize + myLabel[u]]; + myBeta[t*maxGLen + u] = logSumExp(next, current); + } + } + } + __syncthreads(); + } + if (tid == 0) + loss[batch] = -myBeta[0]; + } + +} + +// transudcer loss function (i.e. forward-backward algorithm) with batch loading optimization. +// Compared to the vanilla version, there are two optimizations: +// 1. load x in batch through loop unrolling to reduce the latency. +// 2. Use registers and shared memory to hold alpha and beta values passed from one step the next. +// For simplicity, this kernel currently only supports U <= maxThread, which should be the common +// case. For cases where U > maxThread, the vanilla kernel is used as a fallback option. + +// Detail of this loss function can be found in: +// [1] Sequence Transduction with Recurrent Neural Networks. +// Forward (alpha) and backward (beta) path are launched together. Input is assumed to be converted +// into log scale by the preceding log_softmax layer +// Diagonal wavefront advancing usually used in dynamic programming is leveraged here. +// alpha and beta are of acc_t type, as they are essentially accumulators. + +// This loss function supports packed input where a tensor of shape [B, T, U, H] is packed into +// [B_packed, H]. +// Don't-care region (t > audLen) or (u > txtLen) is removed. +// To support the packed input, the starting offsets for each batch need to be specified with +// batchOffset. +template +__global__ void transducer_loss_batch_load_forward( + const scalar_t* x, + const int* label, + const int* audLen, + const int* txtLen, + const int64_t* batchOffset, + int64_t dictSize, + int64_t blankIdx, + int64_t maxFLen, + int64_t maxGLen, + bool packedInput, + acc_t* alpha, + acc_t* beta, + scalar_t* loss) { + + const int batch = blockIdx.y; + int u = threadIdx.x; + const auto myFLen = audLen[batch]; + const auto myGLen = txtLen[batch] + 1; + const int64_t myBatchOffset = packedInput ? (batch == 0 ? 0 : batchOffset[batch-1]) + : batch * maxFLen * maxGLen; + const int64_t myStrideT = packedInput ? myGLen : maxGLen; + const scalar_t* myX = x + myBatchOffset * dictSize; + scalar_t next[batchLdSize], current[batchLdSize]; + extern __shared__ char smem8[]; + auto smem = reinterpret_cast(smem8); + + if (blockIdx.x == 0){ + // alpha path + acc_t* myAlpha = alpha + batch*maxFLen*maxGLen; + // two SMEM regions for double buffering read and write data to avoid data race + acc_t * const sharedAlpha[2] = {smem, smem+maxGLen}; + + sharedAlpha[0][u] = 0; + __syncthreads(); + + if (u == 0) + myAlpha[0] = 0; + + auto myAlphaLabel = (u == 0) ? 0 : label[batch*(maxGLen-1) + u - 1]; + // register used to pass value to the next step for the same thread + acc_t prvStepAlpha = 0; + for (int64_t step = 1; step < myFLen+myGLen-1+batchLdSize; step += batchLdSize){ + // Move along the diagonal wavefront to leverage available parallelism + // Batch loading X through loop unrolling + #pragma unroll + for (int i = 0; i < batchLdSize; ++i){ + if (step+i= 0 and t < myFLen and u >= 0 and u < myGLen){ + if (u == 0){ + current[i] = myX[currentId]; + } + else if (t == 0){ + next[i] = myX[nextId]; + } + else{ + current[i] = myX[currentId]; + next[i] = myX[nextId]; + } + } + } + } + // main computing loop + for (int i = 0; i < batchLdSize; ++i){ + // swap the pointer for double buffering + auto sharedAlphaRd = sharedAlpha[(step+i-1)%2]; + auto sharedAlphaWr = sharedAlpha[(step+i)%2]; + if (step+i= 0 and t < myFLen and u >= 0 and u < myGLen){ + // Eq(16) in [1] + if (u == 0) + prvStepAlpha = prvStepAlpha+current[i]; + else if (t == 0) + prvStepAlpha = sharedAlphaRd[u-1]+next[i]; + else + prvStepAlpha = logSumExp(prvStepAlpha+current[i], sharedAlphaRd[u-1] + + next[i]); + sharedAlphaWr[u] = prvStepAlpha; + myAlpha[t*maxGLen + u] = prvStepAlpha; + } + } + __syncthreads(); + } + } + } + else if (blockIdx.x == 1){ + // beta path + acc_t* myBeta = beta + batch*maxFLen*maxGLen; + // two SMEM regions for double buffering read and write data to avoid data race + acc_t * const sharedBeta[2] = {smem, smem + maxGLen}; + sharedBeta[0][u] = myX[((myFLen-1)*myStrideT + myGLen - 1) * dictSize + blankIdx]; + __syncthreads(); + + auto myBetaLabel = (u == maxGLen - 1) ? 0 : label[batch*(maxGLen-1) + u]; + // register used to pass value to the next step for the same thread + acc_t prvStepBeta = myX[((myFLen-1)*myStrideT + myGLen - 1) * dictSize + blankIdx]; + if (u == 0) + myBeta[(myFLen-1)*maxGLen + myGLen - 1] = prvStepBeta; + + for (int64_t step = 1; step < myFLen+myGLen-1; step += batchLdSize){ + // Move along the diagonal wavefront to leverage available parallelism + // Batch loading X + #pragma unroll + for (int i = 0; i < batchLdSize; ++i){ + if (step+i= 0 and t < myFLen and u >= 0 and u < myGLen){ + if (u == myGLen - 1){ + current[i] = myX[currentId]; + } + else if (t == myFLen - 1){ + next[i] = myX[nextId]; + } + else{ + current[i] = myX[currentId]; + next[i] = myX[nextId]; + } + } + } + } + // main computing loop + for (int i = 0; i < batchLdSize; ++i){ + // swap the pointer for double buffering + auto sharedBetaRd = sharedBeta[(step+i-1)%2]; + auto sharedBetaWr = sharedBeta[(step+i)%2]; + if (step+i= 0 and t < myFLen and u >= 0 and u < myGLen){ + // Eq(18) in [1] + if (u == myGLen - 1) + prvStepBeta = prvStepBeta+current[i]; + else if (t == myFLen - 1) + prvStepBeta = sharedBetaRd[u+1]+next[i]; + else + prvStepBeta = logSumExp(prvStepBeta+current[i], sharedBetaRd[u+1] + + next[i]); + sharedBetaWr[u] = prvStepBeta; + myBeta[t*maxGLen + u] = prvStepBeta; + } + + } + __syncthreads(); + } + } + if (u == 0) + loss[batch] = -prvStepBeta; + } + +} + +// Vanilla transudcer loss backward operation. +// Detail of this loss function can be found in: +// [1] Sequence Transduction with Recurrent Neural Networks. +// For this backward kernel, bwd op for the preceding softmax is assumed to be handled elsewhere, +// hence only Eq(20) in [1] is implemented in this kernel. + +// Each thread block works on [batch, t, :, :] of data. Each thread works on a specific u at a time +// Since only gradients for the correct token and null token need to be updated, gradients at other +// locations are initialized to 0. + +// To support the packed input, the starting offsets for each batch need to be specified with +// batchOffset. +template +__global__ void transducer_loss_backward( + const scalar_t* x, + const scalar_t* lossGrad, + const int* audLen, + const int* txtLen, + const int* label, + const acc_t* alpha, + const acc_t* beta, + const int64_t* batchOffset, + int64_t dictSize, + int64_t blankIdx, + int64_t maxFLen, + int64_t maxGLen, + bool packedInput, + scalar_t* xGrad) { + + const int tid = threadIdx.x; + const int t = blockIdx.x; + const int batch = blockIdx.y; + const int64_t myFLen = audLen[batch]; + const int64_t myGLen = txtLen[batch] + 1; + const int64_t myBatchOffset = packedInput ? (batch == 0 ? 0 : batchOffset[batch-1]) + : batch * maxFLen * maxGLen; + const int64_t myStrideT = packedInput ? myGLen : maxGLen; + auto myX = x + (myBatchOffset + t*myStrideT)*dictSize; + auto myAlpha = alpha + batch*maxFLen*maxGLen; + auto myBeta = beta + batch*maxFLen*maxGLen; + auto myXGrad = xGrad + (myBatchOffset + t*myStrideT)*dictSize; + auto myLabel = label + batch*(maxGLen-1); + + int64_t u = tid; + while (t < myFLen and u < myGLen){ + // Do the update + // loss = -ln(Pr(y*|x)) + acc_t grad = std::log(lossGrad[batch]) + myAlpha[t*maxGLen + u] - myBeta[0]; + if (u != myGLen - 1) + myXGrad[u*dictSize + myLabel[u]] = -std::exp(grad + myBeta[t*maxGLen + u + 1] + + myX[u*dictSize + myLabel[u]]); + if (t == myFLen - 1 and u == myGLen - 1) + myXGrad[u*dictSize + blankIdx] = -std::exp(grad + myX[u*dictSize + blankIdx]); + else if (t != myFLen - 1) + myXGrad[u*dictSize + blankIdx] = -std::exp(grad + myBeta[(t+1)*maxGLen + u] + + myX[u*dictSize + blankIdx]); + + u += blockDim.x; + } +} + +// Fused transudcer loss backward operation. +// Detail of this loss function can be found in: +// [1] Sequence Transduction with Recurrent Neural Networks. +// The bwd op of the preceding softmax layer is fused in this kernel. +// Each thread block works on [batch, t, u, :] of data. Each thread works on a specific h at a time + +// To support the packed input, the starting offsets for each batch need to be specified with +// batchOffset. +template +__global__ void transducer_loss_fused_backward( + const scalar_t* x, + const scalar_t* lossGrad, + const int* audLen, + const int* txtLen, + const int* label, + const acc_t* alpha, + const acc_t* beta, + const int64_t* batchOffset, + int64_t dictSize, + int64_t blankIdx, + int64_t maxFLen, + int64_t maxGLen, + bool packedInput, + scalar_t* xGrad) { + + const int tid = threadIdx.x; + const int u = blockIdx.x; + const int t = blockIdx.y; + const int batch = blockIdx.z; + const int64_t myFLen = audLen[batch]; + const int64_t myGLen = txtLen[batch] + 1; + const int64_t myBatchOffset = packedInput ? (batch == 0 ? 0 : batchOffset[batch-1]) + : batch * maxFLen * maxGLen; + const int64_t myStrideT = packedInput ? myGLen : maxGLen; + + __shared__ acc_t commonFactor, myBetaTU, myBetaTUp1, myBetaTp1U, myLabelShared; + auto myXGrad = xGrad + (myBatchOffset + t*myStrideT +u)*dictSize; + + if (t < myFLen and u < myGLen){ + auto myX = x + (myBatchOffset + t*myStrideT +u)*dictSize; + auto myAlpha = alpha + batch*maxFLen*maxGLen; + auto myBeta = beta + batch*maxFLen*maxGLen; + auto myLabel = label + batch*(maxGLen-1); + + // load and store shared variables in SMEM + if (tid == 0){ + commonFactor = std::log(lossGrad[batch]) + myAlpha[t*maxGLen + u] - myBeta[0]; + myBetaTU = myBeta[t*maxGLen + u]; + myBetaTUp1 = myBeta[t*maxGLen + u + 1]; + myBetaTp1U = myBeta[(t+1)*maxGLen + u]; + myLabelShared = myLabel[u]; + } + + __syncthreads(); + + for (int64_t h = tid; h < dictSize; h += blockDim.x){ + // Do the update + acc_t grad = commonFactor + myX[h]; // loss = -ln(Pr(y*|x)) + acc_t myGrad = std::exp(grad + myBetaTU); + if (u != myGLen - 1 and h == myLabelShared){ + myGrad -= std::exp(grad + myBetaTUp1); + } + else if (h == blankIdx){ + if (t == myFLen - 1 and u == myGLen - 1) + myGrad -= std::exp(grad); + else if (t != myFLen - 1) + myGrad -= std::exp(grad + myBetaTp1U); + } + myXGrad[h] = myGrad; + } + } + else if (!packedInput){ + // In non-pack mode, need to make sure the gradients for don't-care regions are zero. + for (int64_t h = tid; h < dictSize; h += blockDim.x){ + myXGrad[h] = 0; + } + } +} + + +// Vectorized version of fused transudcer loss backward operation. +// Detail of this loss function can be found in: +// [1] Sequence Transduction with Recurrent Neural Networks. +// The bwd op of the preceding softmax layer is fused in this kernel. +// Each thread block works on [batch, t, u, :] of data. Each thread works on a specific h at a time + +// To support the packed input, the starting offsets for each batch need to be specified with +// batchOffset. +template +__global__ void transducer_loss_fused_vec_backward( + const scalar_t* x, + const scalar_t* lossGrad, + const int* audLen, + const int* txtLen, + const int* label, + const acc_t* alpha, + const acc_t* beta, + const int64_t* batchOffset, + int64_t dictSize, + int64_t blankIdx, + int64_t maxFLen, + int64_t maxGLen, + bool packedInput, + scalar_t* xGrad) { + + const int tid = threadIdx.x; + const int u = blockIdx.x; + const int t = blockIdx.y; + const int batch = blockIdx.z; + const int64_t myFLen = audLen[batch]; + const int64_t myGLen = txtLen[batch] + 1; + const int64_t myBatchOffset = packedInput ? (batch == 0 ? 0 : batchOffset[batch-1]) + : batch * maxFLen * maxGLen; + const int64_t myStrideT = packedInput ? myGLen : maxGLen; + + __shared__ acc_t commonFactor, myBetaTU, myBetaTUp1, myBetaTp1U, myLabelShared; + auto myXGrad = xGrad + (myBatchOffset + t*myStrideT +u)*dictSize; + auto myX = x + (myBatchOffset + t*myStrideT +u)*dictSize; + auto myAlpha = alpha + batch*maxFLen*maxGLen; + auto myBeta = beta + batch*maxFLen*maxGLen; + auto myLabel = label + batch*(maxGLen-1); + + // Variabels for vectorization + scalar_t myXBuffer[V], myXGradBuffer[V]; + auto myXVec = reinterpret_cast(myX); + auto myXGradVec = reinterpret_cast(myXGrad); + auto myXBufferVec = reinterpret_cast(myXBuffer); + auto myXGradBufferVec = reinterpret_cast(myXGradBuffer); + if (t < myFLen and u < myGLen){ + // load and store shared variables in SMEM + if (tid == 0){ + commonFactor = std::log(lossGrad[batch]) + myAlpha[t*maxGLen + u] - myBeta[0]; + myBetaTU = myBeta[t*maxGLen + u]; + if (t != myFLen - 1) + myBetaTp1U = myBeta[(t+1)*maxGLen + u]; + if (u != myGLen - 1){ + myBetaTUp1 = myBeta[t*maxGLen + u + 1]; + myLabelShared = myLabel[u]; + } + } + + __syncthreads(); + + #pragma unroll + for (int64_t h0 = tid*V; h0 < dictSize; h0 += blockDim.x*V){ + // Load myX in a vector form + *myXBufferVec = myXVec[h0/V]; + // Do the update for a vector of input + #pragma unroll + for (int i = 0; i < V; ++i){ + auto h = h0 + i; + acc_t grad = commonFactor + myXBuffer[i]; // loss = -ln(Pr(y*|x)) + acc_t myGrad = std::exp(grad + myBetaTU); + if (u != myGLen - 1 and h == myLabelShared){ + myGrad -= std::exp(grad + myBetaTUp1); + } + else if (h == blankIdx){ + if (t == myFLen - 1 and u == myGLen - 1) + myGrad -= std::exp(grad); + else if (t != myFLen - 1) + myGrad -= std::exp(grad + myBetaTp1U); + } + myXGradBuffer[i] = myGrad; + } + + // Store myXGrad in a vector form + myXGradVec[h0/V] = *myXGradBufferVec; + + } + } + else if (!packedInput){ + // In non-pack mode, need to make sure the gradients for don't-care regions are zero. + for (int64_t h0 = tid*V; h0 < dictSize; h0 += blockDim.x*V){ + myXGradVec[h0/V] = 0; + } + } +} + + +std::vector transducer_loss_cuda_forward( + torch::Tensor x, + torch::Tensor label, + torch::Tensor audLen, + torch::Tensor txtLen, + torch::Tensor batchOffset, + int maxFLen, + int blankIdx, + int opt, + bool packedInput){ + + auto scalarType = x.scalar_type(); + auto tensorOpt = x.options(); + const int batchSize = label.size(0); + const int maxGLen = label.size(1) + 1; + const int dictSize = x.size(-1); + + TORCH_CHECK(blankIdx >= 0 and blankIdx < dictSize, + "Expected blank index to be in the range of 0 to ", + dictSize-1, + ", but got ", + blankIdx); + TORCH_CHECK(opt == -1 or opt == 0 or opt == 1, + "Got an invalid optimization level ", + opt); + + // The data type of alpha and beta will be resolved at dispatch time, + // hence defined here and assigned later + torch::Tensor alpha; + torch::Tensor beta; + torch::Tensor loss = torch::empty({batchSize}, tensorOpt); + const auto deviceProperties = at::cuda::getCurrentDeviceProperties(); + const auto maxThreadPerBlock = deviceProperties->maxThreadsPerBlock; + const auto maxSmemPerBlock = deviceProperties->sharedMemPerBlock; + const auto batchOffsetPtr = packedInput ? batchOffset.data_ptr() : nullptr; + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(scalarType, "transducer_loss_cuda_forward", ([&] { + // resolve accumulation type + using acc_t = at::acc_type; + auto accType = c10::CppTypeToScalarType::value; + auto accTensorOpt = tensorOpt.dtype(accType); + alpha = torch::empty({batchSize, maxFLen, maxGLen}, accTensorOpt); + beta = torch::empty({batchSize, maxFLen, maxGLen}, accTensorOpt); + + // decide what kernel to launch based on the problem size + // if the required SMEM size or number threads exceeds the limit, fall back to the vanilla + // kernel. + const auto smemSize = 2*maxGLen*sizeof(acc_t); + const auto optFallBack = (maxGLen > maxThreadPerBlock or smemSize > maxSmemPerBlock) ? 0 + : (opt == -1) ? 1 : opt; + const int threads = std::min(maxThreadPerBlock, maxGLen); + const dim3 blocks(2, batchSize, 1); + + if (optFallBack == 0) + transducer_loss_forward<<>>( + x.data_ptr(), + label.data_ptr(), + audLen.data_ptr(), + txtLen.data_ptr(), + batchOffsetPtr, + dictSize, + blankIdx, + maxFLen, + maxGLen, + packedInput, + alpha.data_ptr(), + beta.data_ptr(), + loss.data_ptr()); + else if (optFallBack == 1) + transducer_loss_batch_load_forward + <<>>( + x.data_ptr(), + label.data_ptr(), + audLen.data_ptr(), + txtLen.data_ptr(), + batchOffsetPtr, + dictSize, + blankIdx, + maxFLen, + maxGLen, + packedInput, + alpha.data_ptr(), + beta.data_ptr(), + loss.data_ptr()); + + })); + C10_CUDA_CHECK(cudaGetLastError()); + + return {alpha, beta, loss}; +} + + + + +torch::Tensor transducer_loss_cuda_backward( + torch::Tensor x, + torch::Tensor lossGrad, + torch::Tensor alpha, + torch::Tensor beta, + torch::Tensor audLen, + torch::Tensor txtLen, + torch::Tensor label, + torch::Tensor batchOffset, + int maxFLen, + int blankIdx, + int opt, + bool fuseSoftmaxBackward, + bool packedInput){ + + auto dtype = x.scalar_type(); + torch::Tensor xGrad; + const int batchSize = label.size(0); + const int maxGLen = label.size(1) + 1; + const int dictSize = x.size(-1); + const auto deviceProperties = at::cuda::getCurrentDeviceProperties(); + const int maxThreadPerBlock = deviceProperties->maxThreadsPerBlock; + const int warpSize = deviceProperties->warpSize; + const auto batchOffsetPtr = packedInput ? batchOffset.data_ptr() : nullptr; + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (fuseSoftmaxBackward){ + // alloc empty tensors for performance, hence need to ensure zeros are writtern to + // don't-care region in the kernel. + xGrad = torch::empty_like(x); + + // Would like each thread to work on 4 hidden units + const int workPerThread = 4; + // Don't want to have more than 128 threads per thread block + const int maxThreadPerElmt = std::min(128, maxThreadPerBlock); + const int threads = std::min(maxThreadPerElmt, std::max(warpSize, + (dictSize+workPerThread-1)/workPerThread)); + const dim3 blocks(maxGLen, maxFLen, batchSize); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(dtype, "transducer_loss_cuda_backward", ([&] { + using vec_t = uint64_t; + using acc_t = at::acc_type; + constexpr int vectFactor = sizeof(vec_t) / sizeof(scalar_t); + constexpr int vecAlignment = std::alignment_of::value; + // if all input and output tensors meet the alignment requirement + bool memAlign = reinterpret_cast(x.data_ptr()) % vecAlignment == 0 + and reinterpret_cast(xGrad.data_ptr()) + % vecAlignment == 0; + + if (vectFactor > 1 and dictSize%vectFactor == 0 and memAlign){ + transducer_loss_fused_vec_backward + <<>>( + x.data_ptr(), + lossGrad.data_ptr(), + audLen.data_ptr(), + txtLen.data_ptr(), + label.data_ptr(), + alpha.data_ptr(), + beta.data_ptr(), + batchOffsetPtr, + dictSize, + blankIdx, + maxFLen, + maxGLen, + packedInput, + xGrad.data_ptr()); + } + else{ + transducer_loss_fused_backward<<>>( + x.data_ptr(), + lossGrad.data_ptr(), + audLen.data_ptr(), + txtLen.data_ptr(), + label.data_ptr(), + alpha.data_ptr(), + beta.data_ptr(), + batchOffsetPtr, + dictSize, + blankIdx, + maxFLen, + maxGLen, + packedInput, + xGrad.data_ptr()); + + } + })); + } + else{ + // for non-fused kernel, the gradients need to be writtern are very sparse, hence initialize + // the tensor with all zeros. + xGrad = torch::zeros_like(x); + // don't launch more threads than needed. + const int threads = std::min(maxThreadPerBlock, maxGLen); + const dim3 blocks(maxFLen, batchSize); + AT_DISPATCH_FLOATING_TYPES_AND_HALF(dtype, "transducer_loss_cuda_backward", ([&] { + using acc_t = at::acc_type; + transducer_loss_backward<<>>( + x.data_ptr(), + lossGrad.data_ptr(), + audLen.data_ptr(), + txtLen.data_ptr(), + label.data_ptr(), + alpha.data_ptr(), + beta.data_ptr(), + batchOffsetPtr, + dictSize, + blankIdx, + maxFLen, + maxGLen, + packedInput, + xGrad.data_ptr()); + })); + } + C10_CUDA_CHECK(cudaGetLastError()); + + return xGrad; +} diff --git a/apex/apex/contrib/csrc/xentropy/interface.cpp b/apex/apex/contrib/csrc/xentropy/interface.cpp new file mode 100644 index 00000000..72a62979 --- /dev/null +++ b/apex/apex/contrib/csrc/xentropy/interface.cpp @@ -0,0 +1,52 @@ +#include + +// CUDA forward declarations + +std::vector softmax_xentropy_cuda( + const at::Tensor &input, + const at::Tensor &labels, + const float smoothing, + const bool half_to_float); + +at::Tensor softmax_xentropy_backward_cuda( + const at::Tensor &grad_loss, + const at::Tensor &logits, + const at::Tensor &max_log_sum_exp, + const at::Tensor &labels, + const float smoothing); + +// C++ interface + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::vector softmax_xentropy_forward( + const at::Tensor &input, + const at::Tensor &labels, + const float smoothing, + const bool half_to_float) { + CHECK_CUDA(input); + CHECK_INPUT(labels); + + return softmax_xentropy_cuda(input, labels, smoothing, half_to_float); +} + +at::Tensor softmax_xentropy_backward( + const at::Tensor &grad_loss, + const at::Tensor &logits, + const at::Tensor &max_log_sum_exp, + const at::Tensor &labels, + const float smoothing) { + CHECK_CUDA(grad_loss); + CHECK_CUDA(logits); + CHECK_INPUT(max_log_sum_exp); + CHECK_INPUT(labels); + + return softmax_xentropy_backward_cuda(grad_loss, logits, max_log_sum_exp, labels, smoothing); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &softmax_xentropy_forward, "Softmax cross entropy loss with label smoothing forward (CUDA)"); + m.def("backward", &softmax_xentropy_backward, "Softmax cross entropy loss with label smoothing backward (CUDA)"); +} diff --git a/apex/apex/contrib/csrc/xentropy/xentropy_kernel.cu b/apex/apex/contrib/csrc/xentropy/xentropy_kernel.cu new file mode 100644 index 00000000..bb213fdc --- /dev/null +++ b/apex/apex/contrib/csrc/xentropy/xentropy_kernel.cu @@ -0,0 +1,718 @@ +/** + * From PyTorch: + * + * Copyright (c) 2016- Facebook, Inc (Adam Paszke) + * Copyright (c) 2014- Facebook, Inc (Soumith Chintala) + * Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) + * Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) + * Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) + * Copyright (c) 2011-2013 NYU (Clement Farabet) + * Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) + * Copyright (c) 2006 Idiap Research Institute (Samy Bengio) + * Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + * + * From Caffe2: + * + * Copyright (c) 2016-present, Facebook Inc. All rights reserved. + * + * All contributions by Facebook: + * Copyright (c) 2016 Facebook Inc. + * + * All contributions by Google: + * Copyright (c) 2015 Google Inc. + * All rights reserved. + * + * All contributions by Yangqing Jia: + * Copyright (c) 2015 Yangqing Jia + * All rights reserved. + * + * All contributions from Caffe: + * Copyright(c) 2013, 2014, 2015, the respective contributors + * All rights reserved. + * + * All other contributions: + * Copyright(c) 2015, 2016 the respective contributors + * All rights reserved. + * + * Caffe2 uses a copyright model similar to Caffe: each contributor holds + * copyright over their contributions to Caffe2. The project versioning records + * all such contribution and copyright details. If a contributor wants to further + * mark their specific copyright on a particular contribution, they should + * indicate their copyright solely in the commit message of the change when it is + * committed. + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + * and IDIAP Research Institute nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include + +#include +#include + +#include "type_shim.h" +#include "compat.h" + +#define ALIGN_BYTES 16 + +using Tensor = at::Tensor; +using TensorList = at::TensorList; +using ScalarType = at::ScalarType; +using at::acc_type; + +template +struct LogSoftMaxForwardEpilogue { + __device__ __forceinline__ LogSoftMaxForwardEpilogue(AccumT max_input, AccumT sum) + : logsum(max_input + std::log(sum)) {} + + __device__ __forceinline__ LogSoftMaxForwardEpilogue(AccumT max_log_sum_exp) + : logsum(max_log_sum_exp) {} + + __device__ __forceinline__ OutT operator()(T input) const { + return static_cast(input - logsum); + } + + const AccumT logsum; +}; + +template +struct LogSoftMaxBackwardEpilogue { + __device__ __forceinline__ LogSoftMaxBackwardEpilogue(AccumT sum) + : sum(sum) {} + + __device__ __forceinline__ T operator()(OutT gradOutput, OutT output) const { + return static_cast(gradOutput - std::exp(static_cast(output)) * sum); + } + + const AccumT sum; +}; + + + +const int max_threads = 1024; + +inline dim3 SoftMax_getBlockSize(int ILP, uint64_t dim_size) { + uint64_t block_size = 1; + uint64_t max_block_size = std::min(dim_size / ILP, static_cast(max_threads)); + while (block_size < (max_block_size/2)) block_size *= 2; + // Launch at least a single warp - the kernel assumes that. + block_size = std::max(block_size, static_cast(32)); + return dim3(block_size); +} + +template +struct Add { + __device__ __forceinline__ T operator()(T a, T b) const { + return a + b; + } +}; + +template +struct Max { + __device__ __forceinline__ T operator()(T a, T b) const { + return a < b ? b : a; + } +}; + + +//////////////////////////////////////////////////////////////////////////////// +// Regular kernel (fast when dim_size is large; requires inner_size == 1) +//////////////////////////////////////////////////////////////////////////////// + + +template +struct MaxFloat +{ + __device__ __forceinline__ AccumT operator()(AccumT max, T v) const { + return ::max(max, (AccumT)v); + } +}; + +template +struct AddFloat +{ + __device__ __forceinline__ AccumT operator()(AccumT sum, T v) const { + return sum + v; + } +}; + +template +struct SumExpFloat +{ + __device__ __forceinline__ SumExpFloat(AccumT v) + : max_k(v) {} + + __device__ __forceinline__ AccumT operator()(AccumT sum, T v) const { + return sum + std::exp(v - max_k); + } + + const AccumT max_k; +}; + +template class Reduction, typename AccumT> +__device__ __forceinline__ AccumT +blockReduce(AccumT* smem, AccumT val, + const Reduction& r, + AccumT defaultVal) +{ + // To avoid RaW races from chaining blockReduce calls together, we need a sync here + __syncthreads(); + + smem[threadIdx.x] = val; + + __syncthreads(); + + AccumT warpVal = defaultVal; + + // First warp will perform per-warp reductions for the remaining warps + uint32_t mask = (((uint64_t)1) << (blockDim.x / 32)) - 1; + if (threadIdx.x < 32) { + int lane = threadIdx.x % 32; + if (lane < blockDim.x / 32) { +#pragma unroll + for (int i = 0; i < 32; ++i) { + warpVal = r(warpVal, smem[lane * 32 + i]); + } + __syncwarp(mask); + smem[lane] = warpVal; + } + } + + __syncthreads(); + + // First thread will perform a reduction of the above per-warp reductions + AccumT blockVal = defaultVal; + + if (threadIdx.x == 0) { + for (int i = 0; i < blockDim.x / 32; ++i) { + blockVal = r(blockVal, smem[i]); + } + smem[0] = blockVal; + } + + // Sync and broadcast + __syncthreads(); + return smem[0]; +} + +template class Reduction1, template class Reduction2, typename AccumT> +__device__ __forceinline__ void +blockReduce(AccumT* smem, + AccumT* reducVal1, + AccumT val1, + const Reduction1& r1, + AccumT defaultVal1, + AccumT* reducVal2, + AccumT val2, + const Reduction2& r2, + AccumT defaultVal2) +{ + // To avoid RaW races from chaining blockReduce calls together, we need a sync here + __syncthreads(); + + smem[threadIdx.x] = val1; + smem[blockDim.x + threadIdx.x] = val2; + + __syncthreads(); + + AccumT warpVal1 = defaultVal1; + AccumT warpVal2 = defaultVal2; + + // First warp will perform per-warp reductions for the remaining warps + uint32_t mask = (((uint64_t)1) << (blockDim.x / 32)) - 1; + if (threadIdx.x < 32) { + int lane = threadIdx.x % 32; + if (lane < blockDim.x / 32) { +#pragma unroll + for (int i = 0; i < 32; ++i) { + warpVal1 = r1(warpVal1, smem[lane * 32 + i]); + warpVal2 = r2(warpVal2, smem[lane * 32 + i + blockDim.x]); + } + __syncwarp(mask); + smem[lane] = warpVal1; + smem[lane + blockDim.x] = warpVal2; + } + } + + __syncthreads(); + + // First thread will perform a reduction of the above per-warp reductions + AccumT blockVal1 = defaultVal1; + AccumT blockVal2 = defaultVal2; + + if (threadIdx.x == 0) { + for (int i = 0; i < blockDim.x / 32; ++i) { + blockVal1 = r1(blockVal1, smem[i]); + blockVal2 = r2(blockVal2, smem[i + blockDim.x]); + } + smem[0] = blockVal1; + smem[blockDim.x] = blockVal2; + } + + // Sync and broadcast + __syncthreads(); + *reducVal1 = smem[0]; + *reducVal2 = smem[blockDim.x]; + __syncthreads(); +} + +template class Reduction, int ILP, typename T, typename AccumT> +__device__ __forceinline__ AccumT +ilpReduce(int shift, + T* data, + int size, + const Reduction& r, + AccumT defaultVal) +{ + typedef typename std::aligned_storage::type LoadT; + AccumT threadVal = defaultVal; + int offset = threadIdx.x; + + // shift and do 1 + if(shift > 0){ + data -= shift; + size += shift; + if(threadIdx.x >= shift){ + threadVal = r(threadVal, data[offset]); + } + size -= blockDim.x; + data += blockDim.x; + } + int last = size % (ILP * blockDim.x); + + T v[ILP]; + LoadT* value = reinterpret_cast(&v); + + for (; offset * ILP < (size - last); offset += blockDim.x) { + *value = reinterpret_cast(data)[offset]; + + for (int j = 0; j < ILP; ++j) { + threadVal = r(threadVal, v[j]); + } + } + + offset = size - last + threadIdx.x; + // Epilogue + for (; offset < size; offset += blockDim.x) + threadVal = r(threadVal, data[offset]); + + return threadVal; +} + +template class Reduction1, template class Reduction2, int ILP, typename T, typename AccumT> +__device__ __forceinline__ void +ilpReduce(int shift, + T* data, + int size, + AccumT* reducVal1, + const Reduction1& r1, + AccumT defaultVal1, + AccumT* reducVal2, + const Reduction2& r2, + AccumT defaultVal2) +{ + typedef typename std::aligned_storage::type LoadT; + + AccumT threadVal1 = defaultVal1; + AccumT threadVal2 = defaultVal2; + int offset = threadIdx.x; + + // shift and do 1 + if(shift > 0){ + data -= shift; + size += shift; + if(threadIdx.x >= shift){ + threadVal1 = r1(threadVal1, data[offset]); + threadVal2 = r2(threadVal2, data[offset]); + } + size -= blockDim.x; + data += blockDim.x; + } + int last = size % (ILP * blockDim.x); + + T v[ILP]; + LoadT* value = reinterpret_cast(&v); + + for (; offset * ILP < (size - last); offset += blockDim.x) { + *value = reinterpret_cast(data)[offset]; + + for (int j = 0; j < ILP; ++j) { + threadVal1 = r1(threadVal1, v[j]); + threadVal2 = r2(threadVal2, v[j]); + } + } + + offset = size - last + threadIdx.x; + // Epilogue + for (; offset < size; offset += blockDim.x) { + threadVal1 = r1(threadVal1, data[offset]); + threadVal2 = r2(threadVal2, data[offset]); + } + + *reducVal1 = threadVal1; + *reducVal2 = threadVal2; +} + +template class Epilogue> +__global__ void +cunn_SoftMaxXEntropyForward( + accscalar_t *losses, + outscalar_t *max_log_sum_exp, + scalar_t *input, + int64_t *labels, + int64_t classes, + const float smoothing) +{ + extern __shared__ unsigned char smem[]; + auto sdata = reinterpret_cast(smem); + // forward pointers to batch[blockIdx.x] + // each block handles a sample in the mini-batch + input += blockIdx.x * classes; + //output += blockIdx.x * classes; + const int shift = ((uint64_t)input) % ALIGN_BYTES / sizeof(scalar_t); + + int64_t label = labels[blockIdx.x]; + + // find the max and sum + accscalar_t threadMax, threadSum, max_k, sum_k; + ilpReduce( + shift, input, classes, + &threadMax, MaxFloat(), + -at::numeric_limits::max(), + &threadSum, AddFloat(), + static_cast(0)); + + blockReduce( + sdata, + &max_k, threadMax, Max(), + -at::numeric_limits::max(), + &sum_k, threadSum, Add(), + static_cast(0)); + + accscalar_t threadExp = ilpReduce(shift, input, classes, SumExpFloat(max_k), static_cast(0)); + accscalar_t sumAll = blockReduce( + sdata, threadExp, Add(), static_cast(0)); + + Epilogue epilogue(max_k, sumAll); + + // calculate per element loss with label smoothing + // reserve max + log_sum_exp for bprop + if (threadIdx.x == 0) { + accscalar_t log_prob = epilogue(static_cast(input[label])); + losses[blockIdx.x] = (max_k + std::log(sumAll) - sum_k / classes) \ + * smoothing - log_prob * (1 - smoothing); + max_log_sum_exp[blockIdx.x] = max_k + std::log(sumAll); + } +} + +template +__device__ __forceinline__ void +apply(scalar_t *gradInput, + scalar_t *logits, + outscalar_t *max_log_sum_exp, + outscalar_t *gradOutput, + int64_t *labels, + const float smoothing, + int classes) +{ + accscalar_t smooth_positives = 1.0 - smoothing; + accscalar_t smooth_negatives = smoothing / classes; + accscalar_t tmpGradOutput = gradOutput[blockIdx.x]; + int64_t label = labels[blockIdx.x]; + accscalar_t coeff = max_log_sum_exp[blockIdx.x]; + + int offset = threadIdx.x; + int last = classes % (ILP * blockDim.x); + + for (; offset < classes - last; offset += blockDim.x * ILP) { + accscalar_t tmpLogits[ILP]; + +#pragma unroll + for (int j = 0; j < ILP; ++j) { + tmpLogits[j] = static_cast(logits[offset + j * blockDim.x]); + } + +#pragma unroll + for (int j = 0; j < ILP; ++j) + gradInput[offset + j * blockDim.x] = tmpGradOutput * ( + std::exp(tmpLogits[j] - coeff) - static_cast( + (offset + j * blockDim.x == label) ? 1 : 0) * + smooth_positives - smooth_negatives); + } + + for (; offset < classes; offset += blockDim.x) + gradInput[offset] = tmpGradOutput * (std::exp( + static_cast(logits[offset]) - coeff) - + static_cast((offset == label) ? 1 : 0) * + smooth_positives - smooth_negatives); +} + + +template +__device__ __forceinline__ void +aligned_apply(int shift, + scalar_t *gradInput, + scalar_t *logits, + outscalar_t *max_log_sum_exp, + outscalar_t *gradOutput, + int64_t *labels, + const float smoothing, + int classes) +{ + accscalar_t smooth_positives = 1.0 - smoothing; + accscalar_t smooth_negatives = smoothing / classes; + accscalar_t tmpGradOutput = gradOutput[blockIdx.x]; + int64_t label = labels[blockIdx.x]; + accscalar_t coeff = max_log_sum_exp[blockIdx.x]; + + int offset = threadIdx.x; + + // shift and do 1 + if(shift > 0){ + logits -= shift; + gradInput -= shift; + classes += shift; + if(threadIdx.x >= shift){ + gradInput[offset] = tmpGradOutput * (std::exp( + static_cast(logits[offset]) - coeff) - + static_cast(((offset - shift) == label) ? 1 : 0) * + smooth_positives - smooth_negatives); + } + classes -= blockDim.x; + gradInput += blockDim.x; + logits += blockDim.x; + shift -= blockDim.x; + } + + int last = classes % (ILP * blockDim.x); + + typedef typename std::aligned_storage::type LoadT; + // input + scalar_t v[ILP]; + LoadT* value = reinterpret_cast(&v); + // output + scalar_t r[ILP]; + LoadT* result = reinterpret_cast(&r); + + for (; offset * ILP < (classes - last); offset += blockDim.x) { + *value = reinterpret_cast(logits)[offset]; + +#pragma unroll + for (int j = 0; j < ILP; ++j) { + r[j] = tmpGradOutput * (std::exp( + static_cast(v[j]) - coeff) - + static_cast(((ILP * offset + j - shift) == label) ? 1 : 0) * + smooth_positives - smooth_negatives); + } + reinterpret_cast(gradInput)[offset] = *result; + } + + offset = classes - last + threadIdx.x; + for (; offset < classes; offset += blockDim.x) + gradInput[offset] = tmpGradOutput * (std::exp( + static_cast(logits[offset]) - coeff) - + static_cast(((offset - shift) == label) ? 1 : 0) * + smooth_positives - smooth_negatives); + +} + +template class Epilogue> +__global__ void +cunn_SoftMaxXEntropyBackward( + scalar_t *gradInput, + scalar_t *logits, + outscalar_t *max_log_sum_exp, + outscalar_t *gradOutput, + int64_t *labels, + const float smoothing, + int classes) +{ + gradInput += blockIdx.x * classes; + logits += blockIdx.x * classes; + + // Do vectorized load/store when input/output have same alignment + const int shift = ((uint64_t)logits) % ALIGN_BYTES / sizeof(scalar_t); + const int shift_ = ((uint64_t)gradInput) % ALIGN_BYTES / sizeof(scalar_t); + if (shift == shift_){ + aligned_apply(shift, gradInput, logits, max_log_sum_exp, gradOutput, labels, smoothing, classes); + } + else { + apply(gradInput, logits, max_log_sum_exp, gradOutput, labels, smoothing, classes); + } + +} + +template class Epilogue> +std::vector host_softmax_xentropy( + const Tensor & input_, + const Tensor & labels_, + const float smoothing, + const bool half_to_float){ + if (half_to_float) TORCH_CHECK(input_.scalar_type() == ScalarType::Half,"conversion is supported for Half type only"); + TORCH_CHECK(labels_.scalar_type() == ScalarType::Long,"Label type should be CUDA Long"); + + auto input = input_.contiguous(); + Tensor max_log_sum_exp = at::empty_like(labels_, half_to_float ? input.options().dtype(ScalarType::Float) : input.options()); + Tensor losses = at::empty_like(labels_, input_.options().dtype(ScalarType::Float)); + + static_assert(std::is_same, float>::value || + std::is_same, double>::value, + "accscalar_t for half should be float or double"); + TORCH_CHECK(input.dim() == 2, "Currently only 2 dim input supported"); + TORCH_CHECK(labels_.dim() == 1, "Labels should be 1 dimensional"); + TORCH_CHECK(input.size(0) == labels_.size(0), "Input and label should have same number of examples"); + TORCH_CHECK(input.numel() > 0, "Number of classes in input should not be 0"); + + const int64_t dim = 1; + int64_t outer_size = 1; + int64_t dim_size = input.size(dim); + int64_t inner_size = 1; + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + for (int64_t i = 0; i < dim; ++i) + outer_size *= input.size(i); + for (int64_t i = dim + 1; i < input.dim(); ++i) + inner_size *= input.size(i); + // This kernel spawns a block per each element in the batch. + // XXX: it assumes that inner_size == 1 + TORCH_CHECK(inner_size == 1, "Currently only inner size 1 supported"); + + dim3 grid(outer_size); + + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "host_softmax_xentropy", + using accscalar_t = at::acc_type; + const int ILP = sizeof(float4)/sizeof(scalar_t_0); + dim3 block = SoftMax_getBlockSize(ILP, dim_size); + if (!half_to_float) { + cunn_SoftMaxXEntropyForward + <<>>( + losses.DATA_PTR(), max_log_sum_exp.DATA_PTR(), + input.DATA_PTR(), labels_.DATA_PTR(), + dim_size, smoothing + ); + } else { + cunn_SoftMaxXEntropyForward + <<>>( + losses.DATA_PTR(), max_log_sum_exp.DATA_PTR(), + input.DATA_PTR(), labels_.DATA_PTR(), + dim_size, smoothing + ); + } + ); + + C10_CUDA_CHECK(cudaGetLastError()); + + std::vector ret = {losses, max_log_sum_exp}; + return ret; +} + +template class Epilogue> +Tensor host_softmax_xentropy_backward( + const at::Tensor &grad_loss, + const at::Tensor &logits_, + const at::Tensor &max_log_sum_exp, + const at::Tensor &labels, + const float smoothing, + bool half_to_float) { + const int64_t dim = 1; + Tensor gI = at::empty_like(logits_); + if (grad_loss.numel() == 0) { + return gI; + } + + auto grad = grad_loss.contiguous(); + auto logits = logits_.contiguous(); + + static_assert(std::is_same, float>::value || + std::is_same, double>::value, + "accscalar_t for half should be float or double"); + if (grad.dim() == 0) grad = grad.view(1); + + TORCH_CHECK(logits_.dim() == 2, "Currently only 2 dim input supported"); + TORCH_CHECK(labels.dim() == 1, "Labels should be 1 dimensional"); + TORCH_CHECK(logits_.numel() > 0, "Number of classes in input should not be 0"); + TORCH_CHECK(logits_.size(0) == labels.size(0), "Input and label should have same number of examples"); + TORCH_CHECK(labels.size(0) == grad.size(0), "Label and loss should have same number of examples"); + + int64_t outer_size = 1; + int64_t dim_size = logits.size(dim); + int64_t inner_size = 1; + for (int64_t i = 0; i < dim; ++i) + outer_size *= logits.size(i); + for (int64_t i = dim + 1; i < logits.dim(); ++i) + inner_size *= logits.size(i); + // See descriptions of kernels above. + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + TORCH_CHECK(inner_size == 1, "Currently only inner size 1 supported"); + + dim3 grid(outer_size); + + DISPATCH_FLOAT_AND_HALF(gI.scalar_type(), 0, "host_softmax_xentropy_backward", + using accscalar_t = acc_type; + const int ILP = sizeof(float4)/sizeof(scalar_t_0); + dim3 block = SoftMax_getBlockSize(ILP, dim_size); + if (!half_to_float) { + cunn_SoftMaxXEntropyBackward + <<>>( + gI.DATA_PTR(), logits.DATA_PTR(), + max_log_sum_exp.DATA_PTR(), + grad.DATA_PTR(), labels.DATA_PTR(), + smoothing, dim_size + ); + } else { + cunn_SoftMaxXEntropyBackward + <<>>( + gI.DATA_PTR(), logits.DATA_PTR(), + max_log_sum_exp.DATA_PTR(), + grad.DATA_PTR(), labels.DATA_PTR(), + smoothing, dim_size + ); + } + ); + + C10_CUDA_CHECK(cudaGetLastError()); + return gI; +} + +std::vector softmax_xentropy_cuda(const Tensor &input, const Tensor &labels, const float smoothing, const bool half_to_float){ + return host_softmax_xentropy(input, labels, smoothing, half_to_float); +} + +at::Tensor softmax_xentropy_backward_cuda( + const at::Tensor &grad_loss, + const at::Tensor &logits, + const at::Tensor &max_log_sum_exp, + const at::Tensor &labels, + const float smoothing) { + bool half_to_float = grad_loss.scalar_type() != logits.scalar_type(); + if (half_to_float) { + TORCH_CHECK((grad_loss.scalar_type() == ScalarType::Float && logits.scalar_type() == ScalarType::Half), "expected input and grad types to match, or input to be at::Half and grad to be at::Float"); + } + return host_softmax_xentropy_backward(grad_loss, logits, max_log_sum_exp, labels, smoothing, half_to_float); +} diff --git a/apex/apex/contrib/cudnn_gbn/__init__.py b/apex/apex/contrib/cudnn_gbn/__init__.py new file mode 100644 index 00000000..6dd46725 --- /dev/null +++ b/apex/apex/contrib/cudnn_gbn/__init__.py @@ -0,0 +1 @@ +from .batch_norm import GroupBatchNorm2d \ No newline at end of file diff --git a/apex/apex/contrib/cudnn_gbn/batch_norm.py b/apex/apex/contrib/cudnn_gbn/batch_norm.py new file mode 100644 index 00000000..d5f1fabd --- /dev/null +++ b/apex/apex/contrib/cudnn_gbn/batch_norm.py @@ -0,0 +1,144 @@ +import torch +from torch.nn.modules.batchnorm import _BatchNorm +from torch.nn import functional as F +from torch import Tensor +import peer_memory_cuda as pm +import cudnn_gbn_lib +from torch.cuda.amp import custom_fwd, custom_bwd + +class _GroupBatchNorm2d(torch.autograd.Function): + + @staticmethod + @custom_fwd + def forward(ctx, input, weight, bias, running_mean, running_variance, + minibatch_mean, minibatch_inv_var, momentum, eps, group_size, group_rank, fwd_buffers, bwd_buffers): + ctx.save_for_backward(input, weight, minibatch_mean, minibatch_inv_var) + ctx.eps = eps + ctx.bn_group = group_size + ctx.rank_id = group_rank + ctx.peer_buffers = bwd_buffers + return cudnn_gbn_lib.forward(input, weight, bias, running_mean, running_variance, + minibatch_mean, minibatch_inv_var, momentum, eps, group_size, group_rank, fwd_buffers) + + @staticmethod + @custom_bwd + def backward(ctx, grad_output): + x, scale, minibatch_mean, minibatch_inv_var = ctx.saved_variables + eps = ctx.eps + bn_group = ctx.bn_group + rank_id = ctx.rank_id + peer_buffers = ctx.peer_buffers + dx, dscale, dbias = cudnn_gbn_lib.backward(x, + grad_output, + scale, + minibatch_mean, + minibatch_inv_var, + eps, + bn_group, + rank_id, + peer_buffers) + return dx, dscale, dbias, None, None, None, None, None, None, None, None, None, None + + + +class GroupBatchNorm2d(_BatchNorm): + """ + synchronized batch normalization module extented from ``torch.nn.BatchNormNd`` + with the added stats reduction across multiple processes. + + When running in training mode, the layer reduces stats across process groups + to increase the effective batchsize for normalization layer. This is useful + in applications where batch size is small on a given process that would + diminish converged accuracy of the model. + + When running in evaluation mode, the layer falls back to + ``torch.nn.functional.batch_norm``. + + Args: + num_features: :math:`C` from an expected input of size + :math:`(N, C, L)` or :math:`L` from input of size :math:`(N, L)` + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Can be set to ``None`` for cumulative moving average + (i.e. simple average). Default: 0.1 + affine: a boolean value that when set to ``True``, this module has + learnable affine parameters. Default: ``True`` + track_running_stats: a boolean value that when set to ``True``, this + module tracks the running mean and variance, and when set to ``False``, + this module does not track such statistics and always uses batch + statistics in both training and eval modes. Default: ``True`` + + Example:: + + >>> sbn = apex.contrib.GroupBatchNorm2d(100).cuda() + >>> inp = torch.randn(10, 100, 14, 14).cuda() + >>> out = sbn(inp) + >>> inp = torch.randn(3, 100, 20).cuda() + >>> out = sbn(inp) + """ + + def __init__(self, num_features, group_size, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True): + + super(GroupBatchNorm2d, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine, track_running_stats=track_running_stats) + self.group_size = group_size + rank = torch.distributed.get_rank() + self.group_id = rank // group_size + self.group_rank = rank % group_size + self.fwd_peer_buffers = self.get_peer_buffers(num_features) + self.bwd_peer_buffers = self.get_peer_buffers(num_features) + self.minibatch_mean = torch.cuda.FloatTensor(num_features) + self.minibatch_inv_var = torch.cuda.FloatTensor(num_features) + + def get_peer_buffers(self, num_features): + # group_size * 2 (low-latency algo) * 2 (mean+var) * channels * 4 (float32) + peer_size = self.group_size * 4 * num_features * 4 + raw = pm.allocate_raw(peer_size) + # exchange peer pointers with nccl + world_size = torch.distributed.get_world_size() + raw_ipc = pm.get_raw_ipc_address(raw).cuda() + raw_ipcs = [torch.empty_like(raw_ipc) for _ in range(world_size)] + torch.distributed.all_gather(raw_ipcs, raw_ipc) + group_ipcs = [raw_ipcs[x] for x in range(self.group_id * self.group_size, (self.group_id * self.group_size) + self.group_size)] + peer_raw_ipcs = torch.stack(group_ipcs).cpu() + return pm.get_raw_peers(peer_raw_ipcs, self.group_rank, raw) + + def _check_input_dim(self, input): + if input.dim() != 4: + raise ValueError( + "expected 4D input (got {}D input)".format(input.dim()) + ) + + def _check_input_channels(self, input): + if input.size(1) % 8 != 0: + raise ValueError( + "GroupBatchNorm2d number of input channels should be a multiple of 8" + ) + + def forward(self, input : Tensor) -> Tensor: + # currently only GPU input is supported + if not input.is_cuda: + raise ValueError("GroupBatchNorm2d expected input tensor to be on GPU") + if not input.is_contiguous(memory_format=torch.channels_last): + raise ValueError("GroupBatchNorm2d expected input tensor to be in channels last memory format") + if torch.is_autocast_enabled(): + input = input.to(torch.get_autocast_gpu_dtype()) + if input.dtype != torch.float16: + raise ValueError("GroupBatchNorm2d expected input tensor in float16") + self._check_input_dim(input) + self._check_input_channels(input) + + if not self.training: + # fall back to pytorch implementation for inference + return F.batch_norm(input, self.running_mean, self.running_var, self.weight, self.bias, False, self.momentum, self.eps) + + return _GroupBatchNorm2d.apply(input, + self.weight, self.bias, + self.running_mean, self.running_var, + self.minibatch_mean, self.minibatch_inv_var, + self.momentum, + self.eps, + self.group_size, + self.group_rank, + self.fwd_peer_buffers, + self.bwd_peer_buffers) diff --git a/apex/apex/contrib/examples/multihead_attn/func_test_multihead_attn.py b/apex/apex/contrib/examples/multihead_attn/func_test_multihead_attn.py new file mode 100644 index 00000000..10407b99 --- /dev/null +++ b/apex/apex/contrib/examples/multihead_attn/func_test_multihead_attn.py @@ -0,0 +1,108 @@ +import torch +import torch.nn.functional as F +import argparse + +from apex.contrib.multihead_attn import SelfMultiheadAttn +from apex.contrib.multihead_attn import EncdecMultiheadAttn + +parser = argparse.ArgumentParser(description='Multihead Attention Standalone Test') +parser.add_argument('--seq-length', default=64, type=int, help='Sequence Length of Input') +parser.add_argument('--num-seqs-start', default=5, type=int, help='Start Range of Number of Sequences') +parser.add_argument('--num-seqs-stop', default=80, type=int, help='Stop Range of Number of Sequences') +parser.add_argument('--num-seqs-inc', default=5, type=int, help='Range Increment of Number of Sequences') +parser.add_argument('--trials', default=20, type=int, help='Number of Trials to Execute') +parser.add_argument('--warmup-trials', default=5, type=int, help='Warmup Trials to discard') +parser.add_argument('--layers', default=18, type=int, help='Attention Layers to Execute to Gain CPU/GPU Time Overlap') +parser.add_argument('--seed-start', default=1, type=int, help='Attention Layers to Execute to Gain CPU/GPU Time Overlap') +parser.add_argument('--seed-end', default=100, type=int, help='Attention Layers to Execute to Gain CPU/GPU Time Overlap') +parser.add_argument('--hidden-dim', default=1024, type=int, help='Multihead Attention hidden dimension') +parser.add_argument('--heads', default=16, type=int, help='Number of Multihead Attention heads') +parser.add_argument('--encdec-attn', action='store_true', help='Use Encoder-Decoder Attention instead of Self Attention.') +parser.add_argument('--norm-add', action='store_true', help='Include Layer Norm and Dropout-Add in Multihead Attention block.') +parser.add_argument('--ref', action='store_true', help='Reference implementation in python pytorch.') +parser.add_argument('--native', action='store_true', help='torch.nn.MultitheadAttention Version.') +parser.add_argument('--fwd', action='store_true', help='Only execute Fwd Pass.') +parser.add_argument('--eval', action='store_true', help='Inference only, no backward pass.') + +args = parser.parse_args() +assert args.seq_length % 64 == 0, "Sequence Length should be a multiple of 64!" + +if not torch.cuda.is_available(): + raise NotImplementedError('Running on CPU is not supported') +torch.cuda.set_device(0) + +dropout_prob = 0.1 + +for seed in range(args.seed_start, args.seed_end+1) : + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + ref_layer = None + if args.encdec_attn : + ref_layer = EncdecMultiheadAttn(args.hidden_dim, args.heads, dropout=dropout_prob, bias=False, include_norm_add=args.norm_add, impl='default') + else : + ref_layer = SelfMultiheadAttn(args.hidden_dim, args.heads, dropout=dropout_prob, bias=False, include_norm_add=args.norm_add, impl='default') + ref_layer.cuda() + ref_layer.half() + ref_layer.reset_parameters() + + ref_inputs = torch.randn(args.seq_length, args.num_seqs_start, args.hidden_dim, dtype=torch.float16, device=torch.device("cuda")).requires_grad_(True) + ref_inputs_kv = None + if args.encdec_attn : + ref_inputs_kv = torch.randn(args.seq_length, args.num_seqs_start, args.hidden_dim, dtype=torch.float16, device=torch.device("cuda")).requires_grad_(True) + + ref_grads = torch.randn_like(ref_inputs) + + ref_outputs,_ = ref_layer.forward(ref_inputs, + ref_inputs_kv, + ref_inputs_kv, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=(not args.eval)) + + ref_outputs.backward(ref_grads) + + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + tst_layer = None + if args.encdec_attn : + tst_layer = EncdecMultiheadAttn(args.hidden_dim, args.heads, dropout=dropout_prob, bias=False, include_norm_add=args.norm_add, impl='fast') + else: + tst_layer = SelfMultiheadAttn(args.hidden_dim, args.heads, dropout=dropout_prob, bias=False, include_norm_add=args.norm_add, impl='fast') + tst_layer.cuda() + tst_layer.half() + tst_layer.reset_parameters() + + tst_inputs = torch.randn(args.seq_length, args.num_seqs_start, args.hidden_dim, dtype=torch.float16, device=torch.device("cuda")).requires_grad_(True) + tst_inputs_kv = None + if args.encdec_attn : + tst_inputs_kv = torch.randn(args.seq_length, args.num_seqs_start, args.hidden_dim, dtype=torch.float16, device=torch.device("cuda")).requires_grad_(True) + + assert torch.equal(ref_inputs,tst_inputs), "ERROR: Inputs are different!" + + tst_grads = torch.randn_like(tst_inputs) + + tst_outputs,_ = tst_layer.forward(tst_inputs, + tst_inputs_kv, + tst_inputs_kv, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=(not args.eval)) + + tst_outputs.backward(tst_grads) + + fwd_close = torch.equal(ref_outputs, tst_outputs) + bwd_close = torch.equal(ref_inputs.grad, tst_inputs.grad) + + diff_fwd = ref_outputs - tst_outputs + diff_cnt_fwd = diff_fwd.ne(0.0).sum() + diff_accum_fwd = diff_fwd.abs().sum() + + diff_bwd = ref_inputs.grad - tst_inputs.grad + diff_cnt_bwd = diff_bwd.ne(0.0).sum() + diff_accum_bwd = diff_bwd.abs().sum() + + print(">>> Seed: ", seed, fwd_close, diff_cnt_fwd.item(), diff_accum_fwd.item(), bwd_close, diff_cnt_bwd.item(), diff_accum_bwd.item()) diff --git a/apex/apex/contrib/examples/multihead_attn/perf_test_multihead_attn.py b/apex/apex/contrib/examples/multihead_attn/perf_test_multihead_attn.py new file mode 100644 index 00000000..f81522ab --- /dev/null +++ b/apex/apex/contrib/examples/multihead_attn/perf_test_multihead_attn.py @@ -0,0 +1,115 @@ +import torch +import torch.nn.functional as F +import argparse + +from apex.contrib.multihead_attn import SelfMultiheadAttn +from apex.contrib.multihead_attn import EncdecMultiheadAttn + +parser = argparse.ArgumentParser(description='Multihead Attention Standalone Test') +parser.add_argument('--seq-length', default=64, type=int, help='Sequence Length of Input') +parser.add_argument('--num-seqs-start', default=10, type=int, help='Start Range of Number of Sequences') +parser.add_argument('--num-seqs-stop', default=120, type=int, help='Stop Range of Number of Sequences') +parser.add_argument('--num-seqs-inc', default=5, type=int, help='Range Increment of Number of Sequences') +parser.add_argument('--trials', default=20, type=int, help='Number of Trials to Execute') +parser.add_argument('--warmup-trials', default=5, type=int, help='Warmup Trials to discard') +parser.add_argument('--layers', default=18, type=int, help='Attention Layers to Execute to Gain CPU/GPU Time Overlap') +parser.add_argument('--hidden-dim', default=1024, type=int, help='Multihead Attention hidden dimension') +parser.add_argument('--heads', default=16, type=int, help='Number of Multihead Attention heads') +parser.add_argument('--encdec-attn', action='store_true', help='Use Encoder-Decoder Attention instead of Self Attention.') +parser.add_argument('--norm-add', action='store_true', help='Include Layer Norm and Dropout-Add in Multihead Attention block.') +parser.add_argument('--ref', action='store_true', help='Reference implementation in python pytorch.') +parser.add_argument('--native', action='store_true', help='torch.nn.MultitheadAttention Version.') +parser.add_argument('--fwd', action='store_true', help='Only execute Fwd Pass.') +parser.add_argument('--biases', action='store_true', help='Execute multihead attention with Linear Biases.') + +args = parser.parse_args() + +if not torch.cuda.is_available(): + raise NotImplementedError('Running on CPU is not supported') +torch.cuda.set_device(0) + +torch.manual_seed(111) +if torch.cuda.is_available(): + torch.cuda.manual_seed_all(111) + +attn_layers = [] +for idx in range(0, args.layers) : + if args.encdec_attn : + if args.ref : + attn_layers.append(EncdecMultiheadAttn(args.hidden_dim, args.heads, dropout=0.1, bias=args.biases, include_norm_add=False, impl='default')) + else : + attn_layers.append(EncdecMultiheadAttn(args.hidden_dim, args.heads, dropout=0.1, bias=args.biases, include_norm_add=args.norm_add, impl='fast')) + else : + if args.native : + attn_layers.append(torch.nn.MultiheadAttention(args.hidden_dim, args.heads, dropout=0.1, bias=args.biases)) + elif args.ref : + attn_layers.append(SelfMultiheadAttn(args.hidden_dim, args.heads, dropout=0.1, bias=args.biases, include_norm_add=args.norm_add, impl='default')) + else : + attn_layers.append(SelfMultiheadAttn(args.hidden_dim, args.heads, dropout=0.1, bias=args.biases, include_norm_add=args.norm_add, impl='fast')) + attn_layers[idx].cuda() + attn_layers[idx].half() + if not args.native : + attn_layers[idx].reset_parameters() + +start_evt_fwd = [] +start_evt_bwd = [] +stop_evt_bwd = [] +for recorded_trial in range(0, args.trials) : + start_evt_fwd.append(torch.cuda.Event(enable_timing=True)) + start_evt_bwd.append(torch.cuda.Event(enable_timing=True)) + stop_evt_bwd.append(torch.cuda.Event(enable_timing=True)) + +for sequences in range(args.num_seqs_start, args.num_seqs_stop + args.num_seqs_inc, args.num_seqs_inc) : + inputs = torch.randn(args.seq_length, sequences, args.hidden_dim, dtype=torch.float16, device=torch.device("cuda")).requires_grad_(True) + grads = torch.randn_like(inputs) + + for trial in range(0, args.trials + args.warmup_trials) : + layer_inputs = inputs + evt_idx = trial - args.warmup_trials + + if evt_idx >= 0 : + start_evt_fwd[evt_idx].record() + + for lyr_idx in range(0, args.layers) : + if args.native : + outputs,_ = attn_layers[lyr_idx].forward(layer_inputs, + layer_inputs, + layer_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=None) + else : + outputs,_ = attn_layers[lyr_idx].forward(layer_inputs, + layer_inputs, + layer_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True) + layer_inputs = outputs + + if evt_idx >= 0 : + start_evt_bwd[evt_idx].record() + + if not args.fwd : + layer_inputs.backward(grads) + + if evt_idx >= 0 : + stop_evt_bwd[evt_idx].record() + + torch.cuda.synchronize() + elapsed_time_fwd = 0.0 + elapsed_time_bwd = 0.0 + for evt_idx in range(0, args.trials) : + elapsed_time_fwd += start_evt_fwd[evt_idx].elapsed_time(start_evt_bwd[evt_idx]) + elapsed_time_bwd += start_evt_bwd[evt_idx].elapsed_time(stop_evt_bwd[evt_idx]) + + print("[ {} Attn {} ]Total Tokens: {:4d} Sequences: {:3d} Sequence Length: {:3d} Fwd Time / Layer: {:.3f} ms Bwd Time / Layer: {:.3f} ms".format( + 'Encdec' if args.encdec_attn else 'Self', \ + 'Norm&Add' if args.norm_add else '', \ + sequences*args.seq_length, \ + sequences, \ + args.seq_length, \ + elapsed_time_fwd / ( args.trials * args.layers ), \ + elapsed_time_bwd / ( args.trials * args.layers ))) + diff --git a/apex/apex/contrib/fmha/__init__.py b/apex/apex/contrib/fmha/__init__.py new file mode 100644 index 00000000..ec2e9c66 --- /dev/null +++ b/apex/apex/contrib/fmha/__init__.py @@ -0,0 +1 @@ +from .fmha import FMHAFun diff --git a/apex/apex/contrib/fmha/fmha.py b/apex/apex/contrib/fmha/fmha.py new file mode 100644 index 00000000..6aaca804 --- /dev/null +++ b/apex/apex/contrib/fmha/fmha.py @@ -0,0 +1,76 @@ +############################################################################### +# Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +############################################################################### + + +import torch +import torch.nn.functional as F +import fmhalib as mha + +class FMHAFun(torch.autograd.Function): + @staticmethod + def forward(ctx, qkv, cu_seqlens, p_dropout, max_s, is_training, zero_tensors): + batch_size = cu_seqlens.numel() - 1 + if batch_size < 4: + max_s = 512 + context, S_dmask = mha.fwd_nl(qkv, cu_seqlens, p_dropout, max_s, is_training, True, zero_tensors, None) + else: + context, S_dmask = mha.fwd(qkv, cu_seqlens, p_dropout, max_s, is_training, False, zero_tensors, None) + ctx.save_for_backward(qkv, S_dmask) + ctx.cu_seqlens = cu_seqlens + ctx.p_dropout = p_dropout + ctx.max_s = max_s + ctx.zero_tensors = zero_tensors + return context + + @staticmethod + def backward(ctx, dout): + qkv, S_dmask = ctx.saved_tensors + batch_size = ctx.cu_seqlens.numel() - 1 + if batch_size < 4: + dqkv, dp, _ = mha.bwd_nl(dout, qkv, S_dmask, ctx.cu_seqlens, ctx.p_dropout, ctx.max_s, ctx.zero_tensors) + else: + dqkv, dp = mha.bwd(dout, qkv, S_dmask, ctx.cu_seqlens, ctx.p_dropout, ctx.max_s, ctx.zero_tensors) + + return dqkv, None, None, None, None, None + +class FMHA(torch.nn.Module): + + def __init__(self, config): + + super(FMHA, self).__init__() + + self.p_dropout = config.attention_probs_dropout_prob + self.h = config.num_attention_heads + self.hidden_size = config.hidden_size + self.d = self.hidden_size // self.h + assert self.d * self.h == self.hidden_size, "Invalid hidden size/num_heads" + + def forward(self, qkv, cu_seqlens, max_s, is_training=True, zero_tensors=False): + + ctx = FMHAFun.apply(qkv.view(-1, 3, self.h, self.d), cu_seqlens, self.p_dropout, max_s, is_training, zero_tensors) + + return ctx.view(-1, self.hidden_size) diff --git a/apex/apex/contrib/focal_loss/__init__.py b/apex/apex/contrib/focal_loss/__init__.py new file mode 100644 index 00000000..0589eefb --- /dev/null +++ b/apex/apex/contrib/focal_loss/__init__.py @@ -0,0 +1,9 @@ +try: + import torch + import focal_loss_cuda + from .focal_loss import focal_loss + del torch + del focal_loss_cuda + del focal_loss +except ImportError as err: + print("apex was installed without --focal_loss flag, apex.contrib.focal_loss is not available") diff --git a/apex/apex/contrib/focal_loss/focal_loss.py b/apex/apex/contrib/focal_loss/focal_loss.py new file mode 100644 index 00000000..85c6f620 --- /dev/null +++ b/apex/apex/contrib/focal_loss/focal_loss.py @@ -0,0 +1,60 @@ +import torch + +import focal_loss_cuda + + +class FocalLoss(torch.autograd.Function): + @staticmethod + def forward( + ctx, + cls_output, + cls_targets_at_level, + num_positives_sum, + num_real_classes, + alpha, + gamma, + label_smoothing=0.0, + ): + loss, partial_grad = focal_loss_cuda.forward( + cls_output, + cls_targets_at_level, + num_positives_sum, + num_real_classes, + alpha, + gamma, + label_smoothing, + ) + + ctx.save_for_backward(partial_grad, num_positives_sum) + return loss + + @staticmethod + def backward(ctx, grad_loss): + partial_grad, num_positives_sum = ctx.saved_tensors + + # The backward kernel is actually in-place to save memory space, + # partial_grad and grad_input are the same tensor. + grad_input = focal_loss_cuda.backward(grad_loss, partial_grad, num_positives_sum) + + return grad_input, None, None, None, None, None, None + + +def focal_loss( + cls_output: torch.Tensor, + cls_targets_at_level: torch.Tensor, + num_positive_sum: torch.Tensor, + num_real_classes: int, + alpha: float, + gamma: float, + label_smoothing: float = 0.0, +) -> torch.Tensor: + """Fused focal loss function.""" + return FocalLoss.apply( + cls_output, + cls_targets_at_level, + num_positive_sum, + num_real_classes, + alpha, + gamma, + label_smoothing, + ) diff --git a/apex/apex/contrib/groupbn/__init__.py b/apex/apex/contrib/groupbn/__init__.py new file mode 100644 index 00000000..2f857706 --- /dev/null +++ b/apex/apex/contrib/groupbn/__init__.py @@ -0,0 +1,9 @@ +try: + import torch + import bnp + from .batch_norm import BatchNorm2d_NHWC + del torch + del bnp + del batch_norm +except ImportError as err: + print("apex was installed without --bnp flag, contrib.groupbn is not available") diff --git a/apex/apex/contrib/groupbn/batch_norm.py b/apex/apex/contrib/groupbn/batch_norm.py new file mode 100644 index 00000000..17ef196b --- /dev/null +++ b/apex/apex/contrib/groupbn/batch_norm.py @@ -0,0 +1,225 @@ +import torch +import numpy as np +from torch.nn.modules.batchnorm import _BatchNorm + +import bnp + +class bn_NHWC_impl(torch.autograd.Function): + @staticmethod + def forward(ctx, x, s, b, rm, riv, mini_m, mini_riv, ret_cta, mom, epsilon, fuse_relu, is_train, bn_group, my_data, pair_data, magic, pair_data2, pair_data3, fwd_occup, fwd_grid_x, bwd_occup, bwd_grid_x, multi_stream): + if is_train: + ctx.save_for_backward(x, s, b, rm, riv, mini_m, mini_riv) + ctx.epsilon = epsilon + ctx.momentum = mom + ctx.ret_cta = ret_cta + ctx.fuse_relu = fuse_relu + ctx.my_data = my_data + ctx.pair_data = pair_data + ctx.magic = magic + ctx.pair_data2 = pair_data2 + ctx.pair_data3 = pair_data3 + ctx.bn_group = bn_group + ctx.bwd_occup = bwd_occup + ctx.bwd_grid_x = bwd_grid_x + ctx.multi_stream = multi_stream + + res = bnp.bn_fwd_nhwc(x, s, b, rm, riv, mini_m, mini_riv, ret_cta, mom, epsilon, fuse_relu, my_data, pair_data, pair_data2, pair_data3, bn_group, magic, fwd_occup, fwd_grid_x, multi_stream) + return res + else: + return bnp.bn_fwd_eval_nhwc(x, s, b, rm, riv, ret_cta, bn_group, mom, epsilon, fuse_relu) + + @staticmethod + def backward(ctx, grad_y): + x, s, b, rm, riv, mini_m, mini_riv = ctx.saved_variables + epsilon = ctx.epsilon + mom = ctx.momentum + ret_cta = ctx.ret_cta + fuse_relu = ctx.fuse_relu + my_data = ctx.my_data + pair_data = ctx.pair_data + magic = ctx.magic + pair_data2 = ctx.pair_data2 + pair_data3 = ctx.pair_data3 + bn_group = ctx.bn_group + bwd_occup = ctx.bwd_occup + bwd_grid_x = ctx.bwd_grid_x + multi_stream = ctx.multi_stream + + dx, dscale, dbias = bnp.bn_bwd_nhwc(x, grad_y, s, b, rm, riv, mini_m, mini_riv, ret_cta, mom, epsilon, fuse_relu, my_data, pair_data, pair_data2, pair_data3, bn_group, magic, bwd_occup, bwd_grid_x, multi_stream) + + return dx, dscale, dbias, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None + + +class bn_addrelu_NHWC_impl(torch.autograd.Function): + @staticmethod + def forward(ctx, x, z, s, b, rm, riv, mini_m, mini_riv, grid_dim_y, ret_cta, mom, epsilon, is_train, bn_group, my_data, pair_data, magic, pair_data2, pair_data3, fwd_occup, fwd_grid_x, bwd_occup, bwd_grid_x, multi_stream): + if is_train: + bitmask = torch.cuda.IntTensor(((x.numel()+31)//32) * 2 * grid_dim_y) + ctx.save_for_backward(x, s, b, rm, riv, mini_m, mini_riv, bitmask) + ctx.epsilon = epsilon + ctx.momentum = mom + ctx.ret_cta = ret_cta + ctx.my_data = my_data + ctx.pair_data = pair_data + ctx.magic = magic + ctx.pair_data2 = pair_data2 + ctx.pair_data3 = pair_data3 + ctx.bn_group = bn_group + ctx.bwd_occup = bwd_occup + ctx.bwd_grid_x = bwd_grid_x + ctx.multi_stream = multi_stream + + res = bnp.bn_addrelu_fwd_nhwc(x, z, s, b, rm, riv, mini_m, mini_riv, bitmask, ret_cta, mom, epsilon, my_data, pair_data, pair_data2, pair_data3, bn_group, magic, fwd_occup, fwd_grid_x, multi_stream) + return res + else: + return bnp.bn_addrelu_fwd_eval_nhwc(x, z, s, b, rm, riv, ret_cta, bn_group, mom, epsilon) + + @staticmethod + def backward(ctx, grad_y): + x, s, b, rm, riv, mini_m, mini_riv, bitmask = ctx.saved_variables + epsilon = ctx.epsilon + mom = ctx.momentum + ret_cta = ctx.ret_cta + my_data = ctx.my_data + pair_data = ctx.pair_data + magic = ctx.magic + pair_data2 = ctx.pair_data2 + pair_data3 = ctx.pair_data3 + bn_group = ctx.bn_group + bwd_occup = ctx.bwd_occup + bwd_grid_x = ctx.bwd_grid_x + multi_stream = ctx.multi_stream + + dx, dz, dscale, dbias = bnp.bn_addrelu_bwd_nhwc(x, grad_y, s, b, rm, riv, mini_m, mini_riv, bitmask, ret_cta, mom, epsilon, my_data, pair_data, pair_data2, pair_data3, bn_group, magic, bwd_occup, bwd_grid_x, multi_stream) + + return dx, dz, dscale, dbias, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None + + + + + +class BatchNorm2d_NHWC(_BatchNorm): + # if using BatchNorm2d_NHWC simultaneously with multiple streams set multi_stream to True + def __init__(self, num_features, fuse_relu=False, bn_group=1, max_cta_per_sm=2, cta_launch_margin=12, multi_stream=False): + super(BatchNorm2d_NHWC, self).__init__(num_features) + + self.fuse_relu = fuse_relu + self.multi_stream = multi_stream + + self.minibatch_mean = torch.cuda.FloatTensor(num_features) + self.minibatch_riv = torch.cuda.FloatTensor(num_features) + + #defaut to distributed bn disabled + self.bn_group = bn_group + self.max_cta_per_sm = max_cta_per_sm #used only in training fwd and bwd + self.cta_launch_margin = cta_launch_margin #used only in training fwd and bwd + self.my_data = None + self.pair_data = None + self.pair_data2 = None + self.pair_data3 = None + self.local_rank = 0 + self.magic = torch.IntTensor([0]) + + #calculate cta per sm occupancies + assert(max_cta_per_sm>0) # won't be able to do much with 0 CTAs :) + self.fwd_occupancy = min(bnp.bn_fwd_nhwc_occupancy(), max_cta_per_sm) + self.bwd_occupancy = min(bnp.bn_bwd_nhwc_occupancy(), max_cta_per_sm) + self.addrelu_fwd_occupancy = min(bnp.bn_addrelu_fwd_nhwc_occupancy(), max_cta_per_sm) + self.addrelu_bwd_occupancy = min(bnp.bn_addrelu_bwd_nhwc_occupancy(), max_cta_per_sm) + + #calculate grid dimentions based on occupancy numbers + mp_count = torch.cuda.get_device_properties(None).multi_processor_count + self.fwd_grid_dim_x = max(mp_count*self.fwd_occupancy - cta_launch_margin , 1) + self.bwd_grid_dim_x = max(mp_count*self.bwd_occupancy - cta_launch_margin , 1) + self.addrelu_fwd_grid_dim_x = max(mp_count*self.addrelu_fwd_occupancy - cta_launch_margin , 1) + self.addrelu_bwd_grid_dim_x = max(mp_count*self.addrelu_bwd_occupancy - cta_launch_margin , 1) + self.grid_dim_y = (num_features + 63) // 64 + + # allocate scratch space used by implementation + # TODO: scratch space that is not supposed to be exposed at user code. We only need one time initialization, the + # same buffer could be reused in future iterations. Currently we exposed it here instead of requesting new + # buffer from cache allocator to avoid unnecessary initialization at future iterations. + self.ret_cta = torch.cuda.ByteTensor(8192).fill_(0) + + #FIXME: turn pair handles into an array + if bn_group>1: + local_rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + assert(world_size >= bn_group) + assert(world_size % bn_group == 0) + + bn_sync_steps = 1 + if (bn_group==4): + bn_sync_steps = 2 + if (bn_group==8): + bn_sync_steps = 3 + + self.ipc_buffer = torch.cuda.ByteTensor(bnp.get_buffer_size(bn_sync_steps)) + self.my_data = bnp.get_data_ptr(self.ipc_buffer) + # we are walking on very thin ice here by utilizing internal `_share_cuda_()` + self.storage = self.ipc_buffer.storage() + self.share_cuda = self.storage._share_cuda_() + internal_cuda_mem = self.share_cuda + # internal_cuda_mem[1]: ipc_mem_handle + my_handle = torch.cuda.ByteTensor(np.frombuffer(internal_cuda_mem[1], dtype=np.uint8)) + # internal_cuda_mem[3]: offset + my_offset = torch.cuda.IntTensor([internal_cuda_mem[3]]) + + handles_all = torch.empty(world_size, my_handle.size(0), dtype=my_handle.dtype, device=my_handle.device) + handles_l = list(handles_all.unbind(0)) + torch.distributed.all_gather(handles_l, my_handle) + + offsets_all = torch.empty(world_size, my_offset.size(0), dtype=my_offset.dtype, device=my_offset.device) + offsets_l = list(offsets_all.unbind(0)) + torch.distributed.all_gather(offsets_l, my_offset) + + #whom do I actually care about? that would be local_rank XOR 1 + self.pair_handle = handles_l[local_rank ^ 1].cpu().contiguous() + pair_offset = offsets_l[local_rank ^ 1].cpu() + self.pair_data = bnp.get_remote_data_ptr(self.pair_handle, pair_offset) + + if bn_group>2: + self.pair_handle2 = handles_l[local_rank ^ 2].cpu().contiguous() + pair_offset2 = offsets_l[local_rank ^ 2].cpu() + self.pair_data2 = bnp.get_remote_data_ptr(self.pair_handle2, pair_offset2) + + if bn_group>4: + self.pair_handle3 = handles_l[local_rank ^ 4].cpu().contiguous() + pair_offset3 = offsets_l[local_rank ^ 4].cpu() + self.pair_data3 = bnp.get_remote_data_ptr(self.pair_handle3, pair_offset3) + + #FIXME: get magic value into C code and eliminate from here + self.magic = torch.IntTensor([2]) + self.local_rank = local_rank + + + def forward(self, x, z=None): + if z is not None: + assert(self.fuse_relu==True) + return bn_addrelu_NHWC_impl.apply(x, z, + self.weight, self.bias, + self.running_mean, self.running_var, + self.minibatch_mean, self.minibatch_riv, self.grid_dim_y, self.ret_cta, + self.momentum, + self.eps, self.training, self.bn_group, self.my_data, self.pair_data, (self.magic), self.pair_data2, self.pair_data3, + self.addrelu_fwd_occupancy, self.addrelu_fwd_grid_dim_x, + self.addrelu_bwd_occupancy, self.addrelu_bwd_grid_dim_x, + self.multi_stream) + else: + return bn_NHWC_impl.apply(x, + self.weight, self.bias, + self.running_mean, self.running_var, + self.minibatch_mean, self.minibatch_riv, self.ret_cta, + self.momentum, + self.eps, self.fuse_relu, self.training, self.bn_group, self.my_data, self.pair_data, (self.magic), self.pair_data2, self.pair_data3, + self.fwd_occupancy, self.fwd_grid_dim_x, + self.bwd_occupancy, self.bwd_grid_dim_x, + self.multi_stream) + + def __del__(self): + if self.bn_group>1: + bnp.close_remote_data(self.pair_handle) + if self.bn_group>2: + bnp.close_remote_data(self.pair_handle2) + if self.bn_group>4: + bnp.close_remote_data(self.pair_handle3) diff --git a/apex/apex/contrib/index_mul_2d/__init__.py b/apex/apex/contrib/index_mul_2d/__init__.py new file mode 100644 index 00000000..edb63d39 --- /dev/null +++ b/apex/apex/contrib/index_mul_2d/__init__.py @@ -0,0 +1 @@ +from .index_mul_2d import index_mul_2d diff --git a/apex/apex/contrib/index_mul_2d/index_mul_2d.py b/apex/apex/contrib/index_mul_2d/index_mul_2d.py new file mode 100644 index 00000000..1d34fe20 --- /dev/null +++ b/apex/apex/contrib/index_mul_2d/index_mul_2d.py @@ -0,0 +1,144 @@ +import torch + +import fused_index_mul_2d + +class IndexMul2d_(torch.autograd.Function): + ''' + Currently only support index in dimension 0 with a 2-dimension tensor. + The shape of indexed in1 must be same with in2. Now this kernel does not support broadcast. + The datatype must be float32 or float16. + ''' + @staticmethod + def forward(ctx, in1: torch.Tensor, in2: torch.Tensor, idx1: torch.Tensor) -> torch.Tensor: + assert in2.size(0) == idx1.size(0) + if ((in1.dtype != torch.float32 and in1.dtype != torch.half) or in2.dtype != in1.dtype): + raise RuntimeError("input1'dtype and input2's dtype must be fp32 or fp16. And input type must be same") + if (in1.dim() != 2 or in2.dim() != 2): + raise RuntimeError("in1 and in2 must be 2-dimension tensor.") + if (idx1.dim() != 1): + raise RuntimeError("idx1 must be 1-dimension tensor.") + + if not in1.is_contiguous(): + in1 = in1.contiguous() + if not in2.is_contiguous(): + in2 = in2.contiguous() + if not idx1.is_contiguous(): + idx1 = idx1.contiguous() + + assert in1.is_contiguous() + assert in2.is_contiguous() + assert idx1.is_contiguous() + + out = torch.empty_like(in2) + + if (in1.dtype == torch.float32): + fused_index_mul_2d.float_forward( + out, + in1, + in2, + idx1) + elif (in1.dtype == torch.half): + fused_index_mul_2d.half_forward( + out, + in1, + in2, + idx1) + + ctx.for_backwards = (in1, in2, idx1) + return out + + @staticmethod + def backward(ctx, grad_out): + + in1, in2, idx1 = ctx.for_backwards + + grad_in1, grad_in2 = index_mul_2d_backward(in1, in2, idx1, grad_out) + + return grad_in1, grad_in2, None + + +class IndexMul2dBackward_(torch.autograd.Function): + @staticmethod + def forward(ctx, in1: torch.Tensor, in2: torch.Tensor, idx1: torch.Tensor, + grad_out: torch.Tensor) -> torch.Tensor: + if not in1.is_contiguous(): + in1 = in1.contiguous() + if not in2.is_contiguous(): + in2 = in2.contiguous() + if not idx1.is_contiguous(): + idx1 = idx1.contiguous() + if not grad_out.is_contiguous(): + grad_out = grad_out.contiguous() + + assert in1.is_contiguous() + assert in2.is_contiguous() + assert idx1.is_contiguous() + assert grad_out.is_contiguous() + + grad_in1 = torch.zeros_like(in1) + grad_in2 = torch.empty_like(in2) + + if (in1.dtype == torch.float32): + fused_index_mul_2d.float_backward( + grad_in1, + grad_in2, + grad_out, + in1, + in2, + idx1) + elif (in1.dtype == torch.half): + fused_index_mul_2d.half_backward( + grad_in1, + grad_in2, + grad_out, + in1, + in2, + idx1) + + ctx.for_backwards = (in1, in2, idx1, grad_out) + return grad_in1, grad_in2 + + @staticmethod + def backward(ctx, grad_grad_in1, grad_grad_in2): + if not grad_grad_in1.is_contiguous(): + grad_grad_in1 = grad_grad_in1.contiguous() + if not grad_grad_in2.is_contiguous(): + grad_grad_in2 = grad_grad_in2.contiguous() + + assert grad_grad_in1.is_contiguous() + assert grad_grad_in2.is_contiguous() + + in1, in2, idx1, grad_out = ctx.for_backwards + + grad_in1 = torch.zeros_like(in1) + grad_in2 = torch.empty_like(in2) + grad_grad_out = torch.empty_like(grad_out) + + if (in1.dtype == torch.float32): + fused_index_mul_2d.float_backward_backward( + grad_grad_out, + grad_in1, + grad_in2, + grad_out, + grad_grad_in1, + grad_grad_in2, + in1, + in2, + idx1) + elif (in1.dtype == torch.half): + fused_index_mul_2d.half_backward_backward( + grad_grad_out, + grad_in1, + grad_in2, + grad_out, + grad_grad_in1, + grad_grad_in2, + in1, + in2, + idx1) + + return grad_in1, grad_in2, None, grad_grad_out + +index_mul_2d = IndexMul2d_.apply +index_mul_2d_backward = IndexMul2dBackward_.apply + diff --git a/apex/apex/contrib/layer_norm/__init__.py b/apex/apex/contrib/layer_norm/__init__.py new file mode 100644 index 00000000..4bbc4763 --- /dev/null +++ b/apex/apex/contrib/layer_norm/__init__.py @@ -0,0 +1 @@ +from .layer_norm import FastLayerNorm diff --git a/apex/apex/contrib/layer_norm/layer_norm.py b/apex/apex/contrib/layer_norm/layer_norm.py new file mode 100644 index 00000000..b084b1ac --- /dev/null +++ b/apex/apex/contrib/layer_norm/layer_norm.py @@ -0,0 +1,53 @@ +import torch +from torch.nn import init + +from apex._autocast_utils import _cast_if_autocast_enabled +import fast_layer_norm + + +class FastLayerNormFN(torch.autograd.Function): + @staticmethod + def forward(ctx, x, gamma, beta, epsilon): + x = x.contiguous() + gamma = gamma.contiguous() + beta = beta.contiguous() + hidden_size = gamma.numel() + xmat = x.view((-1, hidden_size)) + ymat, mu, rsigma = fast_layer_norm.ln_fwd(xmat, gamma, beta, epsilon) + ctx.save_for_backward(x, gamma, mu, rsigma) + return ymat.view(x.shape) + + @staticmethod + def backward(ctx, dy): + # assert dy.is_contiguous() + dy = dy.contiguous() # this happens! + x, gamma, mu, rsigma = ctx.saved_tensors + + hidden_size = gamma.numel() + xmat = x.view((-1, hidden_size)) + dymat = dy.view(xmat.shape) + dxmat, dgamma, dbeta, _, _ = fast_layer_norm.ln_bwd(dymat, xmat, mu, rsigma, gamma) + dx = dxmat.view(x.shape) + return dx, dgamma, dbeta, None + + +def _fast_layer_norm(x, weight, bias, epsilon): + args = _cast_if_autocast_enabled(x, weight, bias, epsilon) + with torch.cuda.amp.autocast(enabled=False): + return FastLayerNormFN.apply(*args) + + +class FastLayerNorm(torch.nn.Module): + def __init__(self, hidden_size, eps=1e-5): + super().__init__() + self.epsilon = eps + self.weight = torch.nn.Parameter(torch.empty(hidden_size)) + self.bias = torch.nn.Parameter(torch.empty(hidden_size)) + self.reset_parameters() + + def reset_parameters(self): + init.ones_(self.weight) + init.zeros_(self.bias) + + def forward(self, x): + return _fast_layer_norm(x, self.weight, self.bias, self.epsilon) diff --git a/apex/apex/contrib/multihead_attn/MHA_bwd.png b/apex/apex/contrib/multihead_attn/MHA_bwd.png new file mode 100644 index 0000000000000000000000000000000000000000..7069973e0a88d6f459a88dc85c0d6d92f4e85d73 GIT binary patch literal 86630 zcmeFZWmweR_dW^;iYU^cw1FT9sC3RCB_$0K11dd~bc3LRfP~Ur(lAKF451()NJ%$H zGjum+&-4BLzmMQKZ_ahjyPwyE3^054{_MTtUiVsS`zt+_CM2LHz{0{JlzAedf`x?( z!NR(53;!bcjoDq(ci@PA%k zxP=+UFH52TzFmIxM9U5fi;5KUg>91b3oILoB_koG=8U~GdMTPoJ?3=1+)lAwV6OS< zBST_eNkd6urfAr0^8pXte!Lm$K6Q;}5?T!=6n0Pv&1V`CVsEi8N&Wtcdv|Vx%+=M^ zSa7%2(}Z84ZJB1#aeUxdq=U;&JY6*@$L+9BXRa#qyWTzXzP!qt&Vqxrvn1HK#89j= z|IJ^`m@MzlD5zbLlWj_}2f;yVyGi229f>t1qe1pIOa6i+getx^^3n zIE>`nQlZHec=(>=yX$E?e=RWd@#_g`0>-<4EeUFKh4_eNWlDPaua9GVhxILl94aq$ zc1NIC-oGgs_moGPLWchSI09@RhI_B0Kb+kk@Q4lU(6zLR5Qb9ezdnu}2N$A(_3*}D zj~bu@R&B!ofrtF{arbcXJwIc;e01g*FgrgINVcq`|IQW|2m*~9vnhK!oHrK@Y3V9or7Wx z8;uW8B9aNN0r6{hP~QLS5!9OioM-0+@ls;wgu3_XVeM%#k^97jos-?#@tu{?!^OAU zNxl93aVHIt^7#V4PaABvv8#`fEthRSQU^WD``nymS{Dto4!gyn<=x<%iNzG&H= zi?XW}lFoJFY_04@g+SawpbrVcsSMJyxp43e^^)`R+dQ{(^#{E#T*~|W`Fmq9ow=s2 z%{z4s4XL5%m7$Xrr`qDhR1#Ca-ls3|g zpwaiPFbhQySTbz8?9Xs|nVrP1ZL_hl71wwi4prDpEydfl7h+$yxF_S{vSgO@ZH4i` zfMRA7^`idbVLii1=+11r(5kRbiRnj*?#lR;r;8}_?Zw_Qmrdj0BBQ|4va+Gjo8lHb zOM@A!T*pCa_ON8;Bq5mQ3mIHu7JDE#?>ygTG2jT04wRUMiL8CSz-*~Nbq6j+oIYaL zA+pm1IVm*gGir?-G>U>5Bb!?syNHJC{fJNu#kZwCyhl}*n)f4jDWHR3wKCsADz0p8 z@Hs6F*!+@=;JDKcIdx~_R6H`d-C{#*-m|}9gP+=yAKlIb+Y{O96qmW@b&S>@=IPfp zuy(Ge`PAO@y^gH2GHZ3nzO^$&9JZDg-9D3bm<6rDS>GLVD_A%b)pcAQTgJH+d*bA^ z>x7VeVRkPlxn4>+Y4^Jw^I{Db15ls>=x_bP29#nCEjBKaF&Xu;aHdO|!GTILTic6P z$nht)wT1ujiMh=OSC#QV>_e6^}tZ>-1_+3r!b97Q^0FdTCk|9me*WL?9$ z)?a(F-Anb2YxJGxdIQxcUbhU{W6o6COKtQHHJv$jJKVCP0SmeWUKg7_+%O2ct6)p@ z77k0siV8-rPc<#iK~5`!=tYL~He>szf5s9z=tJ3;X4?~n%_PzN1*@h^RMyBy)q5Fs zmEBJpUBu@$^2^8IoV7=`O=_j^SYFH6D_=bNQ{;(GB%BrN3yNMtrLS^x za*8@UR_reCra$YLkr4W9H*iU`XK_uO;UE;|Jo%2f4Z*kva~hrTnXEj~aZr&AIFeKp zatb(lrR0H2{GJ7>Q~O@&r!Ul+6#w*@`}9X?m6d~fBFNJ04PJ|AR9LjFw{Tms?%f6Q zQr`vHU=sGQl@NGa!L$v$D8WiOH%guoCd{%M7=RQZq!%6#@jBU`Q^8?K;D`~e!y5^t`!P-dk0?QvsKEV2YC+OU`t?{-W zpYb`I9%9l?>vT!s*wtxeWP$W{W13beTwR*$<#vjO)q7{jedv}VmF~ghV6E5b(_#-` zyh>wZsi~k{%^W`hY~7FMyiPJv(HjsP_5Kd`SuVQ31&;FCu;aa{5R~Tx1qH><>B;!1 z-Kf5x@duaG6*?l(y<{_w0}_bmk+aa$2d<@@Tttz?aec)6w%5s=m&H_LP`liwnZR*{ zhRAQGaNSDVbdj9_0|b$)rDcBZyG6bib-3Ly9!oA|mzB!7r4Fx?;^9P}yRu`M#?96q zXdN$qM(2D zn`-$TmEHrqxxPHDDGtitbgfy3q3xBQ9a!gH+gh%9-6MOlNZlfk`6^ zdLJb#CW#^r13Bs(zR|k{JIx9A)|~E?*E)<}Ij}&-|9Yr&5-rDB&8_m-$fGGsCm6|{ z6;ba8IeM`0oL8nXbonFI^a_ zsCLpIGm;Xnl`J^VmDX_r6x^kJ-1GL}(b0jWgp2%6_4P4kkI1dTe$fRtWvX9`cOCX; zqJ|}@VAH(m?gFy0><|hM<^<1!g_T3jop@U=+|%^K{hihZx1PDSOR%2DmSa^e>&AAg z*`imdIF`6Y$dZQT1s7J|?8y21L$`T2U-=#Vid;CD zg`CugnBsb!3O&iPD6&tN&`v(0hlNaJJIXby2IZ~hxWw}?;P_qp*Nv@7-rrAVD5=^1 zqV0854>=MaG4jYI;(6Hj2(te}@ppTcdSHMJgNl32LvvFgS= zI@6oLd_3O`DUI&Li(WM;z>6^4@H*Y~66%{wntv&CxZP&$**&A9{FSL}zNz5RvR~k8 zD}n)~yu#}jMF@$S96#CN+l`cx3=QMx^LV6#C-w3TgL3d2Z9K^sPTeR(#UhT*X?W{n zD#F_RHtV`v`=*CG&x{XrT^G`$mQdrzR+%Ik_9oh=+{c3^Nu?p4TWK6?6Kn$!$U`I4 zK^CPb0an)hOWvJZr9yMr?KgtB$!ex@J0gx8KISD$rP7LcdfB(TPtlCen+i-x&v(v! zcQe~VE2l~AOOB+qtDd^_&uM-8{;I2tB3h;ET@@#6R9~Me!O3aFrb$NT$#x>cH_722 zvvd8c(mhRvz>D*|}BDm1(F5a%Oq zyYQwwRdeUN5?j}*<7Mkr`zQ^nr^xrJT?xG7BDa2BL<)p)_>65MoGPzS1#zb_XizUm zIz_m5!J6y?YuEbej7zIJ-}NS)Y%!O6;ytwIV;gDGyEqO|*+_^g-fQwarQhnu&w61( zqjajRmtl9@m(BIj%>t>Ew^(JN*+Ly;%unqLdKV?qR!qIfX)E`B2Pz(j&4j65bxqCe z2}tS7HSZ9$rbmmSdk)NQYSj*fh^d0qzFP+trq8rS^LrZtGZvmL>@C_+=={odTPj+p z(&&_?ViD|Dt4pG*O++MS61>VP_0=zLHyBOn+x6f0G4H;O-Wd+?=xNwyFCY0uvgf=R zH$L*vea9>_!B}=NI~7%pk>~Nkin5#L`jMRaWB2w_cHJ+}6|OtT?%FN%ryI;k>nqFM z#|tkh5FLnH?kFgbzROUmD7u#7@e8u=B2H&A+S^~K(Rz0#vMa%}G_l)_z;@pGtsm9u z%AAnnX$hT&W-iOBnzK&fW}{m|c86s~f;+7DQRgMmEhh!9IPzyIsWy21^NbUqAN#>#Rav2lgbm1j2N~LMKh395?J1P zcR`pFMfb{)$~y~ZG0}F+Wv-Gz*Pk?rRi|dP-s=ZblmyH{YPOqbgs)~j$O^VQ>Xd}m zNWKy@AXEh#8~_euTBaPVz`(;l+!uNWUrtj5l4ubezP1?Nfq z>QmdPGRjkuwJJM_&dwNud!$UtlS5M4!~y87YdbdBy)~*)vg2m;$vM&Z=%u<8%~Q9d++&rEHyj2F|LQsJA}w{ z?G4BKIrJ8?zq%a{Z#(9RTF+ytxgzU_DPN^4-0kOFW$5i)X18_;^cq}`Uhc4N@-14Epz5m$3=B?@l2Qr0K!L|# z?|HfjIkl5U-*h>=qJkGukL@q-QFCm?`gIB71aS5${-eW%Y+gN_O+qxS;Tcs7`yIqJT@*dCg_AA?#xfqJE-!H81gzXS#Cn^A z2n_7-+$m-rRp0DR6<)lSTFVj-@v0%D<^N*v(7|K9;nI^Mv^>PIiAOieKG?{)5+SPT z#8LW5rHf7xFY{%f=MnmrvLux`n?~OIgof&NsZj^-tVp4QOjIM#62hD-Pbx~^7N}{q ztKzL5d1g0+);`}s_oQhHF)hkwLL1un#+PCb&Bq45`i-d~E{yj|e!EusbgjX9xVCSf z+*5J%fqT?-4`T=A1v$-~eoiz)FDsMY#O>iE{n}#-;hH&-Re=S5R(|SsowPp2(=El* z9+EvJy1H(z?dllQ@j4hSnS+^piN#$=d5nEP{wi=QD`7Ycv4~osBF(P3r%9IjzzZK* zes}8$BGd1oxOXIywp;HVB0pp@CDgro=|4xHM_9cR@9|QR7|JM))1sVmw;1k`EY!HD zlNWf6VmqyW)PMr-QXLiXO53etMaXr7K;->-t5}YfBT;jDyt_w4LQJgNu-A31=u6q0 zHCkm}be=u#@S(gwe!QFRmpKapHDS=~`AgP&(w~#`o8)EgA1CmgdWK7v!q*SUt~r`& zF^z1_`LtgcElr~3Hfopp5pFKn^oZ;*Whz%_f)zeVQiG3QpwA9>5B;wH)-Mz7!n=r| zye9f28+eG%1hiRkK&fSK<$RBy6xu4>4zNx1-Ic+v84$izi2mS_ZzV2DBvbX(FG=4! z5OOq0gh0pHI`2)SDOP0LJ!8LS|1>wymq>7bg<4xN2VtYU)8F3&@eRv7*xkaYh_gW?aJD>i9~HF0_@$Z=fHs?{nS#{+uM=ww`UY)B2F#7qp$ zB5c(|?HzylZCG?px?imh`j7#g3K@IlBar(%n0NXO13<-XzN@?kLW@6g$Y?@46C53^ zmCXa!8pKi^SoRY*;X?@vsm9$k^T99qJJWW1sU^H0P?VlDYpkWA5C|WclR&5YVL$0z zRQM~>8BYBJI-*h+CWU_c&l1&+Cm_+cWJVv>Ns9TNo~)ia!isuGB}dim3rRT^5;G^K zqE=wemi@r6>(1$>{(b{w48iHju2{OHq#P=PE$P8av8MD~R!p1WC<}oI7P|EWd5Gqv zJWw3>UzaU|)UKDswC$1`Drgwy10K^-lqGAweok7mhHuJ}i}rn&q>~`iaMCbBP;%67 z&E)>h>XWke88rvhM*Xm&hZrhgvLO$=*AEf#f)-tMx?9?A`hOw*8cZYX;rMgiKF!Ls7nCPP4#B%xg z3JwDtk?;8%E;7Z*?|@D1whLpss*iOb5h}wM-YDT!c*B8YR9RE=jf6sB8iIMu9p(^J zj?k^4vI@GLB*EmqRCgF_dSX#q&{46tq9dQRf(#q$-Q*rAV$$L|k~mD4C?rT`5&M!y zj-*TDafg1Gl`27~spxoVt5ewe_Wa_> z!HlYi@#bb6eE1!=hEILc>Ajwh)!o@+8D@^7r-|4!APc^HC#B zWm|Qg3p?H+hG=|!;d^Vu1z8PGG-*B7#J@!YI;C~R3MIwle@ykXXfet^U%F>1)U*bH zZ0t)DDj@4+9~7)tSoQcl;JkULK@gi^`0bri9ZcbfZWCa`$*DP=_vytU1aSB1+YB}? z7fRy$CZ%-(V2b>&Q=a=?q#fS122O97ib% z`L#bLU&lqaSYQUh9*A&JTmzY@6=fQY1QZL@d9*IX2{~%ICk@j`YcIt*@GgB!AGY( zA~^zizG!rdBuXP$sGoJX%o6A7Ljy`#I;8gkcW4wY=XolnWLl%sGcn5qf;A&WLo zc=R=4)#6U{s$lPs4mM~z4^wUmrr{6lwtb<+tV11Z4;l+MXBSIxVrzH;7FVVB7ZzP| zP#uDr>fv^bV{?yTiXbXuU>cExJveTyJ(80yCuidA-KW%^W1I~GJCYd!bXLIw|xGKQ>D6gv9mbl63X#&6&aSR)tiMr|@Vo^I0} z4j6<~V#WEqXxOMcZg;9NnAC_G;yi9s-KC_!V?X}V1_X2TBH2-9WWprE+LIUWmcx?z z(Rk>~!rjBKw}I8G#aKOv?-DkS7F8!v#49wBr4lgT2a8E zA_ApTQQFFVt8oW8JPjdUM+W_Q8dL(*5qE}X=vJMjo9RKeVHe}A1~cg zcid$zfg{sL(MrJP_xE5U^7+oS(wxJ{=eQ9V28H7?yy@yg#X4+ehfAFy&JLQLu01?o z`3{4%?k}{yx)QuL+Ume&g8jRBb<0sN+HD2;Q+Ih*)>mUiMbMLY}uy`lOZmT@901oEUO;YP9Uf)+JZF?M_7}QaIa`zp?3l4iV6~s40!>G-FETh|dmUM_iHb?*Jso3EiE#Ef&&A^abfaB&8yDkglcEG4?F zESPGgeD$-YXD~t|(~gGvYx+jF`y`vKy$*GDJ2-PG$qzn~yV*=-g2+)tBPnn5qcr`) zV^tSG5e`4(wl11MaV?j0KeandXGleUIoR*;S~+RZ$*kp9MH9}jcG3ws6zt9N)w=vz zvQ_9=ek{fUt6JnJJ5WSRyH{k<}HjW!|&7WPgll;(guaslHXIT>c%jEI5Z_xWqv!70fJZ4F6u=z;`th0^Qv+M ziIx{1-FM+`D`!+Lry75E$9*!@rEat7X}5h)Drrqx&hFNQ`y)yGgMyzFx{+!}!s`~? z+P<;#Y-_}*4s8vH{5T*h9PG8a+_a@2J@uQd@>Fm*V%d*#Yq<9jE5Qf%o#LcYJCK+C zJP{VV#>SX_Mchl1J+arnwCbSNHh=G}i(pNS)<&+Ky=TTv36J1Uuab^gn#9Zl1@$#` z>-GG1#yiMXpGjrM>FXVmq6q0a6y1N5adaiE;-u}XaVyMKe8Tbs1o`TgfLz%anfVh> z`U97`5bDu8@z%`33Ui-0K9OhbEo#=nG1^@pHi~g7Md}P*u+WA?Rd{}iB^=55W9Jyd zmfx!l4e)V=-=yF`hVIOzjfD_en8sQkrz|Y?&Dh}grb1&OP|S;*1q7W!c=jgh?AWA)uv~_RvLCWm z1J)X(p2<0(ctSkGnGOn5TzUwh=~f3NWaR@EOd>I4L?yLL9tcBp{=J-UZb34 zc-$saOh1OSJibMj>w4CA6ESr1jyW7YveqW3Z;$YWjK?Oz#~l#4>0vc& zbxG#zxxZmsbR?^r8QbPABCJaD5H{w>l_M|YVO-8hLV0Lqhjo<$dNc1>VVf>AZ|q0L zzU3lHyyubaP*<3lEBzL2?`WtTAP~S>y2#|rl zaKZl^1LgL9l^Cv8DUzE1u@aObc#pAvRVuGO+$-MwHvx^ul6><84?KvJN%{sg63fY( z^(IJs<1qo^Df!x^2(f?j>l5E!)lmn>Vsaylp*OIR)rO59OaFa}|9SOi%J|>0|GZ-4 z0$lf_sb=l-Wf9P1C?;Vx@13Ik@527)6`09~A;!j#{ja|$C4-M6*!rr({#VYt`2zq4 zKlzI&|N0Ay68Jbg*XLQ3e zaK4GM7sn0(MxD}jd#lM~_Un$&>Cxu$q#uK0{$pxt>gH3Di@yc_T0Al1f&`^fsjP#Y zzoxd1&ggKgo|e`PYsxY4zs~k5D6?p0G|o|{_DG!)1r<;T^E zscSkYN6SohOte(^BaGR}HHq*1qeB@tG5TsfRM@z`IfoiyP*70NP*t7M_BQ=(Sw1=| zc{;-mL0O4_m254BA9N!A$+;bk^cjf1bTFDx~Gyse?B z8I|38S65L>D+Xky`6un~o;PWo-FQV*8Vd`y_jU@Zt3_65{jSj!9UQpwSpvbgnVVEN z|J=mQ@fh=?<(OwSSY))b9^y4}vXu#Oz!l`SSskl(JX|i1Bl-Z!9Fh%gB%Yt`HPjIo zSbx0!vNB%2x)w#R1J%R94o_o{(RR?e9D=k37->MuBq8EmezELdGvkB$D|~aq`N~hx z@aw}(3|=FpJNTDGe2ExVd@Es9&NjYfjqvk33uTOf`rk!pm%^_Jg=*fA`>j<=}sH0if6aclIEB&ZO9a;xBX-pp2=@ zDl=%h);+biK4m%G{1LB~2H?pPVr8-OW_5$@)!y2R;TjLrq0@Y4N^nb_R&gVynl6st z=4F%HyS=v)(l_ZT7SH2#C_UcL@bL0i?+b``em?Zl!L%~ZXz-x;o_D5S;QZyB$pfG| zI&sUx6-1Q>Bz28w+RPp~IJ(mSNDudl@Xk@MQ~xCbwM z6ykW>4tAE~b~Y;Y08~IU4AP%prvN3t<5INwXv)fcx7`(*FeeG$1iI2-5EM9t*w?z1BDrVbf5^T_~jF0Y9ofro)XG0>CYb;ffDp~$*1d+fK!6Khh{( zUfzuAT3MBx=YHDGEKz>QimmC#YiZ03upPp)LX|9y&&|g-jDVqZzlw)Yr}BN|#l@TO+2IQ*92^UgFY0|AO4hFPzj}@g+>PG9 z?70;!{v-GtP9R4?RaL`4ZPH+w<*+e0ceB-}GBW-&DUzdFTZ*GPEXKGH{wwDladYu& zDKo8ds{~M0*&E({J0TS}9YZR!3v)k~1EuDIfx7`NV8VdmdIlu+hELe;>|PRBGhI#9L!=K83kY@`XUiMt=SD@Sq^~ZD~VLk?LYAoo##Ms zIlQx_Ed2!~mO-tnW#fk%ya4ChI)7$QbO#aR50B`8^Rd+(@5o3>Oi}L@*PgyUDea7s zk{&!E@~!Kl?#`c$qVIWtaSL&UxM#%+3x^J)1qhu$WI2!*r>dkS88Ex$!PWx8Tn>n6 zkycD3HW+#XYDjVJ8LQ=>FfB(=fvD}WG>YQeXg7Bris3RrPMXQ+QOD#DbCDj+^WJ1T zSIIxmj?)yvP)r_Z?#_d6h*Y{YB*Nk#o zSca$4(znb!wD~Oo%Af;_@HY)*fN6+)uEl=#$w2Wxi&xWZTO6!&YHhOqqob5qFbFMB z2zGJbtNtZdxs3F|xSgQEbEXeUa@q7VIBBH`@!J&=_byz#`YK;I<7omsA|gVW@wc`9;P#Etyt>UY0Mn z9G>-|2P*QhEdOmfFdKIH!crmhbU-3nLI6+?Jp?+7A)yOF5qLX@)Pyov;7*`yX>cY= z--`N;fO?nU3Tn0HZLvc1gKt(f`?GJmW6nj|p)}sWsvxJ+Y51&4e0_bN(Db%hFmd+Q zf;wy|ZQqy;Hn7vvr=a(zCYzw*MFex z@!e?(cE+GFQsKbUYe5YlpM1WR4>P=!K_?2R5fZ_UOre`;bY9x1YRBY(L#usgL7ier zZb8kEhp`w( zRu^g1N=dbqRZyTk-i7O*xEpqN@8er_h0pY!doTE!K-6dyNXahyXg4zCiFlVw^3vIC6+-o@Xq1*>6md_EQO0l0Yc!|WN|7IK0mtE-*VRELSNw1 z3febw>sp4#)4#bygo&yvfjBDvELC|yE3SWq=MLlEB#-S@1t5x0$2-HL#N`15HK)g` zAyr1(RR2JyBiYY6{hPCG#h9w-49c6-kNMya(hVeG&Z!eu{9Tb)O}G9ufl8e?@P$IF zQNi5YTsbX87+JPkD*#uj?X6e$$(oJ(Z)3;&v}zhjks^NWpHp}c(CK`e$-^}<6?W)oaa(K83Kp9tH5ZLRXSwh;h&eV z%GZcAJ=X;$)xZ!nOf}V3^EBa-+OVOk4G5E*XZY_uoChrni;X^gI{h%G1cg}cA20o+ zRg@&yAFHiY@9G_nu6mllWz(%5v9f6*>~ABhRvZfKJX&OBWk(LF4GzHJ*R<}YAXqRR zF>XnxXC4U~SMZAI7koUC@Soi3%AbC!J{4#n)QW@aW$Zo$E~|icH;9H0NuqURwg#r@ zhlngFPeI2)naN7^Olwq}&s`zMrC0l6;q&%@2Pv~(P}wa2HR8O6&BRpD%gmE5!^307 zp#EIMth(x)6MK-GpYI?>1T#rzZL!XZqzC9&k%z6@trDoL=2I*@jvkMOx#=hNdT&t` z^1xs*ajHWlT65zahs?_0y{xLwCB)hHCnHNnO3b>KfI&pH#c(lqExItA0deyYIMj}g z4FME%**KqWno1~?ZbbuByx+I;W7dQk2canSs!){gI;lfMV-FLRneq* zg~gpoGpB2{q80)Cjcfo4-VwSpH5I_(o z;#PK;IY*9L1acU+b&B$+&TPfJBu;*QJ|Y4j`K(^)qErrc_)JP4J$zeCjByy~W)b_( zgEMRMrsw$Lw5sQJu#mY72t%1&wI6c8H@b*~_k{aE5rANymW^FLd(B>RM#a&XdUg#R0A120AwKQc`YC21ka6gC?+U zi<6`MzH;mFx0?^Ky)6L0*If1heKxdm4gK1cCcTWQ?T4WpnMT;~L^42YoDV(dl2hbo zAR}&$0BWKuQ%rYe^OE1=4Gj%hfCj6b-+`R5pW^Paf)dyA;&#E-i-gqSrj%pSjB{YA zwoltHo?Ys1QhxjSP*A+!RSYcu-Mk9l989kk%!;V=D49{|2^d7MV5&rQcDmQN+S=L{ zK)W!8|yzkMvSpBIXuc5(s+^(&g9T01sG7A~x%ndH%3M-DqmB(WLMVl#OHVORE><%t7mx%3ISAMiub#WPyPn%)k}Wap0n7! zITUJVS6sdIORmgudFZowJh#~=cI}d8pub`5O%k{PVqmU^v`mZd&qOnZ(C6S0toDWg zv|-ZK8L#nxQAHIRG~NAYsyNJ4LN9yHD$*0x2|%3efVMcR*;m)$$13xW5*_$tQAj|=s`EP}~x5b?iXIvZY0 z;Bgx+FZSkI_U5Q(q*#zefGVm1AaXA-y1{*A^>(KLIJCb)Y^XI=R(7^ceSc1`bgGKF zx>06=V=O?zOkMY@LZBfa+1fRA?VivX#-~~VTHnuF&~&hk4qHLu&=a_`U;(k?0}S`P zf^?k6pX9eUJ@XfjLr1rrK@)}B$UCfyL$Xv*-Nz=>&s||(oc;Cd z*QZx=zJxTh+HR)TbqH{(VPJqYKCuk&XdLlgA7t{p1gE&KQOfO+nL}hgFh^LC@1#J>OiS^)D_qH zhqCwB*-I_5a2m54 zZQPDa&*B8^GJgHzx<5!&-q*{Uu!#ISg11VaVzRFURmX9L^B)F{5H45d!?`q=y4x0 z0=IwReICx4Q6aB4oGbuah9`23L6p|4skNoWVq^LT!n_rDI~h7;h^!mLW3Sa)Y6ug7M8#VfIBqAm{dw7Ev7$h+}2fWrU#>C zmT`a^>gkafy2o^%j?XIIeCDC+*8G#!2V`KJCK%`Fp(XU^IPw&XCx>nP0ytV5u!5gO zKsmW1032UStjB9UM!vXzf6U2Z3q+Ru1n{t@;T;gj20#L&?FHZ9`SYF#Zw`DGn`~{DJe_im zUm&Bdoim-@ouN<+a+3Bh0`G%i?{0x%O2N3>|2`JT806t&q)VefZ&}LP%+gY!l8{g_ z7r~Bka^7aL?8eY!evHFzvPq5n^UOmtUi;>%cZ&azA08U|!~s)L`Ar(#o*>i@Y(%uZ z-rZZTq3gh=v(y+!1^tPO82vG9XTEl63$Nu+xNU{@%x1KpLrz;9Uk7jrQHJ^QkFoym zT^rO73izult`>tuOh%n)d3ir)Hk9OuW_EzJg?0dE*i{_D>E`#Z&xVFjWnluM5KzUC zF`|hl!VPbY7qENsGFBz<2Jt+G32|*NGz2{@{{{8)>NuB(=t}@|G6bB6%8C;xqX3Ec zR#0eQ3BuWXb0!x-JE|6DWZ$}>^`9fxaX>`sBo+d|W%@{eON%Vk79i_7?)_6t2ApU6 zRJ-0Eo?6Ks2Y6^Zu@DeWf6hZ`J7q=q`R3(kWjzsqaQ#0$P}dcffs_??0?*H^7zdE# zx8a(Vdj%8yhGLChjKB2f0V##zEcT@)`yneovr-pI%E)IbCRxeCWid2z9e98`YT-Wt zvUe3hNqPA&I6+va_+viG| zk~27BA>rCTfi4x|NRk|30605fi#)mjzRNOT05P?=fxU8*gR$$8e&0+`{Scp{BKG*PyC#u$Miy-Y5-HU$__n{OeQO9^J* zF!&KMl<8PiHn@lHfgorHbXgCA%9kYrhQnv6$1~uDfPV$8`ORx>bM)*CCYS@nv!d1; zlG{=h$xVP!Q2|U1I7ld~cPhST-m}8G-r}fRJ%uuxdmnY)hiAw`v7n(fsXc9qt8Kk(jxf_kw3)iGoWpT4m=PX z9E*#eaX?!$E_#<5BGTc1HK6(ZYh)RC4sF-Hc=v1#VZh|`8RlD$Ccr&2Frqg~!V)mi zgt_+v;BhB*mY#selI+l}8cqOUT|PA)B!X)%yo9Vn^bd#n<5ame|B0v`Z8iuFKRCK?zp@E z-2zN;%-XvWS=b|B0OH;KaT0SlUrL5uN!dHPQ|M4R6)ZIJp4;`5W_-`L6`NFusNfHS zz{2f~u(woBHkkbx!?7kdhI%ZN+H=W!4`;cJu&?Nt$$V_%$p3XyaD*{?(7K7|%;<*y z70U@f&v^Sp_oo)XS*C)*C5mh%9EKe03U%3v6M!REoNq9p9h!drsTGVwI17%J$`ldP z`uW0J&z{tFZGLUKOTyypz0;V6;E@g_a$M53q_nq`$qZN9ku%w;`l+MzEd*dh6lhO= zDoY%T2&mB=--s+Hx3EKEXmps0h|Eg(N|%x6e-+$Y7vDb9JW$(}{4w8_E`K2*Zh&&~ z^1uu8!JWsQIk)HQNRP}%s~hvG^U)|3J)>SK&n$H7oaqlA0#8%^0Za?x=osj&7b$0a z!vHAwVhq*3Jiato3?hZPkYICj^F zZ^6~*+njNxjh_~=HAEM<*@#FJG!ksHBd*cLnD+U&)F%5dP?u>KxHqA%6@kln%&n{( z+h)o6@u`BgR5UeB7Nz$Z`D(hrS1BHcIzlOpH7BA-+d%qWy)R+$WwN^iAYC|WZBHl<6~pqzA3wz3Ge^T(2G-^VLY6Za%R3A$2cn!q0~) zY5%8*;PM=Zrr}1RH2-$d&nt^CnVm0J#Dt4sZVU`sh&xs=q~l!bWL2y(?0j@-c`?C zAyqQr(#7|L?xq}*x?acIJD96xFeU*7ZVZ5ZpYP?F-i^`D81Xz@wtBBa*o|o>ftXOJ zOv`)4gG%)OTE!>O{A_Sb(F?%X6izki(;*(k1S8EcmZLF*L0_r1IIK6u41Y__d*^~q zn`4#XvXDLjy*_vD`1mo5j=Ry~=rP-k+OHxSxCLy;+P;$hWGwSCS+UZiCnaI#eYlsO z`K0R7`v)MAKV!I>G!VLGAm+0lOebrLYuzQH6ErLzu?@U8?&o`eTyLbYi^a+T_zR?r z<6B|II{&T`kEJt0S!atz=d%2b(sV2$7WG96oMkb-l0FZbkS&w9XjK~UqAR#VZ*CgvI2DL|jOWI; zXHhnDtN*y7&e+_h%xmjPJpau8)&$8NR`z8`i5vQUU3%cHy@wco0yu|daL-B#dh8L1 zvcs|>dNx5*f9dk5+pQlhfN^{pn;1d`U$EJ1<3mAsTZ~ez1crz}xcT1$8>D>R6KQd! z*c12B?ar}Z7VJWrm63GpVqel+hHZ;8Oh>y&`;tHRv^2Jt5%5vBrao7T?0;sK zm7Of$)}9S$r$gR+xw|^PiVR{Q6#-m<>Lc>fjc3)a4$@u?7|QOB{J!vO*+-!5wR{MZ zQsxQ&tnP6KjYccxMEcx4Y4>W-&gUraH6r+@G(>bHX6LN$OX!=Mt@@t515=?X?VY-HKz zw1z7DX^rIS^(^zYuGHsEk#l)Q{qSO7&dOz7A|e!@b#}c=cdxsrp`8=J$`=sCE2}k`x2$aA3?TvDsiVmSrfh6eOpphDa(5S^|DNXO3C>rujOQi@-*lO z9G76@%hfE|^u*H(YqtyIjg1V9b!0#}O8;;#1$TML;Lx(z7lWx6IwZKl8div6-;A&e zg8I&@_uZC#CSjuzeNk*Vkqlb>&oBHdiTLE+uaTAab;|A=O^BzJnKELPZUNc6#s5XE zkOg<>+UoLke@S+sp&jc5$7;C)LYr^b2nBNc=#0{gpUFIUELS6IV3(UeX8RpaQ12=Z zy?I%mjw&&hjK+z{4}{M(&#V>6?q$gc7`R}`d?@oyX3O)0tg0!x)n>P`5DAB=KVdKO z_8K^~3abhKKC%8tiAR7XJiy7n9P)I4DG*Ge^5}k=qwi+1fS-38o6Xwqd#3{uGW0d z3AA+`t$&J~lG78fe;Uu<#WN}w{=okGdy;pi3JL$tPCQ@q72~!l=Mn=^N8n1Ic2-gEG3*{X&4~8j<-f(iFy8jG1aT{^f`c|h_^7dC=bG^g2Jl39#OstmH{J&f>?8h$K+Jy(j zpTJv#lWFA4PE!{0Zcc6X_3%2{zfeX8FCPXwxzW5i)WrQ3>|OSJh*9;cK}&RvOPk!U z2s&1zMJ4H%BTBN(XyIZ=uyF;Y? z=IYa*-&*&Nw_HB&+?g{c_St98)kGEg#a6GZ!L$FRNU4}Ijdf(=(O-ld1Ly$dsS5Dt zyBhle>f$Jv+>kN-glo3^ponj1)#^QGb%P5FS%xBu{dqtHZ7{ zLZeq?&5%*Q&auSDnsFEBXI0bp`5an>=g@7P%WSY^WUBHy_^Lt60_D1t%u@c z`>%sB|GH|h{|Hc-y(h-Ab8~Y&x^vZ5_MnsXi|@a+4`XI`o$quTF;Px$v|M5pt1Qo| z!W|i3VBS;%1x7W!IP;f+Htw>r|Jxe+N0i$?(?Yp-V2Dcc08^?u z0Tiv#l<*pRH}yiv6iC%`_?+(&5bR;;ASw>Rc$B1tqQAVQC5fRh>KdI|*ldh{)&nwkNVT~2uZ)dTTxR*I0lQxn!`;|D?D{|KKCV9EitAed1N5O=&9 zlb5cM)#SctK_S0MopkID^xm{lTN62fLR^1*3NMV;#DpTW=Gq4&ri6_lnM!O8$CRiT z><;NmsqdRlU>S*r?CS@MdU~NLsH}3+7d9ECWpCg??1Xdt>1v`S5n#P(3i~_Utt;D8s=< z#`*Ru9$lQMy;W8=j-ZgHufF{?nFyM(A!5OQ*STA(ul zr^v8ROM~}%R!poE~w9ej{W(0!VHU_Dk|3S>CR|T)lV64K*!aS-CLE z4-l*!!1)62ygl+z-1|d1=ICHaU~ExE;4?mll187}^27t$@*i6KLtjaahS6ZzSn5Yg zSBJt595iJUdi%soCq58O9g7Ws2EdiVIz{p*ewyh^RZuufyY0wFOWo63>%Iv|i{JO@ z{HK%aoauQoSD;>3IXbYtvG88=S_O}TsfpjnJ{QM==(D!h1b>~Bt-fqCDNe|QO6>&t z8;~h{4$C4E=GIcELdFc=1(LdDh;fdF%l&AOgUS1xKjgT>**^PRQ*<8t0XPH5o5|aR|4=u)5~+&;Ey!ir7klL(@!>rjz>}IYt`Fd4Kq{}Qtfr^!-(`Nf zk1?TAmh-+2;{{p47iR<1J+?(^A1s0lA0az4E@Rqp+!v(tSfA=km!q)od&f(xCS^Gn zrW&5(m%Nps332)rJ+NTg1_mChI4Qv1M-_91;zS6C;VPbcfK&{?`P~HqK%RrjUyl9! zg_QiUABIPu0P6zfG*;CPG{Jy2x(=8Y1d?&!WXg`hEHD#cT{ zBl5MkK*qHA%JkfbfvUIcJGH8)ci#AM_V{I)OSclyBSe`D#(IlHhQd5?Xlwg2NTJ)S zSE{k-PBOJ*GEBg4S-Jnb{V_}U^FnyESaUg*xRJi`LZ@ygH_vn^dG5#Jqe7X@J()6s zTJS0Hi+y)h73bq93B6U+<3EP-b5bLRLS5l=_<}2J13J2d)B6&DFbHZhg=FR|6@(YqMivm0e!Ljv(V2H2QwtW8T)klBK~iET zDn?@n*CUpdU`93`i@&{P#2>U8ifY2S5tJv*x|CqB0mVL}Qd?a)qseGKUJQLI75e$KNzJA}Sfmwo& zvuv&=i@o_!DL-i$YyoFyfB)8|*3V@jmz9hMl_oX`@25+4d7VjvgmOHZb{OMW4g{*A zww)h)<6{V&m&nx?3M}`{K!80u9(RCI=VUe8VL|(OyUBzP41oO;_a96Sm0=hi(%K;z z1%!u)ja9h6E)ELu56#S&h)2_w)YjHYy_KWcKzdm})Z2^Gkb;41@id;-#U>5Oo{8>f zL*JsdnYzfi^7vrI4jbAL!pGEo9@?~Pr!A3)TV@1DNTq920fIMy`nFX>_L~RSQFTin zRcs|ttZl!6oLKkcTSF$T2&38hq(ms%XTLYKe7-Xx{!+5q0ZObCjUbOL)w2LiIRKm| zSZe$`E(Yxn@+bi^SZnwE7O})K)WafM2vJ1RKlJ^tZ=u|ad!9l(%-qwgC zii`V6BMa#}=~K=Sg$j!*3x2H-;l2*jrK_~zFqbwG-CWO?x07_01N*#O93;;E|s`euepOX+(RpS2p;ysRrA<34p6` z+@KMG?H;#@@bnY!u9w+Fwr5umijwO7!PyQb2we=9XBcT^u<9zk-eD??>XA*WId9?j zOR&#p%Rs>zm23+FK5K8_Ze=PKK3`u5@dE@ggR3v=)^6ZnK?^Pw_}2+&I|zW!ifV++ z_;dA6WpOMf3g=z6A*zTA|1gY*0-AH*=5bh0`x=gJ`C_YCdR@jjEgM6GiIPBlf~-2) z>j&DGc3pwxxpEwqV|o0;Qe2~A)Ys`1u930KjD4l!e%_wK!CVdGIB%-R8qk!*ywUW! zo*a%2hqqO2fgnXmAC-p6Fh~O>(|qhw&AudxeKX$X6P{xUb^SmLuZwsYbJ5Q{Bu6HA{!x|pXYCol zGpnw*_T^jO?H0J6eaqi_BQGAub$XJLEZ*m?idBAQWO@LRuq}NON;uqwmBcZJ)&G1C zJ>oovq)CdgD3P%Mq$ck)oHC%?8T#tunnAY#riGf1cey~T7qrD|c}Td$&+!(H9_AGm z9#D1p-TqDXQMc~H)Hxj5lRoX7jT)8W>|ZYoVh#R*+MU7McaAx6p)*uKhEtMp>$s^` zkGLy0JxRUUFp!JYINX~RpBSZ=L|0&TK3A(UnX%pz1;4MP>m2ZQJb1m;x6SPM{l~fk zTEri|h+K+L7`^xVF?4|O0E~=c??H)O+nT1z#adns`(_qFZgD^!_b-gICdtdXo9MU_ zp!FeOp#L~3JxSo!iId5c4b_ONyl?$ThCZ~?XErS^GBOb-Wqqu#z>(WOW3R$Ub2Bi2&&^0l;I9g`0UK9B>r5h@zbHlkJRCu!19?Tp6s>J7onQKZGIns{BYgQNM#WS&*T>dzwO(lN25NRXd8WInAU6 zFWJ22F2jcuOj3J9rP_48B98Ul{ zc<;>;>HC{}wiiRCF8u+^#+uGUGob{JBt?&zwmRk5H%amysW?SSbM=867r>~_vX|{3 zDn8z;yO`2huDDoK`)weBhskJ}p_PoDj)rt83`^@}d<`WDTt!92IaGxiOpj)#)A#@E&wJ>`D{6N_$yqVlW))(bg!EVbE>S=d5UcEq9h?L#MsT2*w3 z&Fi-rA>a4dr|F(Ose(bKtsfArvX04NWZ@#a(u~*n`DjAn^X&#_vV&?adnpokUmt>W>ck2x7cpXs3e z&Fl_85OqMY2_>PIgC1!7+VI?*sl&WoOdnDzBJS(|XGip~k&!=v(!u-BIw_}uiy*l* z1Cg{Q`{C@C7mV3=cy)*9%qWl2=z^E&F>gMwxB4b5fQmRA zu^hjsY$N8UPB4wPXBRmcy_q=6wdkmrw<3O2mYPWv-%y&4&c6%|#*XgE{R z53~nh%)44qoRI&M89sYzWMot;<4`94NT z9`(p*q4-;V0a*unN>9eRBEu;prn+2b15Qyc@v@MV^BlhlK51cB1&njC`mrS5Y)3M* z3=M#~>SrhNVT7weCdp@g=pJ@20n%dy;5kkK0esUqRtx(!zipJ@6E8}qvF)rRbHMH- z_s}YJZ7p{G8+GoNfnNXQ3P0#u^@KNEJ48xFe(KMV1 z_P}SSYeLCJ-h?Y#i%m^ANRR)mJzWtghwm_?A=;p?x~Bj~wxC(>Sn^gu@9w|_ggl{P z#B%uiAsa+PDOd!O?+K;f8>#G@ZWoJYrf0lGdRbPw98p&K6VHCEZY{&Zsj)MJ#r9y} z`_E|?lq0sn-Myct9rVtJ4WEgW$Q1456iRfb1L;(wgG~D_fo$3;y6`}j>s>gT)zdC( zg?Lwwj&KYz`=<`dmFwL*=yYN)I;6Z|MW3er(5O7TJFE$UlQqCD3sTp@ay&f!WZR)fsK6T-WVkgNrnQx_pf2L)l!cC$Qy0M1}1ric>YNO48CO!66|d z0ENu1tW;#w_sPl0p(<4=keRMHm;ZR|c@N>gPct=1^IP?)AanQEas7Z-#eE6k@E~j~ zQitIV4N{ZzyY&`s+Z71uDc0H=Me<#;iaR@bh1K2(acJEj$KtHk5tq-pI$lRS3~|`+ zjgTc^-#$7<*zHU`EyeK1J6;7AdIo@|eg#`?IY22s12bQeZ#Z2qUbz8Hj<9Wpq@OpN zG59a&5nzb!{rMDMA;xpq1p-AAY>6?gSKa=hfHekpU97#2cI3phX3oBY4f)`dBmL#e z$5Gpy8odQd5{Q6UP>G{Fo8QOrAGV{}67QGKu(DJWN^)fd?9{5LA{H7rlL|Fr9?p6d z`=_0Xq5C5pr=_L&5V-6ySaaMuS*or3-f|nnV9Zuqr6CkCc3&P=h|&E#7h|4G6$6yYmjRa_FYZ##WhQ%h)FX2M=h&Z z&(^hR>!=MwJ=MSC^qu6B6p$sE&nNQdh>GmWJwe82<|m?W4+YGO7`cn= z+}uUsebu1arY2|r z14b)5AB^5dc-GxAu`3<1U87%iUblHCmt}_492hh$$(BUeI=d-#ljsSVc z)(jJ0skB_>Z_dg&zJj5(rBB~>C1W5MMe(Zn6Ve)3uG&V|*F!oucIuPF14}L#AkPzf zd%IG-{+bc^4^MQjyE}Id$D^hA>{Z?-Fl#Cc9Ox3w#QP7Pb#*OCBi0*&Cn5HJ%*dZ!a`5lnz1SrvAGHt+n86M?8(#WyWx`mI(F67<^jtlPe*%_Y>6G6P zb%@$)>NW|hI{;}v^Yx)!%Jd9^ipK@^mXt3dx zrqeJY&}9@Es~kCc)${K!_s^J;onGMq^X{)mf89whnAE_z!102m4P@YvXkA8@;Ajzpk0{?7tDVeZHE z3l%k-&wM(Z+oQj6tp$@D({V>mnCM53EBg@$67PJ4s)(~pC5T3Acd7UkOc)B9yllzE zyAQ;|;r3IYS>ez$6bxx(vXA?Z;@aBDt*zSALlFJc9 z7|L_U=C)3wnXLy~a1o6`oke&Q*T!Fz zuoaUk%s=~Kr^FyDcDpj6hOvT-=XGW3ryaf&QiS#|(48W@XolCr#HKg9&Uon-0H|Lt zz#I#fGTcV1FA@_YBT3af(3I3GS@KxI`8l_A_RsatMX-Fz;NoIyZ{Mr0^X3vo)E%)3 z%$zXduC6dRznxE0lKLpGq877&{#HeUM7nGGh(3e$bB=Wk6CJZzb?n*16rNAwd;F|T zlCYN;%t_G1R0`HGM%61SefSV{5TwFiZ(+pQ2DIn$%4YG@Sp zP$h7=L^lGa*vQh7*3dEFkNGqn`Mz9=k(GHfb^nit0`=oK?F*=Vlv(bo1haTTX+ zF7KN+Fi;WU_O}m*SK>S!@*u?laSsG_+4r+JKIB4x7 zfw}gMHU)-GCs@}IDr0uJ8AyrSnC zg$Vf#qKs~n?&PIy^X?zJ@ir7aMM-$fsiEoF?TR$ogg`iS7VXdvaqd$rpf z>@P4AM5wP}@>YH#Aj=9Y=znt;vefyzURb+*4-ZG50VB`f_O%|2Q6#yB zb8v9z!F>T#qEYa2mDLIY+P?wDtAfu{^TYS^^8o~ccwBtZRYQ?N+-`69mwU%vnd&eL zJ=(FID<)5KXj`k=I4AHfwSo;8qs=j*RL(!woRc%IDG0b13WlsPmUq$nD`15USf^`$ z=_Dr&5}8(urA~veHITLFDXryBJps0hz0w6uB&po%fs%6^8jp~$@V4sW6&754byru{ zHW*wWubY=jx}Mv#G~~Mn%lqe}$Z+5N;p6nOV(j?Iyuy_Z*B8dd#tI4wFFKIV!}%{~ zEuO4uGY z{OHBqqYZx!ar_L>3WAlZUC`_PpYiAU+zY}}3uGa3JPIln!M%1^+uRf{@evGdZ*Lco z^3jd<@|2#lk(Pd7rRT>)ZN|m7W+mvV@1}D9S)M&&i*6OAWF2QFrpj}Al!SUF4qe3= z)|F32b5yNWy2550`wO4>I6K0>8U=1a*brcOJy+U}Y)wriGAs1-F2qHwj8?(4IrZTp zcs+_Nq3W^^5SF$<7R31E;R?Ka{r`58s1eapz=C&At4uD2rj*AuA|k?2PPtl)2#LsO z=Ec&pgUCkwcyQwGey)|>0`!Rq1K|>#C-|eo_;o#>)v-oJe>|Sv_Rs&uC_#ZM+`S$` zRL}pZn1m3c^WJV2b_+cmt7BA=pX-GK*Z2eM_UyMD#C%$=lg6D ziG?;VdXEJlGW=3x&+>kBGhxYhAdkfA?)oq=E~94{`6_SM5OX+de<0sa^zTeeiY4+z z#)4EnT5dBP6SgG)t0PKI{1wk`mf*M7dLqUa7U;1P31}|rxrQ#VH$XExAPS06HXG#s z?{M$IG6^*=TMmIaU;SKbO-)UK3?nD8_>JeUXx0aBp8IFgf$TaCGT&OYdSd1AL3r1S zL|YBlcr;@TAw_qlieJECtK)|fONh|QNtxQm=)PfVqn^g=?H+QS6rX;vsv_yE7vgF+ z+n5eKHaM>kvdM_nkHOp`$8leJ;`_Lmn3y~+MVs4;80x1U{qtCl!EO5OX|$eTeEIq{ zbz{T8#Ka_g{IXTBYqBqaS0@YjyXPtB@fb!QhjnMwKvYXt@1kqDlz^DT7C@vVv2 zId6}n9qntOpd=8VQ%N+m=AZSK@iQi@+m7Nmk>m){kQW!em`)lXfknCd)Fi2Med<>R z2-+~ut*Ud^gl|?;&nMFU$s^lC?g`sjiZLP)sAc$S%H){Qnp;L%@E&=_#Hm>KjSjvZ zp=(D*8z;t!L50-c(up?~bLZL4q$>#6BP=2}km>hqD%f~WBvLt_ej~z%aEiomkdHqv zO_tX2<`c5Bw1pRT(o6q>eb(|21S&Jsq4J1-6knkYPDKv6TC97ZT zb$Y1Z@V&B;FAHq(&tX#qcx_gM`S~9}g(0n-0oHa8j6cUh373~a3j6WXgf9|D9mWPQ z03!eQ1Tzq;=go~p1~mw7Z*Oz`+QyZssj1gqs|pEy+*uuBu!nUeh{clzx}{@CXx^JxtQYxWs=$uzk=NAdKE$06)(iJ?8&X$5cJi49}v zDYtUNBZC4>$qD8w22W~OS3q`X)X$bygukm$WWCRlvPp?jIMcdrGS1{Kw8~hn78^=x zmv%gtwNbm3CZQF=PK88(Zy2o*KxT>QrsDOKwf z4$slaj44c)T|Td+#~HsXctwMmVN{eWN*Xkc`h{Qu9Ez0~ug%A-cv9!RDF&U>aZvt* zwP+C`r#z;xWn^V#m9l{U2x9M_!-)B3w>I22-goRO0lH*Zi4P}y3F^~n;uvZ*Z;cgei{A~0z-kiN^JFPm}k z=^+eJu``0v$Wi3ty*&V#A?w>;3SG{yiyXoIITjEMf_XJHlW1jU7Z#-U1~W5{U#nYL zl{|$^bv+2HG07IJlBWG8Dr)h9U%TMb))EtzZjS$+h(ypq^V>u-tua1}AcbgFU+wLl zPFeIrnKly=)Bu*~tdZ}uj-~^tHxlV(AK*|E_=pA&jOh2t1}?O#|Nda2Vrs-GQx;>j zURi?G)z#e1FVWHDa46@eA|fI(ZqGUBgYU&TIbX%~xH(0$Zz~;+Nc9TG(YCJ}Na{bI zCU)fvWQ`awRljl~h42eEN#aIZf9P`3*1$D?{miypGQFgD%4)w;LctIbo9N1i_c;c* zOfCAuY~RU7IxR3|_Q~X&*^vM2f)*XdlHn_Q$v$s*o`@Wvs5E{lCN7>+Tl=-NtZb68 zRX?F_aG#{>*s#ZLo6fo_0d*r?SG@7=xs4pLBhI6Vh=Jl#4@M??iJ`*}Q!U=Y?~Ca6 z*i{7CXVfdU<&KuZ%LAgL5~Qavn(u=jFH4-9^Yaw`;lr)F>qE_xcA0(s9t10xKbff) zvw#-3;g1Nw+Ypkuz>v>5ecMFB}U4Cw0HR96EtW(N}5b{OsZ0o zJj^jjT{rm!`G1-DH&m^|kdP2$D(ADavu~!Fdk^A9Y{Zb>nTkqFYvyyFcX@)k-HSZ& zj?sx!UkEqyb;&s|y+ z2%gO|#NeBK76~BWHO5?iTAz7x1s6o2hqHgfCG<@fnu~Baf)N{3dYH8t5Pf7AwGs5x7st% ztesrN=HHm-(McwvK>}?Q2V9xgh)OVEd3kMMnJCzZX0~RZCr*m+rOPH#n2+ziRgX$}6xou^wDC!7 z6S?87c;JI!GJTm7^+h7xjsZ;IAb5DOIKS4CZ_9P1e>%u+`Zm~o>f(Dhz~il&HzFXa zBAX(ZUZIzj5-4M&Glc}Q7^oNT5_{*O9dioM;{fPu7GZ76$~S1H(klsB@h1Gn|rV%WX(CWrfR*Is-R?en?z1Vrk^Q5C+wPR>;nqk zK?~CvJ1HyPkM+1>cT;J4A}?j;n$+-W7-a@O``~g?v@?oI>MVZ;@XlE7w65fJ-!FBgyJbdx_|8_ z65;WaC&M7Y)^|pVi_JCOn{RokUoAif8687aUEn?lamM|*oD^nJuX)^>`Q%!!-EW}2 zUMFgHHG@UgI=bTt7N^qI(pHFQU2PI)M~{DSkK%)pA*%%cqbcGKC_&E_7+aLIIT1X0 zFJPI-+y(N)#9wT(JD+7UsjmC^e){xyWlo9yx-!zf3*TV~bkJMivR>gb`?|GU6yN`| zV?^8tM>a?Ns5p&b`3JV>Qf_Sm0OwwwRk?F6N@i?0?Vtl4fg8Y zg_xp?c9)Gg^exM6@LUTI*GV_2?Z&r}oy@XsFZjTWpPyGA|MkQ+h(}1cSP-@fL9-65 zh#pY9VR223K9-DwWpn!=DY7X)gwemTd3DtIQ{Nzmr|6RJ6-rPfY|jD;Ya|4tTA9bl&r5}!!aNYK4YlOMg%M@Aw5 zFpmF;%mWEhH#M^?Bky2I3O`I?y#-r5vWA})B8H`u2Pmd`9>^g}NAFaQ->C;=@(3?+ z)JsXTh+6r;h2$Ei_>3xlZe@%^=ce9Xy!@jD07^ik0I=%Nj~^d8Z>9LXNP{PE^aO$+ z5-2=3#ovtIA{~JGERwyzi6-@65$AyhqsH{#<9iL{k^`Ifc7qMAOy7u&xX?#@$99^4-Qk z$^)C4hO=4H2HRX?2}Skoy(*e>RiNWU+xKax1r7i0nxnJ;!KLzxtFR(Axky4Hqxcj@Vou2Zp#(Jz^=+JWHp}?@wklfpz7Nq0K=fl?I`Z3I>&Ip8~jecc~xPjY{SzyL8WEn>Z7vFv$xcpQUO%t}JaE6GJ1mO_YfF!pQGYq8w*@I=Ba+s^3f{l9Inh)6?D$!AD66JR zP`Z&#OqXTR+s%aa#TQ$Jc>UxR(=;W#9D#f!#!q_j@+K3a^mP$J<2)bp8X#BDK7c35 z{ee`a*DLVC?!U56guC?t@p{&C5~`OkW42$;&(HTU;wug)sytSS4p<@iW84(57rs(1 zE_G)omTY4U6`Ov1Aty^QG&kXJbnb{3LUlbqZYBdBzNC>()q3!3@rOfXRJvNo#ri7$ znZ+Q==h-}s3lVWf8NtT4v{*Hfo63|tqmMn1*Eu~4z!%Vn5aGrWwR|k9=?>L;piVdY z;G_QY+}h{;TCZfBov*Cb9!>u3@F4Jc%F9jt4Dv+@xsn;3YV&D;D4i`WZW|p-Hrrj| zNZxF@r&2Xh%-u|XscQ5KKe%a1e@%Zjs^t;-^GA@uRe#D*j-+pEk`EXAls}V<6&c3e z^HbSX5I&l5UnGkjuR1hcnh81Iax#Q@%@E3 zl;&ZlZsL=SmCj`1KK7Ep^H6|k{}4t-Mn%?SW+u+-VMC*x8Vhu%c_hPv(J;H8n?25} zZb&}@==gP|uC6>-hHCnHw`EPauh#4GjBEs{$yAsw9gEmrngS;5J+Ji_$vqzUO+b7u zuTZxLb{EXmrGc5N40YN0^B~O|??k7KXVjFG$+ng? zn0y7^DeJG8)W91rSs{B>ZTO4_`8jqhqzB+{N?g!q*I#i^yr9d`2&zEp@!9r z3G?+S4@?d*L;{Ops){7Ollh>{4CnzR15nj^qu6uw3XJh>1MU2%S@yFt@ETI=w#~_! z=yv|X@Gze#ODB8v^ATt8WI~Nnf#PMSfsT%MXJ@BD$n}vQkHn3Z&tFFis;8q9KpNf!aBu13Ym9k#%; z*)I190s2tuuVZx%o5M-&$pEjL9#?uQ?Hdf;pW+R~4_VZ-7=`ii@v ^rN+)r%?+U zF}f;siG)fp-d#7w4=dp(ylz2kIJxgfs0g#6`%cr1d3RQkQw0CPn2!A{K!^a*3_XoA`f`3k8_i#>e zScLw}E7_wKYuubomsa^QvxcVA)pVEZo9HG#O|+OiIh<7A! zwp3CX6Z7*5C5h!KZnau^DyjgbAw*Vh{qxJCXMdd)C+x+uQqY3pl^8oiBOVR3hb7kLFT8D!KkBA-rRd%Mv+_2GE2T;J%+<)2;VZAo)&XZ`hI%9w9LN-N-7{~kVAXo{4PERK zWc}SaO2vz---nxSHyCfvz8At>JNjINrSdpiz_~4TzUjD~`rYg=X7j2B$1=K7YNZyi z-hr(dnUdwTe`Bx?R1PZ7rVvcqaQ_$yC_IRpH`XnhZY$4zAlO0TqD#G~;^J&7Sj#K; zn@1OEKS?$TEG#2;h~m-M%boZ&30}4P=FI5I(_dW^swJ;t3uA^#N-~aUx}`=}D!zm$ z9^d=r3GbuIyl7tWBhPD~46$Dj8c5QApl}YSoyK6VvbkYhqbGoX_vxP=H*Y#DP%z&H z^TMw;cEvWA11;ib

U9>|mN=DEh5uJC=m*#1dweUZe!Db1?A2~TzWqfv~btyTCS z`atw5&x1#ri$CPjujlZspy=6m>2W!YOhGH2|2oGXgteBN+L!U0NE&J)=V*%RNfz6S zy)lds5fRIQ)GakVJ+f)*KU>mD(mb58$Iv;5*MfWPq~@|;5gAreEMZ)%TkK|sR^0Sm zlj(bFngBOkC~|`r`r_@T-t?_@hbSQ%GcU^R7W#C;qpm!jOPBQ5 zRRHzQiqOpxLTJ^3vv?o@(8sVb=q1hsgVSk%{`W8?)24grZ#~V+20NwtDT(#XTkB;J zln`$HRlD*E;pR>I;k6>d7n=c^)L>RxG|ZSjGj|^8+{L#E6qS|1?iXTR-fs1LFLz$u zwYc-xr zB+A{SAjin7d`Hv3agm>!3m_+St6g#b&m@#^?@I;}N@Ya=Ze}vhdJCBud#^6|5=y+P za+lDwzhw-pS3m$_tk~@0B4MO35MTzv^KnY|px->#5M#7rtZm9MQh?B`;1*1&e=Ru@ z)E#@5r^c> z-cM9^vp@`~sZ%sD*U6+XDMz<&78VKk=@xfC$V%BErw%=btfoDw*M~X8?^|1SF;SX@ ze&JGH{s=~LH`TV=Bj5A@bK;F^m3b8M4uw?wmuZ5X2f`n}^l!+h7o+^SodB(0qLjzt z+PL=b7YdJ!rIM7CmzOuR=-gx$YPMgZDs4D<**Y~=Wx3%@O4j(}eyOR)Pj5Hw;h*p? zEBo|Yq)PZGZn(IeT&UdkCeSnHr+&v{RDI}1upP1tNhNVh)_)iUt$=Jm3y$X>!v6cs z!KhpN`%%X9rkMo!R$uqQY_x163r;d~@%;Rxz&Yz(aJ->=9-FGV4hPxdDQawQ)ypX! zeTP1y(5hWOti9DIqF4{Oj!Nvx>*(eG-a~;TPx1~FkaZrWa^|61XnF6|ddPtC$3 zBuo~TCllxx82aV#Bc?uRDh;y@x^8PG+;0jq+jQ6k8UtFABbe6|WkGsC#xFGN&5eJ{ zvVAx_krhuOAO1e0slnm_W)So5z|(w!Hk5t=rMsJ7A}ophJ>R^<=B=&l{O7EYd^KaA z`;jp*Z8C1-KVPWDCJ!-Sj}@4*te({fs?q5eHNb_7s{bJ%X70*Vk^!}F{v zHJe~c*}%x@sGMw3>!>v<3&hqELM-@cvEd6pn8l{-yS!Ix0{f~wTmz8f z<)k8(sp&nA!*FZOZ6>^ROMU|{T9*jCH%ivf>cn9+|YZiD4#^8=|N;%~ZOUwAX!LH?|vk!U%8Z@ zQO0dz>8oz-wl1d8@>sQ$O#Kep#r4o%_|ih{Zxu%L5nA^qfWXN%PNoEs;S?Y%5>g#W zr{aonRI|5c&b1x7hpOY3wYd^buGg(e(Sa;KZq693F3BPPX6O{ZiS5ne#T^`O6;;;2 zUcO{j)6IpmSmlZpC#Aq1v}p0tf8ps79D~L}5BE21;Dj+XfF`8gh&u z!JQ~ZgEPSE*n*U$L0o#WKObFCP556Hr5JLrJv2%;l;&@OLL1f$Dq33Y*~v-lGYLS> zlDjBYDd4--^0P;9G}B!($0EZSPtoi&teTP(JOn(!PeTQR4)n`R|2+;joR?P zGPe@>X(nc#*L+tl3Rv{CUUhgSiS_r{-ED*o|b)2%@$udE#5Jgn!J%91}0 zf9T@{Se8OS9R1x~Xz&ctl1*Lryt~Ov)#@v!+5y=i=Mw6VrrkL&R&DkCLhGmVK?b(*BiLo)FbH>@3t+B~hz_cnyvCn^e|8UG7l?}Ih z-Y#)2CNwPXJ6qT*`rz{I@UpB?CAKKR3;Xmn0cYxt)x=wMY_7XJtViLpf)4WvA|8&y z?9Y}CQ@T-p?L99I?8=*)C035bn+?{SloUS9Dj?U$e9_BeSR;i?to zr&Q1Pk1qtA`(U}Q4OkmoC8ifa2YxhZUS06IYLhzte%$Zx0TfrjlLT{;U9Vf=!DDLC z(daQbqr2Y&AvV^jc^5Pqu>Tng`0LTZ_wZt^VU2(eUAMmseMw6q!K9kall9Hbm$TKD zDKxMUQuRB3WMWH4wp{*vHw>S590FIOSrq)kov~ZUx;SJ9Yw5t5}~~x1zF*H^)soEm&Oxb@SKC492LtLyGrQTUhW?)=uY`bCQ!wy3?LW<#>!*pP4U2$sBpFCK zR>!GA9#27OSxx#-xowkVnGvE2x?#U4NS`%O$4{lF=M`SIBZus<(UEY{ z2U4O`iI^b*VxKy<+xdW4m|OfH09$R~H9JSBf~#=mBK> z+kv9Khe7u90dkZQ`1tsIFQXC?Bqudo=*TEYzY|fWzp}R+$CTDglX!=iUf}3Vk5+@P z9WN;OPS#}NvpV0$Rux7pQzG5Vx4@4|bEu!e`j${(Ct#AmXBAuO!CyN#woP&vkn1w; zY#}+k9j>!q?>B9O|9xH;Bv3XKkP-qh7z_uFWJ<=N-lA#p=;F}*7d$=)Ld(+y_iMA# zY16>2W5S*rNHLK2PrlO(gx6Vpd_QSa=pZp@6b;{0gj!DN>T5uKFB;2h_>b3ojy5nbTjBeQ?SAeCyYhF#5)ir!+0YWjl-t^c~C!B}k=gYP7|%*7a51 zgxu%+oo1FxLaiM9OjFtqmZ zoDL!MYN*DTe7Jz%Rt_)hmYhSnR2K9UB`miU2L7Ffb`kO+FHM+qLYHrCp1%e;(zDY^ z$)M-7_6RAYRg60u>5#-bv@VH9NWu=~Kd6xObGzs{O8vSFh;2=CPLK#N;Q|~JAHx$} zLHVwporv8RZ3r(fjP*Q?p^-vKEpV#1GO$a?@R9$SNqg%JK>251$a9yvk) zw>Kk0Fth4m-dZb!?MZb1pd>kteoFFCnPDgt&Brzo;Yeg?BR$l==2%pwDa-&7YBj*QV z7c^3MDk55Pn5hZ4Pwk%<;|RSrWXeZM`-KfaU2StJtkTlf;UNVU1PIG^uUq>grp?ty<1<#A0*-(3{9>DV4Q$JmZ!*)#B3X2IpsrEmHMBA$loC8n9wVHecuq?DHkO+Z9M$_ne6L_XTNFV zL&#p$s5}4?4#57K-FjV+G;3#qYESM6=lOF5F2@Fuq^Vjvn(>K=PNr7g@1D^$q{3Z} zf&t17UB(^?!q>*K)`~dcf&0TIS7&cztppWRwO@}Im+&XWKPwo8=NgQV$Y~Hl69Mre_zK6VW`1ZIHf0}AZx)zE~Y)!@15w^2B`D6%ehAO}@ z|H%o4Xa8@1!$Tf`F9(>?ssAR}b7VPe>j&h74y<>QR3qF2~ye8RipAEp@gr(1OVm|IZk z^w@qo_GVn7WcK5!g+ZXt*?`cJ>izt_JWiHx8rL^h*cb?NenaIcZGL}jQi7JVtBoBlr69&q$ZwL zB+#={@_{zOEkh3FW=%YF%N4lU2>uH^(QiD{H~^B8Q&Xd?Xok3{w!ICA6O=DsQsj-} z4y0HOPj1?W2}i~>$a$?ZISJRMvDmvA^gK^#`}~d{3=0r!LRvqSODbp>$A@fb6@_FL zik7lRZwxaKA_!ZVKGlI2wl_u7H=GQhM2SGP8SL=G$qy5)#`0X;glM&!-puUM?1qNyf+k7T2^90Ull_iluN^CrmwPRbnulV zYl@)1^RTAZ8pGF67=0XY#6pg;>BYz&;5jA#mmcswMkp>2+S9-kVABe@0k7C?#;3{d z>mwuoA@O_ZlfGhNdI5(bWws0tUlFXS0a{(%G1PEyaCx0wZJXs`t7|rECLFZ49IhA- zOu{}KWQ~Rq*LgoCb1y=c(q>pw2&aF9Rw9O7$;mZFOTq^8_8ps{cFhR6L?Q@(J@qDk_+ASCcB2;F@8dW@_q-^nueW0UZvjIu z?pMvyxk1273fWm%6!{F+AIR}yPQWnBJ9%9c6exq|yz@sl>jL5Q*x5tIHASQpU5;Fz zykfm5A$!1VK?5+G9%ibUYnX|PhOpCZg7lw}>i`(@mzs@koSdvu|=CCpSrVd5`X;`C}_IIa$J3|UV@xLrZsNqo%}elnsT$tbttnN!W&6aIUjHP1`%TOg;Iqzjtp&t z?yxOxU~_hor@hmN_KS=%Id`$~FUO>^_qQ__dIOu*LHwUDNv4BADOMEC@;{ElQV^l%&!f5)w+Mba!`ybV#RkcZamJbeE)bgOu>@ z#~B^;d(J=bUgx@;`N5gzvuo|O*Sc5S3>+M|tbdZhrn^LFtMGq|M zBE5lpiZ8~d5?}tuG~+Gtr^1A3B&4XJ9v^G|tuboeBu0^MI++aG0X-Cl{e$Pe0fHhS zPzQP#vVRw^enh#@xC&POVC+OIVCo-lE{C+a3HkE!GV|*YAjhMP5!?C5kmr@o{<6>~ zom0-qN{xeJSlN4h$i{UjH-3CUWc)`8iPm+U;3e(2M{jqPwdC-5?=r|s+@K=%$t>q~ z%*v?U(4Cfw=ELcEW6dgwcLBV3lh zQ6g@3V4bM=Au{&@yJ=mz!eZ!C3)_QW)^D*9YTL1d^^9yq87gX3LU6b?AeB<1g`awR z=3g8LnEKCSASMuQ*(D$l3kTIK>3V(+er@#R&US!{t{vGJ5Q2pN=+nJgN^#0R_y&80F_^m_w@cF}+waTG%Y^nfLAac9{uxvH$ zw4Lq!m>IF)B;q`+x(kG2CqKOMdg=`;=_(ze6^)PSWrVLdZu}cC|1*HXAg#UfySuLM;$J|o)s`T4T8|3B3erO{b?K_)e+$C zVG%$o7RP+a)}3c}vJEz6i5&g?Lp$P2trLQdpTi&UOKhRH`iiT@VX|WjzV+)(pI0Uw zWkwUs>$w=#rnDWE5Vpx6i{tw@m*uB_#0v+Cy(|=KMy&p+#9O-0of(~yLe~*$uDTud z#mD5=)g*;ZJnjCRiMxU7@BNmM3pjO91fjFDDoac;mBX{Gy=`Oy$P*ne9w(LYW>hMe zU2B2B(c`jWkmedQzmdKiBeb5lzx@PTelHhov)x%))T+=bjjiVp%_nqfx>o(;nZ~d5 zXzHJ(f#5kE_73(5l?rZitA#}u=EDp8nScW(oyrUia*`S1!v#ZuzdP2Z<>93c2eO|m zEa*$`ltstae~&=nypB@&rmk!~eea#dGr`d5MxskLRW%Sy$G_N%Vb|Bz-0-|FuQVm) z*pZ!HMI9GGZM4PZ0~3<;?`i=?Y;m9*9Xxp^@i=CKih@E)pxILxP=U@qE(l%n_&3mW z2O68WSsD*;XpYX#Qs-BBGgbhUgwh^Bs=b?fi@5dJ7mgRoU8mD_-43T6g4tSKxgXN* z#}qw$^so&dzln2LOtNW|*Q>RLDdMx>`ZMnr!h|+Hjcfr{1>uPZoxjmj0TDTL3Xk8D%FR?L=L?=mCt*i5SlCm<=w~|e*&xhe}gB$cPcTvk9dW<*h03jcxG?@ z-j0Q4bs>t{wV-1S%kXhpC#Wa=!Tof0ke_%^FDk-jfgs#rWG6#QmRRHnZXqNlQDC!;@()IgJRd^y98(M1xu}&CQZkgbkd-` z{z^OYdzHV7@f4yHNQ!=C?z}^Pf{h*FdEZGHR_4W6qPGtuCvXW48S{!?kT!=^z2WpA z$U6MT$sVvcih-uJzLCrEM1b4n#>aLOzv4a;ave*6 zDAkytRXW=t!vQWD#uM9r_pccNHH=p2T6rw3X^*xf`@xwAuH~ zR$iVWzB^t;;_GcvOo^*c+e7eF3{fPG2(ShFiJWj)WEAts>IKv28#P%eZbhQYi}1)^Do{s7aX3wh% z$@;tsJQe))T-<@$LG`CP;CDKE{?q6Oc7LpxJ7>F_3J1 zX}|xya=xL;qO9rUZduch1>!44>$kpN;zgskxOk&E35b$dtx2OmxK_YXj4H`2K>_0+ zU;47sLlKP82n{W-Ho^TGbj;n9H z$MlStb9!C;X!1%G@snAhX!@o1dTA-KO$(Rt@4O*@ZV0et-px&G|BN8e3cTm9~V$vv##-cK=eA{pt`EeHO zpRj+R!Hs-+GKlCFDzzii>5T$*s$~CEp`eiNRhA?I$w*S=$AQ{OhZ2H+y*?hE1jvM5 zx?a^}bfV%xx_eV_bE_vX(Leinmpy8Tnjf|63R}PkAlcpxmDA$=R8kW5jEZV?TH`r@ zJ*Bxr2Mmr7F|ELyG)pvys=9N6He9atA2v1fa+!^GrrGmJXR*BdrmEezIFcdREw_y} zISDGM1w*~`{Gby&i>8{7L0oFEGC_kH2LtW5;&}$ChZMH6s}}gtE$NEZ+WGA4V;cs< zy4PTZH;KaK{-fSAY8($cb$kyBWYjaA?a@CopU z6(<9g3+doFt6dYNp^9_ec$AD~N@Q(c=QBBEqgK?I&Iwu%l+rrhjjNvL9dC3K@eOJX z^V-5L8AA46Kj=AdMT>h$q%0lc!)2jN34$#&&o+6E;ouSf#UFVz9-%^>kMFK~-`Bgz zQ!-MnkF(oz>9DZ6I#mU^yyF>{n9DVGM+Yro2p*i+a$44+^3Icz&g<1gqr|yn7oS2*z1?>JJh)JYA*5gdWq*?H7GNnk)uf@q#Ey?2Y zPT_{qDON-r{kD&fd-0B2*&Cm_Qu9?^^}$@66_Q%_hNv)rd3GcGo+mS;r-K&A{IuSk zWi@cTsCK)owjSX8?qAL9*m2qNF&|KukfhVoW@Wv+|8JnbhR>aZ4Y;p4O--PKI__Kx zv`7K=jq74p@Hz2)9FyX=jxd32^DQiKRZEiU2g0@`x$6Nhq-_Mv9(nrY)DZhTz@%IG zX5ExS)cz2Y<+uT9yWIa1uWM+2+(2`V*R`@g{s6%)X*GGp*QiL-tk5|bzUsXm(sbVQ zDc*kOq5HpX1*ijU_u*?scJa^;1zw~hXeq-da|3};`&apFTVMb4&h?{%%H>qhm8RP| zS|2&uCkS}`ti)i08)#yi@ptGck_P_7>qT>&j;rALhp@tlQGDndPt6{u`C>3Ati)on z*f$MfG}2RBW-VHf|5%Sgx;UBPT(R`SFjcK7XhG} z27vee>nP#4#5RIOP5{48m$wkIM7_rH-9})*7>}TL;Z8ssT-}u<$FnI zPtZD}I*oNa|HKdK=%*T{L0RA0iUh@?Pok6x$FY`&+@UCit@Mc=MTm9RInllrwl<#* zcuuloM0Qx09;hl{4v_n91*u!eeWCMMoH;q$JF5{%^}P)BtD*#BB+0I}Hw8i$VJMug zHy6-lWPjeBhd*V~++1ote!%s#93+{S-!ntc0woT2pz~Ai2sb+~ zQ&TVApttJ*iW{r6b! zcmRiN$zUpoy01@MQIOC#rZpE5E+M3aI}_-@ zoEUwQ^wJ?9ewxiIL{X9}Pa-kHuu~_k%}>Av`GrJHF&vufK`RXmLN*4oE)4w8K+~S6 zI@-+A$I_2E+(2(3q4pCsupMJ{P|2B)$@Qt2(j^(-r?-9c+t{xq+QcoPbIgT_Or9sd z;sEj(JM7(b|JQp8Em7TS;)aXuHwtWEqk)8jQ_975{}Q$q_*U1lQMlH_3o0iu^#m z&iXyR{a25$>Ln1=$v8Yb40DOr(i0ZG&w0>}&S0p+TI6R0(F8m-FtaWakd6|`E#n3j z#5MN3X)s1i@nR`F4{I-T7+u~Bj%+m2Doxz5Kf~dAU6+n}!}Ef*iD+ZHj^$m*X)7Cn!K$Ik4Aut?OM)6M6Qy-7)bc07{ytiN8Kd z#wc1Nf#9Xl{3{9jIHGiqedbr5v5}{7;(lI9jp<7?PM6@MlS(sU(kJNOLy`8s=#yC- zSbRZrQtUJM?jina+BvHaJH-iG3t4R3bD?Vlnc3OG_zvEH(6W4KxfL)q z!i-}=y%x3r{W`ous=Mc`y*YlQRcs$5k zM=_UIxV2C?8&O_bR}X~>o=^3Gk}S<@AH{i)Y?XLF)*A1nR9p z%> zfv}{It{&<(>of~*m0h@2_V%!6hv)`&s=g>33m{VlCl#KBQ8O~-FErl#p|WNyk0r3b-Z51YF5Lj~jD!o53Ck?8`hPlZEZ+EGIA--Qb} zOJXkhU-fCeKkIXYNfQjd6T?mVSROr2r-5~k5%yW2`3Z`J_l} zH{&KRyEY!sI%8mB4i~A`Q_x$OWcuX@>EyWeN*^_bf0oj5BGFl5u?1VT)+@%-4R0|`L#T6a zxFs<cgp+g;QsbxR0a(tN&{ zq+Ea5+Sp4AjgAlLWSoA*Rw|V*0u?uQhHoze$`adAB}K1~m7H};1!i&jndt8?59%=3 zq>#NCP-Hk^4l{-SgqNIQY`JPitk;5BdwMXSmMT%U{gDEY=H{rCe;oqtH}X|r*(o?V zlR&D9|A?yGOp`>*TEWQmmY?Ci1Iw4-$?jMp~;og<-Av&EZG`>W9@FB8GEN{<>QA}vt2|bmRUZV zFNVG@;_Q$uw78KQ+Up#QjLj0xUw=cFKzc7I@X4H2@_IWc{Bi3N37T6&UDxv|y2p>^ zhDh(hoZ&waW?w1$dQ2uQOPua4BJh4+DP~dq$Zn3|1Nmx-nIWgprz$N0lg{_s%ZR=? zj$ZpKsSzd_R13Us>ypH=I@;l>N=0c)r;N+uWpnhAuV1(Tn4Dpn|Ab?ZUKhp~DO~ee z8hZ$H2;yD{Ja(LaAI1xsZXS8r?7^|I?uyE1acJld=y4cd|1>!vS`PHq6Cim){3Ns4 zC{E)t#)Vxnx}Q&#_Ax9Oe(0E5G@eQrBKCuOOD_r!W+xY#A;GhQFOu&$J`xNDKVqjm z7^e#>wNGFA;6i)ff&5+4iW5I$c_S?IqvU zzEi*l1od#41I2=JpH30o+yWsqc+z5%pxJ(5Z#zE)@Q+yyNyNxmC~Bs1Flj>f+e*Kn zsQUJS8#3uJic>dl%XS`6*zz-xvV&G(StN94raH32vn;daNrxl5yl(9z7m?^-$Vh1* ziFKVp5J&iE+M~Ow@`)TKa`C)dq7j03lyS2nI-A`x-tun30 z1ZebkzjU^^JGYJwUyr+m?|l9K10Rj(voT00dSC70AwD+Yy-r#tE)K7aVs`8RKjzG* zeehlcmC{$lLt9FbdeJ|EG=sz&yDfg6JngxzXu_8d?2lvxU#7iVV~^o}RQg~E**uiO zq;EBt)2ndl(?emrBuThO$svrEac-7L&l^t=iAgp4v-0_H-@OwMP~IO|w+{wtrL7o`&f|4Tz|0J-VTGuX6+9O{h;5dl<1a1($wF|5EB~? zMn~~{a}NfA%>il&)yQ_W)^MhfwaHKO>H)Sf6oLui#69=Y{|z}qjJ8Bb4l=6V1NvFe z0M0>$xM%qr@$_=H;7!;VB5GURjbvBD`DxB4kmF2LH>xrmNcwj9#R4vA>ktVS<}2u4 zft!ZC)~j$5L^&!A+Z<@%6dBSEWga0bz0(4t~L|( zT&P&T&42#(9T-i)J4?a^T}ut`(7$r$CF-dW&QRTiHne^0=eJ3#3pmN*&hSpM^O1FB z+~%dRyyNF|q;RbHu^~RJ@jTShEucSD0nwtJj03Mdk$KMEwHy`!lU&#B+rYaI+^Qk9 zo7CMP1%M*vd}0|R&mCn+y}D=b%7L=A{c5XgZ)x7qC-goYDl-wtMK6jD2j$c6`U(Tq znT9w34R8=7`wetU;gvM1RMI)0PD>jclI4|j$0LH@Z;CWYn>8lPTTb-f)!%uf^D*X3 z5?)?&O35Y@!-N-8<^$h6+%}}R@8=(Xb`YS|wC)8J3_Oxt=PP|vwVMbpuGecoNG?2O;=*n6NMp03vumVWWJ>9vvwC$pH1ecQQeeBXM5X|&_AcGjDiyOgvzsbr zZibX>oedP?p29*9%F?3^MrwZ``Z@ge{WX}!p%?d1@ibA=Bna!;Ko2>_)(H6(F_ju~ z50JZ$$1n}53j>NiDZ#O%MIb6!!v#sy=6sa3x2Xtu)w1mAzaLw1>6Su$g1`!5nz9K* zSL_TMdGSnTv08iR<)ie=_P@0?ivI`qtPW&Ml!(a4ws@ST_Y2i#DFvA5;~vtxeQwk+ zFrnw!NXEzo-}cnS7Uq&p_>RwzYL(~VIZeWq!b_IDwC67UuDR9tzkgOkflIiP{qjIF z59N`JE_2G6+Z6jWiHtt9r^uEyTd+tz)&^QKUHA=A-T*Z!7qjaEQhH6%d<4T*&-7W> zs{YiBmbui~42%J2+yY5-?iqOeD0+~XtodFB&0M6lT5N=+K&L}^>+lpoqjjwck*zU> z*vE&fxoLc2*s_=b7)-x|jv(Sn0>eb(%5l`kh&tyBtY|>E7qmX9j|Xb!)s|o-;_fs_ zS2teKtrWN^*0vQCuJ!K`Z_&7w<$(G|{(V%KG$9`HXHr-U2&)VWQZYXa8roEe`JNK` ziwg%n@ZZyFMEsm8CQcri^O-om4DJSr-?7D$OKGYZdKcuDtOKHxRtma zDx;O{zZom*k!d$cUo5|(9#}@h2yigGDA-p^-^oP>g zp1(r=h3fjOyy@Hp#2${*K1Xajh1t(Mew;y2fQAwK8Jj2-ebq78VXPHSp4ws%3@wtHdFUj?&6uu)_WVDw|w-w}KNS+WNTaPlM7&4LCEY_#y< z%Rl3_p&_q|;B+%&q-|Nk>RPwT8U%*7P{&FmzT-{Y`>hqYfmT@BOYZ3KpL#m%u=xVG zH5LY#ScZdjA8^ZdFCUPSSJFiP>II-;lbvJgLC`H#04jabc-=`jva+)DEiGkejdiC% zq9W6@*Bcwav&;ipJ|nR|qQ_ny>tVeogva+aHiA!>DiBZ809{8neRZVeNcZgkj?6BmTzOJ^oWFV@8NM1H zik1xrNstoobD!BsCNYH(2x*S@L9V2sp^Sj+X9{0t~r zI(t$(^V;`Q^pJS;906JH!xS>E@c*-StF#XkkX|Zu6ZM_#3Hwok4kO17 z#nM$VWlYrZFtoFr_{lLbvKC;Se=9Ff{(xEVwNN5+HU;_2ydKF$=Mc@XT-J!}WEfL4 z4L~9fuLPc*dEp&+`IL27|MEKTQy6iWMqc>A`%lJ#UF>4 zmcc~BTlNColV6}_f@)AvSlKR3R)SwMD|UPYt&OD}NW*&&+ljqHnj<2UQzw7bmHZ=# z75sOW{A_o9)`XXi96@08yU+ds#yKuJQKwN?A#uL{@$=#)dTs7)7!P|A6bZ&5h5n=; zY?F)zRW^_KiWnlDlUGkhCiObEw)y;N2H4D+wYf%Yg+^l z-TRKL?rm6NL_}=tlee>9T@GTa7*Y_0yBcgY-9LR#=pE%oW9u=#E;amg#4Rm2HA383 zAQD0>ZPAhH*Y(kt=;B`o5MuH*pny^7>j&zUgB_=XfRWSQk|1PSgFA!=<34DX zXYB|73D*!c?N^!z7_prh%Adf(P4Q;&0PatOh@NC{pX~Qo2z8=Y&AFS;C9LpVxlFFz;84m9$ zDD&e9*?uc~;MP6``CvBl6FCE-DIa)2g&OG(u9fbSd!m!qu4TJF9&EfROLyf=M1fgD z|B66?TiFsN1Kqn#fdm;DljW?uTRWb+k@?1@buBAf-7s5kVOwtXTR5mGC-y5SVW83> z?G8{kGP?GIhX8W_{^9YslZ^H#RU?uCidNRBsPR4MA*)T@6#oEAum`X75lky#^Sv}E zdXkc4I?H|AY{rXZo`nj-u0#oW*pK>Y8 zmY+S#ZxMfs*tfvOE+WZc7sqzFL*(~H_Je@i)hEkoxg`8D0^9#K4hNYBo@4vhjQ$*BFqjFKlW+%&t+t|n|4dd??WgY>k z0ixAAU5`A`^NPK_eg4as07e24D4~LTf1jOK!vgO$nqN54Xy0gvy1J9CDpc``cc^n zpgjIiCoAAE3n1RIk#W-woSiw|px>=0e-=izhn_Y0^|IUzLDM%i%>q5jHXJmbKUsIx}X9n5Z{Jmf3<>dS8eTL90rG{?bb0W}6w2@L=*Rc>t3E+ViushnElXZXhN% z*hqp#`-;%yixfF*Y-E}Wll<4|+-RTh#{?XM<}b=*AqKq6W!kFQ1fzx zf5domJBsy%X+FNmHhZVB(dEuK%{Oaj|6A0N^w2Qe0YgT7M@1^NW!fIdrO)o9Hd*4p zXTU#l7?+Su!CoX3a&+gfe90sl;7O2?MgClCSM*Zsxo7VrGUn>+S;$WO%!Z$-iTYG= z=-BvhoWW~quKzf*D3sjuBh%m$K11u9N;R$)X!shcYdgLrB$<6GI3U~R>&;Ww9I?8! zy(c+PBCM8x4nsS_J$F^-fU+{0;NKYkzKej}Xr6b9juLcOaO=h>lg{n3=9NylbgVj$K@ zrYSFa%=LJ-@>b%=0nsf^4huw-D}UvRfS78KQm71=&{X`gLEUQ>_W}p$RqVzO@jZiN z@jZFN&LoRRjfPWaguZSM?o+o8{P^aWAtaw@ha;(lUptMSPk5LI_JU% z7d;f;BS_rPxfONY9PQ&>p1UZ%4oX&4nu-%PF}lAyHJ9`5Lu3H*e@tL#n1pBH&z}bl zsPjC2mDj-|;d`|cd)MPB>>90#Ni}TS`1QA)b5{A0D%rv$=Y2Wbt814+9YkO9FYi12 z{^PzOMI}6wM$?spum-&?Y7Q9N+S&c4JKdyy(KlYW5NpF94?EY1%UTfQa^!DPAm$gX zn7w@WKjIJ=Q5v58^wFgPWI*4SgW^&55NGG0)M`5{No_?|PKK2EPy7b|Z+>Wn3%jiIaw9|AIfaa5`Tr_Sb%X z#vnW!ioGa+a7t4!dKi?;Pu_5G%zbvLc&Yb=t=jC}0z!N>E-I5%uB_8zv|FdHLk*@H zgy>fQk+4iOg$m=sA3N`{PFHZOX!Lg?m@AFv`$D)=i<6wCxQ%Aw`9T;)R z#$GiCHxH9OyrYs+6T_WR{;Dc1;nPw~UIK!rw|@If)%~k*W(?uEFgO`YxCT~(@VTQa zYNzkZq^w!f$KgUI=RK~CJI8I+RUszIOfNan34dQ&LMck1Xr5(ebdiINLXl3T)&Fw1 zRDYyOo;9LdY;0bH*>AL6(&#k({{CtCr@5KGe{BHybzyrK9r$&apvK8Go73S*?P>92 z?sA8~aqT$6qnRD0iRpDzmu;i;3xyaYnip=(jJGDACk`eb^@t+D>khPpq(=9l&FNr% z`A&w@rzJFxQj`S-3MM|Fxih7( z#3?1kfDYz|qp|+WwRc_@b|sNW(>=b(+nS&?kc7|so&QuQ4u13i} zwz$i;%VG0z`kL5iIUVik*st9`jW@o@zn85CeQT@ZBfxUA-cB1+LBoA$Y(A9T+6kw( z+P;qAD(`5apnaH0i>Cy`KL+@OwkKw$s~5=|iRic>GWE|C^y8ruDPQzk>gA*Ir1rtA z(J5-(APJc|7;md6S7sHp%${t`Pk+J>nzdZ#;w;p$5;8SF3A#1?_zEyQnd+4>2Q(a0 zRWrGj+;k&dU)i_ME~YlTH~0@Tld4S3$O%RVgto`~q&yhc>*k)t8vv=EQo;SGy{&a zol1Dbon&x2$XRqo=BoXk3%pxw)LNOu>ag~KyeQy-ACIH2tL_PR$}_&p3^R< zX$n`??Nc86YN=eQuPgjuOGh19$aG_@(&jJegj<1V{TV+=@D6x*!s_;`>(DjTH{(0& zRM9BZRkf^I8&AxJ;0<+tVk--^F`h_k*@*kS-w>fm18eqQYkv#TDPGWdiwD=EnWq=+ zf~Obv5lzOg#vXo2hj+9w&>@E*sm7(R$gVV1#4*4p?N)U>UysdYCduYxOtCkc8eeXXGW+R z7_U0we1A9iffT5j&l!S%ob;jr*3hs|2x)iCH-FAy3&~+%;K6?n9uV{zFna4wEIJfS zo`S|jVpI`j?_{cN=lziwC8^ju06epWD-9!UvFLYuHF{ZGc|>nU{#VvqVN4@ z+7L>Kf@_;Os;dg-BL{7TSyc66wu?G&dxtX6WbOhw&kjbJDb-FHhowXeWq;kU{&bJ$ zTc(a|kM_O)rUAw1N#KkK=Fkl{I%ZOFz>6L&0Tt?V+C3`^YvQyO-7lULjf68fHnzV$ zR4dC1oYh-*7li_%tA<~eV}keLsYgfe1T6p1y`(7DlEL{4^U^{ERe7n`j&>%i(0B;; z+xReCN-Sn_gO;(7_vE+s5d}w)e!NzI3I&6PF$u_;a9riu1`S^-Jru)fh_cIj=twnZ z!XrG9sG~n<*ICs#z1H|Z_xG>CcRUVJfc=5L%0mm50p}5w@AXliTShMvai>2OIn<|j z{82^4mmTF;XuQn=>*6GK<@)g}0Zf=~@mmM_A?}Xesg4|>_I-$W_dR5za?!csn@C)3 z42@AzCDIzYmp^T=b(#=g4g-bES)2?1Ih*v>Nx1XKf;*9ux1_Cw0uVOIwSo%yd38|7 zsIC#IS45$&HB*&mQ_1$Wp1+c#4zzdtdk>xogZF0j!y{I~)Nc5kn~+zG(KGqH&HTKf zibwK~wM%RUSk$fMraMlShZ>^4h$4Dw-#&(CZ^5s9Fmb1ZQk7ttaChnn=qy@7=y}Vd zVZ|oa8gpslCnk4i%q^NuRL%X$FLyeb!D_V0d45td_`)s zU71SROU3J#xhJluiE|iar%xEo`-6R21eW8TUHQDK!IuaT-cqSsQuN=8%~K2CcjX+l zRq~dZgX~^X%vyrd==dZ)FC!)_`zg+7)R5vKtoAM5IvAAao&;E*U;vp@W$v5oI!VAP zZ&s-M&CSkXV>mM_{24iEd}VtjVI%_{mc{EKN68qa+n}iu}N!1W`b|V!}!WR~@^o*tw7xj;XwC~;87nCjvp8aLSXvK2fDG0x(GGNAY$_6cb$IEoY z66m#j0TX&hQNy8V{ncS_q0{$WETT3uTE()bSC)a_d`6#6X%aPf@Q@%Ds5Z6CU+a&+ z(G*K4=!?k}46qAx)>j~~QO%9Ibpi`i$)r5>4f1WX6ii`Yxm0JwTdUs@tC?+bJK$UnwjC_qMD#O~*)rnO)7r?PDXSLn*M1 zXJaIaOlEZwPIKKm;ecG5w#n$zxsJTHN3ZM z8u^B7V>^SG|Yv;1y2LAR3d=#(b`f7kNxqGK?|>MWt2o3&_-$Bs!<*ZnO)3!z(b3~c`P+!TEBxGA2onV5!XCzgR|XX_pLs)GXtylNXhJy zr80}B(?8X>bDsBWx=rtN!lh9=e~(TfL|nXTT(*V!0v#DmyaV0_i+ zI0K*9TlTD59^f61&%pgewJgr><**c#kZ^r$r+#xWvD17Pxx#j5I*yG~)?8;eYs?emopCvbx-DYz|4Hg2D=pe=jZ}p~2^kUE0eH*XZqb)2nlfQXWN~TX%mDPP- z+VbBq11;L%t|X0eB{Dd5pQ#L%yOo(7qlR%sHuoh?=pqh3fpX*mAZST-9&Lw?7T%_v8;u1LUKJ7+U6-XucQ+s7MZCdncQtso$ekl#*?o~;kbTLw9DlWZiyJwm%d|awpZBk?V=HjYB=$7dS z{Cq|5-bu0X`#vZI*vmlFNR7qvcB&Urp1dwy(^Z*MBk0PdWWVybF;_!qbbS=GumTLi zSf+}3P<(uZuRSEc6;=p;?kmk+NYtZ})7xz#1I>i$tI8k_M#z)tUbZTu@N4?w|4V z0#B&9oXCEk$dbU3f1`bVxXMc&F81aq*?Ra@6=1*1R6Bz4aE8UCRyC3gO4{lwT4NTG%haS zV(NmH!%LX`%J&{T`N~GOGgn`Y=&XN_`D5Jkcb=5b(|7`|Ka4U7Rla)7b*}y92n!9` z2d?4r0L^J-Z0fY<60y*fzrw?C%&&7DwbA4zRWdZXKXQvFCv4{({@+T77uJ5H&2N~W2_Mf(YNsG(!dxJYSw+u7>DM0 zzN1G{U1B3RHSwx?r3w?^l#OJ)OR$m{xJAwmZZgjsbO3;jiYD&Rh(R*O>LHO_ zOMHlZAHx1UY1yU5SqNIH5#2V^!DIN_^GoH0Vm63l(yDF5hT*_C_E&1-M>JJaMrShE z(Cxe@p=^s7UlaDnvY}uCz_v*z5?GOhp#d9Cf6uj{9hUubkh(r%bdiJ$=IX1^FEl&r zsdq=xeZQ9h;6mu{iva~g3&V=`Q@xA*9Q zBs-h_G=s5*e7=cXqz&6G1o5ZeLjggj-kJ7|c|%NXGX(1#)<9?iqj8fiWv$CkJH`Uf zm3n#k9Tl2(O%9V&{;>8T-a7=iHtqNzp`dXc!F;MT_+d3FwQblgi@Nlo!)JCwPXs&w zFh{HY9wA*f>E3!{8OkGPa#OT6g3k>U5e-!jj$FnK#Oowg?_ za~6mCA;z{V$KS>_4yE(|-y{+ae}Uc(E~wjrds%yLBg2Q!s{ zl_veP(JzP@e_Jmlgd;3&r1IOQLx?GG3xGD1TZCce(cvzXFX1ss+n@}*K|Kg&# zsIdPR1Jx6c#0wjr-E`~|YPGNIE!YBr6s?c|XNiC&tB}G&uy;J((6-^?n$HOYVU*>o z>G!Vo$|Nq|Hf!bMKX{e!g7B~P>kxvQ_W;3z8iTfkR=>4Oe~L}H#JZ5?YRSxX|7^Zf z%%t@=vx#6JvRTj59{xzR-T9ch->K8jPMJp7Wph(<~rGG>?++&Xo6XP_L8m z2$EtX5AnRA{u2HIA|V1a)+Ok{!;u!c;L~{X*dnUlORNWpF_H z9G7{D<-UQc4bE2knv0J;g2Y>bPhgMhQ6Imp4xv$fgUM_lJrX@V9bbdlZ~50n?NI<% z1ncqZTAe~#I28^q+YlEftJ?g=(yxKKlnBl%Gu1-5RfX!`$BS(JfFprzt)}#bzd8Nw zX#oHLZhed5F%Z^O8MH@Xp$oI;;f?!WuF850$_b-~a$OL&utzu7$3N=q*K9b1ekn8t z(o*QQSEUoB&H!-Z+dCuW2tjII6n8OyQ7_BN@Ce1^jr>51_X*Ar#^WLEi#k`-S*fE( zKWyNy+3tIUDhcw4-2OF0X0(Bk8xHGRy#?wYOfZ?SO4AamA@2_?t;%zL-XJTaDu$XeSg5FKj#!h z46cBTXn->$9P-C3d3Yg9)YQ5Sj{Gwu7|4?#N=9Y{0Cs;(0s#$N02Ca?zvd34FcYZi zoDUJiDSr$C8B;0@EX;HAPax~(cb^G@X=i@GIKPF-|6{y*cmkjWqy6)3*y$hL_D8hi z&oyB{LO+Box_`nYU?dMwNYIUvRFLZr)eKo(Dj`@aY$-kZpPK{b{WbthTa=RW@DGpq z=jvd=@lyB)$NjZA;A=7veCYoj5s<830jd9YME^g!Mfz%#ScNOjploS8>vhs5%SX4H z>u~_biO%C5hWxEx1R+YWq-L!zP74rk2O>IylKxs|&_ZYu;s!oev;SzbHBQq~S+4)L zmO%--)31hhf!`O;+*?$)`7D9Mp^S2WsNT+OMmNF47*v3;==5OJANEk0Twk5T7bbQ5 zk9Ko^oVw;0<9~h-!oFJq7}Dy=Xk*k}YjbM@hjGjr-o2Q&Pe*j-IA4jhoi@m>{eJHE z*X)7a11EJv0yQBRh&gq3usjzb$^GN_?N@%Y2Y76wzQ_c-T)2I)YSeRt9vnPqNr3RH_thY%(0n8$EiOT%g`Gk4dc%)}E$F_V-Ao@Jwd&vk>fc z=CBo!_zZR2|Gd_>rvC%zOV7}l)aBIUh5Y$x88l|VUt7uZ02+9RPtQ=jKu6)l%X5WN zO~KDLK!$$7JdyTkKf#qq1*YD=UjjlF58eKVtvIiEg3?DLYq00;LRDXdYp_JKUnj8) z#5R{tPb+N!u1{z&-SEF(p0~a{mhYl^Ca(o7H@qHNt@U+Ju4r$L=H{N9ZjMo%B$EIA z145K0p!3fpOuE8|ax+74UeKP*ayy)_#>|ZXHtVrglBTl(hreeCl9v8GIT_G^rO<5Q z)9wlP&AH^|*>dRIQ83Pn>AT1r8q1xbcE!TM*irF*|2u40kUNg8cB?o1U9S*&@*F)` z)tbnF$Myh&*tys$ez8!S8{Z1}kG?o_6Osr8l~@gMPDE@EW}X#l!2i93w^hyY%tk@+ z1n^OdUt7F(mICQ)Tb|mm)!83t0$7=VAYC|=Y(G4VJrI%va$9Z*Ro&wWHi+RHpQ0Z*8NO!HXm*c{=~=q{v7g^23;@?Sje`E+ z|9s$W0odmN8a9T%l?LDo1)C`#t%?1|+#sLJQ2|5qBsO-yTN`g=17 zrAi?$QPujN@t2>5gaE2W&J2a9cqAGmj0SE&PowEbRxnUc8C+B`a5zVNb_ApC| z2n)iv%c%Z(l+FYRA3%pynGU>bs+=i zni05Pm)r~iYysVPDWH!V#H3R*?v0_V1`r>p!!t4}1l=sg8=dTMSd3$XpU}6m+3l*B zUCT5Oh8|EX`=`T97N@%*PwJ&9ug0Tws;~`lx6#+hR#DnmeanndE3;d0nb7@`xbek7 zu2?N8zCku$ZkIz3P%%~jNNp;W*Q-FWd{{1xEHrnCViP_@@E}w7GrgenC^~N7Qm>J& z-Ha-2QL;JlB zox}Vq&y4Mi3Fu>Cn#E2ang>;;KST@(!Zd!LcgmvsFpQ;__|6aCF~=!+h5PVHC~e5g zGWBoY@ibDp2dHL7U+PA7@44U}9rZAs09<){Sd6X`RD9Hh%BIU!vN?X=H34OGcFap} z+xSO7+t>lm1m7ZEf3n?);uKZa^VK3U<1C%d6MTmtVKiafsp6wcq_?eH=r)1Yg`(v# zIZ6`7b>IbSF)8)L|CZwR$!2quGTYrY2*d#TKZoEM)!mg1_=$^8 zzHan$eq53_tBEL`i5$QFx_8cEsr^0e{>KDPz&lzMjLXXK?R1@=Ik|@QYWE&yNB(e)d_ANBFVa$xJi0qQelHCkN8p=BMow0qN z)BAkR@0|1g_xIoTdtGz6x|m+eb9-#}eLv7ykA_Ojr5w91$?mVTUU^gW`gFU!n19yH z7tcyHvEIiw)jif5WZbLy36%p<#y1J3@3Qkx4wYHEQgVl`ZEQR%d8t*d!#zFn+g9@< zRmGGvonvpdwJ`@*;H_M{;Rn#3xi3@QIr`7hZwmIuD=tkJ9s`M+giDeq88kvOs zY2yrDkoq$o&J@S8)AzZ4xv3rr@|^Apn;(*VZp5$B{&`_9a(ruE7$JB+=f~~gAH?fB ziDT%Fpo+&6D#Hb8s9Fi*KU}Zy`D*l|@Z!>+mV-t5Ou;OB-fNeb1ma|*xIYG2+r9`y zH)-URKfY;Pb{HTtw9UUk07>Yv5*%ZWKHDw3kSw`5Z;sOEa??h)e-wQYtJhV?8Gjvb zy5-{8<(`b^JE@qIR60JjweVw*WyiqiBH`G}Oo@%092-fcB%Mhy4(qF)k2FIL>YC2( z=>*2B*iPKTSRL#>E?I`n3K4%C6+n~MEbd}^{89wgUYqqlL|$quk?;!)vA3}@=ld!q zr2ByLZz$)*TB6tZ zGsQa!7XOvZhS}Z@`~K(d$7Oy4EwKZCZKT$Br|K@mmhbF}M7?hE;lqvDel+K?*vYj& zoJZd`z7P?^)?+(cxC{Br;i*DqbtBgSGocbx^hQa9V&spNiB{cKUOpr^rasC12AaA~ zAcuGW8SxDb))wUu%+gbz*}fLF0mO{`Y+s&x%4C9o+dDbJ7RFiYvvU(gL5w{Dbpul~vl+c(8#3P0 zI-qq%PO(awC;EJIMz%(*Da`$)+*hL;|7}uU*vnC^-i%nbmy4?ts(%D_T0^0A{gWjX zfoz_(4=mvDHs+xHY5a)OHf zR7GxeV)QHEl^E2odeL9H*lcoBnQB42BLG;aBv0X}!irvQBRWtldm@dk9)TA>H2r9J zMb{4W8!cPDOc?S2rV=%U8ZqrsFmmI`^n1p#&#&8dCS$m_-sD;?Gb<0_m;j;036;*O{*tV@2aSa*_IygkfA56sK#{DFK+X&!$q%7Jxjsk11{fB^#wh z$AR@T+Fly7ZgU52)ngWY|`s6 za9F{4sDJ!$BBfgk*a#$)^4=z$4FWn_H@^sSm8u4o&jtJjA;;skWlixf5{Zy7lUPeh z1}%MNK0P1QoHN^))f_mA-9C&x*&>t&5h!Ip1;6}qL>x+RpRSD5ASB;RDa*-&Lj2OH zOoc1AAHHBm;9)$Ce!N_P&G>WWO|G4lYxBvO7+k9tgVrrLp8tx3vjKE{W<9Z&3#@I6 z1#!zv?p?-~cy5Hy(;vj6!)@#MX_)jxUKGde1A)1a;*Zba54lrWY6rj64HYKGJsxP$ zLgD%6a&b>;>3aBUa{YYX?tx01i_;rV^{3T$v*$lItNczWcBA_xHZJLT9kg;Do(=lOo&&-vImvqYXI1nCucOU8y^yxTeV$7u@N^NW9!T=Ap7c(c7u0ZHEr z>VgMz)2Ti6!Jy@ddAf*oMGx~f&4T!smleG%GiL2SL9Pu}yyoZg!B%l2f|vbKYvPbJG=&!{L?lN<~t)Q%5z!J9E;d=J_CEvnw@KODhuS<9%ecgtwHn4E5L|5{< zFxWUfx}en<+wXuOkx3VTX}u>o+|QK9=w4Pg6Uy7!RBc?#!Xm#69r?J&t)3OZ~s$TMmGOgy7RUJb}E`p;g7-v>NO2 z9Vi_wgxx<}H&i@->y^fvKwb`wt&HF7LOks~3Pu6uuBUN4tCPw4Lzb_5_-vj|P_ypK z%$HQM6?z-xef7gBs@7{+l#JlYG!cCdWzk4@(uq{Py{O-@{Y<8xx_2*@k_w8WL> zHq4%iqYxNnL*NJ z@{gxB1x~od{;72z*(aGbYf&p0&zsIKfZFK1hyDy408qVW?gCX5Z$HCWItHEGU6JN} zFEdN&xQ0oKQ+qlhNavh)y?4E%y#>QKJ>jMu6&v9=PE%saR6(*{r~fTBamQBxR%BWB z<7NDJp|B?KPd;2;j5`neHJ`o#NZBWdF{vEz`iX$0IRZ!1WNDCS?7jFu>Tyj9t0(moExy|k2=0n0SzZv%QVC212 z$hlju^eJ3$-Jx_+e$8|);5pmsT6*Ta7&|nf`GkJ0X);cE4?SsqgVs7G7%x2^uZHCOO$Sir# z`E&~eru7K0XDwJrW~$v-570f+OvtR;Ac-*u{5FvfMm-qWZZGzoKs}Z`7Sl<2s+TK>9 zT{-)cd`lu>YekIrTAy?#FG=xdmN#>UbCRGIlKdqC6 zdIK#?TbHq{Ibh7CyL?Y)1WT@4(+!#-KZ>b$9tE-pO$r%$qx+)T0Es~Id|u)7&j)u2Zo!rBpada2)b)&8*M#`YOu^-#d8Ut`Fto107_d zXMk4p@~^?u;hA))zb8MJ-rCUTWA}m?8eK(x#xm6C<+lcpKB|n(`bu+VGeZxz->}qr z^01KROl$Fr^SAIDhXrDC9&dVoIQ{N_nDeqN5$uCDEt#GjHWXQSE0oEtvFQYfsHJOy zaL@ZQWqY{B`fNOu=66eny#UU7zV{j4Zx8I>-;gs5oVNSk&ENA>qtJ1RptaYw2jAAL z$8*dyTxR#!E4bxDtwUb@fwoJfqnu7Y$pUX!&aATL!_$&~?SCER3$#JY%}HAc5JOTG zeRaqUI)3&!i7Klv1JO@g1JXf$&h{OzL?fd*8+3}2k5T*x3GE*{Ic9Cv{qn0`-FM>w zWm5t_5%iDueULwLK})o0C;u5<5&4#1>mZA`|CLZTteM#;zFubB8*}nqy zy^F8L^AX-a^$$O${>p)peZDjewGfAMVT&&@P~F)54N|Fm$E1Gddqxj7n;em!7xoQe z&r+W?HrePDD%aUpk$*!Y&u8~qB%l1#(l@!+mWFWd`ZPR-% zkau)drzMEpEI{`-;Z-BB?gtAze)v_;S+Bf$IbcS%tRs?Y!R%zW8o+C&8S4%tJ>+iF z!W-X(&e)SUTls>G1TDK6H&J;h8{o`Q?RHGXzvq5~Kgxh_^J7h2{E3LeuYhF2iwwxs zLY7P?YazgBzU!T&gnJG-9seEUEMCdv8lk}{)a<6hyg z01{Z$W(?$-qEw(QjIjmzY(fz8KI9eRWzso~tBE+}1mi=8DW(e2Rb$qv=;@;+-)3+2 zPEx+L-|*>_wdLgexlGfJobTk`-$D46Z_>$i9~J+j1#nOc`c!jKpTVD(~D$1nx=KCnfY{!Wo_U&#ddL22nbBK+PHt*A8_g_fmsU zXKYC1z;%~ow$_=W8wBTgqse=_z;9k~e^Vi>qR4hsr|6f9@RUD2XT z*>PK8#WQppb|aCNp} zf7HA`C0bAKi$E8iC>JRR22NyA{`_Uu^Q~ERfAYuE!q{-aWWxP)f{{?J#1WmmE?+-M@?oo76|+(dNohn@ChaGH>dKrtk^IHRDRPmahfOdpdHT zD91Wn01Sa;+McI++XHH;6d$>bZiu*BEj$UA>=xVf+Xx#M5fpd`YhjDc|DBaF`^l!Y z3U>#S1W_3BY)k>m?=qR~6J*#hmW>+nu*jQUvXUTXvAk=a*dvWEiVph#<)xN2E;gU3 zoP$=m;@>Um2a}ChSzx!+yUh@Pz6=i>fW%Xq{M4KJ(q_w~*&n#?H+v7D+vA*1ZWWk8|b2nAg{R`07|pde}NI z<)oK~1oB zGjA5s2JkmSc4j}al)cN1v7@$Sd7hXRcA7k4w8(_~xPwb$%NLmoPhZ1?Hx8j>&(ztO z*%-4M``+2eK9op`hVNtUgrQ+csLV{~XGwaUDF4vmj+9KvDKc`v2>=p3`2>F%Dn~tX zi%ssVw|er+&uJ9sj*Xqp)?+bVZy#1Ld`~pX(-UOMNGoQXVHr1Eyb&mEl0Bi6sHaYN znJ?;u)D$p4^+#*HBhJ=?SRFgL_AEY-SFzm6sIqr*;N_NiEjX*i%DmYK4fy4F&PI{J zp*dfJwoB}``?{=1S4?oMA>Y*Ax0YCDE@C01t+D4Gn{*0#r$wEV!f%@Y9ov^fW^c@`GruG5W@kLw| zgma)aK4%&<@_zCgEhrwCgOAUEGy(YZOd_4vt1HqyV`mI!ig6Pq-gLod%yPBk$lY3y zcfFtax)hjm98|Rp?cS*~!`4yP1r>U^b7`A`0uWL6LYasz`e>Id^0PWwRQPMXOrQSZ z6UF)v3>F&~c7~2uAjBIF(;&?Ibn9nLqSo>RbIY5^IN3fS(Zh#o^8~fGRJ6Fuxk|~5 zU4?R3!^fI@H^^nX9N)eDZr*$Qk>lBdnDooYo^@R(ukPeYClDK7ObENC#U>9u9`RL& zju%T;u~rZg74w3IFLKS~ER=7fZoV+r2@7`2>Tj@DuAoIhI?0;5d3Y+|4eXasUUBAQ zx=-p*92T2(k#yCjIW;!J=Q>4Ku0GMlC#<|+x})y9M@fNPhmXGjmr>|tYULDZr2cG} zehVYP`2~r6<_V_3 zKiqWyl~`bE^+lidlBws!sjROG%)z_BqjSO~!>K*+68U0OnHkj}tCg(fxr{Z<$-FG^ z0$F@Y+tU@>Qwv-?){Q#z^HaRq_WYP1ad3Zd8V5P!AMs}4Nh%Ui8Rl_qqyrkI3W`aEA2h8_gIfT*_l6(trN zGB+;Uc(d1x2_LG&IuE%u8&s6n?^Nn$@Cnhr?I7fuiozM-m`-&PKVPnYbWg!cOrr0+YL!emlvvYz+29&xaQ374BP z*)>j``x7VIV~ziUj1NdFTH90=bIOdb=qY%x)B~@s7=mKqU$7;FD81Sg#I*2_{W9_< zjQJC@o=n~&k^GGVqoaExW+~-gH8Zwcn|o8{gOJ~#tl5GoG7leWTzoC(>oxcT~@Iuj6*3~&y?ZNL@6v;j`+x8#<8wiU}_uI z=4@?-O@sLxE{Y(!ie28@V&gd_g^d_y3?&@2d-YYzPX|eGg-`7Hz#6zAEa&0=*1YtX&s%yDb44jvwt`I1e; zA)DrvHZ6g9mFkt+nQU*qFdEK)$G@8=VCyqkVx(y$tWi~3RoETUd#mQ+sq_0Q4#jI* z4ogIBGAz)3C&+-s*nnm?>Z;d^y>CyNl6|so3)@O>`-l`zzx$E2bhh|n?4ga{)CJ#U zeXdTPvFonY`;-1T6+LA}0uH6un#n%>wXr?*K8MR1i?DSeYu4x6PLg@UwdOKs)2#Dq z^nFSTLr=2!^xMn096N)QjZ7}H^%223Zp^&#*84$F;@UYUBAhLI{9t^3yil_OGI@rN zeKK%d6((OLhJTr|)ak-erHgM_dL`aN+nd(zz4!{Wsnvz^-&SitJT}3b$|7-tq*{(D zEC@M+F`zw0KB!BPp0f5Gr(`T%>&0m&vNj*Ca5=CK@h|z+rdL(# zRq^?Bl7%ag7-d z^Gjle3A4A!2jBRxe|&O${%9yq8uC)VM3kYHvxb5v6E!T3a;5Iu^CFqx5?48Q`XbOv z8^xkhlB?zuo_{RckKTl&WM&-xgMhYCcCMJumutQ-t;wCmeGwav%RAmQlnahH_(TAGKtJzX< zGjtU@$D$zb@z$(4M4)=##{THS9r?xggI1+1>;Z_x0zn*P2hI{#kc_)dYBPhHC?XOF z@nIrLn?)0A!2auLCb|Dga3>yEbH(IphSR%tr)mv7RiVO*NDNQYJSD#f|E0Y=DgPEiNeCM+Mh2&%zI_Yn2Aa zorf|{O4X80@tu23(B}0*O~|Qf<{i!6}zkd#K-6S>57 zO_${l7=QW<>Xu2R$EC9=i*$h1fU-R8P@2_txDB&h45uNx9fBljVjZk-o<`CPuWLiV z7`Smuio?9KMQN-Vxij{+VUmiUpjGaOAk6%gF>WKiT?I-pZRw5fAjTx5Y_OB(+&+JL z5)s9b-LOu~N68nrjz@ zOAx@ym%-o7ju%dRTWZ|4@%^a1_OUqGkW-jIM!Z8;cSom?4nV)|L4pAc5jfUAwDPcN z(fg@oQUF+Cl=ncoYG1H3V3T|LxY5Frjzf^vAEe6bj^&3rKyLA+`kW=?jKmagy>80E z73NC~vxOPVn++T;p5XK^C9+K!vECwU-`-I+*KXJ`24fOOZBqaCO>*4*x)+0Z~7KquOSRa zx*uXh8F&IYFQg1_zAEGM(Mx%_Chc_4ZPhG7oC=9F)l-a{^;h5Pik1E~OWs^tPieKU z^lCrp$1i%^qqiznz;A9;Fd_n$^{#%k-H!81ifTAPI2+Mmg)b9v8t)RY;w>zCYlAIyeg zt$ydVxG6!qsa{{>Kj6UP@MKkOy4e~Z6o@-FPc(}i{;Em$HE`aN=+zyE*fyJC*CGoO zbgxl68`Dc21c+lB;Eh_lr^5He9LXDV?uXn*u%XpiQx)^Qc^+!217Drcv)1K1^QR>F zRzsOYQ6ulPf`X2M&c`<`D-+5X_&D#Gu%5^;#Fm{FFO?mBc2d!6PGX{W|G4R5pU?I; zKBCC?;kuROd03*B8h>h1gKWuEaX2TnxQ|8$adlu%L0~p_+k{}~3lAKiPCfF3l2v++ z_?^cxAl!y$@3t@zX!|>gM3fLwjVy`}f2qc&Anu{w&ZHU+{iyCSi|W~sC&(Qhn;zA4 zvebLGyti5GhN$@wLiLVRLf{)~)YCtFj+YM9K9VaNPC>{er5X&;<;HW!kGQ>!bD`9O zL5qXn=#829)K%vxo9Il|{jCqp2uX-fmr@l$?3*Uc=r_By(@w-ZHuS+BFkEG#u zg1765!io8{BW`B4fmMMsvP927J^uE>`W66})5e)vb=)@k7(H2h0QC)P-fmp3?P0FE zv^iXmDUCb8jav_N(1aTAapz3Rp3rZm?RL5hD>Bhl?}Xq?x8?-%N)B|VHW))C@fGe0 z0s?l(?0}ETz*6uLS4jZ>HUUTNZ0`Pf6BKisXEPm@vLSgNjhOL|AJ-5;^6qCWaSI4^ z7Co^de-e6;D>$o#Ynn~|WmYAOXh6Z|Aon+XU++tJDw0HEmw5Tx^LTI0L1_#LJAD#| zQD5}`Fj+y5>}HoDo1Y;Uqm=yytS_4&jNGmeG-1cM8{rwf&(4FfU&Vou57IW7hx-25 z&tVEGSQZNMroSOM^RvP9Ty=g(zrmn{~tE01&3%hJn5FMu7y7-q> z#aQyx3(33os7;T#o-u_)zJ>5kMIRfm^@Qi7Mja_3>p~KonzDT9M<+5w5FQUKYmGV z8j(MPxk(VtT^4x*|>_eq6zoZ8Sdm8bxq9o2$>yK!E@r931yrT#JwiHd*W&V ze|p3ntQ(xc0$0Ci7@|rHKKK=N^j>s!k|^!DdIJY3k2`x`Ee^-aqfK+p@MQ;pYU}wsA5ER`1C3;#P0uEzZGZk}JZIdfdU%ZLER!fzRNy4BS zUcz`UoyrHw=`=80dZ5&h^-Jo&R|Yy z3N)p!8QAiZTy^)}cUrVvnX=>>oQ+I{=;^)W@Q6>^@pzQ*qR(-WI1s31yu zs@t!H{FwFtV_9m(>I^wf*!jK&=bzJ9eAmX|oh(4Nkkun0VQZCRG!&t2YGJPNW1uM8 zzjKp4H#~dkcADVaZx~+@WPJW3=h7QJzT!HgeE7GKBE-xe3@Xk`w_Ph$u@j;&=OsG7 zQ~lmEtssjEr#rQG)4+%Wc}rS7za-gc_)FWtHgQ?qbhX=7!nn4Fpq0v^)aOwdMnCoB zgLHCeoEXSSDn+^H?<&=y3GXxxQux1@=Dvyxo|C4zEnLa63>(@NO7}LOf1@t#l|N)d zcu(Tg(7kOkD0KBwhb`~XosEORTb~BQdx9km!ok>d!*Xqv&x6cm7VfG&U7a5HPGWF| zIL&|Hod#l5q%3XUNFw3DYGsRg@$l=I*H%`vqDDvCAr z2apWI@-?nX`*IH4t&!K!`bzCKhhAdt1E~(Fx{pmWM@$g@d0eG~w zC+8r^XQJRbGl0Q(`L2~PzHWueqmDMqgAu;DTK$YUa*><#!G{OS!Ngbv7Q|^L`Hh4* zedrz;VBlsTg)xxe>i~SEf+zK&;vkX|Z-Xia!#V3_2_8?;1;;S(Fz1K01{C!9thO=w4(bue@|*~U%*Ur=GN;hivo z<7N@PEraAUfSr-HSiiV-N#ES{Bz3HRVL|f4)DKIHg5#@{F0Yb0#$#SDAiMuT=2?+R-N=|NqKl99!hezsNS;Xkqnjr+UeHJw2OoB za%(Rd0q4N|sedd6Wx*Aj+HUXT*Q{SWtsz5ER5>V{-?nOTi+wOV2<1aQG_&%IaAAuT z!6fcbu#z@o*Ht5H`lC34_dTWGtOBI-m}VlIupz30-ShLgrF+N?&pX4Wi2#ed8l?V@ zf{l`v(&Ais?Q|x7-YYyRB;Nyzc0@i6)lz({bP6>SU#n3g)(Xg0SA*5;jv zE8A8%>9+ws?mfcSfJ=ZMoAK>K)28t7lYP*>wOa9TH6ANi9mN?`b0w^&91+F-F zE)mbh;%hS$I>qme`ldlW+1eT0?cj+B))CNmN@#nuJo9+$h5c*syQ$U7mfFYX9~;`K zLLJ^T5lK%=4~{_HU5ENZ>-yJ&H*#Nv;UevMN>Opm`f{t!8$tAq?h)nF&73^+X(hdF zW%O=C#BK~;g6?v~cYKr94!TfsJQOW3uCMjYZ@$A`t`Mas9LSI(-XqgWt?>S#wG6hG z33-h(cz@O1fplvbfURolY2%z0`_oi7D2UwDs~0p4e*oMO(I=FtyAqreb&RX~`x@+s zlwcCahaZkteB17c?n9q|I4?k%<4-G|I-cwbGepdRW(S{BHv!<&nt|g8vt}~jEPB{@ zzcsRO8o|lC5iiGk^2*I28U_42EK|Vq;px21`q{-QSl(p;wL;&b0b2wlPoL5+?HXbAUT%-Ts8H2Y~b z&C=b|mxCg#Gpx~mk(qDfl;10wAz7ULV4ASykR^r+fMXHq&xkqvQG6?WggM`itUKMV{kb+>gBK-#EcQ3ZdtkMu4ke0Ce)*o!$^<=Xgb}f&F)yPIr*R2Ma(3;`Ey)8sM-D5|srms0YGf8@rVlMY90kpH;+h+`O zN%l#Ji^R>;MtC^pNaSFyp_^EvrJ!M17AoiNa=uL#zyCrt(E-RODygF8LU7fGko%^D z4*uZ61(swvM}Y>pUqi?f!Pnm`eul;jvQOk3EI17T0g2)C5UOiXTiU(uYw>b9f9RIRWRbo{;&{$~aj6LsnQ_rHCjs;_yBUiUq9#R!% zRyW#2Ftx1B^y3WNfWdi4eWuF20N_2^Sajbbnez4%-QR|FKPGZ%)FlSvT^iw@WC=jJ zI(iS|?67ax2X8!_TzK6;A84UEArWcwa@z78^E5-Y*9ci}&)Ful{a!Vx1lZE8$G%C~ z0c+4vfB>u%F*Yd@tzWq9n=9_iQ^HeSRkJDMd9ZCf z+-ae>AMN{|HCrSOtWL%+!s&VRXaMRN!nq&1+q6w=BXdMF76Fjr*8{aWJ5xi%w4repZ@eXo@=Xu^x7?!?aT119 z%%LsbFIz9=8&6`}zP5_*0iwZV{Pv(Nnh3W(TReETaO1wwdarV;cS2iaY`$Jzk1(O( z9x8x8H?IOut1yz2YHVp%B_m0SW(qiS>DEVMOYuS9>m>j6j5xVz)P?2uxOV_16g-OX zy0V<~$CAzO=FbZC%2SKqPo1Lqb{bb*n(XygsY>aP%BV;W;rr6(#??Z$_j=X%9%CrG z$1YlYxBL$I>Lg&NlnE5+W=-oQm<+)>7l$l85GKX(0Dk<6op0L0EH4Mx<*EGMq7hEe z+M)hRH`H68FAxA7)oqndqq*Zdt`$qQ1I**I4@Wt^zx;UW%@ccvz$;bZUpWJg#1c9p zlNZNyA$FzUoEb}>X&=;g^vXKTTkC*QBWxav)Y)wZ)Qzq}<|8N4 zV8pn_e%Vc+nD>{Ul=&iL>0t~{6*b-@l(cZlhl;I=ZXRLkwhe9{@*08 zq|^L6$~l}oe;n2RMKkKV1ZYNVsz}7YXoyN3;DRZyGB5m_Fl`S=OU`EyL;t21^H78{ zZo^Ig&E}514j9g2cyo%<>;ER!rch)qIxyzv{`D7$NWiuFYT$(YzyIZ$!EL~(;z9lY z?}a2hUZqz2M>7F)H55>Lz5{|q3D{uOKxRk^{`H%HXe?bzr#RipL3{hBe`xce$6A=0 zPW|(ADfjFExvJWItWML_ZKxyzG23^2;5}4#Vr|eq=t`MPYRe0fD>r!8Z8`pg77UdV9|88=q*R z_?Ra15P|SxcP7UJ*uk+IG=b!ecOVKQ#z9CJb^7NN3S|@MUpcxq_3cBuEoew(eHeFV z`jRWL>EL3H^w1T;#)0!q)gelRV_bYJFv!@oqXLLhg6VV+%AMw_3tu`K5GN6b?aM4K z&aFq-0$f<^7U+uEn;m?(yE6JxQS?hSPt!Rbi%@vr&c~=ub4dk2CL0@Z>dkiB{^=cm z0MPpqgeK=#1>;dn0hnqHpkHPrsX$?$0T$ZZ0|1r1t!wrOUb5`^Zr3r0vMWM6&*e`4 zBdPe8X?H$Uaesnov-)9wz8ZjK%+ziNfimSNzzD#W*Hvt~0NB|lwC?f5ClyE0r`d*& zEL$>!{#@?8Gjny+qxry!(W&Q+3*1H`X#eTnEGmBCC13ViU<#^X-sMUo3VREG*4{qs zIA6csBfB`2Zd*M!SXAEz(2;L3`;tT3RTv#{ampfAQaf$1j>?aY>H23^*#QM_tE%HZ z!$B=!)b1Hmc2{PqBR**8>afsu7MXtrtcn};7?kdvuQ2)t#ii91oCmWSXf%(ON7Jt3 zQzMdmuW$W(-A<>PoJ4>6S1T&IZ+jr~H_eRHl7)^X3G=KT7{*=>)PKJSGgO$mGIP|PXPL_FFW|98W3IZzj;wXzzzb|E5_z;4G$25# zp6YA^ILigV+i0BEo&tkc4QlKyx+3(ofxu{#CI0|Dct%5xpIF!!)c*ijdqo$qY!{mk zH_P4X7pp5Nyoo1(btBt$LCEwTX!z|0PQ_~Ac5)FGT6-}gOiXesA(S|vvCvV-mOPNY z0KyJ;*ui^DMhAtHP2qvG&^0R@-D`h>1jK0cG!F$o9RmyIXsUi`>yFnzwBwdzAga1d zpQqLjqSpd9{RVuuK%qcP^0eVuZQYU$Bn;r=pIJXv1hjyyrTXO&O2cDcF-)8>;pZ1V z>(}HMBVuqXkP~wa-Ww(Uj5Z>iH6RG^w_)NC)B{Sa$Sa)h4C#_ z&Q3C5SbBXktK`n1h)Nh zpxhVsloCCUF#7jyv*yvO^B~Hw^h4zVQ$l6x;Wov7pFsJIB(m1_>hd4B zI=B2Mou<(=EdPbs-ENN{P>`Js`(WD~B;`7E(0GV=mV1_kyQ)6^y_%#ve)6uL zLF|n{K8HM*%O`=HWwVs!V))d~$2K6!X2o!ox-zAx8Jw=0RFH{)_FGzaB4;N4%7`g# z0+}VuFCftPcl2;dN|LhpSGj;Cg|?spP|<=3HNqLy?mN67XoG|3K964iU>Bx}mrz=d zOctTtWYx>m=(aX)Rfe(~@Uri>`;LCvC3uf@n8h+l{xOJUqE-m7Q(>L)8ugzq)$NuK z2vHsNX}=!zI@7XUujPy`vmRUuW%p8_c)0b!dhoRMLh;vd+b9o<0$$djT|rK2%QfB3 zV)9#q#R~)lm^TCAk-^957hG|g!ey@)*=|476x2lV8trmRF815{8;T2X8n5U4v%yT< zxC;SQzHU56Ck6iq>ae8I<>?W5s3zRjcT?^!g&Xq#SaB1pnn!GZSJ4zb#xyF4?6?vd zzhVmLqyyB7#wS!-j*XoY<(^ZKl`AJ}YC@f?8i?Z%JU115{rW3`sJ$H_N@EhYIm{AaJTK87s3Z6#eEveY~wdN>n& z*tWr{M`wrh>J%tjW;zNI8!bGv2}t-VDoR4ZbNF|1uf?8uh_kqc^UDje-*;B~k!IGl z{VhFb>p17m%6$LUL;3+VVU6aS9Dl`GaF)Q4L8OaX{#9f!XraQnjLQ}o2VclA++l1| z7e7sN>$dJPaZv4jW+$c+wmFs?GiZpGsdfVyXpwK=5VprjMG{X@^Q7RfV-G6|F$Xpn zbZi*sM86fjAD#KI{Q&d;L>!d!U)rP)ur`k;xgl$RnGK##t%9dxe#B<3JU0>>P5tGo z`Lvcn-puH-BEA%z8j*S8FYUNV|NR(b24>e|0>vGjR)FLfN2DD_mWLhm_QTH~b^BphuDjqes2IXlWK4GUX z&9R40TbSY#&9vTG*RIVe4sK69BkpCps-w2Q#&hOWyjtHYTz(0E&0jc7e(bSE=^6p- z{R`l)-cS*{{ExF(c+nu37VedoNe%G%y*2&_(rmKGS5!g$Qr$PghQVSOg+x(yCp<^L z`=(an@9$EZQ+Wz9_=UejH)Tc!rs9Gi8N8JWs+PM}7YBU|T>3h_yOJ!$nW4^;uPMNh zA79++l|B=}aefts3f%>$(@*k3ar}{@} zSD0IvPL*6hGGpw|ydK@aHA%Gr(Mwfl5E!h_$Ky13Aca_vWQbPz&^-f_lon}NQ5nuF zhSUB8(H%O^D0RQWB$+2U%U0bgzD|>3UB+I>TA)3$&kwI1cxg3S2Q_AJ5X?)coTX4T8ZX>%WTK|WQVVY_5Ia7}fIJS_K*7m@%4%aixQuYW{Ub9GJz7%@_d;jg2Y zvs794*v>tqu(^v!PB-2J@FlRFR%bVc`^QG-p-%*Dq=TqO_QIt;2~h4uJUs^-IiJx_ zmyL93C7TVB`n#ox3)Rq2S{}k-4Jf@Cbw_Pvi}VqOJAmphHl3P>;Iji5Yvbho75Y9e z$$gyR)dQm%>cRtU|M~&bbbst}eCE$teIl2nq+wL0!Xj9tOD=iD@<&87VdK_yL4n)? zHif3ScrsNjg=HBnRQ5P${l8X&6_v!Zj>6XeQazNY0CdOJN8=wummUOmy%v*tvT)Cr z&#K!lz?PO$VgynJ`8Ilz@o@QBs?{)hk;koxOBLTlwkY)8NJe%Nx!vH{IN;M%0S;fy z$hEqU!VZEyqXX+Tlr&b4GAE!s4<9dzd`NiP=x=+RW0MBRxRlrQs3JTSfcY78mLeUK zkbCur?B(4!w2zBLDIyGTyc!l|GO3Qm&a{~<_d0=5jVHh>8lHb#ov>_H z1`GwiJm6dPY-5YEy$*^owTb1cdCQ{z2&jifp2Y0-C!EdsM_(@CZ3u3E@46--h3n16 zA)qh*Ca3g$se`k2Wy_CJD*&03X>w)FY<`U4`ktSO>c{w=JlUzbd)wfom)|rVo>cp0 zDt-=OfQ{*>?12aoTnaNicP4XK2PPi=98D7zCHdO-R+K$eUyhg3KMjjT&2N#UnkFE_i3bu@_sRI_?oEC|kw&e-2uKTzi?8c+ySFYP$c5_WwEqe}7ONr&e^{`egP$ zx(fW)cW@MWO%qnpEC1F2I;ptB_`>_A~0K%NI|2vSU+|Gp3;|CX$9{WF~oBz2kXh1;ouiNJ0fVj3XVzt8a ze_Z;X=T=$?U;aOOJe0t*3giTEA~sTt|9V>5pcVFiixc5M7YrET%eSxp%41N1_Y?oR ztpNd87+l@cA2O6h_P=g&ZKuWefB%N^h5v67{^ub5zhM&gkG6VFXh!G9{Lz)U`z7Qx zGe2E+*lF+2A(y+%jd$j9U+X{mhl+h7`{I9p+IRBgu2J3i_vfu9bHb{9Vo;;M z%(=72|NXhFm_l<`>EypZ7hR}s2i+VfH)G=adtC$JlR)mvbsGfI^|nE~g%%vAsltP_e;QcKC6_-WRd-ti$jEs_RKg!eBLqZ6FYaM*< z$+}XWu4T23c^*YO^h)F4yFMU**Z!=NE$En0GnHoflcMNCF>Z{2t9%#I>8asRb(I)B zV7zn^4KlZpXD9Fl_CsTWrq)k&m`LdH0$(Rq8IGYyS=+g7$?{Hf`>USC_@V zQp%Hze?m`3Mlwi!|HuvkNf6#Vc~XNJ1qI&$ik3cc9iWkYfJnNY{}GU~=|FmZy?nS2 zY&xkLu%Zj@(+55Ps&7kRnARd+gID~y3A#AwvwrMUgx0xKf|j3=meZYcn1b8c(G+~|Se8fAT$*i=pDzf|(GoXStSRjY z#(;}73zAA{f+Jx$l*;WGsA(MpQc`Uog11?DMmh!NZkH$bzUb~0_WNDCMBbG#Ut(>H zXw&!pKJiwXMk(D)iPc{hK?)}Oip=BO?IwO|S~=cXs0xq}DNKPr-6WVD_=%4coUs~D z1~OrKPigz^r5SS^_vh7J*?PPE!s%S3N@v8>9s2Z+Euf0wRg@D{CkwI(;Jl;QSt#!>D(pt2kup{?aE00M(M&Qg*<`jHvZT>;5Co2xt5$u zwPG(mOI2J_5xxEUdhM&;HZ?MsK5y@tp3PAP>7RY|9!_ML8q2H}u|d~wVhx?}J53h~ zT>w1%P_}_TwaP1?7(aMsv?V#h;~~fpa3w?Kd(K{dIO_Hx_KZ_MysE^n|u58_!D?!yegAC9DA2uIuBuCMH#z*H^AzQn(sPl#n?( z+*`&NLE{g=8^-O3(Nafj+&af|DuZ7+G0YP3=PU#K6?aA_P~F;po2gT7w6k!!PyBnW z`z!&Zu9vv3E8ge6iSB#+@x>*x=!|A9bJOHhBN=ST*KT3DHfdyDZU!yl1mdLMEv>_Y zZChKjlQT@wbWUwAL~WC=Ix(9`4I2I)c zVO)#5WkC}`uGP(w5{aAO8f{1{_r0w1VM|Tvs&W`hVKH?sqo7{;#Of#)zPyDpf`7r1suasnSx^jQQz6tXip>v3KlQN-4Ek zqa;+UQq(Bg(ilN$wTV@fw$GhsT%YIqUf)0A$q%`5CHI}&Ip;p-yx*_a>%876{6%uB zL9_`T>zo`A@{|O z0?T^h4cH`n2TDln&N9lj1JOJocgWVWc(uwjf0@zOj8c?AC z{$k!hA(Jcm{Y5a)S(h;!!il?yR?X$f1~A|3UlIJ?1~o}l-@d#6$&Rn~9-IsS24om{ z6tKY9?E3jd5*8Ccvn6BaU^Mn+L7YitJ*80FOry5a{Ivkf{tKU9FRAaiq{9Hfx=EiT zBsjl)6^g~S8QY+Fo{Vgh#B*y%VKv*<$Tv^36Wmt0!zF5$>@79AX-KgW;1fe>d3{3* zz-|*B$rh^y`TNh-^h1Ez_O)J!)f zu~do6{8^N{O`Dz7=(mqwrxw<8MLxX{;pakP8bLdLmm4X(l%UHqofKukh;q+fggWw9 z0R+cZR_z35xy6U4R+4d&@=P&rWV`V?6s{5hc|o9CZz_4ux9fB8YJ7*g!LPLMmA`KB znwvT60sU+mtq2G4ToA-FMCle#tA)IAE}a77CXIHNRe4hXpV9YX!G;vl~GRhjS@%{e2-asLz;c*rti4r#QL}67abyR%<7@p6iugyS zWBr1WYjFpiP(1)3|MUr0zV{C3HAJ~tfxNhv{RewUpiin-*Wp@%)lBeMqj|UOgTEYI zc(FpQ04Et(`rt=?TJ%ym=5&bcy97$tQT@riIp+?fz5yYcMhejU|GG_7;nfWjfr#e! zlf=GW*aGB@rU&eM1f z*oJYz*X!xI$r#;au7Po(o0UCuy7N#)L;>P{C}8w))wKEcmM;acXqs_~Gh!dUnBs!_ z0*s>hiPMf|&Cb8ol|GNg&ExY+XHqn(weD3HO>Pv(`;XpH>nu<#b`eQBc8zLdXFI#i ziRIY#1XT81DRe|zy@;Wh5FaU261~|qVL#wk4js)ybt`fvE0Ns$gk05!Je_rh**Ef@ z*69ib^?Xe+)Lf`UlM(yxcg`>@nDdV)Doq1MG8=5%(M)XH3!A_aKS%>feq(XgF%*Pg zu=*^3?B*WKxC*=kX8KP6{Jm=PEJ@`_yQUN-$i3=P5*;1b#8_X)@{nfIMu=kxWE8Yr z(E#PJq+ryQ;Ir?*0B7y#GM0qKtgO;X=E!=DwnbspB&!i_IFWVo%=F`g7j9g@Odd1! zcCzK29fqOC`jGv)b%Bl8$vqD=T$%}!40~M?_I-l@3n^WBNwbUTj;3L7K{sOv2AKqg z&F@f- zabfxf@k>1kKxop@7Nq9=0^e;negIw#KWoNC;)P!j4K_Q!8&Ec$pGTh> z4mE1d$H<|`rNJnH$KS%&%(j%9hdWv+CsjQl{uYz=9t6(fD)zx=9 z71{VW%YMgD)27O#Vhl)Kj*YYjI93_KaQaBt!@?U#EJttTiN0i5`({C&i$`}wYx_pI z4fdq_W58>6kKGBdg>W2tvsMrajtw7=#&V_RGWVK}95@ef3>a1{7L-Zlm*FNmxw*NG zlVKjOSxa0?2kQ7VHwA_zb>EKBkT0GVx91;^A|5@|(E}6wP^e`2Q*s;SpT(tUen!!U z=SQktQwUUr%AXp4_VamM4-kgWzk<%A$jnS9lRZ&A(ECJIv5V|k9P5DtzxaEz`Lxy* zlyWy4YlD`T<<)!_!d|X4xa3S%`<*h&o|ey-7`P1z$#ywt9PF{lsJC7qE>A~}J1h0Z z6}u)`3vO+B#TwwOdHv&TdOy~;_$X~*zgiPT3+=NK2BV2j-KF--gf*uqVYDW{iXdzP zBCdfF2!7oLL1xBsp53O;Ewd4<7~xk_^Vw7HuBoj^hFMS|sC|R`M*vN^HPHA^55aR> zic&AvQZS96C&t}Cd{oZ*`T#s?U}SKg(eEp4J8{p*xuY7a=9bPk5H=r~sr=BiO z7iHE1)wWcM@56H)CPc*Qe(h-#Tz;Ii(7#-sRn4@ zjDobWL~TbFsj2dTb6sz~c<)XvtcY~!8cpJDU1uNcD?r*<`L)(>qsqrEqmI1Hp>Ngh zy+>%xX)r~=ht>+JgEyhWrwCtz%f`$47M*MHWydu8#;BiO<4{if*hfV)n>OP^pGHVE!SPf6Diwce}Fhs+=iV#I9p) z4))pO8eXAvf>)+qwRdn?t$6-?xag4kYKd}G9Mb>?zXbdUbQfrC-(*+c?Z>bKVy{`CC^UWbz&t3O{=p_R?cFZ!7?BA-*@$IET1Y7bF2-MmVQXI&V?e{jt&jEZYJ z^8r|LNZpp1vdKi$y<%ch2Naa11R-RTBRWz0 zoZ9_>V(&rDptvWh*I>!7kK zn?2p`^X#~Hf|@pWAlvq&$&c4Pz6|etyl6MVnSf@FVC#7m&sTI!;*%Vg>;T6XGky}P zS2QcqHF(IXy0rK9n8Fyw{#pUTugt0OR*Hs}qrOEbBOVcsut$8+SP{MomCw@__)#b2 zs^^v2?@*1;*bG;dp}&x>5g8l(^Ld({bz%gI<%tsmgl743#O*v`qtpPY55sOH$w)(A znk{K30N1?nm{h~IjjP2hHtD9Rqzu5L8DyTV>8b$&fIs(s6aV}n^Vt%|MeRhecBR(@oa>{vhROYa-c13BW{7_+QhYoI* zS))P}r$EdSd+yuP7CbEeu%_;#-9#&nv98GrlDqw^b^7Kyoi-vF5io&G0b?n0Or|j% zmmsL`uox!AH#u{S?06oaV&E5Ad9=5lM`^pyak3DkYrD8n(~|VNY!Cfr*QMQ$Pd%X8 z>+=Z57;@NU4HY0~p}LDdd~T2IND551u~!ri=;-J$4-s}R54;WxF-cl}bgEE>EAor5 zHu*6iukt>iE;;h`iQu{$$H_c5xz|}Me8*PYeMJiImMq^+d z9Iy?8@ec8La93J55ZkPnX_vrW%2|Rgm@6<+%b|e3sZ-^JdyCp1v~?CSHnzQCCwh%% z>m6L54+C@lYoO{Q4mz9S$30!^J&VlW04(hZTzWR#&fE^T^EhYjBC1;h{C2#Tv`p(R z6Kc0Cjn!<9(@j!q3z>HCH+z-U z2HhO1Fz>mKJ99VPr!QOX^ES}m4Io=+SLChsfk7*@gn38Dd|^wvRP#ZH62|$W=Wo`o zuPAcIs!G>_MO2SJzzvfAA_g()ox*Y~_vB61S~v8$Z!94mQCR>PXtEe^^Oz^R*>@!+U*er>)2@!GHCw2Fm$2I9az>mD>Zw zeBX%xZ(rbyqca(rsED}`--*5y3pcZm1x0hciFt$}A0!F4UvfQOIv@1-(=c;U!hxWQF>elbB}*}W`wkg3JSQpv{ha{gr#d)^-cA4SDpA-LOJ(qCKJ zR(j?b^62{b<37iI%Et_47T&XX|FYD$oK(s0^Q_re`GbHh=GA+%aD)mdcT3@;QO(_o}ckQ(|mAj4lVLP z{GvGf{@0{eLSeu_aeU4}u=TrX%xR>hNT;GzhqMF68X4|v-b3wiYA;EFT${&dNYuMg zCMTah1AxRe#skgagA5fV2cds+1|h;|Kv);H@5L_)cm=0hNcgz%Wc*Ds92?P*jb3n+ zuq6D4cHl0e<3YO4fU#CXXP+#-4rKTbwGb&G0xUz>UBMcy|2$uNgSGKR#WN<{9nFHt zCzj%WlMs=^qGi*L5=Xzp_h?;no}R}vLL|r5J-%tRrKN@E_~Mn0zpntNbb?Q7_~;ir z@L$I!Ax?YfB*d+zx(m%PTuFd-`DlJuGe}#pV!+@NkN8`n4TC12ZvPl?j02z9DEoK z&Ix?Nli)X{9YHzZFI+nn83~-6X2u!t4`-tXvQOmYaX7)xggE%P^f(02N5CI3T!#OC zmd544IdSxRJRBTfQyl#NeMSNN4gD7e{y^XP=kF77xc~b!_+1>{zn{j3#hv)~=M&H! za2y*Op)n9Wy{l!1gF{CN{lPVkZ3n}KKi+jm15%(qSX4Oe)3gvVAEoo_|+do-Kw|Xgm@;<|wn2`TUt&fB|#Huja zwZmR6Qoyp^;&9i!-|lu4{5dEbFC~(~!oml! zN-VgcGJlL1{8NRHeD5yqSBt+!N?3yPDfnqD%q;l#k%56;!6!r|;TQ<~HezT3&viUU zWlCFEivD_#7d0L}Oas9e+^)amUW9_ zaZP67uj#VB0Hdac|B(K3re04;$oB|0w-Y!1GURIsFlu7fCw~v1lzA~YnVH`-~~w5X`)98>$JXQzgc zCr+N8&PU}Gj~FG2ZjBbi?q@CZ=6z|tHF3IphkKY}(++lH$BjPY0dE45{C>T>tf=`i zHIurup2q#n5$j>Q3(EUHo1@K_+%UpaO;3<;+?Lz*=GH@KhmG=)eV);<5xw~+yXHJq zMbWO$SP{4KjZaH6mBv#Ip~a8BoE#{%Nh~TU8F+nO(&Fcj4=I~$?$XLP=r}eoia&2; z`QhL_f;RTg0+}I-X=$&LVPqp}cVNV^FVP#1CRPdyUUJ!vB!mU$mPlWo4h!0mBL_3&=vpm^3 zZ@XBA(bm!$4-RF;h-Zn1&-tm6Cs{p0^33>7upDmHg@nwiNH4QpQdOX9UAw#VZYC^V z(ic4{^FGFRhA|Oug*XYeTF;iUY3`ApnQ4bh(bTiH4!&5rJ|jtHU9H@@Tf4JIgxtI8 zi-T8t<@J`1nGG|q%nKa6bHvjb+7{c@hr2M>!`+eC{Pj9;A3 z)@9_gB>s9WuJ#!ZX28B=h*YKInrQf5?kBJtFIQ-b8Gd_Zb@Pdg3_+)m71&30Hd%;r zf&S0j!*1EBlqr}on^PYiejfYcyAUM2_)$GmLn{32t*3GFA8ynV=#^NHSna5~?98Mt z?5(ww%ygv6gik3v^)hffz=t8j?%;SC(6DYPXBmf0NW?MVM?^%-Cx`3VYVN&TSZwh) zC~!f@bl!-|&gSkLza5L&uUWI2SQz!^%K5liwON@%B2%$_-&F@5)FQS!KR^{Uqzb>& zv_M7EGE&~;SaxR)V~5ssS`rwp`n;p7(ik1bT{g}n*kQ{RS0i09`i-t?nV?4~TfZ9R zIG0_*K4|(N)|ZHkJU z#XKZX_tM1h2q)p5#h$m}kEsZ;0=S3aO{pTLNllI}Qqx7n2&mpq6;9DbdH!wqYvF?7Kv|EQcO-It|RFi3N2on*M! zGBWVoP4=80{xn<(O|QF7>6D!xrFl1m#ir+ViN9FfbH9t&t_y&Zt!X*^c-%-9_Fpj^$7E>OZFglY;%xlGmaK6 z6M=&BiXJ=f=Dt)}kBABCtTPB&Ma^VHZG1?pSZ(0in0%**-oIj!FzI3TZO@C$jkS%_ zE7cXDhpm0!N3gj#G}jzm^(y1iYuJiu>)JJ!)ycx&w)TLXql znFNUVo;9@=rpl8CVDNp5iY!(`MDK=Nsnnm2|U1jQaux##$N~@`iO}_n?%(rcm5RQFD zj-LqVN)cCj0;N0O#`+H{DhMy|Z4QTWq)MZ6{f zwn7!R^y%u%>?7)uTUsKmtNO5m%Hx)o%GEvLEnM#1dUj3x8tUo}gDaD3+fnn|iT)gY zS+<$eHxb$Jb(grES-XSr0A8`2oU)H(t0?!upbt>S+X6+;*0s_A9K25i*yLVwR8}{O z$9|6olaoHr)ah z3GPU#-HnZ0_vt#DiUb@V6vuQxx@HKgx<~(pyx`EJe9&;!-fH7UBFlbSW0xhOtUX+M zKfU%j>Z^zovCl7fM*Q)3@}%@*Leod|E2@@v@}rv6 z9nV#5jR#nD(5s6p?Bp$8Do}rTQDBQ=QY1}X>GjlJU5H<%xABf@5as^#))cSD%t(@O za31DHPfK!-zQ+)1>kA9B;Aj6Fi#f%$@T*z|u?N`H?EL&2+ST`GnGv_^oNR4Q!?r$= zp)X=PZ?~CnAcI`%wDt8p23xz3K`y!Km9#X3SULh-8a0(RGMC)PI}xk12|JfE3QB_A zsx~Uz@hjywVt&F>^!H~q7iF?M%I`5ETSPZYX=V3!H=Q>3H8*IMyJLqn4`yNyzE@|I z$9^|oD-Ha5U&*tbYc}D%)%RLEfgTNF@)_Of+a61mn`n5}VJTKC73MIdEEg{R?4*=2 zX7^}yd*X(_>`BG?A(!@+4iBt_$DUK6C&?OKxVM7*6PBUB@HDr@S@s z>ZUHMBrGYkW8xXh-l&bAVi&&zY+TJgxp|J6!sL_|PZHNVmhF0(!C;m^zC_+9^fR?E zgBLlvDLNG_2eSdWVRSn-8@{l;PnP=PrxB!Un;qewHmeWaH>WgkSqZp8oYnOm9l8`X zdpd0;V0=|8f!25Ig3?~RY|`%O9Hi?!_5x8w&gD5u=Ef2H zOgntqKd1SH_PNKZ_OI>+h3a#U z@X3pxi6%R29qp?Um|%+CNZ!<@nF?H8;vHVi>#$vqYAOoVR4kfbTN~*cFt~oXD)YB!zOzN%R1CTXni6*re)YRuVi+03h`CRoI5b>rp@>rx>9u>4)>gG z8JosXu-bFIJF90wg}n?qfo1cjEV%f1Bx$Ys+>c3l|%;+ ztP^aDF1Z8`bS4Ii&7x_hPif4nkKf-|&kJ_Xk}J^Lf2#TJX(%Svqe^aBQvboap~loV z2J88^E+>(>IELTQ$x=ZrSw#OV?lnZPLYAO;i-Z?+gr%66K83{zM{j;8#xlRm|x+!+2 zhaIVgW=SeG-IPl3JiL|nKX#JEf}k5|b#A|AFSB|l^Lh4b?@m!=(Q$drhd+d?^VVpR zlu2+xb(x=`-4HHQp$NxROjwRt=GT+3&i)^+ifwKLve<)(F8K~>RyF01>WQAtmpXbv zN~)X~a+@QS$lgfs|C+;V8m7-_7wf2d)Zdls*FvNrLveM zOhL8P!LAw2WD&F4L)(MBZ7WAh%j}t(@=J2a7VJHq1-tziOxGLGQBRBFT@+am#YC`4 zv10HJcun(whYsp~BBLQ-+RHv~tM|rgFi!jO*9~KhIg;FI&TvgE+d>&8#*VCO=QD|g z^O{2@j<4q4kXWbwP~-WrrcJA^q+Tww8w)fGikb%M_5?#DaObu46H|@)2e#pvw!;;y zC1|{v{f|+?PC4qDnlhTIJ#@JjcW@V)S~dh2G!LsfZigSN->A%8)HS=5oWPYrtWZn4 z@7zizPqW{vQ|0C|e__!^Y<7`fUej))7q(P^Sv}kNnN46h{+a49&i-o^<&Tn@X_b_7 z@fdH92=pUogaH=WL3S_q@oL0s_qr8T*XwSh6TQeR^*>$qXg*hTa@*Hq z%hzO7vT+N8nH5zWkXPG7P*XX`-Y>M`i7hBA^v_rV0`w}sj(W3Lb)L8 z%`dPGXY;r@OCjFu4aM^fv24{RyNeC4RZNW1;^HHNK@&Y}mvdVM-=;i)eq)QWeqHQ`L z7u7j4Tj5Qv*JhtPJ)4Gl4z_c_p`#>b|~V^@Ix&Z@qh;k?b43 z)j3z4#j?NR!#4ko)3Gs!dQWN1^^gvJ*mVz)x29hdpm`{`Wk zUdg=}3=KbNi-U53dG54-609|G#*8$<>K3Qj{`KqaWFjn=bb8br{GE7Iv#B0ep$MI8 zcUMM5H%5@#Dtt{I>OYm&tivPr*UkpfF`xMNu!5OH0*`8V;ToM zNrVU~e`Pd2?k!%H-g06>!i7Z>)|sl5!DhEZ%Aiv(z@gZt_ zNOQBo+GBTsM=QsBd^RCLqOefY9JighwSDY8ydbE*q*URia~Vm2(@TwwP{E&8DQ)~c-99;VqN%P(xH zc?MVl2~%ojAW-Y-)aUlE7-W8TS<%sp!eIe9>UC;xj0>JHRbi#2S8}d}9m)`>i~&&6 zsVtPg{<#~QJ5zxnxP8)6M>0LbggQiyx00{ktQ*nN2M_)J@VtjPS+4qns@N9sj4M6W zyXDn(;#NgFYM=GpPbsmg$bREjQSV)NcI)G5fF;U7SYk6>kyIQj0qxZXu>!M>GqVa{p8Z$zW3RUrh#O&*D$yYVHWNDDL4*uzRmUndWR{$ zGB(ZXd-0q#IydM3y{Bh$ow=F#6u*0uq}yYzb9jzjdf9dNu3jUVL#~Rlf!cnlN;uUA zm+bh88ov>4U=k08?~9nVqHv4q7XrMox$;F9u-+`y+fCN@6WcE~L{2!(G-I+ns%M6K zFexz|c~X8aE5`|_R`=aDHs~h1<#Mtg*XVD3rqdPVuuh8m*n8@>LYO#?*El)rn5j*a zOc0(IGtGN`a}BIaL{$`0K6ZVI&!p{Rh@Q*X3%ZK(MU5s?{y}|qSbq_2k%F>ve(Rz| ze-;m?9=Qxcnv&q6?v&f%UKyW=oOSW+QXuPu{F+r4_mj>v<;8d(Hf&Ca>sk?l)FCn_ zadvTiNaLF1wphp@-*7A*|7=)XLQ=`W&M8r{^svUxF1z}l0_sC=Ep=e8D|J~0iY;%F zQqb#6X2l1q+eaNd_RVYD_&TvDa+0Y$+)xsiKUeA%EbeFE^LUnL0d%~M7m={l=6A*+ zGv*CCurA9gJ=gRN9h0@S?S3^mvL!nS+|>cA@+N`?LsBgBb}DTDyV!cVVws!l&~{W{ zwwzFK_2K?fWOY|kcb9P(x+E)^x8zngb zzC95aXQU~AU?Ye#`D-gThvy=uW_{m!B!%K#UamezPO>M1AB+hZP#*dk+I|atAf)-?nhsey{>%H)SXNObuE@G1nE_W&*a^wDm~k1G@=zfz6rM308)Smnj;Ov%X46|}2_ifBF<|}D=Qa}7E8bLpHi(^m@F55wgO}kY;p4)oj1+jH7oHw$V+JJ2 zM5!&c;%j7DpWA1A%y}8>H0-jNQG%-9o7Jqg^%(w?M)0EQ@s+2qt&~+Xc(pJ9WHs7R z-R`Xuqb>PhgC0;Uu86>!E5%A_z?#8`W6P?Y=bT~7ok+>;GxoKLLtL~xy1hm=W$d$# z@vI?uOlZoSgPt9VQQleU_{75|DhW~^?G3l> zewJbi=(b*KT2ztAH~!>LUVj#%TfN~uUqBxI9b>MV7?3_U2rkz7quGu zuYO6h$JhxJGmEouDaEzm6xpt#&miYGd&gRA+L>C6tw-`o^42?R$qrw_%m^H|r^014 zNuIeE<`04(kw6zWCDRsbaE9D#kkGlQrKrexKdXyz8{47iB-{~x9+B*?S+OuplGVf4 zt+NQaC_&Ot40qy;9tov`>k9hymdBpSxx>}GH9q~sw5{9*vxusi?OnFZ1Jd~}{bBwk z_eB^;Nu0ls3YO-G=hbC7tk-U^v25QhZ1hhtWpZ!ZYh8ZM`LFU9Bx6<@JUa zptX*mO?4m=OK0%JQ_{8rNbPY8W^SH3yYj>e?K?Im*>>d~yEMa!f?(KGvnJ$$j&E$EJHmzxUm{g%3ySPwbd-Xxr zN?_B2vfb}r>lyMG5mp?|_J}z|rmK+#OSiId8QFB73aU z;q_TNSzJ=ju_P3AUr&`CQ9A zaQ(#Kq0x_l+$wz2azFps+%UHODxX^H;cLa>z{hJ9?w2wm1xL!&H)*=WHMi6>_i+@u zC8Xu!`RzX%k9{x|-jTECDTK8s>x(&55N|nmIGizH8v1eT@bO}_(;?m3tpMrE=~fx8 z6>CxRB6kGI%c$&cXH#|ZYhrF_4SCOh5mq=ivppKkwzP*_+bX27~kkL1k8t-l4B zd}G~k%lh@NRwKgfl~og!8j}ec3TXA$Ev_+jR&}mfJ1J`s#N&&W1LaL!<}Cv;hZ?z3 zOx77a*3gPz7UXhWnPd_$FE;7J%Ql%8vgfKvclxop@0#5!X+yCZ7=-wkbIY6Fg{hs} zk?En@w=x1!{n1jK);_B)^8mxGblMpyF%16X8kLK&%fP(U3*U*_5No7C~S^hbK8agcu8tcc*Up;u37*mMAu4Pi5opNr02idX{+=Xr8i0h zFouhzQh(8q3Dkfq2uTfnCh^}Tj>otjIU#CgX2jdK(ii|-O-_sJgC=eKkP!3hzEW&) z=O6KB6;U#gF$iRt1O=Rcb_TIJD3ao1&dmaH<$s@5D#W8NvHv9+0S9WwWC zX7Y*xsA2t;%M6l#U6G&yi)>>2eCLm0gRW2k@S2k$?*h;N`{;kC29F0@hU5RsW%A08 zvdVrb;5b@?J5XR8Ov)tGnUR@UAe0{#Mw1w~M#%4H=Q>XG9$5l@ z4J^)e1tun@jjvRSy$MnQ_r(LY*DwR5f`jA*Y9D=dO00@v_vej^-L_YGxI)6h`qz8( ztpO^A3ZD`ATZ=FVL2n;8AJ~9mVgT(h=canIyxex`-tJGqYSV><1*;SM{QLq)tVvJA zmk@=LFah2^1m*k=XmI{&fa+ZvZ%PYA5eWag-!$p2DVv}vWN{@ zRc$}%>8c%gkFHKzKSl7z;^E@2^1?rN7=B|Auz0%3Q@v-q+w4?j2sq@12Z!mYF*l}8 zU*Y8~$wjSaBJf!H)4aL*eo}ldwat3@#}0w>QpHWx`bz@IkJJB_l}u|v?zS;fcK3lx{$MR;axhl?l?)(bi7TiMnp+0};-EY+;=1 zNDa=`E@}WwGV?%*b?p8)UG>MCEh0CqFL!4?$73<`lK9hO;$9O_Z@s@|W1yFD7mO>5 zeyo-&cBeBntGKw^)db?s5P+G1h>!Y>r4eff2M2*ipPx_fxeb0dp1jJkZWQ^iul#Qf zDphL^U4?p}7>Niwi)i*rBN9%HsiJvV`u)1AOu%iQ$6Qs z8#1~&JMYY8GqU6*1BO;89WWfoSm8gO0|-PGun1ac&iJ(I(_n7_cNt6y*F%1XH4bU! ztLDC97VR%@g@H;5VslOgH~5=mGo#> ziPty}q1S^qtk@6#_Sjs!OZ_{5VSN%F8k+s^@oQ|Bu&{7;US1mrNAg36@z7_d$Ra3X z!J?0ZoaS${UFkNW|I3z-FSoY08`!&k|E%Qo3h+CZhP+piuLsz!P~7}egj1i1Go-iL z(D?1o3-Gv9;B;8*xrc{;FR>?NhoRSx%A!o>k76I+%87!#5~|8{xIVzl-6=9EExZ~g471LQ;fyk z_*IhQF`G0Ch@Rhq34WpLv4(hF5X1 zfVr6&v3z!;gsLj1|KAT}Z*6OP1bn*KWaW%ScC%TL|}fQs!hTqn*q^kxwc zd2sTN`FZJaeso&W17Y-;Wq;xKh{(utj`@L-mM|WK879h3A$;b=#{WxNhm(;o3fb2e zEi1Je9v@1Osm99=3uN`V^|l&ghEQyvPI6Yh%uJZy6<4~t{Z;qO_F_Qft_wkse8l$RAf)X(%Al7I4xoh zt#rC&6BCoW=H_2~$x6PzyF36ehHUGRD${HoLK3Pyj^_n%ra!h1dHpj_be_c!x~pip z^;*&FPs+m6XRp-&+i;N$k=dR9pX%t3EcKE?$AVZ8ZH<>ikOUJ^u|JTKlG2{@v3%ss@<`aQlkWG$ zi_;|G2(H^$7(moWmVk1_OBBTD1Q!tyVNT@IZ4iLR?Vncp!-2i@&VO{>w4z&9k_%>& z3BniqnV+9;!W4`LUKhr#GoM{DY?}yZ*6*NTDfg`V>PH3f>e1=}NG)#` zDR<=UTYnhk8ew5kQJDxckbl^h%uKE*y_S@fyQ~;vL5xxgqGR2%sUTAQYq`rkDZ{(F zy2{$Li>-#tkc@>UxB6`+zLjY(1DRfTs$y(h=0?L5m|;Ulv-ltWjD7B@Z9fkLdjuk# zMN-^i?N^BH??>=`ClG6BZ0yh~(&Uv}PSlJ=o%utr$U1^Y>^whMc9VYju5l7=m%L;< z=V&~PR9#M1ANc6blPs@4;sRX73+VL1v(&z{TMbwAX=`i0T z)&B@A`qMs7@kmKYc}C+km_c=9n$BTe*PCaE;0gxnAd2O?a=crg@nPONc^7|QL>xSY zhu{f0!U1^D>fe=gS4Q@-6iDXt?A9OvUWJCP*oko0T&0s1mV;~j5pFs3a2*Vl=g#b>yUGbk;Re59lRY9p`lSqdS` z5MiS-R-%1>WJ!2E!5JnYYRj=aMfuV&`vh#JGeQk#^EDIqanv$U;YJn8=^ z_pl$~c6N4rqj}~%IfdqlZCahlQn+NKs&ZGZf+Y6BK`f2*cRRoZi5PF&el51>8%Gy* z_Vnl&8yjo8GK;yDk~qI&femwVadl{izbMxPtGL1VLAUMM;Vv?zuTMXno19B)nMtp*gD=#ns3FHk#VZ>ru106slw{Ow=@zE#E$FhG6=oJB< zL}P|}rmkGDJ-UdaKK`Na0rg(ok(~L*_T2RJ$ICI!pJ}e z_=JEXIIAhWP}DNq`}f@qipHts2W7U&>E?2+307D2Y7FaNpBL9OEF%Y^>*eT+>$~XP z#d1p^9<5668&bbERkgbqa8j(W$UgFck>-i+-$l$wUr%jE4zF}?X4lO{Qv23>Y8jrs zZ^Jmft|Z?A4#Fs$D;IV(4@lG+3D~K}R4EzPi5iPVwkO|Nm0om+Hnp($;b@ojM~y(v z7Dw{Kx^g|NaVFmWMEX4~tq<~$ql-$vI{OH80O=Z!Q1wSQ_xi6Q_XqQfa$CzSPk0o3GXk;L&F|0ufy?5T%-1S)p8|y5^wZvyA=5tk-mF!a;iKn5Z1gr(R;fKG+S5AY}X+= zfj<@v`bQJ#P&wV=U+4+w$}3eKf@;BPlD24&?1pUs3eyxLI`A!!q0TyAZ~K+YZX6M+ z!tzalE0SW^b*XZ())|thO&c@|H2eG(WxVv*KbrUOgnzXM)QDG*8-QeZrj>Pc7))Gf zIZ2cE%z3%V<(G|K#dtG?K1N{ zp{6cHJ`$NvJpa22pYUoNNhy7bBUJRfeZMi&QAetrslmLzvwqVdu#M%C8y>OS{is{l zd1)?hy)o>D6kAqzbpol}+ob5Goabh&d1BubV9meKA0M(O+zpphGZTa?R}<=RHK{LM zL5#QCJ8P3GDnz=*e~@{>#ah@t1v84_h8S&qzZ`3?(-{DIw=MDT?77;k)vbx(0j}H% z15C72OtgKwEF$dt`ZG7_W?O`2F;Er%dQzq0=TEI@2E(J!b--KN^8;v(cLNG! z*?&iNq`2ZHlesi=3V{wAU+5bsE$FakMUnP#Q(A;x&Y-avPsfkisAmiA{Xzzt-+4cT z+Fh{927TaV-Q5`1#p@34A8r~4MZ0hG5rrF`JT`Fv>{;M}&f7^mI9k~0q);K0-Q(&2 z+j`TlcEsa@U-ZK-w$aeF?l8}l{@kgk*CsNGFjsD6i?+u#U3y7FPRI%>5**5aV-Ew1TFVoMOCM4q$t_}^dGXB$>Py$vTY*0Q z-06+iM67d@Qeb9!5BfNM8>7+|9@qc@#fwnEm0l~uE>$KX9n8F7>vHHMZyRJ9 z<&~0>GF^Wau%Crt_x)ELIc(CsnapO|K=FEC;ENC;_9e*l%#Vyyk6Y6dj=~ob61tCg z5R#@=CJZoz*tQ^$1+qbdO20fkQtcr<@aPggu{fCK`6z>yE`L0^BOMPhzI(z*> zKl<5u)0@hVqpyKfPxwVEA?Ua7%n@`(Kuu;F6dxa}aB_0$=BrFCDYdt+uC5+U?)g11 zz%&KA(Bl+>*DyQO47;v130eYYId_?hNyy&BoH#9UTF$#v>Vbuq7TD{ch_`3)39o1Q&94rEL)&QtI+qKKl zXrX?zq>IQ|e*tbWS{_@F!K6^$w?%&QY75*;yj>jJ#K>s@6XmCpZ21s0cIj%@U2cGE zzT^e_v5G|X_h(fQb3B>>?Dv#6REv9Ubd^}^Gr&+4pp=9%M&%1oR;3d-CnpIQ5FLLO zoYfew+-b=XXgoR+6BF559eq!qGw4WFECJmj`7+_fZxV#|AUd^mJ(D?Zd{1~YL4SY0 z#l*M3oKP7_Ny)f{Oq~F4CScG4&_~a@KCOv`g*`5rgSDxp zC267P)Z%*~hYaYL2-wVYCQP&vI9`*$pl^4t{obG6m4hR+iY*6zdNT_<&J$MTP=;rO z`dZ1|yN8!s=MC^h2*WXB3;VtQRv~5-bL;KS*1;Uu!|p0Ko&oK;9GSE^(S>g}2p
tlk&aUWLSH9J8LGSt9cBaTq6Exg|e;{)Ee@C@YV>4d?VSRaaM!b9ey4 z2f*}~*$%(W|Ay>HT!?#~OdkW4pH`*)aYXk)IjP$d2GXTgteCrmZJ`>K*DxD^!#1PL zgQ$r-?4*C3qXMK=1B@XLh2B(SG;59(N5?@zg(E;x&Hzo*^!&R_2|Q-0A}~pitO{&_1crpdM$)3Bym3 zO@HqP=K#6q`8c2QZyi&;^h0pGBqu-)IM3&%G;`1mLvA-aJNvxx`U!BP6XG4*Ft6y3 z;|g#%Zs2%6s5k&+glOjn%3v$wC>x*aW^yX*PcD#agK!cr>Y(-;zDY@B@95YM_OPy1 za9CK_od7M+!_7k{!T{JmKgM+cb&7q zc}07KF7THRPtQF8{<#8ZHI4(j9~8%+DkZg}>M&arbjY>v7uVBUST!yR-N zSm?}YTS2nkN4LdXVX*r0VDC2r|$ld^crBHEoDFN9t+`w z$)4yU6MVYBNsu$cmd}Dha=x%bp&-w&j$PO193g9~FjyjN^IEg=vHd_!>;=l!esGl2 zVs~>%@xjR3x0F4VMvH@GDHI3}FeZ2w&eG|;3w6v~=GYifR|`OBXn-Q=Vfy4d{vTlx zlcqXs(yyS?Tv{$JE~}yM!NHtgGwB9LSmnZbibxp*E_S7*r`r(8pCY9XvA_M~0??d7 zhb-+5x{?OYvn!*00N`(VFFA7Y2HA(6gN7 z^ZAXhY-&Ymj-Ro1)`=@9=;+SXw+3uw0l*L|MA?4oNlCqkc3b@}VA`Ja{+5SAgB zZtP|ssI(!q4?yIEMmn)es0oIVkAQ_ON%%&g8JC!z{=L$LD=k3~%av1lVV|_%c)2fi z70UcQwNl(4sc0N*J^(L`b{_Xj2?&5FL(uDd3O2m=F96V~0wE3E{rzU&eVXt2WD>4d zxU9{)FS*9bM+9(A)Z88h1jl8MrGB@u0{IyxJ^tkRGsQvXGOAs~} zcWo0+un1VZpBqoenK^nMgwNC!R#ZJFzNZTg6BIuklL(g=V!ChLC%jnt#bB2)#^vXS z9EKkSEn;)%4ge9AGqMf~4IbN{tqcSLR)Z$FRXZ7tD{rQ^x;{(N(K$R)=7t@uJ{qKs zAU^~+{Zjn!6JGdxKE&7FDZ>*i!+W3_K{_Ew-c6r4)~%fM(h&OhT>z}Djre@X0BM^x z6Q1w>wn&Z=Ut2d2Dsa5+Lr})Mxu6s~Ktxzs`)g#HlOAJ+L3!2M-TefL*V%fN{d~Rf zgai_Qz_+4J`RI<$5oQB3TM(Sc@tAHs-xH-6&=PgKV`p=T<*7XFv1T&v7Cv8%r|rT_ zxlID7x1L;%dg|Tu{`xaU7r)o<0T*Km-1Tnc)dCtmZ1y)!%`$Ae+|2<(^Iz6yC6~Y@ zy|~=KmD>x7hPT})0N_EMs0}h*`_y}IP?#_YLK_izr^EEC+{#3ulbTJSX)G7So)&o5 z#%Xph?TtBP$A6u|KHRhFnOU#))LLuoqLZT|`(UNER4iz$e@@k#o0`g&UcCuosWU;y zg$w$U52*I)*Vd!~j~<^XxZB^a4M3&2)rz{DNX=10P$nTRuO6$_y%>q7%NIfZff&90 z^kkWQH|F7-bx9dq2u&@cpx7Rs{r)rnzC``cTMArmI|Jnpvcm@2#AXk&#d)gb`Bb z!9frh4hkBTi7}yBuUaS`umH3L@JrY(IFY!djhZxdA`*;jsn_}(eFJ2u|K*CfY5b6+ z!m6ra!1Rn+y}QIM@3>6t$dOwI2$gr--=Na=NFtLzsb*It2Edj+e=)+U{RQjt=L-J1 zL-rl=0uHmCJdMl86d+y_w6|<0?}k70H}!3<#q)+Ql3c*l!g8fDoTsl z0E*2#UlvsFy0Y3PZ785~%O`?_Ex!1WF0Mct3M%&d*VbLJbd_V7QiQ(@f#W4Xu2d2_ zRPIzB22n8=ll!RMd(@kS_pjvoqP-v@z$=Cl>d8@{_jd$G>h82W^FGV@Z~MP^@W1wd zvFZR6i9ZO0`Si*FMsqg@s`O{OG7}s3mVIHbhBrsrPSt>8{WGy&-T$>0m?m1jhT)Br z*O~n*%M=%7O%sU2LPPri8>Aa2IpPr7A-Rn?a^Lfy?YZShiR{*z0* zqrb>}h@nD|w#=%Fl)dYejD_Yo>&W9#;We+9`|gI6og;^b3|jWy{(AdnA9MnF=`)xs z$qTo-brKsujuOk(yfDy&umQ?+gt+YwtU{m>N0W$lY^1RN@&iC4`M-f4S&cxL8`fzssEjeQ6)AW2C0o)%sk0p{aDy64iSms|g z35PGngrt6H?2Fg2?M~*ZUj_IJ2Z&L`Q}>m~j@hT3TB(G_3R&U2PY4yc?V!1nnbzRIs<1NChjBeTcB zT5L7|3R=avWTNpH>5&>9^lzka$)U*BIsk_E2ix^r?Ve?#~eHwLsxm zzoVNYz}M{8M?pdVU)U67k9%aH?=nRL)vTX|enboRDBK#u_eqoI_<9-|n=_OKBd;BI zPb)g5OpQ^tDH)#}i}kHcc%UbI>L(90Hsgy{ zAaV0l*MX92yxI7|k=dY5`@8$`0tcLx;d`~v5X!Ys)T1X?8VnkezH`P=kCKyPQ7@Oe&>4bja( zLtsJvb&n{RN0mD^k;SNDQ|!t1R}e>rT~Iq<;hJ_XcMrW z@De+h)v=H%f^C&dsRrnx^kyY^*#(q7EFqy-)my;FeFPe!cGTVki%j@$ik}x3AngGt zj%*MJN&+#!&0EsxKJ}v5;s_~_76rcf01SNZ0Z624(3$DPMGqj2(Y^Tzym)7Ks}5$> zqiYw)Ft^ud94miIvATn`H4xAD43jP0?6BKfQ>km0Q$H~9h=S0}#m#7eMTz@R z^+#U+fR%bk543ZL5?a!gEj%m6XQ8072~@D2$ITi)gK z{vaz0Q*g^1Q(^%Hj7GM$5wMxvP~k+E(dXwxJl0I#cPD19L6)wMhI4qcKwUhd+_BHH zWax=%s)ECZJgK-8uZHIfG}iC0U4k8#i?yoa0cpSns3=SK>{n)$R8$P0qr#xvVH>tc zIojrvVD_Ja!3pSn9)q08lpR%N5_0)Xe>@idthr~8fvXQcK3-E#Yv2}1fKYWfugzIE zU}I#I9PPSTCSdUO<$q3e3Ys&3EGM+xMZmEOo(}+y$%bARz=JdXLF906E$4Q8zM-b3 zrt7=1V>n0L0KNbKv%zbXWDpf9s;Y)SKoWEXPzBH$O^bO^E|}H&(-g4npVow`es+bZ zUU%1)l^DLs?4w@uzbrqkfD1f)kQoq>3Z(-8phvgq?qG>(Jp+h>pMj3Y6W)stxa_Oc z+%WIYlv{+HwYIi7GA1-cIqFSm-me<+FRlV0A$C=P>z zniFIX0Ll5yHwULaR8!CNG)>s`^IX4*XnblV6%+(c`e}AD?Sj*AbL|M>VcUQ1QlASK zCxAg6XEivnoi8>H!ffQIOt-(qfAfH|ey9OW?>u;Sh5W5o+1kh-pbG*V|2s<`m!km> zu_`HjA|uric3K=P<%R($%9#GfeQx1S{dRr6b~!`C#NCH3Ew00c;I%v(EkBO~+qbEd zZgkM;yR`{e_ES4W?o=V$WiLZ--uN%rY?4r`ElVn|@Vd{))m`B*-_WdDr4if*XA(*X zNNDY~fS-zP1X{a&fvU3zYR;GcEYv?38EX$7dKC2D{85$_5D+A7*EaneO0+m$pD75X zGG?IC3Q$f4W?d8vTjW!K7nx6qw90R9SJ45dRxZ(NXFcsrC;BhggB1gGhyiq}i0Y&m zgib7>Hp>HRWO|#gu1f(FfR)^T)K~EFds~-_oU^kDDKF?hKLHe(AFZL#e>VZ8J(2U5 z-i%tAga8W&5((fz*PD1n(Q=bOt?v-KlG2bKB~y_gEjr>HJ6A(-c0Q2pJPx>i+~MJP zO-)T3^93#1?(Xh`Q(dbMWaTd_M4fdKqdMxki6vRVz5;~CY$0m(UF`c+kFT^+qf6C? z2h+aoAy-rl=%5!F{FlQ_iROnZ2OR8Vskfka+92FkYVpO|Rg4}1Tg4gUOSG*@vT)K% z-)DiB4B7@joGEZDC^0%@i)iHNo@3H+2Pq=uAneGv{|ij*6iPv5pnsP>q^GB+JYxtF z*aKl8YCQW*fazfv1T{9Kd8yK0d;1a znUfPF7-{-zyo7fMI+JTz<_X&!w7>MmfF4)Cb;IHavjBcIM#3Rz!l8|NxyrQU>>G{R znMq--2dPH>%YXnt2jeaI`x}9LclMj==IS$QF&O_*{5{$-(+XZdPvSu2nAC8YPNxL5 zzf<-Ide7B4;efiM#}n|7M~wP1hKlaKmE*bcR3+mVNTfuFcZ!VJ{?nPMc_jlk(dNQw zg8^`)+#F~+TI{K^jy~A>3@@G-g^i;s|5bf*JoN$}aWd_**;*dUG8cJRgVV$925=aV z=Kg8V#;6I^m%*D}hCf1-*~Qd*cSFOnhN5PA*8VcLQ_HJ`JBl&<*J4IQ&{F#IZh5y#zJmeG5vavGa0bx{1 zn`XsBUv%EZ^0Tzqw3B4ajHLD))5<_SYp))5)WZ_Kf$l{nw)K#2IsT67L|w38U}g+Q z*$k-iCG3gQewD}YEV!#{`NB|niQLCTAfI*2D`Yme4+BNEJ&~hHq2xbR-wMg|3tRDv zJe>4xQysAfg>r7%d#OHb|1AtM*b|R|zDbc=I3!MOlLqH|%CtlufT0pUok6g2@$$Aj zd6#tr?vSfMVIO-wZ56!i#PrCmxjI{= zqN6jn7#{j!e9lV1w+Rha8Xin&&&TqyS#meqe!7>r&jT$?TATW5^7}i>t_*bzO-*A! z4ajYt)%qoGio?P0L>q!?7*a1fMOuE8^dc?#xl9?75qgR(pSs2`g&0)qYIxByXLNRR zO0wKj54yEn_mSoSgN_2smPtz5!c8ajc17n@5vr|qO#Y*FABwfM4vi#w|omD?Yw+myZqA9h& ztB=0f>)f3JTyD-%M$TOmLjI#AgZ7+K7M@uffbh=HE%ySorcp$in%`oj1{1og{_2Yj zrrZZD=j#CZSqSIHflwh_!)dvnb zKnYWzTVfUMz7n8+e-=4lS<(l@Z|`_I5I5Jy4*xx0VA#^~KPYaoUv&4s$^|V2A8`q( z2COCZa{c485P`+>DG2C*9pr}kpfIlV3X)@X=UKZZMf!U)$edtWL?t=+7M1+OzTRy{ z87q{82M4~mu+=s=7}cFe*Fb~Izfb|>H}bd)zb0*gw_{KVL>pkz0NyQceb;HReOys_GLHrDXu;Z0p3e&*0h+*p)d$_QWoQ11# zc6OFXBTvL8Mdp3uDdq{if8NFf^4y#WYYqWR1B-;Zj$ye?n^1e`REmmW`RTj$fPOLc8IoIJRu!0yI& z2Pkvt@xkIuyZ4rhXN`0&c<49`+-@j#=+uss@*xX$J8iMIKwCb=)73CT2oek~9)0;LhmzFxjyKEpRac>tGI)$^mgW~r^O6m*c@OLEn&+Wp zoyyY>l-jpnT)RpAiR}NO>n)(F>bmZ6fs29?_kx7dmy!kn>5^_lI+gD3k`gWMe2YjKLDv3t-W%-MV~g?G6!fK3CU2r6Kx9I z7^X2@NbPFs6y-~8*O;anx7DqeTIWitsqKNN5KT`WXF7aMD*~r>hu1{?!T!kteu!y} zfqU8gcgsjqIv~dT9iLbvEo9XoSe?o?vI#jYd=Td=W%T+^4*WQDL zZ_;hi&k6?EQ&cySiKGdG<6egi3-eRF(ijh0_LMQv{)Ai_VBe)RXB zH5pT9V)!9b?XmV=7j{#it-DY_OF*3~A; zKZD^n5Z;OUnOT;%p~*ped@u-C5;PolNxguGB1oj_4TDU z8NMX&m5-ixGB;^hNIF}iGv^jdXInIv(;x}2b^>7#=@nS~6P-x!1+cjCP3|0(VMJCs zpi6gVzkbOM2}Ku1D5$HO`6f2nG-HqhX0w?0-zO1Wy63VY(-Ea0l{ukOvolwGXlQUR z8nEIoKw1`Hz^2mp>V4v&o~h}N;sx97%+tVuqg%TIA+sS-j=iX4(z;HQlBg6Pl6v~A=QbomT$ zEzq*^Yh%|+Ps_HD2T(t%0ev&&z7L;hZ$}Ym)ES$uyXF3{RAQ4>739MvYbx>$FSSWG zE9TEz&l!|6UBV*~Dp}|O9qhX2nr3KB(f0kYa;2|AL-$HHjoZMq@ED?hPRohxDB&Ex zBe+o@3R-T6|4Cd#U*0Wv9m+~x_#Li2;gVzX$KQs>Vnur=W!l%a42=n8Wte`Rg}JDQt*;Yr3=YEj z$2g1}+Gh06=X$|?Y<0R1z2gRQH>>v|ulyF`zbr1=At~(>pn+Xu?fe5#AoHM%6!ggf zW3bcLfyT25_iUGN*2fGpnQZ}R)fDKIAAJ9WcNH1b54NRKb+)hXIW=UO^GJ`glkz2~ z6M)DL*VpQq8;FuAj*_V`cFlP${rq<4KvHs215AUC;nIR;yTHjwA$)<6s!-?Gb3A17 z!vUISh^{ZnfB$4l)|2NGzTW7Ir9K0OnKRrQlbD)$j{j~zqq>mEDsiHK1bW3Vg&;z7 zzGi>vBEmYc`hH|?0BRI|-S?%ogU80=Bdx0ky1H)t@8|jDZS{L0({D`GX_hPVeB-!v zC3U#&S}2eREbqfK;3&}?)QqKS#-CNk2{9zUuOXc$k%<60k6p_TjrYR7jf1R<04Ptc z82l~v3RqPEp}m)KQOy^ve_8ZL&;T+4w}#-uhd)?lp79W}EoXM+BG*MmJ*GuICY;lWMBi@+&4I!(`eD;da@>j4!rbOb|OrVr3HB2xQrO zD#fHxlA^DnQI{2SB3^*GiR<@0C`Y^!No$>b9Pwp+=PO7o=9i??>{0m@^hu4qxq6?L zltiw`n!Wz=9uaFKYbV*iyHkNEs6~-phJY_ga!MQ+=yP}-d%(OSS8_=4pF1VzvnjQ| z>DEd5-Jvsm(_OMJqX7vOxcWFn3T~bz*J@+TZo8Hx<$nK&qY=$0_yufndSe2%(dDc7 z{SE4>juCn8ZeCD}sS<5B06ES9`+<$db|kk&HFt_)fW#Bhyvsf~e8Lpc>MU!T;yu6d z4}DdJ#J)$?{yS0|E+?@W+5z!{;=8^Y<#ge^vQG~mJAlT)Q*`B!0l+#uk_#A?@y~YM zcLt5%A~UgHMp7gj>rHfU+%y^m$yoU;d)2&|9z$&T#fOXX2PQet;2lxf0a5sLxK#n_ znU{QN47y^aMDLO)&~v4L4&!gVA7Do~DjAQoQeE^|!=3=#*g%8h+8%I`bnBd?(~Uw_ z{Gyd$K{MzHukHijKDPme z{Dvg*{v=~^WcsFn`Y^oCwlfZ_^=LPfjptLI4IKNO^Aj918m;BdnDrX>Iv{_`sbaf+(lN$w47gTt5nt1O@S;0s0N64*x%ki`GX5`4J5`Tl6$UU z3RJCP?%G(%?2ewf;?$VhR#S_QD7m6UfB3MyAzrknxufG;pdNbcT97F1NptHdgC4S0 zA@@CfHWAwTvsCHr;|*TL+A|N}dTZYXTIcqF(NTT%F#q7>NaFvb6VEaL2V8qPQ)Bsj zd}mi+bZ@p=@#9!?@Q$qJ3p((UpiU^*vh!JlrMa_C@3@h#t?azs^e1D>D;w+TnCjk;G%`co zL?<#rjAUeCE4K(d+{}S+wjLA@-2_v~x6y^vkQ}c2gTM3vGGTg6mvP906cB*woxb#4 zE1D%Fc`TcI^%fH8e$i6&NzZS`d9{gydYU`G9w&9uJYzJK9_$Fdu?dGJKcLvSsC|zv|5GYUH~@ z_GwLH7s#q@90z6Va(+oz*G-~C#^+;K$$y)kEi%nGiXvbpvK3Rw6 zCrQK44;n^4)T{T-?1IqSPMD0Ecg+OCPfT zHpi8M!zlQJ(*Q%>ArEpBq6Es)P?A#DjfvPy%ik|-?ORAOJ!r_g85b)LD7YrlEQM$N z=GlACxVuMw5jQQStPq?XfN)r@R9u-EO9=Kc!GKYSh}sGY+U=Cab|im^%x7)|uwy)W z&~)sz_qB<~()9gz=_@Af}tlf?-IK>>besqytTjt$SLhUJEKVxkBfd8w(w(~oo z2G_iOi@Qm~q8|l@1sqfKoy>95;Q7lq=ki6WxsM zoNC=I?JHpZscXLfW4DyWY5qa~W~|*dsc`qynaBj;m!T>G-M~9+d6C=p$r8sNdLBC% zBq|Sz((AVIDt&MuPfxJMpbqDY-u5mML}XWhJKdYowRgora@)S=O-rs{jfGF%j;#BK zDEanM58G{66m{(2IMN@<=}p|#I}Y3_Vbk>0YHXLIYw5( za{pG>$1E8UTvnMyWgtxecJ+FXCxB-5AEdgk*nb?|WpAE*J1=XwXG}D-sm(iE+xzkJh zQ(Pbr(aLU7(BNST8qD*Sk(%89P$9@E?nIYClP70Zl#|yVejGdWo@azk@m*JeOMgZR z0ABfswh-$}ymNtTS~21u`QW8|CAEB;SK)RQInjmk@|OvmHs(3HgTL)cD)ZClZ54-p zVsD}gXUS8z=PQTJT5^Uotmm2!lN*`l#&TFQ04sDrR^j>4%&eu`Jmz0?#*nZVCns+0svu(nipHd&HbJbiIIvy=+S7%^isdWB^^M;0v z%rakrLSna}-H92g77e8MiJH0G`|7&))y$^wIUfmohPB4h)Rack@0%gW2VR2Q-R z&jVbc80r2YZBqnV*!KIswuRc8%E#{i(G{o!fTm`5OediBIZts3*C|Xx#?Nc=Kk-u+ zg_g>8scu|nx(#~iJvLX@OfD-tm$Yw&UuLMv%{v#@8KPVl(z<42z4>Uqxs3~N9RcB) zl13$CdorIUwZRKKH6N`g!8rkHAk|nKS>rAlQc!wem}c|r8&wP^GGjxcJUrnMA$2Wa zThzx_#Vq((C=~!2Uf%2juqji|v712S*|S(q$|pd@@R~Yf>^?Hr{PE*Q8SM->g`z3) z=l$}$n-_>ccx7~psBH80Q6BA_RF&zM1${ByK#rqf=md1@0kX4U4V7$Y=3Jmq_ZClxS)4VJ9R4&BrIS0XH7PT=Jjy7Y( zph394t?Yo8I3{Va_vj3KmYY^d+E-#GkLInzR3s{kmqOt{q)hr=FeH#F!!QDL94kKK z*>uIz%my+F2k8H?@+p!Gj6I@rxF=^)TH5fPj~L%g zh;A@}@*p!aQ;ZGng~-(uB)s%Z%=p~nRD!N6I2>k77vMojsg2#G*zKUtF?BOOcAKB) zvP}r6vOYK}bv+}*qeQ{ONb>L6t=QL^rFVNY);T)|wIN!Zr_XEG(?Zi#!q!pk5k-f! zyXi92)#hayd@SPP`tfPS#ZyEWl{>!2M!ohKd4tjse%!2eViVnd3TThSkYRI_ty#ydtue}jeY9aooQT> z8QCtBeJCVx>m?NmxUMK>gIXTJRT^8(%***R22$+zXhM_T0@Y4r{iRPe%?RFgsdsA! z4t?#OnTZSYRTQe2&1Q1IB2DQ!dc-Evp^#X)p-0eMlb0}Z7wyu)+XJKP!v%+6iiZXj zB~KI*N$jmZz%!PVhyhB6eN|B-8YmXZrCr!nIE0I^QhaeBswXZzz5KP_RZ1tm%{o*5bT z`=9#SY70?9){{fEt}o&<>c}2+TMUv$2PHY}RVBVKHiu0;G0!e`--6)CO6#U3XGumK ze^OH`kc8lB%TE!;bG@JnAcj+}O6zr9fA3WwPfK|^Op*J2XJc@6=&LBXj%h5}hD#9yFOy)7`oc-<>XRP3488J)0yUb>dKOdwvMB$=3Qu*}-jb9zMMT6w(Gmw-jYM^Ea zgA-PO@?*LKvibRK5HNWmj)5i2Sx@|f7d zoO(1rJvO#NC?Ii)(K~IhSQq{tvGN#hZ_+{l6OGcp9ZoH!pK+Drn_+vWF8*?`Q#V60 zid#aGPbygQZMoL#J};X5;rmL*PYek!TR5ihtYyL*9xA+lqWHUxy){tk@#+mMb`Z)N zS=&%0p|b_uWrXozWr}ycvqLiyFwJ&q90*lKUgi zO2TIR`?coO-nf1+UbYHL8;yyNEr8%uu5Q&YtIQj9Jj-L|i^I1JdwBeMJ%B@+lGVsh zCgV9wgthg;NDmlYKFnHxE&-8EOlWw(2Cy^iMJh@Hni|NcQ1OgS0iLI0bN7@^Z81p} zstD+6$QYw9Bhwj>-diqW{|t*m{bjxs6zL52mA)m@$WvJn)52ChjX<)O}-8cVG)A#!=#>5N%|*1?9HII>4pTl+V> zq_YE!L9ZcFoB<7CmsIw5HhLuOjB*g#r%i{g#r14WdykoQYGmx~XRFYG&}J$L^Vltg zmLEdX6PWTr`2)XtTPFTFbw6b7Q1L@bMZ^X|kJ|Ai9+kwR&Pl1QOWE@p@5Eh8EjO73 z9M2TY+MJx629^g>jk=-ic+-NHB9syLVfNTbMi1a3!vTaasDwwHDQLu>8?wpJ9aZUG zigoN0#L^__)R**Y`}OMygkMv6f8A#=nQc0-9zh9|$7$~z4Y8omU9n0C{yp5w);nu= zzu8_KpZ9^~K};5TZ|x1?dHgL*h;+j_RNjjwhkAEyRR><{yHj)YD<3+) z%*<-Uxiq-QI?oj@^1P ztA5c??uKmzl@Epm$D3sMzE+m9sKh)v$R-3-i8CXWEp}SZ?z7 zit$GWu@6fuO9QjHYc3vaGnl2Synhog;~<7(A^exBgbZ-52h(DD;XbC7(15{;W#Wdjq)Ubr$4e`wwiUPw zNkpW!kx$>k#`)6{<7NWbK=)&nZ**`(iFQbo|8-rzeMr|q3@tHJUO@K!XX0gFMJt0k zKHtGxeS#SyN`u+p#XtIMY;EguouPe>=!HPSjrYW0g@EZpVVmOaInO;Ch-P&mRXQp)_UFrIGSgQr#Db^Zoz4g|p%p;2u5dLo34|L6gLp zJ)`ZDS%Ui^FyL7+ZSW;Am>AZe=&OBg3PF_1R*-eR3f-t*Onx0zzbHwWc$k4$+Su5z zI_f=9jTf)R3xg1yiJ?-*`Y%lPyuP6NzM(h{%wjW_uKR-~fvp7Z*0(M2M?0`3 z4+a{uWnJJ}Cev)-C`9jhL1Yc}ydwRKme_oW50iMzk0_QXA>=s6rFWJr8D>gxl#LPV zOq2|?A3qc(_R6eJ6pe)k_Sxrj8N~7&R%)y_QDuctxe)I{29lAn2dBJ-83p$7c6X<$qp3}s8TPcM_)*liQ?(ceXXUlK%+=0vP+*>Z$ zY7F=ru_Q5QO$idFyQ+1Bj8^5z&?RjfSJ#+^Vvuv^Zu^al+y+}nEH(RTS9D+IOJ#8} zQM)`^{W7g!$jLXA#f5Ng8K4oLRLvL=&sjlbAGM{O0}m7E&ASN=02zp<%-HgahG4y-j_X!pk6}e0?zC{~6f=F=T zAAN=jqQ3O9>uP^N}TXg5r?~?6jH*ir@ihT1h>r4^xp>$nI+af)_|rj ze}gffXDZyJdA4@Z=Xv*L%o*jKUZ(bg&OF$FdTCk4g4L4iq7K zlwDSagFNBE%m;aGA&c&!e~_&1(SuHH1v3Q zi$>ReCh(y|v>k5BkW@Ag@pM69C>xny5WS!ny>1J44H%y1h5`6de%(G?E^gLti%4hVPww`O)7f7- zss3&5`Uo%0!V$M#4YOQbO1QM)-8s_xo08K*Co?A}oZxHA5Q%F9AKzm;VBRswPvfIYWYC)nur@h+Yj-RkoC7Cz=(%XhLu zM~PclCG&14R%6{hOzU+RC1V@N!aSVI@nNEK2rYQ8!Uuj=m4`oKyQKVxXw5=FG`8hvf#>0Rou)o7Z$md>%ATU-B55q9% zf%LBz2udlpJ=SXbNN_6mVwM!OO>~1-RU#VN!vrEQ$g75kuruS<9l^>Sqg<}~VbLKT zjH?-K?|zxr<#b*93$?^a;ang=ht@&nr?$4lg^ZlZ0#4iL40z5MQH92qE=qbFh7gRt zp{C|cg7h1U{8Q4h&jcL7@5QMCVA7(2?RH-q2>^-z{k@xRmwRP6%(_NPxs&lo_jWjG z)&9^p?Px@iK{%G2B+1BAl?-Py0Z=nvP%k_2k2d{OcWJvaZ459vI|bAHo4XS~z`5@fP8ipzPlNxZ@@c4u zb&_Z=Gm0JdE>74Q3{!JbbY#d4e2)gojItY&yXCbR?eJv(%(Gx9E+DzY@Kf#jtJSO~o5br&4QL(u8kttCCaqX;G+bp(}% z%&6`k#024rTK6HHVQlT(VY2+3Pc5JCfqW%Yxt2wgr@4tkNsYm>&0F@3oAuEw6%&Z; zt4X4t!ze;=DSDy-kW&%xcwT-6W337;jEI`yVK=Q5IkA-LxA2B$N>VP&E*J{zJ{C-( zAa(y~V^AJ)8aJ%fHFBg9$b3rt2j6WG{Ly)wK#1^oV==nCLWO z+7XlhCr~i;4x5Zt%K7;Krs;WK@#*3VxjZ6mEwD;tD2Y+w7O6mAFjpkL8X%(wbu2$o zlXw*SSruTb<8TB-8k^3w_LRvo1Uo={DlTzg{!;J_RgCX(y-LYf8eU5~yE1Q~`x;CE zk(LBtQ&R=nVtmD z1i0Jq^agSdGmiOy9iBw|G_;}WLV$Hj^kc5Ao~;?}39sNf>h0sh7H_k<`! z!VaRz@092Z=TF9R6TY%-x<%ox^nF&Ts@`XF^`TmF&ZSMS9QYBjvi_@~$-9hnLwJJE z94RaKuxTrl&r@-B^YbghYwP zuW`a=#WFjIxTj)oVRkX0BzJ}^ju<>Wuljv2x$5V4b(;sd9*b@$27FF|R(Wq`g`N;q zia&1noR+X#nI-@xu+*J^l!!~JnNZGoU|)bn?zKh5;zQgzy~@dw_nGIb3;yxR3EGVr zvtcVK=$%aUyW;sy2@@@AyYbrB6?PD*2bVxb>12P1N*X`_>-L~92~;NOMjCqhrcyqv zVjMoUH7&_^Tvya6cnEG(BPw(K--M0H*On0`C!O3F|)83ET=RfYS|vqe%n+HgxUU?e%w< ziD@WNt($QvA>1E{ zmIL&iNB8B<=AMkj`44_eUa>aUq-1&R7~LFbgpL{c7?s2P5>{BRn|XQyyt}Md#P#$f z*4ZU80J*;kPC5DDX6Om{H%HL0_Vd>}!Qahy2XcVh+ymgRZP?L}`L)HH#}6Fp`*gty zSPz7d*fn(68`^+yiHw-upyclXehg9HzL|k@do^@fyhw7dv3WJNpho~d0K*~0iE`D- z&*C-XelyQ=J087Bez?Q_?6b6S#Gwb-J!Qsr9<~ChXJzh=UFZE;@~AZIahP2Rq>Og7 zaIRnyQq~z8Hm4Itv(KtPj>}*Fohn_KZ{xo0&NZ+l6HT%%``D9TUJL1skK|Sb=MXIu z3a?jdc>x;u1yGCd;~@5okx(rdZ9`PJRVrwEx3IDrFSMVZpQqyByB$tuB-!_8vCxc4 zw~y6Gf2_ZTO*PW`#gZ0k`7d!Zf66P>7M@os?(cF}XP*c&e(YOkOp6)QzC{Opz8b1` z7ZZbj(iF5M*>#u#s-gFT!CBaMDtK5!2eAfKFSnY&I-RDTKFv&_#3h9mjce8ly)fws zDKFul5PAz!BFZ0ep8vs3XPw_f8TmHZKVk6BKfD^BD)V~a!0$6$mVbR8xluDOrEXU1 z16cAV{az9KZ;9V_+rk1y$Qvo3ggUSGI;hy&*i7cLu(5?gk_GD2G&CCB$(Sq}n11rm z)b(Q0r}}eSPc4JS<=zwy<5TR)hDO1&8RJm~H#|HCV1rKK{YV5E)c^g6F5o-VQ3U>K zH{+c&5~+|a|AqcIQC1*=-ner6v0zi~41Lh)ea_aKTsb{2F9tjog_C`JLgWs#05MX! zj`7b}6=8vQfMm=iL#47IhASxGe%Lt`T#DUML|y2M*nhkLu*9gD2CHR1q_B}lD!luZ z2#P`2?-P;V*+Ur=z**1-*dOql$pFPM`LuiM2^mZH#ct>iNKiM?lOhRMU0vgB%^`}SWp!Wk1 zEheC>%gI4{yciBx@iid*K|0k6y;)jo6*z33$hRJ-ydyj5$AGst`8@VCND4Mh@Jdoo zN8;K8C_^e-cQtI$jYvU5++{j{2;zuu&f2l9a@J-{82gI;Z|wf_J8=g@4~HZ*)UAEC z0)05hYHYXEf9Q8hr6%(O^cgKbbiXFYyE^A`OoyL(Q)Oqt)|vQ-E#}HK-|yK@ts(M; zyI4>-j1U^lH7WQaGzL-)w8X{H0h@Tm_{y|1HxlT{Hmp)UxF0=M&)|}(%iw@s2=egD z{O6KB=@=AorXhb;B63&!^FgJGd9fe((UXUHsObeCd{8Mx_nfL@ZaJEP2YzU5w9)0xwk~ zv?ZeQ$Yd-;5WyJfQ@LI)W{FvaTlcSX9fezGmgjA0>AwccZHlK14(hCWBHbuTV@ zcv=I170I?6&|?KPto;f0uQ!3e`=ppM_DVOPNoF1>ZZM;E8Wa7`+!RF>7hS%%AjOMx z;S7u#f%Rz(g|QPw2Yoa_WzLO%p&>7CML@1Uj&K(iSg#IEpusQWEhg1ueIzXxXI~6W zC##G@j#d;Ea)EQKk;FvpR9q=^c30dI*}r$55<^kNBD+ z8L9Zh!$|OJ+FvrfEttX|d#4(pc*m>4*O0!>R&{V(iPWO#WL$j7j^p6sxCDOP>uIbK zS|F`F=W-b27qy-)m2bR&vIU}zS{$^f!#SgJ=z9x#-4j@%^A8SI<c^nuo-F9uE2kOIA^qD+~sQoMciv1fLv@vhf-R3L{N=vbG3hk&- z)Im2(4Q&q)l)f%1V)SjsAO*H|STaCMIjytZk9lj16Hg&cDQjF%S+ zAq{9F{&Rd8ONxvkp2z7B&(h=%xQ~pS!Jpjl-|+yn=-^A~kzVC2^UI~>oqdcmho$|* z2O+Y#>HGNnbXL;@<|~jrTji;zy-#5LPabs`zY_8)@%B02$gay`Cs=ANc$lv=bTSsM zj+}eY2gsrX!TtJ~L|27YAc>iQ!FQ1io&^UqfkawKK)gAu#J!JFo>D_oi=cCSfx<-<6)F*ToH8IJLZE-K zhDlCLOl;H_-K+g{d*TSRtKC-?Jc67Ze2oByruY_gU8HcDjkXYKF;=Sjj>&7)UTa~NwxpV>z%L2ex&A*Za4o4a#%)!@ZJ~JOkE&Mxi zk$(x`rC;8&csjj%{;BcC_K;JSoyzeTRcj!>Va<8(*%6;h9YNR#6^d-)J^5&E4{O*V z{eH2mGRJuY>2`rSioE1j0i#m3K!k`x3D_-(RZ%(E{D`@_K0OVL3dexYYOPn_4|~f2 zY_UG5y6sFBN;#MP_f8-~S%%s7tL(>#=ID6Wg| zttc&@r~vxm5xMbIsZu;%8qNp|vfN7swD%70|5OWcq6-KLwCynY&!Fk+JCG7!-=$=H zpb6}GA^m-Eu^n96Xm1_={MWzxj|`gFY1LObM6_py@fn1sFdyrMtYxaWKeYX;=%BSWUu^HJ?vI+`a;AChL znNhX>9TgBnsH~w#s43*&{5NWWt$l8j3!&e<(}PxC(wloC)WV@!PN&^lFMM$97m%1d zr5`5Z0#^SriS}11L^QF15*T;6=jSAz*5CyvtV&<|aqv0)&mKT*3$gKT6-~}57pm#J zntWb;WOr8U;W1BhXzMeH6*fmxk*&2daZozVsdXpWX~c>aGz z_6}-XYu))0TgL8(CA5|g2B8bFoG9nDdI8u?a?N%9a`nr)L~3iP&%9>pxPm-9M4(}tu4^E>prEfMo#ek?J`EHXviD}*RrLAQVy!{wZdK0e zenR>3n6LA4;IZkE>Tvq=NP7dV*Ckx77^i`VCghS@=0$EBSPJ^ypD^i{fvh%ImJc8# zEk7GDJ?5yAsAOdqhtPz`I(`5553)pMZBoxKOT1s_FxTSNCgM$Z!0UmDQ~%A&_PK6a z)O*(A?)4&ta$A6}Aiuc$R#YLJLMeYj2zU)mKTUdpR1b52ROSWFOWpsOoglCv9-+(+ zt8e-Im>tfI?wcJJ5;p)_r&4F;BA^8{WXQQ^~ZugWJ+uhm9 z#0&^a-?cmil{2^~M3j=QJNod|?<*9n4~qZmYLs;tr)A#v^vjp-X>P*G(RO86I_-7K zx_%Dpl-n}ieSVZZ*t42*#0Snt3H+0IQ#$+s426B_{(*J=repyt^vf1}{$-%;XFdtH z1*gZfYI#4{`RHQhbw#Ih#W+m_<}`Crd*Z!;*n+x$9tbY(zbb9QTSfF=Z!O@NO#6mQ z#`%Q#Xzi(3`_x)udq4ta!+GDv%qHWHIkSTVz3XKr<7aWntF_one;?&3wdfHwmz`kv zKa~OUUtc8PS5VAE4r}^h_ugO|$j%H|dW?H>-L|!w7T6%9lB~BuA7L95OLLu8X;Yrm`>ArUyAlF zis3A6yL%V&pScF*LJ6l3qh3S#?-`r@#=pAyfxi7B?k>^qV}93UCu`3U*V{LXUNEm^ zK09eUGkB}>Gw$Iu?e$uUKXoFg($j8rNB41>M*sJPs1c%v?_OE0bbk4=>Z);PXyZb> zSAAxuQleXcgnK38-m~erb<^Va`0gv#O+3fGHR9XM`h+WVwgE9P2zKC)a zm1sb}A|umvuVr(YU>?p3Zl||v0Bga;lch(~#A81c2@HbI7%fzDwHKtHm7w;lsi|qV zs`a8hsV(=#2EITByV_!?or>+V6F8dE_%Es`JK`WI_^}y;+Pk*#mJOkOC;!y>?FH~< zH4QSH#Fp;XSJ^uDu$=$5P8Ivp+UkiMz130hlroB1;LWoqkc5_9Kv ze`-!}EJ`4PDzZ;!pQkIXwHD)%oD9d@DTTQY`J*yo--oVVz6##%`bLlORI9(ly@teZ z)@!%Yf71Eqbs>aMhtI_27G@^togQU&X4ZWuy@N__;r^7LeD@<{a4+9W=$m}?U4YM9 z#l&ac*j4{?zXHhnP4hj|uKymJni>ht@uB48e9o|ywiHD~NQfh#h4aD%sR?o*UF4Hl z2aKp@?DDQS+KLm(nL)#((fwL;EqS23#1nIhiAlna$ADYyFHWVaf54Mf@Ori6_Oyy$ zDwy;?0iKu+H7ec6I9MQ3I=Jm$Z5fTnSGU{kr=sors2FE6 zR>;4Iq`&*i1d4VB(M5ysU&~W zMNtok-cENwLm`%nF6^8@(_Pnt?3}_E;_1<_DOcjC@MFeME)H|r*At8Nz2WWZx1ZXi zWfuOFbe%GktX4F?7|@^7Psx3+t@&ixT8ZPL{%we>t)?AH< z!t)u^h$Q!@mtr1L?s(HDo1 z*_rbBRl*JyJc+`ZiNdkQ-sw!z_fb-u24Auk2{qma7(zxP+h9cU$p8rpP;R`GkbotN zhrzu{yYL8%bCj85pZv-S!+UmGCtPN~l(8nM#t}8WE1a2ocoBfA@G9L2hxXP-wNr1+ zawqp#UYbgs$pzA@ozqqFtbrV3){+2iOt=nKtZP4nh9S)M~t+jAK6Fy!9ccu@ts#*K@{Eb@|8v#Zr-((DODD; zc!DdU?mhrL7Ayqsl-|362>!EYtDwf6eRX^>=pV@>w+B5}H6?h^zsgRV$x(S%fRsRM zNj-c&N^Vj>Uj4dgv?^ujT}luIqjmH#YWQ|+v(jD!^4(?0X4Kac0U_xEI6ne3SBkeY zAAs^Ig}@Ot4MK8TQWSyk8(XslWJ6mgPZ&MhX5-_sR z|pn#Cz{On7Av4?#9nlxulLo^r?Qh57b4DLf_K0pr86YF#Zfpy8|Bd zh?;f5z{@KzqNRQEUFT)(!$#+(gyJCEUFnLlS0q!4hth&lp*e0sj1GwTa6G<*t&Y&u z_HxHSS{UZr*X)I~DXTmC<3Q;cV-bN@FY|b`QNlK&Hf%r`z$u*!n;f1f z6cvYOsihV|qM3#nrLCMSrr;})0lj}iiA0E~o)*$4kBS&2rvQQbDE88;%YY3p{ZpN9 zE}r0GpzE5qsJ;0`kea3X&}-wXU>w^q4vmlHk7B8tc&}S$J7b_g=c87pL{b~w0J}6q z&68@Rd1MWZz?1ka2jy5rq^K(p{fYR*ijWZMeeY@}-Fsw4{V$mn%ldS-64z<78)^2p zRNqC((fZJ6f2x1@Kye9oI1xD$jABI>8d|#Gf>D-R7=4OKzMJO0gybYiR_1t?2~$t0U~jgAXxeQJe~MFMuhN;)R6c?Yf^P| zz287|nuUVQKwM5V-`9S2iGsOVgnAB|<7@Bo?ISvPx=bUN8aR`YGh3eq$H7m@D8|~d zXM-uLTsp22k?aC+WOg!x%mdHLFr_DzMeQ2)nnW}Mdj%?{TJURh#h-eSGpo~gJf?gS z%J3@Ei!%4)j>SzJUlVOZ8lL#%>P$JaBv@HB0>^J{)TAEtGaEmVfF*|Tbi|R)ZzXEQ z$0Xzt6&#)Xma{Qrjf#T66<_dp@>!?i z!#;b@9%!5uNj@&!b4@f#uO@bbB`#e97+=^+ZCnKvd>Es`s~{&30_?$L~HJ{XW8 zl^By6&?Reb)j?h9hJ%-2eeWJ7L|l~oJASd*RyVIw;5I*w-F`h3CJ#&Wd-m2#e4m25 zdkQU?`Q`0+*Auy|YUY4D9bHg$6W?YJBMfFaiDwL9ik^us&z;3LZX>$}e!tqkN=%%G zg83tb<2El3gz;}$hO7K4vmq@QOFvm?`*XfN$4VwJ)=$CrTUwN@0zCxCN&$$B_>Ma7 zqlkF+*duYAtW{9=2Wk4z|;HmtTkRwcv(Q%>9KU8%0zmYef(j7sqZp$CWZ<#y&sTcj3qX?(7tkd?2?w zvnx6y+>2O|^i^Zk_lw7Ibg7v`--Gzm-pEqk1&P$ToKXl?Bo%xWQB2Ry)A9(*a?<6k z%Q&r!z-BLdu4(i;KIbf&z+qeH$o= znjbnS;A-BcFASX+ZhFv9pGtCKym3pGQU{{PxZypLKa#PT-4;sQ%>TLHLa9iFkd$_2 zrZ6z(OxJ}pOod;2>@iLDK*O<1WO<*8Jipc8cx)1$vsB+$quO5b-}yj{sd7y$bwPbO ziN~?1Qx9o@drX$!vnygGiGVOEYKY0>)2Cy{A0L?`$#gGNJPZx;BE22#DuWvK@%lF? z^S*&$bLI0z#u9>Ky=RyVQX&pBSx>?wHg1yK#$I)%J}0bot}sSY_s6e_%{NX4x;Ho) z5{l(Bx=rOIGwg!ZNbydSGAzR%l0)w`qNre2oe5IJ=>$sCdc}E*y~m$KG3bWtn|p!-6QC5~8Glbcu9} zbW3+hNOw0PNDERDA_z!#cXxMpNlW*89?=<{-~0Uq-#cs8EXTE;`@YXU`|Pvh+SkS^ zu`eZjMJMDEv~xelOpRA)laNmMd*d$g%CJey#v3$z_lIv_)Jru1804O<$KE#t5!dFm z!}!imOP1FX`aQDmkn!&+55?y_%f`$wHGcZJ?Ven0ywNMr>0ReLSm>@n+5&w2Vd0qsOb%rT%~+bBs^HCUbm^ zea=nz7(_+Q^jyC!BRBrjvQTi+;j zT~Va7%R_HX-Z7lfMe8p&Ps%$85V*C?Qm`k-5EFUf<-|p06c<}Gfxy204XUx}YGDVs z4Ldu*Mn4)ZFqMFWySh+=XL&9k_bP`uks=jY^=C^&H1_cC3m;3pXSRC5mz|cab@^as zmmAtJYu9$K7ltW{b9Hed(FV;tJ0#myT+0>3tY?7S{^LqI!<1OOs|B0H?VdNlJUq~2 zKz08C(&zi!wzZeyeHI#ScqVj@(!Hao5K$vDTH=*9QpmC(c8-Vv8n`?uo1YyvwF_rM z7&0CbVIY!+bi z?sgiwwc?6#EO13)t==#s%E;JGIF`ft2S!WYL*8{hA!b9}{lShDjUZEr!L%Qmd(b~m z^xy5mcTJ1FR$%^G}o6l zu04vPi69HNO|=-%B2YiWSVNn^aRpEKBT6BgGd%4xqIk#Km_$@qVNlV!fx!qL3UhCu zr5f3G0x3azHWc$CHpTF1ww*_PXUAp5~0hDI}b;b;O2GA`N?vyW~6c-<+22xW}dNuQb zZsX9MNeYtDpEP1KzXf zx=$UinamrFKQrQjR}mD6qMX$+)Fc3Zv(YEmqK}SeNnBf z&_6W6T~b|jR0!JsgB02>&B>7@Bx&^XCxeia+S@UrV4mRMns7zJ=u1X8ZSM{o9q%r% z72l0wBtM2H&@*5-RpHVQ35kReqJZjn8o_b%m_x&UEz6<4R!%?Zhs+kIV)9W#R+#f> z*$3-MyRao?5$$rg@zOlLu|eJj*}C3cG!{~cmWa+8{i>o)6SX2rsI_jo#uBaEmpUkV z_Kx>jZu9V7vZ6$EJLj5!8JyP0|DpqM^S%e<9rLY^E%0m`o-r{ox?eg%fFD)%+kP3QdtRWX|4NYnH+qhs>h5woFcm`FPzCGlmHRq}x$anyJY%$2hnsrANwP}^VMtm_ z>d2&(e>q7djA@nPn-lDdoxZC?qpUYi2zI#$)#7KD3@Ll=8urmN+i4N_yo&sAi#mo_ zpm|3~Bcgu{vEZ~yKw#iZE~)dHr$@7bU^6N@lQ$(GYqC`P(4EMAQ7)kW*`IJQ?-vAV zYpwv)@@mNF*cw{l_eaSgT~9y>V5Er7z6O3^TR^g{anHkM=fbCCd@)H-0o7rX`(ko} z_iZy8Fl|N{`MVlfcVj0bTm0$@hn}NI+a~&Pq@O%gg%>+yGZBKyPw0?-Ycl%~Aw2J1e8PS)42Un6A?PDMrv4;2N72j&e9 z40!p#Dp<-Wfb`c=7Ud>nwIE~`ssET57(~7E)~?le-FX{@O$`OlgJp$h;$nj--^>y9 z4e$y@BYL@Nkx#<-Vqmr|Sd-QCEeg!e`YohgB$*vw+v z&%=KUssJxIhX<(_Vy_ntyR$h%Eno&>h%ucv1o?ARTwStnKY|RfNz`3>5nkq0i@l3k z07@~ZL96F}GUHHSHpSvk5gr|#uIC|=FGnICT*U~1c|*SX1<{*JF4F70dsllC2OYff zs_JW2YafCX@o!G)@tH`sFV(a=k!Z?~tnjU1;gN{6l|}_;cCT?+B3oLt*tK(qqX@F_ zhTK>ZILUBstTeHJi**Y)`?+ah_1X|60JE$W0|~?59h38(by$t!vcw@vb01+gdka9n zzfbKBpvSjZr8`+d8#_BJEZn)AoSZWkpFs}vq|~(i^!0~s=DD+8hj*kSIdb}M(0L$n z4Jkt^$Yr4LA?~Jn|_MEs@m-ki5kHuFw%yOPHWNUD_V(xWD^a9$9F!CYA7$=V=GCBwF42y%Ks8A*RL7GixYl z+PfA)fb-iQ5V#Y9s<`)8^XYQ*P_7^T1e?G-=iJQ@+H8EZgtZ_*Bl8nY>#qmH*ZCQ6H!>4yJofQOI1-7U*4>u6Fs$~sY=dY;<=t!iM&=HiG^Yx0b3_1no#4qu z2q?{k^&f#966x=cGp~dPcLf8Ov0l9LwtszN!wZI*xf4L*AW)}Agm*O`Tk+giHACVW z7b&2=Ak7NOdxEPO^kZh%T?y8RnFlicrTlwl)*)B(4~g1tH5EN%`u)Q4T(Y6lQ>~lHs@_g2g7hbL9kOg zuUcd`hrF(Ci{(Jag`RkM^$mSU)S8|?1I@?C;6xDK_TQ&``2q!80SXF=erTVsDG6Lo z)*`M*{7rGE#NL6jg+)cdlA)Vkxl9;@%i}~j;kjcJU_yp04sUWsJnq5#C&&R>jNoHt zxWYWlb-Sx!ad|(tcZtcDjBQ&)=+WOVRc&kU)jrO=1tqL}uTTZ$|21twwb3r5o@ zhBh_3MU=9_fh1x?xbifj3^>C`*n%I~bUP{aFC(Cm@N$sX&rU@lLB)~Z3`f%HX^)f* zO5(Qa+tg#H{HCg-Fc^lK>v*y5xF`=-@@awJm>o10uW}6D#>AXx6e*sw*VTL8m0q%t zR&C{S%Y~#UfWY*+rimiE%T(7c1y4b>&lU6}iUK&IlT`X(-Wfm$CCa9b0aPSKc*`$% zq4p_!%wXY%{m2{C(g*Z>YguJYPOtOY_&b_%?UtT>)v^riMOrG51?!7PVT(Ja_@Gy% zxdj`^?B|d7OiBT#;E8Sk!hPzlr0|d5OVJ82kJJJ&sbxuvEM@}ob=T>o`G<|tR z!l^bgI9+Exfhv5%V#@8FK55q5d?0q{IOWb^D)M8pHj_=^J%OHOEBqN|O!ceVGA`K6 zx6(wp0Bi6SjXDT8)~5(k&H(xfi*+auo#~H8{KlBfB&c}-?F2I#L9x_vUl+_I(Tw0( z3SzpC?s2vfy$=SvY-~LPo!+6j5f2{&_ARO?I*ouno(m*GkIMqJJ0Kl}tm#&=o3j|t zZWGd_%Jj{~{npQaXblbW7bF+3eP1rFg!>N`mc*3VjB^Z;gb0E-6Kn3hf6({x+?Usf zvTFPrf(xyOL;R1h8G3W00)f&q?Bi|2~MwTg&^QxC}{X-;b3bueP>M7v+8?MZO-|iW{)5*rJCX1n1>Tm?Q#dVj~buul_auZ zwL<~Y&(^5{kygV9{TEQeCd*LM`Sa^V>u4q@EAKEo6&eOKy{E+(_+b#0l*Gee?~id2 zliYlxcw!}p(AJndZHV#enfE5e+flj}>A**w%z}ZB#3Y`*M5!@XvXZ`htqZ?Uod0QX z&v-ZXG~EuWk}NAM`c-&z_N!4v-GS^UF_g}ZUz%rr2?tFDIy@3`T><_`DK+hfL(z%9@ua9ybmmjox3A-4@@ILngqL>A!D2MWYS|bu#O)J>oTYKvcYUJY@lTXJAg| zJZ_Q}wkAsu0hVBn_>E)m6Gk3`?ieqCr-uTi+kXHkx*t8u`3`ekQ6MdU%2m}NgX)YL z_g-GH)3gBW{*t_x`~?mBc{4J;r+h^=Xh#k)dE9}Y7GKy;1Nz0kH+>w3Rg3&?en{_2q zm!wmkPJsxL$}iQPBfL(e6Gu>e5V-Gt@v?ff>dy9_`9OGdtbZP8wYscm5h;{8gnLUr zbsH@~Iv2qziK8$Uy{$orAjr*Z`jxnd_x7_t2-Jf-SHexDFl;m4cNMa4Y>#|i4IQ_Q zGo%Xf9t0^XlrM!oTx|c6Ks6+~E2H-?{GcM7Nk^JqNXeR5RpN|oxx0zS1jXLoKu28d z=_k`!TCN}{gLE~^qecXETUGe*2Q}%X%#${fzt1XYU47!-ELbmB5Q8R;1TR9>Ao3gI zqRj?JDyab@zKv&Fcja>Eeyl71j>w9!yG_-aA{tB1*<8LCMW-}JYvIo&T?;b=cUvk^gS?QHROzxaTGl0~UKU}nKEikUR%W0lEd05AU?2lJk^tT; zDK-OJy&wrMI#DxV_;mom7ujXm-$%s>G`Jru)tt;Wk%V$3&uExeZL>Js8E9Y9{tcc$ zdk8hPGr~cXd5~Y|377hf4F>~G9uXfSRG=bLquhfQrfa zOZ2r5&R9mnxaKF`b)iB?ms1)N18ZvB7F$P4k(4#wUtho%3OWbi zY9(mB>*}Qua28XqRzt9UXDanefcqfQpVoxghj4ck)ri~~%mU2PD9dPh=hnoYZ_A#uml)TB>7mlD@fcb$grRk*}yC$4_^Vz%|` zwEbC+`t(b`@BUHjo^;YuqLDTgr4$egPf>SJzte zMw2NttYY$`I0efxu@b|&orEca^C&Y-j^#GE- zZp4f;pHGc?ky9}~66u=*5EXMI6SG{fxP+Mnf%CGTK6_Wl{jiKJJZ|sxuO{n|>G55= zZJ{CZ8sLLuNvZ*ZB7rZeMXX$SL}VV_@1z0}I+W z^9@WskMCDta6MAXmW`_wb!x_ZO-`8=>@F(+{|GSQ4Bn`eMqo(Fa`JNigLd+lghXZ{ zU~Ixv>6fA+CKgI-`Cr?`2W<{r!vg!=zSN`(rC?{gmfqT2(Lcii(Je;9Yk0)%#`_~eBMUnh_Y>>3t*NOE%<7{$P5+k$ zgfUmCoxzFdSAr?bQy6-Zt~fW>JfKB4LtN~Yg=%INHw#{GLU^S;7@`P;y3otH8`R+h z!v)3iyT$P#N&kRZn=zSncGNh2oLYWe2Kh4jCk5@04h3A*EjA}tl&1>%FOCa-S!ECs zgC^8w$;_$5#Ch}l>eTw+`!A1wB36Lp0m($qPV>8&ZS&0#S{F_EqBp9|;GFX)`6i zyQmV|{?kQuH}Nqj$_51>-vaxD7`$3nHMveSxiz({&{(Rd*JJ-|`cH+Kf&HBtha_c{ zA-!#W9hkKpG(|PdjY@%fzx-)BXju72lZH{FOWS>qbR1yloJs%fyWg0f8(ydk;G+Mn zi-5F+B2*&@qKww`KpDke0ICYo3)D*b^r80(S&ICv&u&YA#U&a32mD{F^h=|Y3J&i4 z$_(=*)fPOyQLe-h=jDYtT6>=KDhrUI@lprE5sJQFsCx^y-uz2?Kb>iTmwb)J&0kl9 zH8lxLSso)%qZzp1CR@Nr-^N{N(J=PRwhM&f$GQ`rnk>&455ux)40|c+X~^tFs3y09 zn#<#vV1M5~VBZ@c6;^@!^q?Q1s%jt19P?DfYfzUeB?A%yiY5tDI^r;&L3VU%sSz>v zCY>zKs$(Oy+|(WS9%@}NXyyNJk-&!3G!?OLREn;`8B>=g{y{boq zd5PNa<3g6$VY{|Hc3t%!KYopobjtWFkJ=D`!6CUKX!z{8fi%HT1vS;nUOF{>)0yehSaBi{abHDgf_I;2KbIYeE5JqY!~*?6`;cVCv?3z!h&!0Nrb2sE_-@B zTi1CD+%Vhj!iq5wZ>e$Q=P%h?y2lrzvHyKs5DPRTt$Q_?VMweIy1&$>w>4fsjNkS9 zLIIhSa{0F*))84lkI1n)gRQWE$M;WLhcoV&D(9j z^cp_Re?=IO$};%|TXL-6cME;*x#M5$ zvp-b^a3@Iw5<>DQax)>T7UXL`eU%hXPl5LYU5bQ_yz-rs+`%*MhUyCO0VdcAi`lsM z7-A`$Pw%xtSXqAkn*`TfeA+<$+oF*~dd@5~hxef@Y@{BI&kZj*d@9l3fM9Jc&ktSw zBT0VW+YoUBk_`%XnBD-|-*PpDR;+qpk;CP0I6LD9&TQS4WOlbqvM`xc!l$(|+JEH= ze(m@a6tS3ac5XG2$jcvNy@Y-5JRNpF8-x4PWVZW7qGCkGj2eKx9Zf|c|5Cr7n(6ug zqLkonGcqv|FgDKI9Vm+WrLns4d=7bc^W3IA*P^!tKnu&Rkzr6*Yf0ze(#sNqUmnTN zGvJTp6@$rZUVFCM-W^i&TYDJL*S8Av_F-Z77JpnB?G|3(w`9L(S3QhTjad250zj4r zK)mPE>PLTx5{(n)<+`INJORB&K4AhkO5NQEWde+LTV4IMxzay(juhN9!UVA3|UflZMB$O1sYm3J#1Gssyzhbx0MlqCBca ze__`tAP5ApNv>ZA%a`!LqqFTF(756k4lc*@`Wl05NJQq;@r_QDv+`qFN+AW7?ta5< z3;69(0v+uX?#;8z3;ws78bVBjt5~|dDUYjop3U}hD=dimiTlfQ?gKmFyKI)_AsgKPpo1S=6Gj` zNi&-02+*)M2c)Q~*!i!2i1_eOT^QhO{xu9}N?<{b*jn6?ZAw3a17+e z4q0w)HJD8-scOuNx32O*o`>_^x)h-r!0pI?<4bW<1l?g&ic_6tg3#feyR3hjsypy- zK&V%9K41vqLI~#$pSbvECp%G5cmUNB6|qzbf$KF9HR8P%#by!#f^~X}aXh60Y-takc}{Vv35`fR`JZ zJ7^oC0NTT7mK++6t254*S2^4=&IN7Zak2h!c?6*o5!T$JHst?`6&!-oC)z(0*j2xO zii?A^vvALB=2rF6y0(YNNwMDp6z(H-kUp$R#uID9Ts5R?;pPy~@U6L9ef~X5hlPoG zQTxRuxC$61;h*DeG)P|;qlDHjXq>Xy850nk@si7)KHS#A?)}N-cB`C?M11=K4Qd6t zzEiY6xeOMbAHNrQV$R?d1-SOH!Spr`gLW_%4|g5RWrS!~ig*2>Lhxy?#i;gGuYKr2 zWKt?`Z8i4IL2yUA0Sk6$&@H_E_URA=g{_NhJeab}oO4?R#1qb4=I%N9%>__cI9Nk3 zGltcHU1U1(4Bv7WlxttlbF=cB17U!IHU5nK<&N*Ra_MTm66OPW^-#4s9)4J+qe=f? zeG$KHHxW<|gpb5VBI+p8WUMFOLMy&z9VwA!b)|&zWafwhk_b%z%G( z%q$**#!{YA!18%`Xsd2FqA>djxx@XvtZvc7yJdAQc*x#5l4v`y?JtL4Ia)dSV7%il zqv{rIl+OLuQ2a@PZHuME(Y&qZ-(w46m^=Lb(!lu~3V+>w&L)85Nb_v8~-Qo>1e zTE(5Y1jB+nSxv(EE)dHS;0OVoBud$10%^$J5I0d+oiw_7;RyI%QJ@VCqGU8cYN% zp{u3LgQmh|FnhB;Qz6T;71vN3urK>ezC0HI?-mJ-yaE(6L3q%#_$<>WTZ*ogU;1+T zhq^&D)!3IC>zO>AlIxo|X+`+Q`#}jn98lamiHV3vVP|>%eEpQL4FNB_qJ^TRrKNS% z$^@PF^#^OvIZ%T)CbA7EkqSkKMvMXp6MOr9w!NWALQ}I4G*~1~+(+l*!UFgM)_BhQ zO0*PyqGrF^O>S?mFur1N3(33%_<`5{RRNd}XuVUGB~B6BEgFSZ0i%t^4y zwEX?3Xf1bK`kF%wF>!G#z^AjG35Km9nLabR_#jN+RbC4Lg8&wl-$)d+J{WKf)@w+c zH6;4XGU;kCls%lQ`FfvHbk#ADs_l1z4Whwleh_1+RSZdOL;PCO$ZasKI!!}|R4xD= zcDZ;+s_F9V09IoW^qm_qrqtFFgs+g%218xkhgyV%z;UGOQ-6%|)R}@IJ$kHLD1H2r zMMJcw9STcU--oNw`+Oe!6M;gmSxImz&-1$eFHPYZB#b=GpO!kQ^K7fi9!3|YrQN@6 z3K!*T^lQ-_(0xDAU6y!X0D^Q{jIRF@r8#|YsF*#m%KdF)OO(C*?E8LuPmomY@o8$? zhkr6JK+Z^HQ21|hAu**hL!O>N7|&sNhW=cwTYF!hu%cqr*e0s?PF2O%{-Ah z475n>C)XkciNN(6g|EJ8(c#*=WU;m76y^he5l34a=ZTksD#EiYF; zHo8bQ$0ZgzS4a=i(&obrrQ5`UzY??txBcb*3cll7k$`5pj9OH#&nLymeyo0K(qH{+ zw_1Tj3GAXkKpZ)sJ_0msWHMz%$PPEa>V-J=8j&$oEoMKPcGSeepmkHpUvwzNWB=pe z)Qddf+t(x2knH_QBa+Nl{{=dh=d4%rQm;dkjl1o+Aiq6^`Prmn{Bs_11QPA&zZQW; z0N%W?zcc{(TZUx#E#B!IxpDK!p#{6d1L^LzP*x@52YMi|_No&H!rA9Pp5Eb)HoA_>f7 zrRnLm&D8!v<3;o`j|&|z+ew4$1|)#;qCgvWN|J+Ro^eX*GTDUyoKS1Q00P5ZiTP!! z&Kivghj6qgQd6C|*K^4J2oyx#(1a9SHBZWko_uyjmwj!ghb62$+1m-G0<#Eye&ccvM)N~r`6>4c&5?^+f$SpElJ zMERgG!aQLB7ZtonQWm<*<+?GmffE_{U)!+yOJDIuYKtD^Jc(Ew&BSNqME2EO`sOjm znk$racn0mBvkRNTrg+Db_Q5CwBQrmr@#}H@XE2z&9Cau(Uf|JQ67V!s5;{(1RM2hK zRwOmG>~8LoZ4){}Ul!Cp&pgj&EY^~|h*kdgL%`&1A-?^T3FGD;BH)4F!t29RyBe3v z)&wm=mRt74`1c&$Cad#{e|#@iSeZ^&3f()q%L?bW&CPMRt|K#)nG`GX@57QZ;3?dn z+Z`#h!Gnhp)2NlwNH`hti%W;Rh)1dB#HLk=qD)2}B|xjS>0Z@I8Sr`5GuHjrT4Et< ziKuOh0&PqEm^M-{jcX3sX_-3N~$qqOF(0+8^aj%s$goybCF9p zy4lCbIJy<=aX6`(QD+Z3_X@xs(fQ-~G{(r1&EnnrqGze|URCm6*@DF>OVy7SD0xG+ z83iqd$3_Q}^=Y*x8v9dv9L}`bs-(N@3T0hM*EPxH!ao+3>FkjZn(m)u(A|~$Q}eX3 zo+h0ZASl(Q(?P+@3(w~`tO`q`aB*nT_~So7&~2+K`Vm_?SZC44#+@pt9^y^j_NnPy zK}q?qhwwpD1N~KofBzVKi$dfb@y=(Gz8h2$YNEPr*-n8>!h=9l^393+DGJzHF}*{c z7WjC6<#xu5(i-?Jn4c%Xs~e8O{q?)SAn=&nm)aeJ`d(zO-w%9fMw#tX3%fF=-proh zHI;v2No1QXr>t6vW72K>&l!SjH{T9W7;E7zDDZ&Zr};+vgN=oSHqkED)w=_X`@t1D zIyx><@1hz4sN%AjC`>DN<{Rgd^7q&Ib+kP^jEj~{HUE6vw@6RgAFA(THK8$#jG+&3 z=-?k9c6A6@gq)=3*G@P%6jL$`wO^)gm*Q$ zIoF!GGyl;Zh>&C+K(&N^vVCW zT|*4MId%Q3!9XY>5Iggb^(5RAhQq7=L0=zSU5$mc<7EYIES_5le@%4Tn*S_bS4+qM z$$8bJ*i+k{mw&FKM*y58r|*s7;7nk_(I^*_UY@-#Yw9_vkyTF6*Yw34F?&>?I>)Br z`-a-l*7TL5hKXa{cYBt|zn0LS3haXI#`mHEr&7mKRNOjos{lCL9AL zh{-}2Y=fE9O@4gdPZsN<6NAo`1$1MGu8KOe|}f;j0^7P_enET9(V`e@K)q3om< zoxo`|A>UNxqU{k&xip(m=!b>*0umo6@u@X^gx?12=Q5I>0rlxrpQeI>fBjN>U@>(e zPg*K)pG(tOPfDS}oXuF9%{FgfA%!ys_xyNAWrXK10RUO_KLQUJZ7juEbW6QOy*ODjUIOJ={4xr zKbog>2EvcmN)LCZm|R_>OB&M=5(zeQ*^|C6#mM}reqhbOo#)LY)8_zprUG;q0u&d4 z+v<$v!Ii~Za`NgiiIE0PC+5=61%$ zXb-Lrf6+haa#P}(F}bD{r!itRs%64Q?78tRf3DZW_u9=xp(OObd$24Ba+&mntJGd} zQrfH2f(xOjby)5v^Km-ha?Rq1OJF6$UUJ%-Oc4&vJZAjxm)wZFs=>ismK?t%>lcLU z{#3;l6!)Bo3!;M=@0pqc^ZIg)jm76?xtnS^^KemUX%wm&+PyTEO=;8qdQLtS0p~Nk;a_;JgHe}xVVPHwARa>mgj}jI%qEJ;Ha*Nu-(*u>eC7!WE0gZU zR6%pJIlo-Q#OC5mt1e1vn!r}fL?*nf|B-6Un?GQT-`j@&ht~kn7u);b$Y6X2tq$-? znojzdJW(_%7~x20g#MnF^)w+G>d`G}eUh^ii=a3A(t z_VdH~CUWC5S|lXR*}kpMvgiqxQ%-cBD@vIpN;ShbS^j?2s5uovAz^`wC6qhMh@PKw zc_sCqvqJ*cyz;O$l(0@=cebW&h`nt{N1?bQg4~bravoL(1pfPn!XbFLEdfeeZp`kQ zdcn+%kX)3bS)UZg#@9VSZPxnw?tLWx7i}QH7*K`pwESTqOaO(_{WuuLm@MCPqVQn~ z$vtv1`?jEoK;s^R)Wc1zsBbfA^AiO#tpZD%z;np?*oyquf$)T>#P9-h2w=E*XY&fc zj2{*RLJ2GWWsxWSdZFCdbX8l0pm(s?5=5ottryQaZ{{pSe7^iTOSm2Woc~Gw>y5(lbA$#_bdJQtQ_UD|iwcKVVnZLs zkcE+vrm)|Jm?3(@Cb#VoyK#mGU;M2gkUKws(D!?TxZZTRRY7tH_wji0Uc);LhZ*Lr z+KZiq1wXi`z#G53sF6(I zU>=?2*vx)#AC;i`m1Hhxq-`++L)U3)^NxDrENXEbi2c9sEi{?T)KCu6s~10)_*}t| z&0MXu?n~`8N(^nYMvvlXgI478hQ`r}Mzb(mz$rBWTr0Sx)7m13TlG~wCs*f(VpA;A913V{iPc%nMqub^4R|@jKGwPMqd9 zHWrc}KYh!75dGHc%Go!E^=nEEmu9oUJyf6B8uf;>5`<684pt9Y;qyXH76WMLYk4>^ z8y$!B?|$>~W`(umt468jItSzR{`Rkdx21|9b4ju12)c-$V0kAf7P?<4W;-%k%vOtu zPR_IrmL071g>Im5?bX{48AfV@_7$Ch913pTZv6=FbrCYCp6OtQ1Kt zo0q~hiOAnIsh@bxwTAzTh0au_ad!1H^TBL}ot26GqZLa0Ds!9eB;p> z8gOeAGC84yBR#S*-5&4BznYm4Zw$1c3-EY=kY~|jl%Px58tqNxj+vjvMrgCzodFk{Y7zk^?2^{x ze76G}l~x(*QRNyg+a*na>se{UU4^VujQw$q5Zq?&0B1FinU#VdiQ<~L9Ck}xRU(qt z9+5wBT+R@^DNCi+_-os_#^Ko9P23;HE`d#m5jbygW|bdmq5FL=S@3awCpouPG{NE4 z^lju~I4#1xxQh3t9r&C6^^TWlK!AUW?78o)MW&BldL^`hF?gygT$0HB<s&NR{YTz0a~dGuOm4W*Vx0#C!=DMdFXkL8xqbo=8SBVSp{fKDVk~)IZ%uY zZ#aY<|Bk^R4i+!C@SvCQ9i|zlZHFX39D{B|be{8p&{86go2!$yJ~i`-6*2K)44GoX zzNjJAy+gK}%Y(aenwC>%p*N?Y5W8O7UPSG&t`gH!*iq@TH1~z@5k4BCrkjZ-Vu9z1 zr3v$o`Q1{#v}O%q94_KgN}7pJ&E6lf)|PM!z7nnGpCb_9!{Nhm6s?W$2J1J;-VIE| z_h)bNTYgZUuGpIfsLTI!P4hE>KSD9g2BjbrN`Maryie=+C zvR`enVzIUC3s9S}CbXC9Pft1uQgk=SrQ>iC-QO4x-}&w$D+6MO7h%1Rwu3hg?s_wDjb8S70{XtrQC<4rCoY${7k)B(aBIw?Ken;+Q>dA<6t{NEIIaQMu9 zZY@%!Sqt-;)x^&rJOTdopUVql_IeRb`xf9aEJi^6Vqc}zue~zKX{{}Q1~7A4i8M&Y zA~cCnQlhj22xQZH^2G7RuBGgvUIf5u>n~UZ`ePbFZ%h-Qltt_1x!AG~$#+@hUI)wg))&Y zSsYo)=~_0r=dLrCFdEmImh~NUxQx4!JsvCR^^-rM2hScyje&6hwVFa&O8*JLo?_Qx%CsIj75+_knH`_)bua8kX+su!6 zC3!~+fA+qU{AZHpXKG{YiRYT%qVQN|kwRLr%_=?05;>!S{+aPJddzB8IEhcu$OQe= zuG&rBV}Npqza+6nua1#O3G>!QF%%H%TY^znr)FO!4{{!YuO{hYY6OxiY13 zt0JBKQ11u%hiBe*!lA!C3F_7wcaD!_es`=vfOq897jiVFNsy8%&Yz$E*Mq>0_Y(te z3%_wyDnZio&Q}_juDJo7nnsMKu13VXBG>I7Z=S)(&sYv8Q8>;|@ZXXDpQ~C?68t{P zpvIx9Xh%WWISNbplyi|}X6KNL_%GiA2;rNLXP442RG-Fwu2P5#V&o1U)1xIE^`F2K zqLW@6AU*Z}e)r#<`+suzfBN!YFXjL5=jbq3Ps<@Ey#7~uLGL?Ya(D}7rYcpWcl>!b z|4Pn&iv}sp8knhZkIs*j3!so1)Yt}il65-?xSijE>O5OK7y+)+?2m4?wXO6&=MSXv z`0L-$tH@-BwSlsCQ^i{1V*uz!L*o$w;rytU#}y1mcjpRmzI}pquILkVbwRfeVeg{e-^kX*uAmZenc1dr)>E{db1p zr;&7pTR;17cg$hVjTPcDt-6BtENov?maWRoTpduS4xQt^+-uSS!P0)s&HqIf?U3{? ziaKbF8(^#P&32g3kB$rpDd;P6q-1UoTMKCUZ$^!Bbf*4L@2RNEuvJ zFGZKJ8%P$cST;-hpUw2jhU-__nJTkToG57*-cPv~V#@RNHdcyR0O}Mt<%fj(%Jp;G zKq$DfhL{x_f<_WN!G7C+>Be^BH( z{hd63Jf!fccKuNHd+Z-mAc_mD;Ms3}&~mcTb9FLDI0sz+s4uoF(U7#r&-eXhJ6?m_ zDhX#)OMFbqbLYeL)3dR@jc@{vt1{`}Q<74QF<;9~I6F8b(q zzbnLBfrHxJ{GNArH}<{ycFGy4(D0`nXvoKJ(OkpMk`cmnsK;uduq7R`4j@;2x7oN=WPYUSlCbD;hF`y>{f8qaS)=MuF#$b=Uh2EkYW zCD!plW`QFS3W5uA)pqo(20XK(X-BYCX!B$5jN`ArwPN1=31a80SS*Z${f(K<=rff;9Oo-WH#<@7V zVsK{9TWiw?(j!`(1;+AbPfhJZK<~LJCeOpIv%%|Ea)UQ^Mho_WpG+fmxC2u;c;uK#AN26ZO0BP>CA~0cY zgoS11&i>inr~>X6BX0C<-`Pr{#X`2xGk09r727gu zyiyI zL*J{_jFnO8kRW&T?bO+b?ZZjoq_Ud3_n=s{99=G$ic4Hc>=FVVg;&q*u zHTvE6Gw@c%34qo|y{$G>{rB80kB18j^Ge3C(C!3qy3&?{n7eh^smzJZUq3t5KvkN$ zZFFMzM=+krml0=G_R`*`c@~(wKb}na9%z{4lt4Ga%Bk$#$rx2Gq)TnPODg?KN2W;T z-DbkWP9*+&+S4(&Zh%vhMHwhm;zaRLKRd%%Rg$JnabWn+7tf?|#8{WTG~j6&J*fQ) zOxZC#{WkVA&J;s>9$+`&K-J~V$!joY0uwT`Yy<~!{p^l-ya+_w<$R4TDu03Nvs!cOnGK9xxPQQhO}h;afG zZ>BYWRMwRR*^va{w*QZmksK#E;uOx~Ko2b2-ragJIxuLt3>xPeq>gaM%G#*kzH6AS zut{-f2T7yGa;v3A0M;WZnsJ!p&NsE5g=ja8FgxV-b#`&mQgWbm?34d{_svpYRBPw* z@=I8hFTC4ZQzSudB90o0PG9Mj&|}U&k1VJdMw_});vzhQfW0dN!zxyHjR@m znd>Z2ePxy&p*Q84d8HUK5^l_yfH$=&w_VoL`=LG?l1*|1spl;2)bUXr&-J^6r+gsS z8qxo3RKTL8)>^$GInlv7+iy0&ZKd2>aCTSudnqg|WwuhS!j~eB#DVkYCltFraUNqt z^%SsOV?5w)T>#E69H=;$E}(S!w*m8<9NydB28Pr?IS5zz1Vo+oxs29C2% zB>v~avP~`L+)f11Ujth&n5THWJCGv$kjv4MXcWP_O_=w5Ab<6fiDI0c!3JPdrC6a5 zhI7PJnh38gn`-1ZoI?^S2=@tBd*W~c>!(FNTIIMy zcR#jZX~@4WdIk&U*tcwh5B}3sf@^)0*pZG6F*s}@B|DXyt2TFRwx2_T!Al_phI^yC zb9El+kttR9<1T4cBStF+{Fg=&+V48s986e=_?qrD-TgXXzu=4W4#KqNy7YJs4vccM z+27-O*vow~{V=SL9*#a5#gV0^4Zid^)!$3_k~FZG^pRhy?c< zw`uW!(1P6Ob9rTu|LuxcJC9{Sp}X|YDz576ase4_KQQlv6D4VX2HboPA@!hlD2WaJ zn}K}Q>Lwp!e>~L-pn~dB`KS$w$_OBrY$wna-6`9Zb<*8gTy3am5Wa@p63Vqo=WBmv8mX;Crz z*cd=WQ%-~Z&mp+$?k{{<168?i*^wPfnS~F-9K`)&!hzpp&(p~kPvZ&)Ojs7HhnUv3EGI)@tyI?ael0Qy(wJz*6 z^4yV@xg6B7cxh)aKledO@*zhSH@ijvR}9=Ri^Rv~3=1dj}$5ozuOAoSldxoLQzL z^rOU#P%YtO#mjZL67yQrl&i~rhhfL@D5@d+p7fln^|v;7!;Y)-QI8UAf~59E$rI$y z0q;ZzXCs7St2}6CK$tM0>ZI-eu=iGRRetN+uL2Sx-Q5k+-QfgD0qF*jE(s^yDIrKo zN=!sr8tD{}Mq0WwCk`3BnGPP_r+XF+uQu zEPp93V|`486m8jvDH$R*0w$?mJA;8ZHO&5$RAAYaht zs97t!sMckrI`-eR-klHR>2e&eE&Bn5epi^rbr3RM$zkp#7`8oz_PqJS zKb!97TJH?FNoX+6w>ADi`xw`s^W6CW4Sv_{Ie4a>yKv;?${#i197+g-@W}$3AJ=E! z&IDb|nVZx=tUlE*=taU?dQVah;#?GryvZhJ^V|-}R1@E`pDqN{5P4D>zr%p_dg1}* zaqOJYt_UW7IFYhL^3T<b!l?}F+{l?z*d4>wQe>G+Fd zLdjIhkN_2fTP2fgjcJey%&iAV!6VpSmdWZt$@`ygtQ8b2uX|)na^k{i>ghe;M{sf6r~}g{g`%XcP}}tTV|ln|13nI$ zfIp1SKgy9h`~tbmX@bte$5XqI;z-U$sCcTfNb4%}3n5_0jM^5L5_SvM@;(Z8HT{CU zzz671wzpYd1^V%yRMG5@$|QiL!OchYeewA)cTDLHiU>Bh&(d!t*hIu#wZ z3uJ9p3v&V?iq!naCXS>~503F%ZiI6>k6)lu=<*`UgMLp#N_~1YQ%RH#&D=8?&+MTE zb~)*CaJ4dG*eY0K1480VPDh>I$UA507s_`}>;^v_m%lD;NNt(J5!%WLUXjg`%4J^Y zI-w4j1W1=LzD-3jtS6jEB1e51CGq$dbPk(IHVhvaRie^~XZpsSOS^8SEmn;f?{!HG`uOP3c~Z_-y;aBfeHlBJ6kUgFDD`U9^WrZS?1`N-$7 z?vu6tST<=e)%A+7MUovWgjj61=(SvA^@YJMlm*C}t-dZ#6J+uBStKs5(EB)_8zOeq znSWt*K+j5nKb)N$b@K@r8KiI8zX{ZMqb~S?u(UhiQ^9b`sf6V>;JYKtmINBC=jpEA zDu^`DvSfW+)5F@&uWa5k9auf-RUIZlPE*9-rgxW1OV*UQW2_whW&}yQ#OSppDAAbX zKJXy1Jjn`_x!Vxel46!%Oa-(=x+Nv=S!dIS;>h+vrLtZ&+T0X+wu)uvE zErd^Oj>1;7R4HQ-7@v;+#2rLIoL<;#T7J3(;q2|w+lhG_CU5!}xO3PuJ^ED-W76c`zcD)Ay5TQj4|5xRL-dXc^9J%t0i1a0UGD{w_ z;!t&J>gAdCe(yeO`6a!KkbHUp*ue?3dl#3k7G>cF6q8pMqQ{M_9&vHv501qcL@O7I zWfT1x&YIn^b1`F(Cdx{!hy+`ChrLJ`qH+H`H&>W-0PgF5ZK-7_D6Gt71_Y3f-tl(g z@dtJFkj#wUzN`nCiXz8I(X2JHn8Nf!{OrFbKVOtcbnqc2Q`-$&Gq21>6h4TFXo`ss z0h734D`u&J(Tc-kO=9+NOmc6mXnQsEa4i15Pt8x`?CFX5i9iSkB*FPPU-L9xnj7QfRFX^ zIBeUT58@K{nh)($#J1`oeTA{rFJATm1w9WQS^hlX>976h#}st!!es#NfH@cl6FAdcWq_x9avs5Ml*&1WVXbg7!)HuQm}Q!T6znQQpUy8pbrF==~<^{cPY5h84k^|r7j9` zJ_H)|fH7ehUB-s$vGBpxRH?d|TwpwQ|FaMmIYfM$T0C9!VdUJZDps^C8t7L(8E|-s z`9Qgnyx2$3AU@I|fn??gl0H!Xk$~rA$?WPW7oG2Tjw?2WqPa%Q3fb#}dHUN0?scC( zilc(jy+;nF(^L`4Zi?Z-eU#OMKmKv{I&YN`L-*n6MgnZL-|lM?EDrg%h0o0{5cD$0B$ zHqN-J_?F@0q!!Fv9pdVIE_o+owOG=#6M7&9i;8))~4Wfm}B8?UyV|-wwB91 zc$7_L0*Szp&=|>5w9z?ph#oq`&Ks30OhK{Zp|n;0Dg8=HS*A_+uno}w$wnlzPbF+kP#FE;rXRdm=d$ZLIu#7tmd+ zZA>stwqzT8S4%fVB7?HOuhAYRz+jno`vg{jfQ=Xs!vikfa}bISSM$lwHh_MRoDb(i zG<-B_$59&Cr;bkkgP#gfD8GB0@Sl8hStnElOZny+6bE8iVr;v}U=X*k9KCJjCM#C; zA}2CqUu9mIF2{l3f|fkD;pQV~zU;8+bN*oVOzA)yQK%O+HBNY#5Nq}L?8h%ZN!IPp zEs`hsvlJG12aPRe60wf*D)AmxvD7v21w1&S031O?b}H`8_O=L-vbWJ)ykDe5D8JU5 z2#k?Ez=twnZ=p@Ku2Fsu4j2-_LZ(Dj_T7$-4~^9(@F_xsksSsMjO!QM?cfk8nsez+ z%}hK{CZS=kflRt@yRbF)ij$~e8zzyccc4e!z-|R|##;28EUIJPEa?xNej4-Kz5R_% z9rsm3)ewdSvWe-UfkyJ0HTgSOK`Pt6m61*^?#g#tJ9m{l{Boh4kHjK_8dFKbue!BY zuDL=FL21V6qbmLs)lp@=4zrGxZo!SnFWx~fle+H6SSI(F3mGeA@1mCiO)M?`gpmEuuJv=M{) z@iqDyL}CoDj+tH}eOnh+@FCi5#I3{@kNr|=Czg?3P^L5-#^Px+%jdv*EfCX0=?NJx z?-=+zfB2i0i_@mQJmRQxf=4Waj612cNo?O&`_?*+)YHo2-zy?B=E4N!>g3BM?M+c|lYEIt`tzzwrn64@O z_gUS4#q%}+G^n|(GazjD-3d0$9v9d|Kf&aiZL~9@+rX~QSp4}KM^fXL#$wUXcx{tO zWK8@@4mB$_wLd8T#rT)jygh!ZS#V1-pT{w9Iz1VUW8P$a>Ae()vO-P8*1t~izNY55 zy5?7-W2UN3k)52MdE-d!r(e%*Bui&IPuZ>?6XRjaJ`83_1 z$#1v>pu4uesUJx@>cWBslE?gHcpN~jPCrf<8q{gP=>^^tPw_PNx`b0z zSryGZ!H!5e@1P)&_Ph*!$)v7+qGYrc7J|@1y5~oCBS|Cel;w+ZLON$J9%Mi4cru%a zu4?!C1)=+gDCF2Z_)Tb{8S!5wtef5%OBzQYJ3Cr z8FS~-zuECtUYxPbs|G#?!UG(=gf?j@C5VxRr&H5W&t=6kVeySri(H+@SSBLR>pveK zS`x%IVF=3k`EVObdVW97lR`<9OdhacARwjpRC6kFE1Ps@V5OLw+;jgmsRu7cJX;Rd zia!VV*{y{LHoAu=#6Q>kWWNv0X&{;cwLt7cGql;h2$YT;P_8kGwjEZK1tmjocFa~K zJ_R)6a8PMz6WkR^FV10`;Ln>-jx*6(&K9 zy@@wN&?bTYog%A#h4ei29$O*yxS^N)u)!7nAFO=0+~$M4$mJ_dq7F$zDVwAS0<^s; zEvE$)U&`=PqKBn%Wo~wczDM-bZ=^iBW#(^kw4oYnHG3aTebpB$cKm(%H62D3!d4;w zI~G4aB^5q(;RI2!Ob^Evjq(sap6EK6hEH3C>k2k<*j99$d6mhetc95#`WXh=UhICT z*rLufovCr}h|sY(lR0k&o&4~qpq>&?n?%N%#MN;<3rAi>mbr4!hur!U^i6_pmR&7* zLn0+hDeItg*M7?*w9#=_8Hc#Sp>_rqIz_lbL%SusnSu96u8W8AhbIY^`h6;A>;>@@ zlQHX$hSz8^v2iZ`Ov}N;x05eaMYwL)S^RXTu*(-4WsLB2u@5?43uKwOtw766`GgF} z%<{(1FIxYIdx&UKD`y=PU`&b))qwjHEC6)Y-js#edRYaPyw;i(U*+u_h-XEJjip&s zwpm@%2)@KbV+&u(TAMoWfr3*ErB}4>iM-Ws3#uF4%kT0M8szo1bl-lo$SCw$_iEGm zyiYFbpEqc|UFXcZa9&smcgHxpE``6*SK@t2m=?~|>d{(sjw6++!;t zu6oHO7D~$c2MMTWB_msnn>vbUKNfBq>t7$nx#pecO)>J&i{4E-w+Wld8hn8|Fk*=0 zTuF;7MkM^jylC`$hciaWtD^CE#|TB6uhYpd75e*6)8FKgi;01pVab&5SQT^{>ND@Q z>DO(Pi4^F<+}08*9?JUFJ7kG5%cE&3=*?n>=S2%*2G#tp$L$@>_=x{qvuJM|LVl(5d6%C3m21z{x(p(3 zX-_vk;>wGPJyry>ZIh2XJ+_=SqME&9EuKjgY!mLb^PAn&?|g7dG`vybK7gJlhew!( zQF04xbr~nUcJ*kMB#ii?Vc7Sns_5&}6c3ahCzfC-R6b~Xa-HA_D_d{3EkW=Y@5XwH zN$kz$)o|lbm?L=m2@@GqG-rQ>B?kZ^FymGQX;M1r!L6m9A?ZoV)pKXs4)>NkuT|+VMcj2RQf@#wY6F`-8oa zmF!pW#2{R^qfn8nrCQAr=osSMpGf|x`qFw@@*_-pu7Rt0DfiveC`t;klS8jpk0L-% zJb#~p?9@T$vmNZFK$-%!uWc}Fa=SuACdVyzPz(CRa7nvK1UD0Q12IO+aX;HTV;2lI zWfV*#liy5`pw{E0@~)-GSF%lvVoryC&zc725>P`fo^vDz1*W`}vFpp5t%maGPyHOT z6*mHxneVe9C3b2-GKo(IcF6z-8s=&6aVA;&D2=jecK@ z3d+XQR?fPZ4X%||y~%B+EvKxgpp{Bn=H~2}&!?(pT8xX!9RoJMRf; zcsF9jpo|!cs1x z*R`eoicw##o>oh+_NDR<}D*vZ?Q+m5y^D%YL{*LFR99)|LiR?LB)9pfwP zK=h{Otrbz*svy(DjS(vM3H2j{!VfM$euez<`$VPGo@v8#k!O65H8&%xXx?klIYLMu z%m?tjOg*W+JD=-CMi-=RFRD;E@$Ws$ z+$2yhLq{L3NoN%5P0frtfOtpws9$tB1aF1<0nWvCHN!)3QQVnzIm9#I(}|!k>mN8p zU0LUHDEX*r?7?7@rU|ZjP!68L`}08{$iY|CQMB!;cA_Z+-e)VnY`LuLWnd<;-biLPQ~a?S0*Ih9A7aOPZIJ7lE^|e z@JnMu4C9&Lqrh%ESn18XgNXM-So%>dYQZ%LE9vfOrMuUW;P`xPNrQx0=I?i)^rxQ4 z`nHheX(Vdd36i25`zxD`rh|R%P|xk#1^V{TH(WnMSj^pWPz|G}`#vLfu*clHwT|eS z{b?Em+o&_R%0u{0^pJ_8R2jqKwID;)kfKlXCa7+nl2-k3gLgsGLw7%1N!#CBS(0j5 z;m29-klBExx7>4pPO->W*utTzjQ+YS*>qG)p3K2bpuXquJdMjtCYmgaNr7jDE=l#L z(%gyHI;ku>HAL$17lXk{>&>NyV-dT!lwC43lcX1X?t>>`Z>o4O5VetV<&jo>G`^)^ zGHbeu@<2+el3JN}S~^WTo!oP5RVWEs^GXJsHBlOobl>$Ol?8cSBg*eX}Lo8JMn_b2yS-$m0M@B~JW5F}1 z;VIdub4hIg%#IR)*%$h9Bti!0Pz%k%1yM6S5NO|YCO^iAVHOvp!16*u_7h?zlB9Qg z4|n+H7Amr$2Tr|UvuP7BFEWT#LWx0U-nHl~7GIU$$MV0FRStWc0UN5WGtpi&n4(~s z`MJ~hk+&Jt&CT~7fVhH3pnpqjOlUoDIJlKhh$2P{qUda}k@b3Q70_&>HNTT;N$iS< zjtv@5&1Bkau65-+Sa`|u`k|($HP_oPy{Rkp!V{O1LBZZ6t+7+0WY)scNoHB8r8*Lq zPcsm2Htr>o;7w{7lwj-U3QQcslq-qU9(4VOw`^48pA)UIZrgH zCby={j-i&;SEqQFfZ@VO9LN=SK5e%AzjDPKsepK{rj1lwkeGb8WozTnR_b^A``Zx; zG-^NM!)npVlFf;AH0;ZFsmS3}eo!qm>(HcVz| z>=*ku^;nu-k6iA)jyWOrk5n&qLBBe^M`3*RCwLR_i%zh2{5&3kmh9zevyyld$wj8h z-S9d=2B`rH)OugosxI8}N33OyFpJ+y_@}p~4g_FvCJE>TNGM?LYPHd9CfG>cji3)| zkwFB}iwu{2=}FnRZdYH}{549v~x0_*p-B(Cy?Vp5NT5S~Za(tdx zf}ljTc5(T(0;OLl;QlqzR5q|%tfm>PADKSBAU4) z0GZ96+HG_Hp*e{K|Ev4B#x>&&!H?JCO96~aT%chS?YiqQnZXCm7`XLJc<36TN zxX>l{-9GpPNA*ZXB0F%+pQ_~DX+1kRrI4qp%h%{gFW+*$5Isgd70XYE6SEPVS&~4H zv1oxP;(5Mwr9K<@rJOJ6qBc6I;Z+Smyz98`zpWaWCg^QJQTr;9Bo0Xa^>=Q)3ns>gMp z+<;sRlbhOj_7B+am7P|1uXoF-q)v(@bbgy z()02OPIzhsGN^s+c|pIlamj&HylI)*=aBT^? z>V(51Up%CSFjeGH-QdkdI*%@ddCdZKI-Bi@KK9%rtSlfc?nOMBf0Ywu69V1AwJ5Cm z!Ly-cJRIk9_3?&P`N32lTopM)$9T97*RD;QeL+Thnt~+VKL zB97!1u@U&dE^Esfu0xUBGTMq6I%c&*aS^{eoE!S zuJWPVdm!~u_qr#!+lcf-oM0O{s0Wu0xz09VNH9i`&rlSt3hVX6R@phJ!Pw1{{HA8I ze!GgO!q zD7d7!f|SrbP7w9z+hyLbWaP4A&e4APifA_8KJ#~3RC+J?u+XYy0t!Z7A ze%|u)FvBeE3yy84e|(qTL9Lr~rWJ*Rz(<9KJw!zA(nUa}_Iw*MZleoM46tRdph-11 zgAkj8hmP?9B)A}<;Pg3CQ6l3zhh%Q_D7NFAU?r)6aaB={XmVkmg>UfcPDTgBbmleJ zK&lD{c3vpX;%Q0~YeJ5$Um{S;_ozgCis2d1rLut$P`||c5I*XUWY@(8nFEt|er=^) zbf9zMW64Mv$PELyB#ooR%v{qT&xQ*0+5HoQ-$04$vQ8B=x!Rq!2hI&;C(*N?Jg?uG z49D4feoRNw$IkiS)`7;9&)3JK|9nc9~v6a}qf4zjD zRt!nCv`1P`?xl9b@`oCq_#Bi>Q*kK$JpKT;C(vb|0<9STMZ6RS(u>2RvbNPceS)+h z-iNqkXPtH=cArPI6L`AO;wW(5560=@4qbN=^% zh$vxfW%aZfZx=k-C?YU-S>s< z`Q)~vpG?=Xim~^u z)~e(ge)#JSnni=_0Aq1SvCDceKK2lvD03q?6%_H-P&HhOvs;{%%qj}ojO9;D!ODt@ z>Ohs5wNaZ2$XN*xnQGljV*RX0L>0@f?+)+lL7ul_tGX9D;tP~mFSDf)HS~LYs8=zL z0l|fS;SW=$n?@%xo2@X}z_bJh3`ZOsK`tXfMUh=E%tD#84W&mQZsyg^yf^a6m3Lkx zi7Q2gSy2N&4uGN#<7B$%Wo&J&UQg}Tt3MRnM!@8?X&d|Rh_HkfiJ0RxDSlpegqcl2 z*lQ8rfobI$GrhlVv>aaG@^wRYk7M}{{+doy5HFtOnbnKG{|_#jOliO*;{EKc{2wf8 zq;RgH+2{YC{=b~JR^$^F|28F!xC3@v1+^HM^Y_VVvBhkFqi$~^NKjKxgdhZ^T%m0@ zkCssI?;lGA0?cj-9Ayb^Mov2t7#K_jwEQoBsslMTNf>a#Y0Wm_ii0C*`SxTuZpH-Y zb3eU3m`um=%r@Y#?0YAleD#K+ja@H}QoJMJAtInS`HsOZeHYwK0f^J8D&ZBmZm)Qx6S&OJ1$2U%?Id7v4E8OOQSQK<3udtX8m)! zO0^!CE5B6zxS6+QQTiGBJ3j<)$kR8YrU0ju+2K+yQGsNNItGPvXXw<0ju|EhT{&iVI|0{BQ} zDl;VUyCfn$hs_!e&UeRmijpS4mbuJka;10)W&&z73SV5r2-VH2!$@_x@uM-IBiNTReh1A3;(upmZa2QzE3zT)*r!W)^^-a0d?I%43k?KAi{HkY#wWww&tfyZjETiIq{gJHc$`Uju)R5*)dpq?Ho- zFLFdkfNnyP;Dfw#Ao`8-6hNF~A1*!hNi`3`emPJbQJ|EfJs2vL?);+z-TasP6vpE1 zh9IJ*QbjM7f79tp{igD>CLyMNJyNUuXJJ}CrjIAlm@=@%wG?6>jD1{p@m=Qbbnj@; zv8b7_;J-@y$AH+#_)uK3@pZGF?cdv^+l?v)T3pLO?C1bpPtq~(O=97On+_qRefgw+M)vF}OmFl=^x&TbKxWn~?6Hs7jBirWX!!|} z@1)&O)6eTNh8$Bm{8M(UcvE}ebqG^&Ln6BI*~+$ueKvT_2KlFy`=ZJxC@5Ls+H@K3 zGu`<hMm*-wi81TznO3=n)i)b_|2q&69yQ0(AuGAl}R8P zjHF@NsKg@zD{}p~B6tgqGE8_Q#R6BP5}f+l_)1&ffzH)G`evNlG34hP$I>Je%o2#~ z?UNM^Afce--X0WOBzJ7whFwy=hKrPcWci-YIB%p`g=wmRS%g)C0!Q9M6bCUW#$GHw z2i}WpQ6xTSyFm5KxiMUw{|Zk0q9*331?B{~%3u~YR6KG-voDmgOC9|>H``mDG{5uh z0(^Fl?xjLHZxCdaS!?vm%5BSZjzQ%4FE8ig zpaD~zV1ZKTdalB=tUdfJkSmYQ1v;=8BT+!2mp;1bg89~JkT_%sUU9fkJu34S?r8IZ ztTlHimu_kovEq370ec%!D3(YQ-1CwO1`bQnw_yq`1vsCM%A+J>^w7b{ND)8oZ(9F| z{!W)e9;3ylf%%-~QDMLd^M-v$fuIECe}9tw&?VT0g*em49`1wa+M~|CI@y$u`v~>{ zNx!L4e`OU<;2v82O=e#q&7JNw^v*S}1p3$|7O)7PZp7W<1*gJD44 z1;wVuhY!U>Qz2Z|BRK(6561<#l8G!yk;jGM*JrCTa74QILu<7yswU!e-NY95v5xIt zryG$NITyl<4GzBvFUNxQb#b~Ys1H{RVjNaT9KG_0aQGbX77(TWjz_{AvH_|C4CT52 z3Ug!|#TXmj2!=ihAcE~J!gD)6f-3YXXueB*6MtOjhizcX->>)ty-DX#!mJTF3j7qwEL96vUMz35V)l+TcrC;Y%I{NIfx?Yode5 ze{(Z2r(bh8%bJ51g1RH#EC;`71rKKBrQ7=>E+F~)OfS2}LL>=q^Pal1u602SxfHHX=R*m$)irfwCa=PKeyR`@GjZ-kxKGnMOb_873Ow@IaiwL;~Cd= z_SqJgBCsO(xB^&=siDT$f{dlw6Z(EHB7!1^79Eg(Tl4y7PjlNi@==}}KM{fh4+G-wR^W9rza{AUtmeC;$&IXF z*K2Pls;TtZu1P7?qZ2<%qj;R=jiA#L~)Q<1Wq$$T!1o)&JV`o z-FLf4fsfe;N~2J|9H|s|m2jhf_=^~mA_6eG(PwN!=KuWjf~k!!Vq!*hT(^JtWBjE) zN2z@ROpEKw@Ro4iPD%7A>cGl;>YMf`{;pN(^FHAF(qZ>n>1n78QevQ+wzCOu*7#Uk zi^H4@rBfb&ANaV1CWngHsxXU06a)vha$6l^n=a29t)WwjsR5mq<0PS&Fo)Hi7`CrJ zj;~il+B=chF0LcU?*2t*`xYiMdgKWlOT4G|-6dy)O%6s6J zm96c-x9xxB`)?EVj^54>MAdKcctY%5&Mc)9O%C_k`W>7k#QWotx1>~4Ihp`rvTI#SR zbc^!iCr4eCEW2D+6E#&M9~vED*8>*>qEewbi<6ihtBaE7t20F=UmzliWc&Tz>SI=d z);&u$BwI(su|OxNoA%VS3$b0q%eTg(Il)R~Q=ZDVx_+K_?n42?+i|OZo<=EzB#kQ^lS75B+Q*5|`th?OZ6id*65 z{WG%wJ`fcd+YmOi9&9xv5k65vg_)RgHLgUltq9Y|nL_>HCv*3_=xoF>{JV33f(WDr0x&|%cugxYCXgx zs1M*^1$f+hiu@^fPGJ{(%M_Pz3Xh0(X?Xpu_dy1GU;o!;d-)|A%I}C+V%WjqW5U_s zU@_$w`& z#wZh@DDr_R!&!#gnqgchq2Pi7O(^ZGp+=i<#nB!xM=HO=r6C!Vfg`d_Smv5s;;Ud| zjoy5WuC0D$@@Ln-KrvN*Swg&I|k;PH2|-e0FWVSzD%4Mibk4v=p)x ze%OBY?(hPHd3n&=my(foNAE5c^AE=Dt}guk;^+f^E9IaR3cz6lkFf%e#MXN7CrhUO zhFSp8V9ovzRz+=(E%lYdB2v9cA|g1zbTo^`aYUWoL}af35{)G$dwp?W5?vhd;;Zv= zM^f?+OQ4jEE&x!(6Wd+i$<0Z9Jzc`2gDvWm6gwD^Imb4I~I<&c2^0=Pcu-I z(3n0&?L!Bs{iw^ReGaGed(%(Xk?oGlJR-p?xn1${B zON*yI{H<#IuT=l2)9zoOivRv!104aoJI_F;91P3&Ki_R7eCmI^`M<8^ua97Ztt&HD z`(HHPzkl0}RsJ7d38|4EM0t)z%~=`$EYkn+S)!;Q@5SEHA>@c;SI zKtrqv`QP{UzdCKWTa(~7jqYb&Z~nszf^lXT|B2jD3XjWoch1)S!wbH+JtX~aumA5K za}3}I_~g&szW+_N1D`$Ta+&|XU*&&YkpFvwz+L~pHwYfN|9`WBbhS|7oQ~z}Ketv; zz^&3r(mWSqs8GbwtuU3o1Xv;hc!sF8(#jYux$5W!4emeAtc>4o|1M-|(%H9sGPToV^U11h#^I&04ghkVQ%=7~ z0Tr&tSdzES_6B>PNwi{I5{+L0O=+iR87RFDd}ac zr$P4937FXbn9CbJm6taiD_#!0ZNZDX>gM$0_5oET$FoaNlfi-)$8PW86)U(I)j%cv zf=DUlDF&P$fp5;J@)7_3;?Mn}C)L<3FQYSPNrVF5)9V7mn~MV~xN-atIGuRxfuEPn zH+vl!DYVRdq1dPZ#%{%nv!!?eHJHEOZM|oM0}{+)_$N`}0?2e)gw|qzP`~R0JpclD zrN8#;;N3THn* zKf_5uxpm!eWCvVg+RQ0}h(w7OO>YKl1TWH`K|xl$dhxv4hSz1t1ki=5wrHfR^me_F zh|`qA>g2!3i@n2s+`u`;pECCI_!@LtLg4=KWy$+hN!yuHn{lsl0f(O@V~_IaVu|lT zwahYU@+uWK(s=9Eu z@DZR)hQNIH@F5cD58ibKqEHgcJ6^&&H|TJR5EPU(5XQ}j3t0F?0znTA!|k-VM*o85 z0F(;E=Ru7s(1yORw07)BLq2gdeaNGrz(?}39jgP-n>)Z=)+wrY+@(c=#E=p+Lz$nR zoVdi_gFB1^uWXvs{qmUGYBAc~UUJMt>eat+=s8^|9@J$^-iD8@ZteWFZEV-0zb-sj zO(CXt83U&eun~UP6w-WE0VW1&E02GV0e9ssyhUM=Znib@DMqZi7qIGhXkYkp(&r(5 zi~+1rezYZzk0Eg&H08O4JJ<0ggvdmA0KrY`^7q?TGF8$2>WX$`vTC8{L0o>I@PGwC zA94EjPP?-5=QOPoQn~6$NU_JOTicu27PQBZkL9sh`TJ$5 z6_L^uApjb-A1Ukh8Rxle4XEgL+3?r=wz`hRGktZ~8Cs;1eYa<2!Dy%!5fZ~Ef*BF< zQ=hK9e5gss;G5d=TC@KCPjeB!@wCVK?~d%3NZR>z``|2^Y=VNLVAcYqx$m6C)V&ma z#dJ4^O-fnhAab8p93F6U+$Mn+hU!?~x1m)=-PG#am1AB~c9o z0S?`-SKG#fTd7ZZz~lnW4p1E#d(l?@ICkzUSpF}dvI=nx&I2c9fHz4X_A0MBtXDbby!3(fG z>_O57p6;)*k5^&C8G^&gT|RPV$S)Kz)K}qTgLyf9p%hZf0?JGAXI$oAXpBk37JKlK zqBPhz-Ws?&!?7b^*J3!#=(eUiETIo04nn!Yc1|{(1jgEK#r;72vY0U*t%26l40s95 zEjl2Jieh!c7+1Fdbwud-0dx%5%=@pFJsJ)b5VQxSqogD+o@T8*W29nq!ctF zNpQkfy$l`oEhx?!+qj{rVT#0o;Sla{1mrt9-RSyp>by@O2AmiTa7g82-XE?oSp!>T z0$QpjM8z1MWp&vFz96H-)@sJK4nd_9&kr;R@gJ4f?lf1G zmqUIPZ)bgW%kEh8uDNA3tdThRHgMO~97NMtc*Cyp%~W&+NfTesHu;9AJSnSmyz=Y-l5q+ux?+92+V;7Ov(zYU4~V|k&n`F=Nd!u7+3c>*xR4r zF{ws-av3Eq%It(%t&FQlW+wDZjlH3h)A|w&s-s+nK-*CHlq6@#Q^f!!o`HiyS>pD{ zJE1JeEl0on@BNdc7|;Q|uT><_Y!(u|NR%2x+ThT|g$?PxI4mynoC@0?`S#W<5i0kG zFpu2^KSahYbZf)yvvtInP`g9&C4C<@zeca5d(+rk!Fgv{=*@l|;sOVpo#1kfsp&^d z^6H-3m=4$q4Q2~26`2xZ_W=Q%Sbp0Sf|9Jx0*SMBF0C5+<8)wevd{cms0CuDWMgB? zTstkpJmJLfj5Opr9EsZTI@@o2vBNit)e8k6(&^}97wp83d zOxDORkYmorLgk3al^#DyJAIoen&vu`74Gd3cX1NzeUagtnt3ua^J~#(Q;?S^|1NFp z{PJkExHtA%D$6;-rDIE^tAr1gIzOf&&{}43d*JCed-wN)3URJr9)mE2yd35beUf>` z4t&JHCxRv?kDc+j^x;3=xl-$d9#WsOj z)ZlJ;>jIYnk%g*@kB!*dtGO0m?yacs1H5bpP>=N4)Ud#=tRX>^P%w>%G0`~$gqX+4@wO9{G9)qxWbTU;Hdf?MX&B`Q zd)c0h1}9TRSIo@MQDLJmeX&;u#xqZ6B)<$eam0C<@3QAy9r<)C-S>KPQIP81@xU~HWSk?- zcfceoyK4B4`pKZleeKL?>B-5@B6B%(dIG8bJ78o?VwK&KLluoDCq~?)@f*=svJBN8 zsc^pDwhi6tHu*u>De=+`qBpm@5Pn><K#_Q&{4A*o~zWghVJ65D{mIBhhwLheyR99!CiV4i!@|iG=Khg<23U+9`Fo zErJM)S713B-gdQ{1bxNMzP|L0RDLu96KiSNTk8H`M@AZnkW1-BYw~%ni~_C<4gHii zo$+$Fs5DHAJyxt$G(KGd89LF)nP8+6lf1+9=X0Ch6_cml69CmFHJ{EB_tw_Y#7xDB zJhTfb#+F2liF?1tGvR%<#7#A8e9@9v{%oPR<_14wq1fWF*1JJe$^lxRLsuv=uKLe& z3A!(cc^#A;^7==A{N>hZ{N6jdMfTO5q*co1np4cJ%&0W3-z52lW0du^tu}Dmk!Ut+ zMrNO0NvPsd zTWoS-V~2xli`l(16pUpbr;R_kIz8g`tXQ8?I4F;`;_?5bPk;;VQ%`7vAlF`GzP zdg?A>HVWg6tO|{s$TRj%nt`dKw5OZ}>dVbXJ!G2iXp&~(oxErH?}^CQT?l2+qn&M) zI=MGy55J)XI$GfHw-KZGejS!1U-L(!m3Z?y+xO7#-hI%j3?o)3p0b*6jFzOdfA70T zu5$!)ZjAt|(#r>2WxK#JoGLNAYO>#WQQ*UIY+U-Q@bJtyvpI3}nJEda#IcD8`xn~9 z@L@U45N485gLMgT!E>!IQi*95I_XaP;`0kyp)Ph#A<4N~hj&KfJVc1cdb*Q) ze3v}W*N}UE&(;*{>FGU3T<1D28JfPBAyU-$qeu6C@8f&Ff583W zo`2w+$2p(RIiK@-K0og_*EqX&dV}4kIV>6~dhyH_vt=O;&>!uk7Bv;s->OXAbH90f zT^16tH>H%Gg&71+vt2G`;C-`dOQ;feqrTve3c2Koug&oKj>))RP8uu;82x6`Sbt^` z{hNJ%+0d}z*U!{Bq@PmO{*U>c4!zaxCVoz+$8EWEE+%(s<%U;%Xh&(XX`dY{%$0I@ zjgSL}po>D4pPnK*p@cX_g9`B}2tT*jQX~B|c(`bEHp6oRa$AV#Vz}}%;V&$-bk0A1 zVr_7f14NS{km_}H_ek#6$mFxakjOn5Q(th0+z!cCE-$@n^zANYl?Z&8ls0H;%08OI ze~#p>4}VKNRNl!f{cunrTs+l`bFNZZQ;9g+%;J9oO8q=3Ht#}H_Tn~vLq#I|ih{wi zosnMc=Z}0;{Ry3xr&w^2qI>sL70k_2RqAwg()28*&1Fu4)|0RL%H_vbPJYqn%2Ecs zKF@!-_I0iFJh`!YeL&kOnUkPFv*1tP48bLG^vu@si+tpyCJ$w(&%euxw^0`@woqjD zW(aPI@f|)hxthF?a{P!P`f>e_nbWQ7rc0#PyqV`~VKyTN`cu51$5y!&VJJ*61R@Xv ziP<(5oUT&PYBpJW6eT?W7TAfk@3D-)0yT0uAyJKX(XrW%h9>2YfLCKp_HsxD_2a7x z;2YTmC@aZ1Mj06uAPC3qr}dBnjU$#@5w&2K-0h7%YS+GNF$V$RjyMK!T{R*QF!99o z^fw2|vS&0j_BslvX;-p@IXoknFE?^wjNaAO%?{aOLI~l%U5tB+0<)8992x%D!t%PNAqy;nM>U*O zql5eCaJnaGM)w+^oXom_r6`fn^JLjp%GtY0sb(ZgZ{$54BeDaqgO)~kS_Dx+AJer`=9&+-b+c|6e`n%8iA>8}af4P9HqZdg9A@3{O|G7+&=a#RXv+@g zF0T^_U-O;zCx%kj_ogvohX0`-hISZx=yU2qfGH9^qUpW&%}2O&41`aZ7*m$e>zNo9 zbO=RotXEahqtEya+?iW!7+D{e`js1)} zZ61+c>E|3NPAc-B2l#+VbKI;TyNhB5tU$}B~_eL3Lh9ujg#CIK} zn)kTDWUk8BeYD>R%c;bSW#XzcM6p0Fh>pCa-7Pq(PSI}`uW`{VZlFT(by59wsa>hx zu}MWAqR2IrawJdDxf#uH;d7Oz1Cq(z8olI?=cwdqD z8AGzikk03I41|n5GGeEteZ42MW8mYh{S5Y)}?GmE3w+P(kQJw4b=`M!Cs%ZL}h(5z?!udh=DQ zkYEAhPuduV3d78MQh6$Rv6I7a#S`%Pm%|=<6|$OJ$FQn#u)BAoVui}2N6jw*KWb1% zjXr(b$G8jxRQt#@mr>{Xpcxu(IrO%aF1>)S_z5yW7W=d;6vNc^KE=f~d;j z1Cq}aO)Ne>>eDhW+FDpu3tN?NDZZdJt!yq6&@HXY@t;4k8b7WvAF9(Q>z~)!8>(Z* zp#4*esmw2EZ%86AQ=v}RR^)r-YoT^?^p)y27X+p1E+ZpvcvCE^bDuA1GRJcI~ra@oaTAU;s%jZw@4B0Z@iFy}fOJ+b-f*4E7t)1lRt z`ktM%(#lx-&X*6(!RyhNA1g0~Q)aUyerXQ15*y*H%@E@1)47+1zf9S~{JYX4C+Ludh3_6eao|iYpREDS<8UFZ zaQ09M>rbGFIk3qk)NYRBGWS!pA;fLbXxDlw+*@eUosGu0!VN0_ z(JcL6_>dq07IrZF&}DEDe+8922!Klef4aZi?Ek5o+5PIcFW6ee0=wB>omsW}uSW>r hFGej2B!Rj=1fD^ Tensor + out = F.dropout(x, p=prob, training=True) + out = residual + out + return out + + +class EncdecMultiheadAttn(nn.Module): + """Multi-headed attention. + + See "Attention Is All You Need" for more details. + """ + + def __init__(self, embed_dim, num_heads, dropout=0.0, bias=False, include_norm_add=False, impl="fast"): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + self.bias = bias + self.include_norm_add = include_norm_add + self.impl = impl + self.scaling = self.head_dim ** -0.5 + + self.in_proj_weight_q = Parameter(torch.empty(embed_dim, embed_dim)) + self.in_proj_weight_kv = Parameter(torch.empty(2 * embed_dim, embed_dim)) + self.out_proj_weight = Parameter(torch.empty(embed_dim, embed_dim)) + if self.bias: + assert impl != "fast", "ERROR! The Fast implementation does not support biases!" + self.in_proj_bias_q = Parameter(torch.empty(embed_dim)) + self.in_proj_bias_kv = Parameter(torch.empty(2 * embed_dim)) + self.out_proj_bias = Parameter(torch.empty(embed_dim)) + else: + self.register_parameter("in_proj_bias_q", None) + self.register_parameter("in_proj_bias_kv", None) + self.in_proj_bias_q = None + self.in_proj_bias_kv = None + self.out_proj_bias = None + if self.include_norm_add: + if impl == "fast": + self.lyr_nrm_gamma_weights = Parameter(torch.empty(embed_dim)) + self.lyr_nrm_beta_weights = Parameter(torch.empty(embed_dim)) + self.lyr_nrm = None + else: + self.register_parameter("lyr_norm_gamma_weights", None) + self.register_parameter("lyr_norm_beta_weights", None) + self.lyr_nrm_gamma_weights = None + self.lyr_nrm_beta_weights = None + self.lyr_nrm = FusedLayerNorm(embed_dim) + self.reset_parameters() + + if self.include_norm_add: + if impl == "fast": + self.attn_func = fast_encdec_attn_norm_add_func + elif impl == "default": + self.attn_func = encdec_attn_func + else: + assert False, "Unsupported impl: {} !".format(impl) + else: + if impl == "fast": + self.attn_func = fast_encdec_attn_func + elif impl == "default": + self.attn_func = encdec_attn_func + else: + assert False, "Unsupported impl: {} !".format(impl) + + def reset_parameters(self): + nn.init.xavier_uniform_(self.in_proj_weight_q) + # in_proj_weight_kv has shape [2 * hidden, hidden] but it should be + # initialized like a [hidden, hidden] matrix. + # sqrt(6 / (hidden + hidden)) / sqrt(6 / (2 * hidden + hidden)) = sqrt(1.5) + # therefore xavier_uniform gain should be set to sqrt(1.5). + nn.init.xavier_uniform_(self.in_proj_weight_kv, gain=math.sqrt(1.5)) + nn.init.xavier_uniform_(self.out_proj_weight) + if self.bias: + nn.init.constant_(self.in_proj_bias_q, 0.0) + nn.init.constant_(self.in_proj_bias_kv, 0.0) + nn.init.constant_(self.out_proj_bias, 0.0) + if self.include_norm_add: + if self.impl == "fast": + nn.init.ones_(self.lyr_nrm_gamma_weights) + nn.init.zeros_(self.lyr_nrm_beta_weights) + else: + self.lyr_nrm.reset_parameters() + + def forward(self, query, key, value, key_padding_mask=None, need_weights=False, attn_mask=None, is_training=True): + """Input shape: Time x Batch x Channel + + Self-attention can be implemented by passing in the same arguments for + query, key and value. Future timesteps can be masked with the + `mask_future_timesteps` argument. Padding elements can be excluded from + the key by passing a binary ByteTensor (`key_padding_mask`) with shape: + batch x src_len, where padding elements are indicated by 1s. + """ + + if key_padding_mask is not None: + assert attn_mask is None, "ERROR attn_mask and key_padding_mask should not be both defined!" + mask = key_padding_mask + elif attn_mask is not None: + mask = attn_mask + else: + mask = None + + if self.include_norm_add: + if self.impl == "fast": + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + query, + key, + self.lyr_nrm_gamma_weights, + self.lyr_nrm_beta_weights, + self.in_proj_weight_q, + self.in_proj_weight_kv, + self.out_proj_weight, + mask, + self.dropout, + ) + else: + lyr_nrm_results = self.lyr_nrm(query) + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + self.scaling, + lyr_nrm_results, + key, + self.in_proj_weight_q, + self.in_proj_weight_kv, + self.out_proj_weight, + self.in_proj_bias_q, + self.in_proj_bias_kv, + self.out_proj_bias, + mask, + self.dropout, + ) + if is_training: + outputs = jit_dropout_add(outputs, query, self.dropout, is_training) + else: + outputs = outputs + query + else: + if self.impl == "fast": + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + query, + key, + self.in_proj_weight_q, + self.in_proj_weight_kv, + self.out_proj_weight, + mask, + self.dropout, + ) + else: + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + self.scaling, + query, + key, + self.in_proj_weight_q, + self.in_proj_weight_kv, + self.out_proj_weight, + self.in_proj_bias_q, + self.in_proj_bias_kv, + self.out_proj_bias, + mask, + self.dropout, + ) + + return outputs, None diff --git a/apex/apex/contrib/multihead_attn/encdec_multihead_attn_func.py b/apex/apex/contrib/multihead_attn/encdec_multihead_attn_func.py new file mode 100644 index 00000000..cef255ba --- /dev/null +++ b/apex/apex/contrib/multihead_attn/encdec_multihead_attn_func.py @@ -0,0 +1,356 @@ +import torch +import torch.nn.functional as F + + +class EncdecAttnFunc(torch.autograd.Function): + @staticmethod + def forward( + ctx, + use_time_mask, + is_training, + heads, + scale, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + input_biases_q, + input_biases_kv, + output_biases, + mask, + dropout_prob, + ): + use_biases_t = torch.tensor([input_biases_q is not None]) + heads_t = torch.tensor([heads]) + scale_t = torch.tensor([scale]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + head_dim = inputs_q.size(2) // heads + + # Input Linear GEMM Q + # input1: (activations) [seql_q, seqs, embed_dim(1024)] + # input2: (weights) [embed_dim (1024), embed_dim (1024)] (transpose [0,1]) + # output: [seql_q, seqs, embed_dim] + # GEMM: ( (seql_q*seqs) x embed_dim ) x ( embed_dim x embed_dim ) = (seql_q*seqs x embed_dim) + if use_biases_t[0]: + input_lin_q_results = torch.addmm( + input_biases_q, + inputs_q.view(inputs_q.size(0) * inputs_q.size(1), inputs_q.size(2)), + input_weights_q.transpose(0, 1), + beta=1.0, + alpha=1.0, + ) + else: + input_lin_q_results = torch.mm( + inputs_q.view(inputs_q.size(0) * inputs_q.size(1), inputs_q.size(2)), input_weights_q.transpose(0, 1) + ) + input_lin_q_results = input_lin_q_results.view(inputs_q.size(0), inputs_q.size(1), input_weights_q.size(0)) + # Input Linear GEMM KV + # input1: (activations) [seql_k, seqs, embed_dim(1024)] + # input2: (weights) [embed_dim*2 (2048), embed_dim (1024)] (transpose [0,1]) + # output: [seql_k, seqs, embed_dim*2] + # GEMM: ( (seql_k*seqs) x embed_dim ) x ( embed_dim x embed_dim*2 ) = (seql_k*seqs x embed_dim*2) + if use_biases_t[0]: + input_lin_kv_results = torch.addmm( + input_biases_kv, + inputs_kv.view(inputs_kv.size(0) * inputs_kv.size(1), inputs_kv.size(2)), + input_weights_kv.transpose(0, 1), + beta=1.0, + alpha=1.0, + ) + else: + input_lin_kv_results = torch.mm( + inputs_kv.view(inputs_kv.size(0) * inputs_kv.size(1), inputs_kv.size(2)), + input_weights_kv.transpose(0, 1), + ) + input_lin_kv_results = input_lin_kv_results.view(inputs_kv.size(0), inputs_kv.size(1), input_weights_kv.size(0)) + + # Slice out k,v from one big Input Linear outuput (should only impact meta data, no copies!) + # Sequences and heads are combined to make the batch of the Batched GEMM + # input_lin_kv_results: [seql_k, seqs, heads(16), 2, head_dim(64)] + # input_lin_kv_results: [seql_k, batches=seqs*heads, 2, head_dim] + queries = input_lin_q_results.view(inputs_q.size(0), inputs_q.size(1) * heads, head_dim) + input_lin_kv_results = input_lin_kv_results.view(inputs_kv.size(0), inputs_kv.size(1) * heads, 2, head_dim) + keys = input_lin_kv_results[:, :, 0, :] + values = input_lin_kv_results[:, :, 1, :] + + # Matmul1 Batched GEMMs + # The output tensor is specified prior to the Batch GEMM because baddbmm requires its specification + # baddbmm is used to apply the scale parameter via the Batched GEMM's alpha parameter instead of + # a separate elementwise operation. + # Input1: (Queries) [seql_q, seqs*heads, head_dim] tranpose(0,1) + # Input2: (Keys) [seql_k, seqs*heads, head_dim] transpose(0,1) + # output: [seqs*heads, seql_q, seql_k] + # GEMM: Per batch: ( seql_q x head_dim ) x ( head_dim x seql_k ) = ( seql_q x seql_k ) + matmul1_results = torch.empty( + (queries.size(1), queries.size(0), keys.size(0)), dtype=queries.dtype, device=torch.device("cuda") + ) + matmul1_results = torch.baddbmm( + matmul1_results, + queries.transpose(0, 1), + keys.transpose(0, 1).transpose(1, 2), + out=matmul1_results, + beta=0.0, + alpha=scale_t[0], + ) + + if mask is not None: + # Self Attention Time Mask + if use_time_mask: + assert len(mask.size()) == 2, "Timing mask is not 2D!" + assert mask.size(0) == mask.size(1), "Sequence length should match!" + mask = mask.to(torch.bool) + matmul1_results = matmul1_results.masked_fill_(mask, float("-inf")) + # Key Padding Mask + else: + batches, seql_q, seql_k = matmul1_results.size() + seqs = int(batches / heads) + matmul1_results = matmul1_results.view(seqs, heads, seql_q, seql_k) + mask = mask.to(torch.bool) + matmul1_results = matmul1_results.masked_fill_(mask.unsqueeze(1).unsqueeze(2), float("-inf")) + matmul1_results = matmul1_results.view(seqs * heads, seql_q, seql_k) + + softmax_results = F.softmax(matmul1_results, dim=-1) + + # Dropout - is not executed for inference + if is_training: + dropout_results, dropout_mask = torch._fused_dropout(softmax_results, p=(1.0 - dropout_prob_t[0])) + else: + dropout_results = softmax_results + dropout_mask = null_tensor + + # Matmul2 Batched GEMMs + # The output tensor specification is needed here to specify the non-standard output. + # Given that pytorch cannot currently perform autograd with an output tensor specified, + # this requires a backward pass specified. + # Input1: from_softmax [seqs*heads, seql_q, seql_k] + # Input2: (values) [seql_v, seqs*heads, head_dim] transpose(0,1) + # Output: [seql_q, seqs*heads, head_dim] transpose(0,1) + # GEMM: Per batch: ( seql_q x seql_k ) x ( seql_k x head_dim ) = (seql_q x head_dim) + matmul2_results = torch.empty( + (dropout_results.size(1), dropout_results.size(0), values.size(2)), + dtype=dropout_results.dtype, + device=torch.device("cuda"), + ).transpose(1, 0) + matmul2_results = torch.bmm(dropout_results, values.transpose(0, 1), out=matmul2_results) + matmul2_results = ( + matmul2_results.transpose(0, 1).contiguous().view(inputs_q.size(0), inputs_q.size(1), inputs_q.size(2)) + ) + + # Output Linear GEMM + # Input1: (activations) [seql_q, seqs, embed_dim=heads*head_dim] + # Input2: (weights) [ embed_dim, embed_dim ] transpose(0,1) + # Output: [ seql_q, seqs, embed_dim ] + # GEMM: ( seql_q*seqs x embed_dim ) x ( embed_dim x embed_dim ) = ( seql_q*seqs x embed_dim ) + if use_biases_t[0]: + outputs = torch.addmm( + output_biases, + matmul2_results.view(inputs_q.size(0) * inputs_q.size(1), inputs_q.size(2)), + output_weights.transpose(0, 1), + beta=1.0, + alpha=1.0, + ) + else: + outputs = torch.mm( + matmul2_results.view(inputs_q.size(0) * inputs_q.size(1), inputs_q.size(2)), + output_weights.transpose(0, 1), + ) + outputs = outputs.view(inputs_q.size(0), inputs_q.size(1), output_weights.size(0)) + + ctx.save_for_backward( + use_biases_t, + heads_t, + scale_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_prob_t, + ) + + return outputs.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + use_biases_t, + heads_t, + scale_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_prob_t, + ) = ctx.saved_tensors + + head_dim = inputs_q.size(2) // heads_t[0] + + # Slice out k,v from one big Input Linear outuput (should only impact meta data, no copies!) + # Sequences and heads are combined to make the batch of the Batched GEMM + # input_lin_kv_results: [seql_k, seqs, heads(16), 2, head_dim(64)] + # input_lin_kv_results: [seql_k, batches=seqs*heads, 2, head_dim] + queries = input_lin_q_results.view(inputs_q.size(0), inputs_q.size(1) * heads_t[0], head_dim) + input_lin_kv_results = input_lin_kv_results.view(inputs_kv.size(0), inputs_kv.size(1) * heads_t[0], 2, head_dim) + keys = input_lin_kv_results[:, :, 0, :] + values = input_lin_kv_results[:, :, 1, :] + + # Slice out k,v from one big set of gradients entering the input linear's bprop (should only impact meta data, no copies!) + # The gradients are identical in size to the Input Linear outputs. + # The tensor is declared before hand to properly slice out query, key, and value grads. + input_lin_kv_results_grads = torch.empty_like(input_lin_kv_results) + queries_grads = torch.empty_like(queries) + keys_grads = input_lin_kv_results_grads[:, :, 0, :] + values_grads = input_lin_kv_results_grads[:, :, 1, :] + + # Output Linear GEMM - DGRAD + # Input1: (data grads) [seql_q, seqs, embed_dim=heads*head_dim] + # Input2: (weights) [ embed_dim, embed_dim ] + # Output: [ seql_q, seqs, embed_dim ] + # GEMM: ( seql_q*seqs x embed_dim ) x ( embed_dim x embed_dim ) = ( seql_q*seqs x embed_dim ) + output_lin_grads = torch.mm( + output_grads.view(output_grads.size(0) * output_grads.size(1), output_grads.size(2)), output_weights + ) + output_lin_grads = output_lin_grads.view(output_grads.size(0), output_grads.size(1), output_weights.size(1)) + # Output Linear GEMM - WGRAD + # Input1: (data grads) [seql_q*seqs, embed_dim=heads*head_dim] transpose(0,1) + # Input2: (activations) [seql_q*seqs, embed_dim ] + # Output: [ seql_q, seqs, embed_dim ] + # GEMM: ( embed_dim x seql_q*seqs ) x ( seql_q*seqs x embed_dim ) = ( embed_dim x embed_dim ) + output_weight_grads = torch.mm( + output_grads.view(output_grads.size(0) * output_grads.size(1), output_grads.size(2)).transpose(0, 1), + matmul2_results.view(matmul2_results.size(0) * matmul2_results.size(1), matmul2_results.size(2)), + ) + output_lin_grads = output_lin_grads.view( + output_grads.size(0), output_grads.size(1) * heads_t[0], head_dim + ).transpose(0, 1) + + if use_biases_t[0]: + output_bias_grads = torch.sum( + output_grads.view(output_grads.size(0) * output_grads.size(1), output_grads.size(2)), 0 + ) + else: + output_bias_grads = None + + # Matmul2 - DGRAD1 + # Input1: (data grads) [seql_q, seqs*heads, head_dim] transpose(0,1) + # Input2: (activations) [seql_k, seqs*heads, head_dim] transpose(0,1).transpose(1,2) + # Output: [seqs*heads, seql_q, seql_k] + # GEMM: Per batch: ( seql_q x head_dim ) x ( head_dim x seql_k ) = ( seql_q x seql_k ) + matmul2_dgrad1 = torch.bmm(output_lin_grads, values.transpose(0, 1).transpose(1, 2)) + # Matmul2 - DGRAD2 + # Input1: (data grads) [seql_q, seqs*heads, head_dim] transpose(0,1) + # Input2: (activations) [seql_k, seqs*heads, head_dim] transpose(0,1).transpose(1,2) + # Output: [seqs*heads, seql_q, seql_k] + # GEMM: Per batch: ( seql_q x head_dim ) x ( head_dim x seql_k ) = ( seql_q x seql_k ) + values_grads = torch.bmm(dropout_results.transpose(1, 2), output_lin_grads, out=values_grads.transpose(0, 1)) + + # Mask and Scaling for Dropout (not a publically documented op) + dropout_grads = torch._masked_scale(matmul2_dgrad1, dropout_mask, 1.0 / (1.0 - dropout_prob_t[0])) + + # Softmax Grad (not a publically documented op) + softmax_grads = torch._softmax_backward_data(dropout_grads, softmax_results, -1, softmax_results.dtype) + + # Matmul1 - DGRAD1 + # Input1: (data grads) [seqs*heads, seql_q, seql_k] + # Input2: (activations) [seql_k, seqs*heads, head_dim] transpose(0,1) + # Output: [seqs*heads, seql_q, head_dim] transpose(0,1) + # GEMM: Per batch: ( seql_q x seql_k ) x ( seql_k x head_dim ) = ( seql_q x head_dim ) + queries_grads = torch.baddbmm( + queries_grads.transpose(0, 1), + softmax_grads, + keys.transpose(0, 1), + out=queries_grads.transpose(0, 1), + beta=0.0, + alpha=scale_t[0], + ) + # Matmul1 - DGRAD2 + # Input1: (data grads) [seqs*heads, seql_q, seql_k] transpose(1,2) + # Input2: (activations) [seql_q, seqs*heads, head_dim] transpose(0,1) + # Output: [seqs*heads, seql_k, head_dim] transpose(0,1) + # GEMM: Per batch: ( seql_k x seql_q ) x ( seql_q x head_dim ) = ( seql_k x head_dim ) + keys_grads = torch.baddbmm( + keys_grads.transpose(0, 1), + softmax_grads.transpose(1, 2), + queries.transpose(0, 1), + out=keys_grads.transpose(0, 1), + beta=0.0, + alpha=scale_t[0], + ) + + # Input Q Linear GEMM - DGRAD + # input1: (data grads) [seql_q, seqs, embed_dim(1024)] + # input2: (weights) [embed_dim (1024), embed_dim (1024)] + # output: [seql_q, seqs, embed_dim] + # GEMM: ( (seql_q*seqs) x embed_dim ) x ( embed_dim x embed_dim ) = (seql_q*seqs x embed_dim) + queries_grads = queries_grads.transpose(0, 1).view(inputs_q.size(0) * inputs_q.size(1), heads_t[0] * head_dim) + input_q_grads = torch.mm(queries_grads, input_weights_q) + input_q_grads = input_q_grads.view(inputs_q.size(0), inputs_q.size(1), inputs_q.size(2)) + # Input KV Linear GEMM - DGRAD + # input1: (data grads) [seql_k, seqs, 2*embed_dim(2048)] + # input2: (weights) [embed_dim*2 (2048), embed_dim (1024)] + # output: [seql_k, seqs, embed_dim] + # GEMM: ( (seql_k*seqs) x 2*embed_dim ) x ( 2*embed_dim x embed_dim ) = (seql_k*seqs x embed_dim) + input_lin_kv_results_grads = input_lin_kv_results_grads.view( + inputs_kv.size(0) * inputs_kv.size(1), heads_t[0] * 2 * head_dim + ) + input_kv_grads = torch.mm(input_lin_kv_results_grads, input_weights_kv) + input_kv_grads = input_kv_grads.view(inputs_kv.size(0), inputs_kv.size(1), inputs_kv.size(2)) + # Input Q Linear GEMM - WGRAD + # input1: (data grads) [seql_q*seqs, embed_dim(1024)] + # input2: (activations) [seql_q*seqs, embed_dim(1024)] + # output: [embed_dim, embed_dim] + # GEMM: ( embed_dim x seql_q*seqs ) x ( seql_q*seqs x embed_dim ) = (embed_dim x embed_dim) + input_weight_q_grads = torch.mm( + queries_grads.transpose(0, 1), inputs_q.view(inputs_q.size(0) * inputs_q.size(1), inputs_q.size(2)) + ) + # Input KV Linear GEMM - WGRAD + # input1: (data grads) [seql_k*seqs, 2*embed_dim(2048)] + # input2: (activations) [seql_k*seqs, embed_dim(1024)] + # output: [2*embed_dim, embed_dim] + # GEMM: ( 2*embed_dim x seql_k*seqs ) x ( seql_k*seqs x embed_dim ) = (2*embed_dim x embed_dim) + input_weight_kv_grads = torch.mm( + input_lin_kv_results_grads.transpose(0, 1), + inputs_kv.view(inputs_kv.size(0) * inputs_kv.size(1), inputs_kv.size(2)), + ) + + if use_biases_t[0]: + input_bias_grads_q = torch.sum(queries_grads, 0) + input_bias_grads_kv = torch.sum(input_lin_kv_results_grads, 0) + else: + input_bias_grads_q = None + input_bias_grads_kv = None + + return ( + None, + None, + None, + None, + input_q_grads, + input_kv_grads, + input_weight_q_grads, + input_weight_kv_grads, + output_weight_grads, + input_bias_grads_q, + input_bias_grads_kv, + output_bias_grads, + None, + None, + ) + + +encdec_attn_func = EncdecAttnFunc.apply diff --git a/apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py b/apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py new file mode 100644 index 00000000..9431a493 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py @@ -0,0 +1,121 @@ +import torch + +import fast_multihead_attn + + +class FastEncdecAttnFunc(torch.autograd.Function): + @staticmethod + def forward( + ctx, + use_time_mask, + is_training, + heads, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + pad_mask, + dropout_prob, + ): + heads_t = torch.tensor([heads]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + use_mask = pad_mask is not None + + ( + input_lin_q_results, + input_lin_kv_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + outputs, + ) = fast_multihead_attn.encdec_multihead_attn_forward( + use_mask, + use_time_mask, + is_training, + heads, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + pad_mask if use_mask else null_tensor, + dropout_prob, + ) + + ctx.save_for_backward( + heads_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_prob_t, + ) + + return outputs.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + heads_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_prob_t, + ) = ctx.saved_tensors + + ( + input_q_grads, + input_kv_grads, + input_weight_q_grads, + input_weight_kv_grads, + output_weight_grads, + ) = fast_multihead_attn.encdec_multihead_attn_backward( + heads_t[0], + output_grads, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + inputs_q, + inputs_kv, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_prob_t[0], + ) + + return ( + None, + None, + None, + input_q_grads, + input_kv_grads, + input_weight_q_grads, + input_weight_kv_grads, + output_weight_grads, + None, + None, + ) + + +fast_encdec_attn_func = FastEncdecAttnFunc.apply diff --git a/apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py b/apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py new file mode 100644 index 00000000..320bebd6 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py @@ -0,0 +1,159 @@ +# Copyright (c) 2017-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the license found in the LICENSE file in +# the root directory of this source tree. An additional grant of patent rights +# can be found in the PATENTS file in the same directory. + +import torch + +import fast_multihead_attn + + +class FastEncdecAttnNormAddFunc(torch.autograd.Function): + @staticmethod + def forward( + ctx, + use_time_mask, + is_training, + heads, + inputs_q, + inputs_kv, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights_q, + input_weights_kv, + output_weights, + pad_mask, + dropout_prob, + ): + heads_t = torch.tensor([heads]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + use_mask = pad_mask is not None + + ( + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + input_lin_q_results, + input_lin_kv_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + dropout_add_mask, + outputs, + ) = fast_multihead_attn.encdec_multihead_attn_norm_add_forward( + use_mask, + use_time_mask, + is_training, + heads, + inputs_q, + inputs_kv, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights_q, + input_weights_kv, + output_weights, + pad_mask if use_mask else null_tensor, + dropout_prob, + ) + + ctx.save_for_backward( + heads_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + inputs_q, + inputs_kv, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_add_mask, + dropout_prob_t, + ) + + return outputs.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + heads_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + inputs_q, + inputs_kv, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_add_mask, + dropout_prob_t, + ) = ctx.saved_tensors + + ( + input_q_grads, + input_kv_grads, + lyr_nrm_gamma_grads, + lyr_nrm_beta_grads, + input_weight_q_grads, + input_weight_kv_grads, + output_weight_grads, + ) = fast_multihead_attn.encdec_multihead_attn_norm_add_backward( + heads_t[0], + output_grads, + matmul2_results, + dropout_results, + softmax_results, + input_lin_q_results, + input_lin_kv_results, + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + inputs_q, + inputs_kv, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights_q, + input_weights_kv, + output_weights, + dropout_mask, + dropout_add_mask, + dropout_prob_t[0], + ) + + # import pdb; pdb.set_trace() + return ( + None, + None, + None, + input_q_grads, + input_kv_grads, + lyr_nrm_gamma_grads, + lyr_nrm_beta_grads, + input_weight_q_grads, + input_weight_kv_grads, + output_weight_grads, + None, + None, + ) + + +fast_encdec_attn_norm_add_func = FastEncdecAttnNormAddFunc.apply diff --git a/apex/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py b/apex/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py new file mode 100644 index 00000000..6b50fe22 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py @@ -0,0 +1,243 @@ +import torch + +import fast_multihead_attn + + +class FastSelfAttnFunc(torch.autograd.Function): + @staticmethod + def forward( + ctx, + use_time_mask, + is_training, + heads, + inputs, + input_weights, + output_weights, + input_biases, + output_biases, + pad_mask, + mask_additive, + dropout_prob, + ): + use_biases_t = torch.tensor([input_biases is not None]) + heads_t = torch.tensor([heads]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + use_mask = pad_mask is not None + mask_additive_t = torch.tensor([mask_additive]) + + if use_biases_t[0]: + if not mask_additive: + ( + input_lin_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + outputs, + ) = fast_multihead_attn.self_attn_bias_forward( + use_mask, + use_time_mask, + is_training, + heads, + inputs, + input_weights, + output_weights, + input_biases, + output_biases, + pad_mask if use_mask else null_tensor, + dropout_prob, + ) + # fast_self_multihead_attn_bias.forward() \ + ctx.save_for_backward( + use_biases_t, + heads_t, + matmul2_results, + dropout_results, + softmax_results, + null_tensor, + null_tensor, + mask_additive_t, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t, + ) + + else: + ( + input_lin_results, + bmm1_results, + dropout_results, + dropout_mask, + matmul2_results, + outputs, + ) = fast_multihead_attn.self_attn_bias_additive_mask_forward( + use_mask, + use_time_mask, + is_training, + heads, + inputs, + input_weights, + output_weights, + input_biases, + output_biases, + pad_mask if use_mask else null_tensor, + dropout_prob, + ) + # fast_self_multihead_attn_bias_additive_mask.forward( \ + ctx.save_for_backward( + use_biases_t, + heads_t, + matmul2_results, + dropout_results, + null_tensor, + bmm1_results, + pad_mask, + mask_additive_t, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t, + ) + + else: + ( + input_lin_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + outputs, + ) = fast_multihead_attn.self_attn_forward( + use_mask, + use_time_mask, + is_training, + heads, + inputs, + input_weights, + output_weights, + pad_mask if use_mask else null_tensor, + dropout_prob, + ) + # fast_self_multihead_attn.forward( \ + ctx.save_for_backward( + use_biases_t, + heads_t, + matmul2_results, + dropout_results, + softmax_results, + null_tensor, + null_tensor, + mask_additive_t, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t, + ) + return outputs.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + use_biases_t, + heads_t, + matmul2_results, + dropout_results, + softmax_results, + bmm1_results, + pad_mask, + mask_additive_t, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t, + ) = ctx.saved_tensors + + if use_biases_t[0]: + if not mask_additive_t[0]: + ( + input_grads, + input_weight_grads, + output_weight_grads, + input_bias_grads, + output_bias_grads, + ) = fast_multihead_attn.self_attn_bias_backward( + heads_t[0], + output_grads, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t[0], + ) + # fast_self_multihead_attn_bias.backward( \ + + else: + ( + input_grads, + input_weight_grads, + output_weight_grads, + input_bias_grads, + output_bias_grads, + ) = fast_multihead_attn.self_attn_bias_additive_mask_backward( + heads_t[0], + output_grads, + matmul2_results, + dropout_results, + bmm1_results, + pad_mask, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t[0], + ) + # fast_self_multihead_attn_bias_additive_mask.backward( \ + + else: + input_bias_grads = None + output_bias_grads = None + input_grads, input_weight_grads, output_weight_grads = fast_multihead_attn.self_attn_backward( + heads_t[0], + output_grads, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t[0], + ) + # fast_self_multihead_attn.backward( \ + return ( + None, + None, + None, + input_grads, + input_weight_grads, + output_weight_grads, + input_bias_grads, + output_bias_grads, + None, + None, + None, + ) + + +fast_self_attn_func = FastSelfAttnFunc.apply diff --git a/apex/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py b/apex/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py new file mode 100644 index 00000000..7f110cb3 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py @@ -0,0 +1,135 @@ +import torch + +import fast_multihead_attn + + +class FastSelfAttnNormAddFunc(torch.autograd.Function): + @staticmethod + def forward( + ctx, + use_time_mask, + is_training, + heads, + inputs, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights, + output_weights, + pad_mask, + dropout_prob, + ): + heads_t = torch.tensor([heads]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + use_mask = pad_mask is not None + + ( + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + input_lin_results, + softmax_results, + dropout_results, + dropout_mask, + matmul2_results, + dropout_add_mask, + outputs, + ) = fast_multihead_attn.self_attn_norm_add_forward( + use_mask, + use_time_mask, + is_training, + heads, + inputs, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights, + output_weights, + pad_mask if use_mask else null_tensor, + dropout_prob, + ) + # fast_self_multihead_attn_norm_add.forward( \ + + ctx.save_for_backward( + heads_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + inputs, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights, + output_weights, + dropout_mask, + dropout_add_mask, + dropout_prob_t, + ) + + return outputs.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + heads_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + inputs, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights, + output_weights, + dropout_mask, + dropout_add_mask, + dropout_prob_t, + ) = ctx.saved_tensors + + ( + input_grads, + lyr_nrm_gamma_grads, + lyr_nrm_beta_grads, + input_weight_grads, + output_weight_grads, + ) = fast_multihead_attn.self_attn_norm_add_backward( + heads_t[0], + output_grads, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + lyr_nrm_results, + lyr_nrm_mean, + lyr_nrm_invvar, + inputs, + lyr_nrm_gamma_weights, + lyr_nrm_beta_weights, + input_weights, + output_weights, + dropout_mask, + dropout_add_mask, + dropout_prob_t[0], + ) + # fast_self_multihead_attn_norm_add.backward( \ + + return ( + None, + None, + None, + input_grads, + lyr_nrm_gamma_grads, + lyr_nrm_beta_grads, + input_weight_grads, + output_weight_grads, + None, + None, + ) + + +fast_self_attn_norm_add_func = FastSelfAttnNormAddFunc.apply diff --git a/apex/apex/contrib/multihead_attn/mask_softmax_dropout_func.py b/apex/apex/contrib/multihead_attn/mask_softmax_dropout_func.py new file mode 100644 index 00000000..b34eec44 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/mask_softmax_dropout_func.py @@ -0,0 +1,64 @@ +import torch + +import fast_multihead_attn + + +class MaskSoftmaxDropout(torch.autograd.Function): + @staticmethod + def forward(ctx, is_training, heads, inputs, pad_mask, mask_additive, dropout_prob): + heads_t = torch.tensor([heads]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + use_mask = pad_mask is not None + use_mask_t = torch.tensor([use_mask]) + mask_additive_t = torch.tensor([mask_additive]) + + if mask_additive: + dropout_results, dropout_mask, softmax_results = fast_multihead_attn.additive_mask_softmax_dropout_forward( + use_mask, is_training, heads, inputs, pad_mask if use_mask else null_tensor, dropout_prob + ) + # fast_additive_mask_softmax_dropout.forward( \ + else: + dropout_results, dropout_mask, softmax_results = fast_multihead_attn.mask_softmax_dropout_forward( + use_mask, is_training, heads, inputs, pad_mask if use_mask else null_tensor, dropout_prob + ) + # fast_mask_softmax_dropout.forward( \ + + ctx.save_for_backward( + use_mask_t, + heads_t, + softmax_results, + dropout_mask, + pad_mask if use_mask else null_tensor, + mask_additive_t, + dropout_prob_t, + ) + + return dropout_results.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + use_mask_t, + heads_t, + softmax_results, + dropout_mask, + pad_mask, + mask_additive_t, + dropout_prob_t, + ) = ctx.saved_tensors + + if mask_additive_t[0]: + input_grads = fast_multihead_attn.additive_mask_softmax_dropout_backward( + use_mask_t[0], heads_t[0], output_grads, softmax_results, dropout_mask, dropout_prob_t[0] + ) + # fast_additive_mask_softmax_dropout.backward( \ + else: + input_grads = fast_multihead_attn.mask_softmax_dropout_backward( + use_mask_t[0], heads_t[0], output_grads, softmax_results, dropout_mask, pad_mask, dropout_prob_t[0] + ) + # fast_mask_softmax_dropout.backward( \ + return None, None, input_grads, None, None, None + + +fast_mask_softmax_dropout_func = MaskSoftmaxDropout.apply diff --git a/apex/apex/contrib/multihead_attn/self_multihead_attn.py b/apex/apex/contrib/multihead_attn/self_multihead_attn.py new file mode 100644 index 00000000..e910a571 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/self_multihead_attn.py @@ -0,0 +1,254 @@ +import math + +import torch +from torch import nn +from torch.nn import Parameter +import torch.nn.functional as F + +from .self_multihead_attn_func import self_attn_func +from .fast_self_multihead_attn_func import fast_self_attn_func +from .fast_self_multihead_attn_norm_add_func import fast_self_attn_norm_add_func +from apex.normalization.fused_layer_norm import FusedLayerNorm + +@torch.jit.script +def jit_dropout_add(x, residual, prob, is_training): + # type: (Tensor, Tensor, float, bool) -> Tensor + out = F.dropout(x, p=prob, training=True) + out = residual + out + return out + + +class SelfMultiheadAttn(nn.Module): + """Multi-headed attention. + + See "Attention Is All You Need" for more details. + """ + + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + bias=False, + include_norm_add=False, + impl="fast", + separate_qkv_params=False, + mask_additive=False, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads" + self.bias = bias + self.include_norm_add = include_norm_add + self.impl = impl + self.scaling = self.head_dim ** -0.5 + self.separate_qkv_params = separate_qkv_params + self.mask_additive = mask_additive + if mask_additive: + assert self.include_norm_add == False, "additive mask not supported with layer norm" + assert impl == "default" or ( + impl == "fast" and bias + ), "additive mask not supported for fast mode without bias" + if separate_qkv_params: + self.q_weight = Parameter(torch.empty(embed_dim, embed_dim)) + self.k_weight = Parameter(torch.empty(embed_dim, embed_dim)) + self.v_weight = Parameter(torch.empty(embed_dim, embed_dim)) + else: + self.in_proj_weight = Parameter(torch.empty(3 * embed_dim, embed_dim)) + self.out_proj_weight = Parameter(torch.empty(embed_dim, embed_dim)) + if self.bias: + if separate_qkv_params: + self.q_bias = Parameter(torch.empty(embed_dim)) + self.k_bias = Parameter(torch.empty(embed_dim)) + self.v_bias = Parameter(torch.empty(embed_dim)) + else: + self.in_proj_bias = Parameter(torch.empty(3 * embed_dim)) + self.out_proj_bias = Parameter(torch.empty(embed_dim)) + else: + if separate_qkv_params: + self.register_parameter("q_bias", None) + self.register_parameter("k_bias", None) + self.register_parameter("v_bias", None) + self.q_bias = None + self.k_bias = None + self.v_bias = None + else: + self.register_parameter("in_proj_bias", None) + self.in_proj_bias = None + self.register_parameter("out_proj_bias", None) + self.out_proj_bias = None + if self.include_norm_add: + if impl == "fast": + self.lyr_nrm_gamma_weights = Parameter(torch.empty(embed_dim)) + self.lyr_nrm_beta_weights = Parameter(torch.empty(embed_dim)) + self.lyr_nrm = None + else: + self.register_parameter("lyr_norm_gamma_weights", None) + self.register_parameter("lyr_norm_beta_weights", None) + self.lyr_nrm_gamma_weights = None + self.lyr_nrm_beta_weights = None + self.lyr_nrm = FusedLayerNorm(embed_dim) + self.reset_parameters() + + if self.include_norm_add: + if impl == "fast": + self.attn_func = fast_self_attn_norm_add_func + elif impl == "default": + self.attn_func = self_attn_func + else: + assert False, "Unsupported impl: {} !".format(impl) + else: + if impl == "fast": + self.attn_func = fast_self_attn_func + elif impl == "default": + self.attn_func = self_attn_func + else: + assert False, "Unsupported impl: {} !".format(impl) + + def reset_parameters(self): + if self.separate_qkv_params: + nn.init.xavier_uniform_(self.q_weight) + nn.init.xavier_uniform_(self.k_weight) + nn.init.xavier_uniform_(self.v_weight) + else: + # in_proj_weight has shape [3 * hidden, hidden] but it should be + # initialized like a [hidden, hidden] matrix. + # sqrt(6 / (hidden + hidden)) / sqrt(6 / (3 * hidden + hidden)) = sqrt(2) + # therefore xavier_uniform gain should be set to sqrt(2). + nn.init.xavier_uniform_(self.in_proj_weight, gain=math.sqrt(2)) + nn.init.xavier_uniform_(self.out_proj_weight) + if self.bias: + if self.separate_qkv_params: + nn.init.constant_(self.q_bias, 0.0) + nn.init.constant_(self.k_bias, 0.0) + nn.init.constant_(self.v_bias, 0.0) + else: + nn.init.constant_(self.in_proj_bias, 0.0) + nn.init.constant_(self.out_proj_bias, 0.0) + if self.include_norm_add: + if self.impl == "fast": + nn.init.ones_(self.lyr_nrm_gamma_weights) + nn.init.zeros_(self.lyr_nrm_beta_weights) + else: + self.lyr_nrm.reset_parameters() + + def forward(self, query, key, value, key_padding_mask=None, need_weights=False, attn_mask=None, is_training=True): + """Input shape: Time x Batch x Channel + + Self-attention can be implemented by passing in the same arguments for + query, key and value. Future timesteps can be masked with the + `mask_future_timesteps` argument. Padding elements can be excluded from + the key by passing a binary ByteTensor (`key_padding_mask`) with shape: + batch x src_len, where padding elements are indicated by 1s. + """ + if self.separate_qkv_params: + input_weights = ( + torch.cat( + [ + self.q_weight.view(self.num_heads, 1, self.head_dim, self.embed_dim), + self.k_weight.view(self.num_heads, 1, self.head_dim, self.embed_dim), + self.v_weight.view(self.num_heads, 1, self.head_dim, self.embed_dim), + ], + dim=1, + ) + .reshape(3 * self.embed_dim, self.embed_dim) + .contiguous() + ) + else: + input_weights = self.in_proj_weight + if self.bias: + if self.separate_qkv_params: + input_bias = ( + torch.cat( + [ + self.q_bias.view(self.num_heads, 1, self.head_dim), + self.k_bias.view(self.num_heads, 1, self.head_dim), + self.v_bias.view(self.num_heads, 1, self.head_dim), + ], + dim=1, + ) + .reshape(3 * self.embed_dim) + .contiguous() + ) + else: + input_bias = self.in_proj_bias + else: + input_bias = None + if key_padding_mask is not None: + assert attn_mask is None, "ERROR attn_mask and key_padding_mask should not be both defined!" + mask = key_padding_mask + elif attn_mask is not None: + assert self.mask_additive == False, "additive mask not supported for time mask" + mask = attn_mask + else: + mask = None + + if self.include_norm_add: + if self.impl == "fast": + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + query, + self.lyr_nrm_gamma_weights, + self.lyr_nrm_beta_weights, + input_weights, + self.out_proj_weight, + mask, + self.dropout, + ) + else: + lyr_nrm_results = self.lyr_nrm(query) + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + self.scaling, + lyr_nrm_results, + input_weights, + self.out_proj_weight, + input_bias, + self.out_proj_bias, + mask, + self.mask_additive, + self.dropout, + ) + if is_training: + outputs = jit_dropout_add(outputs, query, self.dropout, is_training) + else: + outputs = outputs + query + else: + if self.impl == "fast": + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + query, + input_weights, + self.out_proj_weight, + input_bias, + self.out_proj_bias, + mask, + self.mask_additive, + self.dropout, + ) + else: + outputs = self.attn_func( + attn_mask is not None, + is_training, + self.num_heads, + self.scaling, + query, + input_weights, + self.out_proj_weight, + input_bias, + self.out_proj_bias, + mask, + self.mask_additive, + self.dropout, + ) + + return outputs, None diff --git a/apex/apex/contrib/multihead_attn/self_multihead_attn_func.py b/apex/apex/contrib/multihead_attn/self_multihead_attn_func.py new file mode 100644 index 00000000..c27a7203 --- /dev/null +++ b/apex/apex/contrib/multihead_attn/self_multihead_attn_func.py @@ -0,0 +1,308 @@ +import torch +import torch.nn.functional as F + + +class SelfAttnFunc(torch.autograd.Function): + @staticmethod + def forward( + ctx, + use_time_mask, + is_training, + heads, + scale, + inputs, + input_weights, + output_weights, + input_biases, + output_biases, + mask, + is_additive_mask, + dropout_prob, + ): + use_biases_t = torch.tensor([input_biases is not None]) + heads_t = torch.tensor([heads]) + scale_t = torch.tensor([scale]) + dropout_prob_t = torch.tensor([dropout_prob]) + null_tensor = torch.tensor([]) + head_dim = inputs.size(2) // heads + + # Input Linear GEMM + # input1: (activations) [seql_q, seqs, embed_dim(1024)] + # input2: (weights) [embed_dim*3 (3072), embed_dim (1024)] (transpose [0,1]) + # output: [seql_q, seqs, embed_dim*3] + # GEMM: ( (seql_q*seqs) x embed_dim ) x ( embed_dim x embed_dim*3 ) = (seql_q*seqs x embed_dim*3) + if use_biases_t[0]: + input_lin_results = torch.addmm( + input_biases, + inputs.view(inputs.size(0) * inputs.size(1), inputs.size(2)), + input_weights.transpose(0, 1), + beta=1.0, + alpha=1.0, + ) + else: + input_lin_results = torch.mm( + inputs.view(inputs.size(0) * inputs.size(1), inputs.size(2)), input_weights.transpose(0, 1) + ) + input_lin_results = input_lin_results.view(inputs.size(0), inputs.size(1), input_weights.size(0)) + + # Slice out q,k,v from one big Input Linear outuput (should only impact meta data, no copies!) + # Sequences and heads are combined to make the batch of the Batched GEMM + # input_lin_results: [seql_q, seqs, heads(16), 3, head_dim(64)] + # input_lin_results: [seql_q, batches=seqs*heads, 3, head_dim] + input_lin_results = input_lin_results.view(inputs.size(0), inputs.size(1) * heads, 3, head_dim) + queries = input_lin_results[:, :, 0, :] + keys = input_lin_results[:, :, 1, :] + values = input_lin_results[:, :, 2, :] + + # Matmul1 Batched GEMMs + # The output tensor is specified prior to the Batch GEMM because baddbmm requires its specification + # baddbmm is used to apply the scale parameter via the Batched GEMM's alpha parameter instead of + # a separate elementwise operation. + # Input1: (Queries) [seql_q, seqs*heads, head_dim] tranpose(0,1) + # Input2: (Keys) [seql_k, seqs*heads, head_dim] transpose(0,1) + # output: [seqs*heads, seql_q, seql_k] + # GEMM: Per batch: ( seql_q x head_dim ) x ( head_dim x seql_k ) = ( seql_q x seql_k ) + matmul1_results = torch.empty( + (queries.size(1), queries.size(0), keys.size(0)), dtype=queries.dtype, device=torch.device("cuda") + ) + matmul1_results = torch.baddbmm( + matmul1_results, + queries.transpose(0, 1), + keys.transpose(0, 1).transpose(1, 2), + out=matmul1_results, + beta=0.0, + alpha=scale_t[0], + ) + + if mask is not None: + # Self Attention Time Mask + if use_time_mask: + assert len(mask.size()) == 2, "Timing mask is not 2D!" + assert mask.size(0) == mask.size(1), "Sequence length should match!" + mask = mask.to(torch.bool) + matmul1_results = matmul1_results.masked_fill_(mask, float("-inf")) + # Key Padding Mask + else: + batches, seql_q, seql_k = matmul1_results.size() + seqs = int(batches / heads) + matmul1_results = matmul1_results.view(seqs, heads, seql_q, seql_k) + if is_additive_mask: + matmul1_results = matmul1_results + mask.unsqueeze(1).unsqueeze(2) + else: + mask = mask.to(torch.bool) + matmul1_results = matmul1_results.masked_fill_(mask.unsqueeze(1).unsqueeze(2), float("-inf")) + matmul1_results = matmul1_results.view(seqs * heads, seql_q, seql_k) + + softmax_results = F.softmax(matmul1_results, dim=-1) + + # Dropout - is not executed for inference + if is_training: + dropout_results, dropout_mask = torch._fused_dropout(softmax_results, p=(1.0 - dropout_prob_t[0])) + else: + dropout_results = softmax_results + dropout_mask = null_tensor + + # Matmul2 Batched GEMMs + # The output tensor specification is needed here to specify the non-standard output. + # Given that pytorch cannot currently perform autograd with an output tensor specified, + # this requires a backward pass specified. + # Input1: from_softmax [seqs*heads, seql_q, seql_k] + # Input2: (values) [seql_v, seqs*heads, head_dim] transpose(0,1) + # Output: [seql_q, seqs*heads, head_dim] transpose(0,1) + # GEMM: Per batch: ( seql_q x seql_k ) x ( seql_k x head_dim ) = (seql_q x head_dim) + matmul2_results = torch.empty( + (dropout_results.size(1), dropout_results.size(0), values.size(2)), + dtype=dropout_results.dtype, + device=torch.device("cuda"), + ).transpose(1, 0) + matmul2_results = torch.bmm(dropout_results, values.transpose(0, 1), out=matmul2_results) + matmul2_results = ( + matmul2_results.transpose(0, 1).contiguous().view(inputs.size(0), inputs.size(1), inputs.size(2)) + ) + + # Output Linear GEMM + # Input1: (activations) [seql_q, seqs, embed_dim=heads*head_dim] + # Input2: (weights) [ embed_dim, embed_dim ] transpose(0,1) + # Output: [ seql_q, seqs, embed_dim ] + # GEMM: ( seql_q*seqs x embed_dim ) x ( embed_dim x embed_dim ) = ( seql_q*seqs x embed_dim ) + if use_biases_t[0]: + outputs = torch.addmm( + output_biases, + matmul2_results.view(inputs.size(0) * inputs.size(1), inputs.size(2)), + output_weights.transpose(0, 1), + beta=1.0, + alpha=1.0, + ) + else: + outputs = torch.mm( + matmul2_results.view(inputs.size(0) * inputs.size(1), inputs.size(2)), output_weights.transpose(0, 1) + ) + outputs = outputs.view(inputs.size(0), inputs.size(1), output_weights.size(0)) + + ctx.save_for_backward( + use_biases_t, + heads_t, + scale_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t, + ) + + return outputs.detach() + + @staticmethod + def backward(ctx, output_grads): + ( + use_biases_t, + heads_t, + scale_t, + matmul2_results, + dropout_results, + softmax_results, + input_lin_results, + inputs, + input_weights, + output_weights, + dropout_mask, + dropout_prob_t, + ) = ctx.saved_tensors + + head_dim = inputs.size(2) // heads_t[0] + + # Slice out q,k,v from one big Input Linear outuput (should only impact meta data, no copies!) + # Sequences and heads are combined to make the batch of the Batched GEMM + # input_lin_results: [seql_q, seqs, heads(16), 3, head_dim(64)] + # input_lin_results: [seql_q, batches=seqs*heads, 3, head_dim] + input_lin_results = input_lin_results.view(inputs.size(0), inputs.size(1) * heads_t[0], 3, head_dim) + queries = input_lin_results[:, :, 0, :] + keys = input_lin_results[:, :, 1, :] + values = input_lin_results[:, :, 2, :] + + # Slice out q,k,v from one big set of gradients entering the input linear's bprop (should only impact meta data, no copies!) + # The gradients are identical in size to the Input Linear outputs. + # The tensor is declared before hand to properly slice out query, key, and value grads. + input_lin_results_grads = torch.empty_like(input_lin_results) + queries_grads = input_lin_results_grads[:, :, 0, :] + keys_grads = input_lin_results_grads[:, :, 1, :] + values_grads = input_lin_results_grads[:, :, 2, :] + + # Output Linear GEMM - DGRAD + # Input1: (data grads) [seql_q, seqs, embed_dim=heads*head_dim] + # Input2: (weights) [ embed_dim, embed_dim ] + # Output: [ seql_q, seqs, embed_dim ] + # GEMM: ( seql_q*seqs x embed_dim ) x ( embed_dim x embed_dim ) = ( seql_q*seqs x embed_dim ) + output_lin_grads = torch.mm( + output_grads.view(output_grads.size(0) * output_grads.size(1), output_grads.size(2)), output_weights + ) + output_lin_grads = output_lin_grads.view(output_grads.size(0), output_grads.size(1), output_weights.size(1)) + # Output Linear GEMM - WGRAD + # Input1: (data grads) [seql_q*seqs, embed_dim=heads*head_dim] transpose(0,1) + # Input2: (activations) [seql_q*seqs, embed_dim ] + # Output: [ seql_q, seqs, embed_dim ] + # GEMM: ( embed_dim x seql_q*seqs ) x ( seql_q*seqs x embed_dim ) = ( embed_dim x embed_dim ) + output_weight_grads = torch.mm( + output_grads.view(output_grads.size(0) * output_grads.size(1), output_grads.size(2)).transpose(0, 1), + matmul2_results.view(matmul2_results.size(0) * matmul2_results.size(1), matmul2_results.size(2)), + ) + output_lin_grads = output_lin_grads.view(inputs.size(0), inputs.size(1) * heads_t[0], head_dim).transpose(0, 1) + + if use_biases_t[0]: + output_bias_grads = torch.sum( + output_grads.view(output_grads.size(0) * output_grads.size(1), output_grads.size(2)), 0 + ) + else: + output_bias_grads = None + + # Matmul2 - DGRAD1 + # Input1: (data grads) [seql_q, seqs*heads, head_dim] transpose(0,1) + # Input2: (activations) [seql_k, seqs*heads, head_dim] transpose(0,1).transpose(1,2) + # Output: [seqs*heads, seql_q, seql_k] + # GEMM: Per batch: ( seql_q x head_dim ) x ( head_dim x seql_k ) = ( seql_q x seql_k ) + matmul2_dgrad1 = torch.bmm(output_lin_grads, values.transpose(0, 1).transpose(1, 2)) + # Matmul2 - DGRAD2 + # Input1: (data grads) [seql_q, seqs*heads, head_dim] transpose(0,1) + # Input2: (activations) [seql_k, seqs*heads, head_dim] transpose(0,1).transpose(1,2) + # Output: [seqs*heads, seql_q, seql_k] + # GEMM: Per batch: ( seql_q x head_dim ) x ( head_dim x seql_k ) = ( seql_q x seql_k ) + values_grads = torch.bmm(dropout_results.transpose(1, 2), output_lin_grads, out=values_grads.transpose(0, 1)) + + # Mask and Scaling for Dropout (not a publically documented op) + dropout_grads = torch._masked_scale(matmul2_dgrad1, dropout_mask, 1.0 / (1.0 - dropout_prob_t[0])) + + # Softmax Grad (not a publically documented op) + softmax_grads = torch._softmax_backward_data(dropout_grads, softmax_results, -1, softmax_results.dtype) + + # Matmul1 - DGRAD1 + # Input1: (data grads) [seqs*heads, seql_q, seql_k] + # Input2: (activations) [seql_k, seqs*heads, head_dim] transpose(0,1) + # Output: [seqs*heads, seql_q, head_dim] transpose(0,1) + # GEMM: Per batch: ( seql_q x seql_k ) x ( seql_k x head_dim ) = ( seql_q x head_dim ) + queries_grads = torch.baddbmm( + queries_grads.transpose(0, 1), + softmax_grads, + keys.transpose(0, 1), + out=queries_grads.transpose(0, 1), + beta=0.0, + alpha=scale_t[0], + ) + # Matmul1 - DGRAD2 + # Input1: (data grads) [seqs*heads, seql_q, seql_k] transpose(1,2) + # Input2: (activations) [seql_q, seqs*heads, head_dim] transpose(0,1) + # Output: [seqs*heads, seql_k, head_dim] transpose(0,1) + # GEMM: Per batch: ( seql_k x seql_q ) x ( seql_q x head_dim ) = ( seql_k x head_dim ) + keys_grads = torch.baddbmm( + keys_grads.transpose(0, 1), + softmax_grads.transpose(1, 2), + queries.transpose(0, 1), + out=keys_grads.transpose(0, 1), + beta=0.0, + alpha=scale_t[0], + ) + + # Input Linear GEMM - DGRAD + # input1: (data grads) [seql_q, seqs, 3*embed_dim(3072)] + # input2: (weights) [embed_dim*3 (3072), embed_dim (1024)] + # output: [seql_q, seqs, embed_dim] + # GEMM: ( (seql_q*seqs) x 3*embed_dim ) x ( 3*embed_dim x embed_dim ) = (seql_q*seqs x embed_dim) + input_lin_results_grads = input_lin_results_grads.view( + inputs.size(0) * inputs.size(1), heads_t[0] * 3 * head_dim + ) + input_grads = torch.mm(input_lin_results_grads, input_weights) + input_grads = input_grads.view(inputs.size(0), inputs.size(1), inputs.size(2)) + # Input Linear GEMM - WGRAD + # input1: (data grads) [seql_q*seqs, 3*embed_dim(3072)] + # input2: (activations) [seql_q*seqs, embed_dim(1024)] + # output: [3*embed_dim, embed_dim] + # GEMM: ( 3*embed_dim x seql_q*seqs ) x ( seql_q*seqs x embed_dim ) = (3*embed_dim x embed_dim) + input_weight_grads = torch.mm( + input_lin_results_grads.transpose(0, 1), inputs.view(inputs.size(0) * inputs.size(1), inputs.size(2)) + ) + + if use_biases_t[0]: + input_bias_grads = torch.sum(input_lin_results_grads, 0) + else: + input_bias_grads = None + + return ( + None, + None, + None, + None, + input_grads, + input_weight_grads, + output_weight_grads, + input_bias_grads, + output_bias_grads, + None, + None, + None, + ) + + +self_attn_func = SelfAttnFunc.apply diff --git a/apex/apex/contrib/optimizers/__init__.py b/apex/apex/contrib/optimizers/__init__.py new file mode 100644 index 00000000..1933b2fd --- /dev/null +++ b/apex/apex/contrib/optimizers/__init__.py @@ -0,0 +1,3 @@ +from .fp16_optimizer import FP16_Optimizer +from .fused_adam import FusedAdam +from .fused_lamb import FusedLAMB diff --git a/apex/apex/contrib/optimizers/distributed_fused_adam.py b/apex/apex/contrib/optimizers/distributed_fused_adam.py new file mode 100644 index 00000000..79cd2c8f --- /dev/null +++ b/apex/apex/contrib/optimizers/distributed_fused_adam.py @@ -0,0 +1,2136 @@ +import collections +import contextlib +import enum +import inspect +import io +import itertools +import threading +import types + +import torch +from torch.distributed.distributed_c10d import _get_default_group +from apex.multi_tensor_apply import multi_tensor_applier +import amp_C +import distributed_adam_cuda + +# Fallback to private functions if using PyTorch <1.13.0 +try: + from torch.distributed.distributed_c10d import get_global_rank +except ImportError: + from torch.distributed.distributed_c10d import _get_global_rank + get_global_rank = _get_global_rank +try: + from torch.distributed.distributed_c10d import reduce_scatter_tensor +except ImportError: + from torch.distributed.distributed_c10d import _reduce_scatter_base + reduce_scatter_tensor = _reduce_scatter_base +try: + from torch.distributed.distributed_c10d import all_gather_into_tensor +except ImportError: + from torch.distributed.distributed_c10d import _all_gather_base + all_gather_into_tensor = _all_gather_base + +# Add args to coalescing manager if using PyTorch <=1.13.1 +from torch.distributed.distributed_c10d import _coalescing_manager +if 'device' not in inspect.signature(_coalescing_manager).parameters.keys(): + _coalescing_manager_nodevice = _coalescing_manager + @contextlib.contextmanager + def _coalescing_manager(group, device, reqs): + with _coalescing_manager_nodevice(group, reqs): + yield + +# Import optional CUDA kernels +_FOUND_DEPRECATED_FUSED_ADAM = False +try: + import fused_adam_cuda + _FOUND_DEPRECATED_FUSED_ADAM = True +except ImportError: + import warnings + warnings.warn( + 'Could not find recommended CUDA kernels when importing ' + '`DistributedFusedAdam`. ' + 'For best performance, Apex should be installed with ' + '`--deprecated_fused_adam`.' + ) + +def _ceildiv(numer, denom): + """Assumes arguments are positive integers""" + return (numer + denom - 1) // denom + +def _round_to_multiple(number, multiple, round_up=True): + """Assumes arguments are positive integers""" + return (number+multiple-1 if round_up else number) // multiple * multiple + +def _devices_match(device1, device2): + """Whether two PyTorch devices are equivalent""" + device1 = torch.device(device1) + device2 = torch.device(device2) + if device1.type != device2.type: + return False + if device1.type == 'cuda': + index1 = device1.index + index2 = device2.index + if index1 is None: + index1 = torch.cuda.current_device() + if index2 is None: + index2 = torch.cuda.current_device() + if index1 != index2: + return False + return True + +def _multi_tensor_copy( + buffers_in, + buffers_out, + dummy_overflow_buf=None, +): + """Copy between corresponding buffers + + Uses fused copy kernel if possible. + """ + + # Group buffers by device and dtype + buffer_groups = collections.defaultdict(list) + for buf_in, buf_out in zip(buffers_in, buffers_out): + if buf_in.data_ptr() == buf_out.data_ptr() or buf_in.numel() == 0: + # Nothing to be done if input and output buffers are same + # or have no entries + continue + if buf_in.dtype == buf_out.dtype: + # Just copy bytes if dtypes are same + buf_in = buf_in.view(torch.uint8) + buf_out = buf_out.view(torch.uint8) + key = (buf_in.is_cuda, buf_in.dtype, buf_out.is_cuda, buf_out.dtype) + buffer_groups[key].append((buf_in, buf_out)) + + # Copy each group of buffers + for key, buffers in buffer_groups.items(): + + # Check if buffers support fused kernel + is_cuda_in, dtype_in, is_cuda_out, dtype_out = key + supported_dtypes = (torch.float32, torch.float16) + use_fused_kernel = ( + (dtype_in in supported_dtypes and dtype_out in supported_dtypes) + or + (dtype_in == torch.uint8 and dtype_out == torch.uint8) + ) + use_fused_kernel = use_fused_kernel and is_cuda_in and is_cuda_out + + # Copy buffers + if use_fused_kernel and _FOUND_DEPRECATED_FUSED_ADAM: + if dummy_overflow_buf is None: + dummy_overflow_buf = torch.zeros([1], dtype=torch.int32, device='cuda') + multi_tensor_applier( + fused_adam_cuda.maybe_cast_mt, + dummy_overflow_buf, + list(zip(*buffers)), + ) + else: + for buf_in, buf_out in buffers: + buf_out.copy_(buf_in) + +@contextlib.contextmanager +def _disable_pre_forward_hook(param): + """Prevent parameter from calling pre-forward hook""" + hook_is_enabled = getattr( + param, + '_pre_forward_hook_is_enabled', + False, + ) + if hook_is_enabled: + param._pre_forward_hook_is_enabled = False + try: + yield + finally: + if hook_is_enabled: + param._pre_forward_hook_is_enabled = True + +class DistributedFusedAdam(torch.optim.Optimizer): + """Adam optimizer with ZeRO algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``python setup.py install --cuda_ext --cpp_ext --distributed_adam --deprecated_fused_adam``. + + This implements the ZeRO-2 algorithm, which distributes the + optimizer state and gradients between parallel processes. In + particular, the parameters are flattened, grouped into fixed-size + buckets, and the optimizer state for each bucket is sharded over + the parallel processes. Options are provided to overlap the + gradient synchronization with the backward pass compute. + + Adam was proposed in `Adam: A Method for Stochastic + Optimization`_, AdamW in `Decoupled Weight Decay Regularization`_, + and ZeRO in `ZeRO: Memory Optimizations Toward Training Trillion + Parameter Models`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts + defining parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + bias_correction (bool, optional): apply correction factor to + moment estimates. (default: True) + betas (Tuple[float, float], optional): coefficients used for + computing running averages of gradient and its square. + (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + adam_w_mode (boolean, optional): Decouple weight decay + regularization (also known as AdamW algorithm) (default: + True) + weight_decay (float, optional): weight decay (L2 penalty) + (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad + variant of this algorithm from the paper + `On the Convergence of Adam and Beyond`_ (default: False). + This is not yet supported. + dtype (torch.dtype, optional): datatype for optimizer state + (default: torch.float32) + grad_sync_dtype (torch.dtype, optional): datatype for gradient + synchronization (default: same as dtype) + param_sync_dtype (torch.dtype, optional): datatype for + parameter synchronization (default: same as dtype) + device (torch.device, optional): device for optimizer state + (default: cuda). Currently only supports GPU with one GPU + per process. + process_group (torch.distributed.ProcessGroup, optional): + parallel processes participating in optimizer (default: + default group in torch.distributed). This group is + interpreted as a 2D grid with dimensions + distributed_size x redundant_size. + distributed_process_group (torch.distributed.ProcessGroup, + optional): parallel processes to distribute optimizer + state over (default: same as process_group) + redundant_process_group (torch.distributed.ProcessGroup, + optional): parallel processes to replicate optimizer state + over (default: group only containing calling process) + average_grad_sync (bool, optional): whether to use average + reduction for gradient synchronization rather than sum + (default: True) + overlap_grad_sync(boolean, optional): whether to overlap + gradient synchronization with backward pass compute + (default: True) + overlap_param_sync(boolean, optional): whether to overlap + parameter synchronization with forward pass compute + (default: False). This is an experimental feature. + bucket_cap_mb (float, optional): bucket size in megabytes + (default: 100) + pipeline_size (int, optional): number of buckets to process + simultaneously in optimizer step (default: 2) + contiguous_param_buffer (bool, optional): convert parameters + into views into a large persistent buffer (default: + False). This enables some performance optimizations (e.g. + avoiding some memory copies), but may add memory overhead + (e.g. if the memory allocator can't reuse the original + parameter buffers). + contiguous_grad_buffer (bool, optional): allocate gradient + buckets out of a large persistent buffer (default: False). + This allows individual parameter gradients to be accessed + externally (see grad_buffer_view function). It enables + some performance optimizations (e.g. avoiding some memory + copies), but prevents some memory optimizations (e.g. the + memory allocator can't reuse buffers for gradient + buckets). + store_params (bool, optional): store a distributed copy of the + parameters as optimizer state (default: True). This may be + desirable if the optimizer dtype has higher precision than + the parameter dtype. + store_param_remainders (bool, optional): if model is BF16 and + optimizer is FP32, store bits required to reconstruct FP32 + params (default: False). This is an experimental feature. + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + .. _Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101 + .. _ZeRO\: Memory Optimizations Toward Training Trillion Parameter Models: + https://arxiv.org/abs/1910.02054 + + """ + + class ParameterFragment: + """Buffer ranges for a parameter fragment + + Describes corresponding regions in parameter buffer and + parameter bucket. + + """ + def __init__( + self, + param_group_id, + param_id, + bucket_id, + param_range, + bucket_range, + in_local_shard, + shard_range, + shard_bucket_range, + shard_param_range, + ): + # Parameter group index + self.param_group_id = param_group_id + # Parameter index within parameter group + self.param_id = param_id + # Bucket index + self.bucket_id = bucket_id + # Range within flattened parameter buffer + self.param_range = param_range + # Range within bucket + self.bucket_range = bucket_range + # Whether fragment is in local shard of bucket + self.in_local_shard = in_local_shard + # Range within local shard + self.shard_range = shard_range + # Range of local fragment shard within bucket + self.shard_bucket_range = shard_bucket_range + # Range of local fragment shard within parameter + self.shard_param_range = shard_param_range + + class StateBucket: + """Optimizer state for a bucket""" + def __init__( + self, + bucket_size, + shard_size, + dtype, + device, + contiguous_buffer_offset=0, + store_params=False, + store_param_remainders=False, + ): + # Size of parameter bucket + self.bucket_size = bucket_size + # Size of local shard of parameter bucket + self.shard_size = shard_size + # Size of the filled region in the bucket + self.filled_size = 0 + # Offset to bucket in contiguous buffers + self.contiguous_buffer_offset = contiguous_buffer_offset + # Buffer ranges corresponding to parameter fragments + self.fragments = [] + # Local shard of parameters + self.params_shard = None + if store_params: + self.params_shard = torch.zeros( + [shard_size], dtype=dtype, device=device, + ) + # Local shard of parameter remainders + self.param_remainders_shard = None + if store_param_remainders: + self.param_remainders_shard = torch.zeros( + [shard_size], dtype=torch.int16, device=device, + ) + # Local shard of first moment estimate + self.exp_avg_shard = torch.zeros( + [shard_size], dtype=dtype, device=device, + ) + # Local shard of second moment estimate + self.exp_avg_sq_shard = torch.zeros( + [shard_size], dtype=dtype, device=device, + ) + + class GradientStatus(enum.Enum): + """Status of gradients within a bucket""" + # Gradients are ready to use + READY = enum.auto() + # Bucket is partially filled with unreduced gradients + PARTIALLY_FILLED = enum.auto() + # Bucket is fully filled with unreduced gradients + FULLY_FILLED = enum.auto() + # Asynchronous reduction is in progress + SYNCING = enum.auto() + + class GradientBucket: + """Gradient buffers and state for a bucket""" + def __init__(self): + # Local shard of gradients + self.grads_shard = None + # Local contribution to gradients + self.grads_bucket = None + # Buffer for gradient reduce-scatter + self.sync_grads_shard = None + # Status of gradients + self.status = DistributedFusedAdam.GradientStatus.READY + # Request object for asynchronous communication + self.sync_request = None + # Params that have generated grads + self.grads_generated = set() + + def sync_wait(self): + """Wait for asynchronous communication to finish""" + if self.sync_request is not None: + self.sync_request.wait() + self.sync_request = None + + class ParameterStatus(enum.Enum): + """Status of parameters within a bucket""" + # Parameters are sharded between processes + SHARDED = enum.auto() + # Asynchronous communication is in progress + SYNCING = enum.auto() + # Parameters are ready to use + READY = enum.auto() + + class ParameterBucket: + """Parameter buffers and state for a bucket""" + def __init__(self): + # Local shard of parameters + self.params_shard = None + # Gathered parameter values + self.params_bucket = None + # Status of parameters + self.status = DistributedFusedAdam.ParameterStatus.SHARDED + # Request object for asynchronous communication + self.sync_request = None + # Params that have been updated + self.params_updated = set() + + def sync_wait(self): + """Wait for asynchronous communication to finish""" + if self.sync_request is not None: + self.sync_request.wait() + self.sync_request = None + + # Enable custom logic for AMP grad scaling + _step_supports_amp_scaling = True + _custom_amp_unscale_grads = True + + def __init__(self, + params, + lr=1e-3, + bias_correction=True, + betas=(0.9, 0.999), + eps=1e-8, + adam_w_mode=True, + weight_decay=0., + amsgrad=False, + dtype=torch.float32, + grad_sync_dtype=None, + param_sync_dtype=None, + device='cuda', + process_group=None, + distributed_process_group=None, + redundant_process_group=None, + average_grad_sync=True, + overlap_grad_sync=True, + overlap_param_sync=False, + bucket_cap_mb=100, + pipeline_size=2, + contiguous_param_buffer=False, + contiguous_grad_buffer=False, + store_params=True, + store_param_remainders=False, + ): + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay) + super().__init__(params, defaults) + + # Adam options + self.adam_w_mode = adam_w_mode + if amsgrad: + raise RuntimeError('DistributedFusedAdam does not support the AMSGrad variant.') + + # Datatype options + if grad_sync_dtype is None: + grad_sync_dtype = dtype + if param_sync_dtype is None: + param_sync_dtype = dtype + supported_dtypes = (torch.float32, torch.float16, torch.bfloat16) + if (dtype not in supported_dtypes + or grad_sync_dtype not in supported_dtypes + or param_sync_dtype not in supported_dtypes): + raise RuntimeError( + 'Unsupported dtypes for DistributedFusedAdam ' + f'(dtype={dtype}, ' + f'grad_sync_dtype={grad_sync_dtype}, ' + f'param_sync_dtype={param_sync_dtype}))' + ) + self.dtype = dtype + self.grad_sync_dtype = grad_sync_dtype + self.param_sync_dtype = param_sync_dtype + + # Device options + if not _devices_match(device, 'cuda'): + raise RuntimeError( + 'Invalid device for DistributedFusedAdam ' + f'(device={device})' + ) + self.device = torch.device('cuda', torch.cuda.current_device()) + + # Process groups + self.process_group = ( + _get_default_group() + if process_group is None + else process_group + ) + self.distributed_process_group = ( + self.process_group + if distributed_process_group is None + else distributed_process_group + ) + self.redundant_process_group = redundant_process_group + self.process_group_size = torch.distributed.get_world_size(self.process_group) + self.distributed_rank = torch.distributed.get_rank(self.distributed_process_group) + self.distributed_size = torch.distributed.get_world_size(self.distributed_process_group) + self.redundant_size = ( + 1 + if self.redundant_process_group is None + else torch.distributed.get_world_size(self.redundant_process_group) + ) + if self.process_group_size != self.distributed_size * self.redundant_size: + raise RuntimeError( + 'Invalid process group configuration ' + f'(process group size = {self.process_group_size}, ' + f'distributed process group size = {self.distributed_size}, ' + f'redundant process group size = {self.redundant_size})' + ) + self.process_group_root = get_global_rank(self.process_group, 0) + + # Use average reduction for grad sync + self.average_grad_sync = average_grad_sync + # Copy param grads to bucket as soon as available + self.greedy_grad_copy = True + # Synchronize grad buckets as soon as their grads are available + self.overlap_grad_sync = overlap_grad_sync + # Try synchronizing param buckets just before param is needed + self.overlap_param_sync = overlap_param_sync + # Number of buckets to synchronize at a time + self.pipeline_size = pipeline_size + + # Store params or param remainders + if store_param_remainders: + if store_params: + raise RuntimeError( + 'Attempted to construct DistributedFusedAdam ' + 'with store_params=True and store_param_remainders=True' + ) + if (self.dtype != torch.float32 + or self.param_sync_dtype != torch.bfloat16): + raise RuntimeError( + 'DistributedFusedAdam requires ' + 'BF16 params and FP32 optimizer state ' + 'when storing parameter remainders ' + f'(dtype={self.dtype}, ' + f'param_sync_dtype={self.param_sync_dtype}))' + ) + self.store_params = store_params + self.store_param_remainders = store_param_remainders + + # Determine bucket sizes + dtype_size = torch.finfo(self.grad_sync_dtype).bits // 8 + self.alignment = 128 // dtype_size + bucket_size = 1024*1024*bucket_cap_mb / dtype_size + shard_size = int(bucket_size / self.distributed_size) + shard_size = _round_to_multiple(shard_size, self.alignment, round_up=False) + shard_size = max(shard_size, self.alignment) + self.default_shard_size = shard_size + + # Optimizer state + self.state['buckets'] = [] + self.state['step'] = 0 + + # Gradient state + self._grads_buckets = collections.defaultdict(self.GradientBucket) + # Param state + self._params_buckets = collections.OrderedDict() + + # Whether to allocate contiguous buffer for parameters + self.contiguous_param_buffer = contiguous_param_buffer + # Whether to allocate contiguous buffer for gradients + self.contiguous_grad_buffer = contiguous_grad_buffer + # Contiguous buffer for parameters + self._param_buffer = None + # Contiguous buffer for gradients + self._grad_buffer = None + + # Whether to allocate contiguous buffer for gradients + self.contiguous_grad_buffer = contiguous_grad_buffer + # Contiguous buffer for gradients + self._grad_buffer = None + + # Side streams for optimizer step and communication + self._pipeline_streams = [torch.cuda.Stream() for _ in range(self.pipeline_size+1)] + + # Scale by factor before optimizer step. Used for grad + # clipping and gradient scaler. + self._grad_scale = torch.full([], 1.0, dtype=torch.float32, device=self.device) + # Norm of parameter gradients. Used for gradient clipping and + # gradient scaler. + self._grad_norm = None + + # Dummy flag for multi-tensor kernels + # Note: Apex multi-tensor kernels have a noop_flag argument + # that is intended to detect non-finite values. It shouldn't + # have any effect with the kernels used in the optimizer, but + # we still set it to zero out of an abundance of caution. + self._dummy_overflow_buf = torch.zeros([1], dtype=torch.int32, device=self.device) + + # Check if collectives have no_copy option + self._gather_no_copy = ( + 'no_copy' in inspect.getfullargspec(torch.distributed.gather).args + ) + + # Make sure parameter values are same across processes + self._broadcast_params() + + # Lock for callbacks + self._lock = threading.Lock() + # Attach hooks for gradient synchronization + self._register_post_backward_hooks() + # Attach hooks for param synchronization + if self.overlap_param_sync: + self._register_pre_forward_hooks() + + def _broadcast_params(self): + """Broadcast parameter values from root rank""" + sync_requests = [] + process_group = self.process_group + with _coalescing_manager(process_group, self.device, sync_requests): + for param_group in self.param_groups: + for param in param_group['params']: + sync_requests.append( + torch.distributed.broadcast( + param, + src=self.process_group_root, + group=process_group, + async_op=True, + ) + ) + for req in sync_requests: + req.wait() + + def _make_post_backward_hook(self, param, param_group_id, param_id): + """Create callback function to call after param generates grad + + Lazily initialize parameter and try launching grad sync. + + """ + def post_backward_hook(*unused): + if getattr(param, '_pre_forward_hook_is_enabled', False): + raise RuntimeError( + 'A parameter called its post-backward hook ' + 'before its pre-forward hook. ' + 'Please manually interact with the parameter ' + 'before the forward pass (e.g. by calling data_ptr) ' + 'or run DistributedFusedAdam with overlap_param_sync=False.' + ) + with self._lock: + need_to_initialize = 'fragments' not in self.state[param] + if need_to_initialize: + self._init_param_state(param, param_group_id, param_id) + if self.greedy_grad_copy: + self._grad_copy(param) + if self.overlap_grad_sync: + self._try_start_bucket_grad_sync( + params=[param], + ignore_last_bucket=need_to_initialize, + ) + return post_backward_hook + + def _register_post_backward_hooks(self): + """Attach hooks for gradient synchronization""" + self._grad_accs = [] + for param_group_id, group in enumerate(self.param_groups): + for param_id, param in enumerate(group['params']): + if param.requires_grad: + param_tmp = param.expand_as(param) + grad_acc = param_tmp.grad_fn.next_functions[0][0] + hook = self._make_post_backward_hook( + param, + param_group_id, + param_id, + ) + grad_acc.register_hook(hook) + self._grad_accs.append(grad_acc) + + def _make_pre_forward_hook(self, param, param_group_id, param_id): + """Create callback function to call before param forward pass + + Make sure param has been synchronized and try launching next + param sync. + + """ + def pre_forward_hook(*unused): + with self._lock: + if 'fragments' not in self.state[param]: + return + self._param_copy(param) + if self.overlap_param_sync: + self._try_start_bucket_param_sync() + return pre_forward_hook + + def _register_pre_forward_hooks(self): + """Attach hooks for parameter synchronization + + If _pre_forward_hook_is_enabled is set in a parameter, then + the callback will be called the first time any of its + attributes are accessed. This is hackily done by + monkey-patching the parameter class, so proceed with caution. + + """ + for param_group_id, group in enumerate(self.param_groups): + for param_id, param in enumerate(group['params']): + + # Monkey-patch parameter class + cls = param.__class__ + if not getattr(cls, '_has_pre_forward_hook', False): + + # Monkey-patch magic methods to call __getattribute__ + special_funcs = [ + '__abs__', '__add__', '__and__', + '__bool__', '__complex__', '__contains__', + '__deepcopy__', '__delitem__', '__div__', + '__eq__', '__float__', '__floordiv__', + '__ge__', '__getitem__', '__gt__', '__iadd__', + '__iand__', '__idiv__', '__ifloordiv__', + '__ilshift__', '__imod__', '__imul__', + '__index__', '__int__', '__invert__', + '__ior__', '__ipow__', '__irshift__', + '__isub__', '__iter__', '__itruediv__', + '__ixor__', '__le__', '__len__', '__long__', + '__lshift__', '__lt__', '__matmul__', + '__mod__', '__mul__', '__neg__', + '__nonzero__', '__or__', '__pos__', '__pow__', + '__radd__', '__rand__', '__rdiv__', + '__reduce__', '__reduce_ex__', '__reversed__', + '__rfloordiv__', '__rlshift__', '__rmatmul__', + '__rmod__', '__rmul__', '__ror__', '__rpow__', + '__rrshift__', '__rshift__', '__rsub__', + '__rtruediv__', '__rxor__', '__setitem__', + '__sizeof__', '__sub__', '__torch_function__', + '__truediv__', '__xor__', + ] + for func_name in special_funcs: + def make_augmented_func(): + base_func_name = f'_base_{func_name}' + def augmented_func(self, *args, **kwargs): + return getattr(self, base_func_name)(*args, **kwargs) + return augmented_func + setattr(cls, f'_base_{func_name}', getattr(cls, func_name)) + setattr(cls, func_name, make_augmented_func()) + + # Monkey-patch __getattribute__ to call pre-forward hook + def make_getattribute(): + special_attrs = { + '_pre_forward_hook_is_enabled', + '_pre_forward_hook', + '__del__', '__delattr__', '__dir__', '__getattr__', + '__getattribute__', '__hash__', + '__init__', '__new__', '__setattr__', + } + def getattribute_with_pre_forward_hook(self, name): + """Variant of __getattribute__ that can call pre-forward hook""" + if name not in special_attrs: + if getattr(self, '_pre_forward_hook_is_enabled', False): + self._pre_forward_hook_is_enabled = False + self._pre_forward_hook() + return object.__getattribute__(self, name) + return getattribute_with_pre_forward_hook + cls.__getattribute__ = make_getattribute() + cls._has_pre_forward_hook = True + + # Register pre-forward callback + param._pre_forward_hook_is_enabled = False + param._pre_forward_hook = self._make_pre_forward_hook( + param, + param_group_id, + param_id, + ) + + def init_param_buffer(self): + """Allocate contiguous buffer for param buckets + + This converts the parameters into views into the contiguous + buffer. This enables some performance optimizations (e.g. + avoiding some memory copies), but may add memory overhead + (e.g. if the memory allocator can't reuse the original + parameter buffers). To minimize memory overhead, this buffer + should be initialized before the first training step. + + """ + + # Make sure all params are initialized + self.contiguous_param_buffer = True + self.init_params() + + # Construct param buffer + if self.state['buckets']: + buffer_size = max( + bucket.contiguous_buffer_offset + bucket.bucket_size + for bucket in self.state['buckets'] + ) + else: + buffer_size = 0 + self._param_buffer = torch.zeros( + [buffer_size], + dtype=self.param_sync_dtype, + device=self.device, + ) + + # Figure out corresponding positions in params and param buffer + params = list(self.parameters()) + param_flat_views = [] + param_buffer_views = [] + for i, param in enumerate(params): + fragment = self.state[param]['fragments'][0] + bucket_id = fragment.bucket_id + param_size = param.numel() + bucket_start, _ = fragment.bucket_range + buffer_offset = self.state['buckets'][bucket_id].contiguous_buffer_offset + buffer_start = buffer_offset + bucket_start + buffer_end = buffer_start + param_size + buffer_view = self._param_buffer[buffer_start:buffer_end].detach() + if not _devices_match(buffer_view.device, param.device): + raise RuntimeError( + 'Attempted to change a parameter with device={param.device} ' + f'into a buffer view with device={view_buffer.device}' + ) + if buffer_view.dtype != param.dtype: + raise RuntimeError( + f'Attempted to change a parameter with dtype={param.dtype} ' + f'into a buffer view with dtype={view_buffer.dtype}' + ) + param_flat_views.append(param.detach().view(-1)) + param_buffer_views.append(buffer_view) + + # Copy values into param buffer + _multi_tensor_copy( + param_flat_views, + param_buffer_views, + dummy_overflow_buf=self._dummy_overflow_buf, + ) + + # Make all params a view into the param buffer + for param, buffer_view in zip(params, param_buffer_views): + param.data = buffer_view.view(param.size()) + + def _init_grad_buffer(self): + """Allocate contiguous buffer for grad buckets""" + self.contiguous_grad_buffer = True + self.init_params() # Make sure all params are initialized + if self.state['buckets']: + buffer_size = max( + bucket.contiguous_buffer_offset + bucket.bucket_size + for bucket in self.state['buckets'] + ) + else: + buffer_size = 0 + self._grad_buffer = torch.zeros( + [buffer_size], + dtype=self.grad_sync_dtype, + device=self.device, + ) + + def parameters(self): + """Returns an iterator over optimizer parameters""" + return itertools.chain.from_iterable( + group['params'] for group in self.param_groups + ) + + def init_params(self, params=None): + """Initialize optimizer state for parameters + + Ignores parameters that have already been initialized. + + Arguments: + params (iterable, optional): parameters to initialize + (default: all parameters) + + """ + + # Default cases + if params is None: + params = self.parameters() + elif isinstance(params, torch.Tensor): + params = [params] + + # Ignore parameters that have already been initialized + params = [ + param + for param in params + if 'fragments' not in self.state[param] + ] + if not params: + return + + # Get indices corresponding to parameters + id_map = dict() + for param_group_id, group in enumerate(self.param_groups): + for param_id, param in enumerate(group['params']): + id_map[param] = (param_group_id, param_id) + + # Initialize parameters + for param in params: + if param in id_map: + param_group_id, param_id = id_map[param] + self._init_param_state(param, param_group_id, param_id) + + def init_params_bucket(self, params): + """Initialize optimizer state for parameters in one effective bucket + + The buckets corresponding to the provided parameters are + configured so they all perform communication together. Ignores + parameters that have already been initialized. + + Arguments: + params (iterable): parameters to initialize + + """ + + # Ignore parameters that have already been initialized + if isinstance(params, torch.Tensor): + params = [params] + params = [ + param + for param in params + if 'fragments' not in self.state[param] + ] + if not params: + return + + # Get indices corresponding to parameters + id_map = dict() + for param_group_id, group in enumerate(self.param_groups): + for param_id, param in enumerate(group['params']): + id_map[param] = [param_group_id, param_id] + param_ids = [tuple([param] + id_map[param]) for param in params] + + # Mark existings bucket as fully filled + for bucket in self.state['buckets']: + bucket.filled_size = bucket.bucket_size + + # Initialize optimizer state for parameters + start_bucket_id = len(self.state['buckets']) + self.init_params(params) + end_bucket_id = len(self.state['buckets']) + + # Make sure all added buckets depend on provided params + for bucket_id in range(start_bucket_id, end_bucket_id): + bucket = self.state['buckets'][bucket_id] + bucket_size = bucket.bucket_size + bucket.filled_size = bucket_size + ids_in_bucket = set( + (fragment.param_group_id, fragment.param_id) + for fragment in bucket.fragments + ) + for param, param_group_id, param_id in param_ids: + if (param_group_id, param_id) not in ids_in_bucket: + param_size = param.numel() + fragment = self.ParameterFragment( + param_group_id=param_group_id, + param_id=param_id, + bucket_id=bucket_id, + param_range=(param_size, param_size), + bucket_range=(bucket_size, bucket_size), + in_local_shard=False, + shard_range=(None, None), + shard_bucket_range=(None, None), + shard_param_range=(None, None), + ) + self.state[param]['fragments'].append(fragment) + bucket.fragments.append(fragment) + + def _init_param_state( + self, + param, + param_group_id, + param_id, + ): + """Initialize optimizer state for a parameter""" + + # Return immediately if already initialized + if 'fragments' in self.state[param]: + return + self.state[param]['fragments'] = [] + + # Make sure there is at least one bucket + if not self.state['buckets']: + shard_size = self.default_shard_size + bucket_size = shard_size * self.distributed_size + buffer_offset = 0 + self.state['buckets'].append( + self.StateBucket( + bucket_size, + shard_size, + self.dtype, + self.device, + contiguous_buffer_offset=buffer_offset, + store_params=self.store_params, + store_param_remainders=self.store_param_remainders, + ) + ) + + # Split parameter values into fragments + # Note: Each fragment resides within a bucket + param_start = 0 + param_size = param.numel() + while param_start < param_size: + + # Get current bucket + bucket_id = len(self.state['buckets']) - 1 + bucket = self.state['buckets'][bucket_id] + fragment_id = len(bucket.fragments) + bucket_size = bucket.bucket_size + shard_size = bucket.shard_size + + # Determine fragment position within bucket + bucket_start = _round_to_multiple( + bucket.filled_size, + self.alignment, + round_up=True, + ) + fragment_size = min(param_size-param_start, bucket_size-bucket_start) + param_end = param_start + fragment_size + bucket_end = bucket_start + fragment_size + + # Create new bucket if current one is full + if fragment_size <= 0: + shard_size = self.default_shard_size + bucket_size = shard_size * self.distributed_size + buffer_offset = bucket.contiguous_buffer_offset + bucket.bucket_size + self.state['buckets'].append( + self.StateBucket( + bucket_size, + shard_size, + self.dtype, + self.device, + contiguous_buffer_offset=buffer_offset, + store_params=self.store_params, + store_param_remainders=self.store_param_remainders, + ) + ) + continue + + # Fragment position within local shard + shard_id = self.distributed_rank + shard_start = bucket_start - shard_size*shard_id + shard_end = bucket_end - shard_size*shard_id + shard_start = min(max(shard_start, 0), shard_size) + shard_end = min(max(shard_end, 0), shard_size) + in_local_shard = shard_start < shard_end + if in_local_shard: + shard_bucket_start = shard_start + shard_size*shard_id + shard_bucket_end = shard_bucket_start + shard_end - shard_start + shard_param_start = shard_bucket_start - bucket_start + param_start + shard_param_end = shard_param_start + shard_end - shard_start + else: + shard_start, shard_end = None, None + shard_bucket_start, shard_bucket_end = None, None + shard_param_start, shard_param_end = None, None + + # Record fragment info + fragment = self.ParameterFragment( + param_group_id=param_group_id, + param_id=param_id, + bucket_id=bucket_id, + param_range=(param_start,param_end), + bucket_range=(bucket_start,bucket_end), + in_local_shard=in_local_shard, + shard_range=(shard_start,shard_end), + shard_bucket_range=(shard_bucket_start,shard_bucket_end), + shard_param_range=(shard_param_start,shard_param_end), + ) + self.state[param]['fragments'].append(fragment) + bucket.fragments.append(fragment) + bucket.filled_size = bucket_end + param_start = param_end + + # Initialize main param buffer + if self.store_params: + for fragment in self.state[param]['fragments']: + if fragment.in_local_shard: + bucket = self.state['buckets'][fragment.bucket_id] + param_start, param_end = fragment.shard_param_range + shard_start, shard_end = fragment.shard_range + model_param_fragment = param.detach().view(-1)[param_start:param_end] + main_param_fragment = bucket.params_shard[shard_start:shard_end] + main_param_fragment.copy_(model_param_fragment) + + def zero_grad(self, set_to_none=False): + """Clear parameter gradients""" + + # Reset bucket buffers + self._grads_buckets.clear() + + # Construct views into contiguous grad buffer, if needed + if self.contiguous_grad_buffer: + if self._grad_buffer is None: + self._init_grad_buffer() + self._grad_buffer.zero_() + for bucket_id, bucket in enumerate(self.state['buckets']): + bucket_size = bucket.bucket_size + buffer_start = bucket.contiguous_buffer_offset + buffer_end = buffer_start + bucket_size + grad_buffer = self._grad_buffer[buffer_start:buffer_end] + self._grads_buckets[bucket_id].grads_bucket = grad_buffer + + # Reset param grads + for param in self.parameters(): + with _disable_pre_forward_hook(param): + if set_to_none: + param.grad = None + elif (self.contiguous_grad_buffer + and param.dtype == self.grad_sync_dtype + and _devices_match(param.device, self.device)): + param.grad = self.grad_buffer_view(param) + elif param.grad is not None: + param.grad.zero_() + + # Reset other state + self._grad_scale = torch.full([], 1.0, dtype=torch.float32, device=self.device) + self._grad_norm = None + self._dummy_overflow_buf = torch.zeros([1], dtype=torch.int32, device=self.device) + + def _grad_copy(self, param): + """Copy parameter gradients to buckets""" + + # Initialize parameter if needed + if 'fragments' not in self.state[param]: + for param_group_id, group in enumerate(self.param_groups): + for param_id, param_ in enumerate(group['params']): + if param == param_: + self._init_param_state(param, param_group_id, param_id) + if 'fragments' not in self.state[param]: + raise RuntimeError( + 'Could not initialize DistributedFusedAdam with parameter' + ) + + # Copy param grad to buckets + for fragment in self.state[param]['fragments']: + + # Get fragment position + bucket_id = fragment.bucket_id + bucket = self._grads_buckets[bucket_id] + bucket_size = self.state['buckets'][bucket_id].bucket_size + grad_start, grad_end = fragment.param_range + bucket_start, bucket_end = fragment.bucket_range + + # Set reduction status + if bucket.status == self.GradientStatus.SYNCING: + self._finish_bucket_grad_sync() + bucket.status = self.GradientStatus.PARTIALLY_FILLED + + # Allocate gradient buffer if needed + if bucket.grads_bucket is None and self.contiguous_grad_buffer: + if self._grad_buffer is None: + self._init_grad_buffer() + buffer_start = self.state['buckets'][bucket_id].contiguous_buffer_offset + buffer_end = buffer_start + bucket_size + grad_buffer = self._grad_buffer[buffer_start:buffer_end] + if (bucket.grads_shard is None + or bucket.grads_shard.data_ptr() != grad_buffer.data_ptr()): + bucket.grads_bucket = grad_buffer + bucket.grads_bucket.zero_() + if bucket.grads_bucket is None: + bucket.grads_bucket = torch.zeros( + [bucket_size], + dtype=self.grad_sync_dtype, + device=self.device, + ) + + # Copy param grad to bucket + if param.grad is not None: + grad_in = param.grad.detach().view(-1)[grad_start:grad_end] + grad_out = bucket.grads_bucket[bucket_start:bucket_end] + if grad_in.data_ptr() != grad_out.data_ptr(): + grad_out.add_(grad_in) + + # Free param grad buffer + param.grad = None + + def _param_copy(self, params): + """Update parameters with values from parameter buckets""" + + # Get parameter fragments to be synchronized + if isinstance(params, torch.Tensor): + params = [params] + fragments = [] + for param in params: + if 'fragments' in self.state[param]: + fragments.extend( + fragment + for fragment in self.state[param]['fragments'] + if fragment.bucket_id in self._params_buckets + ) + + # Make sure all needed buckets have been synchronized + buckets = collections.OrderedDict() + for fragment in fragments: + bucket_id = fragment.bucket_id + bucket = self._params_buckets[bucket_id] + buckets[bucket] = bucket.status + if any(status != self.ParameterStatus.READY + for bucket, status in buckets.items()): + self._start_bucket_param_sync(buckets.keys()) + self._finish_bucket_param_sync() + + # Copy values from bucket buffers to params + params_in = [] + params_out = [] + for fragment in fragments: + bucket_id = fragment.bucket_id + param_group_id = fragment.param_group_id + param_id = fragment.param_id + bucket_start, bucket_end = fragment.bucket_range + param_start, param_end = fragment.param_range + if param_end > param_start: + bucket = self._params_buckets[bucket_id] + param = self.param_groups[param_group_id]['params'][param_id] + params_in.append(bucket.params_bucket[bucket_start:bucket_end]) + params_out.append(param.detach().view(-1)[param_start:param_end]) + _multi_tensor_copy( + params_in, + params_out, + dummy_overflow_buf=self._dummy_overflow_buf, + ) + + # Delete buckets if possible + for fragment in fragments: + bucket_id = fragment.bucket_id + bucket = self._params_buckets[bucket_id] + bucket_fragments = self.state['buckets'][bucket_id].fragments + param_group_id = fragment.param_group_id + param_id = fragment.param_id + param = self.param_groups[param_group_id]['params'][param_id] + bucket.params_updated.add(param) + if len(bucket.params_updated) == len(bucket_fragments): + del self._params_buckets[bucket_id] + + def grad_buffer_view(self, param): + """Construct view into grad buffer corresponding to param + + Assumes optimizer is using a contiguous grad buffer. + + """ + + # Initialize contiguous grad buffer if needed + assert self.contiguous_grad_buffer + if self._grad_buffer is None: + self._init_grad_buffer() + + # Figure out corresponding position in grad buffer + fragment = self.state[param]['fragments'][0] + bucket_id = fragment.bucket_id + param_size = param.numel() + bucket_start, _ = fragment.bucket_range + buffer_offset = self.state['buckets'][bucket_id].contiguous_buffer_offset + buffer_start = buffer_offset + bucket_start + buffer_end = buffer_start + param_size + + # Construct view into grad buffer + flat_buffer = self._grad_buffer[buffer_start:buffer_end] + return flat_buffer.detach().view(param.size()) + + def _force_bucket_grad_sync(self): + """Ensure that all gradient buckets are synchronized""" + + # Synchronize all unsynchronized buckets + Status = self.GradientStatus + buckets = [] + for bucket_id, bucket in sorted(self._grads_buckets.items()): + if bucket.status not in (Status.READY, Status.SYNCING): + buckets.append(bucket) + if bucket.grads_bucket is None: + bucket_size = self.state['buckets'][bucket_id].bucket_size + bucket.grads_bucket = torch.zeros( + [bucket_size], + dtype=self.grad_sync_dtype, + device=self.device, + ) + if buckets: + self._start_bucket_grad_sync(buckets) + self._finish_bucket_grad_sync() + + # Fill any unsynchronized gradients with zeros + for bucket_id in range(len(self.state['buckets'])): + bucket = self._grads_buckets[bucket_id] + if bucket.grads_shard is None: + shard_size = self.state['buckets'][bucket_id].shard_size + bucket.grads_shard = torch.zeros( + [shard_size], + dtype=self.grad_sync_dtype, + device=self.device, + ) + + def _try_start_bucket_grad_sync( + self, + params=[], + ignore_last_bucket=False, + ): + """Attempt to launch gradient synchronization + + Launches gradient synchronization if any bucket has receieved + all its expected gradients. Gradient synchronization is + asynchronous. + + Arguments: + params (iterable): parameters that have had their + gradients copied to buckets + ignore_last_bucket (bool): avoid synchronizing last bucket + until all gradients have been generated. This avoids + excessive synchronization when initializing buckets in + the first backward pass. + + """ + + # Register params that have generated grads + for param in params: + for fragment in self.state[param]['fragments']: + bucket_id = fragment.bucket_id + bucket = self._grads_buckets[bucket_id] + bucket_fragments = self.state['buckets'][bucket_id].fragments + bucket.grads_generated.add(param) + if len(bucket.grads_generated) == len(bucket_fragments): + bucket.status = self.GradientStatus.FULLY_FILLED + if bucket.grads_bucket is None: + bucket_size = self.state['buckets'][bucket_id].bucket_size + bucket.grads_bucket = torch.zeros( + [bucket_size], + dtype=self.grad_sync_dtype, + device=self.device, + ) + + # Launch reductions if enough buckets are ready + filled_buckets = [] + for bucket_id, bucket in sorted(self._grads_buckets.items()): + if ignore_last_bucket and bucket_id == len(self.state['buckets'])-1: + continue + if bucket.status == self.GradientStatus.FULLY_FILLED: + filled_buckets.append(bucket) + if filled_buckets: + self._start_bucket_grad_sync(filled_buckets) + + def _start_bucket_grad_sync(self, buckets): + """Synchronize gradient buckets + + Gradient synchronization is asynchronous. Involves + reduce-scatter over distributed process group and allreduce + over redundant process group. Assumes grad bucket buffers are + already initialized. + + """ + + # Complete any outstanding grad syncs + # Note: Not needed with contiguous grad buffer since there is + # no memory benefit from eagerly freeing grad buffers. + if not self.contiguous_grad_buffer: + self._finish_bucket_grad_sync() + + # Reduction operation + if self.average_grad_sync: + reduce_op = torch.distributed.ReduceOp.AVG + else: + reduce_op = torch.distributed.ReduceOp.SUM + + # Initialize grad state and buffers + for bucket in buckets: + if bucket.status == self.GradientStatus.SYNCING: + self._finish_bucket_grad_sync() + bucket.status = self.GradientStatus.SYNCING + bucket.grads_generated.clear() + if self.distributed_size == 1: + bucket.sync_grads_shard = bucket.grads_bucket + else: + bucket_size = bucket.grads_bucket.numel() + shard_size = bucket_size // self.distributed_size + bucket.sync_grads_shard = torch.empty( + [shard_size], + dtype=self.grad_sync_dtype, + device=self.device, + ) + + # Side stream for communication + main_stream = torch.cuda.current_stream() + comm_stream = self._pipeline_streams[-1] + comm_stream.wait_stream(main_stream) + + # Reduce-scatter over distributed process group + if self.distributed_size > 1: + with torch.cuda.stream(comm_stream): + for bucket in buckets: + bucket.sync_wait() + sync_requests = [] + group = self.distributed_process_group + with _coalescing_manager(group, self.device, sync_requests): + for bucket in buckets: + bucket.sync_request = ( + reduce_scatter_tensor( + bucket.sync_grads_shard, + bucket.grads_bucket, + op=reduce_op, + group=group, + async_op=True, + ) + ) + sync_requests.append(bucket.sync_request) + + # All-reduce over redundant process group + if self.redundant_size > 1: + with torch.cuda.stream(comm_stream): + for bucket in buckets: + bucket.sync_wait() + sync_requests = [] + group = self.redundant_process_group + with _coalescing_manager(group, self.device, sync_requests): + for bucket in buckets: + bucket.sync_request = ( + torch.distributed.all_reduce( + bucket.sync_grads_shard, + op=reduce_op, + group=group, + async_op=True, + ) + ) + sync_requests.append(bucket.sync_request) + + def _finish_bucket_grad_sync(self): + """Wait for any gradient synchronizations that are in progress""" + main_stream = torch.cuda.current_stream() + comm_stream = self._pipeline_streams[-1] + main_stream.wait_stream(comm_stream) + for bucket_id, bucket in sorted(self._grads_buckets.items()): + if bucket.status == self.GradientStatus.SYNCING: + + # Finish asynchronous communication + bucket.sync_wait() + + # Accumulate gradient in local shard + if bucket.grads_shard is None: + bucket.grads_shard = bucket.sync_grads_shard + else: + bucket.grads_shard.add_(bucket.sync_grads_shard) + bucket.grads_bucket = None + bucket.sync_grads_shard = None + + # Reset status + bucket.status = self.GradientStatus.READY + + # Cached gradient norm has been invalidated + self._grad_norm = None + + def _try_start_bucket_param_sync( + self, + params=None, + ): + """Attempt to launch parameter synchronization + + Launches parameter synchronization for buckets corresponding + to provided parameters, if needed. If parameters are not + provided and no other synchronizations are in progress, + attempts to find a parameter that still requires + synchronization. Parameter synchronization is asynchronous. + + Arguments: + params (iterable, optional): parameters to synchronize + + """ + + # Default behavior: only launch param sync if no other syncs + # are in progress + if params is None: + params = [] + if any(bucket.status == self.ParameterStatus.SYNCING + for bucket in self._params_buckets.values()): + return + for bucket_id, bucket in self._params_buckets.items(): + if bucket.status == self.ParameterStatus.SHARDED: + fragment = self.state['buckets'][bucket_id].fragments[-1] + param_group_id = fragment.param_group_id + param_id = fragment.param_id + param = self.param_groups[param_group_id]['params'][param_id] + params.append(param) + break + + # Find buckets corresponding to params + bucket_ids = set() + for param in params: + bucket_ids.update( + fragment.bucket_id + for fragment in self.state[param]['fragments'] + ) + buckets = [ + self._params_buckets[bucket_id] + for bucket_id in sorted(bucket_ids) + if bucket_id in self._params_buckets + ] + buckets = [ + bucket + for bucket in buckets + if bucket.status == self.ParameterStatus.SHARDED + ] + + # Launch param sync if needed + if buckets: + self._start_bucket_param_sync(buckets) + + def _start_bucket_param_sync(self, buckets): + """Synchronize parameter buckets + + Parameter synchronization is asynchronous. Involves all-gather + over distributed process group. Assumes param shard buffers + are already initialized. + + """ + + # Complete any outstanding param syncs + self._finish_bucket_param_sync() + + # Initialize param state and buffers + buckets = [ + bucket + for bucket in buckets + if bucket.status == self.ParameterStatus.SHARDED + ] + for bucket in buckets: + bucket.status = self.ParameterStatus.SYNCING + if self.distributed_size == 1: + bucket.params_bucket = bucket.params_shard + elif bucket.params_bucket is None: + shard_size = bucket.params_shard.numel() + bucket_size = shard_size * self.distributed_size + bucket.params_bucket = torch.empty( + [bucket_size], + dtype=self.param_sync_dtype, + device=self.device, + ) + + # Side stream for communication + main_stream = torch.cuda.current_stream() + comm_stream = self._pipeline_streams[-1] + comm_stream.wait_stream(main_stream) + + # All-gather over distributed process group + if self.distributed_size > 1: + with torch.cuda.stream(comm_stream): + for bucket in buckets: + bucket.sync_wait() + sync_requests = [] + group = self.distributed_process_group + with _coalescing_manager(group, self.device, sync_requests): + for bucket in buckets: + bucket.sync_request = ( + all_gather_into_tensor( + bucket.params_bucket, + bucket.params_shard, + group=group, + async_op=True, + ) + ) + sync_requests.append(bucket.sync_request) + + def _finish_bucket_param_sync(self): + """Wait for any param synchronizations that are in progress""" + main_stream = torch.cuda.current_stream() + comm_stream = self._pipeline_streams[-1] + main_stream.wait_stream(comm_stream) + for bucket_id, bucket in self._params_buckets.items(): + if bucket.status == self.ParameterStatus.SYNCING: + bucket.sync_wait() + bucket.params_shard = None + bucket.status = self.ParameterStatus.READY + + @contextlib.contextmanager + def no_sync(self, greedy_grad_copy=False): + """Disable overlapped gradient synchronization + + Context manager that is similar to + torch.nn.parallel.DistributedDataParallel.no_sync. The + gradients can be synchronized by calling grad_sync or step. If + overlapped gradient synchronization is enabled, gradients can + also be synchronized by leaving the context and performing a + backward pass. + + Arguments: + greedy_grad_copy (bool, optional): copy parameter + gradients to buckets as soon as they are generated + (default: False) + + """ + old_greedy_grad_copy = self.greedy_grad_copy + old_overlap_grad_sync = self.overlap_grad_sync + self.greedy_grad_copy = greedy_grad_copy + self.overlap_grad_sync = False + try: + yield + finally: + self.greedy_grad_copy = old_greedy_grad_copy + self.overlap_grad_sync = old_overlap_grad_sync + + def grad_sync(self): + """Ensure that all gradients are synchronized""" + for bucket in self.state['buckets']: + for fragment in bucket.fragments: + param_group_id = fragment.param_group_id + param_id = fragment.param_id + param = self.param_groups[param_group_id]['params'][param_id] + if param.grad is not None: + self._grad_copy(param) + if not self.contiguous_grad_buffer: + self._try_start_bucket_grad_sync( + params=[param], + ignore_last_bucket=False, + ) + self._force_bucket_grad_sync() + + def param_sync(self): + """Ensure that all parameters are synchronized""" + if self.contiguous_param_buffer: + self._param_copy(self.parameters()) + else: + while self._params_buckets: + bucket_id, bucket = next(iter((self._params_buckets.items()))) + for fragment in reversed(self.state['buckets'][bucket_id].fragments): + param_id = fragment.param_id + param_group_id = fragment.param_group_id + param = self.param_groups[param_group_id]['params'][param_id] + self._param_copy(param) + self._params_buckets.clear() + + def _local_grad_norm(self, parameters=None, norm_type=2.0): + """Local contribution to parameter gradient norm + + Returns square of 2-norm. Other norms are not yet supported. + + If no parameters are provided, the norm is computed for all + parameters in optimizer. Provided parameters are assumed to be + in optimizer and to require gradients. + + """ + norm_type = float(norm_type) + assert norm_type == 2.0 + + # Make sure that gradients have been reduced + self.grad_sync() + + # Check if provided parameters are subset of all parameters + if parameters is not None: + params_set = set(parameters) + all_params_set = set() + for bucket in self.state['buckets']: + for fragment in bucket.fragments: + param_group_id = fragment.param_group_id + param_id = fragment.param_id + all_params_set.add( + self.param_groups[param_group_id]['params'][param_id] + ) + if not params_set.issubset(all_params_set): + raise RuntimeError( + 'Attempted to compute gradient norm for a parameter ' + 'that is not managed by DistributedFusedAdam' + ) + if params_set == all_params_set: + parameters = None + + if parameters is None: + # Compute norm of all local gradients + grad_norm_sq = multi_tensor_applier( + amp_C.multi_tensor_l2norm, + self._dummy_overflow_buf, + [[bucket.grads_shard for bucket in self._grads_buckets.values()]], + False, + )[0] ** 2 + else: + # Compute norm of selected local gradients + grads = [] + for param in parameters: + if 'fragments' not in self.state[param]: + continue + for fragment in self.state[param]['fragments']: + if fragment.in_local_shard: + bucket = self._grads_buckets[fragment.bucket_id] + shard_start, shard_end = fragment.shard_range + if shard_end > shard_start: + grads.append(bucket.grads_shard[shard_start:shard_end]) + if grads: + grad_norm_sq = multi_tensor_applier( + amp_C.multi_tensor_l2norm, + self._dummy_overflow_buf, + [grads], + False, + )[0] ** 2 + else: + grad_norm_sq = torch.zeros([1], dtype=self.dtype, device=self.device) + + grad_norm_sq = grad_norm_sq.detach() + grad_norm_sq = grad_norm_sq.to(dtype=self.dtype, device=self.device) + grad_norm_sq = grad_norm_sq.view([]) + return grad_norm_sq + + def grad_norm(self, parameters=None, norm_type=2.0, force=False): + """Gradient norm of parameters in optimizer + + The norm is computed over all gradients together, as if they + were concatenated into a single vector. All provided + parameters must be managed by optimizer. + + The computed value is cached to avoid redundant communication. + + Arguments: + parameters (iterable, optional): an iterable of parameters + in optimizer (default: all parameters in optimizer). + norm_type (float or int, optional): type of the used + p-norm (default: 2). Only 2-norm is currently + supported. + force (bool, optional): ignore cached value and force norm + computation (default: False). + + """ + if force or self._grad_norm is None: + norm_type = float(norm_type) + assert norm_type == 2.0 + grad_norm_sq = self._local_grad_norm( + parameters=parameters, + norm_type=norm_type, + ) + torch.distributed.all_reduce( + grad_norm_sq, + op=torch.distributed.ReduceOp.SUM, + group=self.distributed_process_group, + ) + self._grad_norm = grad_norm_sq.sqrt() + grad_norm = self._grad_norm * self._grad_scale + return grad_norm.detach() + + def clip_grad_norm(self, max_norm, parameters=None, norm_type=2.0): + """Clips gradient norm of parameters in optimizer + + The norm is computed over all gradients together, as if they + were concatenated into a single vector. The scaling is + deferred until the optimizer step, which should be called + immediately after this function. + + The computed grad norm is cached to avoid redundant + communication. + + Arguments: + max_norm (float or int): max norm of the gradients + parameters (iterable, optional): an iterable of parameters + in optimizer (default: all parameters in optimizer). + norm_type (float or int, optional): type of the used + p-norm (default: 2) + + """ + assert max_norm > 0 + total_norm = self.grad_norm(parameters=parameters, norm_type=norm_type) + clip_coef = max_norm / (total_norm + 1e-6) + clip_coef_clamped = torch.clamp(clip_coef, max=1.0) + self._grad_scale *= clip_coef_clamped + return total_norm + + def unscale_grads(self, inv_scale, *args): + """Custom unscale function for use by AMP gradient scaler + + Overflow checking is deferred to optimization step. + + Arguments: + inv_scale (torch.Tensor): factor to multiply gradients + + """ + self._grad_scale *= inv_scale.view([]) + return { self.device: torch.zeros(1, dtype=torch.float32, device=self.device) } + + def step(self, closure=None, *, grad_scaler=None): + """Apply Adam optimizer step + + Arguments: + closure (callable, optional): closure to recompute loss + (default: None) + grad_scaler (torch.cuda.amp.GradScaler, optional): + gradient scaler (default: None) + + """ + + # Apply closure + loss = None + if closure is not None: + loss = closure() + + # Make sure that parameters and gradients are synchronized + self.param_sync() + self.grad_sync() + + # Apply gradient scaler if provided + # Note: We compute gradient norm to check for non-finite + # values. This is more conservative and compute intensive than + # directly checking, but it avoids extra communication if we + # have already computed gradient norm e.g. for gradient + # clipping. + if grad_scaler is not None: + grad_scaler_state = grad_scaler._per_optimizer_states[id(self)] + GradScalerOptState = torch.cuda.amp.grad_scaler.OptState + if grad_scaler_state['stage'] is GradScalerOptState.READY: + assert grad_scaler._scale is not None + self._grad_scale /= grad_scaler._scale.view([]) + grad_norm = self.grad_norm() + found_inf = torch.logical_not(torch.isfinite(grad_norm)) + scaler_state = grad_scaler._per_optimizer_states[id(self)] + scaler_state['found_inf_per_device'] = {found_inf.device: found_inf.float()} + if found_inf.item(): + return + self._grad_scale = self._grad_scale.to(dtype=torch.float32, device=self.device) + + # Initialize param shard buffers + for bucket_id in reversed(range(len(self.state['buckets']))): + bucket = self.ParameterBucket() + self._params_buckets[bucket_id] = bucket + shard_size = self.state['buckets'][bucket_id].shard_size + if self.contiguous_param_buffer: + if self._param_buffer is None: + self.init_param_buffer() + bucket_size = self.state['buckets'][bucket_id].bucket_size + buffer_start = self.state['buckets'][bucket_id].contiguous_buffer_offset + buffer_end = buffer_start + bucket_size + bucket.params_bucket = ( + self._param_buffer[buffer_start:buffer_end] + ) + bucket_start = self.distributed_rank * shard_size + bucket_end = bucket_start + shard_size + bucket.params_shard = ( + bucket.params_bucket[bucket_start:bucket_end] + ) + else: + bucket.params_shard = torch.empty( + [shard_size], + dtype=self.param_sync_dtype, + device=self.device, + ) + + # Apply optimizer step and synchronize params + self.state['step'] += 1 + if self.distributed_size > 1 and self.overlap_param_sync and self.state['buckets']: + # Local step and non-blocking param sync + # Note: Overlap param sync of first buckets with optimizer + # step of remaining buckets. + + # Get buckets containing "first" parameter + fragment = self.state['buckets'][-1].fragments[-1] + param_group_id = fragment.param_group_id + param_id = fragment.param_id + param = self.param_groups[param_group_id]['params'][param_id] + first_bucket_ids = sorted( + fragment.bucket_id + for fragment in self.state[param]['fragments'] + ) + + # Local step and launch param sync for first buckets + self._local_step(first_bucket_ids) + self._start_bucket_param_sync( + self._params_buckets[bucket_id] + for bucket_id in first_bucket_ids + ) + + # Local step for remaining buckets + first_bucket_ids = set(first_bucket_ids) + self._local_step( + bucket_id + for bucket_id in range(len(self.state['buckets'])) + if bucket_id not in first_bucket_ids + ) + + # Enable pre-forward hook + for param in self.parameters(): + param._pre_forward_hook_is_enabled = True + + else: + # Local step and blocking param sync + self._local_step(list(range(len(self.state['buckets'])))) + self.param_sync() + + return loss + + def _local_step(self, bucket_ids): + """Apply optimizer step to local shard of parameter buckets + + Arguments: + bucket_ids (iterable): bucket indices + + """ + + # Optimized implementation with BF16 params and 16-bit param + # remainders + if self.store_param_remainders: + self._local_step_with_param_remainders(bucket_ids) + return + + # Find param fragments for each bucket + buffers = collections.defaultdict(list) # p_in, m, v, g, p_out + for bucket_id in bucket_ids: + + # Optimizer state buffers for local shard + fragments = self.state['buckets'][bucket_id].fragments + exp_avg = self.state['buckets'][bucket_id].exp_avg_shard + exp_avg_sq = self.state['buckets'][bucket_id].exp_avg_sq_shard + grads = self._grads_buckets[bucket_id].grads_shard + params_out = self._params_buckets[bucket_id].params_shard + + # Find param fragments in local shard + for fragment in fragments: + if fragment.in_local_shard: + param_group_id = fragment.param_group_id + shard_start, shard_end = fragment.shard_range + if self.store_params: + params_shard = self.state['buckets'][bucket_id].params_shard + param_fragment = params_shard[shard_start:shard_end] + else: + param_id = fragment.param_id + param = self.param_groups[param_group_id]['params'][param_id] + param_start, param_end = fragment.shard_param_range + param_fragment = param.detach().view(-1)[param_start:param_end] + param_fragment = param_fragment.to(dtype=self.dtype, device=self.device) + if shard_end > shard_start: + buffers[param_group_id].append([ + param_fragment, + exp_avg[shard_start:shard_end], + exp_avg_sq[shard_start:shard_end], + grads[shard_start:shard_end], + params_out[shard_start:shard_end], + ]) + + # Apply optimizer step to each param group + for group_id, group_buffers in buffers.items(): + group = self.param_groups[group_id] + beta1, beta2 = group['betas'] + multi_tensor_applier( + distributed_adam_cuda.multi_tensor_fused_adam, + self._dummy_overflow_buf, + list(zip(*group_buffers)), + self._grad_scale, + group['lr'], + beta1, + beta2, + group['eps'], + self.state['step'], + 1 if self.adam_w_mode else 0, + 1 if group['bias_correction'] else 0, + group['weight_decay'], + ) + + def _local_step_with_param_remainders(self, bucket_ids): + """Apply optimizer step to local shard of parameter bucket + + This is an experimental implementation that expects + store_params=False and store_param_remainders=True. The + optimizer dtype must be FP32 and the params must all be BF16 + and GPU. + + Arguments: + bucket_ids (iterable): bucket indices + + """ + + # Find param fragments for each bucket + buffers = collections.defaultdict(list) # p_in, p_rem, m, v, g, p_out + for bucket_id in bucket_ids: + + # State buffers for local shard + fragments = self.state['buckets'][bucket_id].fragments + param_remainders_shard = self.state['buckets'][bucket_id].param_remainders_shard + exp_avg = self.state['buckets'][bucket_id].exp_avg_shard + exp_avg_sq = self.state['buckets'][bucket_id].exp_avg_sq_shard + grads = self._grads_buckets[bucket_id].grads_shard + params_out = self._params_buckets[bucket_id].params_shard + + # Find param fragments in local shard + for fragment in fragments: + if fragment.in_local_shard: + param_group_id = fragment.param_group_id + param_id = fragment.param_id + param_start, param_end = fragment.shard_param_range + shard_start, shard_end = fragment.shard_range + param = self.param_groups[param_group_id]['params'][param_id] + param_fragment = param.detach().view(-1)[param_start:param_end] + param_fragment = param_fragment.to(dtype=torch.bfloat16, device=self.device) + if shard_end > shard_start: + buffers[param_group_id].append([ + param_fragment, + param_remainders_shard[shard_start:shard_end], + exp_avg[shard_start:shard_end], + exp_avg_sq[shard_start:shard_end], + grads[shard_start:shard_end], + params_out[shard_start:shard_end], + ]) + + # Apply optimizer step to each param group + for group_id, group_buffers in buffers.items(): + group = self.param_groups[group_id] + beta1, beta2 = group['betas'] + multi_tensor_applier( + distributed_adam_cuda.multi_tensor_fused_adam_with_param_remainders, + self._dummy_overflow_buf, + list(zip(*group_buffers)), + self._grad_scale, + group['lr'], + beta1, + beta2, + group['eps'], + self.state['step'], + 1 if self.adam_w_mode else 0, + 1 if group['bias_correction'] else 0, + group['weight_decay'], + ) + + def state_dict(self, gather_on_root=True): + """Get dictionary containing optimizer state + + Default behavior is to perform communication so that the + entire optimizer state is returned on the root rank in the + process group. In this case, all ranks in the process group + must enter this function and no value is returned on non-root + ranks. + + Arguments: + gather_on_root (bool, optional): Gather state from all + ranks on the root rank (default: True) + + """ + state_dict = super().state_dict() + if not gather_on_root: + return state_dict + + # Finish any asynchronous communication + self.grad_sync() + self.param_sync() + + # Export local state to byte string + state_bytes = io.BytesIO() + torch.save(state_dict, state_bytes) + state_bytes.seek(0) + state_bytes_view = state_bytes.getbuffer() + + # Get data sizes on all ranks + local_state_size = len(state_bytes_view) + state_sizes = [None] * self.distributed_size + torch.distributed.all_gather_object( + state_sizes, + local_state_size, + group=self.process_group, + ) + max_state_size = max(state_sizes) + + # Construct workspace buffers + chunk_size = self.default_shard_size * torch.finfo(self.grad_sync_dtype).bits // 8 + if self.distributed_rank == 0: + gathered_state_bytes = [ + torch.empty([size], dtype=torch.uint8, device='cpu') + for size in state_sizes + ] + gathered_state_bytes[0].copy_( + torch.frombuffer(state_bytes_view, dtype=torch.uint8) + ) + gathered_chunks_buffers = [ + torch.empty( + [chunk_size * self.distributed_size], + dtype=torch.uint8, + device=self.device, + ) + for _ in range(self.pipeline_size) + ] + else: + chunk_buffers = [ + torch.empty( + [chunk_size], + dtype=torch.uint8, + device=self.device, + ) + for _ in range(self.pipeline_size) + ] + + # Split data into chunks and gather on root rank + # Note: Assuming we are using the NCCL backend, communication + # must happen on the GPU. We split the data into fixed-size + # chunks to limit GPU memory usage. + # TODO: Avoid chunking with direct communication between CPUs + main_stream = torch.cuda.current_stream() + for stream in self._pipeline_streams: + stream.wait_stream(main_stream) + for stream_id, offset in enumerate(range(0, max_state_size, chunk_size)): + stream_id %= self.pipeline_size + stream = self._pipeline_streams[stream_id] + with torch.cuda.stream(stream): + + # Buffers for chunk + if self.distributed_rank == 0: + gathered_chunks = [ + gathered_chunks_buffers[stream_id][i*chunk_size:(i+1)*chunk_size] + for i in range(self.distributed_size) + ] + else: + chunk = chunk_buffers[stream_id] + + # Copy to GPU + if self.distributed_rank != 0 and offset < local_state_size: + local_chunk_size = min(chunk_size, local_state_size-offset) + chunk[:local_chunk_size].copy_( + torch.frombuffer( + state_bytes_view, + dtype=torch.uint8, + count=local_chunk_size, + offset=offset, + ), + non_blocking=True, + ) + + # Gather on root + # Note: Call in main stream to avoid memory pool + # overheads from internal memory allocations in + # gather. + main_stream.wait_stream(stream) + with torch.cuda.stream(main_stream): + if self.distributed_rank == 0: + if self._gather_no_copy: + no_copy_kwarg = { 'no_copy': True } + else: + no_copy_kwarg = {} + torch.distributed.gather( + gathered_chunks[0], + gathered_chunks, + dst=self.process_group_root, + group=self.process_group, + **no_copy_kwarg, + ) + else: + torch.distributed.gather( + chunk, + dst=self.process_group_root, + group=self.process_group, + ) + stream.wait_stream(main_stream) + + # Copy back to CPU + if self.distributed_rank == 0: + for rank in range(1, self.distributed_size): + rank_chunk_start = offset + rank_chunk_end = min(offset + chunk_size, state_sizes[rank]) + rank_chunk_size = rank_chunk_end - rank_chunk_start + if rank_chunk_size > 0: + src = gathered_chunks[rank][:rank_chunk_size] + dst = gathered_state_bytes[rank][rank_chunk_start:rank_chunk_end] + dst.copy_(src, non_blocking=True) + + # Synchronize GPU + for stream in self._pipeline_streams: + main_stream.wait_stream(stream) + main_stream.synchronize() + + # Return gathered state data on root rank + if self.distributed_rank == 0: + return {'gathered_states': gathered_state_bytes} + else: + return None + + def load_state_dict(self, state_dict): + """Load optimizer state""" + + # State dict contains state for all ranks + if 'gathered_states' in state_dict: + + # Deallocate distributed optimizer state to reduce GPU + # memory usage + if 'buckets' in self.state: + del self.state['buckets'] + + # Get state for current rank and parse byte string + state_bytes = state_dict['gathered_states'][self.distributed_rank] + state_bytes = io.BytesIO(state_bytes.numpy()) + state_dict = torch.load(state_bytes) + + return super().load_state_dict(state_dict) diff --git a/apex/apex/contrib/optimizers/distributed_fused_lamb.py b/apex/apex/contrib/optimizers/distributed_fused_lamb.py new file mode 100644 index 00000000..7c87ef36 --- /dev/null +++ b/apex/apex/contrib/optimizers/distributed_fused_lamb.py @@ -0,0 +1,1061 @@ +import os +import math +import inspect +import torch +import importlib +import amp_C +from apex.multi_tensor_apply import multi_tensor_applier + +import torch.distributed.distributed_c10d as c10d + +# Fallback to private fields if using older PyTorch version +try: + import torch.distributed.distributed_c10d.get_process_group_ranks +except ImportError: + def get_process_group_ranks(group): + return list(c10d._pg_group_ranks[group].keys()) + +_make_nccl_premul_sum = getattr(torch.distributed, "_make_nccl_premul_sum", None) +# Ref: https://github.com/pytorch/pytorch/pull/81272 +if _make_nccl_premul_sum is None: + if hasattr(torch.distributed, "make_nccl_premul_sum"): + _make_nccl_premul_sum = torch.distributed.make_nccl_premul_sum + +class DistributedFusedLAMB(torch.optim.Optimizer): + + """Implements LAMB algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``. + + This version of fused LAMB implements 2 fusions. + + * Fusion of the LAMB update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.optimizers.FusedLAMB`'s usage is identical to any ordinary Pytorch optimizer:: + + opt = apex.optimizers.FusedLAMB(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedLAMB` may be used with or without Amp. If you wish to use :class:`FusedLAMB` with Amp, + you may choose any ``opt_level``:: + + opt = apex.optimizers.FusedLAMB(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + + In general, ``opt_level="O1"`` is recommended. + + LAMB was proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its norm. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + NOT SUPPORTED now! (default: False) + adam_w_mode (boolean, optional): Apply L2 regularization or weight decay + True for decoupled weight decay(also known as AdamW) (default: True) + grad_averaging (bool, optional): whether apply (1-beta2) to grad when + calculating running averages of gradient. (default: True) + set_grad_none (bool, optional): whether set grad to None when zero_grad() + method is called. (default: True) + max_grad_norm (float, optional): value used to clip global grad norm + (default: 1.0) + use_nvlamb (boolean, optional): Apply adaptive learning rate to 0.0 + weight decay parameter (default: False) + step_supports_amp_scaling(boolean, optional): whether to use customized + gradient unscaling logic (default: True) + + .. _Large Batch Optimization for Deep Learning - Training BERT in 76 minutes: + https://arxiv.org/abs/1904.00962 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + class AtomicCounter(object): + def __init__(self): + self.value = 0 + self.order = [] + import threading + self._lock = threading.Lock() + + def add(self, idx): + with self._lock: + self.value += 1 + self.order.append(idx) + + def __init__(self, params, + lr=1e-3, bias_correction = True, grad_averaging=True, + betas=(0.9, 0.999), eps=1e-8, + weight_decay=0., max_grad_norm=0., + adam_w_mode=True, use_nvlamb=False, + step_supports_amp_scaling=True, overlap_reductions=True, + dwu_group_size=0, dwu_num_blocks=4, dwu_num_chunks=4, + dwu_num_rs_pg=1, dwu_num_ar_pg=4, dwu_num_ag_pg=0, fused_norm=False, + e5m2_allgather=False, verbose=False, clip_after_ar=True, + full_ar=False, set_param_views_to_flat_buffer=False, skip_allgather=False, + fuse_scale=False, param_order=None, nccl_allgather_channels=0): + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay, + grad_averaging=grad_averaging, + max_grad_norm=max_grad_norm) + + super(DistributedFusedLAMB, self).__init__(params, defaults) + + global fused_adam_cuda, distributed_lamb_cuda + fused_adam_cuda = importlib.import_module("fused_adam_cuda") + distributed_lamb_cuda = importlib.import_module("distributed_lamb_cuda") + + self._overflow_buf = torch.cuda.IntTensor([0]) + self._has_overflow = False + self.multi_tensor_lamb_compute_update_term = distributed_lamb_cuda.multi_tensor_lamb_compute_update_term + self.multi_tensor_lamb_update_weights = distributed_lamb_cuda.multi_tensor_lamb_update_weights + import amp_C + self.multi_tensor_l2norm = amp_C.multi_tensor_l2norm + + self._grad_averaging = grad_averaging + self._adam_w_mode = 1 if adam_w_mode else 0 + self._use_nvlamb = use_nvlamb + self._step_supports_amp_scaling = step_supports_amp_scaling + self._is_accumulation_step = False + self._last_step = False + self._overlap_reductions = overlap_reductions + self._global_scale = None + self._num_blocks = dwu_num_blocks + self._num_chunks = dwu_num_chunks + self._e5m2_allgather = e5m2_allgather + self._verbose = verbose + self._clip_after_ar = clip_after_ar + self._full_ar = full_ar + self._fuse_scale = fuse_scale + self._L2_grad_norm = None + self._set_flat_param_view = set_param_views_to_flat_buffer + self._skip_ag = skip_allgather + self._fused_norm = fused_norm if not clip_after_ar else False + self._current_process_group = c10d._get_default_group() + self._available_ranks = get_process_group_ranks(self._current_process_group) + self._group_size = torch.cuda.device_count() if dwu_group_size <= 0 else dwu_group_size + self._world_size = torch.distributed.get_world_size() + self._num_groups = self._world_size // self._group_size + self._rank_in_group = torch.distributed.get_rank() % self._group_size + + self._lr = torch.tensor(0.0, dtype=torch.float32, device='cuda') + + self._resume_from_checkpoint = False + self._step = torch.cuda.IntTensor([0]) + + # Master weight, moment, gradient buffers + self._fp32_p, self._fp32_m, self._fp32_v, self._fp16_p, self._fp16_g = None, None, None, None, None + + # Check if collectives have no_copy option + self._reduce_scatter_no_copy = ( + 'no_copy' in inspect.getfullargspec(torch.distributed.reduce_scatter).args + ) + self._all_gather_no_copy = ( + 'no_copy' in inspect.getfullargspec(torch.distributed.all_gather).args + ) + + if "reduce_scatter_tensor" not in dir(torch.distributed): + torch.distributed.reduce_scatter_tensor = torch.distributed._reduce_scatter_base + if "all_gather_into_tensor" not in dir(torch.distributed): + torch.distributed.all_gather_into_tensor = torch.distributed._all_gather_base + + self._num_rs_pg = dwu_num_rs_pg + self._num_ar_pg = dwu_num_ar_pg + self._num_ag_pg = dwu_num_ag_pg + + if self._full_ar: # full all reduce, only need AR and AG groups + # l2_grad_norm may be reduced within a node to limit from memory reads + for group_i in range(self._num_groups): + ranks = [group_i*self._group_size+j for j in range(self._group_size)] + l2_grad_norm_pg = torch.distributed.new_group(ranks=ranks) + if torch.distributed.get_rank() in ranks: + self._l2_grad_norm_pg = l2_grad_norm_pg + + self._ar_pg = [] + # consider all the ranks + ranks = list(range(0, self._world_size)) + for i in range(self._num_ar_pg): + if self._verbose: + print(f"creating new AR group {i}: {ranks}") + grp = torch.distributed.new_group(ranks=ranks) + if grp != torch.distributed.GroupMember.NON_GROUP_MEMBER: + if self._verbose: + print(f"group {i}: init barrier (device: {torch.cuda.current_device()})") + torch.distributed.barrier(group=grp, device_ids=[torch.cuda.current_device()]) + if self._verbose: + print(f"created new AR group {i}: {ranks}") + + if torch.distributed.get_rank() in ranks: + self._ar_pg.append(grp) + self._ar_st = [torch.cuda.Stream() for _ in range(self._num_ar_pg)] + if nccl_allgather_channels > 0: + os.putenv('NCCL_MAX_NCHANNELS', str(nccl_allgather_channels)) + if self._num_ag_pg == 0: + self._ag_pg = self._ar_pg + self._ag_st = self._ar_st + self._num_ag_pg = self._num_ar_pg + else: + self._ag_pg = [] + ranks = [] + stride = torch.cuda.device_count() + for i in range(self._num_groups): + rs = list(range(i*stride, (i+1)*stride)) + ranks.append(rs) + for rs in ranks: + for i in range(self._num_ag_pg): + grp = torch.distributed.new_group(ranks=rs) + if torch.distributed.get_rank() in rs: + if self._verbose: + print(f"creating AG group {i}: {rs}") + self._ag_pg.append(grp) + + self._ag_st = [torch.cuda.Stream() for _ in range(self._num_ag_pg)] + else: # reduce-scatter + all-reduce, need RS, AR, AG groups + if self._num_groups > 1: + self._ar_pg = [] + for dev_i in range(self._group_size): + ranks = [dev_i+j*self._group_size for j in range(self._num_groups)] + for i in range(self._num_ar_pg): + if self._verbose: + print(f"creating new AR group {i}: {ranks}") + grp = torch.distributed.new_group(ranks=ranks) + if grp != torch.distributed.GroupMember.NON_GROUP_MEMBER: + if self._verbose: + print(f"group {i}: init barrier (device: {torch.cuda.current_device()})") + torch.distributed.barrier(group=grp, device_ids=[torch.cuda.current_device()]) + if self._verbose: + print(f"created new AR group {i}: {ranks}") + + if torch.distributed.get_rank() in ranks: + self._ar_pg.append(grp) + self._ar_st = [torch.cuda.Stream() for _ in range(self._num_ar_pg)] + rs_ranks = [] + for group_i in range(self._num_groups): + rs_ranks.append([group_i*self._group_size+j for j in range(self._group_size)]) + self._rs_pg = [] + for group_i in range(self._num_groups): + ranks = rs_ranks[group_i] + for i in range(self._num_rs_pg): + grp = torch.distributed.new_group(ranks=ranks) + if torch.distributed.get_rank() in ranks: + self._rs_pg.append(grp) + if self._verbose: + print(f"creating RS group : {ranks}") + l2_grad_norm_pg = torch.distributed.new_group(ranks=ranks) + if torch.distributed.get_rank() in ranks: + self._l2_grad_norm_pg = l2_grad_norm_pg + self._rs_st = [torch.cuda.Stream() for _ in range(self._num_rs_pg)] + if self._num_ag_pg == 0: + self._ag_pg = self._rs_pg + self._ag_st = self._rs_st + self._num_ag_pg = self._num_rs_pg + else: + self._ag_pg = [] + for group_i in range(self._num_groups): + ranks = rs_ranks[group_i] + for i in range(self._num_ag_pg): + grp = torch.distributed.new_group(ranks=ranks) + if torch.distributed.get_rank() in ranks: + self._ag_pg.append(grp) + if self._verbose: + print(f"creating AG group : {ranks}") + self._ag_st = [torch.cuda.Stream() for _ in range(self._num_ag_pg)] + for ag_pg in self._ag_pg: + torch.distributed.barrier(group=ag_pg) + + self._l2_grad_norm_st = torch.cuda.Stream() + self._completion_st = torch.cuda.Stream() + self._step.record_stream(self._completion_st) + + self._reductions_works = [None]*self._num_blocks + self._allgather_works = [None]*self._num_blocks + + self._one = torch.cuda.IntTensor([1]) + + self._first_step = True + self._lazy_init_stage1_done, self._lazy_init_stage2_done = False, False + self._param_order = self.AtomicCounter() + + p_offset = 0 + p_i = 0 + self._model_params = [] + self._grad_accs = [] + self._group_properties = [] + for group in self.param_groups: + prev = None + beta1, beta2 = group['betas'] + beta3 = 1.0 - beta1 if self._grad_averaging else 1.0 + bias_correction = 1 if group['bias_correction'] else 0 + eps = group['eps'] + weight_decay = group['weight_decay'] + for p in group['params']: + if not p.requires_grad: + continue + self._model_params.append(p) + self._group_properties.append(( + weight_decay, + bias_correction, + beta1, + beta2, + beta3, + eps + )) + p_grads_size = p.numel() + if self._set_flat_param_view: + if param_order: + # this is executed when param_order is specified by the user + self._param_order.add(param_order[p]) + else: + self._param_order.add(p_i) + p_offset += p_grads_size + # Only enforce 128b alignment (64 * fp16) for non-consecutive parameters + # RNN is one example of consecutive parameters: + # (weight_ih, weight_hh, bias_ih, bias_hh) + if prev is not None and (prev.data_ptr() + prev.numel() * prev.element_size() != p.data_ptr()): + p_offset = ((p_offset + 63) // 64) * 64 + prev = p + p_i += 1 + if param_order: + self._param_order.order = torch.argsort(torch.tensor(self._param_order.order)).tolist() + self._grads_generated = [False]*len(self._model_params) + self._grads_fp16, self._grads_fp32 = [], [] + if self._overlap_reductions: + self._current_block = self._num_blocks + + self._net_total_param_size = p_offset + self._total_param_size = p_offset + dwu_min_page_size = 256 * self._num_blocks * self._num_chunks * self._group_size + self._total_param_size = ((self._total_param_size + dwu_min_page_size - 1) // dwu_min_page_size) * dwu_min_page_size + self._new_params = torch.zeros([self._total_param_size], dtype=torch.uint8 if self._e5m2_allgather else torch.float16, device='cuda') + + + + def _lazy_init_stage1(self): + if self._lazy_init_stage1_done: return + + p_i = 0 + #self._model_params = [] + #self._grad_accs = [] + #self._group_properties = [] + for group in self.param_groups: + for p in group['params']: + torch.distributed.broadcast(p, 0) + if not p.requires_grad: + continue + def wrapper(param, param_i): + param_tmp = param.expand_as(param) + grad_acc = param_tmp.grad_fn.next_functions[0][0] + def allreduce_hook(*unused): + if not self._set_flat_param_view: + if self._first_step: + # first time + self._param_order.add(param_i) + else: + idx = self._param_order.order.index(param_i) + self._do_overlapped_reduction(idx, param) + else: + if not self._first_step: + idx = self._param_order.order.index(param_i) + self._do_overlapped_reduction(idx, param) + grad_acc.register_hook(allreduce_hook) + self._grad_accs.append(grad_acc) + wrapper(p, p_i) + p_i += 1 + + self._block_size = self._total_param_size // self._num_blocks + self._chunk_size = self._block_size // self._num_chunks + self._shard_size = self._chunk_size // self._group_size + + self._flat_grads = torch.zeros([self._total_param_size], dtype=torch.float16, device='cuda') + self._mega_shard_size = self._num_blocks * self._num_chunks * self._shard_size + # initialize master weights, moments buffers if not loaded from checkpoint + if self._fp32_p is None: + self._fp32_p = torch.zeros([self._mega_shard_size], dtype=torch.float32, device='cuda') + self._fp32_m = torch.zeros([self._mega_shard_size], dtype=torch.float32, device='cuda') + self._fp32_v = torch.zeros([self._mega_shard_size], dtype=torch.float32, device='cuda') + self._fp32_u = torch.zeros([self._mega_shard_size], dtype=torch.float32, device='cuda') + # FIXME: Rethink fp16 label since it's either uint8 or fp16 + self._fp16_p = torch.zeros([self._mega_shard_size], dtype=torch.uint8 if self._e5m2_allgather else torch.float16, device='cuda') + self._fp16_g = torch.zeros([self._mega_shard_size], dtype=torch.float16, device='cuda') + + def _flat_split(p): + def __blockify(p): + return [p[block_id*self._block_size:(block_id+1)*self._block_size] for block_id in range(self._num_blocks)] + def __chunkify(p): + return [p[chunk_id*self._chunk_size:(chunk_id+1)*self._chunk_size] for chunk_id in range(self._num_chunks)] + def __shardify(p): + return [p[shard_id*self._shard_size:(shard_id+1)*self._shard_size] for shard_id in range(self._group_size)] + list_of_blocks = __blockify(p) + list_of_list_of_chunks = [__chunkify(block) for block in list_of_blocks] + list_of_list_of_list_of_shards = [[__shardify(chunk) for chunk in chunks] for chunks in list_of_list_of_chunks] + return list_of_blocks, list_of_list_of_chunks, list_of_list_of_list_of_shards + + # note(crcrpar): the function below doesn't seem to be used at all. + # def _flat_split_no_shards(p): + # def __blockify(p): + # return [p[block_id*self._block_size:(block_id+1)*self._block_size] for block_id in range(self._num_blocks)] + # def __chunkify(p): + # return [p[chunk_id*self._chunk_size:(chunk_id+1)*self._chunk_size] for chunk_id in range(self._num_chunks)] + # list_of_blocks = __blockify(self._flat_grads) + # list_of_list_of_chunks = [__chunkify(block) for block in list_of_blocks] + # return list_of_blocks, list_of_list_of_chunks + + def _full_packed_split(p): + def __shardify(p): + return [p[mega_shard*self._mega_shard_size:(mega_shard+1)*self._mega_shard_size] for mega_shard in range(self._group_size)] + def __blockify(p): + return [p[block_id*self._num_chunks*self._shard_size:(block_id+1)*self._num_chunks*self._shard_size] for block_id in range(self._num_blocks)] + def __chunkify(p): + return [p[chunk_id*self._shard_size:(chunk_id+1)*self._shard_size] for chunk_id in range(self._num_chunks)] + list_of_mega_shards = __shardify(p) + list_of_list_of_mega_blocks = [__blockify(mega_shard) for mega_shard in list_of_mega_shards] + list_of_list_of_list_of_mega_chunks = [[__chunkify(mega_block) for mega_block in mega_blocks] for mega_blocks in list_of_list_of_mega_blocks] + return list_of_mega_shards, list_of_list_of_mega_blocks, list_of_list_of_list_of_mega_chunks + def _packed_split(p): + def __packed_blockify(p): + packed_block_size = self._num_chunks*self._shard_size + return [p[block_id*packed_block_size:(block_id+1)*packed_block_size] for block_id in range(self._num_blocks)] + def __packed_chunkify(p): + # in the packed format, each chunk contains one shard, so packed_chunk_size == self._shard_size + return [p[chunk_id*self._shard_size:(chunk_id+1)*self._shard_size] for chunk_id in range(self._num_chunks)] + list_of_blocks = __packed_blockify(p) + list_of_list_of_chunks = [__packed_chunkify(block) for block in list_of_blocks] + return list_of_blocks, list_of_list_of_chunks + def _split_assign(shards): + packed_block_size = self._num_chunks*self._shard_size + list_of_list_of_chunks=[] + for block_id in range(self._num_blocks): + list_of_chunks=[] + for chunk_id in range(self._num_chunks): + #self._fp16_g[block_id*packed_block_size+chunk_id*self._shard_size:block_id*packed_block_size+(chunk_id+1)*self._shard_size] = shards[block_id][chunk_id][self._rank_in_group] + list_of_chunks.append( shards[block_id][chunk_id][self._rank_in_group]) + list_of_list_of_chunks.append(list_of_chunks) + return list_of_list_of_chunks + + self._new_params_mega_shards, self._new_params_mega_blocks, self._new_params_mega_chunks = _full_packed_split(self._new_params) + # this splitting scheme is needed when allgather needs to be split into multiple chunks in a contiguous way + self._new_params2_blocks, self._new_params2_chunks, self._new_params2_shards = _flat_split(self._new_params) + + self._fp32_p_blocks, self._fp32_p_chunks = _packed_split(self._fp32_p) + self._fp32_m_blocks, self._fp32_m_chunks = _packed_split(self._fp32_m) + self._fp32_v_blocks, self._fp32_v_chunks = _packed_split(self._fp32_v) + self._fp32_u_blocks, self._fp32_u_chunks = _packed_split(self._fp32_u) + self._fp16_p_blocks, self._fp16_p_chunks = _packed_split(self._fp16_p) + + if self._full_ar: + # for gradient all-reduce + self._flat_grads_blocks, self._flat_grads_chunks, self._flat_grads_shards = _flat_split(self._flat_grads) + # for weight update + self._fp16_g_chunks = _split_assign(self._flat_grads_shards) + else: + self._flat_grads_blocks, self._flat_grads_chunks, self._flat_grads_shards = _flat_split(self._flat_grads) + self._fp16_g_blocks, self._fp16_g_chunks = _packed_split(self._fp16_g) + + self._lazy_init_stage1_done = True + + def _lazy_init_stage2(self): + if self._lazy_init_stage2_done: return + if not self._set_flat_param_view: + # reversing is needed for overlapping allreduce and backprop, but currently not supported for flat param view + self._param_order.order.reverse() + + # re-order model_params, grad_accs, group_properties lists + self._model_params = [self._model_params[i] for i in self._param_order.order] + self._grad_accs = [self._grad_accs[i] for i in self._param_order.order] + self._group_properties = [self._group_properties[i] for i in self._param_order.order] + + def _get_flat_view(param): + if param.is_contiguous(memory_format=torch.channels_last): + K, C, H, W = param.shape + pv = param.as_strided(size=(K,H,W,C), stride=(H*W*C, W*C, C, 1)) + elif param.is_contiguous(memory_format=torch.channels_last_3d): + K, C, D, H, W = param.shape + pv = param.as_strided(size=(K,D,H,W,C), stride=(D*H*W*C, H*W*C, W*C, C, 1)) + else: + pv = param + return pv.view(-1) + + # re-collect grads info (size, offset) after ordering + prev = None + p_offset = 0 + self._grads_info = [] + self._individual_flat_grads = [] + for i, p in enumerate(self._model_params): + p_grads_size = p.numel() + self._grads_info.append({"param_grads_size":p_grads_size, "param_offset":p_offset}) + self._individual_flat_grads.append(self._flat_grads[p_offset:p_offset+p_grads_size].view_as(p)) + # for the first iteration + self._do_overlapped_reduction(i, p) + p_offset += p_grads_size + # Only enforce 128b alignment (64 * fp16) for non-consecutive parameters + # RNN is one example of consecutive parameters: + # (weight_ih, weight_hh, bias_ih, bias_hh) + if prev is not None and (prev.data_ptr() + prev.numel() * prev.element_size() != p.data_ptr()): + p_offset = ((p_offset + 63) // 64) * 64 + prev = p + + self._low_param_i = [0]*self._num_blocks + for block_id in range(self._num_blocks-1,-1,-1): + p_i = len(self._grads_info)-1 + while p_i > 0 and self._grads_info[p_i]["param_offset"] > block_id*self._block_size: + p_i -= 1 + self._low_param_i[block_id] = p_i + #print("self._low_param_i", self._low_param_i) + + # This paragraph does two things: + # 1) Copy model parameters into master buffer + # 2) Create tensor lists for unpacking new parameter tensor after all-gather + self._packed_flat_to_model_params_fp16 = [] + self._packed_flat_to_model_params_fp32 = [] + self._model_params_num = len(self._model_params) + self._contrib_tensor_list = [] + self._contrib_min_param_i, self._contrib_max_param_i = -1, -1 + self._contrib_update_frag_for_norm = [] + self._contrib_model_param_for_norm_fp16 = [] + self._contrib_model_param_for_norm_fp32 = [] + self._contrib_model_param_for_norm_is_fp16 = [] + self._model_param_is_contrib = [] + self._contrib_group_properties = [] + for shard_id in range(self._group_size): + for block_id in range(self._num_blocks): + for chunk_id in range(self._num_chunks): + flat_shard_start = (((block_id * self._num_chunks + chunk_id) * self._group_size) + shard_id) * self._shard_size + flat_shard_end = flat_shard_start + self._shard_size + for param_i, (p, grads_info, group_props) in enumerate(zip(self._model_params, self._grads_info, self._group_properties)): + flat_grad_start = grads_info["param_offset"] + flat_grad_end = flat_grad_start + grads_info["param_grads_size"] + clipped_start = (lambda a,b: a if a > b else b)(flat_grad_start, flat_shard_start) + clipped_end = (lambda a,b: a if a < b else b)(flat_grad_end, flat_shard_end) + if clipped_start < clipped_end: + grad_offset = clipped_start - flat_grad_start + grad_length = clipped_end - clipped_start + shard_offset = clipped_start - flat_shard_start + pf = _get_flat_view(p) + model_param_fragment = pf[grad_offset:grad_offset+grad_length] + new_param_packed_fragment = self._new_params_mega_chunks[shard_id][block_id][chunk_id][shard_offset:shard_offset+grad_length] + if model_param_fragment.dtype == torch.float16: + self._packed_flat_to_model_params_fp16.append( (new_param_packed_fragment, model_param_fragment) ) + else: + self._packed_flat_to_model_params_fp32.append( (new_param_packed_fragment, model_param_fragment) ) + if shard_id == self._rank_in_group: + self._model_param_is_contrib.append(param_i) + # copy model parameters into master buffer + master_param_fragment = self._fp32_p_chunks[block_id][chunk_id][shard_offset:shard_offset+grad_length] + opti_state_m_fragment = self._fp32_m_chunks[block_id][chunk_id][shard_offset:shard_offset+grad_length] + opti_state_v_fragment = self._fp32_v_chunks[block_id][chunk_id][shard_offset:shard_offset+grad_length] + opti_state_u_fragment = self._fp32_u_chunks[block_id][chunk_id][shard_offset:shard_offset+grad_length] + opti_state_g_fragment = self._fp16_g_chunks[block_id][chunk_id][shard_offset:shard_offset+grad_length] + opti_state_p_fragment = self._fp16_p_chunks[block_id][chunk_id][shard_offset:shard_offset+grad_length] + #print("model_param_fragment.size()=%s, new_param_packed_fragment.size()=%s, master_param_fragment.size()=%s" % (str(model_param_fragment.size()), str(new_param_packed_fragment.size()), str(master_param_fragment.size()))) + if not self._resume_from_checkpoint: + master_param_fragment.copy_(model_param_fragment) + self._contrib_group_properties.append(group_props) + self._contrib_tensor_list.append((master_param_fragment, opti_state_m_fragment, opti_state_v_fragment, opti_state_u_fragment, opti_state_g_fragment, opti_state_p_fragment)) # p, m, v, u, g, p_copy + self._contrib_update_frag_for_norm.append(opti_state_u_fragment) + if p.dtype == torch.float16: + self._contrib_model_param_for_norm_fp16.append(p) + else: + self._contrib_model_param_for_norm_fp32.append(p) + self._contrib_model_param_for_norm_is_fp16.append(True if p.dtype == torch.float16 else False) + if self._contrib_min_param_i < 0: self._contrib_min_param_i = param_i + self._contrib_max_param_i = param_i + self._contrib_model_param_for_norm_num = len(self._contrib_model_param_for_norm_is_fp16) + if len(self._contrib_model_param_for_norm_fp16) == 0: self._contrib_model_param_for_norm_fp16 = None + if len(self._contrib_model_param_for_norm_fp32) == 0: self._contrib_model_param_for_norm_fp32 = None + self._contrib_model_param_for_norm_is_fp32 = torch.tensor([not is_fp16 for is_fp16 in self._contrib_model_param_for_norm_is_fp16], dtype=torch.bool, device='cuda') + self._contrib_model_param_for_norm_is_fp16 = torch.tensor([is_fp16 for is_fp16 in self._contrib_model_param_for_norm_is_fp16], dtype=torch.bool, device='cuda') + self._offsets = torch.tensor(self._model_param_is_contrib, dtype=torch.int64, device='cuda') + + p, m, v, u, g, p_copy = list(zip(*self._contrib_tensor_list)) + self._contrib_compute_update_term_tensor_list = [g, p, m, v, u] + self._contrib_update_weights_tensor_list = [u, p, p_copy] + + math_type = self._fp32_u.dtype + decay, bias_correction, beta1, beta2, beta3, epsilon = list(zip(*self._contrib_group_properties)) + self._contrib_beta1 = torch.tensor(beta1, dtype=math_type, device='cuda') + self._contrib_beta2 = torch.tensor(beta2, dtype=math_type, device='cuda') + self._contrib_beta3 = torch.tensor(beta3, dtype=math_type, device='cuda') + self._contrib_bias_correction = torch.tensor(bias_correction, dtype=torch.int, device='cuda') + self._contrib_epsilon = torch.tensor(epsilon, dtype=math_type, device='cuda') + self._contrib_weight_decay = torch.tensor(decay, dtype=math_type, device='cuda') + + self._packed_flat_to_model_params_fp16 = list(zip(*self._packed_flat_to_model_params_fp16)) if len(self._packed_flat_to_model_params_fp16) > 0 else None + self._packed_flat_to_model_params_fp32 = list(zip(*self._packed_flat_to_model_params_fp32)) if len(self._packed_flat_to_model_params_fp32) > 0 else None + + self._lazy_init_stage2_done = True + + self.complete_reductions() + self._first_step = False + + def set_is_accumulation_step(self, is_accumulation_step): + self._is_accumulation_step = is_accumulation_step + + def set_last_step(self, last_step): + self._last_step = last_step + + def _get_flush_block(self): + flush_block = [] + if self._current_block > 0 and self._grads_generated[self._low_param_i[self._current_block-1]]: + num_grads = len(self._grads_generated) + contiguous_idx = num_grads + while contiguous_idx > 0 and self._grads_generated[contiguous_idx-1]: + contiguous_idx -= 1 + + if contiguous_idx < num_grads and self._grads_info[contiguous_idx]["param_offset"] <= (self._current_block-1)*self._block_size: + self._current_block -= 1 + start = self._current_block * self._block_size + end = (self._current_block+1) * self._block_size + flush_block = [start, end] + + return flush_block + + def _full_all_reduce_scale(self, block_id, scale): + works = [None]*self._num_chunks + if self._clip_after_ar: + for chunk_id in range(self._num_chunks): + glob_chunk_id = block_id * self._num_chunks + chunk_id + ar_stream = self._ar_st[glob_chunk_id%self._num_ar_pg] + ar_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(ar_stream): + works[chunk_id] = torch.distributed.all_reduce(self._flat_grads_chunks[block_id][chunk_id],group=self._ar_pg[glob_chunk_id%self._num_ar_pg],async_op=True,op=_make_nccl_premul_sum(scale)) + else: + glob_chunk_id = block_id + ar_stream = self._ar_st[glob_chunk_id%self._num_ar_pg] + ar_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(ar_stream): + works0 = torch.distributed.all_reduce(self._flat_grads_blocks[block_id],group=self._ar_pg[glob_chunk_id%self._num_ar_pg],async_op=True,op=_make_nccl_premul_sum(scale)) + for i in range(self._num_chunks): + works[i]=works0 + self._reductions_works[block_id] = works + + def _full_all_reduce(self, block_id): + works = [None]*self._num_chunks + + for chunk_id in range(self._num_chunks): + glob_chunk_id = block_id * self._num_chunks + chunk_id + ar_stream = self._ar_st[glob_chunk_id%self._num_ar_pg] + ar_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(ar_stream): + works[chunk_id] = torch.distributed.all_reduce(self._flat_grads_chunks[block_id][chunk_id],group=self._ar_pg[glob_chunk_id%self._num_ar_pg],async_op=True) + self._reductions_works[block_id] = works + + def _reduce_scatter_and_all_reduce_scale(self, block_id, scale): + # Reduction within each node + # Changes gradient format from [block * chunk * shard] to [shard * block * chunk] + # The output format is the same as the fp32 master parameters + works = [None]*self._num_chunks + for chunk_id in range(self._num_chunks): + glob_chunk_id = block_id * self._num_chunks + chunk_id + rs_stream = self._rs_st[glob_chunk_id%self._num_rs_pg] + rs_stream.wait_stream(torch.cuda.current_stream()) + rs_stream.wait_stream(self._l2_grad_norm_st) + with torch.cuda.stream(rs_stream): + if self._reduce_scatter_no_copy: + works[chunk_id] = torch.distributed.reduce_scatter( + output=self._fp16_g_chunks[block_id][chunk_id], + input_list=self._flat_grads_shards[block_id][chunk_id], + group=self._rs_pg[glob_chunk_id%self._num_rs_pg], + async_op=True, + no_copy=True, + op=_make_nccl_premul_sum(scale), + ) + else: + works[chunk_id] = torch.distributed.reduce_scatter_tensor( + output=self._fp16_g_chunks[block_id][chunk_id], + input=self._flat_grads_chunks[block_id][chunk_id], + group=self._rs_pg[glob_chunk_id%self._num_rs_pg], + async_op=True, + op=_make_nccl_premul_sum(scale), + ) + + # Reduction across nodes for each rank + if self._num_groups > 1: + for chunk_id in range(self._num_chunks): + glob_chunk_id = block_id * self._num_chunks + chunk_id + ar_stream = self._ar_st[glob_chunk_id%self._num_ar_pg] + with torch.cuda.stream(ar_stream): + works[chunk_id].wait() + works[chunk_id] = torch.distributed.all_reduce(self._fp16_g_chunks[block_id][chunk_id],group=self._ar_pg[glob_chunk_id%self._num_ar_pg],async_op=True) + self._reductions_works[block_id] = works + + def _reduce_scatter_and_all_reduce(self, block_id): + # Reduction within each node + # Changes gradient format from [block * chunk * shard] to [shard * block * chunk] + # The output format is the same as the fp32 master parameters + works = [None]*self._num_chunks + for chunk_id in range(self._num_chunks): + glob_chunk_id = block_id * self._num_chunks + chunk_id + rs_stream = self._rs_st[glob_chunk_id%self._num_rs_pg] + rs_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(rs_stream): + if self._reduce_scatter_no_copy: + works[chunk_id] = torch.distributed.reduce_scatter( + output=self._fp16_g_chunks[block_id][chunk_id], + input_list=self._flat_grads_shards[block_id][chunk_id], + group=self._rs_pg[glob_chunk_id%self._num_rs_pg], + async_op=True, + no_copy=True, + ) + else: + works[chunk_id] = torch.distributed.reduce_scatter_tensor( + output = self._fp16_g_chunks[block_id][chunk_id], + input = self._flat_grads_chunks[block_id][chunk_id], + group = self._rs_pg[glob_chunk_id%self._num_rs_pg], + async_op = True, + ) + + # Reduction across nodes for each rank + if self._num_groups > 1: + for chunk_id in range(self._num_chunks): + glob_chunk_id = block_id * self._num_chunks + chunk_id + ar_stream = self._ar_st[glob_chunk_id%self._num_ar_pg] + with torch.cuda.stream(ar_stream): + works[chunk_id].wait() + works[chunk_id] = torch.distributed.all_reduce(self._fp16_g_chunks[block_id][chunk_id],group=self._ar_pg[glob_chunk_id%self._num_ar_pg],async_op=True) + self._reductions_works[block_id] = works + + def _pipeline_block_reductions(self, block_id): + if self._clip_after_ar: + self._flatten_grad_mt(1.0/self._world_size) + + if self._full_ar: + self._full_all_reduce(block_id) + else: + self._reduce_scatter_and_all_reduce(block_id) + + # Compute L2 grad norm + if block_id == 0: + with torch.cuda.stream(self._l2_grad_norm_st): + for block_id in range(self._num_blocks): + for chunk_id in range(self._num_chunks): + self._reductions_works[block_id][chunk_id].wait() + # Since the packed format is contiguous after reductions, only one norm is needed + l2_grad_norm_sq = torch.empty([1], device='cuda') + if self._full_ar: + # this flattening of lists is to keep multi_tensor_apply function happy, it wants depth=1 for l2 norm computation + flat_list = [item for sublist in self._fp16_g_chunks for item in sublist] + l2_grad_norm_sq = multi_tensor_applier(self.multi_tensor_l2norm, self._overflow_buf, [flat_list], False)[0]**2 + else: + l2_grad_norm_sq = self._fp16_g.norm(dtype=torch.float32, p=2)**2 + torch.distributed.all_reduce(l2_grad_norm_sq, group=self._l2_grad_norm_pg) + self._L2_grad_norm = l2_grad_norm_sq.sqrt() + else: + # Copy model grads to flat grads buffer + self._flatten_grad_mt(1.0) + + # Compute L2 grad norm + self._l2_grad_norm_st.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(self._l2_grad_norm_st): + if not self._fused_norm: + self._L2_grad_norm = self._flat_grads.norm(dtype=torch.float16, p=2).float() + torch.cuda.current_stream().wait_stream(self._l2_grad_norm_st) + + # Apply clipping & pre-reduction scaling on grads + loss_scale = self.global_scale + max_grad_norm = loss_scale*self.defaults['max_grad_norm'] + coeff = max_grad_norm /(1e-6+self.L2_grad_norm) + coeff = (coeff>1) * self._one + (coeff<=1) * coeff + tmp = torch.cat(((self._one), (coeff))) + index = (coeff+1>coeff).int() + scale = tmp.index_select(0, index).half()/self._world_size + if not self._fuse_scale: + self._flat_grads.mul_(scale) + + if self._full_ar: + if self._fuse_scale: + self._full_all_reduce_scale(block_id, scale) + else: + self._full_all_reduce(block_id) + else: + if self._fuse_scale: + self._reduce_scatter_and_all_reduce_scale(block_id, scale) + else: + self._reduce_scatter_and_all_reduce(block_id) + + if block_id == 0: + for block_id in range(self._num_blocks): + for chunk_id in range(self._num_chunks): + self._reductions_works[block_id][chunk_id].wait() + + def __compute_contrib_param_norm(self): + if self._contrib_model_param_for_norm_fp16 is not None and self._contrib_model_param_for_norm_fp32 is not None: + gnorm_fp16 = multi_tensor_applier(self.multi_tensor_l2norm, self._overflow_buf, [self._contrib_model_param_for_norm_fp16], True)[1] + gnorm_fp32 = multi_tensor_applier(self.multi_tensor_l2norm, self._overflow_buf, [self._contrib_model_param_for_norm_fp32], True)[1] + gnorm = torch.empty(size=[self._contrib_model_param_for_norm_num], dtype=torch.bool, device='cuda') + gnorm.masked_scatter_(self._contrib_model_param_for_norm_is_fp16, gnorm_fp16) + gnorm.masked_scatter_(self._contrib_model_param_for_norm_is_fp32, gnorm_fp32) + elif self._contrib_model_param_for_norm_fp16 is not None: + gnorm = multi_tensor_applier(self.multi_tensor_l2norm, self._overflow_buf, [self._contrib_model_param_for_norm_fp16], True)[1] + elif self._contrib_model_param_for_norm_fp32 is not None: + gnorm = multi_tensor_applier(self.multi_tensor_l2norm, self._overflow_buf, [self._contrib_model_param_for_norm_fp32], True)[1] + return gnorm + + def __compute_contrib_update_norm(self): + l2_norm = torch.zeros(size=[self._model_params_num], dtype=torch.float32, device='cuda') + local_contrib_l2_norm = multi_tensor_applier(self.multi_tensor_l2norm, self._overflow_buf, [self._contrib_update_frag_for_norm], True)[1] ** 2 + l2_norm.scatter_(dim=0, index=self._offsets, src=local_contrib_l2_norm) + torch.distributed.all_reduce(l2_norm, group=self._ag_pg[0]) + l2_norm = torch.sqrt(l2_norm) + return l2_norm + + def _pipeline_step(self): + global_scale = self.global_scale + # if clip before ar, set max_grad_norm to 0 + max_grad_norm = self.defaults['max_grad_norm'] * self._clip_after_ar + self._completion_st.wait_stream(self._l2_grad_norm_st) + global_grad_norm = self.L2_grad_norm + + # check global_grad_norm and fill overflow_buf + is_finite = (global_grad_norm + 1 > global_grad_norm).int() + self._overflow_buf = self._one * (is_finite ^ self._one) # toggle between 0 and 1 + + if not self._clip_after_ar: + torch.distributed.all_reduce(is_finite, + op=torch.distributed.ReduceOp.MIN, + group=self._current_process_group) + torch.distributed.all_reduce(self._overflow_buf, + op=torch.distributed.ReduceOp.MAX, + group=self._current_process_group) + + # increment step counter if no overflow + self._step += is_finite + self._completion_st.wait_stream(torch.cuda.current_stream()) + self._completion_st.wait_stream(self._l2_grad_norm_st) + + # Call step kernel once per step + # Call all-gather once per step + with torch.cuda.stream(self._completion_st): + for block_id in range(self._num_blocks): + for chunk_id in range(self._num_chunks): + self._reductions_works[block_id][chunk_id].wait() + param_norm = self.__compute_contrib_param_norm() + multi_tensor_applier(self.multi_tensor_lamb_compute_update_term, + self._overflow_buf, + self._contrib_compute_update_term_tensor_list, # g, p, m, v, u + self._contrib_beta1, + self._contrib_beta2, + self._contrib_beta3, + self._contrib_bias_correction, + self._step, + self._contrib_epsilon, + self._adam_w_mode, + self._contrib_weight_decay, + global_scale, + global_grad_norm, + max_grad_norm) + upd_norm = self.__compute_contrib_update_norm() + multi_tensor_applier(self.multi_tensor_lamb_update_weights, + self._overflow_buf, + self._contrib_update_weights_tensor_list, # u, p, p_copy + param_norm, + upd_norm, + self._offsets, + self._lr, + self._contrib_weight_decay, + global_grad_norm, + self._use_nvlamb) + if not self._skip_ag: + # allgather chunking is currently not supported for clip after allreduce + if not self._clip_after_ar: + for block in range(self._num_blocks): + for chunk in range(self._num_chunks): + if self._all_gather_no_copy: + torch.distributed.all_gather( + tensor_list = self._new_params2_shards[block][chunk], + tensor = self._fp16_p_chunks[block][chunk], + group = self._ag_pg[0], + no_copy = True, + ) + else: + torch.distributed.all_gather_into_tensor( + output_tensor = self._new_params2_blocks[block], + input_tensor = self._fp16_p_chunks[block][chunk], + group = self._ag_pg[0], + ) + else: + if self._all_gather_no_copy: + torch.distributed.all_gather( + tensor_list = self._new_params_mega_shards, + tensor = self._fp16_p, + group = self._ag_pg[0], + no_copy = True, + ) + else: + torch.distributed.all_gather_into_tensor( + output_tensor = self._new_params, + input_tensor = self._fp16_p, + group = self._ag_pg[0], + ) + + def _flatten_grad_mt(self, scale): + if len(self._grads_fp16) > 0: + self._overflow_buf.zero_() + if not self._fused_norm: + multi_tensor_applier( + amp_C.multi_tensor_scale, + self._overflow_buf, + list(zip(*self._grads_fp16)), + scale) + else: + self._L2_grad_norm=multi_tensor_applier( + amp_C.multi_tensor_l2norm_scale, + self._overflow_buf, + list(zip(*self._grads_fp16)), + scale, False)[0].float() + + self._grads_fp16 = [] + if len(self._grads_fp32) > 0: + self._overflow_buf.zero_() + if not self._fused_norm: + multi_tensor_applier( + amp_C.multi_tensor_scale, + self._overflow_buf, + list(zip(*self._grads_fp32)), + scale) + else: + self._L2_grad_norm=multi_tensor_applier( + amp_C.multi_tensor_l2norm_scale, + self._overflow_buf, + list(zip(*self._grads_fp32)), + scale, False)[0].float() + self._grads_fp32 = [] + + def _do_overlapped_reduction(self, param_i, param): + if not self._is_accumulation_step: + # handle overlapped reductions + if param.dtype == torch.float16: + self._grads_fp16.append( (param.grad, self._individual_flat_grads[param_i]) ) + else: + self._grads_fp32.append( (param.grad, self._individual_flat_grads[param_i]) ) + self._grads_generated[param_i]=True + if not self._first_step and not self._last_step: + if self._overlap_reductions: + flush_block = self._get_flush_block() + while flush_block: + block_id = flush_block[0] // self._block_size + self._pipeline_block_reductions(block_id) + flush_block = self._get_flush_block() + + def set_global_scale(self, global_scale): + """Set global scale. + """ + self._global_scale = global_scale + + @property + def global_scale(self): + return self._global_scale + + @property + def L2_grad_norm(self): + torch.cuda.current_stream().wait_stream(self._l2_grad_norm_st) + return self._L2_grad_norm + + def complete_reductions(self): + """Complete reductions if full pipeline is not selected or overlap is not allowed. + """ + if self._last_step: + # zero out gradients that have not been completed yet + for param_i, grad_generated in enumerate(self._grads_generated): + if not grad_generated: + grad_info = self._grads_info[param_i] + param_offset = grad_info["param_offset"] + param_size = grad_info["param_grads_size"] + self._flat_grads[param_offset:param_offset+param_size].zero_() + self._grads_generated[param_i] = True + + if self._first_step or self._last_step or not self._overlap_reductions: + # nothing done so far, run full pipeline after reductions + for block_id in range(self._num_blocks-1,-1,-1): + self._pipeline_block_reductions(block_id) + + torch.cuda.current_stream().wait_stream(self._l2_grad_norm_st) + + self._current_block = self._num_blocks + self._grads_generated = [False]*len(self._grads_info) + + def step(self, closure=None, grad_scaler=None): + loss = None + if closure is not None: + loss = closure() + + self._pipeline_step() + + if grad_scaler is not None: + found_inf = self._overflow_buf.float() + optimizer_state = grad_scaler._per_optimizer_states[id(self)] + current_device = torch.device('cuda', torch.cuda.current_device()) + optimizer_state["found_inf_per_device"][current_device] = found_inf + + self._completion_st.wait_stream(torch.cuda.current_stream()) + if not self._set_flat_param_view: + with torch.cuda.stream(self._completion_st): + # Copy self._new_params to model params + with torch.no_grad(): + if self._packed_flat_to_model_params_fp16 is not None: + multi_tensor_applier( + fused_adam_cuda.maybe_cast_mt, + self._overflow_buf, + self._packed_flat_to_model_params_fp16) + if self._packed_flat_to_model_params_fp32 is not None: + multi_tensor_applier( + fused_adam_cuda.maybe_cast_mt, + self._overflow_buf, + self._packed_flat_to_model_params_fp32) + + torch.cuda.current_stream().wait_stream(self._completion_st) + + self._reductions_works = [None]*self._num_blocks + self._allgather_works = [None]*self._num_blocks + + return loss + + def state_dict(self): + """ + Returns a dict containing the current state of this :class:`DistributedFusedAdam` instance. + Example:: + checkpoint = {} + checkpoint['model'] = model.state_dict() + checkpoint['optimizer'] = optimizer.state_dict() + torch.save(checkpoint, "saved.pth") + """ + # save step, master weights and first/second moments + state_dict = {} + state_dict['step'] = self._step + state_dict['fp32_p'] = self._fp32_p + state_dict['fp32_m'] = self._fp32_m + state_dict['fp32_v'] = self._fp32_v + return state_dict + + def load_state_dict(self, state_dict): + """ + Loads a state_dict created by an earlier call to state_dict(). + If an DistributedFusedAdam instance was constructed from some ``init_optimizer``, + whose parameters in turn came from ``model``, it is expected that the user + will call ``model.load_state_dict()`` before + ``optimizer.load_state_dict()`` is called. + Example:: + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + checkpoint = torch.load("saved.pth") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + """ + # restore step, master weights and first/second moments + self._step = state_dict['step'] + self._fp32_p = state_dict['fp32_p'].to(device="cuda") + self._fp32_m = state_dict['fp32_m'].to(device="cuda") + self._fp32_v = state_dict['fp32_v'].to(device="cuda") + self._resume_from_checkpoint = True diff --git a/apex/apex/contrib/optimizers/fp16_optimizer.py b/apex/apex/contrib/optimizers/fp16_optimizer.py new file mode 100755 index 00000000..0cbb63b8 --- /dev/null +++ b/apex/apex/contrib/optimizers/fp16_optimizer.py @@ -0,0 +1,243 @@ +import torch +from apex.multi_tensor_apply import multi_tensor_applier + +class FP16_Optimizer(object): + """ + :class:`FP16_Optimizer` A cutdown version of apex.fp16_utils.FP16_Optimizer. + Designed only to wrap apex.contrib.optimizers.FusedAdam, FusedSGD. + Refer to apex.fp16_utils documents for more information. + Example:: + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = apex.contrib.optimizers.FusedSGD(model.parameters()) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + # loss.backward() becomes: + optimizer.backward(loss) + ... + Example with dynamic loss scaling:: + ... + optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) + # optional arg to control dynamic loss scaling behavior + # dynamic_loss_args={'scale_window' : 500}) + # Usually, dynamic_loss_args is not necessary. + """ + + def __init__(self, + init_optimizer, + static_loss_scale=1.0, + dynamic_loss_scale=False, + dynamic_loss_args=None, + verbose=True): + + print("\nThis fp16_optimizer is designed to only work with apex.contrib.optimizers.*") + print("To update, use updated optimizers with AMP.") + # The fused optimizer does all the work. We need this layer for two reason: + # 1. maintain same user API from apex.fp16_utils + # 2. keep common stuff here in case we need to add new fused optimizer later + + if not torch.cuda.is_available: + raise SystemError("Cannot use fp16 without CUDA.") + self.optimizer = init_optimizer + + self.fp16_groups = [] # model params + self.fp32_groups = [] # master weights + + # iterate over param_groups + for param_group in self.optimizer.param_groups: + fp16_group = [] + fp32_group = [] + for p in param_group['params']: + fp16_group.append(p) + fp32_group.append(p.clone().float().detach()) + self.fp16_groups.append(fp16_group) + self.fp32_groups.append(fp32_group) + param_group['params'] = fp32_group + + if multi_tensor_applier.available: + import amp_C + self.overflow_buf = torch.cuda.IntTensor([0]) + self.multi_tensor_l2norm=amp_C.multi_tensor_l2norm + else: + raise RuntimeError('FP16_Optimizer requires cuda extensions') + + # we may have a way of fusing dynamic scale. Do not support for now + if dynamic_loss_scale: + if dynamic_loss_args is not None: + raise SystemError("Do not support dynamic loss scale args for now.") + self.dynamic_loss_scale = True + self.cur_scale = 2**16 + self.cur_iter = 0 + self.last_overflow_iter = -1 + self.scale_factor = 2 + self.scale_window = 1000 + else: + self.dynamic_loss_scale = False + self.cur_iter = 0 + self.cur_scale = static_loss_scale + self.verbose = verbose + + def zero_grad(self, set_grads_to_None=True): + """ + Zero FP16 parameter grads. + """ + # FP32 grad should never exist. + # For speed, set model fp16 grad to None by default + for group in self.fp16_groups: + for p in group: + if set_grads_to_None: + p.grad = None + else: + if p.grad is not None: + p.grad.detach_() + p.grad.zero_() + + def step(self, closure=None): + """ + Not supporting closure. + """ + fp16_grads = [] + norm_groups = [] + skip = False + + for group in self.fp16_groups: + fp16_grad = [] + for i, p in enumerate(group): + fp16_grad.append(p.grad) + fp16_grads.append(fp16_grad) + + # nan check + self.overflow_buf.zero_() + for fp16_grad in fp16_grads: + if len(fp16_grad) > 0: + norm, norm_per_tensor = multi_tensor_applier(self.multi_tensor_l2norm, + self.overflow_buf, + [fp16_grad], True) + norm_groups.append(norm) + if self.overflow_buf.item() != 0: + skip = True + + if skip: + self._update_scale(skip) + return + + # norm is in fact norm*cur_scale + self.optimizer.step(grads=fp16_grads, + output_params=self.fp16_groups, + scale=self.cur_scale, + grad_norms=norm_groups) + + self._update_scale(False) + return + + def backward(self, loss): + """ + :attr:`backward` performs the following steps: + 1. fp32_loss = loss.float() + 2. scaled_loss = fp32_loss*loss_scale + 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's fp16 leaves + """ + scaled_loss = (loss.float()) * self.cur_scale + scaled_loss.backward() + + def _update_scale(self, skip): + if self.dynamic_loss_scale: + if skip: + if self.verbose: + print("\nGrad overflow on iteration", self.cur_iter) + print("Using dynamic loss scale of", self.cur_scale) + self.cur_scale = max(self.cur_scale/self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.cur_scale *= self.scale_factor + else: + if skip: + print("\nGrad overflow on iteration", self.cur_iter) + print("Using static loss scale of", self.cur_scale) + self.cur_iter +=1 + return + + # Promote state so it can be retrieved or set via "fp16_optimizer_instance.state" + def _get_state(self): + return self.optimizer.state + + def _set_state(self, value): + self.optimizer.state = value + + state = property(_get_state, _set_state) + + # Promote param_groups so it can be retrieved or set via "fp16_optimizer_instance.param_groups" + # (for example, to adjust the learning rate) + def _get_param_groups(self): + return self.optimizer.param_groups + + def _set_param_groups(self, value): + self.optimizer.param_groups = value + + param_groups = property(_get_param_groups, _set_param_groups) + + def state_dict(self): + """ + Returns a dict containing the current state of this :class:`FP16_Optimizer` instance. + This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict + of the contained Pytorch optimizer. + Example:: + checkpoint = {} + checkpoint['model'] = model.state_dict() + checkpoint['optimizer'] = optimizer.state_dict() + torch.save(checkpoint, "saved.pth") + """ + state_dict = {} + state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale + state_dict['cur_scale'] = self.cur_scale + state_dict['cur_iter'] = self.cur_iter + if state_dict['dynamic_loss_scale']: + state_dict['last_overflow_iter'] = self.last_overflow_iter + state_dict['scale_factor'] = self.scale_factor + state_dict['scale_window'] = self.scale_window + state_dict['optimizer_state_dict'] = self.optimizer.state_dict() + state_dict['fp32_groups'] = self.fp32_groups + return state_dict + + def load_state_dict(self, state_dict): + """ + Loads a state_dict created by an earlier call to state_dict(). + If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``, + whose parameters in turn came from ``model``, it is expected that the user + will call ``model.load_state_dict()`` before + ``fp16_optimizer_instance.load_state_dict()`` is called. + Example:: + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + checkpoint = torch.load("saved.pth") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + """ + # I think it should actually be ok to reload the optimizer before the model. + self.dynamic_loss_scale = state_dict['dynamic_loss_scale'] + self.cur_scale = state_dict['cur_scale'] + self.cur_iter = state_dict['cur_iter'] + if state_dict['dynamic_loss_scale']: + self.last_overflow_iter = state_dict['last_overflow_iter'] + self.scale_factor = state_dict['scale_factor'] + self.scale_window = state_dict['scale_window'] + self.optimizer.load_state_dict(state_dict['optimizer_state_dict']) + # At this point, the optimizer's references to the model's fp32 parameters are up to date. + # The optimizer's hyperparameters and internal buffers are also up to date. + # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still + # out of date. There are two options. + # 1: Refresh the master params from the model's fp16 params. + # This requires less storage but incurs precision loss. + # 2: Save and restore the fp32 master copies separately. + # We choose option 2. + # + # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device + # of their associated parameters, because it's possible those buffers might not exist yet in + # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been + # constructed in the same way as the one whose state_dict we are loading, the same master params + # are guaranteed to exist, so we can just copy_() from the saved master params. + for current, saved in zip(self.fp32_groups, state_dict['fp32_groups']): + for _current, _saved in zip(current, saved): + _current.data.copy_(_saved.data) diff --git a/apex/apex/contrib/optimizers/fused_adam.py b/apex/apex/contrib/optimizers/fused_adam.py new file mode 100644 index 00000000..a823e7be --- /dev/null +++ b/apex/apex/contrib/optimizers/fused_adam.py @@ -0,0 +1,206 @@ +import types +import torch +import importlib +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedAdam(torch.optim.Optimizer): + + """Implements Adam algorithm. Currently GPU-only. Requires Apex to be installed via + ``python setup.py install --cuda_ext --cpp_ext``. + + It has been proposed in `Adam: A Method for Stochastic Optimization`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) NOT SUPPORTED in FusedAdam! + eps_inside_sqrt (boolean, optional): in the 'update parameters' step, + adds eps to the bias-corrected second moment estimate before + evaluating square root instead of adding it to the square root of + second moment estimate as in the original paper. (default: False) + use_mt (boolean, optional): use multi tensor apply for lower launch + latency. (default: False) + + .. _Adam - A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__(self, params, + lr=1e-3, bias_correction = True, + betas=(0.9, 0.999), eps=1e-8, eps_inside_sqrt = False, + weight_decay=0., max_grad_norm=0., amsgrad=False, use_mt=False, + amp_scale_adjustment=1.0): + global fused_adam_cuda + fused_adam_cuda = importlib.import_module("fused_adam_cuda") + + self._use_multi_tensor = False + if use_mt: + if not multi_tensor_applier.available: + print("Warning: multi_tensor_applier is unavailable") + else: + self._use_multi_tensor = True + self._overflow_buf = torch.cuda.IntTensor([0]) + + self._amp_scale_adjustment = amp_scale_adjustment + + if amsgrad: + raise RuntimeError('FusedAdam does not support the AMSGrad variant.') + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay, + max_grad_norm=max_grad_norm) + super(FusedAdam, self).__init__(params, defaults) + self.eps_mode = 0 if eps_inside_sqrt else 1 + + def step(self, closure=None, grads=None, output_params=None, scale=1., grad_norms=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + grads (list of tensors, optional): weight gradient to use for the + optimizer update. If gradients have type torch.half, parameters + are expected to be in type torch.float. (default: None) + output params (list of tensors, optional): A reduced precision copy + of the updated weights written out in addition to the regular + updated weights. Have to be of same type as gradients. (default: None) + scale (float, optional): factor to divide gradient tensor values + by before applying to weights. (default: 1) + """ + loss = None + if closure is not None: + loss = closure() + + if hasattr(self, "_amp_stash"): + grads = self._amp_stash.grads + output_params = self._amp_stash.output_params + scale = self._amp_stash.scale*self._amp_scale_adjustment + grad_norms = self._amp_stash.grad_norms + + if grads is None: + grads_group = [None]*len(self.param_groups) + # backward compatibility + # assuming a list/generator of parameter means single group + elif isinstance(grads, types.GeneratorType): + grads_group = [grads] + elif type(grads[0])!=list: + grads_group = [grads] + else: + grads_group = grads + + if output_params is None: + output_params_group = [None]*len(self.param_groups) + elif isinstance(output_params, types.GeneratorType): + output_params_group = [output_params] + elif type(output_params[0])!=list: + output_params_group = [output_params] + else: + output_params_group = output_params + + if grad_norms is None: + grad_norms = [None]*len(self.param_groups) + + for group, grads_this_group, output_params_this_group, grad_norm in zip(self.param_groups, grads_group, output_params_group, grad_norms): + if grads_this_group is None: + grads_this_group = [None]*len(group['params']) + if output_params_this_group is None: + output_params_this_group = [None]*len(group['params']) + + # compute combined scale factor for this group + combined_scale = scale + if group['max_grad_norm'] > 0: + # norm is in fact norm*scale + clip = ((grad_norm / scale) + 1e-6) / group['max_grad_norm'] + if clip > 1: + combined_scale = clip * scale + + bias_correction = 1 if group['bias_correction'] else 0 + + if self._use_multi_tensor: + if output_params: + tensorlists = [[],[],[],[],[]] + else: + tensorlists = [[],[],[],[]] + tensordevice = None + + for p, grad, output_param in zip(group['params'], grads_this_group, output_params_this_group): + #note: p.grad should not ever be set for correct operation of mixed precision optimizer that sometimes sends None gradients + if p.grad is None and grad is None: + continue + if grad is None: + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('FusedAdam does not support sparse gradients, please consider SparseAdam instead') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + out_p = torch.tensor([], dtype = torch.float) if output_param is None else output_param + if self._use_multi_tensor: + pl = [p.data, exp_avg, exp_avg_sq, grad] + if output_param is not None: + pl.append(out_p) + + for tl, t in zip(tensorlists, pl): + tl.append(t) + + if tensordevice is None: + tensordevice = p.device + elif tensordevice != p.device: + raise RuntimeError('FusedAdam does not support use_mt with tensors on multiple device') + + else: + with torch.cuda.device(p.device): + fused_adam_cuda.adam(p.data, + out_p, + exp_avg, + exp_avg_sq, + grad, + group['lr'], + beta1, + beta2, + group['eps'], + combined_scale, + state['step'], + self.eps_mode, + bias_correction, + group['weight_decay']) + + if self._use_multi_tensor: + with torch.cuda.device(tensordevice): + multi_tensor_applier( + fused_adam_cuda.adam_mt, + self._overflow_buf, + tensorlists, + group['lr'], + beta1, + beta2, + group['eps'], + combined_scale, + state['step'], + self.eps_mode, + bias_correction, + group['weight_decay']) + + return loss diff --git a/apex/apex/contrib/optimizers/fused_lamb.py b/apex/apex/contrib/optimizers/fused_lamb.py new file mode 100644 index 00000000..81d86822 --- /dev/null +++ b/apex/apex/contrib/optimizers/fused_lamb.py @@ -0,0 +1,208 @@ +import torch +import importlib +import math +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedLAMB(torch.optim.Optimizer): + + """Implements LAMB algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" --global-option="--deprecated_fused_lamb" ./``. + + This version of fused LAMB implements 2 fusions. + + * Fusion of the LAMB update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.contrib.optimizers.FusedLAMB`'s usage is identical to any ordinary Pytorch optimizer:: + + opt = apex.contrib.optimizers.FusedLAMB(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedLAMB` may be used with or without Amp. If you wish to use :class:`FusedLAMB` with Amp, + you may choose any ``opt_level``:: + + opt = apex.optimizers.FusedLAMB(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + + In general, ``opt_level="O1"`` is recommended. + + LAMB was proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its norm. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + NOT SUPPORTED now! (default: False) + adam_w_mode (boolean, optional): Apply L2 regularization or weight decay + True for decoupled weight decay(also known as AdamW) (default: True) + grad_averaging (bool, optional): whether apply (1-beta2) to grad when + calculating running averages of gradient. (default: True) + set_grad_none (bool, optional): whether set grad to None when zero_grad() + method is called. (default: True) + max_grad_norm (float, optional): value used to clip global grad norm + (default: 1.0) + + .. _Large Batch Optimization for Deep Learning - Training BERT in 76 minutes: + https://arxiv.org/abs/1904.00962 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__(self, params, lr=1e-3, bias_correction=True, + betas=(0.9, 0.999), eps=1e-6, weight_decay=0.01, + amsgrad=False, adam_w_mode=True, + grad_averaging=True, set_grad_none=True, + max_grad_norm=1.0): + if amsgrad: + raise RuntimeError('FusedLAMB does not support the AMSGrad variant.') + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay, + grad_averaging=grad_averaging, + max_grad_norm=max_grad_norm) + super(FusedLAMB, self).__init__(params, defaults) + if multi_tensor_applier.available: + import amp_C + self.multi_tensor_l2norm=amp_C.multi_tensor_l2norm + self._dummy_overflow_buf = torch.cuda.IntTensor([0]) + fused_lamb_cuda = importlib.import_module("fused_lamb_cuda") + self.multi_tensor_lamb = fused_lamb_cuda.lamb + else: + raise RuntimeError('apex.contrib.optimizers.FusedLAMB requires cuda extensions') + + self.adam_w_mode = 1 if adam_w_mode else 0 + self.set_grad_none = set_grad_none + + def zero_grad(self): + if self.set_grad_none: + for group in self.param_groups: + for p in group['params']: + p.grad = None + else: + super(FusedLAMB, self).zero_grad() + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + # create separate grad lists for fp32 and fp16 params + g_all_32, g_all_16 = [], [] + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + if p.dtype == torch.float32: + g_all_32.append(p.grad.data) + elif p.dtype == torch.float16: + g_all_16.append(p.grad.data) + else: + raise RuntimeError('FusedLAMB only support fp16 and fp32.') + + g_norm_32, g_norm_16 = 0.0, 0.0 + # compute grad norm for two lists + if len(g_all_32) > 0: + g_norm_32 = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [g_all_32], False)[0].item() + if len(g_all_16) > 0: + g_norm_16 = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [g_all_16], False)[0].item() + + # blend two grad norms to get global grad norm + global_grad_norm = math.sqrt(g_norm_32 * g_norm_32 + g_norm_16 * g_norm_16) + max_grad_norm = self.defaults['max_grad_norm'] + + for group in self.param_groups: + bias_correction = 1 if group['bias_correction'] else 0 + beta1, beta2 = group['betas'] + grad_averaging = 1 if group['grad_averaging'] else 0 + + # assume same step across group now to simplify things + # per parameter step can be easily support by making it tensor, or pass list into kernel + if 'step' in group: + group['step'] += 1 + else: + group['step'] = 1 + + # create lists for multi-tensor apply + g_16, p_16, m_16, v_16 = [], [], [], [] + g_32, p_32, m_32, v_32 = [], [], [], [] + + for p in group['params']: + if p.grad is None: + continue + if p.grad.data.is_sparse: + raise RuntimeError('FusedLAMB does not support sparse gradients, please consider SparseAdam instead') + + state = self.state[p] + # State initialization + if len(state) == 0: + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + if p.dtype == torch.float16: + g_16.append(p.grad.data) + p_16.append(p.data) + m_16.append(state['exp_avg']) + v_16.append(state['exp_avg_sq']) + elif p.dtype == torch.float32: + g_32.append(p.grad.data) + p_32.append(p.data) + m_32.append(state['exp_avg']) + v_32.append(state['exp_avg_sq']) + else: + raise RuntimeError('FusedLAMB only support fp16 and fp32.') + + if(len(g_16) > 0): + multi_tensor_applier(self.multi_tensor_lamb, + self._dummy_overflow_buf, + [g_16, p_16, m_16, v_16], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.adam_w_mode, + global_grad_norm, + max_grad_norm) + if(len(g_32) > 0): + multi_tensor_applier(self.multi_tensor_lamb, + self._dummy_overflow_buf, + [g_32, p_32, m_32, v_32], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.adam_w_mode, + global_grad_norm, + max_grad_norm) + + return loss diff --git a/apex/apex/contrib/optimizers/fused_sgd.py b/apex/apex/contrib/optimizers/fused_sgd.py new file mode 100644 index 00000000..83587c6a --- /dev/null +++ b/apex/apex/contrib/optimizers/fused_sgd.py @@ -0,0 +1,211 @@ +import types +import torch +from torch.optim.optimizer import Optimizer, required + +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedSGD(Optimizer): + r"""Implements stochastic gradient descent (optionally with momentum). + + This version of fused SGD implements 2 fusions. + * Fusion of the SGD update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.contrib.optimizers.FusedSGD` should be used without AMP. + + :class:`apex.contrib.optimizers.FusedSGD` only works in the case where all parameters require grad. + + Nesterov momentum is based on the formula from + `On the importance of initialization and momentum in deep learning`__. + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float): learning rate + momentum (float, optional): momentum factor (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + dampening (float, optional): dampening for momentum (default: 0) + nesterov (bool, optional): enables Nesterov momentum (default: False) + + Example: + model = ... + model.half() + optimizer = apex.contrib.optimizers.FusedSGD(model.parameters()) + # wrap with FP16_Optimizer + optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True) + optimizer.zero_grad() + ... + optimizer.backward(loss) + optmizer.step() + + __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + + .. note:: + The implementation of SGD with Momentum/Nesterov subtly differs from + Sutskever et. al. and implementations in some other frameworks. + + Considering the specific case of Momentum, the update can be written as + + .. math:: + v = \rho * v + g \\ + p = p - lr * v + + where p, g, v and :math:`\rho` denote the parameters, gradient, + velocity, and momentum respectively. + + This is in contrast to Sutskever et. al. and + other frameworks which employ an update of the form + + .. math:: + v = \rho * v + lr * g \\ + p = p - v + + The Nesterov version is analogously modified. + """ + + def __init__(self, params, lr=required, momentum=0, dampening=0, + weight_decay=0, nesterov=False, + wd_after_momentum=False, + materialize_master_grads=True): + if lr is not required and lr < 0.0: + raise ValueError("Invalid learning rate: {}".format(lr)) + if momentum < 0.0: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if weight_decay < 0.0: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + + defaults = dict(lr=lr, momentum=momentum, dampening=dampening, + weight_decay=weight_decay, nesterov=nesterov) + if nesterov and (momentum <= 0 or dampening != 0): + raise ValueError("Nesterov momentum requires a momentum and zero dampening") + super(FusedSGD, self).__init__(params, defaults) + + self.wd_after_momentum = wd_after_momentum + + if multi_tensor_applier.available: + import amp_C + # Skip buffer + self._dummy_overflow_buf = torch.cuda.IntTensor([0]) + self.multi_tensor_sgd = amp_C.multi_tensor_sgd + else: + raise RuntimeError('apex.contrib.optimizers.FusedSGD requires cuda extensions') + + def __setstate__(self, state): + super(FusedSGD, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('nesterov', False) + + def get_momentums(self, params): + momentums = [] + first_run = True + for p in params: + param_state = self.state[p] + # torch.optim.SGD initializes momentum in the main loop, we have + # to do it here, and track whether or not we've done so, so that + # momentum application can be skipped in the main kernel. + if 'momentum_buffer' not in param_state: + first_run = True + buf = param_state['momentum_buffer'] = torch.zeros_like(p.data) + momentums.append(buf) + else: + first_run = False + momentums.append(param_state['momentum_buffer']) + return momentums, first_run + + def step(self, closure=None, grads=None, output_params=None, scale=1., grad_norms=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + grads (list of tensors, optional): weight gradient to use for the + optimizer update. If gradients have type torch.half, parameters + are expected to be in type torch.float. (default: None) + output_params (list of tensors, optional): A reduced precision copy + of the updated weights written out in addition to the regular + updated weights. Have to be of same type as gradients. (default: None) + scale (float, optional): factor to divide gradient tensor values + by before applying to weights. (default: 1) + """ + if hasattr(self, "_amp_stash"): + raise RuntimeError('apex.contrib.optimizers.FusedSGD should not be used with AMP.') + + loss = None + if closure is not None: + loss = closure() + + if grads is None: + raise RuntimeError('apex.contrib.optimizers.FusedSGD must be wrapped \ + with apex.contrib.optimizers.FP16_Optimizer \ + which provides grads.') + # backward compatibility + # assuming a list/generator of parameter means single group + elif isinstance(grads, types.GeneratorType): + grads_group = [grads] + elif type(grads[0])!=list: + grads_group = [grads] + else: + grads_group = grads + + if output_params is None: + raise RuntimeError('apex.contrib.optimizers.FusedSGD must be wrapped \ + with apex.contrib.optimizers.FP16_Optimizer \ + which provides output_params.') + elif isinstance(output_params, types.GeneratorType): + output_params_group = [output_params] + elif type(output_params[0])!=list: + output_params_group = [output_params] + else: + output_params_group = output_params + + for group, grads_this_group, output_params_this_group in zip(self.param_groups, + grads_group, + output_params_group): + if grads_this_group is None or output_params_this_group is None: + raise RuntimeError('apex.contrib.optimizers.FusedSGD only works \ + when all parameters require grad.') + + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + lr = group['lr'] + + first_runs = [True, True] + + # output_params_this_group: original weights (either fp16 or fp32) + # group['params']: master weights (fp32) + + # grad_type, param_to_update_type, momentum_type, requires_fp16_model_copy + # fp32, fp32, fp32, No + fp32_grads = [g for (p, g) in zip(output_params_this_group, grads_this_group) if p.dtype == torch.float32] + fp32_params = [p2 for (p1, p2) in zip(output_params_this_group, group['params']) if p1.dtype == torch.float32] + fp32_momentums, first_runs[1] = self.get_momentums(fp32_params) + fp32_set = [fp32_grads, fp32_params, fp32_momentums] + + # fp16, fp32, fp32, Yes + fp16_grads = [g for (p, g) in zip(output_params_this_group, grads_this_group) if p.dtype == torch.float16] + fp32_from_fp16_params = [p2 for (p1, p2) in zip(output_params_this_group, group['params']) if p1.dtype == torch.float16] + fp32_from_fp16_momentums, first_runs[0] = self.get_momentums(fp32_from_fp16_params) + fp16_params = [p1 for (p1, p2) in zip(output_params_this_group, group['params']) if p1.dtype == torch.float16] + fp16_set = [fp16_grads, fp32_from_fp16_params, fp32_from_fp16_momentums, fp16_params] + + launch_sets = [fp16_set, fp32_set] + + for launch_set, first_run in zip(launch_sets, first_runs): + assert len(launch_set[0]) == len(launch_set[1]) + assert len(launch_set[0]) == len(launch_set[2]) + if len(launch_set[0]) > 0: + multi_tensor_applier( + self.multi_tensor_sgd, + self._dummy_overflow_buf, + launch_set, + weight_decay, + momentum, + dampening, + lr, + nesterov, + first_run, + self.wd_after_momentum, + 1.0/scale) + + return loss diff --git a/apex/apex/contrib/peer_memory/__init__.py b/apex/apex/contrib/peer_memory/__init__.py new file mode 100644 index 00000000..8d6fa548 --- /dev/null +++ b/apex/apex/contrib/peer_memory/__init__.py @@ -0,0 +1,3 @@ +from .peer_memory import PeerMemoryPool +from .peer_halo_exchanger_1d import PeerHaloExchanger1d + diff --git a/apex/apex/contrib/peer_memory/peer_halo_exchanger_1d.py b/apex/apex/contrib/peer_memory/peer_halo_exchanger_1d.py new file mode 100644 index 00000000..943d886d --- /dev/null +++ b/apex/apex/contrib/peer_memory/peer_halo_exchanger_1d.py @@ -0,0 +1,74 @@ +import torch +from apex.contrib.peer_memory import PeerMemoryPool +import peer_memory_cuda as pm + +class PeerHaloExchanger1d: + def __init__(self, ranks, rank_in_group, peer_pool, half_halo): + self.peer_group_size = len(ranks) + self.ranks = ranks + self.peer_rank = rank_in_group + self.low_neighbor = (self.peer_rank + self.peer_group_size - 1) % self.peer_group_size + self.high_neighbor = (self.peer_rank + 1) % self.peer_group_size + self.low_zero = True if self.peer_rank == 0 else False + self.high_zero = True if self.peer_rank == self.peer_group_size - 1 else False + + self.peer_pool = peer_pool + self.half_halo = half_halo + + def _allocate_peer_tensor(self, halo): + + # Compute size in bytes + # Note: Pad buffer so each CUDA block gets required buffer size + size = 4 * halo.numel() * halo.element_size() + size_per_block = 128 * 2 * 16 # 128 threads each require two 128b buffers + size = (size + size_per_block - 1) // size_per_block * size_per_block + + # Construct dtype peer buffer with desired size + shape = [1, 1, 1, size // halo.element_size()] + return self.peer_pool.allocate_peer_tensors(shape, halo.dtype, False, True) + + def __call__(self, y, H_split=True, explicit_nhwc=False, numSM=0, diagnostics=False): + channels_last = y.is_contiguous(memory_format=torch.channels_last) and not explicit_nhwc + if H_split: + if explicit_nhwc: + _, Hs, _, _ = list(y.shape) + H = Hs - 2*self.half_halo + low_out_halo = y[:,self.half_halo:2*self.half_halo,:,:] + low_tx = self._allocate_peer_tensor(low_out_halo) + low_inp_halo = y[:,:self.half_halo,:,:] + high_out_halo = y[:,H:H+self.half_halo,:,:] + high_tx = self._allocate_peer_tensor(high_out_halo) + high_inp_halo = y[:,H+self.half_halo:H+2*self.half_halo,:,:] + else: + _, _, Hs, _ = list(y.shape) + H = Hs - 2*self.half_halo + low_out_halo = y[:,:,self.half_halo:2*self.half_halo,:] + low_tx = self._allocate_peer_tensor(low_out_halo) + low_inp_halo = y[:,:,:self.half_halo,:] + high_out_halo = y[:,:,H:H+self.half_halo,:] + high_tx = self._allocate_peer_tensor(high_out_halo) + high_inp_halo = y[:,:,H+self.half_halo:H+2*self.half_halo,:] + else: + if explicit_nhwc: + _, _, Ws, _ = list(y.shape) + W = Ws - 2*self.half_halo + low_out_halo = y[:,:,self.half_halo:2*self.half_halo,:] + low_tx = self._allocate_peer_tensor(low_out_halo) + low_inp_halo = y[:,:,:self.half_halo,:] + high_out_halo = y[:,:,W:W+self.half_halo,:] + high_tx = self._allocate_peer_tensor(high_out_halo) + high_inp_halo = y[:,:,W+self.half_halo:W+2*self.half_halo,:] + else: + _, _, _, Ws = list(y.shape) + W = Ws - 2*self.half_halo + low_out_halo = y[:,:,:,self.half_halo:2*self.half_halo] + low_tx = self._allocate_peer_tensor(low_out_halo) + low_inp_halo = y[:,:,:,:self.half_halo] + high_out_halo = y[:,:,:,W:W+self.half_halo] + high_tx = self._allocate_peer_tensor(high_out_halo) + high_inp_halo = y[:,:,:,W+self.half_halo:W+2*self.half_halo] + pm.push_pull_halos_1d( + diagnostics, explicit_nhwc, numSM, self.peer_rank, + self.low_zero, low_out_halo, low_tx[self.peer_rank], high_tx[self.low_neighbor], low_inp_halo, + self.high_zero, high_out_halo, high_tx[self.peer_rank], low_tx[self.high_neighbor], high_inp_halo, + ) diff --git a/apex/apex/contrib/peer_memory/peer_memory.py b/apex/apex/contrib/peer_memory/peer_memory.py new file mode 100644 index 00000000..adb21821 --- /dev/null +++ b/apex/apex/contrib/peer_memory/peer_memory.py @@ -0,0 +1,87 @@ +import torch +import numpy as np +import peer_memory_cuda as pm + +class PeerMemoryPool(object): + + def __init__(self, static_size, dynamic_size, peer_ranks=None): + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + ngpus = min(torch.cuda.device_count(), world_size) + peer_group_size = ngpus + peer_group = rank // ngpus + peer_rank_base = peer_group * ngpus + peer_rank = rank - peer_rank_base + if peer_ranks is None: + peer_ranks = [i+peer_rank_base for i in range(peer_group_size)] + peer_rank_start = peer_rank_base + peer_rank_end = peer_rank_start + peer_group_size - 1 + for pr in peer_ranks: + assert(pr >= peer_rank_start and pr <= peer_rank_end), "%d :: peer_rank %d not on same node (ranks=[%d,%d])" % (rank, pr, peer_rank_start, peer_rank_end) + + self.alignment = 256 + self.static_size = ((static_size + self.alignment - 1) // self.alignment) * self.alignment + self.dynamic_size = ((dynamic_size + self.alignment - 1) // self.alignment) * self.alignment + + # allocate giant pool of device memory + self.raw = pm.allocate_raw(self.static_size+self.dynamic_size) + + # exchange peer pointers with nccl + raw_ipc = pm.get_raw_ipc_address(self.raw).cuda() + peer_raw_ipcs = [torch.empty_like(raw_ipc) for _ in range(world_size)] + torch.distributed.all_gather(peer_raw_ipcs, raw_ipc) + peer_raw_ipcs = torch.stack(peer_raw_ipcs).cpu() + + # extract IPC pointers for ranks on same node + peer_raw = pm.get_raw_peers(peer_raw_ipcs[peer_rank_base:peer_rank_base+ngpus], peer_rank, self.raw) + self.peer_raw = [peer_raw[peer_rank-peer_rank_base] for peer_rank in peer_ranks] + self.static_offset = 0 + self.dynamic_offset = 0 + self.peer_ranks = peer_ranks + + def __del__(self): + pm.free_raw(self.raw) + + def reset(self): + self.dynamic_offset = 0 + + def allocate_peer_tensors(self, shape, dtype, channels_last, dynamic): + nels = np.prod(shape) + if dtype == torch.float16: + elem_size = 2 + if dynamic: + start = ((self.dynamic_offset + self.alignment - 1) // self.alignment) * self.alignment + self.dynamic_offset = start + nels * elem_size + assert(self.dynamic_offset < self.dynamic_size), "Dynamic peer memory pool exhausted" + return [pm.blob_view_half(pr + self.static_size + start, shape, channels_last) for pr in self.peer_raw] + else: + start = ((self.static_offset + self.alignment - 1) // self.alignment) * self.alignment + self.static_offset = start + nels * elem_size + assert(self.static_offset < self.static_size), "Static peer memory pool exhausted" + return [pm.blob_view_half(pr + start, shape, channels_last) for pr in self.peer_raw] + if dtype == torch.float32: + elem_size = 4 + if dynamic: + start = ((self.dynamic_offset + self.alignment - 1) // self.alignment) * self.alignment + self.dynamic_offset = start + nels * elem_size + assert(self.dynamic_offset < self.dynamic_size), "Dynamic peer memory pool exhausted" + return [pm.blob_view_float(pr + self.static_size + start, shape, channels_last) for pr in self.peer_raw] + else: + start = ((self.static_offset + self.alignment - 1) // self.alignment) * self.alignment + self.static_offset = start + nels * elem_size + assert(self.static_offset < self.static_size), "Static peer memory pool exhausted" + return [pm.blob_view_float(pr + start, shape, channels_last) for pr in self.peer_raw] + if dtype == torch.int32: + elem_size = 4 + if dynamic: + start = ((self.dynamic_offset + self.alignment - 1) // self.alignment) * self.alignment + self.dynamic_offset = start + nels * elem_size + assert(self.dynamic_offset < self.dynamic_size), "Dynamic peer memory pool exhausted" + return [pm.blob_view_int(pr + self.static_size + start, shape, channels_last) for pr in self.peer_raw] + else: + start = ((self.static_offset + self.alignment - 1) // self.alignment) * self.alignment + self.static_offset = start + nels * elem_size + assert(self.static_offset < self.static_size), "Static peer memory pool exhausted" + return [pm.blob_view_int(pr + start, shape, channels_last) for pr in self.peer_raw] + else: + assert(False), "dtype %s not supported" % (str(dtype)) diff --git a/apex/apex/contrib/sparsity/COPYRIGHT b/apex/apex/contrib/sparsity/COPYRIGHT new file mode 100644 index 00000000..ff4d4092 --- /dev/null +++ b/apex/apex/contrib/sparsity/COPYRIGHT @@ -0,0 +1 @@ +Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. diff --git a/apex/apex/contrib/sparsity/README.md b/apex/apex/contrib/sparsity/README.md new file mode 100644 index 00000000..bf2ccc03 --- /dev/null +++ b/apex/apex/contrib/sparsity/README.md @@ -0,0 +1,138 @@ +# Introduction to ASP + +This serves as a quick-start for ASP (Automatic SParsity), a tool that enables sparse training and inference for PyTorch models by adding 2 lines of Python. + +For details on "[Channel Permutations for N:M Sparsity](https://proceedings.neurips.cc/paper/2021/hash/6e8404c3b93a9527c8db241a1846599a-Abstract.html)," please see the [permutation_tests](permutation_tests/README.md) directory. + +## Importing ASP + +``` +from apex.contrib.sparsity import ASP +``` + +## Initializing ASP + +Apart from the import statement, it is sufficient to add just the following line of code before the training phase to augment the model and the optimizer for sparse training/inference: + +``` +ASP.prune_trained_model(model, optimizer) +``` + +In the context of a typical PyTorch training loop, it might look like this: + +``` +ASP.prune_trained_model(model, optimizer) + +x, y = DataLoader(args) +for epoch in range(epochs): + y_pred = model(x) + loss = loss_function(y_pred, y) + loss.backward() + optimizer.step() + +torch.save(...) +``` + +The `prune_trained_model` step calculates the sparse mask and applies it to the weights. This is done once, i.e., sparse locations in the weights matrix remain fixed after this step. + +## Generate a Sparse Network + +The following approach serves as a guiding example on how to generate a pruned model that can use Sparse Tensor Cores in the NVIDIA Ampere Architecture. This approach generates a model for deployment, i.e. inference mode. + +``` +(1) Given a fully trained (dense) network, prune parameter values in a 2:4 sparse pattern. +(2) Fine-tune the pruned model with optimization method and hyper-parameters (learning-rate, schedule, number of epochs, etc.) exactly as those used to obtain the trained model. +(3) (If required) Quantize the model. +``` + +In code, below is a sketch on how to use ASP for this approach (steps 1 and 2 above). + +``` +model = define_model(..., pretrained=True) # define model architecture and load parameter tensors with trained values (by reading a trained checkpoint) +criterion = ... # compare ground truth with model predition; use the same criterion as used to generate the dense trained model +optimizer = ... # optimize model parameters; use the same optimizer as used to generate the dense trained model +lr_scheduler = ... # learning rate scheduler; use the same schedule as used to generate the dense trained model + +from apex.contrib.sparsity import ASP +ASP.prune_trained_model(model, optimizer) #pruned a trained model + +x, y = DataLoader(args) +for epoch in range(epochs): # train the pruned model for the same number of epochs as used to generate the dense trained model + y_pred = model(x) + loss = criterion(y_pred, y) + lr_scheduler.step() + loss.backward() + optimizer.step() + +torch.save(...) # saves the pruned checkpoint with sparsity masks +``` + +## Non-Standard Usage + +If your goal is to easily perpare a network for accelerated inference, please follow the recipe above. However, ASP can also be used to perform experiments in advanced techniques like training with sparsity from initialization. For example, in order to recompute the sparse mask in between training steps, use the following method: + +``` +ASP.compute_sparse_masks() +``` + +A more thorough example can be found in `./test/toy_problem.py`. + +## Advanced Usage: Channel Permutation + +We introduce channel permutations as an advanced method to maximize the accuracy of structured sparse networks. By permuting weight matrices along their channel dimension and adjusting the surrounding layers appropriately, we demonstrate accuracy recovery for even small, parameter-efficient networks, without affecting inference run-time. + +The final accuracy has a strong relationship with the quality of permutations. We provide the default algorithms to search for high-quality permutations. The permutation search process can be accelerated by the Apex CUDA extension: `apex.contrib.sparsity.permutation_search_kernels` + +If you want to use the GPU to accelerate the permutation search process, we recommend installing Apex with permutation search CUDA extension via + +``` +pip install -v --disable-pip-version-check --no-cache-dir --global-option="--permutation_search" ./ +``` + +If you want to disable the permutation search process, please pass the `allow_permutation=False` to `init_model_for_pruning` function. For example: + +``` +ASP.init_model_for_pruning(model, mask_calculator="m4n2_1d", verbosity=2, whitelist=[torch.nn.Linear, torch.nn.Conv2d], allow_recompute_mask=False, allow_permutation=False) +``` + +Please notice, when using multi-GPUs we should set the identical random seed for all GPUs to make sure the same results generated in permutation search. The library has implemented the `set_identical_seed` function in `permutation_lib.py`, and be called in ASP library. We still suggest the users to set the identical random seed when using multi-GPUs in their code, the example code is as follows: + +``` +import torch +import numpy +import random + +torch.manual_seed(identical_seed) +torch.cuda.manual_seed_all(identical_seed) +numpy.random.seed(identical_seed) +random.seed(identical_seed) +torch.backends.cudnn.deterministic = True +torch.backends.cudnn.benchmark = False +``` + +## Reference Papers + +More details about sparsity support on the NVIDIA Ampere GPU with Sparse Tensor Cores can refer to our [white paper](https://arxiv.org/abs/2104.08378). + +``` +@article{mishra2021accelerating, + title={Accelerating sparse deep neural networks}, + author={Mishra, Asit and Latorre, Jorge Albericio and Pool, Jeff and Stosic, Darko and Stosic, Dusan and Venkatesh, Ganesh and Yu, Chong and Micikevicius, Paulius}, + journal={arXiv preprint arXiv:2104.08378}, + year={2021} +} +``` + +The details about sparsity with permutation can refer to our [paper](https://proceedings.neurips.cc/paper/2021/hash/6e8404c3b93a9527c8db241a1846599a-Abstract.html) published in *Thirty-fourth Conference on Neural Information Processing Systems* (**NeurIPS 2021**): + +``` +@inproceedings{pool2021channel, + author = {Pool, Jeff and Yu, Chong}, + booktitle = {Advances in Neural Information Processing Systems ({NeurIPS})}, + title = {Channel Permutations for {N:M} Sparsity}, + url = {https://proceedings.neurips.cc/paper/2021/file/6e8404c3b93a9527c8db241a1846599a-Paper.pdf}, + volume = {34}, + year = {2021} +} + +``` diff --git a/apex/apex/contrib/sparsity/__init__.py b/apex/apex/contrib/sparsity/__init__.py new file mode 100644 index 00000000..661fd4ae --- /dev/null +++ b/apex/apex/contrib/sparsity/__init__.py @@ -0,0 +1,2 @@ +from .sparse_masklib import create_mask +from .asp import ASP diff --git a/apex/apex/contrib/sparsity/asp.py b/apex/apex/contrib/sparsity/asp.py new file mode 100644 index 00000000..42de945c --- /dev/null +++ b/apex/apex/contrib/sparsity/asp.py @@ -0,0 +1,311 @@ +import types +import torch +from .sparse_masklib import create_mask +from .permutation_lib import Permutation + +torchvision_imported=True +try: + import torchvision +except ImportError: + print("[ASP][Warning] torchvision cannot be imported.") + torchvision_imported=False + +import json +import os +import string +import time + +def eligible_modules(model, whitelist_layer_types, allowed_layer_names, disallowed_layer_names): + eligible_modules_list = [] + for name, mod in model.named_modules(): + if isinstance(mod, whitelist_layer_types) and name not in disallowed_layer_names: + if allowed_layer_names is not None and name not in allowed_layer_names: + continue + eligible_modules_list.append((name, mod)) + return eligible_modules_list + + +class ASP: + __model = None + __verbosity = 0 + __optimizer = None + __sparse_parameters = [] + __calculate_mask = None + __allow_permutation = True + __all_parameters = [] + __save_permutation_graph = False + __permutation_output_dir = '' + + @classmethod + def init_model_for_pruning(cls, model, mask_calculator="m4n2_1d", + verbosity=3, + whitelist=[torch.nn.Linear, torch.nn.Conv1d, torch.nn.Conv2d, torch.nn.Conv3d, torch.nn.MultiheadAttention], + allowed_layer_names=None, disallowed_layer_names=[], + allow_recompute_mask=False, custom_layer_dict={}, + allow_permutation=True): + """Call this method to modify your model to take advantage of sparse matrix multiplication. + Note that this call alone only augments the model with additional buffers needed for sparse MMA, + it does not enable use of sparse MMA. + + If you are starting with a fresh model: + + model = ... + ASP.init_model_for_pruning(model, mask_calculator, ...) + if (training) ASP.init_optimizer_for_pruning(optimizer) + ASP.compute_sparse_masks() // sparsity is off by default, call when youy want to enable it. + + If you are starting from a checkpoint: + + model = ... + ASP.init_model_for_pruning(model, mask_calculator, ...) + torch.load(...) + if (training) ASP.init_optimizer_for_pruning(optimizer) + + Arguments: + model The model + mask_calculator Either callable that computes mask given a tensor OR pattern string for sparse mask lib. + verbosity Integer controling verbosity level. + 0 -> Only errors. + 1 -> Errors and warnings. + 2 -> Errors, warnings and info. + 3 -> Errors, warnings, info and debug. + whitelist Module types approved for sparsity. + allowed_layer_names If not None, only layer names that appear in this list are considered for sparsity. + disallowed_layer_names If not [], only layer names that do not appear in this list are considered for sparsity. + allow_recompute_mask If True, stores pruned values so that dense weights can be restored. + Pruned weights are stored in CPU memory, hence this option does not increase GPU memory usage. + custom_layer_dict Dictionary of additional layer paremeters to sparsify. e.g. {CustomLinear: ['weight']} + allow_permutation If True, allow the input channel permutation to ease the influence of weight pruning. + + [Future] Support for allow_recompute_mask can be removed, it is not part of sparse inference recipe. + """ + assert (cls.__model is None), "ASP has been initialized already." + cls.__model = model + cls.__verbosity = verbosity + cls.__allow_permutation = allow_permutation + + if isinstance(mask_calculator, str): + def create_mask_from_pattern(param): + return create_mask(param, mask_calculator).bool() + cls.__calculate_mask = create_mask_from_pattern + else: + cls.__calculate_mask = mask_calculator #user defined function + + # function to extract variables that will be sparsified. + # idea is that you will add one of these functions for each module type that can be sparsified. + if torchvision_imported: + print("[ASP] torchvision is imported, can work with the MaskRCNN/KeypointRCNN from torchvision.") + torchvision_version = str(torchvision.__version__) + torchvision_version_major = int(torchvision_version.split('.')[0]) + torchvision_version_minor = int(torchvision_version.split('.')[1]) + if torchvision_version_major == 0 and torchvision_version_minor < 12: + sparse_parameter_list = {torch.nn.Linear: ['weight'], torch.nn.Conv1d: ['weight'], torch.nn.Conv2d: ['weight'], torch.nn.Conv3d: ['weight'], torch.nn.modules.linear.NonDynamicallyQuantizableLinear: ['weight'], torch.nn.MultiheadAttention: ['q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight'], torchvision.ops.misc.Conv2d: ['weight']} + else: # Torchvision remove APIs that were deprecated before 0.8 (#5386) in 0.12.0, torchvision.ops.misc.Conv2d is removed + sparse_parameter_list = {torch.nn.Linear: ['weight'], torch.nn.Conv1d: ['weight'], torch.nn.Conv2d: ['weight'], torch.nn.Conv3d: ['weight'], torch.nn.modules.linear.NonDynamicallyQuantizableLinear: ['weight'], torch.nn.MultiheadAttention: ['q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight']} + else: + sparse_parameter_list = {torch.nn.Linear: ['weight'], torch.nn.Conv1d: ['weight'], torch.nn.Conv2d: ['weight'], torch.nn.Conv3d: ['weight'], torch.nn.modules.linear.NonDynamicallyQuantizableLinear: ['weight'], torch.nn.MultiheadAttention: ['q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight']} + if custom_layer_dict: # Update default list to include user supplied custom (layer type : parameter tensor), make sure this tensor type is something ASP knows how to prune + sparse_parameter_list.update(custom_layer_dict) + whitelist += list(custom_layer_dict.keys()) + + for module_type in whitelist: + assert (module_type in sparse_parameter_list), "Module %s :: Don't know how to sparsify module." % module.dtype() + + + # find all sparse modules, extract sparse parameters and decorate + def add_sparse_attributes(module_name, module): + sparse_parameters = sparse_parameter_list[type(module)] + for p_name, p in module.named_parameters(): + if p_name in sparse_parameters and p.requires_grad: + # check for NVIDIA's TC compatibility: we check along the horizontal direction + if p.dtype == torch.float32 and ((p.size()[0] % 8) != 0 or (p.size()[1] % 16) != 0): #User defines FP32 and APEX internally uses FP16 math + print("[ASP] Auto skipping pruning %s::%s of size=%s and type=%s for sparsity" % (module_name, p_name, str(p.size()), str(p.dtype))) + continue + if p.dtype == torch.float16 and ((p.size()[0] % 8) != 0 or (p.size()[1] % 16) != 0): #For Conv2d dim= K x CRS; we prune along C + print("[ASP] Auto skipping pruning %s::%s of size=%s and type=%s for sparsity" % (module_name, p_name, str(p.size()), str(p.dtype))) + continue + + if cls.__verbosity >= 3: + print("[ASP] Sparsifying %s::%s of size=%s and type=%s for sparsity" % (module_name, p_name, str(p.size()), str(p.dtype))) + + mask = torch.ones_like(p).bool() + buffname = p_name.split(".")[-1] # buffer names cannot contain "." + module.register_buffer('__%s_mma_mask' % buffname, mask) + if allow_recompute_mask: + pruned = torch.zeros_like(p).cpu() + module.register_buffer('__%s_mma_pruned_p' % buffname, pruned) + else: + pruned = None + cls.__sparse_parameters.append((module_name, module, p_name, p, mask, pruned)) + else: + if cls.__verbosity >= 3: + print("[ASP] Not sparsifying %s::%s of size=%s and type=%s" % (module_name, p_name, str(p.size()), str(p.dtype))) + + for name, sparse_module in eligible_modules(model, tuple(whitelist), allowed_layer_names, disallowed_layer_names): + add_sparse_attributes(name, sparse_module) + + if allow_permutation: # find all named modules, extract parameters and decorate, used for offline permutation in K dim + for module_name, module in model.named_modules(): + module_type_str = str(type(module)).split("\'")[1] + if module_type_str == 'torch.nn.modules.container.Sequential' or module_type_str.startswith('torchvision.models'): + # filter out the 'torch.nn.modules.container.Sequential' type and the whole model, like 'torchvision.models.vgg.VGG' + continue + for p_name, p in module.named_parameters(): + cls.__all_parameters.append((module_name, module, p_name, p)) + if module_type_str == 'torch.nn.modules.batchnorm.BatchNorm2d': + # need to get the running_mean and running_var from model.state_dict(), as they are not the learnable parameters + module_mean_name = module_name + '.running_mean' + module_var_name = module_name + '.running_var' + for param_key in model.state_dict(): + if module_mean_name == param_key or module_var_name == param_key: + cls.__all_parameters.append((module_name, module, param_key.split(".")[-1], model.state_dict()[param_key])) + # add the __permutation_output_dir field to save the intermediate results for permutation + cls.__permutation_output_dir = '.' + # Set the corresponding params from ASP class to the Permutation class + permutation_verbosity = 5 + Permutation.set_permutation_params_from_asp(cls.__model, cls.__sparse_parameters, cls.__all_parameters, permutation_verbosity) + # Set the identical random seed for all GPUs to make sure the same results generated in permutation search + Permutation.set_identical_seed() + + + @classmethod + def already_init_asp_model(cls): + """Call this method to check whether ASP has been initialized already. + """ + if cls.__model is None: + if cls.__verbosity >= 3: + print("[ASP] ASP has not been initialized.") + return False + else: + if cls.__verbosity >= 3: + print("[ASP] ASP has been initialized already.") + return True + + @classmethod + def init_optimizer_for_pruning(cls, optimizer): + """Call this method to monkey patch optimizer step function so that masks can be applied to + gradients and weights during training. + You must call init_model_for_pruning(...) before calling init_optimizer_for_pruning(...) + """ + assert (cls.__optimizer is None), "ASP has initialized optimizer already." + assert (cls.__calculate_mask is not None), "Called ASP.init_optimizer_for_pruning before ASP.init_model_for_pruning." + + # store pointer to original optimizer step method + cls.__optimizer = optimizer + cls.__optimizer.__step = optimizer.step + + def __step(opt_self, *args, **kwargs): + # prune gradients before step method + with torch.no_grad(): + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + if p.grad is not None: #thx pjudd + p.grad.mul_(mask) + # call original optimizer step method + rval = opt_self.__step(*args, **kwargs) + # prune parameters after step method + with torch.no_grad(): + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + p.mul_(mask) + return rval + cls.__optimizer.step = types.MethodType(__step, cls.__optimizer) + + @classmethod + def compute_sparse_masks(cls): + """Call this method to enable sparsity. + If init(...) was called with allow_recompute_mask=False AND sparsity is disabled, pruned field can be None. + """ + with torch.no_grad(): + if cls.__allow_permutation: + # Step 1: use the Torch.FX library to build the graph + # Step 2: permutation search with the customized kernel + # The simplest without user intervention: + # A. try to import with the distributed mode of the original model + # B. if meet the error, import with the none-distributed mode of the original model + start_time_permute = time.perf_counter() + successful_permutation = False + try: + successful_permutation = Permutation.permute_model(cls.__model.module, dump_fx_graph=cls.__save_permutation_graph, save_dumped_fx_graph=os.path.join(cls.__permutation_output_dir, 'model_offline_permutation_graph.json')) + if successful_permutation: + print("\n[compute_sparse_masks] permuted the (distributed) model.") + except AttributeError: + successful_permutation = Permutation.permute_model(cls.__model, dump_fx_graph=cls.__save_permutation_graph, save_dumped_fx_graph=os.path.join(cls.__permutation_output_dir, 'model_offline_permutation_graph.json')) + if successful_permutation: + print("\n[compute_sparse_masks] permuted the model.") + + if successful_permutation: + duration_build_offline_permutation_graph = time.perf_counter() - start_time_permute + print("[compute_sparse_masks] Take {:.4f} seconds to find and apply permutations.".format(duration_build_offline_permutation_graph)) + + + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + if mask.sum() < mask.numel(): # when recalculating masks + # restore dense parameter if allow_recompute_mask is enabled + assert (pruned is not None), "Unable to restore dense parameter because allow_recompute_mask == False" + p.add_(pruned.cuda()) + + mask.set_(cls.__calculate_mask(p)) + + if pruned is not None: # stow away pruned weights to cpu + pruned.set_((p * (~mask)).cpu()) + + p.mul_(mask) # in-place multiplication, so pruned weights are 0-values, hence checkpoint will have 0s for pruned weights + if cls.__verbosity >= 2: + print("[ASP] Enabled %.2f%% sparsity for %s::%s of size=%s and type=%s with magnitude %s" % (100.0-100.0*mask.sum()/mask.numel(), module_name, p_name, str(p.size()), str(p.dtype), torch.sum(torch.abs(p)))) + + @classmethod + def restore_pruned_weights(cls): + """Call this method to disable sparsity and restore all weights. + This will only work if init(...) was called with allow_recompute=True. + """ + with torch.no_grad(): + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + if mask.sum() < mask.numel(): + assert (pruned is not None), "Unable to restore dense parameter because allow_recompute_mask == False" + p.add_(pruned.cuda()) + mask.fill_(1) + pruned.zero_() + if cls.__verbosity >= 2: + print("[ASP] Disabled sparsity for %s::%s (dense weights restored)" % (module_name, p_name)) + + @classmethod + def is_sparsity_enabled(cls): + """Call this method to determine if sparsity is enabled in the model. + The typical use case is right after checkpoint has been loaded. + """ + total,sp100,sp50 = 0,0,0 + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + total += 1 + mask_sum = mask.sum() + mask_numel = mask.numel() + if mask_sum == mask_numel: + sp100 += 1 + elif mask_sum*2 == mask_numel: + sp50 += 1 + + assert (total == sp100 or total == sp50), "Inconsistent model sparsity" + if total == sp100: + return False + elif total == sp50: + return True + + @classmethod + def prune_trained_model(cls, model, optimizer): + # add mask buffers to model (init_model_for_pruning), augment optimizer (init_optimizer_for_pruning) and compute masks (compute_sparse_masks) + cls.init_model_for_pruning(model, mask_calculator="m4n2_1d", verbosity=2, whitelist=[torch.nn.Linear, torch.nn.Conv2d, torch.nn.MultiheadAttention], allow_recompute_mask=False) + cls.init_optimizer_for_pruning(optimizer) + cls.compute_sparse_masks() + + @classmethod + def set_permutation_saving_params(cls, allow_permutation=True, save_permutation_graph=False, permutation_output_dir='.'): + """This function is used to set the permutation saving related parameters in ASP class and inside of the Permutation class.""" + print("\n[ASP][set_permutation_saving_param] Set permutation saving related parameters") + print("\n[set_permutation_saving_param] Set permutation saving related parameters") + cls.__allow_permutation = allow_permutation + print("[set_permutation_saving_param]\t Allow permutation: {}".format(cls.__allow_permutation)) + cls.__save_permutation_graph = save_permutation_graph + print("[set_permutation_saving_param]\t Save permutation graphs: {}".format(cls.__save_permutation_graph)) + cls.__permutation_output_dir = permutation_output_dir + print("[set_permutation_saving_param]\t Permutation graphs saving dir: {}".format(cls.__permutation_output_dir)) + + Permutation.set_permutation_saving_params(allow_permutation, save_permutation_graph, permutation_output_dir) + diff --git a/apex/apex/contrib/sparsity/permutation_lib.py b/apex/apex/contrib/sparsity/permutation_lib.py new file mode 100644 index 00000000..cc8e94bf --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_lib.py @@ -0,0 +1,1683 @@ +import os +import torch +import json +import string +import time +import numpy as np +import sys +import builtins as __builtin__ +import io +try: + from .permutation_search_kernels import accelerated_search_for_good_permutation, sum_after_2_to_4 + print("[ASP][Info] permutation_search_kernels can be imported.") +except ImportError: + print("[ASP][Warning] permutation_search_kernels cannot be imported.") + print("[ASP][Warning] If you want to accelerate the permutation search process by GPU, please build APEX by following the instructions at https://github.com/NVIDIA/apex/blob/master/apex/contrib/sparsity/README.md") + +def convert_fx_node_name(fx_node_name): + """Standardize punctuation of a node's name: replace all '_' with '.'""" + return fx_node_name.replace('_', '.') + +def get_node_parent_children(fx_node): + """Populate lists of all direct parents and children of a node""" + # get node parent list, and convert node name to module name + node_parent_name_converted = [] + if len(fx_node.all_input_nodes) > 0: + node_parent = fx_node.all_input_nodes + for item in node_parent: + converted_item = convert_fx_node_name(item.name) + node_parent_name_converted.append(converted_item) + else: + node_parent = [] + + # get node children list, and convert node name to module name + node_children_name_converted = [] + if len(list(fx_node.users.keys())) > 0: + node_children = list(fx_node.users.keys()) + for item in node_children: + converted_item = convert_fx_node_name(item.name) + node_children_name_converted.append(converted_item) + else: + node_children = [] + + return node_parent_name_converted, node_children_name_converted + +def node_name_matches(node_name, module_name): + """Check for a match between graph node name and stored module name, accounting for formatting and DDP training differences""" + + # process: remove all punctuation, everything to lower case + def process(name): + return ''.join(c for c in name if c not in string.punctuation).lower() + + processed_node_name = process(node_name) + processed_module_name = process(module_name) + + # module names start with 'module.' in distributed data-parallel training, but fx graph node names don't; check for both + distributed_node_name = 'module.' + node_name + distributed_processed_node_name = 'module' + processed_node_name + + return (node_name == module_name) or (distributed_node_name == module_name) or (processed_node_name == processed_module_name) or (distributed_processed_node_name == processed_module_name) + +def replicate_sequence(sequence, replications): + """Replicate a permutation to apply it to an even multiple of channel counts""" + replicated_sequence = [] + + for rep in range(replications): + offset = len(sequence) * rep + for c in sequence: + replicated_sequence.append(c+offset) + + return replicated_sequence + +class Permutation: + __model = None + __sparse_parameters = [] + __allow_permutation = False + __all_parameters = [] + __verbosity = 0 ## 0: errors only, 1: also high-level details, warnings, 2: also intermediate steps, 3: everything + __params_permuted_in_C = [] + __params_permuted_in_K = [] + __unpermuted_dims = [] + + __save_permutation_graph = False + __permutation_output_dir = '' + __manual_seed = None + __tcpstore_port = 2341 + + # these module types may be the target of permutations (have potentially sparse weights or are attributes with no parents) + __permutation_target_module_types = ['torch.nn.modules.conv.Conv1d', + 'torch.nn.modules.conv.Conv2d', + 'torch.nn.modules.linear.Linear', + 'torch.nn.modules.linear.LazyLinear', + 'torch.nn.modules.linear.NonDynamicallyQuantizableLinear', + 'torch.nn.modules.activation.MultiheadAttention', + 'get_attr'] + + # these module types are not permuted, but must pass any permutation seen by a child's C or passed-thru K to the parents' K + __simple_passthru_module_types = ['torch.nn.modules.activation.ReLU6', + 'torch.nn.modules.activation.ReLU', + 'torch.nn.modules.dropout.Dropout', + 'torch.nn.modules.dropout.Dropout1d', + 'torch.nn.modules.dropout.Dropout2d', + 'torch.nn.modules.dropout.Dropout3d', + 'torch.nn.modules.dropout.AlphaDropout', + 'torch.nn.modules.dropout.FeatureAlphaDropout', + 'torch.nn.modules.pooling.MaxPool2d', + 'torch.nn.modules.pooling.AdaptiveAvgPool2d', + 'torch.nn.modules.pooling.AvgPool2d', + 'torch.nn.modules.activation.Hardsigmoid', + 'torch.nn.modules.activation.Hardswish', + 'torch.nn.modules.activation.GELU', + 'torch.nn.modules.normalization.LocalResponseNorm', + 'torch.nn.modules.activation.Softmin', + 'torch.nn.modules.activation.Softmax', + 'torch.nn.modules.activation.Softmax2d', + 'torch.nn.modules.activation.LogSoftmax', + 'torch.nn.modules.activation.AdaptiveLogSoftmaxWithLoss', + 'torch.nn.modules.activation.SiLU', + 'torch.nn.modules.activation.Sigmoid', + 'concat', + 'torch.nn.modules.flatten.Flatten' # if it's a problem, it'll be handled via dimension mismatch check + ] + + # these module types have parameters that must be permuted along K as well as need to pass the permutation thru to parents' K + __permute_K_and_passthru_module_types = ['torch.nn.modules.batchnorm.BatchNorm2d', + 'torch.nn.modules.normalization.LayerNorm', + 'torch.nn.modules.instancenorm.InstanceNorm2d', + 'torch.nn.modules.batchnorm.SyncBatchNorm'] + + # these module types cannot be permuted safely (today), and cause neighboring layers to have permutations disabled + __disallow_permutations_module_types = ['torch.nn.modules.normalization.GroupNorm', # to handle: influence GCD of real children's sibling group + 'torch.nn.modules.linear.Bilinear', # need to permute one input along in1_features and the other along in2_features + 'torch.nn.modules.activation.GLU', # may work OOTB, but might need to explicitly handle dimsionality change + ] + + @classmethod + def set_identical_seed(cls, identical_seed=1): + """Make all GPUs in DDP use the same seed to find identical permutations and not require syncing parameters later""" + + if cls.__verbosity > 0: + print("[set_identical_seed] Set the identical seed: {:} for all GPUs to make sure the same results generated in permutation search".format(identical_seed)) + + cls.__manual_seed = identical_seed + cls.reset_seed() + + @classmethod + def reset_seed(cls): + """To find the same permutations no matter how many GPUs are used, we reset the seed before every search""" + + identical_seed = cls.__manual_seed + assert identical_seed is not None, "Must call set_identical_seed() before it can be reset" + + torch.manual_seed(identical_seed) + torch.cuda.manual_seed(identical_seed) + import random + np.random.seed(identical_seed) + random.seed(identical_seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + @classmethod + def set_tcpstore_port(cls, tcpstore_port): + """Override the default port if it is in use in a distributed training session""" + + cls.__tcpstore_port = tcpstore_port + if cls.__verbosity > 0: + print(f"[set_tcpstore_port] TCPStore port set to {cls.__tcpstore_port} .") + + @classmethod + def set_permutation_saving_params(cls, allow_permutation=False, save_permutation_graph=False, permutation_output_dir='.'): + """This function is used to set the permutation saving related parameters.""" + + cls.__allow_permutation = allow_permutation + cls.__save_permutation_graph = save_permutation_graph + cls.__permutation_output_dir = permutation_output_dir + + if cls.__verbosity > 0: + print(f"[permutation_lib][set_permutation_saving_param] Set permutation saving related parameters\n\tAllow permutation: {cls.__alow_permutation}\n\tSave permutation graphs: {cls.__save_permutation_graph}\n\tPermutation graphs saving dir: {cls.__permutation_output_dir}") + + @classmethod + def set_permutation_params_from_asp(cls, model, sparse_parameters, all_parameters, verbosity): + """This function is used to set the permutation needed parameters from ASP class.""" + cls.__verbosity = verbosity + + if cls.__verbosity > 0: + print("[set_permutation_params_from_asp] Set permutation needed parameters") + cls.__model = model + cls.__sparse_parameters = sparse_parameters + cls.__all_parameters = all_parameters + + if cls.__verbosity > 1: + sparse_param_names = [module_name+":"+p_name for (module_name, module, p_name, p, mask, pruned) in cls.__sparse_parameters] + all_param_names = [module_name+":"+p_name for (module_name, module, p_name, p) in cls.__all_parameters] + print(f"\tSparse parameter names: {sparse_param_names}\n\tAll parameter names: {all_param_names}") + + cls.__params_permuted_in_C = [] + cls.__params_permuted_in_K = [] + cls.__unpermuted_dims = [] + + @classmethod + def permute_model(cls, model, dump_fx_graph=False, save_dumped_fx_graph='./model_permutation_graph.json'): + """Permute a model's weights in order to maintain more magnitude after enforcing the sparsity constraint.""" + + if cls.__verbosity > 0: + print("\n[permute_model] Permuting the model") + + # extract the output_dir, so all the intermediate fx_graph can be saved under that path + extract_output_dir=os.path.split(save_dumped_fx_graph)[0] + cls.__permutation_output_dir = extract_output_dir + fx_graph, success_in_build_fx_graph = cls.build_fx_graph(model, dump_fx_graph=dump_fx_graph, save_dumped_fx_graph=save_dumped_fx_graph) + + if success_in_build_fx_graph: + + fx_graph_after_init_flags = cls.init_permutation_flags(fx_graph) + fx_graph_after_find_real_parents = cls.find_real_parents(fx_graph_after_init_flags) + fx_graph_after_find_real_children = cls.find_real_children(fx_graph_after_find_real_parents) + fx_graph_after_making_groups = cls.make_sibling_coparent_groups(fx_graph_after_find_real_children) + fx_graph_after_fixup_concats = cls.fixup_concats(fx_graph_after_making_groups) + fx_graph_after_enforce_dimension_agreement = cls.enforce_dimension_agreement(fx_graph_after_fixup_concats) + fx_graph_after_propagate_flags = cls.propagate_permutation_flags(fx_graph_after_enforce_dimension_agreement) + + start_time_search_for_good_permutation = time.perf_counter() + fx_graph_after_find_permutations = cls.find_permutations(fx_graph_after_propagate_flags) + + if torch.distributed.is_initialized(): + if cls.__verbosity > 0: + duration_search_for_good_permutation = time.perf_counter() - start_time_search_for_good_permutation + print(f"[permute_model] Rank {torch.distributed.get_rank()} completed search in {duration_search_for_good_permutation:.2f}s, waiting for others.", force=True) + torch.distributed.barrier() + + duration_search_for_good_permutation = time.perf_counter() - start_time_search_for_good_permutation + if cls.__verbosity > 0: + print("\n[permute_model] Take {:.4f} seconds to finish search_for_good_permutation function.".format(duration_search_for_good_permutation)) + + fx_graph_after_sync_permutations = cls.sync_permutations(fx_graph_after_find_permutations) + fx_graph_after_apply_permutations = cls.apply_permutations(fx_graph_after_sync_permutations) + cls.check_graph_for_unpermuted_nodes(fx_graph_after_apply_permutations) + + fx_graph = fx_graph_after_apply_permutations + + if cls.__save_permutation_graph: + cls.save_graph_to_json(fx_graph, save_dumped_graph_path_with_name=os.path.join(cls.__permutation_output_dir, './model_graph_permutation_graph.json')) # save the intermediate graph as JSON file for debugging + + return success_in_build_fx_graph + + + @classmethod + def get_permutation_stats(cls): + """Return statistics for how many permutations were applied in various dimensions, used for testing""" + + return cls.__params_permuted_in_C, cls.__params_permuted_in_K, cls.__unpermuted_dims + + + @classmethod + def apply_permutation_in_C_dim(cls, node_name, permutation_sequence, dryrun): + """This function is used to permutation for a node in C dim. (Only need to handle the weight of the node) """ + + if cls.__verbosity > 1 and dryrun: + print("[apply_permutation_in_C_dim] Permutation for node: \'{:}\' in C dim".format(node_name)) + + if len(permutation_sequence) == 0: + if cls.__verbosity >= 0: + print(f"ERROR: [apply_permutation_in_C_dim] the permutation sequence for node {node_name} is empty, fail to apply permutation in C dim.") + return False + + is_node_in_sparse_parameters = False + success_permutation = False + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + + if node_name_matches(node_name, module_name): + if cls.__verbosity > 2 and dryrun: + print("[apply_permutation_in_C_dim] find the node: \'{:}\' \'{:}\' in cls.__sparse_parameters, succeed to apply permutation in C dim.".format(node_name, p_name)) + is_node_in_sparse_parameters = True + permutation_to_apply = permutation_sequence + if p.shape[1] != len(permutation_sequence): # assumed to be grouped convolutions or concatenated weights + if p.shape[1] % len(permutation_sequence) != 0: + return False + + permutation_to_apply = replicate_sequence(permutation_sequence, p.shape[1] // len(permutation_sequence)) + + if not dryrun: + p.data.copy_(p[:, permutation_to_apply, ...]) + cls.__params_permuted_in_C.append(node_name + "." + p_name) + + success_permutation = True + if not is_node_in_sparse_parameters: + # A special case: if the node itself not in sparse_module_names but one of its real_siblings in sparse_module_names, then the node will not do the permutation search, but it may need to apply the offline permutation in C dim according to the searched permutation sequence from its real_siblings in sparse_module_names + try: + for module_name_from_all_parameters, module_from_all_parameters, p_name_from_all_parameters, p_from_all_parameters in cls.__all_parameters: + + if node_name_matches(node_name, module_name_from_all_parameters) and p_name_from_all_parameters == "weight": + if cls.__verbosity > 3 and dryrun: + print("[apply_permutation_in_C_dim] cannot find the node: \'{:}\' \'{:}\' in cls.__sparse_parameters, but can find in cls.__all_parameters.".format(node_name, p_name_from_all_parameters)) + permutation_to_apply = permutation_sequence + if p_from_all_parameters.shape[1] != len(permutation_sequence): # assumed to be grouped convolutions + if p_from_all_parameters.shpae[1] % len(permutation_sequence) != 0: + return False + + permutation_to_apply = replicate_sequence(permutation_sequence, p_from_all_parameters.shape[1] // len(permutation_sequence)) + + if not dryrun: + p_from_all_parameters.data.copy_(p_from_all_parameters[:, permutation_to_apply, ...]) + cls.__params_permuted_in_C.append(node_name + "." + p_name_from_all_parameters) + + success_permutation = True + if cls.__verbosity > 2 and dryrun: + print("[apply_permutation_in_C_dim] cannot find the node: \'{:}\' in cls.__sparse_parameters, after trying with cls.__all_parameters, succeed to apply permutation in C dim.".format(node_name)) + except: + success_permutation = False + if cls.__verbosity >= 0: + print("ERROR: [apply_permutation_in_C_dim] cannot find the node: \'{:}\' in cls.__sparse_parameters, after trying with cls.__all_parameters, still fail to apply permutation in C dim.".format(node_name)) + return success_permutation + + @classmethod + def permute_attr(cls, node_name, permutation_sequence, fx_graph, dryrun): + """ Permute a node's attributes. Somewhat hacky, assumes that we'll find exactly one dimension with a length matching the permutation's """ + + assert 'attr' in fx_graph[node_name].keys() + attr = fx_graph[node_name]['attr'] + if cls.__verbosity > 1: + print(f"Found attribute {node_name} of shape {attr.shape}") + found_perm = False + for dim in range(len(attr.shape)): + if attr.shape[dim] == len(permutation_sequence): + if found_perm: + if cls.__verbosity > 0: + print(f"\tWARNING: {node_name} has already been permuted, but it's trying to happen again along another dimension {dim}.") + + return False + + found_perm = True + if cls.__verbosity > 1 and dryrun: + print(f"\tpermuting along dimension {dim}") + + if not dryrun: + # permute the dimension of interest to the front, permute within that dimension, then reset it + order = [c for c in range(len(attr.shape))] + order[0] = dim + order[dim] = 0 + prmt = tuple(order) + + temp_weight = torch.clone(attr) + temp_weight = torch.permute(temp_weight, prmt) + temp_weight.copy_(temp_weight[permutation_sequence, ...]) + temp_weight = torch.permute(temp_weight, prmt) + attr.data.copy_(temp_weight) + + cls.__params_permuted_in_K.append(node_name + "_" + str(dim)) + + return found_perm + + + @classmethod + def apply_permutation_in_K_dim(cls, node_name, permutation_sequence, fx_graph, dryrun): + """This function is used to permutation for a node in K dim. (Need to handle the weight/bias/running_mean/running_var of the node)""" + + if cls.__verbosity > 1: + print("[apply_permutation_in_K_dim] Permutation for node: \'{:}\' in K dim".format(node_name)) + + if len(permutation_sequence) == 0: + if cls.__verbosity >= 0: + print("ERROR: [apply_permutation_in_K_dim] the permutation sequence is empty, fail to apply permutation in K dim.") + return False + + # permute attribute nodes + if 'attr' in fx_graph[node_name].keys(): + return cls.permute_attr(node_name, permutation_sequence, fx_graph, dryrun) + + # if we didn't store the attribute already, look in the modules' parameters + is_node_in_all_parameters = False + success_permutation = False + + for module_name, module, p_name, p in cls.__all_parameters: + + if node_name_matches(node_name, module_name): + + if cls.__verbosity > 1 and dryrun: + print("[apply_permutation_in_K_dim] find the node: \'{:}\' with \'{:}\' in cls.__all_parameters, may succeed to apply permutation in K dim.".format(node_name, p_name)) + is_node_in_all_parameters = True + permutation_to_apply = permutation_sequence + + if p.shape[0] != len(permutation_sequence): # assumed to be grouped convolutions + if cls.__verbosity > 2 and dryrun: + print(f"Mismatch in K dimension between found module {module_name} {p_name} for node {node_name}: permutation length {len(permutation_sequence)} but parameter shape in K {p.shape[0]}") + + if p.shape[0] % len(permutation_sequence) != 0: + return False + + permutation_to_apply = replicate_sequence(permutation_sequence, p.shape[0] // len(permutation_sequence)) + + if cls.__verbosity > 1 and dryrun: + print("[apply_permutation_in_K_dim] the node: \'{:}\' with shape: \'{:}\' required replicating the permutation sequence with len \'{:}\' {:} times to succeed in applying the permutation in the K dimension.".format(node_name, p.shape, len(permutation_sequence), p.shape[0] // len(permutation_sequence))) + else: + if cls.__verbosity > 1 and dryrun: + print("[apply_permutation_in_K_dim] the node: \'{:}\' with shape: \'{:}\', can match the size of permutation sequence with len: \'{:}\', succeed to apply permutation in K dim.".format(node_name, p.shape, len(permutation_sequence))) + + if not dryrun: + p.data.copy_(p[permutation_to_apply, ...]) + cls.__params_permuted_in_K.append(node_name + "." + p_name) + + success_permutation = True + + if not is_node_in_all_parameters: + if cls.__verbosity >= 0: + print("ERROR: [apply_permutation_in _K_dim] cannot find the node: \'{:}\' in cls.__all_parameters, fail to apply permutation in K dim.".format(node_name)) + success_permutation = False + + return success_permutation + + + @classmethod + def check_graph_for_unpermuted_nodes(cls, fx_graph): + """Make sure that all permutable nodes/parameters were actually permuted and all GPUs agree""" + + for node_name in fx_graph.keys(): + node = fx_graph[node_name] + + if 'C_permutable' in node.keys() and node['C_permutable'] and not node['C_permuted']: + sibling_group_id = node['sibling_group_id'] + if node['is_real'] and cls.__group_data['skipped_sibling_groups'][sibling_group_id] is None: + if cls.__verbosity >= 0: + print(f"{node_name} was C_permutable in a not skipped sibling group but was not permuted along C! {node}") + cls.__unpermuted_dims.append(node_name + "_C") + + if 'K_permutable' in node.keys() and node['K_permutable'] and not node['K_permuted']: + coparent_group_id = node['coparent_group_id'] + if node['is_real'] and cls.__group_data['skipped_coparent_groups'][coparent_group_id] is None: + if cls.__verbosity >= 0: + print(f"{node_name} was K_permutable in a not skipped coparent group but was not permuted along K! {node}") + cls.__unpermuted_dims.append(node_name + "_K") + + if cls.__verbosity > 0: + print(f"[check_graph_for_unpermuted_nodes] found nodes that missed permutations along {len(cls.__unpermuted_dims)} dimensions.") + + # make sure all GPUs agree + if torch.distributed.is_initialized(): + cls.__unpermuted_dims = sorted(cls.__unpermuted_dims) + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + dist_store = torch.distributed.TCPStore("127.0.0.1", cls.__tcpstore_port, world_size, rank==0) + torch.distributed.barrier() + + dist_store.set(str(rank), ','.join(cls.__unpermuted_dims)) + torch.distributed.barrier() + + if rank == 0: + my_list = dist_store.get('0').decode() + + for peer in range(1, world_size): + peer_list = dist_store.get(str(peer)).decode() + assert my_list == peer_list, f"peer {peer} disagreed with rank 0's list of unpermuted nodes: \n{my_list}\n{peer_list}" + + + @classmethod + def find_sparse_parameters_for_node(cls, node_name): + """If the node has parameters that are in the trackd sparse parameter list, find them and reshape to a 2D tensor with channels last""" + node_weight = None + + # check the sparse parameters + for module_name, module, p_name, p, mask, pruned in cls.__sparse_parameters: + + if node_name_matches(node_name, module_name): + node_weight = torch.zeros_like(p) + node_weight.copy_(p) + + # if we found something, reshape to concatenate along the same dimension + if node_weight is not None: + # Need to handle the concat for layers with different R & S + shape = node_weight.shape + # 1d-tensor + if len(shape) == 1: + node_weight = node_weight.view(1, shape[0]) + # 2d-tensor (K, C) + elif len(shape) == 2: + node_weight = node_weight.view(shape[0], shape[1]) + # 3d-tensor (K, C, R) + elif len(shape) == 3: + node_weight = node_weight.permute(0,2,1).contiguous().view(shape[0]*shape[2], shape[1]) + # 4d-tensor (K, C, R, S) + elif len(shape) == 4: + # convs + node_weight = node_weight.permute(2,3,0,1).contiguous().view(shape[2]*shape[3]*shape[0], shape[1]) + + return node_weight + + @classmethod + def find_permutation_for_matrix_group(cls, matrix_group): + """Find a good permutation for some matrix (which may be concatenated matrices that require the same permutation)""" + + if cls.__verbosity > 1: + print(f"Searching for a good permutation for this sibling group of shape {matrix_group.shape}") + + permutation_found = False + num_channels = matrix_group.shape[1] + group_permutation = [c for c in range(num_channels)] + + # automatic check for skipping the permutation search process + original_magnitude = (torch.abs(matrix_group)).sum(dtype=torch.float64) + pruned_magnitude = sum_after_2_to_4(matrix_group.cpu().detach().numpy()) + diff_ratio = abs(original_magnitude - pruned_magnitude)/original_magnitude + epsilon = 1e-3 + + if cls.__verbosity > 1: + print("\n[search_for_good_permutation] Original element abs sum: {:}, Pruned element abs sum: {:}, Diff ratio: {:}".format(original_magnitude, pruned_magnitude, diff_ratio)) + + start_time_accelerated_search_for_good_permutation = time.perf_counter() + if diff_ratio < epsilon: + if cls.__verbosity > 2: + print("[search_for_good_permutation] Original element abs sum is almost same as the pruned element abs sum, further permutation search will not help, skipping!") + + else: + if cls.__verbosity > 2: + print("[search_for_good_permutation] Original element abs sum is different from the pruned element abs sum, further permutation search will help, continue with the permutation search!") + + # call the permutation search CUDA kernels as ASP extension. + # users can provide prefer search strategy by providing a valid 'search_options' as a dictionary, + # or users can implement their customized 'accelerated_search_for_good_permutation' function. + search_options = {} + # No.1 Strategy: Exhaustive Search + search_options['strategy'] = 'exhaustive' + search_options['stripe_group_size'] = 8 + search_options['escape_attempts'] = 100 + # No.2 Strategy: Progressive Channel Swap Search + # search_options['strategy'] = 'progressive channel swap' + # search_options['progressive_search_time_limit'] = 10 + # search_options['improvement_threshold'] = 1e-9 + + # permutation search time is too long for matrix_group with large channel num + # change from Exhaustive Search to Progressive Channel Swap Search based on input matrix_group size + if num_channels > 2048: + search_options = {} + search_options['strategy'] = 'progressive channel swap' + search_options['progressive_search_time_limit'] = 120 + search_options['improvement_threshold'] = 1e-9 + + if cls.__verbosity > 1: + print(f"[search_for_good_permutation] search options: {search_options}") + + group_permutation = accelerated_search_for_good_permutation(matrix_group, options=search_options, verbosity=cls.__verbosity) + permutation_found = True + + if cls.__verbosity > 1: + duration_accelerated_search_for_good_permutation = time.perf_counter() - start_time_accelerated_search_for_good_permutation + permuted_magnitude = sum_after_2_to_4(matrix_group.cpu().detach().numpy()[:,group_permutation]) + print("[search_for_good_permutation] Take {:.4f} seconds to finish accelerated_search_for_good_permutation function and with final magnitude {:}.".format(duration_accelerated_search_for_good_permutation, permuted_magnitude)) + + return group_permutation, permutation_found + + @classmethod + def skip_sibling_group(cls, fx_graph, sibling_group_id, reason): + """Keep track of sibling groups that do not have permutations applied""" + + # grab a parent to get the coparent group id + sibling_group = cls.__group_data['sibling_groups'][sibling_group_id] + a_sibling = list(sibling_group)[0] + a_parent = fx_graph[a_sibling]['real_parents'][0] + coparent_group_id = fx_graph[a_parent]['coparent_group_id'] + + if cls.__verbosity > 1: + print(f"Skipping permutations for Sibling Group {sibling_group_id} and Coparent Group {coparent_group_id}: {reason}") + + cls.__group_data['skipped_sibling_groups'][sibling_group_id] = reason + cls.__group_data['skipped_coparent_groups'][coparent_group_id] = reason + + @classmethod + def collect_sparse_weights(cls, fx_graph, sibling_group, sibling_group_C_param): + """Gather all sparse weights for a sibling group (to serve as input to the permutation search)""" + + matrix_group = None + + for sibling in sibling_group: + node_weight = cls.find_sparse_parameters_for_node(sibling) + + if node_weight is not None: + # reshape due to siblings with grouped convolutions of different sizes + assert node_weight.shape[1] % sibling_group_C_param == 0, f"sibling {sibling}'s weights' C={node_weight.shape[1]} must be even multiple of the sibling group's C parameter {sibling_group_C_param}" + node_weight = torch.reshape(node_weight, (-1, sibling_group_C_param)) + + if matrix_group is None: + matrix_group = node_weight + else: + try: + matrix_group = torch.cat((matrix_group, node_weight), dim = 0) # concat the weights in the K dimension, keep the same C dimension + + except: + if cls.__verbosity >= 0: + print("ERROR: [search_for_good_permutation][warning] cannot merge the weight for node: \'{:}\', with its weight shape: \'{:}\', the matrix_group shape: \'{:}\'.".format(sibling, node_weight.size(), matrix_group.size())) + continue + if cls.__verbosity > 2: + print("[search_for_good_permutation] have merged the weight for node: \'{:}\', with its weight shape: \'{:}\', the matrix_group shape: \'{:}\'.".format(sibling, node_weight.size(), matrix_group.size())) + else: + if cls.__verbosity > 2: + print(f"[search_for_good_permutation] not adding dense weights for node {sibling} to the group") + + return matrix_group + + @classmethod + def find_sibling_group_permutation(cls, fx_graph, sibling_group_id): + """"Find a good permutation for some sibling group""" + + if cls.__verbosity > 1: + print(f"Finding permutation for sibling group {sibling_group_id}") + + cls.reset_seed() + + sibling_group = cls.__group_data['sibling_groups'][sibling_group_id] + sibling_group_C_param = int(cls.__group_data['sibling_group_C_params'][sibling_group_id]) + + if sibling_group_C_param % 4 != 0 or sibling_group_C_param < 8: + cls.skip_sibling_group(fx_graph, sibling_group_id, f"Useless C: {sibling_group_C_param}") + return + + # collect *sparse* weights from all siblings, get the coparent group + matrix_group = cls.collect_sparse_weights(fx_graph, sibling_group, sibling_group_C_param) + + # early-out if no siblings are sparse + if matrix_group is None: + cls.skip_sibling_group(fx_graph, sibling_group_id, 'Dense') + return + + # find a good permutation + group_permutation, found = cls.find_permutation_for_matrix_group(matrix_group) + + # if no permutation was found, we didn't need it (input already sparse) + if not found: + cls.skip_sibling_group(fx_graph, sibling_group_id, 'Not needed') + return + + if cls.__verbosity > 2: + print(f"Permutation for sibling group {sibling_group_id}: {group_permutation}") + + cls.__group_data['sibling_group_permutations'][sibling_group_id] = group_permutation + + + @classmethod + def permute_sibling_group(cls, fx_graph, sibling_group_id, group_permutation): + """Apply a permutation to some sibling group""" + + if cls.__verbosity > 1: + print(f"Attempting to permute sibling group {sibling_group_id}") + + sibling_group = cls.__group_data['sibling_groups'][sibling_group_id] + + # apply the permutation in two steps: first, a dry run to find any issues. + # if there were no issues, actually apply the permutation in the second step. + success = True + coparent_group_id = None + for dryrun in [True, False]: + # apply that permutation to the siblings' C dimension + for sibling in sibling_group: + assert fx_graph[sibling]['C_permutable'] and not fx_graph[sibling]['C_permuted'] + sibling_permuted = cls.apply_permutation_in_C_dim(sibling, group_permutation, dryrun) + if dryrun: + success = success and sibling_permuted + else: + assert sibling_permuted, "shouldn't fail permuting siblings after the dry run" + fx_graph[sibling]['C_permuted'] = sibling_permuted + + a_parent = fx_graph[sibling]['real_parents'][0] + if coparent_group_id is None: + coparent_group_id = fx_graph[a_parent]['coparent_group_id'] + else: + assert coparent_group_id == fx_graph[a_parent]['coparent_group_id'], f"parent {a_parent} must belong to the same coparent group {coparent_group_id}, not {fx_graph[a_parent]['coparent_group_id']}" + + # grab the parents (and co-parents) and apply to their K dimension + coparents = cls.__group_data['coparent_groups'][coparent_group_id] + for coparent in coparents: + assert fx_graph[coparent]['K_permutable'] and not fx_graph[coparent]['K_permuted'] + coparent_permuted = cls.apply_permutation_in_K_dim(coparent, group_permutation, fx_graph, dryrun) + if dryrun: + success = success and coparent_permuted + else: + assert coparent_permuted, "shouldn't fail permuting coparents after the dry run" + fx_graph[coparent]['K_permuted'] = coparent_permuted + + children_permuted = cls.apply_permutation_in_K_dim_to_children(fx_graph, coparent, group_permutation, dryrun) + if dryrun: + success = success and children_permuted + else: + assert children_permuted, "shouldn't fail permuting coparents' children after the dry run" + + if not success: + cls.skip_sibling_group(fx_graph, sibling_group_id, "dryrun_failure") + + if cls.__verbosity > 0: + print(f"There was an issue permuting sibling group {sibling_group_id}, skipping it to preserve network quality.") + + break + + + @classmethod + def apply_permutation_in_K_dim_to_children(cls, fx_graph, node_name, permutation, dryrun): + """Apply a permutation along K to the children of some node""" + + success = True + children = fx_graph[node_name]['children'] + if cls.__verbosity > 2 and dryrun: + print(f"Applying a permutation in K to children of {node_name} : {children}") + + # apply the permutation along K to children as necessary + for child in children: + if 'is_real' in fx_graph[child].keys() and fx_graph[child]['is_real']: + if cls.__verbosity > 3 and dryrun: + print(f"\tFound a real child {child}, not permuting it or its children along K") + else: + if 'module_type' not in fx_graph[child].keys() or fx_graph[child]['module_type'] == 'None': + if cls.__verbosity > 3 and dryrun: + print(f"\tPermuting children of non-module {child} along K") + success = success and cls.apply_permutation_in_K_dim_to_children(fx_graph, child, permutation, dryrun) + elif not fx_graph[child]['C_permutable']: + if fx_graph[child]['K_permutable'] and not fx_graph[child]['K_permuted']: + if cls.__verbosity > 2 and dryrun: + print(f"\tPermuting {child} along K") + child_permuted = cls.apply_permutation_in_K_dim(child, permutation, fx_graph, dryrun) + success = success and child_permuted + if not dryrun: + fx_graph[child]['K_permuted'] = child_permuted + assert fx_graph[child]['K_passthru'] + + if fx_graph[child]['K_passthru']: + success = success and cls.apply_permutation_in_K_dim_to_children(fx_graph, child, permutation, dryrun) + else: + if cls.__verbosity >= 0: + print(f"\t!! ERROR {child} was a not real module that was not K_passthru") + + return success + + @classmethod + def defer_prints(cls): + """Collect prints from this rank in distributed mode to avoid interleaved output""" + + if torch.distributed.is_initialized() and torch.distributed.get_world_size() > 1: + cls.__new_stdout = io.StringIO(str(torch.distributed.get_rank())) + cls.__builtin_print = __builtin__.print + + def deferred_print(*args, **kwargs): + try: # see if torchvision examples has suppressed other ranks with the force argument + cls.__builtin_print(*args, file=cls.__new_stdout, force=True, **kwargs) + except: + cls.__builtin_print(*args, file=cls.__new_stdout, **kwargs) + + __builtin__.print = deferred_print + + + @classmethod + def resume_prints(cls): + """Emit the collected outputs from this rank, resume immediate printing""" + + if torch.distributed.is_initialized() and torch.distributed.get_world_size() > 1: + output = cls.__new_stdout.getvalue() + __builtin__.print = cls.__builtin_print + + try: + print(output, force=True) + except: + print(output) + + @classmethod + def find_permutations(cls, fx_graph): + """Search for permutations for all sibling groups""" + + for sibling_group_id in cls.__group_data['sibling_groups'].keys(): + + search_this_group = True + if torch.distributed.is_initialized(): + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + + if sibling_group_id % world_size != rank: + search_this_group = False + + cls.__group_data['sibling_group_permutations'][sibling_group_id] = None + if search_this_group: + + cls.defer_prints() + + sibling_group = cls.__group_data['sibling_groups'][sibling_group_id] + test_node_name = list(sibling_group)[0] + if not fx_graph[test_node_name]['C_permutable']: + if cls.__verbosity > 1: + print(f"Skipping permutation for sibling group {sibling_group_id} since it does not allow permutations along C") + + else: + if cls.__verbosity > 1: + print(f"Sibling group {sibling_group_id} can permute along C, permuting it") + + cls.find_sibling_group_permutation(fx_graph, sibling_group_id) + + cls.resume_prints() + + return fx_graph + + @classmethod + def sync_permutations(cls, fx_graph): + """If multiple GPUs were involved in finding permutations, make sure everyone's in sync""" + + if not torch.distributed.is_initialized(): + return fx_graph + + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + dist_store = torch.distributed.TCPStore("127.0.0.1", cls.__tcpstore_port, world_size, rank==0) + + if cls.__verbosity > 0: + print(f"Syncing permutations found among world size {world_size}") + + torch.distributed.barrier() + for sibling_group_id in sorted(cls.__group_data['sibling_groups'].keys()): + src_rank = sibling_group_id % world_size + + if src_rank == rank: + to_send = cls.__group_data['sibling_group_permutations'].get(sibling_group_id, None) + skip_reason = None + if to_send is None: + skip_reason = cls.__group_data['skipped_sibling_groups'].get(sibling_group_id, None) + if skip_reason is None: + to_send = '' + else: + to_send = 'skip' + else: + to_send = ','.join(str(c) for c in to_send) + + dist_store.set(str(sibling_group_id), to_send) + if skip_reason is not None: + dist_store.set(f"skip {sibling_group_id}", skip_reason) + + if cls.__verbosity > 1: + print(f"{rank}: stored permutation for sibling group {sibling_group_id}", force=True) + + torch.distributed.barrier() + for sibling_group_id in sorted(cls.__group_data['sibling_groups'].keys()): + permutation = dist_store.get(str(sibling_group_id)).decode() + + if permutation == 'skip': + permutation = None + skip_reason = dist_store.get(f"skip {sibling_group_id}").decode() + cls.skip_sibling_group(fx_graph, sibling_group_id, skip_reason) + else: + if len(permutation) == 0: + permutation = None + else: + permutation = [int(c) for c in permutation.split(',')] + + cls.__group_data['sibling_group_permutations'][sibling_group_id] = permutation + + + if cls.__verbosity > 1: + print(f"Got permutation for sibling group {sibling_group_id}") + + torch.distributed.barrier() + return fx_graph + + @classmethod + def apply_permutations(cls, fx_graph): + """Apply all the permutations that were found to the network appropriately""" + + for sibling_group_id in cls.__group_data['sibling_group_permutations'].keys(): + + permutation = cls.__group_data['sibling_group_permutations'][sibling_group_id] + + if permutation is not None: + cls.permute_sibling_group(fx_graph, sibling_group_id, permutation) + + return fx_graph + + @staticmethod + def insert_MHA_out_proj(fx_graph, MHA_node, verbosity): + """MHA nodes have a hidden out_proj node, so insert it and fix up neighboring nodes""" + + if verbosity > 1: + print(f"Inserting MHA out_proj for node {MHA_node}") + out_proj_node_name = MHA_node + ".out_proj" + # insert the new node + fx_graph[out_proj_node_name] = {} + fx_graph[out_proj_node_name]['parents'] = [MHA_node] + fx_graph[out_proj_node_name]['children'] = fx_graph[MHA_node]['children'] + fx_graph[MHA_node]['children'] = [out_proj_node_name] + + # set the new node's properties + fx_graph[out_proj_node_name]['fx_op'] = 'call_module' + fx_graph[out_proj_node_name]['module_type'] = 'torch.nn.modules.linear.Linear' + fx_graph[out_proj_node_name]['groups_param'] = 'None' + fx_graph[out_proj_node_name]['C_param'] = fx_graph[MHA_node]['C_param'] + fx_graph[out_proj_node_name]['K_param'] = fx_graph[MHA_node]['K_param'] + fx_graph[out_proj_node_name]['sibling_group_id'] = None + fx_graph[out_proj_node_name]['coparent_group_id'] = None + + # set permutation flags + fx_graph[out_proj_node_name]['C_permutable'] = False + fx_graph[MHA_node]['K_permutable'] = False + fx_graph[MHA_node]['C_permutable'] = True + fx_graph[out_proj_node_name]['K_permutable'] = True + fx_graph[out_proj_node_name]['K_passthru'] = False + fx_graph[out_proj_node_name]['C_permuted'] = False + fx_graph[out_proj_node_name]['K_permuted'] = False + fx_graph[out_proj_node_name]['is_real'] = True + + if verbosity > 2: + print(f"\tUpdated: {MHA_node}: {fx_graph[MHA_node]}") + print(f"\tAdded: {out_proj_node_name}: {fx_graph[out_proj_node_name]}") + + # update any nodes that thought their parent was the MHA node + for node in fx_graph.keys(): + parents = fx_graph[node]['parents'] + if node != out_proj_node_name and MHA_node in parents: + parents.remove(MHA_node) + parents.append(out_proj_node_name) + fx_graph[node]['parents'] = parents + if verbosity > 2: + print(f"\tUpdated parents of {node}: {fx_graph[node]}") + + return fx_graph + + @staticmethod + def init_grouped_conv_permutation_flags(fx_graph, node_name, node_groups, verbosity): + """Handle grouped convolutions to make dimensions match""" + + node_C = int(fx_graph.get(node_name).get('C_param')) + node_K = int(fx_graph.get(node_name).get('K_param')) + node_groups = int(node_groups) + + if verbosity > 2: + print(f"\t{node_name} pre-divide C: {node_C}, K: {node_K}, G: {node_groups}") + assert node_C % node_groups == 0 + node_C = int(node_C / node_groups) + fx_graph[node_name]['C_param'] = str(node_C) + if verbosity > 2: + print(f"\t{node_name} post-divide C: {node_C}, K: {node_K}, G: {node_groups}") + + if node_C == 1: # G == C (C is pre-divided by G) + if node_groups == node_K: # true depthwise, G == C == K (C will be pre-divided by G) + fx_graph[node_name]['K_permutable'] = True + fx_graph[node_name]['K_permuted'] = False + fx_graph[node_name]['K_passthru'] = True + fx_graph[node_name]['is_real'] = False + #else: # G != K, handling a permutation along K would be very tricky and not likely useful + + else: # G != C + if node_C > 4 and node_C % 4 == 0: # permutations only help if there's more than one 2:4 pruning group + fx_graph[node_name]['C_permutable'] = True + fx_graph[node_name]['C_permuted'] = False + + + @classmethod + def init_permutation_flags(cls, fx_graph): + """Set the permutation flags for each node based only on that node's module type and parameters""" + + if cls.__verbosity > 0: + print("\n[init_permutation_flags] Initialize the permutation flags for each node according to module type and parameters") + + # initialize some graph-wide trackers + cls.__group_data = {} + cls.__group_data['next_sibling_group_id'] = 0 + cls.__group_data['next_coparent_group_id'] = 0 + cls.__group_data['sibling_groups'] = {} + cls.__group_data['sibling_group_permutations'] = {} + cls.__group_data['sibling_group_C_params'] = {} + cls.__group_data['skipped_sibling_groups'] = {} + cls.__group_data['coparent_groups'] = {} + cls.__group_data['skipped_coparent_groups'] = {} + + # track MHA nodes + MHA_nodes = [] + + # initialize each node's details + for node_name in fx_graph.keys(): + fx_node = fx_graph.get(node_name) + node_module_type = fx_node.get('module_type') + if cls.__verbosity > 1: + if node_module_type == 'get_attr': + print(f"Initializing node {node_name} of type {node_module_type}") + else: + print(f"Initializing node {node_name} of type {node_module_type}: {fx_node}") + + # default for all nodes: don't allow anything + if node_module_type is not None: + fx_graph[node_name]['C_permutable'] = False # does this node have parameters that can be permuted in C + fx_graph[node_name]['K_permutable'] = False # does this node have parameters that can be permuted in K + fx_graph[node_name]['K_passthru'] = False # does this node need to pass a K permutation to its parents + fx_graph[node_name]['is_real'] = False + fx_graph[node_name]['C_permuted'] = False + fx_graph[node_name]['K_permuted'] = False + + # initialize sibling and coparent groups + fx_graph[node_name]['sibling_group_id'] = None + fx_graph[node_name]['coparent_group_id'] = None + + # update each node to be more permissive if supported + if node_module_type in cls.__permutation_target_module_types: + fx_graph[node_name]['is_real'] = True + node_groups = fx_graph.get(node_name).get('groups_param') + + if (node_groups in ['None', '1']): # no groups, no constraints + fx_graph[node_name]['C_permutable'] = True + fx_graph[node_name]['K_permutable'] = True + + else: # handle groups + Permutation.init_grouped_conv_permutation_flags(fx_graph, node_name, node_groups, cls.__verbosity) + + elif node_module_type in cls.__permute_K_and_passthru_module_types: + fx_graph[node_name]['K_permutable'] = True + fx_graph[node_name]['K_passthru'] = True + fx_graph[node_name]['is_real'] = False + + elif node_module_type in cls.__simple_passthru_module_types: + fx_graph[node_name]['K_passthru'] = True + fx_graph[node_name]['is_real'] = False + + elif node_module_type in cls.__disallow_permutations_module_types: + fx_graph[node_name]['is_real'] = True + fx_graph[node_name]['C_param'] = 1 + fx_graph[node_name]['K_param'] = 1 + fx_graph[node_name]['groups_param'] = 1 + + elif 'activation' in node_module_type: + if cls.__verbosity > 0: + print(f"WARNING: how should permutation flags be initialized for node {node_name} of module type {node_module_type}? Found 'activation', assuming simple passthru behavior.") + fx_graph[node_name]['K_passthru'] = True + fx_graph[node_name]['is_real'] = False + + else: + if cls.__verbosity > 0: + print(f"WARNING: how should permutation flags be initialized for node {node_name} of module type {node_module_type}? Defaulting to strict, disallowing permutations around it.") + # is_real coupled with disallowed C and K permutations will poison real parents and real children + fx_graph[node_name]['is_real'] = True + # dummy entries: + fx_graph[node_name]['C_param'] = 1 + fx_graph[node_name]['K_param'] = 1 + fx_graph[node_name]['groups_param'] = 1 + + # MHA nodes only handle the in_proj, need to add out_proj nodes explicitly + # keep track here so we can iterate directly and change fx_graph keys + if node_module_type == 'torch.nn.modules.activation.MultiheadAttention': + MHA_nodes.append(node_name) + + if cls.__verbosity > 1: + if node_module_type == 'get_attr': + print(f"\tInitialized node {node_name} of type {node_module_type}") + else: + print(f"\tInitialized node {node_name} of type {node_module_type}: {fx_graph[node_name]}") + + for MHA_node in MHA_nodes: + fx_graph = Permutation.insert_MHA_out_proj(fx_graph, MHA_node, cls.__verbosity) + + return fx_graph + + @staticmethod + def collect_siblings(fx_graph, node_name, all_siblings): + """Recursively build a set of some node's siblings in the graph""" + + # find all siblings of the requested node + siblings = set() + parents = fx_graph.get(node_name).get('real_parents') + for parent in parents: + children = fx_graph.get(parent).get('real_children') + for child in children: + siblings.add(child) + + # separate the new siblings, since we'll need to process them recursively + new_siblings = siblings.difference(all_siblings) + # update the final list with just the new elements + all_siblings.update(new_siblings) + + for new_sibling in new_siblings: + all_siblings = Permutation.collect_siblings(fx_graph, new_sibling, all_siblings) + + return all_siblings + + @staticmethod + def propagate_sibling_group(fx_graph, all_siblings, verbosity): + """Check a sibling group for ability to be permuted, disallow all siblings and coparents if there's an issue""" + + made_change = False + allow_C = True + for sibling in all_siblings: + pre_check = allow_C + allow_C = allow_C and fx_graph[sibling]['C_permutable'] + if allow_C != pre_check: + if verbosity > 2: + if fx_graph[sibling]['module_type'] == 'get_attr': + print(f"\tnode {sibling} has poisoned the sibling group of {all_siblings}") + else: + print(f"\tnode {sibling} has poisoned the sibling group of {all_siblings}: {fx_graph[sibling]}") + break + + if not allow_C: + for sibling in all_siblings: + made_change = made_change or fx_graph[sibling]['C_permutable'] + fx_graph[sibling]['C_permutable'] = False + + # only disable permutation along K for parents if this node cannot passthru, either + if not fx_graph[sibling]['K_passthru']: + sibling_parents = fx_graph[sibling]['real_parents'] + for sibling_parent in sibling_parents: + made_change = made_change or fx_graph[sibling_parent]['K_permutable'] or fx_graph[sibling_parent]['K_passthru'] + fx_graph[sibling_parent]['K_permutable'] = False + fx_graph[sibling_parent]['K_passthru'] = False + + return made_change + + @staticmethod + def collect_coparents(fx_graph, node_name, all_coparents): + """Recursively build a set of all coparents of a particular node in the graph""" + + # find all coparents of the requested node + coparents = set() + children = fx_graph.get(node_name).get('real_children') + for child in children: + parents = fx_graph.get(child).get('real_parents') + for parent in parents: + coparents.add(parent) + + # coparents are used to restrict what nodes can be permuted along C, so we need to track if the current parents also pass their K permutations up + if fx_graph[parent]['K_passthru']: + grandparents = fx_graph[parent]['real_parents'] + for grandparent in grandparents: + coparents = coparents.union(Permutation.collect_coparents(fx_graph, grandparent, coparents)) + + # separate the new coparents, since we'll need to process them recursively + new_coparents = coparents.difference(all_coparents) + # update the final list with just the new elements + all_coparents.update(new_coparents) + + for new_coparent in new_coparents: + all_coparents = Permutation.collect_coparents(fx_graph, new_coparent, all_coparents) + + return all_coparents + + @staticmethod + def propagate_coparent_group(fx_graph, all_coparents, verbosity): + """Check a coparent group for ability to be permuted, disallow all fellow coparents and children if there's an issue""" + + # see if all coparents agree that K can be permuted + allow_K = True + made_change = False + for coparent in all_coparents: + pre_check = allow_K + allow_K = allow_K and (fx_graph[coparent]['K_permutable'] or fx_graph[coparent]['K_passthru']) + if allow_K != pre_check: + if verbosity > 2: + if fx_graph[coparent]['module_type'] == 'get_attr': + print(f"\tnode {coparent} has poisoned the coparent group of {all_coparents}") + else: + print(f"\tnode {coparent} has poisoned the coparent group of {all_coparents}: {fx_graph[coparent]}") + break + + # if anyone says no, force everyone to 'no', keep track of updated state + if not allow_K: + for coparent in all_coparents: + # all coparents can no longer be permuted along K + if fx_graph[coparent]['K_permutable'] or fx_graph[coparent]['K_passthru']: + made_change = True + + fx_graph[coparent]['K_permutable'] = False + fx_graph[coparent]['K_passthru'] = False + + # children of coparents can't be permuted along C + coparent_children = fx_graph[coparent]['real_children'] + for coparent_child in coparent_children: + if fx_graph[coparent_child]['C_permutable']: + fx_graph[coparent_child]['C_permutable'] = False + made_change = True + + return made_change + + @classmethod + def fixup_concats(cls, fx_graph): + """concat operations/modules may concatenate along the channel dimension, which requires special handling (like grouped convs)""" + + if cls.__verbosity > 0: + print("[fixup_concats]") + + for node_name in fx_graph.keys(): + fx_node = fx_graph[node_name] + if fx_node.get('module_type') == 'concat': + # get real parents, find GCD of their Ks + node_real_parents = fx_node['real_parents'] + + # some concats are at the front of networks (googlenet) + if len(node_real_parents) == 0: + continue + + parents_K_params = [] + for parent in node_real_parents: + parent_K_param = int(fx_graph[parent]['K_param']) + parents_K_params.append(parent_K_param) + fx_graph[parent]['allow_K_mismatch'] = 'concat op' + + # if grouped convolutions make the input channels different among siblings different sizes, + # restrict the permutation atom to the greatest common divisor so it can be tiled as needed for each sibling (and parent) + if cls.__verbosity > 2: + print(f"\tfixing up concat node {node_name}, found parents' {node_real_parents} Ks: {parents_K_params}") + + children_GCD_param = str(np.gcd.reduce(parents_K_params)) + + # set this to GCD of children's sibling group + sibling_group_id = -1 + node_real_children = fx_node['real_children'] + for child in node_real_children: + sibling_group_id = fx_graph[child]['sibling_group_id'] + fx_graph[child]['C_param'] = children_GCD_param + + old_children_GCD = cls.__group_data['sibling_group_C_params'][sibling_group_id] + cls.__group_data['sibling_group_C_params'][sibling_group_id] = children_GCD_param + + # fixup this node's dimensions + # use the functionality of grouped convolutions + fx_node['C_param'] = children_GCD_param + fx_node['K_param'] = old_children_GCD + fx_node['groups_param'] = str(int(old_children_GCD) // int(children_GCD_param)) + + if cls.__verbosity > 2: + print(f"\tfixed up concat node {node_name}, found GCD of parents' {node_real_parents} K to be {children_GCD_param}, updated children's {node_real_children} C_params and sibling group {sibling_group_id} GCD") + print(f"\tthis node now: {fx_node}") + + return fx_graph + + @classmethod + def enforce_dimension_agreement(cls, fx_graph): + """Check all nodes' channel dimensions against parents and children to make sure they agree; e.g. flatten ops may change these dimensions""" + + if cls.__verbosity > 0: + print("[enforce_dimension_agreement]") + + for node_name in fx_graph.keys(): + fx_node = fx_graph[node_name] + if 'is_real' in fx_node.keys() and fx_node['is_real']: + + # enforce this node's input dimension matches its parents' output dimensions + node_C = int(fx_node['C_param']) + node_K = int(fx_node['K_param']) + + if fx_graph[node_name]['groups_param'] not in ['1', 'None']: + node_C = node_C * int(fx_node['groups_param']) + + node_real_parents = fx_node['real_parents'] + if len(node_real_parents) == 0: + if cls.__verbosity > 1: + print(f"\t{node_name} has no real parents, disabling permutations along C") + fx_graph[node_name]['C_permutable'] = False + else: + for real_parent in node_real_parents: + parent_K = int(fx_graph[real_parent]['K_param']) + ignore_mismatch = fx_graph[real_parent].get('allow_K_mismatch') + + if ignore_mismatch is not None: + if cls.__verbosity > 1: + print(f"\tIgnoring dimension mismatch between {node_name} (C={node_C}) and its parent {real_parent} (K={parent_K}) as requested: {ignore_mismatch}") + + elif parent_K >= 0 and node_C != parent_K: + if cls.__verbosity > 1: + print(f"\tDimensions mismatch between {node_name} (C={node_C}) and its parent {real_parent} (K={parent_K}), disallowing the relevant permutations") + + fx_graph[node_name]['C_permutable'] = False + fx_graph[real_parent]['K_permutable'] = False + + if cls.__verbosity > 2: + print(f"\t{fx_graph[node_name]}\n\t{fx_graph[real_parent]}") + + if len(fx_graph[node_name]['real_children']) == 0: + if cls.__verbosity > 1: + print(f"\t{node_name} has no real children, disabling permutations along K") + fx_graph[node_name]['K_permutable'] = False + + return fx_graph + + @classmethod + def make_sibling_coparent_groups(cls, fx_graph): + """Traverse all real nodes in the graph and collect their siblings and coparents""" + + if cls.__verbosity > 0: + print("[make_sibling_coparent_groups]") + + for node_name in fx_graph.keys(): + fx_node = fx_graph[node_name] + + if 'is_real' in fx_node.keys() and fx_node['is_real']: + + sibling_group_id = fx_node['sibling_group_id'] + if sibling_group_id is None: # need to make a new sibling group for this node + all_siblings = cls.collect_siblings(fx_graph, node_name, set([node_name])) + all_siblings = sorted(all_siblings) # deterministic order for DDP setups + sibling_group_id = cls.__group_data['next_sibling_group_id'] + cls.__group_data['sibling_groups'][sibling_group_id] = all_siblings + cls.__group_data['next_sibling_group_id'] = sibling_group_id + 1 + + sibling_group_C_params = [] + for sibling in all_siblings: + fx_graph[sibling]['sibling_group_id'] = sibling_group_id + sibling_C_param = int(fx_graph[sibling]['C_param']) + sibling_group_C_params.append(sibling_C_param) + + # if grouped convolutions make the input channels different among siblings different sizes, + # restrict the permutation atom to the greatest common divisor so it can be tiled as needed for each sibling (and parent) + sibling_group_C_param = str(np.gcd.reduce(sibling_group_C_params)) + cls.__group_data['sibling_group_C_params'][sibling_group_id] = sibling_group_C_param + cls.__group_data['skipped_sibling_groups'][sibling_group_id] = None + + if cls.__verbosity > 1: + print(f"New sibling group {sibling_group_id} with GCD(C) of {sibling_group_C_param}: {all_siblings}") + + + coparent_group_id = fx_node['coparent_group_id'] + if coparent_group_id is None: + all_coparents = cls.collect_coparents(fx_graph, node_name, set([node_name])) + coparent_group_id = cls.__group_data['next_coparent_group_id'] + cls.__group_data['coparent_groups'][coparent_group_id] = all_coparents + cls.__group_data['next_coparent_group_id'] = coparent_group_id + 1 + cls.__group_data['skipped_coparent_groups'][coparent_group_id] = None + + for coparent in all_coparents: + fx_graph[coparent]['coparent_group_id'] = coparent_group_id + + if cls.__verbosity > 1: + print(f"New coparent group {coparent_group_id}: {all_coparents}") + return fx_graph + + @classmethod + def propagate_permutation_flags(cls, fx_graph): + """Disallow sibling groups from having different C_permutable flags and coparent groups from having different K_permutable flags within the groups""" + + made_change = True # will we need to repeat this propagation? + # TODO: just propagate to sibling groups and coparent groups directly, instead of iteratively to direct real_parents and siblings + while made_change: + made_change = False + + if cls.__verbosity > 0: + print("Making a pass at propagating permutation flags") + + for node_name in fx_graph.keys(): + + fx_node = fx_graph.get(node_name) + + node_parents = fx_graph.get(node_name).get('parents') + node_real_parents = fx_graph.get(node_name).get('real_parents') + node_children = fx_graph.get(node_name).get('children') + node_real_children = fx_graph.get(node_name).get('real_children') + + # input layers can't be permuted along C without a runtime fixup, skip them + if node_parents is None or ('x' in node_parents and 'C_permutable' in fx_graph[node_name].keys() and fx_graph[node_name]['C_permutable']): + if cls.__verbosity > 1: + print(f"{node_name} has no parents, or only an input, disabling permutations in C") + made_change = True + fx_graph[node_name]['C_permutable'] = False + + # output layers can't be permuted along K without a runtime fixup, skip them + if node_children is None or ('output' in node_children and 'K_permutable' in fx_graph[node_name].keys() and fx_graph[node_name]['K_permutable']): + if cls.__verbosity > 1: + print(f"{node_name} has no children, or only an output, disabling permutations in K") + made_change = True + fx_graph[node_name]['K_permutable'] = False + fx_graph[node_name]['K_passthru'] = False + + if 'is_real' in fx_node.keys() and fx_node['is_real']: + # siblings must share C-flags; if one cannot be permuted along C, none can + sibling_group_id = fx_graph[node_name]['sibling_group_id'] + all_siblings = cls.__group_data['sibling_groups'][sibling_group_id] + made_change = cls.propagate_sibling_group(fx_graph, all_siblings, cls.__verbosity) or made_change + + # coparents must share K-flags; if one cannot be permuted along K, none can + coparent_group_id = fx_graph[node_name]['coparent_group_id'] + all_coparents = cls.__group_data['coparent_groups'][coparent_group_id] + made_change = cls.propagate_coparent_group(fx_graph, all_coparents, cls.__verbosity) or made_change + + return fx_graph + + + @classmethod + def find_node_real_children(cls, fx_graph, node_name, found_children): + """Collect the real children of some node""" + + if 'real_children' in fx_graph[node_name].keys(): + return found_children.union(fx_graph[node_name]['real_children']) + + children = fx_graph[node_name]['children'] + for child in children: + if child in fx_graph.keys(): # not the output node + if cls.__verbosity > 3: + print(f"\tchecking child {child} of node {node_name}") + + # if it's a real node, just add it + if 'is_real' in fx_graph[child].keys() and fx_graph[child]['is_real']: + found_children.add(child) + else: # otherwise, search its children + found_children = cls.find_node_real_children(fx_graph, child, found_children) + + return found_children + + @classmethod + def find_real_children(cls, fx_graph): + """Collect the real children of all nodes in the graph""" + + if cls.__verbosity > 0: + print("\n[find_real_children] Find the real children for each node according to the whole network graph built with Torch.FX") + + reversible_fx_graph_keys = list(fx_graph.keys()) + for node_name in reversed(reversible_fx_graph_keys): # as the optimization, we need to find the real children from back to front, to use the already saved 'real_children' + node_children = fx_graph.get(node_name).get('children') + + if cls.__verbosity > 2: + print("[find_real_children] node_name: \'{:}\', children: {:}".format(node_name, node_children)) + + real_children = cls.find_node_real_children(fx_graph, node_name, set()) + + if cls.__verbosity > 1: + print(f"[find_real_children] {node_name} has {len(real_children)} real children: {real_children}") + + fx_graph[node_name]['real_children'] = sorted(real_children) + + if cls.__save_permutation_graph: + cls.save_graph_to_json(fx_graph, save_dumped_graph_path_with_name=os.path.join(cls.__permutation_output_dir, './model_graph_find_real_children.json')) # save the intermediate graph as JSON file for debugging + return fx_graph + + @classmethod + def find_node_real_parents(cls, fx_graph, node_name, found_parents): + """Collect the real parents of some node""" + + if 'real_parents' in fx_graph[node_name].keys(): + return found_parents.union(fx_graph[node_name]['real_parents']) + + parents = fx_graph[node_name]['parents'] + for parent in parents: + if parent in fx_graph.keys(): # not the input node + if cls.__verbosity > 3: + print(f"\tchecking parent {parent} of node {node_name}") + + # if it's a real node, just add it + if 'is_real' in fx_graph[parent].keys() and fx_graph[parent]['is_real']: + found_parents.add(parent) + else: # otherwise, search its parents + found_parents = cls.find_node_real_parents(fx_graph, parent, found_parents) + + return found_parents + + + @classmethod + def find_real_parents(cls, fx_graph): + """Collect the real parents of all nodes in the graph""" + + if cls.__verbosity > 0: + print("\n[find_real_parents] Find the real parents for each node according to the whole network graph built with Torch.FX") + + for node_name in fx_graph.keys(): + node_real_parents_name = [] + node_real_parents_module_type = [] + + real_parents = cls.find_node_real_parents(fx_graph, node_name, set()) + + if cls.__verbosity > 1: + print(f"[find_real_parents] {node_name} has {len(real_parents)} real parents: {real_parents}") + + fx_graph[node_name]['real_parents'] = sorted(real_parents) + + if cls.__save_permutation_graph: + cls.save_graph_to_json(fx_graph, save_dumped_graph_path_with_name=os.path.join(cls.__permutation_output_dir, './model_graph_find_real_parent.json')) # save the intermediate graph as JSON file for debugging + return fx_graph + + @classmethod + def build_fx_graph(cls, model, dump_fx_graph=False, save_dumped_fx_graph='./model_fx_graph.json'): + """Build the whole network graph with Torch.FX.""" + + network_fx_graph = {} + success = True + torch_version = str(torch.__version__) + torch_version_major = int(torch_version.split('.')[0]) + torch_version_minor = int(torch_version.split('.')[1]) + try: + torch_version_minimum = int(torch_version.split('.')[2]) + except ValueError: # support the none standard version + torch_version_minimum = torch_version.split('.')[2] + if cls.__verbosity > 2: + print("[build_fx_graph] The torch version is: {}, version major is: {}, version minor is: {}, version minimum is: {}".format(torch_version, torch_version_major, torch_version_minor, torch_version_minimum)) + + if torch_version_major >= 2 or (torch_version_major >= 1 and torch_version_minor >= 8): + if cls.__verbosity > 1: + print("[build_fx_graph] The Torch.FX is supported.") + else: # Torch.FX is introduced in torch 1.8.0 + if cls.__verbosity >= 0: + print("[build_fx_graph] The Torch.FX is not supported. So cannot build the Torch.FX graph.") + success = False + return network_fx_graph, success + + if cls.__verbosity > 2: + print("\n[build_fx_graph] Print the model structure with pure PyTorch function") + print(model) + + graph_module = cls.trace_and_print_raw_fx_graph(model, print_tabular=cls.__verbosity > 1) # needs "tabulate" library + if graph_module is None: + success = False + return network_fx_graph, success + + if cls.__verbosity > 0: + print("\n[build_fx_graph] Build the module name and type dictionary") + + module_name_type_dict = {} + module_name_group_conv_dict = {} + module_name_C_dict = {} + module_name_K_dict = {} + for name, mod in model.named_modules(): + if cls.__verbosity > 1: + print("[build_fx_graph] module_name: {}, module type: {}".format(name, type(mod))) + module_name_type_dict[name] = str(type(mod)).split("\'")[1] + try: + module_name_C_dict[name] = str(mod.in_channels) + except: + try: + module_name_C_dict[name] = str(mod.in_features) + except: + try: + module_name_C_dict[name] = str(mod.embed_dim) + except: + module_name_C_dict[name] = 'None' + + try: + module_name_K_dict[name] = str(mod.out_channels) + except: + try: + module_name_K_dict[name] = str(mod.out_features) + except: + try: + module_name_K_dict[name] = str(mod.embed_dim) + except: + module_name_K_dict[name] = 'None' + + try: + module_name_group_conv_dict[name] = str(mod.groups) + if cls.__verbosity > 1: + print("[build_fx_graph] this module has \'group\' param with value: {}".format(mod.groups)) + except: + module_name_group_conv_dict[name] = 'None' + continue + + # keep track of children and parents for each layer (could be call_module or call_function) + if cls.__verbosity > 0: + print("\n[build_fx_graph] Print the children and parents relationship for each layer") + network_fx_graph = {} + for node in graph_module.graph.nodes: + if node.op == 'placeholder': + if cls.__verbosity > 2: + print("[build_fx_graph] This is the \'input\' node: {:}".format(node.target)) + continue + elif node.op == 'get_attr': + if cls.__verbosity > 2: + print("[build_fx_graph] This is the \'get_attr\' node: {:}".format(node.target)) + node_parent, node_children = get_node_parent_children(node) + converted_node_name=convert_fx_node_name(node.target) + + network_fx_graph[converted_node_name] = {} + network_fx_graph[converted_node_name]['parents'] = node_parent + network_fx_graph[converted_node_name]['children'] = node_children + network_fx_graph[converted_node_name]['module_type'] = 'get_attr' + network_fx_graph[converted_node_name]['groups_param'] = 'None' + + # inspired by https://pytorch.org/docs/stable/fx.html + def fetch_attr(target : str, mod): + target_atoms = target.split('.') + attr_itr = mod + for i, atom in enumerate(target_atoms): + if not hasattr(attr_itr, atom): + raise RuntimeError(f"Node referenced nonexistant target {'.'.join(target_atoms[:i])}") + attr_itr = getattr(attr_itr, atom) + return attr_itr + + attr = fetch_attr(node.target, graph_module) + network_fx_graph[converted_node_name]['C_param'] = 1 + network_fx_graph[converted_node_name]['K_param'] = -1 + network_fx_graph[converted_node_name]['attr'] = attr + + elif node.op == 'call_function': # e.g. 'adaptive.avg.pool2d', 'add', 'cat', 'flatten', 'floordiv', 'getattr', 'getitem', 'hardsigmoid', 'mean', 'mul', 'relu', 'transpose' + node_parent, node_children = get_node_parent_children(node) + converted_node_name=convert_fx_node_name(node.name) + if cls.__verbosity > 2: + print("[build_fx_graph] This is the \'call_function\' node: {:}, its parent list: {:}, its children list: {:}".format(converted_node_name, node_parent, node_children)) + network_fx_graph[converted_node_name] = {} + network_fx_graph[converted_node_name]['parents'] = node_parent + network_fx_graph[converted_node_name]['children'] = node_children + network_fx_graph[converted_node_name]['fx_op'] = 'call_function' + + ### "convert" some ops to modules + + # concatenating along K can be handled by reducing the size of the childrens' C appropriately + # see fixup_concats, if no dim arg, default is 0 (handled automatically) + if node.target == torch.cat and len(node.args) > 1 and node.args[1] == 1: + network_fx_graph[converted_node_name]['fx_op'] = 'call_module' + network_fx_graph[converted_node_name]['module_type'] = 'concat' + network_fx_graph[converted_node_name]['groups_param'] = 'N/A' # just need placeholders + network_fx_graph[converted_node_name]['C_param'] = 'N/A' + network_fx_graph[converted_node_name]['K_param'] = 'N/A' + + elif node.op == 'call_method': # e.g. 'chunk', 'contiguous', 'mean', 'size', 'unsqueeze', 'view' + node_parent, node_children = get_node_parent_children(node) + converted_node_name=convert_fx_node_name(node.name) + if cls.__verbosity > 2: + print("[build_fx_graph] This is the \'call_method\' node: {:}, its parent list: {:}, its children list: {:}".format(converted_node_name, node_parent, node_children)) + network_fx_graph[converted_node_name] = {} + network_fx_graph[converted_node_name]['parents'] = node_parent + network_fx_graph[converted_node_name]['children'] = node_children + network_fx_graph[converted_node_name]['fx_op'] = 'call_method' + continue + + elif node.op == 'call_module': + node_parent, node_children = get_node_parent_children(node) + converted_node_name=convert_fx_node_name(node.name) + # check whether the converted_node_name is same as node.target, especially for ReLU case + if converted_node_name != node.target: + if cls.__verbosity > 2: + print("[build_fx_graph][warning] The target name from Torch.FX is \'{:}\', the manually converted node name is \'{:}\', not the same one, choose the converted node name".format(node.target, converted_node_name)) + + # assume the modules share the same target name have the same type, because converted_node_name may not be obtained by model.named_modules(), like some ReLU (defined in forward function) + node_type = module_name_type_dict[node.target] + if cls.__verbosity > 2: + print("[build_fx_graph] This is the \'call_module\' node: {:}, its parent list: {:}, its children list: {:}, its type: {:}".format(converted_node_name, node_parent, node_children, node_type)) + network_fx_graph[converted_node_name] = {} + network_fx_graph[converted_node_name]['parents'] = node_parent + network_fx_graph[converted_node_name]['children'] = node_children + network_fx_graph[converted_node_name]['fx_op'] = 'call_module' + network_fx_graph[converted_node_name]['module_type'] = node_type + network_fx_graph[converted_node_name]['groups_param'] = module_name_group_conv_dict[node.target] + network_fx_graph[converted_node_name]['C_param'] = module_name_C_dict[node.target] + network_fx_graph[converted_node_name]['K_param'] = module_name_K_dict[node.target] + + + elif node.op == 'output': + if cls.__verbosity > 2: + print("[build_fx_graph] This is the \'output\' node: {:}".format(node.target)) + continue + + if dump_fx_graph: + if cls.__verbosity > 0: + print("\n[build_fx_graph] Dump the overall dict for children and parents relationship into JSON file") + cls.save_graph_to_json(network_fx_graph, save_dumped_graph_path_with_name=save_dumped_fx_graph) + + return network_fx_graph, success + + @classmethod + def trace_and_print_raw_fx_graph(cls, model, print_tabular=False, generate_python_code=False): + """This function is used to find and print the intermediate representation (IR) - Graph representation with Torch.FX features.""" + + from torch.fx import symbolic_trace + import traceback + + # Symbolic tracing frontend - captures the semantics of the module + try: + symbolic_traced : torch.fx.GraphModule = symbolic_trace(model) + except Exception as ex: + if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + if cls.__verbosity > 0: + print(ex) + print(''.join(traceback.format_exception(etype=type(ex), value=ex, tb=ex.__traceback__))) + print("\n[print_raw_fx_graph] Meet the fatal fault when trying to symbolic trace the model with Torch.FX") + return None + + # High-level intermediate representation (IR) - Graph representation + if cls.__verbosity > 1: + print("\n[print_raw_fx_graph] Print the intermediate representation (IR) with Torch.FX") + print(symbolic_traced.graph) + + if print_tabular: + print("\n[print_raw_fx_graph] Print the intermediate representation (IR) with Torch.FX in a table format") + try: + from tabulate import tabulate + symbolic_traced.graph.print_tabular() + except ImportError: + if cls.__verbosity > 1: + print("[print_raw_fx_graph][Warning] \'print_tabular\' relies on the library `tabulate`; run `pip install tabulate` to install it.") + except AttributeError: # to avoid the AttributeError: 'Graph' object has no attribute 'print_tabular' + if cls.__verbosity > 1: + print("[print_raw_fx_graph][Warning] \'print_tabular\' function is not supported in current Torch version. Skip!") + + # Code generation - valid Python code + if generate_python_code: + print("\n[print_raw_fx_graph] Create valid Python code matching the IR/Graph's semantics with Torch.FX") + print(symbolic_traced.code) + + return symbolic_traced + + @classmethod + def save_graph_to_json(cls, graph, save_dumped_graph_path_with_name='./model_fx_graph.json'): + """This function is used to save the graph into JSON file for inspection.""" + + # use dumps to transfer the dict to JSON string + json_graph_str = json.dumps(graph) + with open(save_dumped_graph_path_with_name, 'w', encoding='utf-8') as dumped_graph_file: + dumped_graph_file.write(json_graph_str) # write the transferred JSON string into JSON file diff --git a/apex/apex/contrib/sparsity/permutation_search_kernels/CUDA_kernels/permutation_search_kernels.cu b/apex/apex/contrib/sparsity/permutation_search_kernels/CUDA_kernels/permutation_search_kernels.cu new file mode 100644 index 00000000..cdbd109b --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_search_kernels/CUDA_kernels/permutation_search_kernels.cu @@ -0,0 +1,701 @@ +#include +#include +#include +namespace py = pybind11; + +#define gpuErrchk(ans) { gpuAssert((ans), __FILE__, __LINE__); } +inline void gpuAssert(cudaError_t code, const char *file, int line, bool abort=true) +{ + if (code != cudaSuccess) + { + fprintf(stderr,"GPUassert %d: %s %s %d\n", (int)code, cudaGetErrorString(code), file, line); + if (abort) exit(code); + } +} + + +// find the magnitude after enforcing the 2:4 sparsity constraint on a group of 4 values +__device__ float group_2_to_4(float4 vals) +{ + vals.x = fabs(vals.x); + vals.y = fabs(vals.y); + vals.z = fabs(vals.z); + vals.w = fabs(vals.w); + + float sum0 = vals.x + vals.y; + float sum1 = vals.x + vals.z; + float sum2 = vals.x + vals.w; + float sum3 = vals.y + vals.z; + float sum4 = vals.y + vals.w; + float sum5 = vals.z + vals.w; + + float best_sum0 = fmax(sum0, sum1); + float best_sum1 = fmax(sum2, sum3); + float best_sum2 = fmax(sum4, sum5); + float best_sum = fmax(fmax(best_sum0, best_sum1), best_sum2); + + return best_sum; +} + +inline float* float_ptr_from_numpy(py::array_t& py_float) +{ + return (float*)py_float.data(); +} + +inline unsigned int* uint_ptr_from_numpy(py::array_t& py_uint) +{ + return (unsigned int*)py_uint.data(); +} + +/********************************************************** +* Check for the best permutation for an entire matrix +**********************************************************/ +__global__ void permute_and_sum_after_2_to_4(float* matrix, + unsigned int rows, + unsigned int cols, + unsigned int* stripes, + unsigned int total_stripes, + unsigned int* permutations, + float* output) +{ + // vectorize + float4* mat4 = (float4*) matrix; + cols /= 4; + + // each thread in a block takes some number of rows + size_t num_rows = max((int)ceilf((float)rows / (float)blockDim.x), 1); + size_t row_offset = num_rows * threadIdx.x; + size_t num_stripes = total_stripes; // total_stripes / gridDim.x; + size_t stripe_offset = 0; //num_stripes * blockIdx.x; + unsigned int localStart = stripe_offset; + unsigned int localEnd = localStart + num_stripes; + + // each block takes care of one permutation + unsigned int p = blockIdx.x; + unsigned int* permutation = &permutations[p*total_stripes*4]; + + float sum = 0.0f; + extern __shared__ float s[32][32]; + float4* local_stripes = (float4*)&s[threadIdx.x]; + float* local_columns = (float*)&s[threadIdx.x]; + float4* permuted_local_stripes = (float4*)&local_stripes[num_stripes]; + float* permuted_local_columns = (float*)&local_columns[num_stripes*4]; + + for ( unsigned int r = row_offset; r < row_offset + num_rows; ++r) { + if (r >= rows) + break; + + // load into smem + for ( unsigned int s = localStart; s < localEnd; ++s) { + unsigned int stripe = stripes[s]; + local_stripes[s] = mat4[r*cols+stripe]; + } + + // now permute + #pragma unroll 4 + for ( unsigned int c = 0; c < num_stripes*4; ++c) { + permuted_local_columns[c] = local_columns[permutation[c]]; + } + + // now sum 2:4 + for ( unsigned int s = 0; s < num_stripes; ++s) { + sum += group_2_to_4(permuted_local_stripes[s]); + } + } + + atomicAdd(&output[p], sum); +} + +void free_permutation_memory(float** dmatrix, + unsigned int** dstripe_groups, + unsigned int** dpermutations, + float** dresults, + float** hresults) +{ + cudaFree(*dmatrix); + cudaFree(*dresults); + cudaFree(*dpermutations); + cudaFree(*dstripe_groups); + free(*hresults); +} + +int set_up_check_permutation_memory(float** dmatrix, + unsigned int rows, + unsigned int cols, + unsigned int** dstripe_groups, + unsigned int group_width, + unsigned int num_groups, + unsigned int** dpermutations, + unsigned int num_permutations, + float** dresults, + float** hresults) +{ + static unsigned int setupRows = 0; + static unsigned int setupCols = 0; + static unsigned int setupGroupWidth = 0; + static unsigned int setupNumGroups = 0; + static unsigned int setupNumPermutations = 0; + static bool allocated = false; + int fresh_alloc = 0; + if (!allocated || + setupRows != rows || + setupCols != cols || + setupGroupWidth != group_width || + setupNumGroups != num_groups || + setupNumPermutations != num_permutations) { + + if (allocated) { + free_permutation_memory(dmatrix, dstripe_groups, dpermutations, dresults, hresults); + } + + gpuErrchk(cudaMalloc( (void**) dmatrix, rows*cols*sizeof(float))); + gpuErrchk(cudaMalloc( (void**) dstripe_groups, group_width*num_groups*sizeof(unsigned int))); + gpuErrchk(cudaMalloc( (void**) dpermutations, num_permutations*group_width*4*sizeof(unsigned int))); + gpuErrchk(cudaMalloc( (void**) dresults, num_permutations*sizeof(float))); + *hresults = (float*) malloc(num_permutations*sizeof(float)); + + allocated = true; + setupRows = rows; + setupCols = cols; + setupGroupWidth = group_width; + setupNumGroups = num_groups; + setupNumPermutations = num_permutations; + fresh_alloc = 1; + } + + return fresh_alloc; +} + +int run_check_permutations(py::array_t& py_matrix, + unsigned int rows, + unsigned int cols, + py::array_t& py_stripe_groups, // groups of stripes, group_width = stripes per group, num_groups = groups in the array + unsigned int group_width, + unsigned int num_groups, + py::array_t& py_permutations, // array of permutations to try, group_width*4 values per each of num_permutations permutations + unsigned int num_permutations, + py::array_t& py_improvement, // improvment offered by the best permutation + py::array_t& py_permutation // the best permutation + ) +{ + const unsigned int threads = 32; + static float* d_matrix; + static unsigned int* d_permutations; + static unsigned int* d_stripes; + static float* d_results; + static float* results; + + float* matrix = float_ptr_from_numpy(py_matrix); + unsigned int* stripe_groups = uint_ptr_from_numpy(py_stripe_groups); + unsigned int* permutations = uint_ptr_from_numpy(py_permutations); + float* improvement = float_ptr_from_numpy(py_improvement); + unsigned int* permutation = uint_ptr_from_numpy(py_permutation); + + int fresh_alloc = set_up_check_permutation_memory(&d_matrix, rows, cols, &d_stripes, group_width, num_groups, &d_permutations, num_permutations, &d_results, &results); + if (fresh_alloc == 1){ + gpuErrchk(cudaMemcpy( d_permutations, permutations, num_permutations*group_width*4*sizeof(unsigned int), cudaMemcpyHostToDevice)); + gpuErrchk(cudaMemcpy( d_stripes, stripe_groups, group_width*num_groups*sizeof(unsigned int), cudaMemcpyHostToDevice)); + } + + // initialize results, new matrix + gpuErrchk(cudaMemset (d_results, 0, num_permutations*sizeof(float))); + gpuErrchk(cudaMemcpy (d_matrix, matrix, rows*cols*sizeof(float), cudaMemcpyHostToDevice )); + + // get results for all permutations + permute_and_sum_after_2_to_4<<>>(d_matrix, rows, cols, d_stripes, group_width, d_permutations, d_results); + gpuErrchk(cudaDeviceSynchronize()); + + gpuErrchk(cudaMemcpy( results, d_results, num_permutations*sizeof(float), cudaMemcpyDeviceToHost )); + + // find the best permutation - could reduce on GPU + unsigned int best_permutation = 0; + float best_improvement = 0.0f; + for (unsigned int p = 1; p < num_permutations; ++p) { + float cur_improvement = results[p] - results[0]; + if (best_improvement < cur_improvement) { + best_permutation = p; + best_improvement = cur_improvement; + } + } + + *improvement = best_improvement; + *permutation = best_permutation; + + return 0; +} + +/////////////////////////////////////////////////////////// + +/********************************************************** +* Get the magnitude of a matrix after applying 2:4 +**********************************************************/ +// find the magnitude after enforcing the 2:4 sparsity constraint on a subset of the columns of an input matrix +__global__ void subset_sum_after_2_to_4(float* matrix, + unsigned int rows, + unsigned int cols, + unsigned int start_col, + unsigned int end_col, + float* output) +{ + // vectorize + float4* mat4 = (float4*) matrix; + cols /= 4; + start_col /= 4; + end_col /= 4; + + // each thread in a block takes some number of rows + size_t num_rows = max((int)ceilf((float)rows / (float)blockDim.x), 1); + size_t row_offset = num_rows * threadIdx.x; + // each block takes some number of columns + size_t num_cols = (end_col - start_col) / gridDim.x; + size_t col_offset = num_cols * blockIdx.x; + start_col += col_offset; + end_col = start_col + num_cols; + + float sum = 0.0f; + for ( unsigned int r = row_offset; r < row_offset + num_rows; ++r ) { + if (r < rows) { + for ( unsigned int c = start_col; c < end_col; c++ ) { + sum += group_2_to_4(mat4[r * cols + c]); + } + } + } + + atomicAdd(output, sum); +} + + +// build the entire permute map at once +// each block handles one group of stripes +// each threads in the block handle all handle the same permutation at the same time on different rows before moving to the next permutation +__global__ void build_permute_map(float* matrix, + unsigned int rows, + unsigned int cols, + unsigned int* stripes, + unsigned int group_width, + unsigned int* permutations, + unsigned int num_permutations, + unsigned int perm_length, + float* output, + unsigned int* best_indices) +{ + // vectorize + float4* mat4 = (float4*) matrix; + cols /= 4; + + // each block handles a group of stripes + unsigned int* stripe_group = (unsigned int*)&stripes[blockIdx.x*group_width]; + + // shared memory: 32 threads each need 16*2 + extern __shared__ float pm_shared[32][32]; + float4* local_stripes = (float4*)&pm_shared[threadIdx.x]; + float* local_columns = (float*) &pm_shared[threadIdx.x]; + float4* permuted_stripes = (float4*) &local_stripes[4]; + float* permuted_columns = (float*) &local_columns[16]; + + // each thread handles all permutations in the row before moving on to the next row + size_t num_rows = max((int)ceilf((float)rows / (float)blockDim.x), 1); + size_t row_offset = num_rows * threadIdx.x; + + for ( unsigned int r = row_offset; r < row_offset + num_rows; ++r) { + if (r >= rows) + break; + + // load a row into smem + for ( unsigned int s = 0; s < group_width; ++s) { + unsigned int const stripe = stripe_group[s]; + local_stripes[s] = mat4[r*cols+stripe]; + } + + for ( unsigned int p = 0; p < num_permutations; ++p) { + unsigned int* permutation = &permutations[p*perm_length]; + float sum = 0.0f; + + // permute + #pragma unroll 4 + for ( unsigned int c = 0; c < group_width*4; ++c) { + permuted_columns[c] = local_columns[permutation[c]]; + } + + // sum 2:4 + for ( unsigned int s = 0; s < group_width; ++s) { + sum += group_2_to_4(permuted_stripes[s]); + } + + // update the running sum for this stripe group's permutation + atomicAdd(&output[blockIdx.x*num_permutations + p], sum); + } + } + + // at this point, each permutation's sum in this stripe group has been calculated + // now, find the best option + __syncthreads(); + + if (threadIdx.x == 0) { + unsigned int best_permutation = 0; + float best_magnitude = output[blockIdx.x*num_permutations]; + float base_magnitude = best_magnitude; + + //#pragma unroll 32 + for (unsigned int p = 1; p < num_permutations; ++p) { + float magnitude = output[blockIdx.x*num_permutations+p]; + if (magnitude > best_magnitude) { + best_permutation = p; + best_magnitude = magnitude; + } + } + + output[blockIdx.x*num_permutations] = best_magnitude - base_magnitude; + best_indices[blockIdx.x] = best_permutation; + } +} + + +void free_sum_after_2_to_4_memory(float** dmatrix, + float** dresult) +{ + cudaFree(*dmatrix); + cudaFree(*dresult); +} + +int set_up_sum_after_2_to_4_memory(float** dmatrix, + unsigned int rows, + unsigned int cols, + float** dresult) +{ + static unsigned int setupRows = 0; + static unsigned int setupCols = 0; + static bool allocated = false; + + int fresh_allocation = 0; + if (!allocated || + setupRows != rows || + setupCols != cols) + { + if (allocated) + free_sum_after_2_to_4_memory(dmatrix, dresult); + + gpuErrchk(cudaMalloc( (void**) dmatrix, rows*cols*sizeof(float))); + gpuErrchk(cudaMalloc( (void**) dresult, sizeof(float))); + + setupRows = rows; + setupCols = cols; + + fresh_allocation = 1; + } + + allocated = true; + + return fresh_allocation; +} + +int run_subset_sum_after_2_to_4(py::array_t& py_matrix, + unsigned int rows, + unsigned int cols, + unsigned int start_col, + unsigned int end_col, + unsigned int blocks, + unsigned int threads, + py::array_t& py_output) +{ + + static float* d_matrix; + static float* d_result; + + int fresh_allocation = set_up_sum_after_2_to_4_memory(&d_matrix, rows, cols, &d_result); + + float* matrix = float_ptr_from_numpy(py_matrix); + float* output = float_ptr_from_numpy(py_output); + + gpuErrchk(cudaMemcpy( d_matrix, matrix, rows*cols*sizeof(float), cudaMemcpyHostToDevice )); + gpuErrchk(cudaMemset( d_result, 0, sizeof(float))); + + subset_sum_after_2_to_4<<>>(d_matrix, rows, cols, start_col, end_col, d_result); + gpuErrchk(cudaDeviceSynchronize()); + + gpuErrchk(cudaMemcpy( output, d_result, sizeof(float), cudaMemcpyDeviceToHost )); + + return 0; +} + +void set_up_permute_map_memory(float** dmatrix, + unsigned int rows, + unsigned int cols, + unsigned int** dstripes, + unsigned int num_groups, + unsigned int group_width, + unsigned int** dpermutations, + unsigned int num_permutations, + unsigned int perm_length, + float** doutput, + unsigned int** dindices, + float** hresult, + unsigned int** hindices) +{ + static unsigned int setUpRows = 0; + static unsigned int setUpCols = 0; + static unsigned int setUpGroupWidth = 0; + static unsigned int setUpNumGroups = 0; + static unsigned int setUpNumPerms = 0; + static unsigned int setUpPermLength = 0; + + if (setUpRows != rows || + setUpCols != cols) { + if (*dmatrix != NULL) { gpuErrchk(cudaFree(*dmatrix)); *dmatrix = NULL; } + gpuErrchk(cudaMalloc( (void**) dmatrix, rows*cols*sizeof(float))); + } + + if (setUpGroupWidth < group_width || + setUpNumGroups < num_groups) { + if (*dstripes != NULL) { gpuErrchk(cudaFree(*dstripes)); *dstripes = NULL; } + gpuErrchk(cudaMalloc( (void**) dstripes, num_groups*group_width*sizeof(unsigned int))); + + if (setUpNumGroups < num_groups) { + if (*dindices != NULL) { gpuErrchk(cudaFree(*dindices)); *dindices = NULL; } + gpuErrchk(cudaMalloc( (void**) dindices, num_groups*sizeof(unsigned int))); + if (*hindices != NULL) { free(*hindices); *hindices = NULL; } + *hindices = (unsigned int*) malloc (num_groups*sizeof(unsigned int)); + } + } + + if (setUpNumPerms < num_permutations || + setUpPermLength < perm_length) { + if (*dpermutations != NULL) { gpuErrchk(cudaFree(*dpermutations)); *dpermutations = NULL; } + gpuErrchk(cudaMalloc( (void**) dpermutations, perm_length*num_permutations*sizeof(unsigned int))); + } + + if (setUpNumPerms < num_permutations || + setUpNumGroups < num_groups) { + if (*doutput != NULL) { gpuErrchk(cudaFree(*doutput)); *doutput = NULL; } + gpuErrchk(cudaMalloc( (void**) doutput, num_permutations*num_groups*sizeof(float))); + if (*hresult != NULL) { free(*hresult); *hresult = NULL; } + *hresult = (float*) malloc(num_permutations*num_groups*sizeof(float)); + } + + setUpRows = rows; + setUpCols = cols; + setUpGroupWidth = group_width; + setUpNumGroups = num_groups; + setUpNumPerms = num_permutations; + setUpPermLength = perm_length; +} + +int run_build_permute_map(py::array_t& py_matrix, + unsigned int rows, + unsigned int cols, + py::array_t& py_stripes, + unsigned int num_groups, + unsigned int group_width, + py::array_t& py_permutations, + unsigned int perm_length, + py::array_t& py_improvements, + py::array_t& py_best_indices) +{ + static float* d_matrix = NULL; + static unsigned int* d_stripes = NULL; + static unsigned int* d_permutations = NULL; + static float* d_output = NULL; + static unsigned int* d_indices = NULL; + static float* hresult = NULL; + static unsigned int* hindices = NULL; + + const unsigned int num_permutations = py_permutations.size() / perm_length; + + const unsigned int MAX_GROUPS_PER_LAUNCH = num_permutations <= 5775 ? 1820 : 40; + const unsigned int full_launches = num_groups / MAX_GROUPS_PER_LAUNCH; + const unsigned int final_launch = num_groups % MAX_GROUPS_PER_LAUNCH; + const unsigned int launches = full_launches + (final_launch != 0 ? 1 : 0); + + set_up_permute_map_memory(&d_matrix, rows, cols, &d_stripes, min(num_groups,MAX_GROUPS_PER_LAUNCH), group_width, &d_permutations, num_permutations, perm_length, &d_output, &d_indices, &hresult, &hindices); + + float* matrix = float_ptr_from_numpy(py_matrix); + unsigned int* stripes = uint_ptr_from_numpy(py_stripes); + unsigned int* permutations = uint_ptr_from_numpy(py_permutations); + float* improvements = float_ptr_from_numpy(py_improvements); + unsigned int* best_indices = uint_ptr_from_numpy(py_best_indices); + + gpuErrchk(cudaMemcpy( d_matrix, matrix, rows*cols*sizeof(float), cudaMemcpyHostToDevice )); + gpuErrchk(cudaMemcpy( d_permutations, permutations, num_permutations*perm_length*sizeof(unsigned int), cudaMemcpyHostToDevice )); + + unsigned int group_offset = 0; + for (unsigned int l = 0; l < launches; ++l) + { + unsigned int groups_this_launch = (l < full_launches) ? MAX_GROUPS_PER_LAUNCH : final_launch; + + gpuErrchk(cudaMemcpy( d_stripes, &stripes[group_offset*group_width], groups_this_launch*group_width*sizeof(unsigned int), cudaMemcpyHostToDevice )); + gpuErrchk(cudaMemset( d_output, 0, groups_this_launch*num_permutations*sizeof(float))); + gpuErrchk(cudaMemset( d_indices, 0, groups_this_launch*sizeof(unsigned int))); + + unsigned int shmem = 32*(32)*sizeof(float); + build_permute_map<<>>(d_matrix, rows, cols, d_stripes, group_width, d_permutations, num_permutations, perm_length, d_output, d_indices); + gpuErrchk(cudaDeviceSynchronize()); + + gpuErrchk(cudaMemcpy( hresult, d_output, num_permutations*groups_this_launch*sizeof(float), cudaMemcpyDeviceToHost )); + gpuErrchk(cudaMemcpy( hindices, d_indices, groups_this_launch*sizeof(unsigned int), cudaMemcpyDeviceToHost )); + + // thread0 stuck the minimum in the first slot of each group + for (unsigned int g = 0; g < groups_this_launch; ++g) { + improvements[group_offset+g] = hresult[g*num_permutations]; + best_indices[group_offset+g] = hindices[g]; + } + + group_offset += groups_this_launch; + } + + return 0; + +} + +/********************************************************** +* Build the swap map for channel_swaps +**********************************************************/ +// find the magnitude improvement if some columns were swapped (check all pairs of columns in all the stripe_pairs) +__global__ void swap_columns_sum_after_2_to_4(float* matrix, + unsigned int rows, + unsigned int cols, + unsigned int* stripe_pairs, + float* output) +{ + // vectorize + float4* mat4 = (float4*) matrix; + cols /= 4; + + // each thread takes some number of rows + size_t const num_rows = max((int)ceilf((float)rows / (float)blockDim.x), 1); + size_t const row_offset = num_rows * threadIdx.x; + + // each block is repsonsible for a pair of stripes + unsigned int const stripe0 = stripe_pairs[2*blockIdx.x]; + unsigned int const stripe1 = stripe_pairs[2*blockIdx.x+1]; + // space for 32 threads, 8 values (2 stripes) in use at a time, plus 16 partial sums and one base sum + extern __shared__ float cs[32][32]; + float4* local_stripe0 = (float4*)&cs[threadIdx.x][0]; + float* local_cols0 = (float*) &cs[threadIdx.x][0]; + float4* local_stripe1 = (float4*)&cs[threadIdx.x][4]; + float* local_cols1 = (float*) &cs[threadIdx.x][4]; + float* local_psum = (float*) &cs[threadIdx.x][8]; + float* base_psum = (float*) &cs[threadIdx.x][24]; + + *base_psum = 0.0f; + for ( unsigned int s = 0; s < 16; ++s) { + local_psum[s] = 0.0f; + } + + for ( unsigned int r = row_offset; r < row_offset + num_rows; ++r) { + if (r >= rows) + break; + *local_stripe0 = mat4[r*cols+stripe0]; + *local_stripe1 = mat4[r*cols+stripe1]; + *base_psum += group_2_to_4(*local_stripe0) + group_2_to_4(*local_stripe1); + unsigned int swap_idx = 0; + for (unsigned int c0 = 0; c0 < 4; ++c0) { + for (unsigned int c1 = 0; c1 < 4; ++c1) { + // swap c0 and c1 + float tmp = local_cols0[c0]; + local_cols0[c0] = local_cols1[c1]; + local_cols1[c1] = tmp; + + // grab the sum + local_psum[swap_idx] += group_2_to_4(*local_stripe0) + group_2_to_4(*local_stripe1); + + // swap back + local_cols1[c1] = local_cols0[c0]; + local_cols0[c0] = tmp; + + swap_idx++; + } + } + } + + // reduce partial sums, store local diffs in the output + __syncthreads(); + if (threadIdx.x == 0) { + for ( unsigned int t = 1; t < blockDim.x; ++t) { + for (unsigned int swap = 0; swap < 16; ++swap) { + local_psum[swap] += cs[t][8+swap]; + } + *base_psum += cs[t][24]; + } + + for ( unsigned int swap = 0; swap < 16; ++swap) { + atomicAdd(&output[blockIdx.x*16 + swap], local_psum[swap] - (*base_psum)); + } + } +} + +void set_up_swap_map_memory(float** dmatrix, + unsigned int rows, + unsigned int cols, + unsigned int** dstripe_pairs, + unsigned int num_pairs, + float** dresult) +{ + static unsigned int setupRows = 0; + static unsigned int setupCols = 0; + static unsigned int setupPairs = 0; + + if (*dmatrix == NULL || + setupRows != rows || + setupCols != cols) + { + if (*dmatrix != NULL) { gpuErrchk(cudaFree(*dmatrix)); *dmatrix = NULL; } + gpuErrchk(cudaMalloc( (void**) dmatrix, rows*cols*sizeof(float))); + setupRows = rows; + setupCols = cols; + } + + if (*dstripe_pairs == NULL || + *dresult == NULL || + setupPairs < num_pairs) + { + if (*dstripe_pairs != NULL) { gpuErrchk(cudaFree(*dstripe_pairs)); *dstripe_pairs = NULL; } + if (*dresult != NULL) { gpuErrchk(cudaFree(*dresult)); *dresult = NULL; } + gpuErrchk(cudaMalloc( (void**) dstripe_pairs, num_pairs*2*sizeof(unsigned int))); + gpuErrchk(cudaMalloc( (void**) dresult, num_pairs*16*sizeof(float))); + + + setupPairs = num_pairs; + } +} + + +int run_build_swap_map(py::array_t& py_matrix, + unsigned int rows, + unsigned int cols, + py::array_t& py_stripe_pairs, + py::array_t& py_output) +{ + static float* d_matrix = NULL; + static float* d_result = NULL; + static unsigned int* d_stripe_pairs = NULL; + + float* matrix = float_ptr_from_numpy(py_matrix);//(float*)py_matrix.data(); + unsigned int* stripe_pairs = uint_ptr_from_numpy(py_stripe_pairs);//(unsigned int*)py_stripe_pairs.data(); + float* output = float_ptr_from_numpy(py_output);//(float*)py_output.data(); + + unsigned int num_pairs = py_stripe_pairs.size()/2; + + set_up_swap_map_memory(&d_matrix, rows, cols, &d_stripe_pairs, num_pairs, &d_result); + gpuErrchk(cudaMemcpy( d_matrix, matrix, rows*cols*sizeof(float), cudaMemcpyHostToDevice )); + gpuErrchk(cudaMemcpy( d_stripe_pairs, stripe_pairs, 2*num_pairs*sizeof(unsigned int), cudaMemcpyHostToDevice )); + gpuErrchk(cudaMemset( d_result, 0, num_pairs*16*sizeof(float))); + + unsigned int shmem = 32*(32)*sizeof(float); + swap_columns_sum_after_2_to_4<<>>(d_matrix, rows, cols, d_stripe_pairs, d_result); + gpuErrchk(cudaDeviceSynchronize()); + + gpuErrchk(cudaMemcpy( output, d_result, num_pairs*16*sizeof(float), cudaMemcpyDeviceToHost )); + + return 0; +} +/////////////////////////////////////////////////////////// + + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) +{ + m.def("sum_after_2_to_4", &run_subset_sum_after_2_to_4, "matrix sum after applying 2:4 (CUDA)"); + m.def("build_permute_map", &run_build_permute_map, "optimize stripe groups (CUDA)"); + m.def("check_permutations", &run_check_permutations, "exhaustively check all permutations (CUDA)"); + m.def("build_swap_map", &run_build_swap_map, "channel swaps (CUDA)"); +} diff --git a/apex/apex/contrib/sparsity/permutation_search_kernels/__init__.py b/apex/apex/contrib/sparsity/permutation_search_kernels/__init__.py new file mode 100644 index 00000000..bd29c6de --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_search_kernels/__init__.py @@ -0,0 +1,2 @@ +from .call_permutation_search_kernels import accelerated_search_for_good_permutation +from .permutation_utilities import sum_after_2_to_4 \ No newline at end of file diff --git a/apex/apex/contrib/sparsity/permutation_search_kernels/call_permutation_search_kernels.py b/apex/apex/contrib/sparsity/permutation_search_kernels/call_permutation_search_kernels.py new file mode 100644 index 00000000..1570e477 --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_search_kernels/call_permutation_search_kernels.py @@ -0,0 +1,73 @@ +import numpy as np +from .permutation_utilities import * +from .exhaustive_search import Exhaustive_Search + +def accelerated_search_for_good_permutation(matrix_group, options=None, verbosity=0): + """This function is used to call the permutation search CUDA kernels. + users can provide prefer search strategy by providing a valid 'options' as a dictionary, + or users can implement their customized 'accelerated_search_for_good_permutation' function. + """ + input_matrix = matrix_group.cpu().detach().numpy() + if verbosity > 1: + print("\n[accelerated_search_for_good_permutation] input matrix shape: \'{:}\'.".format(input_matrix.shape)) + + result = np.copy(input_matrix) + # init a sequential permutation search sequence + input_channel_num = matrix_group.size(1) + permutation_sequence = [n for n in range(input_channel_num)] + duration = 0.0 + + if options == None: + options = {} + if 'strategy' not in options: # right now, the default permutation search strategy is: 'exhaustive' search + options['strategy'] = 'exhaustive' + + if verbosity > 1: + print("[accelerated_search_for_good_permutation] the permutation strategy is: \'{:} search\'.".format(options['strategy'])) + + # define sub options for each search strategy + if options['strategy'] == 'exhaustive': + # right now, the default options for 'exhaustive' search is: 'exhaustive,8,100' + if 'stripe_group_size' not in options: + options['stripe_group_size'] = 8 + if 'escape_attempts' not in options: + options['escape_attempts'] = 100 + elif options['strategy'] == 'progressive channel swap': + # just swaps meaningful channels, keeping the good swaps, until the search time limit expires. + if 'progressive_search_time_limit' not in options: + options['progressive_search_time_limit'] = 60 + if 'improvement_threshold' not in options: + options['improvement_threshold'] = 1e-9 + + # execute the requested strategy + if options['strategy'] == 'exhaustive': + result, duration, permutation_sequence = Exhaustive_Search(result, stripe_group_size=options['stripe_group_size'], escape_attempts=options['escape_attempts']) + elif options['strategy'] == 'progressive channel swap': + real_swap_num = 0 + start_time = time.perf_counter() + while time.perf_counter() - start_time < options['progressive_search_time_limit']: + src = np.random.randint(result.shape[1]) + dst = np.random.randint(result.shape[1]) + src_group = int(src/4) + dst_group = int(dst/4) + if src_group == dst_group: # channel swapping within a stripe does nothing + continue + new_sum, improvement = try_swap(result, dst, src) + if improvement > options['improvement_threshold']: + result[...,[src,dst]] = result[...,[dst,src]] + permutation_sequence[src], permutation_sequence[dst] = permutation_sequence[dst], permutation_sequence[src] + real_swap_num += 1 + duration = time.perf_counter() - start_time + if verbosity > 1: + print("\tFinally swap {} channel pairs until the search time limit expires.".format(real_swap_num)) + elif options['strategy'] == 'user defined': # need to get the permutated matrix (result) by applying customized permutation search function + if verbosity > 1: + print("[accelerated_search_for_good_permutation] Use the user customized permutation search function!") + else: + if verbosity >= 0: + print("[accelerated_search_for_good_permutation] Cannot find the implementation of the required strategy!") + + if verbosity > 1: + print("[accelerated_search_for_good_permutation] Take {:.4f} seconds to search the permutation sequence.".format(duration)) + + return permutation_sequence diff --git a/apex/apex/contrib/sparsity/permutation_search_kernels/channel_swap.py b/apex/apex/contrib/sparsity/permutation_search_kernels/channel_swap.py new file mode 100644 index 00000000..f722c292 --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_search_kernels/channel_swap.py @@ -0,0 +1,213 @@ +from .permutation_utilities import * + +################################################################################################################ +# Greedy Channel Swaps - iterative, deterministic, can be parallelized +# 1. Build a map of the magnitude improvement of involved stripes for all pairs of channel swaps +# 2. Sort the map, march through by decreasing improvement, skipping entries whose stripes have been modified +# 3. Repeat until there's no entry with positive improvement (convergence) +################################################################################################################ + +## try swapping columns and tracking magnitude after pruning +def try_swap(matrix, dst, src): + src_base = sum_after_2_to_4(matrix[...,int(src/4)*4:int(src/4)*4+4]) + dst_base = sum_after_2_to_4(matrix[...,int(dst/4)*4:int(dst/4)*4+4]) + + # swap + matrix[...,[src,dst]] = matrix[...,[dst,src]] + + # check the Nx4 slices of the swapped columns + src_sum = sum_after_2_to_4(matrix[...,int(src/4)*4:int(src/4)*4+4]) + dst_sum = sum_after_2_to_4(matrix[...,int(dst/4)*4:int(dst/4)*4+4]) + + # swap back + matrix[...,[src,dst]] = matrix[...,[dst,src]] + + return src_sum + dst_sum, (src_sum + dst_sum) - (src_base + dst_base) + + +## convert stripe and a swap indices to columns +def stripes_and_swap_idx_to_columns(stripe0, stripe1, idx): + i = 0 + for c0 in range(4): + for c1 in range(4): + if i == idx: + return stripe0*4+c0, stripe1*4+c1 + i += 1 + return None + +## convert columns to stripe and swap indices +def columns_to_stripes_and_swap_idx(col0, col1): + stripe0 = int(col0/4) + col0 %= 4 + stripe1 = int(col1/4) + col1 %= 4 + + idx = 0 + for c0 in range(4): + for c1 in range(4): + if c0 == col0 and c1 == col1: + return stripe0, stripe1, idx + idx += 1 + return None + +## build a list of stripe pairs that need their benefits recomputed because one stripe was modified +def build_stripe_pairs(matrix, used_stripes): + stripe_pairs = [] + total_stripes = int(matrix.shape[1]/4) + + used_stripes = np.sort(used_stripes) + for stripe0 in range(total_stripes-1): + for stripe1 in range(stripe0, total_stripes): + if stripe0 in used_stripes or stripe1 in used_stripes: + stripe_pairs.append([stripe0,stripe1]) + + return np.asarray(stripe_pairs) + +## compute the benefit of swapping each pair of columns in the matrix using the GPU +## only update stripes' columns that appear in used_stripes to avoid unnecessary computations +def compute_swap_map(matrix, used_stripes): + do_gpu = use_gpu() + assert(do_gpu) + + stripe_pairs = build_stripe_pairs(matrix, used_stripes).astype(np.uint32) + matrix_view = matrix.astype(np.float32).flatten() + stripe_pairs_view = stripe_pairs.flatten() + output = np.zeros((len(stripe_pairs)*16), dtype=np.float32).flatten() + result = permutation_search_cuda_kernels.build_swap_map(matrix_view, matrix.shape[0], matrix.shape[1], stripe_pairs_view, output) + + # translate the flat array from the GPU to a map + pair_improvement_map = {} + for i,pair in enumerate(stripe_pairs): + for swap_idx in range(16): + col0, col1 = stripes_and_swap_idx_to_columns(pair[0], pair[1], swap_idx) + pair_improvement_map[(col0, col1)] = output[i*16+swap_idx] + return pair_improvement_map + +## build the full swap map +def build_swap_map(matrix, swap_map, swap_ids, used_stripes, verbosity): + improvements = None + + # if we have a GPU and built kernels, pre-compute the needed values + do_gpu = use_gpu() + if do_gpu: + if len(swap_map) == 0: + used_stripes = [s for s in range(int(matrix.shape[1]/4))] + improvements = compute_swap_map(matrix, used_stripes) + + idx = 0 + updates = 0 + for src in range(matrix.shape[1]-1): # parallelize these loops + for dst in range(src+1, matrix.shape[1]): + + # swapping within a stripe does nothing + if int(src/4) == int(dst/4): + continue + + # if we touched this stripe last time, update it + if (int(src/4) in used_stripes) or (int(dst/4) in used_stripes) or len(swap_map) <= idx: + tmp_improvement = 0.0 + + # use the pre-computed values from the GPU if possible, otherwise compute on the CPU + if do_gpu: + tmp_improvement = improvements[(src,dst)] + else: + tmp_mag, tmp_improvement = try_swap(matrix, src, dst) + updates += 1 + + if len(swap_map) <= idx: + swap_map.append(tmp_improvement) + swap_ids.append((src,dst)) + else: + swap_map[idx] = tmp_improvement + swap_ids[idx] = (src,dst) + + idx += 1 + + if verbosity > 15: + print(f"\tupdated {updates} map entries") + return swap_map, swap_ids + +def use_swap_map(matrix, swap_map, swap_ids, threshold, used_escape_attempts, escape_attempts, permutation, verbosity): + used_stripes = [] + swaps = 0 + improvement = 0.0 + + # set the traversal order and threshold + ix = np.flip(np.argsort(swap_map)) # small to large -> large to small + threshold = min(max(swap_map[ix[0]] * threshold, 0.0001),1.0) + + # iterate through the potential swaps in benefit order + for swap in range(len(ix)): + swap_id = ix[swap] + src = swap_ids[swap_id][0] + dst = swap_ids[swap_id][1] + + # early-out of swaps that are below the threshold (don't be so greedy) + if swap_map[ix[swap]] < threshold: + # see if an arbitrary swap helps things if we've converged + if len(used_stripes) == 0 and used_escape_attempts < escape_attempts: + swap_id = np.random.randint(len(swap_ids)) + if verbosity > 15: + print(F"converged, attempt #{used_escape_attempts+1} to jiggle out, using index {swap_id} into the sorted list={ix[swap_id]}") + swap_id =ix[swap_id] + src = swap_ids[swap_id][0] + dst = swap_ids[swap_id][1] + used_escape_attempts += 1 + else: + break + + # skip swaps that include a stripe we've already modified + if int(src/4) in used_stripes or int(dst/4) in used_stripes: + continue + + # we'll need to update these stripes later + used_stripes.append(int(src/4)) + used_stripes.append(int(dst/4)) + + # make the swap + if verbosity > 20: + print(F"\t{swap}\t{src},{dst} {swap_map[swap_id]:.4f}") + matrix[...,[src,dst]] = matrix[...,[dst,src]] + permutation[src],permutation[dst] = permutation[dst],permutation[src] + improvement += swap_map[swap_id] + swaps += 1 + + return matrix, swaps, swap_map, swap_ids, used_stripes, improvement, used_escape_attempts, permutation + +def Channel_Swap(matrix, escape_attempts=0, verbosity=0, permutation=None): + threshold = 0.00001 + used_escape_attempts = 0 + + # initialize + if permutation is None: + permutation = [c for c in range(matrix.shape[1])] + swap_map = [] + swap_ids = [] + used_stripes = [] + swap_count = 0 + iterations = 0 + agg_improvement = 0. + cur_total_sum = sum_after_2_to_4(matrix) + start_time = time.perf_counter() + + # do the work + swapped = 1 # just start with nonzero value to fall into the loop + while swapped > 0: + swap_map, swap_ids = build_swap_map(matrix, swap_map, swap_ids, used_stripes, verbosity) + matrix, swapped, swap_map, swap_ids, used_stripes, improvement, used_escape_attempts, permutation = use_swap_map(matrix, swap_map, swap_ids, threshold, used_escape_attempts, escape_attempts, permutation, verbosity) + agg_improvement += improvement + + # keep track of statistics, print occasionally + swap_count += swapped + if verbosity > 10: + iterations += 1 + cur_total_sum += agg_improvement + duration = time.perf_counter() - start_time + print(F"\t{iterations:8} {cur_total_sum:7.2f} {agg_improvement:7.2f} {swap_count:4} {agg_improvement/max(swap_count,1):5.2f} {duration:7.2f}") + agg_improvement = 0. + swap_count = 0 + + # final status + seconds = time.perf_counter() - start_time + + return matrix, seconds, permutation diff --git a/apex/apex/contrib/sparsity/permutation_search_kernels/exhaustive_search.py b/apex/apex/contrib/sparsity/permutation_search_kernels/exhaustive_search.py new file mode 100644 index 00000000..f13e233b --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_search_kernels/exhaustive_search.py @@ -0,0 +1,368 @@ +from .permutation_utilities import * + +################################################################################################################ +# Exhaustive +# Try them all +# - order of columns within a group doesn't matter +# - order of groups doesn't matter +# - we can eliminate effective duplicates by defining aunique combination to be a sorted list of sorted groups +################################################################################################################ + +#################################################################### +# generate unique permutations +#################################################################### + +# check if adding a column index to a current permutation would keep it in canonical form +# assumes that perm is in canonical form already! +def is_canonical(perm, col): + # if it's a new group + if len(perm) % 4 == 0: + # every column ID < col needs to be in the permutation already + for val in range(col): + if val not in perm: + return False + # this new group needs to be sorted w.r.t. the previous group + return col > perm[-4] + + # not a new group, just check to see if it will still be sorted + return col > perm[-1] + + +# recursive: build a unique permutation one column index at a time +def generate_unique_combinations(built_permutation, remaining_columns, full_permutation_list, group_width): + + # base case: nothing else to add + if len(remaining_columns) == 0: + full_permutation_list.append(np.copy(built_permutation)) + if len(full_permutation_list) % 1000000 == 0: + print(f"{len(full_permutation_list)} unique permutations found so far") + + # still more choices to make, so add each remaining column in turn column if it keeps everything sorted + else: + for c in range(len(remaining_columns)): + # to satisfy our immutables (values within groups are sorted, groups are globally sorted), + # only add this column if either: + # it's starting a new group and is larger than the previous group's first entry + # OR + # it's larger than the last value in the built_permutation + col_to_add = remaining_columns[c] + + if is_canonical(built_permutation, col_to_add): + # add the column to the running permutation, remove it from remaining columns + built_permutation.append(col_to_add) + remaining_columns.pop(c) + # recurse + generate_unique_combinations(built_permutation, remaining_columns, full_permutation_list, group_width) + # remove the most recent column and put it back on the remaining column list where we found it (sorted) + remaining_columns.insert(c, built_permutation.pop(-1)) + +import pickle +import os.path +from os import path +master_unique_permutation_list = {} +def generate_all_unique_combinations(C, M, must_use_all_groups = False): + global master_unique_permutation_list + if len(master_unique_permutation_list) == 0 and path.exists("master_list.pkl"): + with open("master_list.pkl","rb") as cache: + master_unique_permutation_list = pickle.load(cache) + + if (C,M) not in master_unique_permutation_list: + full_permutation_list = [] + generate_unique_combinations([0], [c for c in range(1,C)], full_permutation_list, M) + master_unique_permutation_list[(C,M)] = full_permutation_list + + with open("master_list.pkl", "wb") as cache: + pickle.dump(master_unique_permutation_list, cache) + + unique_permutations = master_unique_permutation_list[(C,M)] + + return unique_permutations + +# analytical solution +import math +def predict_unique_combinations(C, M): + assert(C%M==0) + G = int(C/M) + return int(int(math.factorial(C)) / (int(math.pow(math.factorial(M),G)) * math.factorial(G))) + +################################################################# +# exhaustively try all unique permutations +################################################################# + +# exhaustively search the entire matrix +def search_matrix(matrix, group_width): + # give up quickly if we'd go on forever + prediction = predict_unique_combinations(matrix.shape[1], group_width) + best_permutation = [c for c in range(matrix.shape[1])] + if prediction > 1e10: + print(f"There are {prediction} unique combinations with {matrix.shape[1]} columns and a group width of {group_width}, not searching.") + return matrix, prediction, best_permutation + + start_time = time.perf_counter() + full_permutation_list = generate_all_unique_combinations(matrix.shape[1], group_width) + + # found them, now try them + best_improvement = 0.0 + use_cuda = use_gpu() + if use_cuda and matrix.shape[1] >= 8 and group_width == 4: # CUDA path only works for a group width of 4 + best_improvement, best_permutation = try_permutations_on_matrix(matrix, full_permutation_list) + else: + base_sum = sum_after_2_to_4(matrix) + for i in range(1,len(full_permutation_list)): + permutation = full_permutation_list[i] + permuted = matrix[:, permutation] + cur_improvement = sum_after_2_to_4(permuted) - base_sum + + if (cur_improvement > best_improvement): + best_improvement = cur_improvement + best_permutation = permutation + seconds = time.perf_counter() - start_time + return matrix[:, best_permutation], seconds, best_permutation, best_improvement + + +############# +# Stripe group handling +############# + +# gather stripes from a larger matrix into a single matrix +def collect_stripes(matrix, stripes, group_width): + subset = np.zeros((matrix.shape[0], len(stripes)*group_width)) + for s,stripe in enumerate(stripes): + subset[...,s*group_width:s*group_width+group_width] = matrix[...,stripe*group_width:stripe*group_width+group_width] + return subset + +# apply the stripe group permutation to the entire permutation +def apply_stripe_group_permutation(sgp, stripes, group_width, permutation): + new_permutation = permutation.copy() + for subset_idx in range(len(sgp)): + dst_stripe_idx = stripes[int(subset_idx / group_width)] + dst_col_idx = subset_idx % group_width + + subset_val = sgp[subset_idx] + src_stripe_idx = stripes[int(subset_val / group_width)] + src_col_idx = subset_val % group_width + + new_permutation[dst_stripe_idx*group_width + dst_col_idx] = permutation[src_stripe_idx*group_width + src_col_idx] + + return new_permutation + +# generate all possible stripe groups +def generate_stripe_groups(num_stripes, window_size): + stripe_array = [[c] for c in range(num_stripes)] + + next_stripe_array = [] + for w in range(1, window_size): + for g in range(len(stripe_array)): + start_c = stripe_array[g][w-1]+1 + group = stripe_array[g] + for c in range(start_c, num_stripes): + new_group = group.copy() + new_group.append(c) + next_stripe_array.append(new_group) + stripe_array = next_stripe_array + next_stripe_array = [] + + return set(tuple(stripe_array[g]) for g in range(len(stripe_array))) + +# It is not safe to just reset the stripe_set as None here. +# When calling the Exhaustive_Search in E2E search, the stripe_set will not be reset as None. +stripe_set = None +stripe_set_config = None +# build the stripe map +def build_stripe_map(matrix, group_width, window_size, stripe_map, stripe_ids, perm_map, used_stripes): + global stripe_set, stripe_set_config + + window_size = int(window_size / group_width) + + if stripe_set is None or stripe_set_config is None or stripe_set_config != (group_width, window_size): + num_stripes = int(matrix.shape[1] / group_width) + assert(group_width * num_stripes == matrix.shape[1]) + stripe_set = generate_stripe_groups(num_stripes, window_size) + stripe_set_config = (group_width, window_size) + + # step through each, update the stripe_map/stripe_ids if necessary + updates = 0 + use_cuda = use_gpu() + gpu_list = [] + gpu_groups = [] + for i,s in enumerate(stripe_set): + sg = [] # build the group of stripes, check if any members changed + need_update = i >= len(stripe_map) + for stripe in s: + sg.append(stripe) + if stripe in used_stripes: + need_update = True + + # pre-populate if we're building fresh + if i >= len(stripe_map): + stripe_ids.append(sg) + stripe_map.append(0.) + perm_map.append([c for c in range(group_width * window_size)]) + + # update entries if needed (only stripe_map and perm_map) + if need_update: + updates += 1 + + if not use_cuda: # do the work here if using the CPU + subset = collect_stripes(matrix, sg, group_width) + sub_result, sub_duration, permutation, improvement = search_matrix(subset, group_width) + stripe_map[i] = improvement + perm_map[i] = permutation + else: # otherwise, just track the work needed to farm off to the GPU + gpu_groups.append(sg) + gpu_list.append(i) + + if use_cuda: # if using the GPU, perform the work + matrix_view = np.copy(matrix).astype(np.float32).flatten() + all_permutations = generate_all_unique_combinations(window_size*group_width, group_width) + num_permutations = len(all_permutations) + permutation_view = np.copy(np.asarray(all_permutations)).astype(np.uint32).flatten() + stripe_groups_view = np.asarray(gpu_groups).astype(np.uint32).flatten() + num_gpu_groups = len(gpu_list) + gpu_improvement = np.zeros((num_gpu_groups), dtype=np.float32).flatten() + gpu_permutation = np.zeros((num_gpu_groups), dtype=np.uint32).flatten() + + result = permutation_search_cuda_kernels.build_permute_map(matrix_view, + matrix.shape[0], + matrix.shape[1], + stripe_groups_view, + num_gpu_groups, + window_size, + permutation_view, + window_size * group_width, + gpu_improvement, + gpu_permutation) + + # put the data where python expects it + for i in range(len(gpu_list)): + stripe_map[gpu_list[i]] = gpu_improvement[i] + perm_map[gpu_list[i]] = all_permutations[gpu_permutation[i]] + + return stripe_map, stripe_ids, perm_map + + +# start performing stripe checks +sm_perturbations = 0 +sm_perturbation_limit = 0 +def use_stripe_map(matrix, group_width, stripe_map, stripe_ids, perm_map, permutation): + global sm_perturbations, sm_perturbation_limit + used_stripes = [] + stripe_groups_optimized = 0 + improvement = 0.0 + + # set the traversal order + ix = np.flip(np.argsort(stripe_map)) # small to large --> large to small + + for i in range(len(ix)): + stripe_group_id = ix[i] + perm = perm_map[stripe_group_id].copy() + + if stripe_map[stripe_group_id] <= np.finfo(np.float16).tiny*5.: + # perturbations + if len(used_stripes) == 0 and sm_perturbations < sm_perturbation_limit: + sm_perturbations += 1 + # use this permutation, but swap two channels from left/right halves to include two stripes, no matter the group size + stripe_group_id = ix[np.random.randint(len(ix))] + perm = perm_map[stripe_group_id].copy() + # a little easier to escape from + src = np.random.randint(int(len(perm)/2)) + dst = int(len(perm)/2) + np.random.randint(int(len(perm)/2)) + perm[src],perm[dst] = perm[dst],perm[src] + else: + break + + stripe_group = stripe_ids[stripe_group_id] + + # don't work on stripes we've already touched + touched_stripe = False + for stripe in stripe_group: + if stripe in used_stripes: + touched_stripe = True + if touched_stripe: + continue + + # apply the permutation we've already found to this stripe group + subset = collect_stripes(matrix, stripe_group, group_width) + sub_result = subset[...,perm] + permutation = apply_stripe_group_permutation(perm, stripe_group, group_width, permutation) + + # scatter the results, track what changed + for s,stripe in enumerate(stripe_group): + # see if this group is in canonical form (entry 0 a multiple of 4, contiguous values)) + group = perm[s*group_width:s*group_width+group_width] # columns in this group of the used permutation + changed = False + if group[0] % 4 != 0: + changed = True + for c in range(1,group_width): + if group[c] != group[c-1]+1: + changed = True + break + # if it's not, then it changed + if changed: + used_stripes.append(stripe_group[s]) + + matrix[...,stripe*group_width:stripe*group_width+group_width] = sub_result[...,s*group_width:s*group_width+group_width] + + improvement += stripe_map[stripe_group_id] + stripe_groups_optimized += 1 + + return matrix, stripe_groups_optimized, stripe_map, stripe_ids, used_stripes, improvement, permutation + +# entry point for exhaustive searches - both the entire matrix, as well as stripe groups +def Exhaustive_Search(matrix, stripe_group_size=-1, escape_attempts=0, permutation=None): + global sm_perturbation_limit, sm_perturbations + sm_perturbations = 0 + sm_perturbation_limit = escape_attempts + if permutation is None: + permutation = [c for c in range(matrix.shape[1])] + + # It is much safer to reset the stripe_set as None in the entry point of Exhaustive_Search + global stripe_set, stripe_set_config + stripe_set = None + stripe_set_config = None + + # only support N:4 for now + group_width = 4 + + result = np.copy(matrix) + + # if the matrix is too large for a window size of 12, subdivide, then fix up with a global optimization with a window size of 8 + if group_width==4 and stripe_group_size==12 and matrix.shape[1] > 512: + stripe_split = int(matrix.shape[1]/2/group_width) + col_split = stripe_split * group_width + result[:,:col_split], durationL, permutation[:col_split] = Exhaustive_Search(result[:,:col_split], stripe_group_size=stripe_group_size, escape_attempts=escape_attempts, permutation=permutation[:col_split]) + result[:,col_split:], durationR, permutation[col_split:] = Exhaustive_Search(result[:,col_split:], stripe_group_size=stripe_group_size, escape_attempts=escape_attempts, permutation=permutation[col_split:]) + escape_attempts = max(escape_attempts, 100)*10 + result,duration,permutation = Exhaustive_Search(result, stripe_group_size=8, escape_attempts=escape_attempts, permutation=permutation) + return result, durationL+durationR+duration, permutation + + # small enough to optimize the entire matrix at once + if stripe_group_size != -1 and stripe_group_size < matrix.shape[1]: + stripe_map = [] + stripe_ids = [] + perm_map = [] + used_stripes = [] + + # in practice, this work will be cached ahead of time; doing it now. + # (Reading the cached list from disk can take several seconds, which shouldn't be counted against the search, but amortized over every layer in a network) + generate_all_unique_combinations(stripe_group_size, group_width) + + start_time = time.perf_counter() + + while True: + #print("[Debug][Exhaustive_Search] Before entering the build_stripe_map function.") + #print("[Debug][Exhaustive_Search] Now the stripe_set value is: {}".format(stripe_set)) + stripe_map, stripe_ids, perm_map = build_stripe_map(result, group_width, stripe_group_size, stripe_map, stripe_ids, perm_map, used_stripes) + result, stripe_groups_optimized, stripe_map, stripe_ids, used_stripes, improvement, permutation = use_stripe_map(result, group_width, stripe_map, stripe_ids, perm_map, permutation) + + # converged? + if len(used_stripes) == 0: + break + + duration = time.perf_counter() - start_time + + else: # no sliding window, single iteration + print(f"Matrix has {matrix.shape[1]} columns and the search window is only {stripe_group_size}: searching exhaustively") + result, duration, permutation, improvement = search_matrix(matrix, group_width) + + return result, duration, permutation diff --git a/apex/apex/contrib/sparsity/permutation_search_kernels/permutation_utilities.py b/apex/apex/contrib/sparsity/permutation_search_kernels/permutation_utilities.py new file mode 100644 index 00000000..a4635a62 --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_search_kernels/permutation_utilities.py @@ -0,0 +1,556 @@ +import numpy as np +import time +import subprocess +import math + +gpus_tested = False +gpus_found = 0 +kernels_found = True +try: + import permutation_search_cuda as permutation_search_cuda_kernels + print(f"Found permutation search CUDA kernels") +except ImportError: + + try: + from . import permutation_search_cuda as permutation_search_cuda_kernels + print(f"Found permutation search CUDA kernels for standalone testing") + + except ImportError: + print(f"Could not find permutation search CUDA kernels, falling back to CPU path") + kernels_found = False + +def use_gpu(initial_override = True): + global gpus_tested, gpus_found, kernels_found + if not gpus_tested: + if not initial_override: + gpus_tested = True + return False + + try: + gpus_found = str(subprocess.check_output(["nvidia-smi", "-L"])).count('UUID') + print(f"Found {gpus_found} gpus") + except: + gpus_found = 0 + print(f"Could not find nvidia-smi, please check your cuda installation") + + gpus_tested = True + + return gpus_found > 0 and kernels_found + +############################################################################################## +# pruning utilities +############################################################################################## +## apply 2:4 to some matrix +def apply_2_to_4(matrix): + for row in range(matrix.shape[0]): + for col in range(0,matrix.shape[1],4): + ix = np.argsort(np.abs(matrix[row,col:col+4])) + matrix[row,col+ix[0]] = 0.0 + matrix[row,col+ix[1]] = 0.0 + return matrix + +## find the sum of magnitudes if 2:4 were applied to a matrix +def sum_after_2_to_4(matrix): + cur_sum = 0.0 + use_cuda = use_gpu() + if not use_cuda: + for row in range(matrix.shape[0]): + for col in range(0,matrix.shape[1],4): + ix = np.argsort(np.abs(matrix[row,col:col+4])) + cur_sum += abs(matrix[row,col+ix[2]]) + cur_sum += abs(matrix[row,col+ix[3]]) + else: + matrix = matrix.astype(np.float32) + cuda_sum = np.zeros((1), dtype=np.float32) + matrix_view = np.copy(matrix).flatten() + sum_view = cuda_sum.flatten() + blocks = max(int(matrix.shape[1]/4/2), 1) + threads = min(max(math.ceil(matrix.shape[0]/4), 1), 1024) + result = permutation_search_cuda_kernels.sum_after_2_to_4(matrix_view, + matrix.shape[0], + matrix.shape[1], + 0, + matrix.shape[1], + blocks, + threads, + sum_view) + cur_sum = sum_view[0] + return cur_sum + +# perform unstructured pruning on some matrix +def unstructured_prune(matrix, sparsity): + shp = matrix.shape + matrix = matrix.flatten() + ix = np.argsort(matrix) + ix = ix[:int(len(ix)*sparsity)] + matrix[ix] = 0.0 + matrix = np.reshape(matrix, shp) + return matrix + +## try swapping columns and tracking magnitude after pruning +def try_swap(matrix, dst, src): + src_base = sum_after_2_to_4(matrix[...,int(src/4)*4:int(src/4)*4+4]) + dst_base = sum_after_2_to_4(matrix[...,int(dst/4)*4:int(dst/4)*4+4]) + + # swap + matrix[...,[src,dst]] = matrix[...,[dst,src]] + + # check the Nx4 slices of the swapped columns + src_sum = sum_after_2_to_4(matrix[...,int(src/4)*4:int(src/4)*4+4]) + dst_sum = sum_after_2_to_4(matrix[...,int(dst/4)*4:int(dst/4)*4+4]) + + # swap back + matrix[...,[src,dst]] = matrix[...,[dst,src]] + + return src_sum + dst_sum, (src_sum + dst_sum) - (src_base + dst_base) + +## magnitude improvement from the naive 2:4 matrix / how much was lost by naive 2:4 compared to the optimal +def efficacy(optimal_lost_magnitude, base_lost_magnitude, cur_lost_magnitude): + if base_lost_magnitude == optimal_lost_magnitude: + eff = 1.0 + else: + eff = (base_lost_magnitude - cur_lost_magnitude) / (base_lost_magnitude - optimal_lost_magnitude) + return eff + +## find the magnitude if the rows of a matrix were pruned independently, without structure +def magnitude_after_pruning_rows(matrix, rate=0.5): + magnitude = 0. + cols = matrix.shape[1] + for r in range(matrix.shape[0]): + rowVals = matrix[r] + rowVals = np.sort(np.abs(rowVals)) + magnitude += np.sum(rowVals[int(cols*rate):]) + + return magnitude + + + +############################################################################################## +# permutation utilities +############################################################################################## + +## exhaustively search an entire matrix on the GPU +def try_permutations_on_matrix(matrix, permutations): + use_cuda = use_gpu() + assert(use_cuda) # caller should have checked + matrix = np.copy(matrix) + matrix = matrix.astype(np.float32) + matrix_view = np.copy(matrix).flatten() + permutations_view = np.copy(np.asarray(permutations)).astype(np.uint32).flatten() + + stripe_groups = np.asarray([[s for s in range(int(matrix.shape[1]/4))]]).astype(np.uint32) + stripe_groups_view = stripe_groups.flatten() + + improvement = np.zeros((1), dtype=np.float32).flatten() + permutation = np.zeros((1), dtype=np.uint32).flatten() + + result = permutation_search_cuda_kernels.check_permutations(matrix_view, + matrix.shape[0], + matrix.shape[1], + stripe_groups_view, + len(stripe_groups[0]), + len(stripe_groups), + permutations_view, + len(permutations), + improvement, + permutation) + return improvement[0], permutations[permutation[0]] + +## find the permutation needed to make matrix A look like matrix B +def find_permutation(A, B): + permutation = [] + for col in range(A.shape[1]): + Avals = A[...,col] + for bcol in range(B.shape[1]): + if np.all(Avals - B[...,bcol] == np.zeros(Avals.shape)): + permutation.append(bcol) + break + return permutation + + +######################################## +# reasonable method to find distance between permutations +# this is used to generate permutations "between" two other permutations to divide efficacy space +####################################### + +## separate a flat permutation array into its groups, sort each group and the overall order to +## put the output into a canonical order: if two permutations have the same groups, they should appear identical +def make_grouped(A): + groups = [] + for x in range(0,len(A),4): + group = [] + for c in range(4): + group.append(A[x+c]) + group = np.sort(group) + + groups.append(group) + return groups + +## given two permutations, find the groups they have in common +def common_groups(A, B): + Ag = make_grouped(A) + Bg = make_grouped(B) + + # convert to sets to take the intersection + As = set(tuple(Ag[g]) for g in range(len(Ag))) + Bs = set(tuple(Bg[g]) for g in range(len(Bg))) + common = As.intersection(Bs) + + # flatten + C = [] + for s in common: + for v in s: + C.append(v) + + # group + return make_grouped(C) + +## given two permutations, remove the groups that are common between them +def remove_common_groups(A, B): + Ag = make_grouped(A) + Bg = make_grouped(B) + + # convert to sets to take set difference + As = set(tuple(Ag[g]) for g in range(len(Ag))) + Bs = set(tuple(Bg[g]) for g in range(len(Bg))) + Ad = As - Bs + Bd = Bs - As + + # turn the differences back into flat arrays + A = [] + for s in Ad: + for v in s: + A.append(v) + B = [] + for s in Bd: + for v in s: + B.append(v) + + # group to put into canonical order, re-flatten + A = make_grouped(A) + B = make_grouped(B) + A = [item for sublist in A for item in sublist] + B = [item for sublist in B for item in sublist] + + return A,B + +## given two permutations, find which elements in B need to go where to look like A +def group_differences(A, B): + Ag = make_grouped(A) + Bg = make_grouped(B) + + wrong_entries = [] + #for g,group in enumerate(Bg): + for g in range(len(Bg)): + group = Bg[g] + for i in range(len(group)): + val = group[i] + if val not in Ag[g]: + group_in_a = int(np.where(A == val)[0][0] / 4) + wrong_entries.append((val, g, group_in_a)) + + return wrong_entries + +## (val, cur_group, desired_group) ==> dict[(cur_group, desired_group)] = [vals] +def dictify(wrong_entries): + result = {} + for entry in wrong_entries: + key = (entry[1], entry[2]) + if key in result: + result[key].append(entry[0]) + else: + result[key] = [entry[0]] + return result + +## move groups of B to where they best match A's groups +def move_groups_to_match(B, A, debug=False): + Ag = make_grouped(A) + Bg = make_grouped(B) + + new_Bg = [[] for g in range(len(Ag))] + wrong_entry_dict = dictify(group_differences(A, B)) + + if debug: + print(f"MGTM:\n\tAg: {Ag}\n\tBg: {Bg}\n\tWED: {wrong_entry_dict}") + + moved_groups = [] + + keys_to_del = [] + # move triples to the right spot + for k in wrong_entry_dict.keys(): + if k[0] in moved_groups: + keys_to_del.append(k) + continue + + if len(wrong_entry_dict[k]) == 3: + new_Bg[k[1]] = Bg[k[0]] + moved_groups.append(k[0]) + keys_to_del.append(k) + if debug: + print(f"MGTM: moved triple {wrong_entry_dict[k]} from group {k[0]} to group {k[1]}") + + for k in keys_to_del: + del wrong_entry_dict[k] + keys_to_del = [] + + # move doubles + for k in wrong_entry_dict.keys(): + # if we've already moved the group to which this key belongs, remove it + if k[0] in moved_groups: + keys_to_del.append(k) + continue + + if len(wrong_entry_dict[k]) == 2: + if len(new_Bg[k[1]]) == 0: # move it to its requested destination if possible + new_Bg[k[1]] = Bg[k[0]] + keys_to_del.append(k) + assert(k[0] not in moved_groups) + moved_groups.append(k[0]) + if debug: + print(f"MGTM: moved double {wrong_entry_dict[k]} from group {k[0]} to its preferred group {k[1]}") + elif len(new_Bg[k[0]]) == 0: # otherwise leave it where it is (if possible) + new_Bg[k[0]] = Bg[k[0]] + keys_to_del.append(k) + assert(k[0] not in moved_groups) + moved_groups.append(k[0]) + if debug: + print(f"MGTM: left double {wrong_entry_dict[k]} where it was in group {k[0]}") + for k in keys_to_del: + del wrong_entry_dict[k] + keys_to_del = [] + + # move singles + # try to leave things where they are to prevent oscillating + for k in wrong_entry_dict.keys(): + if k[0] in moved_groups: + keys_to_del.append(k) + continue + + if len(new_Bg[k[1]]) == 0: # requested destination + new_Bg[k[1]] = Bg[k[0]] + keys_to_del.append(k) + assert(k[0] not in moved_groups) + moved_groups.append(k[0]) + if debug: + print(f"MGTM: moved single {wrong_entry_dict[k]} from group {k[0]} to its preferred group {k[1]}") + + elif len(new_Bg[k[0]]) == 0: + new_Bg[k[0]] = Bg[k[0]] + keys_to_del.append(k) + assert(k[0] not in moved_groups) + moved_groups.append(k[0]) + if debug: + print(f"MGTM: left group {wrong_entry_dict[k]} where it was in group {k[0]}") + + for k in keys_to_del: + del wrong_entry_dict[k] + keys_to_del = [] + + # put what's left where it'll fit + for k in wrong_entry_dict.keys(): + if k[0] in moved_groups: + keys_to_del.append(k) + continue + + for dst in range(len(new_Bg)): + if len(new_Bg[dst]) == 0: + new_Bg[dst] = Bg[k[0]] + keys_to_del.append(k) + assert(k[0] not in moved_groups) + moved_groups.append(k[0]) + if debug: + print(f"MGTM: put group {wrong_entry_dict[k]} where it found a spot in group {dst}") + break + + for k in keys_to_del: + del wrong_entry_dict[k] + keys_to_del = [] + + assert(len(wrong_entry_dict) == 0) + Agsize = sum( [ len(group) for group in Ag] ) + Bgsize = sum( [ len(group) for group in new_Bg] ) + assert(Agsize == Bgsize) + new_B = [item for sublist in new_Bg for item in sublist] + return new_B + +## swap two permutation entries and put the permutation into unique order +def swap_and_correct(permutation, src, tgt): + permutation[src],permutation[tgt] = permutation[tgt],permutation[src] + grouped = make_grouped(permutation) + grouped = [item for sublist in grouped for item in sublist] + return grouped + +## make a swap that will move B in the direction of A +num_diffs = 0 +def move_permutation_towards(B, A, debug=False): + global num_diffs + B = move_groups_to_match(B, A, debug) + wrong_entries = group_differences(A, B) + num_diffs = len(wrong_entries) + + # nothing to do, early out + if len(wrong_entries) == 0: + if debug: + print("MPT: early out") + return B + + if debug: + print(f"MPT: checking {len(wrong_entries)} diffs: {wrong_entries}") + + # look for a group of three wrong entries that want to do the same thing + entry_dict = dictify(wrong_entries) + for k in entry_dict.keys(): + entry = entry_dict[k] + if len(entry) == 3: + if debug: + print(f"MPT: found a triple swap at {k}: {entry_dict[k]}") + (src, dst) = k + # find the index of the one needed to complete the group + # the value is the value in A[dst] that's not in B[src] + # it's already in the destination group and may or may not need to move + group_id = dst + Ag = make_grouped(np.copy(A)) + Bg = make_grouped(np.copy(B)) + value = -1 + for c in range(4): + if Ag[dst][c] not in Bg[src]: + value = Ag[dst][c] + if debug: + print(f"\tMPT: found the missing value {value} in A group {dst} offset {c}") + break + assert(value != -1) + + # now find that value in B + idx0 = np.where(B == value)[0][0] + # find the index of the one this group doesn't need + # it's a member of the group but not in the dict entry + group_id = src + for c in range(4): + if B[group_id*4+c] not in entry_dict[k]: + if debug: + print(f"\tMPT: swapping {idx0} and {group_id*4+c}") + return swap_and_correct(B, idx0, group_id*4+c) + + # look for a group of two entries that are heading to the same place as another wrong entry + victim_loner_pair = None + for k in entry_dict.keys(): + entry = entry_dict[k] + if len(entry) == 2: + if debug: + print(f"MPT: found a double swap at {k}: {entry_dict[k]}") + (src, dst) = k + # find a wrong entry whose dst is the same + for k2 in entry_dict.keys(): + if k2 == k: + continue + + # k2 is a key whose value also belongs in stripe k2[1] (dst2) + if dst == k2[1]: + if debug: + print(f"\tMPT: found a loner going in the same direction at {k2}: {entry_dict[k2][0]}") + # instead of moving these three to where they're headed, start merging them by moving the loner into the double + + # look for a complement: something moving from src to src2 + (src2, dst2) = k2 + complement_key = (src, src2) + if complement_key in entry_dict: + complement = entry_dict[complement_key][0] + if debug: + print(f"\t\tMPT: found a complement to the loner:{complement}") + return swap_and_correct(B, np.where(B == entry_dict[k2][0])[0][0], np.where(B == complement)[0][0]) + # didn't find a complement, choose one of the two in the src group that don't belong + elif victim_loner_pair is None: + for k3 in entry_dict.keys(): + if k3 == k: + continue + + if k3[0] == src: # found the victim + victim = entry_dict[k3][0] + if debug: + print(f"\t\tMPT: found a victim for the double swap:{k3} -> {victim}") + victim_loner_pair = (victim, entry_dict[k2][0]) + #return swap_and_correct(B, np.where(B == entry_dict[k2][0])[0][0], np.where(B == victim)[0][0]) + + if victim_loner_pair is not None: + if debug: + print(f"\t\tMPT: couldn't find any complements for double swaps, so going with a loner to make a triple: {victim_loner_pair}") + return swap_and_correct(B, np.where(B == victim_loner_pair[0])[0][0], np.where(B == victim_loner_pair[1])[0][0]) + + # look for one swap that will correct two entries + candidate_second = None + for we in range(len(wrong_entries)): + cur_entry = wrong_entries[we] + #if debug: + # print(f"\tMPT: checking {cur_entry} for complement") + for we2 in range(0,len(wrong_entries)): + pos_swap = wrong_entries[we2] + #if debug: + # print(f"\t\tMPT: is {pos_swap}?") + if cur_entry[1] == pos_swap[2] and cur_entry[2] == pos_swap[1]: + if debug: + print(f"\t\tfound complements: swapping {cur_entry} and {pos_swap}") + return swap_and_correct(B, np.where(B == cur_entry[0])[0][0], np.where(B == pos_swap[0])[0][0]) + elif wrong_entries[0][2] == pos_swap[1]: # if pos_swap is currently where we[0] wants to go, keep it in mind + candidate_second = pos_swap + + # fall back on picking the first one we come across + assert(candidate_second is not None) + if debug: + print(f"No complement, swapping two entries: {wrong_entries[0]} {candidate_second}") + return swap_and_correct(B, np.where(B == wrong_entries[0][0])[0][0], np.where(B == candidate_second[0])[0][0]) + +## find a shortest path from permutation A to B +def permutation_distance(A, B, matrix=None, magnitude_targets=None, debug=False, verbosity=0): + global num_diffs + swaps = 0 + debug = False + + swap_limit = int(math.pow(2,int(len(A)/4)-1)) + num_diffs = swap_limit + common = [] + target_results = None + if magnitude_targets is not None: + assert matrix is not None + cur_mag = sum_after_2_to_4(matrix[:,A]) + target_results = [(cur_mag, A) for i in range(len(magnitude_targets))] + + if verbosity > 0 and matrix is not None: + print(f"swap {'0':>4} {sum_after_2_to_4(matrix[:, B]):>15.3f}") + if verbosity > 5: + print(f"swap {0:>4}, {make_grouped(A)} {make_grouped(B)}") + + while not np.all(np.array(A)-np.array(B) == np.zeros(np.array(A).shape)): + cGroups = common_groups(A, B) + for g in cGroups: + common.append(g) + A, B = remove_common_groups(A, B) + if len(A) == 0: + break + + B = move_permutation_towards(np.array(B), np.array(A), debug=debug) + swaps += 1 + + if matrix is not None: + total_cur_permute = [c for c in B] + + for c in [item for sublist in common for item in sublist]: + total_cur_permute.append(c) + + if verbosity > 0 or magnitude_targets is not None: + cur_mag = sum_after_2_to_4(matrix[:,total_cur_permute]) + for i in range(len(target_results)): + result = target_results[i] + if abs(magnitude_targets[i] - result[0]) > abs(magnitude_targets[i] - cur_mag): + target_results[i] = (cur_mag, total_cur_permute) + if verbosity > 0: + print(f"swap {swaps:>4} {cur_mag:>15.3f}") + + if verbosity > 5 or swaps > swap_limit: + print(f"swap {swaps:>4}, {A} {B}, {num_diffs} diffs remain") + + # safety net + if swaps > swap_limit+3: + return swaps, target_results + + return swaps, target_results + diff --git a/apex/apex/contrib/sparsity/permutation_tests/README.md b/apex/apex/contrib/sparsity/permutation_tests/README.md new file mode 100644 index 00000000..71374d84 --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_tests/README.md @@ -0,0 +1,155 @@ +# ChannelPermutations + +Standalone code to reproduce results in "[Channel Permutations for N:M Sparsity](https://proceedings.neurips.cc/paper/2021/hash/6e8404c3b93a9527c8db241a1846599a-Abstract.html)," Jeff Pool and Chong Yu, NeurIPS 2021. + +Three search strategies are supported: randomly generating permutations and checking quality, greedily swapping columns until convergence (i.e. TETRIS adapted for 2:4 sparsity), and the technique presented in the above paper, optimizing stripe groups. This tool will apply these strategies, as configured below, to either a randomly-generated matrix or an .npy file (typically from a real network) and report the efficacy and runtime of the strategy. + +## Quick Start + +### Installation + +#### GPU path + +Requirements: +- CUDA +- pybind11 + +A container such as `nvcr.io/nvidia/pytorch:21.12-py3` satisfies these requirements. + +Installation (from this directory): +``` +pushd ../permutation_search_kernels/CUDA_kernels +nvcc -O3 -shared -Xcompiler -fPIC -Xcompiler -DTORCH_EXTENSION_NAME=permutation_search_cuda -std=c++11 $(python3 -m pybind11 --includes) permutation_search_kernels.cu -o ../permutation_search_cuda$(python3-config --extension-suffix) +popd +``` + +#### CPU path + +Only NumPy is required for CPU-only execution. + +### Important arguments + +`python3 permutation_test.py` will tell you all the available arguments and alert you about required arguments: +``` + usage: permutation_test.py [-h] [--infile INFILE] [--channels CHANNELS] [--filters FILTERS] + [--verbosity VERBOSITY] [--seed SEED] [--pretty_print PRETTY_PRINT] + [--unstructured UNSTRUCTURED] [--gpu GPU] [--check_permutation CHECK_PERMUTATION] + [--intermediate_steps INTERMEDIATE_STEPS] [--print_permutation PRINT_PERMUTATION] + strategy [strategy ...] + permutation_test.py: error: the following arguments are required: strategy +``` + +Detailed information about each argument: + +- `--infile` (string) accepts .npy files with weights dumped from some model checkpoint. By default, the input file is `'random'`, which will generate a random 2D matrix with `CHANNELS` columns and `FILTERS` rows. +- `--channels` and `--filters` (unsigned integers) specify the size of the randomly-generated matrix if there is no input file specified. +- `--verbosity` (unsigned integer) controls the amount of debug and status information printed. `0` is just the important data, `11` can give periodic status details, and higher integers provide increasingly more detail. +- `--seed` (unsigned integer) allows for changing the random seed, which will affect the random matrix generation, random permutations generated, and columns swapped for bounded regressions. +- `--pretty_print` (bool) prints a pretty graph by default (below), but disabling will generate output friendly for redirecting to a .csv file. +- `--unstructured` (float) will apply unstructured pruning to the matrix before searching for permutations. A negative value will find the minimum unstructured sparsity for which a search strategy can find a perfect permutation and not create any extra zeros. +- `--gpu` (bool) uses CUDA kernels by default (if they are built and there is a GPU available), but you can override this to run on the CPU. +- `--check_permutation` (bool) makes sure the permutation tracked during the search process matches the one that's recovered directly from the permuted matrix. +- `--intermediate_steps` (unsigned integer) will emit permutations with efficacies equally dividing the distance between the default order and the best permutation found. +- `--print_permutation` (bool) prints the permutation found for each strategy. + +Finally, after these optional arguments, provide the search strategies desired. There are three strategies offered: +- `random,` +- `channel_swaps,` +- `optimize_stripe_groups,,` + +### Launch a test with interesting search strategies + +Now that kernels are built, you can use them to accelerate the search, which can be quite time-consuming without using the GPU. Below, we report results on a number of interesting strategies for a 64-column, 128-row random matrix using a V100 accelerator. + + $ python3 permutation_test.py --channels 64 --filters 128 channel_swap,0 channel_swap,100 channel_swap,1000 optimize_stripe_groups,8,0 optimize_stripe_groups,8,100 optimize_stripe_groups,8,1000 optimize_stripe_groups,12,0 random,1000 random,10000 random,100000 + Found permutation search CUDA kernels for standalone testing + Found 2 gpus + strategy , magnitude, efficacy, duration + unpruned , 4083.169, - , - + unstructured , 3060.238, - , - + 50% rows , 3042.332, 100.0, - + default 2:4 , 2852.376, 0.0, 0.000 + channel_swap,0 , 2913.352, 32.1, 0.214 + channel_swap,100 , 2914.174, 32.5, 2.249 + channel_swap,1000 , 2920.694, 36.0, 20.248 + optimize_stripe_groups,8,0 , 2919.757, 35.5, 0.013 + optimize_stripe_groups,8,100 , 2919.758, 35.5, 0.152 + optimize_stripe_groups,8,1000 , 2919.935, 35.6, 1.387 + optimize_stripe_groups,12,0 , 2921.947, 36.6, 0.860 + random,1000 , 2873.380, 11.1, 0.116 + random,10000 , 2873.603, 11.2, 1.149 + random,100000 , 2879.129, 14.1, 11.510 + +For this particular input, the `channel_swap` strategy requires 1000 bounded regressions in order to surpass the efficacy of optimizing two stripe groups (8 columns) without any bounded regressions, but allowing 1000 bounded regressions when optimizing two stripe groups is slightly worse than swapping channels with 1000 bounded regressions. Optimizing *three* stripe groups at a time outperforms all the other approaches by a wide margin. Testing many random permutations is inefficient and ineffective. + +Without GPU acceleration, these tests would be much slower (though they find the same final permutations): + + $ python3 permutation_test.py --gpu 0 --channels 64 --filters 128 channel_swap,0 channel_swap,100 optimize_stripe_groups,8,0 optimize_stripe_groups,8,100 random,1000 + strategy , magnitude, efficacy, duration + unpruned , 4083.169, - , - + unstructured , 3060.238, - , - + 50% rows , 3042.332, 100.0, - + default 2:4 , 2852.377, 0.0, 0.016 + channel_swap,0 , 2913.351, 32.1, 55.972 + channel_swap,100 , 2914.174, 32.5, 450.025 + optimize_stripe_groups,8,0 , 2919.759, 35.5, 60.653 + optimize_stripe_groups,8,100 , 2919.759, 35.5, 465.709 + random,1000 , 2873.381, 11.1, 14.889 + + +### Perform the ablation study from Table 1 + +`bash ablation_studies.sh` will generate the results for the ablation study, showing the relative importance of the bounded regressions and stripe group greedy phase. + +### Generate the runtime results from Table 3 + +`bash runtime_table.sh` will generate the search strategies' efficacies and runtime shown in Table 3. + +### Traverse permutation space (as in Figure 3) + +We developed a heuristic approach to interpolating between permutations which allows us to find permutations with efficacies that evenly divide some range. The `--intermediate_steps ` argument can be used to emit such a sequence of permutations: + + $ python3 permutation_test.py --channels 64 --filters 128 --intermediate_steps 7 --print_permutation 1 optimize_stripe_groups,8,0 + Found permutation search CUDA kernels for standalone testing + Found 2 gpus + strategy , magnitude, efficacy, duration + unpruned , 4083.169, - , - + unstructured , 3060.238, - , - + 50% rows , 3042.332, 100.0, - + default 2:4 , 2852.377, 0.0, 0.000 + (2859.8855, [2, 8, 14, 24, 9, 12, 13, 15, 4, 5, 6, 7, 0, 1, 3, 46, 40, 41, 42, 43, 32, 33, 34, 35, 25, 26, 27, 55, 16, 17, 18, 58, 20, 21, 22, 23, 38, 60, 61, 63, 11, 44, 45, 47, 36, 37, 39, 62, 10, 28, 29, 30, 31, 52, 53, 54, 19, 56, 57, 59, 48, 49, 50, 51]) + (2870.1387, [5, 6, 7, 41, 9, 12, 13, 35, 0, 1, 3, 46, 30, 40, 42, 43, 2, 32, 33, 34, 25, 26, 27, 55, 16, 17, 18, 58, 20, 21, 22, 23, 38, 60, 61, 63, 11, 44, 45, 47, 36, 37, 39, 62, 4, 10, 28, 29, 31, 52, 53, 54, 19, 56, 57, 59, 15, 48, 49, 50, 8, 14, 24, 51]) + (2878.0679, [36, 37, 39, 62, 9, 12, 13, 35, 0, 3, 16, 46, 30, 40, 42, 43, 2, 5, 32, 33, 23, 26, 27, 55, 1, 20, 21, 22, 38, 60, 61, 63, 11, 44, 45, 47, 6, 7, 25, 41, 4, 10, 28, 29, 31, 52, 53, 54, 19, 56, 57, 59, 15, 48, 49, 50, 8, 14, 24, 51, 17, 18, 34, 58]) + (2884.8323, [9, 12, 35, 54, 0, 3, 16, 46, 30, 40, 42, 43, 2, 5, 32, 33, 23, 26, 27, 55, 11, 44, 45, 47, 36, 37, 39, 62, 4, 10, 28, 29, 31, 52, 53, 60, 19, 21, 56, 57, 15, 48, 49, 50, 8, 14, 24, 51, 17, 18, 34, 58, 6, 7, 25, 41, 1, 13, 20, 22, 38, 59, 61, 63]) + (2894.9697, [9, 12, 33, 35, 0, 3, 16, 46, 2, 5, 32, 52, 23, 26, 27, 55, 11, 44, 45, 47, 36, 37, 39, 62, 4, 10, 28, 29, 19, 21, 50, 56, 15, 43, 48, 49, 8, 14, 24, 51, 17, 18, 34, 58, 6, 7, 25, 41, 1, 13, 20, 22, 38, 59, 61, 63, 30, 40, 42, 54, 31, 53, 57, 60]) + (2901.5115, [9, 12, 35, 56, 0, 3, 16, 46, 23, 26, 27, 55, 33, 36, 37, 39, 4, 10, 28, 29, 19, 21, 45, 50, 8, 14, 24, 51, 17, 18, 34, 58, 6, 7, 25, 41, 1, 13, 20, 22, 38, 59, 61, 63, 30, 40, 42, 54, 31, 53, 57, 60, 2, 5, 32, 52, 15, 43, 49, 62, 11, 44, 47, 48]) + (2910.2043, [4, 10, 28, 37, 9, 12, 35, 56, 0, 3, 16, 46, 23, 33, 36, 39, 8, 14, 24, 51, 17, 18, 34, 58, 6, 7, 25, 41, 1, 13, 20, 22, 38, 59, 61, 63, 30, 40, 42, 54, 31, 53, 57, 60, 2, 5, 32, 52, 15, 43, 49, 62, 11, 44, 47, 48, 19, 21, 45, 50, 26, 27, 29, 55]) + optimize_stripe_groups,8,0 , 2919.757, 35.5, 0.015 + [0, 9, 12, 35, 4, 10, 28, 37, 50, 19, 45, 21, 34, 17, 18, 58, 16, 46, 39, 3, 49, 43, 15, 62, 6, 7, 41, 25, 48, 11, 44, 47, 13, 20, 22, 1, 55, 29, 26, 27, 5, 2, 32, 52, 40, 30, 42, 54, 53, 57, 60, 31, 36, 56, 23, 33, 59, 38, 61, 63, 51, 24, 14, 8] + +### Transform unstructured sparsity to structured sparsity (as in Figure 4) + +If you have a directory with .npy weight files for each layer of a network, `bash unstructured_study.sh ` will perform a binary search for each file to find the minimum unstructured sparsity required to transparently transform that layer with a number of permutation search techniques; this file was used to generate Figure 4, using weights dumped from a pre-trained ResNet50 in Torchvision. + +## References + +The baseline algorithm which we adapated for use with 2:4 sparsity and upon which we improved is "[TETRIS](https://papers.nips.cc/paper/2018/hash/89885ff2c83a10305ee08bd507c1049c-Abstract.html): TilE-matching the TRemendous Irregular Sparsity," Ji et al., NeurIPS 2018. + +If you want to use this technique when generating a 2:4 sparse network for inference, we've packaged it into our [ASP](https://github.com/NVIDIA/apex/tree/master/apex/contrib/sparsity) library - this will perform the permutation searches for each layer as required, as well as fix up neighboring layers so there are no extra operations inserted at runtime. + +## Citation + +If you use this idea or code in your own research, please cite the [paper](https://proceedings.neurips.cc/paper/2021/hash/6e8404c3b93a9527c8db241a1846599a-Abstract.html) that describes it: + +``` +@inproceedings{pool2021channel, + author = {Pool, Jeff and Yu, Chong}, + booktitle = {Advances in Neural Information Processing Systems ({NeurIPS})}, + title = {Channel Permutations for {N:M} Sparsity}, + url = {https://proceedings.neurips.cc/paper/2021/file/6e8404c3b93a9527c8db241a1846599a-Paper.pdf}, + volume = {34}, + year = {2021} +} + +``` + diff --git a/apex/apex/contrib/sparsity/permutation_tests/ablation_studies.sh b/apex/apex/contrib/sparsity/permutation_tests/ablation_studies.sh new file mode 100755 index 00000000..1a8b12b0 --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_tests/ablation_studies.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +OUTDIR="results/ablation_logs" +mkdir -p $OUTDIR + +R1000=random,1000 +CS=channel_swap,0 +CS_100=channel_swap,100 +CS_1000=channel_swap,1000 +OSG2=optimize_stripe_groups,8,0 +OSG2_100=optimize_stripe_groups,8,100 +OSG2_1000=optimize_stripe_groups,8,1000 +OSG3=optimize_stripe_groups,12,0 +OSG3_100=optimize_stripe_groups,12,100 +OSG3_1000=optimize_stripe_groups,12,1000 +optimal=optimize_stripe_groups,16,0 + +# Table 1 +for seed in {0..24}; do + echo $seed + python3 permutation_test.py --channels 16 --filters 32 --seed $seed --pretty_print=False $R1000 $CS $CS_100 $CS_1000 $OSG2 $OSG2_100 $OSG2_1000 $OSG3 $OSG3_100 $OSG3_1000 $optimal | tee "${OUTDIR}/ablations_32x16_$seed.log" + python3 permutation_test.py --channels 128 --filters 64 --seed $seed --pretty_print=False $R1000 $CS $CS_100 $CS_1000 $OSG2 $OSG2_100 $OSG2_1000 $OSG3 $OSG3_100 $OSG3_1000 | tee "${OUTDIR}/ablations_64x128_$seed.log" +done + +echo "Gathering results ..." + +################# collect results into a .csv file +# get mean and stddev of efficacy from all seeds for one strategy +get_mean_stddev() { + local strategy=$1 + local OUTFILE=$2 + + # get the strategy's line, pull out efficacy and time, use sum-of-squares to compute stddev and mean in a single pass + grep "$strategy," $OUTDIR/ablations_64x128_*.log | awk -F "," '{print $3,$4}' | awk '{sum += $1; sumsq += ($1)^2; timesum += $2} END {printf "%.1f,%.1f,%.2f,", sum/NR, sqrt((sumsq-sum^2/NR)/NR), timesum/NR}' >> $OUTFILE +} + +# get the number of times some strategy matched the optimal solution +get_num_optimal() { + local strategy=$1 + local OUTFILE=$2 + + matches=0 + for seed in {0..24}; do + # compare floats with epsilon: add one thousandth to the efficacy under test + this_eff=$(grep "$strategy," "${OUTDIR}/ablations_32x16_${seed}.log" | awk -F "," '{print int($3 * 1000 + 1)}') + best_eff=$(grep "optimize_stripe_groups_16_0," "${OUTDIR}/ablations_32x16_${seed}.log" | awk -F "," '{print int($3 * 1000)}') + if [ "$this_eff" -ge "$best_eff" ]; then + let "matches = $matches + 1" + fi + done + + printf "$matches," >> $OUTFILE +} + +# populate a row of the ablation study table +populate_row() { + local greedy=$1 + local escape=$2 + local strategy=$(echo "$3" | sed 's/,/_/g') + local OUTFILE=$4 + + printf "$greedy,$escape," >> $OUTFILE + get_mean_stddev "$strategy" "$OUTFILE" + printf "," >> $OUTFILE + get_num_optimal "$strategy" "$OUTFILE" + printf "\n" >> $OUTFILE +} + +# prepare output file header +OUTFILE="results/ablation_studies.csv" +printf ",,25x 64x128,,,,25x 32x16\n" > $OUTFILE +printf ",,Efficacy,,Runtime,,Optimal\n" >> $OUTFILE +printf "Greedy Phase,Escape Phase,Mean,StdDev,Mean,,# Found\n" >> $OUTFILE + +# finally, gather the data for each strategy into a row of the table +populate_row "Random 1000" "-" "$R1000" "$OUTFILE" +populate_row "Channel Swap" "-" "$CS" "$OUTFILE" +populate_row "Channel Swap" "BR(100)" "$CS_100" "$OUTFILE" +populate_row "Channel Swap" "BR(1000)" "$CS_1000" "$OUTFILE" +populate_row "OSG(2)" "-" "$OSG2" "$OUTFILE" +populate_row "OSG(2)" "BR(100)" "$OSG2_100" "$OUTFILE" +populate_row "OSG(2)" "BR(1000)" "$OSG2_1000" "$OUTFILE" +populate_row "OSG(3)" "-" "$OSG3" "$OUTFILE" +populate_row "OSG(3)" "BR(100)" "$OSG3_100" "$OUTFILE" +populate_row "OSG(3)" "BR(1000)" "$OSG3_1000" "$OUTFILE" + +echo "Done! $OUTFILE" diff --git a/apex/apex/contrib/sparsity/permutation_tests/permutation_test.py b/apex/apex/contrib/sparsity/permutation_tests/permutation_test.py new file mode 100644 index 00000000..aae255ca --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_tests/permutation_test.py @@ -0,0 +1,270 @@ +import numpy as np +import time +import sys + +# permutation-specifics +sys.path.append("../") +from permutation_search_kernels.permutation_utilities import * +from permutation_search_kernels.exhaustive_search import Exhaustive_Search +from permutation_search_kernels.channel_swap import Channel_Swap + +# Arguments +import argparse +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + +parser = argparse.ArgumentParser(description='Test channel permutations') +parser.add_argument('--infile', default='random', type=str, help='input file or "random"') +parser.add_argument('--channels', default=384, type=int, help='random input channel count (C)') +parser.add_argument('--filters', default=96, type=int, help='random input filter count (K)') +parser.add_argument('--verbosity', default=0, type=int, help='print status updates') +parser.add_argument('--seed', default=1, type=int, help='random seed') +parser.add_argument('--pretty_print', default=True, type=str2bool, help='print the table for pretty viewing (as opposed to strict .csv)') +parser.add_argument('--unstructured', default=0.0, type=float, help='perform unstructured pruning to a target sparsity before processing, emulate an unstructured sparse network. "-1" will find the minimum sparsity required to achieve a perfect permutation') +parser.add_argument('--gpu', default=True, type=str2bool, help='uses a gpu to accelerate the search if possible') +parser.add_argument('--check_permutation', default=False, type=str2bool, help='check that the tracked permutation matches the recovered permutation') +parser.add_argument('--intermediate_steps', default=0, type=int, help='find roughly evenly-spaced permutations in efficacy') +parser.add_argument('--print_permutation', default=False, type=str2bool, help='print the final permutation found by each strategy') +parser.add_argument('strategies', metavar='strategy', type=str, nargs='+', help='strategies to try') + +## binary search for the minimum sparsity necessary to achieve a perfect permutation with some strategy +def find_minimum_sparsity(matrix, search_function, **kwargs): + duration = 0 + min_sparsity = 50 + max_sparsity = 100 + sparsity = 75 + verbosity = 0 + if 'verbosity' in kwargs: + verbosity = kwargs['verbosity'] + + while min_sparsity < max_sparsity: + if verbosity > 5: + print(f"\tlooking now at {sparsity} (between {min_sparsity} and {max_sparsity})") + + # prepare unstructured sparse matrix, get row sparsity magnitude + tmp_result = unstructured_prune(result, sparsity/100.0) + local_unpruned_magnitude = np.sum(np.abs(tmp_result)) + local_unstructured_rows_magnitude = magnitude_after_pruning_rows(tmp_result, rate=0.5) + + # quick check to see if this sparsity is trivially too low + if local_unstructured_rows_magnitude*1.0001 < local_unpruned_magnitude: + if verbosity > 5: + print(f"Skipping sparsity {sparsity} since there's no perfect permutation (unstructured mag {local_unpruned_magnitude} is larger than sparse rows {local_unstructured_rows_magnitude}).") + min_sparsity = sparsity+1 + sparsity = int(min_sparsity + (max_sparsity - min_sparsity)/2.0) + continue + + tmp_result, tmp_duration, found_permutation = search_function(tmp_result, **kwargs) + duration += tmp_duration + nonzeros = np.count_nonzero(tmp_result) + tmp_result = apply_2_to_4(tmp_result) + nonzeros_after_2to4 = np.count_nonzero(tmp_result) + if nonzeros == nonzeros_after_2to4: # found a winner, are we done? + if verbosity > 3: + print(f"Found an unstructured sparsity that we can turn into 2:4: {sparsity}") + + max_sparsity = sparsity + if max_sparsity <= min_sparsity and verbosity > 0: + print(f"Found the minimum unstructured sparsity that we can turn into 2:4: {sparsity}") + break + else: + if verbosity > 5: + print(f"Unstructured sparsity {sparsity} was insufficient to produce 2:4 sparsity") + min_sparsity = sparsity+1 + if max_sparsity <= min_sparsity and verbosity > 0: + print(f"Found the minimum unstructured sparsity that we can turn into 2:4: {max_sparsity}") + sparsity = max_sparsity + break + + sparsity = int(min_sparsity + (max_sparsity - min_sparsity)/2.0) + + return sparsity, duration + + +# Entry point +if __name__ == "__main__": + args = parser.parse_args() + verbosity = args.verbosity + np.random.seed(seed=args.seed) + use_gpu(initial_override=args.gpu) + + # get or create the input matrix + input_vals = np.random.rand(args.filters, args.channels) + if args.infile != "random": + if 'npy' in args.infile: + input_vals = np.load(args.infile, 'r') + shp = input_vals.shape + shp_str = str(shp).replace(",","x") + newshp_str = '' + if len(shp) == 4: # K,C,R,S -> RSK,C + input_vals = np.transpose(input_vals,(2,3,0,1)).flatten().reshape((shp[2]*shp[3]*shp[0], shp[1])) + newshp_str = str(input_vals.shape).replace(",","x") + print(f"{args.infile},{shp_str},{newshp_str}") + if input_vals.shape[1] % 4 != 0: + print(f"Unfriendly shape {input_vals.shape}, not pruning.") + sys.exit() + + # unstructured prune if requested + if args.unstructured > 0.0: + args.unstructured = min(args.unstructured, 1.0) + input_vals = unstructured_prune(input_vals, args.unstructured) + print(f"{args.infile} pruned to {args.unstructured*100.:>.1f} sparsity, shape is {input_vals.shape}") + + # calculate some early metrics + sorted_magnitudes = np.sort(np.abs(input_vals), axis=None) + unpruned_magnitude = np.sum(sorted_magnitudes) + num_weights = sorted_magnitudes.size + unstructured_magnitude = np.sum(sorted_magnitudes[int(num_weights/2):]) + unstructured_rows_magnitude = magnitude_after_pruning_rows(input_vals, rate=0.5) + simple_2to4 = apply_2_to_4(np.copy(input_vals)) + simple_2to4_magnitude = sum_after_2_to_4(input_vals) + tmp_time = time.perf_counter() + simple_2to4_magnitude = sum_after_2_to_4(input_vals) + default_duration = time.perf_counter() - tmp_time + best_magnitude = unstructured_rows_magnitude + + best_lost_magnitude = unpruned_magnitude - best_magnitude + base_lost_magnitude = unpruned_magnitude - simple_2to4_magnitude + + # prep results table + final_metric = 'efficacy' + if args.unstructured < 0.0: + final_metric = 'min_sparsity' + if args.pretty_print: + print(f"{'strategy':<35},{'magnitude':>15},{final_metric:>15},{'duration':>15}") + print(f"{'unpruned':<35},{unpruned_magnitude:>15.3f},{'-':^15},{'-':^15}") + print(f"{'unstructured':<35},{unstructured_magnitude:>15.3f},{'-':^15},{'-':^15}") + print(f"{'50% rows':<35},{unstructured_rows_magnitude:>15.3f},{'100.0':>15},{'-':^15}") + print(f"{'default 2:4':<35},{simple_2to4_magnitude:>15.3f},{'0.0':>15},{default_duration:>15.3f}") + else: + print(f"strategy,magnitude,{final_metric},duration") + print(f"unpruned,{unpruned_magnitude},-,-") + print(f"unstructured,{unstructured_magnitude},-,-") + print(f"50%_rows,{unstructured_rows_magnitude},100.0,-") + print(f"2:4,{simple_2to4_magnitude},0.0,{default_duration}") + + + # try the requested strategies + for i,strategy in enumerate(args.strategies): + result = np.copy(input_vals) + np.random.seed(seed=args.seed) + + duration = 0.0 + min_sparsity = 0.0 + strat_split = strategy.split(",") + found_permutation = None + + # optimize stripe groups + if strat_split[0] == 'optimize_stripe_groups': + stripe_group_size_in_cols = 8 + if len(strat_split) >= 2: + stripe_group_size_in_cols = int(strat_split[1]) + escape_attempts = 100 + if len(strat_split) >= 3: + escape_attempts = int(strat_split[2]) + + if args.unstructured >= 0.0: # just perform the search on the current matrix + result,duration,found_permutation = Exhaustive_Search(result, stripe_group_size=stripe_group_size_in_cols, escape_attempts=escape_attempts) + else: # find the minimum sparsity needed to transparently transform the input + min_sparsity,duration = find_minimum_sparsity(result, Exhaustive_Search, stripe_group_size=stripe_group_size_in_cols, escape_attempts=escape_attempts) + result = unstructured_prune(result, min_sparsity/100.0) + + # channel swaps + elif strat_split[0] == 'channel_swap': + escape_attempts= 0 + if len(strat_split) >= 2: + escape_attempts = int(strat_split[1]) + + if args.unstructured >= 0.0: # just perform the search on the current matrix + result,duration,found_permutation = Channel_Swap(result, escape_attempts=escape_attempts, verbosity=verbosity) + else: # find the minimum sparsity needed to transparently transform the input + min_sparsity,duration = find_minimum_sparsity(result, Channel_Swap, escape_attempts=escape_attempts, verbosity=verbosity) + result = unstructured_prune(result, min_sparsity/100.0) + + # random permutations + elif strat_split[0] == 'random': + if args.unstructured < 0.0: # searching for minimum sparsity not supported for random permutations + continue + + num_perms = 10 + if len(strat_split) >= 2 and int(strat_split[1]) >= 1: + num_perms = int(strat_split[1]) + + # try the seeds/permutations + permutation = [c for c in range(result.shape[1])] + best_sum = sum_after_2_to_4(result) + best_perm = permutation.copy() + start_time = time.perf_counter() + for x in range(num_perms): + permutation = np.random.permutation(permutation) + cur_sum = sum_after_2_to_4(result[:,permutation]) + if cur_sum > best_sum: + best_sum = cur_sum + best_perm = permutation.copy() + if verbosity > 0: + print(f"\tnew best permutation {x} found with magnitude {best_sum:>15.3f}") + elif verbosity > 5: + print(f"\tpermutation {x} magnitude too low: {cur_sum:>15.3f}") + duration = time.perf_counter() - start_time + result = result[:,best_perm] + found_permutation = best_perm + + else: + print(f"Unknown strategy: {strategy}!") + sys.exit() + + + # report stats for this strategy + cur_mag = sum_after_2_to_4(result) + cur_eff = efficacy(best_lost_magnitude, base_lost_magnitude, unpruned_magnitude - cur_mag)*100.0 + final_metric = cur_eff + if args.unstructured < 0.0: + final_metric = min_sparsity + perm_distance = "" + + error = None + if args.check_permutation and found_permutation is not None: + recovered_perm = find_permutation(result, input_vals) + + error = False + for c in range(len(recovered_perm)): + if recovered_perm[c] != found_permutation[c]: + if verbosity > 0: + print(f"tracked permutation at index {c} was {found_permutation[c]}, but the recovered permutation thought it was {recovered_perm[c]}") + error = True + + # if requested, generate permutations that divide the efficacy space into equal steps + if args.intermediate_steps != 0: + magnitude_targets = None + if args.intermediate_steps != 0: + ratios = [step/float(args.intermediate_steps+1) for step in range(1,args.intermediate_steps+1)] + mag_diff = cur_mag - (unpruned_magnitude - base_lost_magnitude) + magnitude_targets = [(unpruned_magnitude - base_lost_magnitude) + mag_diff * ratio for ratio in ratios] + perm_distance, target_permutations = permutation_distance(found_permutation, [c for c in range(result.shape[1])], matrix=input_vals, magnitude_targets=magnitude_targets, debug=False, verbosity=verbosity) + if target_permutations is not None: + for target_permutation in target_permutations: + print(target_permutation) + + error_str = "" + if error is not None: + error_str = ", correct" + if error: + error_str = ", mismatch" + + if args.pretty_print: + print(f"{strategy:35},{cur_mag:>15.3f},{final_metric:>15.1f},{duration:>15.3f}{error_str:>15}") + else: + strat_string = strategy.replace(",","_") + print(f"{strat_string},{cur_mag},{final_metric},{duration}{error_str}") + + if args.print_permutation and found_permutation is not None: + print(found_permutation) + + diff --git a/apex/apex/contrib/sparsity/permutation_tests/runtime_table.sh b/apex/apex/contrib/sparsity/permutation_tests/runtime_table.sh new file mode 100755 index 00000000..af9986e8 --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_tests/runtime_table.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +OUTDIR="results/runtime_logs" +mkdir -p $OUTDIR + +R1000=random,1000 +CS=channel_swap,0 +CS_100=channel_swap,100 +OSG2=optimize_stripe_groups,8,0 +OSG2_100=optimize_stripe_groups,8,100 +OSG2_1000=optimize_stripe_groups,8,1000 +OSG3=optimize_stripe_groups,12,0 +OSG3_100=optimize_stripe_groups,12,100 +OSG3_1000=optimize_stripe_groups,12,1000 + +for cols in "32" "64" "128" "256"; do + echo "$cols x $cols" + python3 permutation_test.py --channels $cols --filters $cols --pretty_print=False $R1000 $CS $CS_100 $OSG2 $OSG2_100 $OSG2_1000 $OSG3 $OSG3_100 $OSG3_1000 | tee "${OUTDIR}/runtime_${cols}x${cols}.log" + let "rows = $cols * 2" + echo "$cols x $rows" + python3 permutation_test.py --channels $cols --filters $rows --pretty_print=False $R1000 $CS $CS_100 $OSG2 $OSG2_100 $OSG2_1000 $OSG3 $OSG3_100 $OSG3_1000 | tee "${OUTDIR}/runtime_${cols}x${rows}.log" +done + +# 2048x2048 is too large for OSG3 +echo "2048 x 2048" +python3 permutation_test.py --channels 2048 --filters 2048 --pretty_print=False $R1000 $CS $CS_100 $OSG2 $OSG2_100 $OSG2_1000 | tee "${OUTDIR}/runtime_2048x2048.log" + + +############### collect results into a .csv file +echo "Gathering results ..." + +# efficacy and runtime from one strategy and size +get_results() { + local strategy=$1 + local cols=$2 + local rows=$3 + local OUTFILE=$4 + + grep "$strategy," "$OUTDIR/runtime_${cols}x${rows}.log" | awk -F "," '{printf "%s,%s,",$3,$4}' >> $OUTFILE +} + +# prepare output file headers +OUTFILE="results/runtimes.csv" +printf "Columns," > $OUTFILE +for cols in "32" "64" "128" "256"; do + printf "$cols,$cols,$cols,$cols," >> $OUTFILE +done +printf "2048,2048\n" >> $OUTFILE + +printf "Rows," >> $OUTFILE +for cols in "32" "64" "128" "256"; do + let "rows = $cols * 2" + printf "$cols,$cols,$rows,$rows," >> $OUTFILE +done +printf "2048,2048\n" >> $OUTFILE + +printf "Metric," >> $OUTFILE +for cols in "32" "64" "128" "256"; do + printf "Efficacy,Runtime,Efficay,Runtime," >> $OUTFILE +done +printf "Efficacy,Runtime\n" >> $OUTFILE + +# gather data in a reasonable order +for strategy in "$R1000" "$CS" "$CS_100" "$OSG2" "$OSG2_100" "$OSG2_1000" "$OSG3" "$OSG3_100" "$OSG3_1000"; do + strategy=$(echo "$strategy" | sed 's/,/_/g') # replace commas with underscores, as they'll appear in the results logs + printf "$strategy," >> $OUTFILE + for cols in "32" "64" "128" "256"; do + get_results "$strategy" "$cols" "$cols" "$OUTFILE" + let "rows = $cols * 2" + get_results "$strategy" "$cols" "$rows" "$OUTFILE" + done + + get_results "$strategy" "2048" "2048" "$OUTFILE" + + printf "\n" >> $OUTFILE +done + +echo "Done! $OUTFILE" diff --git a/apex/apex/contrib/sparsity/permutation_tests/unstructured_study.sh b/apex/apex/contrib/sparsity/permutation_tests/unstructured_study.sh new file mode 100755 index 00000000..c55b11fa --- /dev/null +++ b/apex/apex/contrib/sparsity/permutation_tests/unstructured_study.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Please specify both the source directory and a run tag: bash unstructured_study.sh " + exit +fi + +dir=$1 # or set to the directory containing .npy files of interest +tag=$2 # or set to an identifier, e.g. "network_name" + +resdir="results/unstructured_logs/${tag}" +mkdir -p $resdir + +CS=channel_swap,0 +OSG2=optimize_stripe_groups,8,0 +OSG2_100=optimize_stripe_groups,8,100 +OSG2_1000=optimize_stripe_groups,8,1000 +OSG3=optimize_stripe_groups,12,0 + +CS_successes=() +OSG2_successes=() +OSG2_100_successes=() +OSG2_1000_successes=() +OSG3_successes=() + +for sparsity in {50..100}; do + CS_successes+=(0) + OSG2_successes+=(0) + OSG2_100_successes+=(0) + OSG2_1000_successes+=(0) + OSG3_successes+=(0) +done + +update_successes () { + strategy=$1 + local -n _successes=$2 + logfile=$3 + + limit=$(grep "${strategy}," $logfile | awk -F "," '{print $3}') + + echo $logfile, $strategy, $limit + for (( sparsity=$limit; sparsity<=100; sparsity++ )); do + let "entry = $sparsity - 50" + let "value = ${_successes[$entry]} + 1" + _successes[$entry]=$value + done +} + +# Figure 4 +for filename in $dir/*.npy; do + out=$(basename -- "$filename") + echo "Searching for minimum sparsities for $out" + out=$resdir/$out.unstructured + python3 permutation_test.py --infile=$filename --pretty_print=False --unstructured=-1 $CS $OSG2 $OSG2_100 $OSG2_1000 $OSG3 > $out + + update_successes "channel_swap_0" CS_successes "$out" + update_successes "optimize_stripe_groups_8_0" OSG2_successes "$out" + update_successes "optimize_stripe_groups_8_100" OSG2_100_successes "$out" + update_successes "optimize_stripe_groups_8_1000" OSG2_1000_successes "$out" + update_successes "optimize_stripe_groups_12_0" OSG3_successes "$out" +done + +#################### save the table +# log a single strategy in as a row in the table +log_success () { + strategy=$1 + local -n _successes=$2 + OUTFILE=$3 + + printf "$strategy," >> $OUTFILE + for sparsity in {50..100}; do + let "entry = $sparsity - 50" + printf "%d," ${_successes[$entry]} >> $OUTFILE + done + printf "\n" >> $OUTFILE +} + +# prepare the header +OUTFILE="results/unstructured.csv" +printf "Sparsity," > $OUTFILE +for sparsity in {50..100}; do + printf "%d," $sparsity >> $OUTFILE +done +printf "\n" >> $OUTFILE + +# add data for each strategy +log_success "channel_swap_0" CS_successes "$OUTFILE" +log_success "optimize_stripe_groups_8_0" OSG2_successes "$OUTFILE" +log_success "optimize_stripe_groups_8_100" OSG2_100_successes "$OUTFILE" +log_success "optimize_stripe_groups_8_1000" OSG2_1000_successes "$OUTFILE" +log_success "optimize_stripe_groups_12_0" OSG3_successes "$OUTFILE" + +echo "Done! ${OUTFILE}" diff --git a/apex/apex/contrib/sparsity/sparse_masklib.py b/apex/apex/contrib/sparsity/sparse_masklib.py new file mode 100644 index 00000000..fce77cce --- /dev/null +++ b/apex/apex/contrib/sparsity/sparse_masklib.py @@ -0,0 +1,187 @@ +import sys +import torch +import numpy as np +import collections +from itertools import permutations + + +""" compute density (helper fn to compute % NNZs in a tensor) """ +def fill(x): + return float(x.nonzero().size(0))/torch.numel(x) + +""" reshape matrix into m-dimensional vectors: (h,w) -> (hw/m, m) """ +def reshape_1d(matrix, m): + # If not a nice multiple of m, fill with zeroes. + if matrix.shape[1] % m > 0: + mat = torch.cuda.FloatTensor(matrix.shape[0], matrix.shape[1] + (m-matrix.shape[1]%m)).fill_(0) + mat[:, :matrix.shape[1]] = matrix + shape = mat.shape + return mat.view(-1,m),shape + else: + return matrix.view(-1,m), matrix.shape + +""" return all possible m:n patterns in a 1d vector """ +valid_m4n2_1d_patterns = None +def compute_valid_1d_patterns(m,n): + # Early exit if patterns was already created. + global valid_m4n2_1d_patterns + + if m==4 and n==2 and valid_m4n2_1d_patterns is not None: return valid_m4n2_1d_patterns + patterns = torch.zeros(m) + patterns[:n] = 1 + valid_patterns = torch.tensor(list(set(permutations(patterns.tolist())))) + if m == 4 and n == 2: valid_m4n2_1d_patterns = valid_patterns + return valid_patterns + +""" m:n 1d structured best """ +def mn_1d_best(matrix, m, n): + # Find all possible patterns. + patterns = compute_valid_1d_patterns(m,n).cuda() + + # Find the best m:n pattern (sum of non-masked weights). + mask = torch.cuda.IntTensor(matrix.shape).fill_(1).view(-1,m) + mat,shape = reshape_1d(matrix,m) + pmax = torch.argmax(torch.matmul(mat.abs(),patterns.t()), dim=1) + mask[:] = patterns[pmax[:]] + mask = mask.view(matrix.shape) + return mask + +def m4n2_1d(mat, density): + return mn_1d_best(mat, 4, 2) + +""" + Below 2d-masking related code is targeted more for training (from scratch). + 2d-pruning of a weight tensor is done to accelerate DGRAD step during backprop + phase of training algorithm. Acceleration comes from using SpMMA instructions in + Tensor Cores of NVIDIA Ampere GPU Architecture + (note: this code does not do the acceleration, GPU kernels are required for this). + 1d pruning of weight tensor helps speed up FPROP step by pruning in 2:4 pattern + along the horizontal (logical) direction. + During DGRAD step, weight tensor is transposed. 2d pruning functions below, mask + weight tensor such that their transposed versions are also 2:4 sparse along the + horizontal (logical) direction. Thus, with 2d pruning, weight tensors are + 2:4 sparse along row and column directions. + """ + +""" m:n 2d structured pruning: greedy method to select mask """ +def mn_2d_greedy(matrix, m, n): + # Convert to numpy + mat = matrix.cpu().detach().numpy() + mask = np.ones(mat.shape, dtype=int) + + rowCount = int(mat.shape[0]/m) * m + colCount = int(mat.shape[1]/m) * m + for rowStartIdx in range(0, rowCount, m): + rowEndIdx = rowStartIdx + m + for colStartIdx in range(0, colCount, m): + colEndIdx = colStartIdx + m + matrixSub = np.absolute(np.squeeze(mat[rowStartIdx:rowEndIdx, colStartIdx:colEndIdx])) + maskSub = np.squeeze(mask[rowStartIdx:rowEndIdx, colStartIdx:colEndIdx]) + maskSub.fill(0.0) + matrixVecView = matrixSub.reshape(-1) + maskVecView = maskSub.reshape(-1) + linearIdx = np.argsort(matrixVecView) + matrixIdx = [(int(x/m), x % m) for x in linearIdx] + rowCounter = collections.Counter() + colCounter = collections.Counter() + for currIdx in range(len(linearIdx) - 1, -1, -1): + currMatrixEntry = matrixIdx[currIdx] + if (rowCounter[currMatrixEntry[0]] == n) or (colCounter[currMatrixEntry[1]] == n): + continue + #end if + maskSub[currMatrixEntry[0], currMatrixEntry[1]] = 1.0 + rowCounter[currMatrixEntry[0]] += 1 + colCounter[currMatrixEntry[1]] += 1 + + return torch.tensor(mask.cuda()) + +def m4n2_2d_greedy(mat, density): + return mn_2d_greedy(mat, 4, 2) + +""" return all possible m:n patterns in a mxn block. """ +valid_m4n2_2d_patterns = None +def compute_valid_2d_patterns(m,n): + # Early exit if patterns was already created. + global valid_m4n2_2d_patterns + if valid_m4n2_2d_patterns is not None: return valid_m4n2_2d_patterns + + patterns = torch.zeros(m) + patterns[:n] = 1 + patterns = list(set(permutations(patterns.tolist()))) + patterns = patterns + patterns + patterns = torch.empty(list(set(permutations(patterns,m)))) + + valid = ((patterns.sum(dim=1) <= n).sum(dim=1) == m).nonzero().view(-1) + valid_patterns = torch.empty(valid.shape[0],m,m) + valid_patterns[:] = patterns[valid[:]] + + if m == 4 and n == 2: valid_m4n2_2d_patterns = valid_patterns + return valid_patterns + +""" m:n 2d structured pruning: exhaustive method to select best mask """ +def mn_2d_best(matrix, m, n): + # Find all possible patterns. + patterns = compute_valid_2d_patterns(m,n).cuda() + + # Find the best m:n pattern (sum of non-masked weights). + mask = torch.cuda.IntTensor(matrix.shape).fill_(1) + mat = reshape_2d(matrix,m,m).abs() + pmax = torch.argmax(torch.matmul(mat,patterns.view(patterns.shape[0],m*m).t()), dim=2) + + # Copy best m:n patterns into mask. + mat = mat.view(mat.shape[0]*mat.shape[1],-1) + pmax = pmax.view(pmax.shape[0]*pmax.shape[1]).unsqueeze(1).expand(-1,mat.shape[1]) + patterns = patterns.view(patterns.shape[0],patterns.shape[1]*patterns.shape[2]) + mat = torch.gather(patterns,0,pmax) + mat = reshape_2d_inv(mat.view(matrix.shape[0]//m,matrix.shape[1]//m,m,m)) + mask.copy_(mat.type(mask.type())) + return mask + +def m4n2_2d_best(mat, density): + return mn_2d_best(mat, 4, 2) + + +""" returns a sparse mask """ +def create_mask(tensor, pattern="m4n2_1d", density=0.5): + # Reshape tensor and mask. + shape = tensor.shape + ttype = tensor.type() + t = tensor.float().contiguous() + + # 1d-tensor + if len(shape) == 1: + t = t.view(1, shape[0]) + func = getattr(sys.modules[__name__], pattern, None) + mask = func(t, density) + return mask.view(shape).type(ttype) + # 2d-tensor (K, C) + elif len(shape) == 2: + # linear + t = t.view(shape[0], shape[1]) + func = getattr(sys.modules[__name__], pattern, None) + mask = func(t, density) + return mask.view(shape).type(ttype) + # 3d-tensor (K, C, R) + elif len(shape) == 3: + # 1d convs + t = t.permute(0,2,1).contiguous().view(shape[0]*shape[2], shape[1]) + func = getattr(sys.modules[__name__], pattern, None) + mask = func(t, density) + mask = mask.view(shape[0], shape[2], shape[1]).permute(0,2,1).contiguous() + return mask.view(shape).type(ttype) + # 4d-tensor (K, C, R, S) + elif len(shape) == 4: + """ + # transformers (bmm) + t = t.view(shape[0]*shape[1]*shape[2], shape[3]) + func = getattr(sys.modules[__name__], pattern, None) + mask = func(t, density) + return mask.view(shape).type(ttype) + """ + # 2d convs + t = t.permute(2,3,0,1).contiguous().view(shape[2]*shape[3]*shape[0], shape[1]) + func = getattr(sys.modules[__name__], pattern, None) + mask = func(t, density) + mask = mask.view(shape[2], shape[3], shape[0], shape[1]).permute(2,3,0,1).contiguous() + return mask.view(shape).type(ttype) + diff --git a/apex/apex/contrib/sparsity/test/checkpointing_test_part1.py b/apex/apex/contrib/sparsity/test/checkpointing_test_part1.py new file mode 100644 index 00000000..34232b82 --- /dev/null +++ b/apex/apex/contrib/sparsity/test/checkpointing_test_part1.py @@ -0,0 +1,94 @@ +from collections import OrderedDict + +import torch +from apex.optimizers import FusedAdam +from apex.contrib.sparsity import ASP + +def build_model(args): + od = OrderedDict() + for i in range(args.num_layers): + if i == 0: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.input_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + elif i == args.num_layers-1: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.output_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.output_features]) + else: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + return torch.nn.Sequential(od) + +def train_step(args, model, optimizer, input_batch, target_batch, step): + predicted_target = model(input_batch) + loss = ((predicted_target-target_batch)**2).sum() + loss.backward() + optimizer.step() + optimizer.zero_grad() + step = step + 1 + #print("Step %d :: loss=%e" % (step, loss.item())) + return step + +def train_loop(args, model, optimizer, step, num_steps): + for i in range(num_steps): + input_batch = torch.randn([args.batch_size, args.input_features]).cuda() + target_batch = torch.randn([args.batch_size, args.output_features]).cuda() + step = train_step(args, model, optimizer, input_batch, target_batch, step) + return step + +def main(args): + # + # PART1 + # + + torch.manual_seed(args.seed) + + model = build_model(args).cuda() + one_ll = next(model.children()).weight + optimizer = FusedAdam(model.parameters()) + ASP.init_model_for_pruning(model, args.pattern, verbosity=args.verbosity, whitelist=args.whitelist, allow_recompute_mask=args.allow_recompute_mask) + ASP.init_optimizer_for_pruning(optimizer) + + step = 0 + + # train for a few steps with dense weights + print("DENSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_dense_steps) + + # simulate sparsity by inserting zeros into existing dense weights + ASP.compute_sparse_masks() + + # train for a few steps with sparse weights + print("SPARSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_sparse_steps) + + torch.save({ + 'step': step, + 'verbosity': args.verbosity, + 'seed2': args.seed2, + 'pattern': args.pattern, + 'whitelist': args.whitelist, + 'allow_recompute_mask': args.allow_recompute_mask, + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + }, args.checkpoint_path) + +if __name__ == '__main__': + class Args: + verbosity=3 + seed = 4873 + seed2 = 99875 + pattern = "m4n2_2d_best" + whitelist = [torch.nn.Linear] + allow_recompute_mask = True + batch_size = 32 + input_features = 8 + output_features = 8 + hidden_features = 32 + num_layers = 4 + num_dense_steps = 2000 + num_sparse_steps = 3000 + num_sparse_steps_2 = 1000 + checkpoint_path = "part1.chkp" + args = Args() + + main(args) diff --git a/apex/apex/contrib/sparsity/test/checkpointing_test_part2.py b/apex/apex/contrib/sparsity/test/checkpointing_test_part2.py new file mode 100644 index 00000000..d2b161cb --- /dev/null +++ b/apex/apex/contrib/sparsity/test/checkpointing_test_part2.py @@ -0,0 +1,79 @@ +from collections import OrderedDict + +import torch +from apex.optimizers import FusedAdam +from apex.contrib.sparsity import ASP + +def build_model(args): + od = OrderedDict() + for i in range(args.num_layers): + if i == 0: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.input_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + elif i == args.num_layers-1: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.output_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.output_features]) + else: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + return torch.nn.Sequential(od) + +def train_step(args, model, optimizer, input_batch, target_batch, step): + predicted_target = model(input_batch) + loss = ((predicted_target-target_batch)**2).sum() + loss.backward() + optimizer.step() + optimizer.zero_grad() + step = step + 1 + #print("Step %d :: loss=%e" % (step, loss.item())) + return step + +def train_loop(args, model, optimizer, step, num_steps): + for i in range(num_steps): + input_batch = torch.randn([args.batch_size, args.input_features]).cuda() + target_batch = torch.randn([args.batch_size, args.output_features]).cuda() + step = train_step(args, model, optimizer, input_batch, target_batch, step) + return step + +def main(step, args, model_state_dict, optimizer_state_dict): + # + # PART2 + # + + model = build_model(args).cuda() + one_ll = next(model.children()).weight + optimizer = FusedAdam(model.parameters()) + ASP.init_model_for_pruning(model, args.pattern, verbosity=args.verbosity, whitelist=args.whitelist, allow_recompute_mask=args.allow_recompute_mask) + ASP.init_optimizer_for_pruning(optimizer) + + torch.manual_seed(args.seed2) + model.load_state_dict(model_state_dict) + optimizer.load_state_dict(optimizer_state_dict) + + print("Model sparsity is %s" % ("enabled" if ASP.is_sparsity_enabled() else "disabled")) + + # train for a few steps with sparse weights + print("SPARSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_sparse_steps_2) + +if __name__ == '__main__': + checkpoint = torch.load("part1.chkp") + class Args: + verbosity = checkpoint['verbosity'] + seed = 4873 + seed2 = checkpoint['seed2'] + pattern = checkpoint['pattern'] + whitelist = checkpoint['whitelist'] + allow_recompute_mask = checkpoint['allow_recompute_mask'] + batch_size = 32 + input_features = 8 + output_features = 8 + hidden_features = 32 + num_layers = 4 + num_dense_steps = 2000 + num_sparse_steps = 3000 + num_sparse_steps_2 = 1000 + checkpoint_path = "part1.chkp" + args = Args() + + main(checkpoint['step'], args, checkpoint['model_state_dict'], checkpoint['optimizer_state_dict']) diff --git a/apex/apex/contrib/sparsity/test/checkpointing_test_reference.py b/apex/apex/contrib/sparsity/test/checkpointing_test_reference.py new file mode 100644 index 00000000..57ea51a3 --- /dev/null +++ b/apex/apex/contrib/sparsity/test/checkpointing_test_reference.py @@ -0,0 +1,96 @@ +from collections import OrderedDict + +import torch +from apex.optimizers import FusedAdam +from apex.contrib.sparsity import ASP + +# +# Reference run for checkpointing test (part1 + part2) +# + +def build_model(args): + od = OrderedDict() + for i in range(args.num_layers): + if i == 0: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.input_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + elif i == args.num_layers-1: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.output_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.output_features]) + else: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + return torch.nn.Sequential(od) + +def train_step(args, model, optimizer, input_batch, target_batch, step): + predicted_target = model(input_batch) + loss = ((predicted_target-target_batch)**2).sum() + loss.backward() + optimizer.step() + optimizer.zero_grad() + step = step + 1 + #print("Step %d :: loss=%e" % (step, loss.item())) + return step + +def train_loop(args, model, optimizer, step, num_steps): + for i in range(num_steps): + input_batch = torch.randn([args.batch_size, args.input_features]).cuda() + target_batch = torch.randn([args.batch_size, args.output_features]).cuda() + step = train_step(args, model, optimizer, input_batch, target_batch, step) + return step + +def main(args): + # + # PART1 + # + + torch.manual_seed(args.seed) + + model = build_model(args).cuda() + one_ll = next(model.children()).weight + optimizer = FusedAdam(model.parameters()) + ASP.init_model_for_pruning(model, args.pattern, whitelist=args.whitelist, allow_recompute_mask=args.allow_recompute_mask) + ASP.init_optimizer_for_pruning(optimizer) + + step = 0 + + # train for a few steps with dense weights + print("DENSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_dense_steps) + + # simulate sparsity by inserting zeros into existing dense weights + ASP.compute_sparse_masks() + + # train for a few steps with sparse weights + print("SPARSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_sparse_steps) + + # + # PART 2 + # + + torch.manual_seed(args.seed2) + + # train for a few steps with sparse weights + print("SPARSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_sparse_steps_2) + +if __name__ == '__main__': + class Args: + seed = 4873 + seed2 = 99875 + pattern = "m4n2_2d_best" + whitelist = [torch.nn.Linear] + allow_recompute_mask = True + batch_size = 32 + input_features = 8 + output_features = 8 + hidden_features = 32 + num_layers = 4 + num_dense_steps = 2000 + num_sparse_steps = 3000 + num_sparse_steps_2 = 1000 + checkpoint_path = "part1.chkp" + args = Args() + + main(args) diff --git a/apex/apex/contrib/sparsity/test/test_permutation_application.py b/apex/apex/contrib/sparsity/test/test_permutation_application.py new file mode 100644 index 00000000..27a96e58 --- /dev/null +++ b/apex/apex/contrib/sparsity/test/test_permutation_application.py @@ -0,0 +1,792 @@ +import torch +import torch.onnx +from apex.contrib.sparsity.permutation_lib import Permutation + +""" +Functional and behavioral correctness checking for network permutations +Each test class is a torch.nn.Module with three required members: +- self.input_shape is used to populate a dummy input +- self.expected_C_params indicates how many parameters are expected to be permuted in the C dimension +- self.expected_K_params indicates how many parameters are expected to be permuted in the K dimension + +A test is successful if and only if: +1. The output of the un-permuted module matches (within a tolerance) the ouput of the permuted module +2. The number of parameters permuted in C, as reported by the Permutation class, matches the expected value in the test module +3. The number of parameters permuted in K, as reported by the Permutation class, matches the expected value in the test module + +This file has all the test modules defined first, followed by the common test routine to check each module's correctness, and finally the main/entry point. +""" + +class simple_convs(torch.nn.Module): + """Stack of 2d convolutions with different normalization and activation functions""" + + def __init__( + self, + num_convs: int, + channels: int, + normalization: str = 'none', + activation: str = 'ReLU', + ): + super().__init__() + self.num_convs = num_convs + self.channels = channels + self.normalization = normalization + self.activation = activation + + self.input_shape = [4, channels, 7, 7] + + # we'll permute all convs' weights along C except the first + self.expected_C_params = -1 + self.expected_K_params = 0 + + self.conv_stack = torch.nn.Sequential() + for c in range(self.num_convs-1): + self.conv_stack.add_module(f"conv_{c}", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1)) + self.expected_C_params += 1 + self.expected_K_params += 2 + + if self.normalization == 'BatchNorm2d': + self.conv_stack.add_module(f"norm_{c}", torch.nn.BatchNorm2d(self.channels, track_running_stats=False)) + self.expected_K_params += 2 + elif self.normalization == 'LazyBatchNorm2d': + self.conv_stack.add_module(f"norm_{c}", torch.nn.LazyBatchNorm2d(track_running_stats=False)) + self.expected_K_params += 2 + elif self.normalization == 'GroupNorm': + self.conv_stack.add_module(f"norm_{c}", torch.nn.GroupNorm(4, self.channels, affine=True)) + self.expected_C_params -= 1 # GN prevents permutations of the neighboring convs + self.expected_K_params -= 2 + elif self.normalization == 'InstanceNorm2d': + self.conv_stack.add_module(f"norm_{c}", torch.nn.InstanceNorm2d(self.channels, affine=True, track_running_stats=False)) + self.expected_K_params += 2 + elif self.normalization == 'LocalResponseNorm': + self.conv_stack.add_module(f"norm_{c}", torch.nn.LocalResponseNorm(16)) + elif self.normalization == 'LayerNorm1': + self.conv_stack.add_module(f"norm_{c}", torch.nn.LayerNorm(7)) + elif self.normalization == 'LayerNorm2': + self.conv_stack.add_module(f"norm_{c}", torch.nn.LayerNorm([7, 7])) + elif self.normalization == 'LayerNorm3': + self.conv_stack.add_module(f"norm_{c}", torch.nn.LayerNorm([self.channels, 7, 7])) + self.expected_K_params += 2 + elif self.normalization == 'SyncBatchNorm': + self.conv_stack.add_module(f"norm_{c}", torch.nn.SyncBatchNorm(self.channels, track_running_stats=False)) + self.expected_K_params += 2 + + self.conv_stack.add_module(f"act_{c}", torch.nn.ReLU()) + + self.conv_stack.add_module("conv_out", torch.nn.Conv2d(self.channels, 8, kernel_size=(1,1))) + self.expected_C_params += 1 + + def forward(self, x: torch.Tensor): + + x = self.conv_stack(x) + + return x + +class conv_1d(torch.nn.Module): + """1D convolutions in isolation and with siblings""" + + def __init__( + self, + with_2d = False, + ): + super().__init__() + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + self.with_2d = with_2d + + self.input_conv = torch.nn.Conv2d(self.input_shape[1], 32, kernel_size=(3,3), padding=1) + self.expected_K_params += 2 + + self.branch_a_1D = torch.nn.Conv1d(32, 32, kernel_size=3, padding=1) + self.expected_C_params += 1 + self.expected_K_params += 2 + if self.with_2d: + self.branch_b_2D = torch.nn.Conv2d(32, 32, kernel_size=(3,3), padding=1) + self.expected_C_params += 1 + self.expected_K_params += 2 + + self.out_conv = torch.nn.Conv2d(32, 8, kernel_size=(1,1)) + self.expected_C_params += 1 + + def forward(self, x: torch.Tensor): + + step0 = self.input_conv(x) + + s0shape = step0.shape + step1 = self.branch_a_1D(step0.view(s0shape[0], s0shape[1], s0shape[2]*s0shape[3])).view(s0shape) + if self.with_2d: + step1 = step1 + self.branch_b_2D(step0) + + return self.out_conv(step1) + +class grouped_convs(torch.nn.Module): + """Stack of 2d convolutions with different types of grouped convolutions""" + + def __init__( + self, + ): + super().__init__() + self.channels = 128 + self.input_shape = [4, self.channels, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.conv_stack = torch.nn.Sequential() + self.conv_stack.add_module("conv_in", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1)) + + # dw conv will let previous and this layers' weights and biases permute along K + self.expected_K_params += 4 + self.conv_stack.add_module("conv_dw", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1, groups=self.channels)) + + # regular conv permutes both + self.expected_C_params += 1 + self.expected_K_params += 2 + self.conv_stack.add_module("conv_0", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1, groups=1)) # explicit '1' groups for extra coverage + + # only 2 groups should allow permutations only in C + self.expected_C_params += 1 + self.conv_stack.add_module("conv_gr2", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1, groups=2)) + + # another regular conv, this one can't do anything + self.conv_stack.add_module("conv_1", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1)) + + # finally, grouped conv with small groups + self.conv_stack.add_module("conv_gr64", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1, groups=self.channels//2)) + + def forward(self, input: torch.Tensor): + + return self.conv_stack(input) + +class simple_forks_joins(torch.nn.Module): + """Some simple residual connections to test collecting parameters into a single group. Four sections: input, blocka + residual, blockb + blockc, output""" + + def __init__( + self, + ): + super().__init__() + self.channels = 64 + self.input_shape = [4, self.channels, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.input_convs = torch.nn.Sequential() + # input conv can only permute along K + self.expected_K_params += 2 + self.input_convs.add_module("conv_in0", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1)) + # the next conv can permute along both C and K + self.expected_C_params += 1 + self.expected_K_params += 2 + self.input_convs.add_module("conv_in1", torch.nn.Conv2d(self.channels, self.channels, kernel_size=(3,3), padding=1)) + # BN will permute 2 more along K + self.expected_K_params += 2 + self.input_convs.add_module("bn_in1", torch.nn.BatchNorm2d(self.channels, track_running_stats=False)) + + self.block_a = torch.nn.Sequential() + # cut channels in half, then back to full, two fully permutable convs + self.expected_C_params += 2 + self.expected_K_params += 4 + self.block_a.add_module("conv_a0", torch.nn.Conv2d(self.channels, self.channels // 2, kernel_size=(3,3), padding=1)) + self.block_a.add_module("conv_a1", torch.nn.Conv2d(self.channels // 2, self.channels, kernel_size=(3,3), padding=1)) + + self.block_b = torch.nn.Sequential() + # cut channels in half, then back to full, two fully permutable convs + self.expected_C_params += 2 + self.expected_K_params += 4 + self.block_b.add_module("conv_b0", torch.nn.Conv2d(self.channels, self.channels // 2, kernel_size=(3,3), padding=1)) + self.block_b.add_module("conv_b1", torch.nn.Conv2d(self.channels // 2, self.channels, kernel_size=(3,3), padding=1)) + + self.block_c = torch.nn.Sequential() + # cut channels in half, then back to full, two fully permutable convs + self.expected_C_params += 2 + self.expected_K_params += 4 + self.block_c.add_module("conv_c0", torch.nn.Conv2d(self.channels, self.channels // 2, kernel_size=(3,3), padding=1)) + self.block_c.add_module("conv_c1", torch.nn.Conv2d(self.channels // 2, self.channels, kernel_size=(3,3), padding=1)) + + self.output_conv = torch.nn.Sequential() + self.expected_C_params += 1 + self.output_conv.add_module("conv_out", torch.nn.Conv2d(self.channels, 8, kernel_size=(3,3), padding=1)) + + def forward(self, input: torch.Tensor): + step0 = self.input_convs(input) + step1 = step0 + self.block_a(step0) + step2 = self.block_b(step1) + self.block_c(step1) + return self.output_conv(step2) + +class different_grouped_convs(torch.nn.Module): + """Convolutions with different group sizes need to use the GCD of the input channel counts if siblings""" + + def __init__( + self, + ): + super().__init__() + self.channels = 16 + self.input_shape = [4, self.channels, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.input_conv = torch.nn.Sequential() + self.expected_K_params += 2 + self.input_conv.add_module("input_conv", torch.nn.Conv2d(self.channels, 128, kernel_size=(3,3), padding=1)) + + self.expected_C_params += 4 + # 4 parallel blocks with decreasing group size from "left" to "right" + self.block_a = torch.nn.Sequential() + self.block_a.add_module("conv_a", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + self.block_b = torch.nn.Sequential() + self.block_b.add_module("conv_b", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1, groups=2)) + self.block_c = torch.nn.Sequential() + self.block_c.add_module("conv_c", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1, groups=4)) + self.block_d = torch.nn.Sequential() + self.block_d.add_module("conv_d", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1, groups=8)) + + # output can't permute along C, disallowed by parents + self.output_conv = torch.nn.Sequential() + self.output_conv.add_module("output_conv", torch.nn.Conv2d(128, 8, kernel_size=(3,3), padding=1)) + + def forward(self, input: torch.Tensor): + step0 = self.input_conv(input) + step1 = self.block_a(step0) + self.block_b(step0) + self.block_c(step0) + self.block_d(step0) + return self.output_conv(step1) + +class siblings_poison(torch.nn.Module): + """A single sibling that cannot permute along C poisons all other siblings in its group""" + + def __init__( + self, + ): + super().__init__() + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.input_conv = torch.nn.Sequential() + self.input_conv.add_module("input_conv", torch.nn.Conv2d(self.input_shape[1], 128, kernel_size=(3,3), padding=1)) + + # two parallel block: conv->flatten->linear | flatten->linear + self.expected_K_params += 4 # two linears will have their output channels permuted for the output layer + self.block_a = torch.nn.Sequential() + self.block_a.add_module("conv_a", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + self.block_a.add_module("flatten_a", torch.nn.Flatten(1)) + self.block_a.add_module("linear_a", torch.nn.Linear(6272, 128)) + + self.block_b = torch.nn.Sequential() + self.block_b.add_module("flatten_b", torch.nn.Flatten(1)) + self.block_b.add_module("linear_b", torch.nn.Linear(6272, 128)) + + self.output = torch.nn.Sequential() + self.expected_C_params += 1 # output layer will have its C dimension permuted + self.output.add_module("output", torch.nn.Linear(128, 8)) + + def forward(self, input: torch.Tensor): + step0 = self.input_conv(input) + step1 = self.block_a(step0) + self.block_b(step0) + return self.output(step1) + +class coparent_poison(torch.nn.Module): + """A single coparent that cannot permute along K poisons all other coparents in its group""" + + def __init__( + self, + ): + super().__init__() + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.input_conv = torch.nn.Sequential() + self.expected_K_params += 2 + self.input_conv.add_module("input_conv", torch.nn.Conv2d(self.input_shape[1], 128, kernel_size=(3,3), padding=1)) + + # two parallel block: conv | conv-> grouped conv + self.expected_C_params += 3 # all convs permute along C + self.expected_K_params += 2 # only conv_b0 permutes along K + self.block_a = torch.nn.Sequential() + self.block_a.add_module("conv_a", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + + self.block_b = torch.nn.Sequential() + self.block_b.add_module("conv_b0", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + self.block_b.add_module("conv_b1", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1, groups=4)) + + self.output = torch.nn.Sequential() + self.output.add_module("output", torch.nn.Conv2d(128, 8, kernel_size=(1,1))) + + def forward(self, input: torch.Tensor): + step0 = self.input_conv(input) + step1 = self.block_a(step0) + self.block_b(step0) + return self.output(step1) + + +class depthwise_child_is_sibling(torch.nn.Module): + """The child of a depthwise convolution should act as a sibling""" + + def __init__( + self, + ): + super().__init__() + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.input_conv = torch.nn.Sequential() + self.expected_K_params += 2 + self.input_conv.add_module("input_conv", torch.nn.Conv2d(self.input_shape[1], 128, kernel_size=(3,3), padding=1)) + + # two parallel block: conv | depthwise->conv + self.expected_C_params += 2 + self.expected_K_params += 4 + 2 + self.block_a = torch.nn.Sequential() + self.block_a.add_module("conv_a", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + + self.block_b = torch.nn.Sequential() + self.block_b.add_module("conv_b_dw", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1, groups=128)) + self.block_b.add_module("conv_b_1", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + + self.output_conv = torch.nn.Sequential() + self.expected_C_params += 1 + self.output_conv.add_module("output_conv", torch.nn.Conv2d(128, 8, kernel_size=(1,1))) + + def forward(self, input: torch.Tensor): + step0 = self.input_conv(input) + step1 = self.block_a(step0) + self.block_b(step0) + return self.output_conv(step1) + + +class module_attribute(torch.nn.Module): + """Attributes of some module must be permuted if they feed some operation that is permuted""" + + def __init__( + self, + complexity: int = 0, + ): + super().__init__() + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + self.complexity = complexity + + self.input_conv = torch.nn.Sequential() + self.expected_K_params += 3 # conv weight, conv bias, input_offset C (counts as K since it's acting as a parent) + self.input_offset = torch.nn.Parameter(torch.zeros(128,7,7)) + torch.nn.init.normal_(self.input_offset.data, mean=0.0, std=2.0) + self.input_conv.add_module("conv_input", torch.nn.Conv2d(self.input_shape[1], 128, kernel_size=(3,3), padding=1)) + + # add a couple more layers, and let the same offset affect another layer, as well + if complexity == 1: + self.expected_C_params += 2 + self.expected_K_params += 4 + self.stack_a = torch.nn.Sequential() + self.stack_a.add_module("conv_a", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + + self.stack_b = torch.nn.Sequential() + self.stack_b.add_module("conv_b", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + + + self.output_conv = torch.nn.Sequential() + self.expected_C_params += 1 + self.output_conv.add_module("conv_output", torch.nn.Conv2d(128, 8, kernel_size=(3,3))) + + def forward(self, input: torch.Tensor): + batch_input_offset = self.input_offset.expand(input.shape[0], -1, -1, -1) + x = self.input_conv(input) + batch_input_offset + if self.complexity == 1: + x = self.stack_a(x) + batch_input_offset + x = self.stack_b(x) + batch_input_offset + return self.output_conv(x) + + +class square_attribute(torch.nn.Module): + """Attributes with multiple dimensions matching the permutation length should only be permuted along the correct dimension""" + # TODO: currently, such an attribute will disallow permutations around it, but with effort, it could be handled correctly. + + def __init__( + self, + ): + super().__init__() + self.input_shape = [4, 16, 16] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.input_linear = torch.nn.Sequential() + #self.expected_K_params += 2 # if handled correctly, the linear's K and the offset's K should both be permuted + self.input_linear.add_module("linear_input", torch.nn.Linear(self.input_shape[1], 16)) + self.input_offset = torch.nn.Parameter(torch.zeros(16, 16)) + torch.nn.init.normal_(self.input_offset.data, mean=0.0, std=2.0) + + self.output_linear = torch.nn.Sequential() + #self.expected_C_params += 1 # if handled correctly, this should be permuted + self.output_linear.add_module("linear_output", torch.nn.Linear(16, 8)) + + def forward(self, input: torch.Tensor): + batch_input_offset = self.input_offset.expand(input.shape[0], -1, -1) + x = self.input_linear(input) + torch.permute(batch_input_offset, (0, 2, 1)) + return self.output_linear(x) + + +class MHA_test(torch.nn.Module): + """MultiheadAttention modules are unique, we need to check permutations for input and ouput projections""" + + def __init__( + self, + hidden_dim: int = 256, + seq_len: int = 64, + num_heads: int = 16 + ): + super().__init__() + self.hidden_dim = hidden_dim + self.seq_len = seq_len + self.num_heads = num_heads + self.input_shape = [4, self.seq_len, self.hidden_dim] + + self.expected_C_params = 1 + self.expected_K_params = 2 + + self.MHA0 = torch.nn.MultiheadAttention(self.hidden_dim, self.num_heads, dropout=False, batch_first=True) + self.MHA1 = torch.nn.MultiheadAttention(self.hidden_dim, self.num_heads, dropout=False, batch_first=True) + + def forward(self, input: torch.Tensor): + step0,_ = self.MHA0(input, input, input) + step1,_ = self.MHA1(step0, step0, step0) + return step1 + + +class one_sparse_sibling(torch.nn.Module): + """If only one of two siblings is sparse, both need to be permuted""" + + def __init__( + self, + ): + super().__init__() + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.in_conv = torch.nn.Sequential() + self.expected_K_params += 2 + self.in_conv.add_module("conv_in", torch.nn.Conv2d(self.input_shape[1], 128, kernel_size=(3,3), padding=1)) + + self.block_a = torch.nn.Sequential() + self.expected_C_params += 1 # only conv_a0 will be permuted along C + self.expected_K_params += 2 # only conv_a1 will be permuted along K + self.block_a.add_module("conv_a0", torch.nn.Conv2d(128, 3, kernel_size=(1,1))) + self.block_a.add_module("conv_a1", torch.nn.Conv2d(3, 128, kernel_size=(3,3), padding=1)) + + self.block_b = torch.nn.Sequential() + self.expected_C_params += 2 # even though conv_a0 will not be sparse (only 3 output channels), conv_b0 can still be permuted along C + self.expected_K_params += 4 + self.block_b.add_module("conv_b0", torch.nn.Conv2d(128, 128, kernel_size=(3,3), padding=1)) + self.block_b.add_module("conv_b1", torch.nn.Conv2d(128, 128, kernel_size=(1,1))) + + self.out_conv = torch.nn.Sequential() + self.expected_C_params += 1 + self.out_conv.add_module("conv_out", torch.nn.Conv2d(128, 8, kernel_size=(1,1))) + + def forward(self, input: torch.Tensor): + step0 = self.in_conv(input) + step1 = self.block_a(step0) + self.block_b(step0) + return self.out_conv(step1) + + +class test_concat(torch.nn.Module): + """If concats are along the channel dimension (dim1 of NCHW), downstream layers can still be permuted despite C!=parentK""" + + def __init__( + self, + ratio = 1, # ratio between # channels in either path to be concatenated + dim = 1, # dimension to concatenate, K by default + depth = 1, # number of concats to stack + ): + super().__init__() + assert dim == 1 or ratio == 1 ,"can't concat along dimensions other than K if K's don't match" + self.dim = dim + self.depth = depth + self.input_shape = [4, 16, 7, 7] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.in_conv = torch.nn.Sequential() + self.expected_K_params += 2 + self.in_conv.add_module("conv_in", torch.nn.Conv2d(self.input_shape[1], 64, kernel_size=(1,1))) + + self.left_paths = torch.nn.ModuleList([torch.nn.Conv2d(64, 64, kernel_size=(1,1))]) + self.expected_C_params += 1 + self.expected_K_params += 2 + + in_C = 64 + out_C = 64 + for d in range(1,depth,1): + self.expected_C_params += 1 + self.expected_K_params += 2 + if dim == 1: + out_C += 64 + self.left_paths.append(torch.nn.Conv2d(in_C+64, out_C, kernel_size=(1,1))) + if dim == 1: + in_C += 64 + + self.right_path = torch.nn.Sequential() + self.expected_C_params += 1 + self.expected_K_params += 2 + self.right_path.add_module("conv_b", torch.nn.Conv2d(64, 64*ratio, kernel_size=(1,1))) + + self.out_conv = torch.nn.Sequential() + self.expected_C_params += 1 + if dim == 1: + out_C += 64*ratio + self.out_conv.add_module("conv_out", torch.nn.Conv2d(out_C, 16, kernel_size=(1,1))) + + def forward(self, input: torch.Tensor): + step0 = self.in_conv(input) + step1 = step0 + for d, layer in enumerate(self.left_paths): + if d == 0: + step1 = layer(step1) + else: + step1 = layer(torch.cat([step1, step0], 1)) + + step2 = torch.cat([step1, self.right_path(step0)], self.dim) + return self.out_conv(step2) + +class test_flatten_op(torch.nn.Module): + """flatten ops may change the effective channel count, typically by collapsing N,C,H,W into N,C*H*W before a classifier""" + + def __init__( + self, + change_dims = True, + ): + super().__init__() + self.change_dims = change_dims + self.input_shape = [4, 16, 3, 3] + self.expected_C_params = 0 + self.expected_K_params = 0 + + if not self.change_dims: + self.input_shape = [4, 16, 1, 1] + self.expected_C_params = 1 + self.expected_K_params = 2 + + self.flattened_C = self.input_shape[2] * self.input_shape[3] * 64 + + self.in_conv = torch.nn.Conv2d(self.input_shape[1], 64, kernel_size=(1,1)) + self.out_gemm = torch.nn.Linear(self.flattened_C, 16) + + + def forward(self, input: torch.Tensor): + step0 = self.in_conv(input) + step1 = torch.flatten(step0, start_dim=1) + return self.out_gemm(step1) + +class test_flatten_module(torch.nn.Module): + """flatten modules may change the effective channel count, typically by collapsing N,C,H,W into N,C*H*W before a classifier""" + + def __init__( + self, + change_dims = True, + ): + super().__init__() + self.change_dims = change_dims + self.input_shape = [4, 16, 3, 3] + self.expected_C_params = 0 + self.expected_K_params = 0 + + if not self.change_dims: + self.input_shape = [4, 16, 1, 1] + self.expected_C_params = 1 + self.expected_K_params = 2 + + self.flattened_C = self.input_shape[2] * self.input_shape[3] * 64 + self.stack = torch.nn.Sequential() + self.stack.add_module("conv_in", torch.nn.Conv2d(self.input_shape[1], 64, kernel_size=(1,1))) + self.stack.add_module("flatten", torch.nn.Flatten(1)) + self.stack.add_module("gemm_out", torch.nn.Linear(self.flattened_C, 16)) + + def forward(self, input: torch.Tensor): + return self.stack(input) + +class test_trace_failure(torch.nn.Module): + """make sure tracing failures are handled gracefully""" + + def __init__( + self + ): + super().__init__() + self.input_shape = [4, 16, 1, 1] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.in_conv = torch.nn.Conv2d(self.input_shape[1], 64, kernel_size=(1,1)) + self.out_conv = torch.nn.Conv2d(64, 16, kernel_size=(1,1)) + + def forward(self, input: torch.Tensor): + step0 = self.in_conv(input) + #NCHW = 4,64,1,1 + channels = step0.size(1) + channel_offset = torch.arange(channels, dtype=torch.long, device=step0.device) + channel_offset = channel_offset.unsqueeze(0).unsqueeze(2).unsqueeze(3).expand_as(step0) + step0.add_(channel_offset) + return self.out_conv(step0) + +class already_sparse(torch.nn.Module): + """if weights are already sparse, permutations should be skipped""" + + def __init__( + self + ): + super().__init__() + self.input_shape = [4, 16, 3, 3] + self.expected_C_params = 0 + self.expected_K_params = 0 + + self.in_conv = torch.nn.Conv2d(self.input_shape[1], 64, kernel_size=(1,1)) + self.out_conv = torch.nn.Conv2d(64, 16, kernel_size=(1,1)) + + # apply 2:4 to the output weights, it will not require a permutation + out_weights = torch.ones_like(self.out_conv.weight) + out_weights[:,0::2,...] = 0 + assert torch.sum(out_weights) == torch.numel(out_weights)/2 + self.out_conv.weight.data.copy_(out_weights) + + def forward(self, input: torch.Tensor): + step0 = self.in_conv(input) + return self.out_conv(step0) + +def test_model(model, tag, verbosity=0, save_onnx=False): + Permutation.set_identical_seed() + x = torch.rand(model.input_shape) + if save_onnx: + torch.onnx.export(model, x, f"{tag}.onnx", verbose=False) + + base_out = model(x) + + sparse_parameters = [] + all_parameters = [] + + module_to_params = {} + module_to_params[torch.nn.MultiheadAttention] = ('q_proj_weight', 'k_proj_weight', 'v_proj_weight', 'in_proj_weight') + + for module_name, module in model.named_modules(): + module_type_str = str(type(module)).split("\'")[1] + if module_type_str == 'torch.nn.modules.container.Sequential' or module_type_str.startswith('torchvision.models'): + # filter out the 'torch.nn.modules.container.Sequential' type and the whole model, like 'torchvision.models.vgg.VGG' + continue + for p_name, p in module.named_parameters(): + all_parameters.append((module_name, module, p_name, p)) + + if isinstance(module, (torch.nn.Linear, torch.nn.Conv1d, torch.nn.Conv2d, torch.nn.Conv3d, torch.nn.MultiheadAttention, torch.nn.modules.linear.NonDynamicallyQuantizableLinear)): + allowed_names = ('weight',) + if type(module) in module_to_params.keys(): + allowed_names = module_to_params[type(module)] + + if p_name not in allowed_names: + continue + + if len(p.size()) >= 2 and (p.size()[0] % 8) == 0 and (p.size()[1] % 16) == 0: + mask = torch.ones_like(p).bool() + buffname = p_name.split(".")[-1] + module.register_buffer('__%s_mma_mask' % buffname, mask) + sparse_parameters.append((module_name, module, p_name, p, mask, None)) + + if module_type_str == 'torch.nn.modules.batchnorm.BatchNorm2d': + # need to get the running_mean and running_var from model.state_dict(), as they are not the learnable parameters + module_mean_name = module_name + '.running_mean' + module_var_name = module_name + '.running_var' + for param_key in model.state_dict(): + if module_mean_name == param_key or module_var_name == param_key: + all_parameters.append((module_name, module, param_key.split(".")[-1], model.state_dict()[param_key])) + + if verbosity > 1: + sparse_param_names = [module_name+":"+p_name for (module_name, module, p_name, p, mask, pruned) in sparse_parameters] + all_param_names = [module_name+":"+p_name for (module_name, module, p_name, p) in all_parameters] + print(f"\tSparse parameter names: {sparse_param_names}\n\tAll parameter names: {all_param_names}") + + Permutation.set_permutation_params_from_asp(model, sparse_parameters, all_parameters, verbosity) + Permutation.permute_model(model) + + C_params, K_params, missed_dims = Permutation.get_permutation_stats() + + success = True + fail_str = "" + succ_str = "" + if len(C_params) != model.expected_C_params: + success = False + fail_str = fail_str + f"\n\tC expected {model.expected_C_params}, got {len(C_params)} ({C_params})" + elif verbosity > 0: + succ_str = succ_str + f"\n\tC expected {model.expected_C_params}, got {len(C_params)} ({C_params})" + + if len(K_params) != model.expected_K_params: + success = False + fail_str = fail_str + f"\n\tK expected {model.expected_K_params}, got {len(K_params)} ({K_params})" + elif verbosity > 0: + succ_str = succ_str + f"\n\tK expected {model.expected_K_params}, got {len(K_params)} ({K_params})" + + if len(missed_dims) != 0: + success = False + fail_str = fail_str + f"\n\tMissed permutations along {len(missed_dims)} dimensions ({missed_dims})" + + perm_out = model(x) + + atol = 1e-5 + rtol = 1e-4 + outs_match = torch.allclose(base_out.data, perm_out.data, atol=atol, rtol=rtol) + if not outs_match: + fail_str = fail_str + f"\n\tOutputs matched: {outs_match}" + if success: + diffs = base_out - perm_out + diff_locs = (diffs >= atol).nonzero(as_tuple=True) + fail_str = fail_str + f"\n{diff_locs}\n{diffs[diff_locs]}" + success = False + + if success: + print(f"{tag}: Success\t{succ_str}") + else: + print(f"{tag}: FAIL\t{fail_str}") + + return success + +def main(): + + global_success = True + + global_success &= test_model(simple_convs(2,16), "smoke test") + global_success &= test_model(simple_convs(5, 64), "simple 5 64") + global_success &= test_model(simple_convs(10, 32), "simple 10 32") + # normalization + for norm in ['BatchNorm2d', 'LazyBatchNorm2d', 'InstanceNorm2d', 'LazyInstanceNorm2d', 'LayerNorm3', 'LocalResponseNorm']: + global_success &= test_model(simple_convs(4, 128, norm), norm) + # disallowed normalization + for norm in ['GroupNorm']: + global_success &= test_model(simple_convs(4, 128, norm), norm) + + global_success &= test_model(conv_1d(), "conv1d") + global_success &= test_model(conv_1d(with_2d=True), "conv1d and conv2d") + global_success &= test_model(grouped_convs(), "grouped convs") + global_success &= test_model(simple_forks_joins(), "forks and joins") + global_success &= test_model(different_grouped_convs(), "GCD") + global_success &= test_model(siblings_poison(), "sibling poison") + global_success &= test_model(coparent_poison(), "coparent poison") + global_success &= test_model(depthwise_child_is_sibling(), "dw child is sibling") + global_success &= test_model(module_attribute(complexity=0), "single attribute") + global_success &= test_model(module_attribute(complexity=1), "single attribute thrice") + global_success &= test_model(MHA_test(hidden_dim=256, seq_len=64, num_heads=16), "stacked MHA") + global_success &= test_model(one_sparse_sibling(), "one sparse sibling") + global_success &= test_model(test_concat(), "simple concat") # concat along K + global_success &= test_model(test_concat(dim=0), "concat dim0") # concat along C + global_success &= test_model(test_concat(ratio=2), "concat ratio2") # concat along K with different K values + global_success &= test_model(test_concat(depth=2), "concat depth2") # concat along K multiple times + global_success &= test_model(test_concat(depth=3), "concat depth3") + global_success &= test_model(test_concat(ratio=3, depth=4), "concat ratio3 depth4") + global_success &= test_model(test_concat(dim=0, depth=3), "concat dim0 depth3") + global_success &= test_model(test_flatten_op(), "flatten op") + global_success &= test_model(test_flatten_op(change_dims=False), "useless flatten op") + global_success &= test_model(test_flatten_module(), "flatten module") + global_success &= test_model(test_flatten_module(change_dims=False), "useless flatten module") + global_success &= test_model(test_trace_failure(), "trace failure") + global_success &= test_model(already_sparse(), "skip already sparse") + global_success &= test_model(square_attribute(), "square attributes") + + if global_success: + print("All tests completed successfully.") + else: + print("There was at least one failure.") + +if __name__ == '__main__': + main() diff --git a/apex/apex/contrib/sparsity/test/toy_problem.py b/apex/apex/contrib/sparsity/test/toy_problem.py new file mode 100644 index 00000000..2145323d --- /dev/null +++ b/apex/apex/contrib/sparsity/test/toy_problem.py @@ -0,0 +1,87 @@ +from collections import OrderedDict + +import torch +from apex.optimizers import FusedAdam +from apex.contrib.sparsity import ASP + +def build_model(args): + od = OrderedDict() + for i in range(args.num_layers): + if i == 0: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.input_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + elif i == args.num_layers-1: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.output_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.output_features]) + else: + od['linear_layer_%d' % (i+1)] = torch.nn.Linear(args.hidden_features, args.hidden_features) + od['layer_norm_%d' % (i+1)] = torch.nn.LayerNorm([args.batch_size, args.hidden_features]) + return torch.nn.Sequential(od) + +def train_step(args, model, optimizer, input_batch, target_batch, step): + predicted_target = model(input_batch) + loss = ((predicted_target-target_batch)**2).sum() + loss.backward() + optimizer.step() + optimizer.zero_grad() + step = step + 1 + #print("Step %d :: loss=%e" % (step, loss.item())) + return step + +def train_loop(args, model, optimizer, step, num_steps): + for i in range(num_steps): + input_batch = torch.randn([args.batch_size, args.input_features]).cuda() + target_batch = torch.randn([args.batch_size, args.output_features]).cuda() + step = train_step(args, model, optimizer, input_batch, target_batch, step) + return step + +def main(args): + model = build_model(args).cuda() + one_ll = next(model.children()).weight + optimizer = FusedAdam(model.parameters()) + # only prune linear layers, even though we also support conv1d, conv2d and conv3d + ASP.init_model_for_pruning(model, "m4n2_1d", whitelist=[torch.nn.Linear], allow_recompute_mask=True) + ASP.init_optimizer_for_pruning(optimizer) + + step = 0 + + # train for a few steps with dense weights + print("DENSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_dense_steps) + + # simulate sparsity by inserting zeros into existing dense weights + ASP.compute_sparse_masks() + + # train for a few steps with sparse weights + print("SPARSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_sparse_steps) + + # recompute sparse masks + ASP.compute_sparse_masks() + + # train for a few steps with sparse weights + print("SPARSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_sparse_steps_2) + + # turn off sparsity + print("SPARSE :: ",one_ll) + ASP.restore_pruned_weights() + + # train for a few steps with dense weights + print("DENSE :: ",one_ll) + step = train_loop(args, model, optimizer, step, args.num_dense_steps_2) + +if __name__ == '__main__': + class Args: + batch_size = 32 + input_features = 16 + output_features = 8 + hidden_features = 40 + num_layers = 4 + num_dense_steps = 2000 + num_sparse_steps = 3000 + num_sparse_steps_2 = 1000 + num_dense_steps_2 = 1500 + args = Args() + + main(args) diff --git a/apex/apex/contrib/test/__init__.py b/apex/apex/contrib/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/bottleneck/__init__.py b/apex/apex/contrib/test/bottleneck/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/bottleneck/test_bottleneck_module.py b/apex/apex/contrib/test/bottleneck/test_bottleneck_module.py new file mode 100644 index 00000000..83cf019b --- /dev/null +++ b/apex/apex/contrib/test/bottleneck/test_bottleneck_module.py @@ -0,0 +1,326 @@ +import unittest + +import torch +from torch.testing._internal import common_utils + +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +SKIP_TEST = None +try: + from apex.contrib.bottleneck import Bottleneck, SpatialBottleneck + from apex.contrib.bottleneck import HaloExchangerPeer + from apex.contrib.peer_memory import PeerMemoryPool +except ImportError as e: + SKIP_TEST = e + + +def ground_truth_bottleneck(C, dtype, explicit_nhwc): + bottleneck = Bottleneck(C, C, C, use_cudnn=True, explicit_nhwc=explicit_nhwc) + bottleneck.to(dtype=dtype, device="cuda") + for p in bottleneck.parameters(): + torch.distributed.broadcast(p, 0) + for b in bottleneck.buffers(): + torch.distributed.broadcast(b, 0) + return bottleneck + + +def print_bottleneck_p_and_b(bottleneck): + with torch.no_grad(): + for n, p in bottleneck.named_parameters(): + print("%s :: %s" % (n, str(p.norm(p=2, dtype=torch.float32)))) + for n, p in bottleneck.named_buffers(): + print("%s :: %s" % (n, str(p.norm(p=2, dtype=torch.float32)))) + + +def has_nan(x): + if isinstance(x, list) or isinstance(x, tuple): + for xx in x: + if torch.any(torch.isnan(xx)): + return True + return False + elif isinstance(x, dict): + for k, v in x.items(): + if torch.any(torch.isnan(v)): + return True + else: + return torch.any(torch.isnan(x)) + + +def rel_diff_t(xx1, xx2): + return ((xx1 - xx2).norm(p=2, dtype=torch.float32) / (xx1 + xx2).norm(p=2, dtype=torch.float32)).item() + + +def rel_diff(x1, x2): + if isinstance(x1, list) or isinstance(x1, tuple): + return [rel_diff_t(xx1, xx2) for xx1, xx2 in zip(x1, x2)] + elif isinstance(x1, dict): + return [rel_diff_t(xx1, xx2) for (k1, xx1), (k2, xx2) in zip(x1.items(), x2.items())] + else: + return rel_diff_t(x1, x2) + + +def graph_it(bottleneck, x): + print("Graphing") + with torch.no_grad(): + x = x.clone() + x.grad = None + x.requires_grad = True + return torch.cuda.make_graphed_callables(bottleneck, (x,)) + + +def clone_inputs(bottleneck, x, dy=None): + with torch.no_grad(): + x = x.clone() + x.grad = None + x.requires_grad = True + if dy is None: + y = bottleneck(x) + dy = torch.randn_like(y) / 1e2 + torch.distributed.broadcast(dy, 0) + return x, dy + + +def fprop_and_bprop(bottleneck, x, dy): + y = bottleneck(x) + y.backward(dy) + dgrad = x.grad.detach() + wgrad = {} + for n, p in bottleneck.named_parameters(): + wgrad[n] = p.grad.detach() + return x, y, dy, dgrad, wgrad + + +def ground_truth(N, C, H, W, dtype, memory_format, bottleneck): + if memory_format == 1: + # 1 -> explicit nhwc + explicit_nhwc = True + with torch.no_grad(): + x = torch.randn([N, H, W, C], dtype=dtype, device="cuda") + torch.distributed.broadcast(x, 0) + x, dy = clone_inputs(bottleneck, x) + return fprop_and_bprop(bottleneck, x, dy) + else: + # 2 -> native nhwc + # 3 -> nchw + explicit_nhwc = False + assert False, "Not implemented yet" + + +def print_ground_truth(gt): + x, y, dy, dgrad, wgrad = gt + if has_nan(y) or has_nan(dgrad) or has_nan(wgrad): + print("Error! Ground truth has NAN") + else: + print("Ok! No NAN found in ground truth") + + +def apply_to_different_bottleneck(gt, bottleneck): + with torch.no_grad(): + x, _, dy, _, _ = gt + x, dy = clone_inputs(bottleneck, x, dy) + return fprop_and_bprop(bottleneck, x, dy) + + +def compare_single_field(results, f1, f2, l0, l1, l2): + if has_nan(f1) and has_nan(f2): + results[l0] = "both NAN" + elif has_nan(f1): + results[l0] = "%s.%s NAN" % (l1, l0) + elif has_nan(f2): + results[l0] = "%s.%s NAN" % (l2, l0) + else: + results[l0] = "%s" % (str(rel_diff(f1, f2))) + + +def compare(gt, bt): + x1, y1, dy1, dgrad1, wgrad1 = gt + x2, y2, dy2, dgrad2, wgrad2 = bt + results = {} + compare_single_field(results, y1, y2, "y", "gt", "bt") + compare_single_field(results, dy1, dy2, "dy", "gt", "bt") + compare_single_field(results, dgrad1, dgrad2, "dgrad", "gt", "bt") + compare_single_field(results, wgrad1, wgrad2, "wgrad", "gt", "bt") + for i in range(torch.distributed.get_world_size()): + if i == torch.distributed.get_rank(): + print(i, results) + torch.distributed.barrier() + + +def spatial_parallel_bottleneck(C, dtype, explicit_nhwc, gt_bottleneck, spatial_parallel_args): + spatial_bottleneck = SpatialBottleneck( + C, + C, + C, + use_cudnn=True, + explicit_nhwc=explicit_nhwc, + spatial_parallel_args=spatial_parallel_args, + ) + spatial_bottleneck.to(dtype=dtype, device="cuda") + with torch.no_grad(): + sp = {} + for n, p in spatial_bottleneck.named_parameters(): + sp[n] = p + for n, p in gt_bottleneck.named_parameters(): + sp[n].copy_(p) + sb = {} + for n, b in spatial_bottleneck.named_buffers(): + sb[n] = b + for n, b in gt_bottleneck.named_buffers(): + sb[n].copy_(b) + return spatial_bottleneck + + +def n_way_spatial(halex, gt_bottleneck, gt, explicit_nhwc, world_size, rank, fp32_reduce=False): + assert explicit_nhwc, "Only tested for explicit nhwc" + + x, _, dy, _, _ = gt + N, H, W, C = list(x.shape) # Tensor is already shaped properly for n-way parallel + dtype = x.dtype + + spatial_group_size = world_size + spatial_group_rank = rank + spatial_communicator = None + spatial_halo_exchanger = halex + spatial_method = 1 # 1 -> overlap halo and main conv, 2 -> wait for halo, conv on padded x + use_delay_kernel = False + spatial_parallel_args = ( + spatial_group_size, + spatial_group_rank, + spatial_communicator, + spatial_halo_exchanger, + spatial_method, + use_delay_kernel, + ) + spatial_bottleneck = spatial_parallel_bottleneck(C, dtype, explicit_nhwc, gt_bottleneck, spatial_parallel_args) + + with torch.no_grad(): + Hs = H // spatial_group_size + xs = x[:, spatial_group_rank * Hs : (spatial_group_rank + 1) * Hs, :, :].clone() + dys = dy[:, spatial_group_rank * Hs : (spatial_group_rank + 1) * Hs, :, :].clone() + xs.requires_grad = True + + spatial_bottleneck = graph_it(spatial_bottleneck, xs) + _, y, _, dgrad, wgrad = fprop_and_bprop(spatial_bottleneck, xs, dys) + + # gather output pieces + for n, p in wgrad.items(): + if fp32_reduce: + p32 = p.float() + torch.distributed.all_reduce(p32) + p.copy_(p32.half()) + else: + torch.distributed.all_reduce(p) + ys = [torch.empty_like(y) for _ in range(spatial_group_size)] + torch.distributed.all_gather(ys, y) + y = torch.cat(ys, dim=1) + dgrads = [torch.empty_like(dgrad) for _ in range(spatial_group_size)] + torch.distributed.all_gather(dgrads, dgrad) + dgrad = torch.cat(dgrads, dim=1) + return x, y, dy, dgrad, wgrad + + +def main(): + torch.use_deterministic_algorithms(True) + + torch.distributed.init_process_group("nccl") + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + torch.cuda.set_device(rank) + + explicit_nhwc = True + + dtype = torch.float16 + N, C, H, W = 1, 64, 200, 336 + Hs = ((H + 8 * world_size - 1) // (8 * world_size)) * 8 + H = Hs * world_size + gt_bottleneck = ground_truth_bottleneck(C, dtype, explicit_nhwc) + gt = ground_truth(N, C, H, W, dtype, 1, gt_bottleneck) + + # verify that spatial bottleneck with group_size 1 produces same results as ground truth bottleneck + spatial_bottleneck = spatial_parallel_bottleneck(C, dtype, explicit_nhwc, gt_bottleneck, None) + bt = apply_to_different_bottleneck(gt, spatial_bottleneck) + compare(gt, bt) + # print_bottleneck_p_and_b(gt_bottleneck) + # print_bottleneck_p_and_b(spatial_bottleneck) + + group_size = world_size + group = rank // group_size + ranks = [group * group_size + i for i in range(group_size)] + rank_in_group = rank % group_size + + spatial_group_size = world_size + spatial_communicator = None + + peer_pool = PeerMemoryPool(0, 64 * 1024 * 1024, ranks) + + # class HaloExchangerNoComm(HaloExchanger): + # def __init__(self, ranks, rank_in_group): + # class HaloExchangerAllGather(HaloExchanger): + # def __init__(self, ranks, rank_in_group, comm): + # class HaloExchangerSendRecv(HaloExchanger): + # def __init__(self, ranks, rank_in_group): + # class HaloExchangerPeer(HaloExchanger): + # def __init__(self, ranks, rank_in_group, peer_pool, explicit_nhwc, numSM=1): + + # halex = HaloExchangerAllGather(ranks, rank_in_group) + # halex = HaloExchangerSendRecv(ranks, rank_in_group) + + halex = HaloExchangerPeer(ranks, rank_in_group, peer_pool, explicit_nhwc, numSM=0) + # print("halex.signals = %s" % (str(halex.signals))) + # Make sure peer memory halo exchanger has finished initializing flags on all ranks before proceeding + # torch.cuda.synchronize() + # torch.distributed.barrier() + + bt2 = n_way_spatial(halex, gt_bottleneck, gt, explicit_nhwc, world_size, rank, fp32_reduce=True) + compare(gt, bt2) + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TestBottleneck(NcclDistributedTestBase): + # PyTorch's float16 tolerance values, see https://pytorch.org/docs/stable/testing.html#torch.testing.assert_close + fp16_tolerance = {"atol": 1e-5, "rtol": 1e-3} + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 2) + + def test_bottleneck_without_peer_memory(self) -> None: + explicit_nhwc: bool = True + dtype: torch.dtype = torch.float16 + N, C, H, W = 1, 64, 200, 336 + Hs = ((H + 8 * self.world_size - 1) // (8 * self.world_size)) * 8 + H = Hs * self.world_size + + gt_bottleneck = ground_truth_bottleneck(C, dtype, explicit_nhwc) + gt = ground_truth(N, C, H, W, dtype, 1, gt_bottleneck) + + spatial_bottleneck = spatial_parallel_bottleneck(C, dtype, explicit_nhwc, gt_bottleneck, None) + bt = apply_to_different_bottleneck(gt, spatial_bottleneck) + self.assertEqual(gt, bt, **self.fp16_tolerance) + + def test_bottleneck_with_peer_memory(self) -> None: + + explicit_nhwc: bool = True + dtype: torch.dtype = torch.float16 + N, C, H, W = 1, 64, 200, 336 + Hs = ((H + 8 * self.world_size - 1) // (8 * self.world_size)) * 8 + H = Hs * self.world_size + + gt_bottleneck = ground_truth_bottleneck(C, dtype, explicit_nhwc) + gt = ground_truth(N, C, H, W, dtype, 1, gt_bottleneck) + + group = self.rank // self.world_size + ranks = [group * self.world_size + i for i in range(self.world_size)] + rank_in_group = self.rank % self.world_size + + spatial_group_size, spatial_communicator = self.world_size, None + peer_pool = PeerMemoryPool(0, 64 * 1024 * 1024, ranks) + halo_exchanger_peer = HaloExchangerPeer(ranks, rank_in_group, peer_pool, explicit_nhwc, numSM=0) + bt2 = n_way_spatial( + halo_exchanger_peer, gt_bottleneck, gt, explicit_nhwc, self.world_size, self.rank, fp32_reduce=True + ) + # TODO(crcrpar): Investigate the implementation to mitigate the numerical errors. + # NOTE(crcrpar): This assert often fails due to numerical errors. + # self.assertEqual(gt, bt2, **self.fp16_tolerance) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/apex/contrib/test/clip_grad/__init__.py b/apex/apex/contrib/test/clip_grad/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/clip_grad/test_clip_grad.py b/apex/apex/contrib/test/clip_grad/test_clip_grad.py new file mode 100644 index 00000000..5153c225 --- /dev/null +++ b/apex/apex/contrib/test/clip_grad/test_clip_grad.py @@ -0,0 +1,172 @@ +import random +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.clip_grad import clip_grad_norm_ +except ImportError as e: + SKIP_TEST = e + + +def make_params( + num_params, + sizes=[1,2,3,4,5], + num_dims=[1,2,3], + dtypes=[torch.float32], + devices=['cuda'], + make_copy=False, +): + """Construct parameters with random configurations""" + + # Construct parameters + params = [] + for _ in range(num_params): + dims = [random.choice(sizes) for _ in range(random.choice(num_dims))] + dtype = random.choice(dtypes) + device = random.choice(devices) + p = torch.nn.Parameter(torch.randn(dims, dtype=dtype, device=device)) + p.grad = torch.randn_like(p) + params.append(p) + + # Copy parameters if needed + if make_copy: + params_copy = [] + for p in params: + p_copy = p.clone().detach() + p_copy.grad = p.grad.clone().detach() + params_copy.append(p_copy) + return params, params_copy + else: + return params + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class ClipGradNormTest(unittest.TestCase): + + def setUp(self, seed=1234): + super().setUp() + random.seed(seed) + torch.manual_seed(seed) + + def test_matches_pytorch( + self, + num_params=41, + dtypes=[torch.float32, torch.float16, torch.float64], + devices=['cuda', 'cpu'], + max_norm=0.54321, + norm_type=2.0, + rtol=1e-3, + atol=1e-20, + ): + """Make sure PyTorch and Apex gradient clipping produce same results""" + + # Construct identical sets of parameters + torch_params, apex_params = make_params( + num_params, + dtypes=dtypes, + devices=devices, + make_copy=True, + ) + + # Apply gradient clipping + torch_norm = torch.nn.utils.clip_grad_norm_( + torch_params, + max_norm, + norm_type=norm_type, + ) + apex_norm = clip_grad_norm_( + apex_params, + max_norm, + norm_type=norm_type, + ) + + # Make sure PyTorch and Apex get same results + torch.testing.assert_close( + apex_norm, torch_norm, + rtol=rtol, + atol=atol, + check_dtype=False, + ) + for torch_p, apex_p in zip(torch_params, apex_params): + torch.testing.assert_close( + apex_p, torch_p, + rtol=0, + atol=0, + ) # Params should be unaffected + torch.testing.assert_close( + apex_p.grad, torch_p.grad, + rtol=rtol, + atol=atol, + ) + + def test_matches_pytorch_fp16(self): + self.test_matches_pytorch(num_params=11, dtypes=[torch.float16]) + + def test_matches_pytorch_fp32(self): + self.test_matches_pytorch(dtypes=[torch.float32], rtol=1e-6) + + def test_matches_pytorch_fp64(self): + self.test_matches_pytorch(dtypes=[torch.float64], rtol=1e-15) + + def test_matches_pytorch_cpu(self): + self.test_matches_pytorch(devices=['cpu']) + + def test_matches_pytorch_infnorm(self): + self.test_matches_pytorch(norm_type=float('inf')) + + def test_matches_pytorch_1norm(self): + self.test_matches_pytorch(norm_type=1.0) + + def test_raises_on_mismatch(self): + + # Construct different sets of parameters + torch_params, apex_params = make_params(7, make_copy=True) + with torch.no_grad(): + torch_params[0].grad.view(-1)[0] = 1.23 + apex_params[0].grad.view(-1)[0] = 3.21 + + # Apply gradient clipping + torch_norm = torch.nn.utils.clip_grad_norm_( + torch_params, + 0.54321, + ) + apex_norm = clip_grad_norm_( + apex_params, + 0.54321, + ) + + # Make sure PyTorch and Apex get different results + self.assertRaises( + AssertionError, + torch.testing.assert_close, + apex_norm, torch_norm, + rtol=1e-3, + atol=1e-20, + check_dtype=False, + ) + for torch_p, apex_p in zip(torch_params, apex_params): + self.assertRaises( + AssertionError, + torch.testing.assert_close, + apex_p.grad, torch_p.grad, + rtol=1e-3, + atol=1e-20, + ) + + def test_raises_on_nan(self): + params = make_params(5, num_dims=[1]) + params[2].grad[-1] = float('NaN') + self.assertRaises( + RuntimeError, clip_grad_norm_, params, 1.0, error_if_nonfinite=True) + + def test_raises_on_inf(self): + params = make_params(5, num_dims=[1]) + params[2].grad[-1] = float('inf') + self.assertRaises( + RuntimeError, clip_grad_norm_, params, 1.0, error_if_nonfinite=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/conv_bias_relu/__init__.py b/apex/apex/contrib/test/conv_bias_relu/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/conv_bias_relu/test_conv_bias_relu.py b/apex/apex/contrib/test/conv_bias_relu/test_conv_bias_relu.py new file mode 100644 index 00000000..ad77b27e --- /dev/null +++ b/apex/apex/contrib/test/conv_bias_relu/test_conv_bias_relu.py @@ -0,0 +1,129 @@ +import copy +import math +import random +import unittest + +import torch +import torch.nn.functional as F + +HAS_CONV_BIAS_RELU = None +try: + from apex.contrib.conv_bias_relu import ConvBiasReLU, ConvBias, ConvBiasMaskReLU, ConvFrozenScaleBiasReLU +except ImportError as e: + HAS_CONV_BIAS_RELU = False +else: + HAS_CONV_BIAS_RELU = True + + +@unittest.skipIf(not HAS_CONV_BIAS_RELU, "`apex.contrib.conv_bias_relu` is not found.") +class FusedDenseTest(unittest.TestCase): + def setUp(self, seed=0): + super().setUp() + torch.manual_seed(seed) + + self.batch_size = random.randint(1, 64) + self.in_channels = random.randint(1, 64) * 8 + self.out_channels = random.randint(1, 64) * 8 + self.in_height = self.in_width = random.randint(5, 100) + self.conv_kernel_size = random.randint(1, 5) + self.conv_pad = random.randint(0, int(self.conv_kernel_size / 2)) + self.conv_stride = random.randint(1, 5) + self.conv_dilation = 1 + self.out_height = self.out_width = \ + math.floor((self.in_height + 2 * self.conv_pad - \ + self.conv_dilation * (self.conv_kernel_size - 1) - 1) / self.conv_stride + 1) + + self.x = torch.randint(low=-16, high=16, + size=[self.batch_size, self.in_channels, self.in_height, self.in_width]) \ + .cuda().to(memory_format=torch.channels_last).float() + self.x_ = self.x.clone() + self.x.requires_grad_() + self.x_.requires_grad_() + + self.mask = torch.randn([self.batch_size, self.out_channels, self.out_height, self.out_width]).cuda().to(memory_format=torch.channels_last) + self.mask = (self.mask > 0).to(torch.int8) + self.mask_ = self.mask.clone() + + self.scale = torch.randn([1, self.out_channels, 1, 1]).half().cuda() + self.scale_ = self.scale.clone() + self.bias = torch.randn([1, self.out_channels, 1, 1]).half().cuda() + self.bias_ = self.bias.clone() + + self.conv1 = torch.nn.Conv2d(self.in_channels, self.out_channels, self.conv_kernel_size, + stride=self.conv_stride, padding=self.conv_pad).cuda().to(memory_format=torch.channels_last) + self.conv1_ = copy.deepcopy(self.conv1) + + self.conv2 = torch.nn.Conv2d(self.in_channels, self.out_channels, self.conv_kernel_size, + stride=self.conv_stride, padding=self.conv_pad, bias=False).cuda().to(memory_format=torch.channels_last) + self.conv2_ = copy.deepcopy(self.conv2) + + print() + print('> input=[{}, {}, {}, {}]'.format(self.batch_size, self.in_channels, self.in_height, self.in_width)) + print('> kernel=[{}, {}, {}, {}], stride={}, pad={}'.format(self.out_channels, self.in_channels, + self.conv_kernel_size, self.conv_kernel_size, + self.conv_stride, self.conv_pad)) + + def test_conv_bias_relu(self): + with torch.cuda.amp.autocast(dtype=torch.half): + out = ConvBiasReLU(self.x, self.conv1.weight, self.conv1.bias.reshape(1, -1, 1, 1), self.conv_pad, self.conv_stride) + loss = (out.float()**2).sum() / out.numel() + loss.backward() + with torch.cuda.amp.autocast(dtype=torch.half): + out_ = F.relu(self.conv1_(self.x_)) + loss_ = (out_**2).sum() / out_.numel() + loss_.backward() + + torch.testing.assert_close(out_, out, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.conv1_.bias.grad, self.conv1.bias.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.conv1_.weight.grad, self.conv1.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.x_.grad, self.x.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + def test_conv_bias(self): + with torch.cuda.amp.autocast(dtype=torch.half): + out = ConvBias(self.x, self.conv1.weight, self.conv1.bias.reshape(1, -1, 1, 1), self.conv_pad, self.conv_stride) + loss = (out.float()**2).sum() / out.numel() + loss.backward() + + with torch.cuda.amp.autocast(dtype=torch.half): + out_ = self.conv1_(self.x_) + loss_ = (out_**2).sum() / out_.numel() + loss_.backward() + + torch.testing.assert_close(out, out_, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.conv1_.bias.grad, self.conv1.bias.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.conv1_.weight.grad, self.conv1.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.x_.grad, self.x.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + def test_conv_bias_mask_relu(self): + with torch.cuda.amp.autocast(dtype=torch.half): + out = ConvBiasMaskReLU(self.x, self.conv1.weight, self.conv1.bias.reshape(1, -1, 1, 1), self.mask, self.conv_pad, self.conv_stride) + loss = (out.float()**2).sum() / out.numel() + loss.backward() + with torch.cuda.amp.autocast(dtype=torch.half): + out_ = F.relu(self.conv1_(self.x_) * self.mask_) + loss_ = (out_**2).sum() / out_.numel() + loss_.backward() + + torch.testing.assert_close(out, out_, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.conv1_.bias.grad, self.conv1.bias.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.conv1_.weight.grad, self.conv1.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.x_.grad, self.x.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + def test_conv_frozen_scale_bias_relu(self): + with torch.cuda.amp.autocast(dtype=torch.half): + out = ConvFrozenScaleBiasReLU(self.x, self.conv2.weight, self.scale, self.bias, self.conv_pad, self.conv_stride) + loss = (out.float()**2).sum() / out.numel() + loss.backward() + with torch.cuda.amp.autocast(dtype=torch.half): + out_ = F.relu(self.conv2_(self.x_) * self.scale_ + self.bias_) + loss_ = (out_**2).sum() / out_.numel() + loss_.backward() + + torch.testing.assert_close(out, out_, atol=2.5e-3, rtol=2.5e-3, equal_nan=True) + torch.testing.assert_close(self.conv2_.weight.grad, self.conv2.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(self.x_.grad, self.x.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + +if __name__ == '__main__': + unittest.main() + diff --git a/apex/apex/contrib/test/cudnn_gbn/__init__.py b/apex/apex/contrib/test/cudnn_gbn/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/cudnn_gbn/test_cudnn_gbn_with_two_gpus.py b/apex/apex/contrib/test/cudnn_gbn/test_cudnn_gbn_with_two_gpus.py new file mode 100644 index 00000000..cf662a67 --- /dev/null +++ b/apex/apex/contrib/test/cudnn_gbn/test_cudnn_gbn_with_two_gpus.py @@ -0,0 +1,147 @@ +import copy +import typing +import unittest + +import torch +import torch.nn as nn +from torch.testing._internal import common_utils + +SKIP_TEST = None +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +try: + from apex.contrib.cudnn_gbn import GroupBatchNorm2d as GBN +except ImportError as e: + SKIP_TEST = e + + +# Usage: python /path/to/cudnn_gbn/test_gbn_with_two_gpus.py + +input_shapes = [ + [1, 1024, 48, 72], + [1, 128, 192, 288], + [1, 128, 384, 576], + [1, 1536, 48, 72], + [1, 2048, 48, 72], + [1, 256, 1, 1], + [1, 256, 192, 288], + [1, 256, 384, 576], + [1, 256, 48, 72], + [1, 256, 96, 144], + [1, 32, 384, 576], + [1, 48, 192, 288], + [1, 64, 384, 576], + [1, 728, 48, 72], + [1, 728, 96, 144], +] + + +class BNModelRef(nn.Module): + def __init__(self, num_features, num_layers=1000): + super().__init__() + self.fwd = nn.Sequential( + *[ + nn.BatchNorm2d(num_features, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + for _ in range(num_layers) + ] + ) + + def forward(self, x): + return self.fwd(x) + + +class BNModel(nn.Module): + def __init__(self, num_features, num_layers=1000): + super().__init__() + self.fwd = nn.Sequential( + *[ + GBN(num_features, group_size=2, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + for _ in range(num_layers) + ] + ) + + def forward(self, x): + return self.fwd(x) + + +def get_rand_tensors(global_shape, device): + inp_t = torch.rand(global_shape, dtype=torch.float32, device=device).to(memory_format=torch.channels_last) + weight = torch.rand(global_shape[1], dtype=torch.float32, device=device) + bias = torch.rand(global_shape[1], dtype=torch.float32, device=device) + _grad_out = torch.rand(global_shape, dtype=torch.float32, device=device).to(memory_format=torch.channels_last) + return inp_t, weight, bias, _grad_out + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TestCudnnGBN(NcclDistributedTestBase): + def _prep(self): + torch.cuda.manual_seed(333) + torch.manual_seed(333) + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 2) + + @torch.backends.cudnn.flags(enabled=True, benchmark=True) + def _test_cudnn_gbn( + self, + num_layers: int, + shape: typing.List[int], + *, + memory_format: torch.memory_format = torch.channels_last, + ) -> None: + global_shape = copy.deepcopy(shape) + global_shape[0] = self.world_size + + device = torch.device("cuda", self.rank) + cudnn_gbn_model = BNModel( + num_features=shape[1], + num_layers=num_layers, + ).to(device=device, memory_format=memory_format) + ref_model = BNModelRef( + num_features=shape[1], + num_layers=num_layers, + ).to(device=device, memory_format=memory_format) + + input, weight, bias, grad_out = get_rand_tensors(global_shape, device) + with torch.no_grad(): + ref_model.fwd[0].weight.copy_(weight) + ref_model.fwd[0].bias.copy_(bias) + cudnn_gbn_model.fwd[0].weight.copy_(weight) + cudnn_gbn_model.fwd[0].bias.copy_(bias) + + ref_input = input.clone().detach().requires_grad_() + input = input[self.rank : self.rank + 1, ...].clone().detach().requires_grad_() + + ref_grad_out = grad_out.half().clone().detach() + grad_out = grad_out[self.rank : self.rank + 1, ...].half().clone().detach() + + with torch.cuda.amp.autocast(): + out = cudnn_gbn_model(input) + ref_out = ref_model(ref_input.half()) + out.backward(grad_out) + ref_out.backward(ref_grad_out) + + kwargs = {"rtol": 3.5e-3, "atol": 3e-2, "msg": f"shape: {shape}"} + + torch.testing.assert_close(ref_out[self.rank : self.rank + 1], out, **kwargs) + torch.testing.assert_close(ref_input.grad[self.rank : self.rank + 1], input.grad, **kwargs) + # compensating the averaging over processes done by DDP + # in order to produce mathematically equivalent result + # https://github.com/NVIDIA/apex/issues/134#issuecomment-458307368 + torch.testing.assert_close( + ref_model.fwd[0].weight.grad / self.world_size, cudnn_gbn_model.fwd[0].weight.grad, **kwargs + ) + torch.testing.assert_close( + ref_model.fwd[0].bias.grad / self.world_size, cudnn_gbn_model.fwd[0].bias.grad, **kwargs + ) + + def test_cudnngbn(self): + if self.world_size != 2: + self.skipTest(f"This test is written for world_size of 2 but {self.world_size}") + for shape in input_shapes: + self._prep() + self._test_cudnn_gbn(1, shape) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/apex/contrib/test/fmha/__init__.py b/apex/apex/contrib/test/fmha/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/fmha/test_fmha.py b/apex/apex/contrib/test/fmha/test_fmha.py new file mode 100644 index 00000000..1bf1ad55 --- /dev/null +++ b/apex/apex/contrib/test/fmha/test_fmha.py @@ -0,0 +1,149 @@ +############################################################################### +# Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the NVIDIA CORPORATION nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +############################################################################### + +import math +import unittest + +import torch +import numpy as np + +SKIP_TEST = None +try: + import fmhalib as mha +except ImportError as e: + SKIP_TEST = e + + +def _get_device_properties(device = torch.device("cuda")): + # type: (str or torch.device) -> Tuple[int, int] + properties = torch.cuda.get_device_properties(device) + return properties.major, properties.minor + + +def py_mha(qkv, amask, b, s, h, d): + qkv = qkv.view(b, s, h, 3, d) + q = qkv[:, :, :, 0, :].permute(0,2,1,3) + k = qkv[:, :, :, 1, :].permute(0,2,1,3) + v = qkv[:, :, :, 2, :].permute(0,2,1,3) + p = torch.matmul(q.float(), k.permute(0,1,3,2).float()) + p_masked = p / math.sqrt(d) + (1.0 - amask) * -10000.0 + s = torch.softmax(p_masked, -1).to(qkv.dtype) + ctx = torch.matmul(s, v) + ctx = ctx.permute(0,2,1,3).contiguous() + + ctx.retain_grad() + + return ctx + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +@unittest.skipIf(not _get_device_properties() == (8, 0), "FMHA only supports sm80") +class TestFMHA(unittest.TestCase): + + def run_test(self, s: int, b: int, zero_tensors: bool): + print(f'Test s={s} b={b}, zero_tensors={zero_tensors}') + + torch.manual_seed(1234) + torch.cuda.manual_seed(1234) + + dtype = torch.float16 + device = torch.device('cuda') + + h = 16 + d = 64 + + slens = [s] * b + a = torch.tensor(np.array([0] + slens), dtype=torch.int32) + amask = torch.ones(b,h,s,s, dtype=dtype, device=device) + seqlens = torch.tensor(slens, dtype=torch.int32, device=device) + cu_seqlens = torch.cumsum(a, 0).to(dtype=torch.int32, device=device) + total = cu_seqlens[-1].item() + + qkv = torch.randn((b,s,h,3,d), device=device, dtype=dtype) + + qkv_vs = qkv.permute(0,1,3,2,4).contiguous().view(b*s, 3, h,d) + + qkv.requires_grad = True + + if b < 4: + ctx, S_ = mha.fwd(qkv_vs, cu_seqlens, 0.0, s, True, True, zero_tensors, None) + else: + ctx, S_ = mha.fwd(qkv_vs, cu_seqlens, 0.0, s, True, False, zero_tensors, None) + ctx = ctx.view(b,s,h,d) + + ctx_ref = py_mha(qkv, amask, b,s,h,d) + self.assertTrue(torch.allclose(ctx_ref.float(), ctx.float(), atol=1e-3)) + + labels = torch.randn_like(ctx_ref) + diff = ctx_ref - labels + l = (diff * diff).sum() / b + l.backward() + + dw = ctx_ref.grad.permute(0,2,1,3) + + dw2 = dw.permute(0,2,1,3).clone().detach().contiguous() + + if b < 4: + dqkv2, _, _ = mha.bwd_nl(dw2, qkv_vs, S_, cu_seqlens, 0.0, s, zero_tensors) + else: + dqkv2, _ = mha.bwd(dw2, qkv_vs, S_, cu_seqlens, 0.0, s, zero_tensors) + + dqkv2 = dqkv2.permute(0,2,1,3).view(b,s, h,3,d) + + self.assertTrue(torch.allclose(qkv.grad.float(), dqkv2.float(), atol=1e-3)) + + def test_128(self): + self.run_test(128, 32, False) + self.run_test(128, 32, True) + self.run_test(128, 56, False) + self.run_test(128, 56, True) + + def test_256(self): + self.run_test(256, 32, False) + self.run_test(256, 32, True) + self.run_test(256, 56, False) + self.run_test(256, 56, True) + + def test_384(self): + self.run_test(384, 32, False) + self.run_test(384, 32, True) + self.run_test(384, 56, False) + self.run_test(384, 56, True) + + def test_512(self): + self.run_test(512, 32, False) + self.run_test(512, 32, True) + self.run_test(512, 56, False) + self.run_test(512, 56, True) + self.run_test(512, 2, False) + self.run_test(512, 2, True) + self.run_test(512, 3, False) + self.run_test(512, 3, True) + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/apex/contrib/test/focal_loss/__init__.py b/apex/apex/contrib/test/focal_loss/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/focal_loss/test_focal_loss.py b/apex/apex/contrib/test/focal_loss/test_focal_loss.py new file mode 100644 index 00000000..a6338139 --- /dev/null +++ b/apex/apex/contrib/test/focal_loss/test_focal_loss.py @@ -0,0 +1,74 @@ +import unittest + +import torch +import torch.nn.functional as F + +reference_available = True +try: + from torchvision.ops.focal_loss import sigmoid_focal_loss +except ImportError: + reference_available = False + +SKIP_TEST = None +try: + from apex.contrib.focal_loss import focal_loss +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +@unittest.skipIf(not reference_available, "Reference implementation `torchvision.ops.focal_loss.sigmoid_focal_loss` is not available.") +class FocalLossTest(unittest.TestCase): + + N_SAMPLES = 12 + N_CLASSES = 8 + ALPHA = 0.24 + GAMMA = 2.0 + REDUCTION = "sum" + + def test_focal_loss(self) -> None: + if not reference_available: + self.skipTest("This test needs `torchvision` for `torchvision.ops.focal_loss.sigmoid_focal_loss`.") + else: + x = torch.randn(FocalLossTest.N_SAMPLES, FocalLossTest.N_CLASSES).cuda() + with torch.no_grad(): + x_expected = x.clone() + x_actual = x.clone() + x_expected.requires_grad_() + x_actual.requires_grad_() + + classes = torch.randint(0, FocalLossTest.N_CLASSES, (FocalLossTest.N_SAMPLES,)).cuda() + with torch.no_grad(): + y = F.one_hot(classes, FocalLossTest.N_CLASSES).float() + + expected = sigmoid_focal_loss( + x_expected, + y, + alpha=FocalLossTest.ALPHA, + gamma=FocalLossTest.GAMMA, + reduction=FocalLossTest.REDUCTION, + ) + + actual = sum([focal_loss.FocalLoss.apply( + x_actual[i:i+1], + classes[i:i+1].long(), + torch.ones([], device="cuda"), + FocalLossTest.N_CLASSES, + FocalLossTest.ALPHA, + FocalLossTest.GAMMA, + 0.0, + ) for i in range(FocalLossTest.N_SAMPLES)]) + + # forward parity + torch.testing.assert_close(expected, actual) + + expected.backward() + actual.backward() + + # grad parity + torch.testing.assert_close(x_expected.grad, x_actual.grad) + + +if __name__ == "__main__": + torch.manual_seed(42) + unittest.main() diff --git a/apex/apex/contrib/test/fused_dense/test_fused_dense.py b/apex/apex/contrib/test/fused_dense/test_fused_dense.py new file mode 100644 index 00000000..c293aeeb --- /dev/null +++ b/apex/apex/contrib/test/fused_dense/test_fused_dense.py @@ -0,0 +1,62 @@ +import unittest +import os + +import torch +from torch.testing._internal import common_utils +from torch.testing._internal.common_device_type import instantiate_device_type_tests + +SKIP_TEST = None +try: + from apex import fused_dense +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class FusedDenseTest(common_utils.TestCase): + + def _test_fused_dense(self, dtype, seed=0): + + os.environ["TORCH_ALLOW_TF32_CUBLAS_OVERRIDE"] = "0" + torch.manual_seed(seed) + + seq_length = 512 + sequences = 3 + hidden_dim = 1024 + + ref_inputs = torch.randn(sequences*seq_length, hidden_dim, + dtype=dtype, device=torch.device("cuda")).requires_grad_(True) + + tst_inputs = ref_inputs.clone().detach().requires_grad_(True) + dense = fused_dense.FusedDense(1024, 3072) + dense.to(dtype=dtype) + dense.cuda() + + y_tst = dense(tst_inputs) + y_ref = torch.matmul(ref_inputs, dense.weight.t())+dense.bias + dy = torch.randn_like(y_tst).to(dtype=dtype) + y_tst.backward(dy) + dw_ref = torch.matmul(dy.t(), ref_inputs) + dx_ref = torch.matmul(dy, dense.weight.clone()) + db_ref = dy.sum(0, False) + + torch.testing.assert_close( + ref_inputs, tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close( + y_ref, y_tst, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close( + dw_ref, dense.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close( + dx_ref, tst_inputs.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close( + db_ref, dense.bias.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + @common_utils.parametrize("dtype", [torch.half, torch.float, torch.bfloat16]) + def test_fused_dense(self, dtype): + self._test_fused_dense(dtype) + + +instantiate_device_type_tests(FusedDenseTest, globals(), only_for=("cuda",)) + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/apex/contrib/test/index_mul_2d/__init__.py b/apex/apex/contrib/test/index_mul_2d/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/index_mul_2d/test_index_mul_2d.py b/apex/apex/contrib/test/index_mul_2d/test_index_mul_2d.py new file mode 100644 index 00000000..1c16fd33 --- /dev/null +++ b/apex/apex/contrib/test/index_mul_2d/test_index_mul_2d.py @@ -0,0 +1,104 @@ +import random +import unittest + +import torch + +HAS_INDEX_MUL_2D_RELU = None +try: + from apex.contrib.index_mul_2d import index_mul_2d +except ImportError as e: + HAS_INDEX_MUL_2D_RELU = False +else: + HAS_INDEX_MUL_2D_RELU = True + + +@unittest.skipIf(not HAS_INDEX_MUL_2D_RELU, "`apex.contrib.index_mul_2d` is not found.") +class IndexMul2dTest(unittest.TestCase): + def setUp(self, seed=0): + torch.manual_seed(seed) + + self.input1_size = random.randint(1, 1000) + self.input2_size = random.randint(1, 100000) + self.feature_size = random.randint(1, 256) + + self.input1_float = torch.randn(size=(self.input1_size, self.feature_size),).cuda() + self.input2_float = torch.randn(size=(self.input2_size, self.feature_size),).cuda() + self.index1 = torch.randint(low=0, high=self.input1_size, size=(self.input2_size,)).cuda() + + self.input1_float_ = self.input1_float.clone() + self.input2_float_ = self.input2_float.clone() + + self.input1_float.requires_grad_() + self.input1_float_.requires_grad_() + self.input2_float.requires_grad_() + self.input2_float_.requires_grad_() + + self.input1_half = torch.randn(size=(self.input1_size, self.feature_size),).cuda().half() + self.input2_half = torch.randn(size=(self.input2_size, self.feature_size),).cuda().half() + + self.input1_half_ = self.input1_half.clone() + self.input2_half_ = self.input2_half.clone() + + self.input1_half.requires_grad_() + self.input2_half.requires_grad_() + self.input1_half_.requires_grad_() + self.input2_half_.requires_grad_() + + def test_index_mul_float(self): + out = index_mul_2d(self.input1_float, self.input2_float, self.index1) + energy = (out.float()**2).sum() / out.numel() + force = torch.autograd.grad( + energy, + self.input1_float, + grad_outputs=torch.ones_like(energy), + create_graph=True, + )[0] + loss = (out.float()**2).sum() / out.numel() + (force.float()**2).sum() + loss.backward() + + out_ = self.input1_float_[self.index1] * self.input2_float_ + energy_ = (out_.float()**2).sum() / out.numel() + force_ = torch.autograd.grad( + energy_, + self.input1_float_, + grad_outputs=torch.ones_like(energy), + create_graph=True, + )[0] + loss = (out_.float()**2).sum() / out_.numel() + (force_.float()**2).sum() + loss.backward() + + self.assertTrue(torch.allclose(self.input1_float, self.input1_float_, atol=1e-3, rtol=1e-3, equal_nan=True)) + self.assertTrue(torch.allclose(self.input2_float, self.input2_float_, atol=1e-3, rtol=1e-3, equal_nan=True)) + self.assertTrue(torch.allclose(self.input1_float.grad, self.input1_float_.grad, atol=1e-3, rtol=1e-3, equal_nan=True)) + self.assertTrue(torch.allclose(self.input2_float.grad, self.input2_float_.grad, atol=1e-3, rtol=1e-3, equal_nan=True)) + + def test_index_mul_half(self): + out = index_mul_2d(self.input1_half, self.input2_half, self.index1) + energy = (out.float()**2).sum() / out.numel() + force = torch.autograd.grad( + energy, + self.input1_half, + grad_outputs=torch.ones_like(energy), + create_graph=True, + )[0] + loss = (out.float()**2).sum() / out.numel() + (force.float()**2).sum() + loss.backward() + + out_ = self.input1_half_[self.index1] * self.input2_half_ + energy_ = (out_.float()**2).sum() / out.numel() + force_ = torch.autograd.grad( + energy_, + self.input1_half_, + grad_outputs=torch.ones_like(energy), + create_graph=True, + )[0] + loss = (out_.float()**2).sum() / out_.numel() + (force_.float()**2).sum() + loss.backward() + + self.assertTrue(torch.allclose(self.input1_half, self.input1_half_, atol=1e-3, rtol=1e-3, equal_nan=True)) + self.assertTrue(torch.allclose(self.input2_half, self.input2_half_, atol=1e-3, rtol=1e-3, equal_nan=True)) + self.assertTrue(torch.allclose(self.input1_half.grad, self.input1_half_.grad, atol=1e-3, rtol=1e-3, equal_nan=True)) + self.assertTrue(torch.allclose(self.input2_half.grad, self.input2_half_.grad, atol=1e-3, rtol=1e-3, equal_nan=True)) + +if __name__ == '__main__': + unittest.main() diff --git a/apex/apex/contrib/test/layer_norm/__init__.py b/apex/apex/contrib/test/layer_norm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/layer_norm/test_fast_layer_norm.py b/apex/apex/contrib/test/layer_norm/test_fast_layer_norm.py new file mode 100644 index 00000000..9f6ee798 --- /dev/null +++ b/apex/apex/contrib/test/layer_norm/test_fast_layer_norm.py @@ -0,0 +1,279 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.layer_norm.layer_norm import FastLayerNorm + import fast_layer_norm as fln +except ImportError as e: + SKIP_TEST = e + + +class GPUTimer: + def __init__(self, stream): + self.start_ = torch.cuda.Event(enable_timing=True) + self.stop_ = torch.cuda.Event(enable_timing=True) + self.stream_ = stream + + def start(self): + self.stream_.record_event(self.start_) + + def stop(self): + self.stream_.record_event(self.stop_) + + def sync(self): + self.stream_.synchronize() + + def millis(self): + return self.start_.elapsed_time(self.stop_) + + +def size_in_bytes(t): + return torch.numel(t) * t.element_size() + + +def metrics(y_ref, y, epsilon=1e-6): + y_ref = y_ref.float() + y = y.float() + relerr, mse = ( + (y_ref - y).abs().sum() / (y_ref.abs().sum() + epsilon), + (y_ref - y).square().mean(), + ) + return relerr.item(), mse.item() + + +device = torch.device("cuda") +fp32 = torch.float32 +fp16 = torch.float16 +bf16 = torch.bfloat16 + + +def backward_(dz, x, mu, rs, gamma): + + wtype = gamma.dtype + itype = x.dtype + otype = dz.dtype + ctype = mu.dtype + mu = mu.unsqueeze(1) + rs = rs.unsqueeze(1) + + hidden_size = gamma.numel() + y = rs * (x.to(ctype) - mu) + dbeta = dz.view(-1, hidden_size).sum(0, dtype=ctype) + dgamma = (dz * y).view(-1, hidden_size).sum(0, dtype=ctype) + dy = dz.view(-1, hidden_size).to(ctype) * gamma.unsqueeze(0).to(ctype) + mdy = dy.mean(1, keepdim=True, dtype=ctype) + + mdyy = (dy * y).mean(1, keepdim=True, dtype=ctype) + dx = rs * (dy - mdyy * y - mdy) + + return dx.to(itype), dgamma.to(wtype), dbeta.to(wtype) + + +def benchmark_(S, B, hidden_size, itype, wtype, runs=100): + epsilon = 1e-5 + + x = torch.randn((S * B, hidden_size), dtype=itype, device=device) + beta = torch.randn(hidden_size, dtype=wtype, device=device) + gamma = torch.randn(hidden_size, dtype=wtype, device=device) + dz = torch.randn(x.shape, dtype=wtype, device=device) + + stream = torch.cuda.Stream() + with torch.cuda.stream(stream): + + timer = GPUTimer(stream) + + # warmup + for r in range(runs): + z, mu, rsigma = fln.ln_fwd(x, gamma, beta, epsilon) + + timer.start() + for r in range(runs): + z, mu, rsigma = fln.ln_fwd(x, gamma, beta, epsilon) + timer.stop() + timer.sync() + + total_bytes_fwd = sum([size_in_bytes(t) for t in [x, z, gamma, beta, mu, rsigma]]) + + ms_fwd = timer.millis() / runs + + print( + "[FWD] Time: {:.4f}ms Throughput: {:.4f} GB/sec".format( + ms_fwd, total_bytes_fwd * 1e-6 / ms_fwd + ) + ) + + timer.start() + for r in range(runs): + dx, dgamma, dbeta, dbp, dgp = fln.ln_bwd(dz, x, mu, rsigma, gamma) + timer.stop() + timer.sync() + + total_bytes_bwd = sum( + [ + size_in_bytes(t) + for t in [dz, x, mu, rsigma, gamma, dx, dgamma, dbeta, dbp, dbp, dgp, dgp] + ] + ) + + ms_bwd = timer.millis() / runs + + print( + "[BWD] Time: {:.4f}ms Throughput: {:.4f} GB/sec".format( + ms_bwd, total_bytes_bwd * 1e-6 / ms_bwd + ) + ) + + +def _test_impl(S, B, hidden_size, itype, wtype, ctype=fp32): + + seed = 1243 + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + otype = wtype + print("========================================================") + print(f"S={S} B={B} Hidden={hidden_size} {itype} {wtype}") + print("--------------------------------------------------------") + + x = torch.randn(S * B, hidden_size, dtype=itype, device=device) + gamma = torch.randn(hidden_size, dtype=wtype, device=device) * 0.2 + beta = torch.randn(hidden_size, dtype=wtype, device=device) * 0.2 + epsilon = 1e-5 + + x.requires_grad = True + gamma.requires_grad = True + beta.requires_grad = True + + mu_ref = x.mean(1, dtype=ctype, keepdim=True) + v = torch.square(x - mu_ref).mean(1, dtype=ctype, keepdim=True) + rs_ref = torch.rsqrt(v + epsilon) + y_ref = rs_ref * (x.to(ctype) - mu_ref) + z_ref = (gamma.unsqueeze(0) * (y_ref).to(otype) + beta.unsqueeze(0)).to(otype) + + mu_ref = mu_ref.flatten() + rs_ref = rs_ref.flatten() + + dz = torch.randn_like(z_ref) + + # z_ref.backward(dz) + # dx_ref = x.grad + # dgamma_ref = gamma.grad + # dbeta_ref = beta.grad + + dx_ref, dg_ref, db_ref = backward_(dz, x, mu_ref, rs_ref, gamma) + + z, mu, rs = fln.ln_fwd(x, gamma, beta, epsilon) + dx, dg, db, dg_part, db_part = fln.ln_bwd(dz, x, mu, rs, gamma) + + re_z, mse_z = metrics(z_ref, z) + re_mu, mse_mu = metrics(mu_ref, mu) + re_rs, mse_rs = metrics(rs_ref, rs) + + re_dx, mse_dx = metrics(dx_ref, dx) + re_dg, mse_dg = metrics(dg_ref, dg) + re_db, mse_db = metrics(db_ref, db) + + print(f" z: relerr={re_z :.4e} mse={mse_z :.4e}") + print(f"mu: relerr={re_mu:.4e} mse={mse_mu:.4e}") + print(f"rs: relerr={re_mu:.4e} mse={mse_mu:.4e}") + + print(f"dx: relerr={re_dx:.4e} mse={mse_dx:.4e}") + print(f"dg: relerr={re_dg:.4e} mse={mse_dg:.4e}") + print(f"db: relerr={re_db:.4e} mse={mse_db:.4e}") + + def check_err(x, relerr): + tol = 1e-3 if x.dtype == torch.float16 else 5e-6 + return relerr < tol + + return [ + check_err(x, re) + for x, re in zip([z, mu, rs, dx, dg, db], [re_z, re_mu, re_rs, re_dx, re_dg, re_db]) + ] + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TestFastLayerNorm(unittest.TestCase): + # TODO(crcrpar): Try `torch.testing.assert_close` instead and migrate to it if it's working. + def assertAll(self, l): + if not all(l): + print(l) + for x in l: + self.assertTrue(x) + + def test_all_configs(self): + + hidden_sizes = [ + 768, + 1024, + 1536, + 2048, + 2304, + 3072, + 3840, + 4096, + 5120, + 6144, + 8192, + 10240, + 12288, + 12800, + 14336, + 15360, + 16384, + 18432, + 20480, + 24576, + 25600, + 30720, + 32768, + 40960, + 49152, + 65536, + ] + + for h in hidden_sizes: + with self.subTest(f"hidden_size={h}"): + self.assertAll(_test_impl(256, 2, h, fp32, fp32)) + self.assertAll(_test_impl(256, 2, h, fp16, fp16)) + self.assertAll(_test_impl(256, 2, h, fp32, fp16)) + self.assertAll(_test_impl(256, 2, h, bf16, bf16)) + self.assertAll(_test_impl(256, 2, h, fp32, bf16)) + + def test_run_benchmark(self): + for (S, B, hidden_size, runs) in ( + (512, 32, 768, 1000), + (512, 32, 1024, 1000), + (512, 8, 4096, 1000), + (512, 8, 5120, 1000), + (512, 8, 6144, 1000), + (256, 2, 20480, 500), + (256, 2, 25600, 500), + (256, 2, 40960, 250), + (256, 2, 65536, 250), + ): + with self.subTest(f"(S, B, hidden_size)=({S}, {B}, {hidden_size})"): + benchmark_(S, B, hidden_size, fp16, fp16, runs) + + def test_compat_with_autocast(self): + autocast_dtypes = ( + (torch.half, torch.bfloat16) if torch.cuda.is_bf16_supported() else (torch.half,) + ) + input_shape = (512, 32, 768) + layer_norm = FastLayerNorm(input_shape[-1]).cuda() + input = torch.randn(input_shape).cuda() + + for dtype in autocast_dtypes: + layer_norm.zero_grad(set_to_none=True) + with self.subTest(f"autocast_dtype={dtype}"): + with torch.cuda.amp.autocast(enabled=True, dtype=dtype): + out = layer_norm(input) + self.assertEqual(dtype, out.dtype) + grad = torch.randn_like(out) + out.backward(grad) + self.assertEqual(torch.float32, layer_norm.weight.grad.dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/multihead_attn/__init__.py b/apex/apex/contrib/test/multihead_attn/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn.py b/apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn.py new file mode 100644 index 00000000..cc7ae510 --- /dev/null +++ b/apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn.py @@ -0,0 +1,161 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.multihead_attn import EncdecMultiheadAttn +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class EncdecMultiheadAttnTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + + self.seq_length = 80 + self.sequences = 10 + self.hidden_dim = 1024 + self.heads = 16 + self.dropout_prob = 0.0 + + self.ref_layer = EncdecMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=False, impl="default" + ) + self.ref_layer.cuda().half() + self.ref_layer.reset_parameters() + self.ref_inputs_q = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + self.ref_inputs_k = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + # Reset seed so parameters are identical + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.tst_layer = EncdecMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=False, impl="fast" + ) + self.tst_layer.cuda().half() + self.tst_layer.reset_parameters() + + self.tst_inputs_q = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + self.tst_inputs_k = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + def test_encdec_multihead_attn(self): + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs_q, + self.ref_inputs_k, + self.ref_inputs_k, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs_q, + self.tst_inputs_k, + self.tst_inputs_k, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + torch.testing.assert_close(self.ref_inputs_q, self.tst_inputs_q, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(self.ref_inputs_k, self.tst_inputs_k, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + + with torch.no_grad(): + ref_grads = torch.randn_like(ref_outputs) + tst_grads = ref_grads.clone() + ref_outputs.backward(ref_grads) + tst_outputs.backward(tst_grads) + torch.testing.assert_close(self.ref_inputs_q.grad, self.tst_inputs_q.grad, atol=1e-3, rtol=1e-3) + + def test_encdec_multihead_attn_time_mask(self): + grads = torch.randn_like(self.tst_inputs_q) + time_mask_byte = torch.triu( + torch.ones( + self.tst_inputs_q.size(0), self.tst_inputs_k.size(0), device=torch.device("cuda"), dtype=torch.uint8 + ), + 1, + ) + time_mask_bool = time_mask_byte.to(torch.bool) + + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs_q, + self.ref_inputs_k, + self.ref_inputs_k, + key_padding_mask=None, + need_weights=False, + attn_mask=time_mask_bool, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs_q, + self.tst_inputs_k, + self.tst_inputs_k, + key_padding_mask=None, + need_weights=False, + attn_mask=time_mask_byte, + is_training=True, + ) + + self.ref_inputs_q.backward(grads) + self.tst_inputs_q.backward(grads) + + torch.testing.assert_close(self.ref_inputs_q, self.tst_inputs_q, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(self.ref_inputs_k, self.tst_inputs_k, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs_q.grad, self.tst_inputs_q.grad, atol=1e-3, rtol=1e-3) + + def test_encdec_multihead_attn_pad_mask(self): + grads = torch.randn_like(self.tst_inputs_q) + pad_mask_byte = torch.tril( + torch.ones( + self.tst_inputs_k.size(1), self.tst_inputs_k.size(0), device=torch.device("cuda"), dtype=torch.uint8 + ), + 1, + ) + pad_mask_bool = pad_mask_byte.to(torch.bool) + + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs_q, + self.ref_inputs_k, + self.ref_inputs_k, + key_padding_mask=pad_mask_bool, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs_q, + self.tst_inputs_k, + self.tst_inputs_k, + key_padding_mask=pad_mask_byte, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + self.ref_inputs_q.backward(grads) + self.tst_inputs_q.backward(grads) + + torch.testing.assert_close(self.ref_inputs_q, self.tst_inputs_q, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(self.ref_inputs_k, self.tst_inputs_k, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs_q.grad, self.tst_inputs_q.grad, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn_norm_add.py b/apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn_norm_add.py new file mode 100644 index 00000000..c46345e0 --- /dev/null +++ b/apex/apex/contrib/test/multihead_attn/test_encdec_multihead_attn_norm_add.py @@ -0,0 +1,87 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.multihead_attn import EncdecMultiheadAttn +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class EncdecMultiheadAttnNormAddTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.seq_length = 80 + self.sequences = 10 + self.hidden_dim = 1024 + self.heads = 16 + self.dropout_prob = 0.0 + + self.ref_layer = EncdecMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=True, impl="default" + ) + self.ref_layer.cuda().half() + self.ref_layer.reset_parameters() + self.ref_inputs_q = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + self.ref_inputs_k = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + # Reset seed so parameters are identical + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.tst_layer = EncdecMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=True, impl="fast" + ) + self.tst_layer.cuda().half() + self.tst_layer.reset_parameters() + + self.tst_inputs_q = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + self.tst_inputs_k = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + def test_encdec_multihead_attn_norm_add(self): + grads = torch.randn_like(self.tst_inputs_q) + + for _ in range(5): + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs_q, + self.ref_inputs_k, + self.ref_inputs_k, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs_q, + self.tst_inputs_k, + self.tst_inputs_k, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + self.ref_inputs_q.backward(grads) + self.tst_inputs_q.backward(grads) + + torch.testing.assert_close(self.ref_inputs_q, self.tst_inputs_q, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(self.ref_inputs_k, self.tst_inputs_k, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs_q.grad, self.tst_inputs_q.grad, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/multihead_attn/test_fast_self_multihead_attn_bias.py b/apex/apex/contrib/test/multihead_attn/test_fast_self_multihead_attn_bias.py new file mode 100644 index 00000000..a7dcfd28 --- /dev/null +++ b/apex/apex/contrib/test/multihead_attn/test_fast_self_multihead_attn_bias.py @@ -0,0 +1,93 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.multihead_attn import SelfMultiheadAttn +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class SelfMultiheadAttnTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.seq_length = 80 + self.sequences = 10 + self.hidden_dim = 1024 + self.heads = 16 + self.dropout_prob = 0.0 + + self.ref_layer = SelfMultiheadAttn( + self.hidden_dim, + self.heads, + dropout=self.dropout_prob, + bias=True, + include_norm_add=False, + separate_qkv_params=True, + mask_additive=True, + impl="default", + ) + self.ref_layer.cuda().half() + self.ref_layer.reset_parameters() + self.ref_inputs = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + # Reset seed so parameters are identical + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.tst_layer = SelfMultiheadAttn( + self.hidden_dim, + self.heads, + dropout=self.dropout_prob, + bias=True, + include_norm_add=False, + separate_qkv_params=True, + mask_additive=True, + impl="fast", + ) + self.tst_layer.cuda().half() + self.tst_layer.reset_parameters() + + self.tst_inputs = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + def test_self_multihead_attn_additive_mask(self): + grads = torch.randn_like(self.tst_inputs) + mask = ((torch.randn(self.sequences, self.seq_length) > 0) * -10000.0).half().cuda() + + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs, + self.ref_inputs, + self.ref_inputs, + key_padding_mask=mask, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs, + self.tst_inputs, + self.tst_inputs, + key_padding_mask=mask, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + self.ref_inputs.backward(grads) + self.tst_inputs.backward(grads) + + torch.testing.assert_close(self.ref_inputs, self.tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs.grad, self.tst_inputs.grad, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/multihead_attn/test_mha_fused_softmax.py b/apex/apex/contrib/test/multihead_attn/test_mha_fused_softmax.py new file mode 100644 index 00000000..97c0bec5 --- /dev/null +++ b/apex/apex/contrib/test/multihead_attn/test_mha_fused_softmax.py @@ -0,0 +1,55 @@ +import unittest + +import torch +import torch.nn.functional as F + +SKIP_TEST = None +try: + from apex.contrib.multihead_attn import fast_mask_softmax_dropout_func +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class FusedSoftmaxTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.seq_length = 80 + self.sequences = 10 + self.hidden_dim = 1024 + self.heads = 16 + self.dropout_prob = 0.0 + + self.mask = (torch.randn(self.sequences, self.seq_length) > 0).cuda() + self.mask = self.mask.half() * -10000 + self.ref_inputs = torch.randn( + self.heads * self.sequences, + self.seq_length, + self.seq_length, + dtype=torch.float16, + device=torch.device("cuda"), + ).requires_grad_(True) + + self.tst_inputs = self.ref_inputs.clone().detach().requires_grad_(True) + + def test_fused_softmax(self): + grads = torch.randn_like(self.tst_inputs) + y_ref = self.ref_inputs.view(self.sequences, self.heads, self.seq_length, self.seq_length) + y_ref = y_ref + self.mask.unsqueeze(1).unsqueeze(2) + y_ref = y_ref.view(self.sequences * self.heads, self.seq_length, self.seq_length) + y_ref = F.softmax(y_ref, dim=-1) + y_ref = torch._fused_dropout(y_ref, 1.0) + + y_tst = fast_mask_softmax_dropout_func(True, self.heads, self.tst_inputs, self.mask, True, 0.0) + y_ref[0].backward(grads) + y_tst.backward(grads) + + torch.testing.assert_close(self.ref_inputs, self.tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(y_ref[0], y_tst, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs.grad, self.tst_inputs.grad, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/multihead_attn/test_self_multihead_attn.py b/apex/apex/contrib/test/multihead_attn/test_self_multihead_attn.py new file mode 100644 index 00000000..2ff488ae --- /dev/null +++ b/apex/apex/contrib/test/multihead_attn/test_self_multihead_attn.py @@ -0,0 +1,155 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.multihead_attn import SelfMultiheadAttn +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class SelfMultiheadAttnTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.seq_length = 80 + self.sequences = 10 + self.hidden_dim = 1024 + self.heads = 16 + self.dropout_prob = 0.0 + + self.ref_layer = SelfMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=False, impl="default" + ) + self.ref_layer.cuda().half() + self.ref_layer.reset_parameters() + self.ref_inputs = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + # Reset seed so parameters are identical + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.tst_layer = SelfMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=False, impl="fast" + ) + self.tst_layer.cuda().half() + self.tst_layer.reset_parameters() + + self.tst_inputs = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + def test_self_multihead_attn(self): + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs, + self.ref_inputs, + self.ref_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs, + self.tst_inputs, + self.tst_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + torch.testing.assert_close(self.ref_inputs, self.tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + + with torch.no_grad(): + ref_grads = torch.randn_like(self.tst_inputs) + tst_grads = ref_grads.clone() + + ref_outputs.backward(ref_grads) + tst_outputs.backward(tst_grads) + torch.testing.assert_close(self.ref_inputs.grad, self.tst_inputs.grad, atol=1e-3, rtol=1e-3) + + def test_self_multihead_attn_time_mask(self): + grads = torch.randn_like(self.tst_inputs) + time_mask_byte = torch.triu( + torch.ones( + self.tst_inputs.size(0), self.tst_inputs.size(0), device=torch.device("cuda"), dtype=torch.uint8 + ), + 1, + ) + time_mask_bool = time_mask_byte.to(torch.bool) + + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs, + self.ref_inputs, + self.ref_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=time_mask_bool, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs, + self.tst_inputs, + self.tst_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=time_mask_byte, + is_training=True, + ) + + self.ref_inputs.backward(grads) + self.tst_inputs.backward(grads) + + torch.testing.assert_close(self.ref_inputs, self.tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs.grad, self.tst_inputs.grad, atol=1e-3, rtol=1e-3) + + def test_self_multihead_attn_pad_mask(self): + grads = torch.randn_like(self.tst_inputs) + pad_mask_byte = torch.tril( + torch.ones( + self.tst_inputs.size(1), self.tst_inputs.size(0), device=torch.device("cuda"), dtype=torch.uint8 + ), + 1, + ) + pad_mask_bool = pad_mask_byte.to(torch.bool) + + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs, + self.ref_inputs, + self.ref_inputs, + key_padding_mask=pad_mask_bool, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs, + self.tst_inputs, + self.tst_inputs, + key_padding_mask=pad_mask_byte, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + self.ref_inputs.backward(grads) + self.tst_inputs.backward(grads) + + torch.testing.assert_close(self.ref_inputs, self.tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs.grad, self.tst_inputs.grad, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/multihead_attn/test_self_multihead_attn_norm_add.py b/apex/apex/contrib/test/multihead_attn/test_self_multihead_attn_norm_add.py new file mode 100644 index 00000000..81441135 --- /dev/null +++ b/apex/apex/contrib/test/multihead_attn/test_self_multihead_attn_norm_add.py @@ -0,0 +1,79 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.multihead_attn import SelfMultiheadAttn +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class SelfMultiheadAttnNormAddTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + + self.seq_length = 80 + self.sequences = 10 + self.hidden_dim = 1024 + self.heads = 16 + self.dropout_prob = 0.0 + + self.ref_layer = SelfMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=True, impl="default" + ) + self.ref_layer.cuda().half() + self.ref_layer.reset_parameters() + self.ref_inputs = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + # Reset seed so parameters are identical + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + self.tst_layer = SelfMultiheadAttn( + self.hidden_dim, self.heads, dropout=self.dropout_prob, bias=False, include_norm_add=True, impl="fast" + ) + self.tst_layer.cuda().half() + self.tst_layer.reset_parameters() + + self.tst_inputs = torch.randn( + self.seq_length, self.sequences, self.hidden_dim, dtype=torch.float16, device=torch.device("cuda") + ).requires_grad_(True) + + def test_self_multihead_attn_norm_add(self): + grads = torch.randn_like(self.tst_inputs) + + for _ in range(0, 5): + ref_outputs, _ = self.ref_layer.forward( + self.ref_inputs, + self.ref_inputs, + self.ref_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + tst_outputs, _ = self.tst_layer.forward( + self.tst_inputs, + self.tst_inputs, + self.tst_inputs, + key_padding_mask=None, + need_weights=False, + attn_mask=None, + is_training=True, + ) + + self.ref_inputs.backward(grads) + self.tst_inputs.backward(grads) + + torch.testing.assert_close(self.ref_inputs, self.tst_inputs, atol=1e-5, rtol=1e-5) + torch.testing.assert_close(ref_outputs, tst_outputs, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(self.ref_inputs.grad, self.tst_inputs.grad, atol=1e-3, rtol=1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/apex/contrib/test/optimizers/__init__.py b/apex/apex/contrib/test/optimizers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/optimizers/test_dist_adam.py b/apex/apex/contrib/test/optimizers/test_dist_adam.py new file mode 100644 index 00000000..d67168f7 --- /dev/null +++ b/apex/apex/contrib/test/optimizers/test_dist_adam.py @@ -0,0 +1,470 @@ +from contextlib import contextmanager +import io +import unittest + +import torch +from torch.testing._internal import common_utils + +SKIP_TEST = None +try: + from apex.contrib.optimizers.distributed_fused_adam import DistributedFusedAdam +except ImportError as e: + SKIP_TEST = e +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase + + +class SimpleModel(torch.nn.Module): + def __init__(self, num_layers, size): + super().__init__() + self.params = torch.nn.ParameterList([ + torch.nn.Parameter(torch.rand(1, size) + 1) + for _ in range(num_layers) + ]) + def forward(self, x): + y = 0 + for i, param in enumerate(self.params): + y += (i+1) * param * x + return y + + +def make_models( + num_layers, + size, + adam_w_mode=True, + model_dtype=torch.float32, + optim_dtype=None, + grad_sync_dtype=None, + param_sync_dtype=None, + device='cuda', + overlap_communication=True, + contiguous_buffers=False, + store_params=False, + store_param_remainders=False, +): + + # Construct models with same parameters + ref_model = SimpleModel(num_layers, size).to(dtype=model_dtype, device=device) + dist_model = SimpleModel(num_layers, size).to(dtype=model_dtype, device=device) + with torch.no_grad(): + for ref_param, dist_param in zip(dist_model.parameters(), + ref_model.parameters()): + dist_param.copy_(ref_param) + + # Initialize reference model with data-parallelism + rank = torch.distributed.get_rank() + ref_model = torch.nn.parallel.DistributedDataParallel( + ref_model, + device_ids=[rank] if device=='cuda' else None, + output_device=rank if device=='cuda' else None, + ) + + # Construct optimizers with same hyperparameters + if optim_dtype is None: + optim_dtype = model_dtype + optim_args = dict(lr=0.1, betas=(0.1,0.2), eps=0.25, weight_decay=0.1) + ref_optim_class = torch.optim.AdamW if adam_w_mode else torch.optim.Adam + ref_optim = ref_optim_class( + [ + {'params': list(ref_model.parameters())[1::2], 'lr': 0.2}, + {'params': list(ref_model.parameters())[0::2]}, + ], + **optim_args, + ) + dist_optim = DistributedFusedAdam( + [ + {'params': list(dist_model.parameters())[1::2], 'lr': 0.2}, + {'params': list(dist_model.parameters())[0::2]}, + ], + adam_w_mode=adam_w_mode, + overlap_grad_sync=overlap_communication, + overlap_param_sync=overlap_communication, + bucket_cap_mb=71/(4*1024*1024), + dtype=optim_dtype, + grad_sync_dtype=grad_sync_dtype, + param_sync_dtype=param_sync_dtype, + contiguous_param_buffer=contiguous_buffers, + contiguous_grad_buffer=contiguous_buffers, + store_params=store_params, + store_param_remainders=store_param_remainders, + **optim_args, + ) + + return ref_model, ref_optim, dist_model, dist_optim + + +@contextmanager +def dummy_context(): + try: + yield + finally: + pass + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TestDistributedFusedAdam(NcclDistributedTestBase): + + seed = 1234 + + def test_matches_pytorch( + self, + rtol=None, + atol=None, + num_layers=11, + layer_size=7, + batch_size=3, + num_steps=3, + micro_batch_steps=3, + adam_w_mode=True, + overlap_communication=True, + use_nosync=True, + model_dtype=torch.float32, + optim_dtype=None, + grad_sync_dtype=None, + param_sync_dtype=None, + device='cuda', + contiguous_buffers=False, + store_params=False, + store_param_remainders=False, + ): + + torch.manual_seed(self.seed + self.rank) + + # Identical models with data-parallel and ZeRO + ref_model, ref_optim, dist_model, dist_optim = make_models( + num_layers, + layer_size, + adam_w_mode=adam_w_mode, + model_dtype=model_dtype, + optim_dtype=optim_dtype, + grad_sync_dtype=grad_sync_dtype, + param_sync_dtype=param_sync_dtype, + device=device, + overlap_communication=overlap_communication, + contiguous_buffers=contiguous_buffers, + store_params=store_params, + store_param_remainders=store_param_remainders, + ) + + # Training loop + for step in range(num_steps): + + # Reset gradients + ref_optim.zero_grad() + dist_optim.zero_grad() + + # Forward and backward passes + for micro_step in range(micro_batch_steps): + + # Synthetic data + x = torch.rand(batch_size, layer_size) - 0.5 + dy = torch.rand_like(x) - 0.5 + x = x.to(dtype=model_dtype, device=device) + dy = dy.to(dtype=model_dtype, device=device) + + # Reference implementation + x_ref = x.detach().clone().requires_grad_(True) + y_ref = ref_model(x_ref) + y_ref.backward(dy) + + # Distributed implementation + x_dist = x.detach().clone().requires_grad_(True) + y_dist = dist_model(x_dist) + backward_context = dummy_context + if use_nosync and micro_step < micro_batch_steps-1: + backward_context = dist_optim.no_sync + with backward_context(): + y_dist.backward(dy) + + # Check that data tensors match + torch.testing.assert_close( + y_dist, y_ref, rtol=rtol, atol=atol) + torch.testing.assert_close( + x_dist.grad, x_ref.grad, rtol=rtol, atol=atol) + + # Optimization step + ref_optim.step() + dist_optim.step() + + # Check that parameters match + for ref_param, dist_param in zip(ref_model.parameters(), + dist_model.parameters()): + torch.testing.assert_close( + dist_param, ref_param, rtol=rtol, atol=atol) + + def test_matches_pytorch_l2_reg(self): + self.test_matches_pytorch(adam_w_mode=False) + + def test_matches_pytorch_no_overlap(self): + self.test_matches_pytorch( + overlap_communication=False, + use_nosync=False, + ) + + def test_matches_pytorch_sync_every_step(self): + self.test_matches_pytorch(use_nosync=False) + + def test_matches_pytorch_contiguous_buffers(self): + self.test_matches_pytorch(contiguous_buffers=True) + + def test_matches_pytorch_fp64(self): + self.test_matches_pytorch( + rtol=1.3e-6, + atol=1e-5, + model_dtype=torch.float64, + optim_dtype=torch.float32, + ) + + def test_matches_pytorch_fp16(self): + self.test_matches_pytorch( + rtol=5e-3, + atol=1e-5, + micro_batch_steps=1, + model_dtype=torch.float16, + optim_dtype=torch.float16, + ) + + def test_matches_pytorch_bf16(self): + self.test_matches_pytorch( + rtol=5e-2, + atol=1e-5, + micro_batch_steps=1, + model_dtype=torch.bfloat16, + optim_dtype=torch.bfloat16, + ) + + def test_matches_pytorch_fp16_params(self): + self.test_matches_pytorch( + rtol=5e-3, + atol=1e-5, + micro_batch_steps=1, + model_dtype=torch.float16, + optim_dtype=torch.float32, + param_sync_dtype=torch.float16, + store_params=True, + ) + + def test_matches_pytorch_bf16_grads(self): + self.test_matches_pytorch( + rtol=5e-2, + atol=1e-5, + micro_batch_steps=1, + model_dtype=torch.float32, + optim_dtype=torch.float32, + grad_sync_dtype=torch.bfloat16, + ) + + def test_matches_pytorch_bf16_param_remainders(self): + self.test_matches_pytorch( + rtol=5e-2, + atol=1e-5, + micro_batch_steps=1, + model_dtype=torch.bfloat16, + optim_dtype=torch.float32, + param_sync_dtype=torch.bfloat16, + store_params=False, + store_param_remainders=True, + ) + + def test_raises_on_mismatch(self): + + torch.manual_seed(self.seed + self.rank) + + # Identical models with data-parallel and ZeRO + num_layers = 11 + layer_size = 7 + ref_model, ref_optim, dist_model, dist_optim = make_models( + num_layers, + layer_size, + ) + + # Only perform training step with distributed model + dist_optim.zero_grad() + x = torch.rand(3, layer_size) - 0.5 + x = x.to(dtype=torch.float32, device='cuda') + dy = torch.rand_like(x) - 0.5 + y = dist_model(x) + y.backward(dy) + dist_optim.step() + + # Check that parameters do not match + for ref_param, dist_param in zip(ref_model.parameters(), + dist_model.parameters()): + self.assertRaises( + AssertionError, + torch.testing.assert_close, + dist_param, ref_param, + ) + + def test_clip_grad_norm(self): + + torch.manual_seed(self.seed + self.rank) + + # Identical models with data-parallel and ZeRO + ref_model, ref_optim, dist_model, dist_optim = make_models(1, 1) + + # Training steps with pre-determined gradients + xs = [3, 1, 4, 1, 5, 9] + dys = [1, -1, 1, -1, 1, -1] + for x, dy in zip(xs, dys): + x = torch.tensor([[x]], dtype=torch.float32, device='cuda') + dy = torch.tensor([[dy]], dtype=torch.float32, device='cuda') + + # Reference implementation + ref_optim.zero_grad() + y_ref = ref_model(x.detach()) + y_ref.backward(dy.detach()) + ref_grad_norm = torch.nn.utils.clip_grad_norm_(ref_model.parameters(), 3.5) + ref_optim.step() + + # Distributed implementation + dist_optim.zero_grad() + y_dist = dist_model(x.detach()) + y_dist.backward(dy.detach()) + dist_grad_norm = dist_optim.clip_grad_norm(3.5) + dist_optim.step() + + # Check that parameters match + torch.testing.assert_close(dist_grad_norm, ref_grad_norm) + for ref_param, dist_param in zip(ref_model.parameters(), + dist_model.parameters()): + torch.testing.assert_close(dist_param, ref_param) + + def test_grad_scaler(self): + + torch.manual_seed(self.seed + self.rank) + + # Identical models with data-parallel and ZeRO + ref_model, ref_optim, dist_model, dist_optim = make_models(1, 1) + grad_scaler_args = dict( + init_scale=3.21, + growth_factor=1.23, + backoff_factor=0.876, + growth_interval=1, + ) + ref_scaler = torch.cuda.amp.GradScaler(**grad_scaler_args) + dist_scaler = torch.cuda.amp.GradScaler(**grad_scaler_args) + + # Training steps with pre-determined gradients + xs = [3, 1, 4, 1, 5, 9] + dys = [1, float('inf'), 1, 1, float('nan'), -1] + for x, dy in zip(xs, dys): + x = torch.tensor([[x]], dtype=torch.float32, device='cuda') + dy = torch.tensor([[dy]], dtype=torch.float32, device='cuda') + + # Reference implementation + ref_optim.zero_grad() + y_ref = ref_model(x.detach()) + ref_scaler.scale(y_ref).backward(dy.detach()) + ref_scaler.step(ref_optim) + ref_scaler.update() + + # Distributed implementation + dist_optim.zero_grad() + y_dist = dist_model(x.detach()) + dist_scaler.scale(y_dist).backward(dy.detach()) + dist_scaler.step(dist_optim) + dist_scaler.update() + + # Check that parameters match + for ref_param, dist_param in zip(ref_model.parameters(), + dist_model.parameters()): + torch.testing.assert_close(dist_param, ref_param) + + def test_checkpoint(self): + + # Construct two models with same config and different params + num_layers = 5 + layer_size = 2 + torch.manual_seed(self.seed + self.rank) + _, _, model_save, optim_save = make_models(num_layers, layer_size) + _, _, model_load, optim_load = make_models(num_layers, layer_size) + + # Train one of the models + num_steps = 3 + micro_batch_steps = 2 + batch_size = 4 + for step in range(num_steps): + optim_save.zero_grad() + for micro_step in range(micro_batch_steps): + x = torch.rand(batch_size, layer_size) - 0.5 + dy = torch.rand_like(x) - 0.5 + x = x.cuda() + dy = dy.cuda() + y = model_save(x) + y.backward(dy) + optim_save.step() + + # Make sure models are different + for param_save, param_load in zip(model_save.parameters(), + model_load.parameters()): + self.assertRaises( + AssertionError, + torch.testing.assert_close, + param_load, param_save, + ) + + # Save state on root rank and load on all ranks + state_dict = { + 'model': model_save.state_dict(), + 'optim': optim_save.state_dict(), + } + if self.rank == 0: + state_bytes = io.BytesIO() + torch.save(state_dict, state_bytes) + state_bytes = [state_bytes.getvalue()] + else: + state_bytes = [None] + torch.distributed.broadcast_object_list(state_bytes, src=0) + state_bytes = io.BytesIO(state_bytes[0]) + state_dict = torch.load(state_bytes) + model_load.load_state_dict(state_dict['model']) + optim_load.load_state_dict(state_dict['optim']) + + # Make sure models are identical + for param_save, param_load in zip(model_save.parameters(), + model_load.parameters()): + torch.testing.assert_close(param_load, param_save) + + # Train both models + num_steps = 3 + micro_batch_steps = 3 + batch_size = 5 + for step in range(num_steps): + + # Reset gradients + optim_save.zero_grad() + optim_load.zero_grad() + + # Forward and backward passes + for micro_step in range(micro_batch_steps): + + # Synthetic data + x = torch.rand(batch_size, layer_size) - 0.5 + dy = torch.rand_like(x) - 0.5 + x = x.cuda() + dy = dy.cuda() + + # Forward and backward pass + x_save = x.detach().clone().requires_grad_(True) + y_save = model_save(x_save) + y_save.backward(dy) + x_load = x.detach().clone().requires_grad_(True) + y_load = model_load(x_load) + y_load.backward(dy) + + # Check that data tensors match + torch.testing.assert_close(y_load, y_save) + torch.testing.assert_close(x_load.grad, x_save.grad) + + # Optimizer step + optim_save.step() + optim_load.step() + + # Check that parameters match + for param_save, param_load in zip(model_save.parameters(), + model_load.parameters()): + torch.testing.assert_close(param_load, param_save) + +if __name__ == "__main__": + # Assume script has been run with torchrun + common_utils.run_tests() diff --git a/apex/apex/contrib/test/optimizers/test_distributed_fused_lamb.py b/apex/apex/contrib/test/optimizers/test_distributed_fused_lamb.py new file mode 100644 index 00000000..f38f371b --- /dev/null +++ b/apex/apex/contrib/test/optimizers/test_distributed_fused_lamb.py @@ -0,0 +1,125 @@ +import os +import inspect +import torch +from torch.cuda.amp import GradScaler +from torch.testing._internal import common_utils +from apex.parallel.distributed import flat_dist_call +from apex.contrib.optimizers.distributed_fused_lamb import DistributedFusedLAMB +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase + +def get_init_weights_func(): + @torch.no_grad() + def init_weights(m): + if isinstance(m, torch.nn.Linear): + m.weight.fill_(1.0) + return init_weights + +class ModelFoo(torch.nn.Module): + def __init__(self): + super(ModelFoo, self).__init__() + self.linear = torch.nn.Linear(128, 128, bias = False) + self.loss = torch.nn.MSELoss() + + def forward(self, input_tensor, gt): + y = self.linear(input_tensor) + loss = self.loss(y, gt) + return loss + +# A test for distributed fused Lamb optimizer: run several iterations and see if loss decreases +# There are two instances of the same test because based on `world_size` the optimizer decides what collectives operation to use. +# If torch.distributed.get_world_size() == torch.cuda.device_count() it uses only `all_gather`. +# If torch.distributed.get_world_size() < torch.cuda.device_count() it uses both `all_gather` and `reduce_scatter`. +class NcclDistributedFusedLAMB(NcclDistributedTestBase): + @property + def world_size(self) -> int: + return torch.cuda.device_count() + + @common_utils.parametrize("no_copy", [False, True]) + @common_utils.parametrize("opt_kwargs", [ + dict(overlap_reductions=True, dwu_num_blocks=2, dwu_num_chunks=2, + fused_norm=False, fuse_scale=False, clip_after_ar=True, + full_ar=False), + dict(overlap_reductions=False, dwu_num_blocks=1, dwu_num_chunks=1, + fused_norm=True, fuse_scale=True, clip_after_ar=False), + ]) + def test_distributed_fused_lamb(self, no_copy, opt_kwargs): + if no_copy and 'no_copy' not in inspect.getfullargspec(torch.distributed.reduce_scatter).args: + self.skipTest("does not support no_copy") + if no_copy and 'no_copy' not in inspect.getfullargspec(torch.distributed.all_gather).args: + self.skipTest("does not support no_copy") + + assert torch.distributed.is_initialized() + gpu_count = torch.distributed.get_world_size() + + init_scale = 100 + lr = torch.tensor(0.1).cuda() + grad_scaler = GradScaler(init_scale=init_scale, growth_interval=1000) + + model = ModelFoo() + model = model.cuda().half() + model.apply(get_init_weights_func()) + + param_optimizer = list(model.named_parameters()) + no_decay = ['bias', 'gamma', 'beta', 'LayerNorm'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, + {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + + if 'full_ar' not in opt_kwargs: + opt_kwargs['full_ar'] = gpu_count == torch.cuda.device_count() + + # Aidyn-A: not sure what parameters are the best for testing purposes, + # setting up whatever I think appropriate. + optimizer = DistributedFusedLAMB( + optimizer_grouped_parameters, + lr=0.1, + betas=(0.9, 0.9), + eps=1e-6, + max_grad_norm=1.0, + dwu_group_size=gpu_count, + dwu_num_rs_pg=1, + dwu_num_ar_pg=1, + dwu_num_ag_pg=1, + use_nvlamb=False, + set_param_views_to_flat_buffer=False, + e5m2_allgather=False, + **opt_kwargs + ) + optimizer.set_global_scale(init_scale) + + optimizer._reduce_scatter_no_copy = no_copy + optimizer._all_gather_no_copy = no_copy + + flat_dist_call([param.data for param in model.parameters()], torch.distributed.broadcast, (0,) ) + + x = torch.randn(4096, 128, dtype=torch.float16).cuda() + y = torch.randn(4096, 128, dtype=torch.float16).cuda() + + losses = [] + for _ in range(10): + loss = model(x, y) + optimizer._lazy_init_stage1() + grad_scaler.scale(loss).backward() + optimizer._lazy_init_stage2() + optimizer._lr = lr + optimizer.complete_reductions() + optimizer.set_global_scale(grad_scaler._get_scale_async()) + grad_scaler.step(optimizer) + grad_scaler.update() + optimizer.zero_grad(set_to_none=True) + + losses.append(loss.item()) + + self.assertTrue(losses == sorted(losses, reverse=True)) + +common_utils.instantiate_parametrized_tests(NcclDistributedFusedLAMB) + +class NcclDistributedFusedLAMB_partial_ar(NcclDistributedFusedLAMB): + @property + def world_size(self) -> int: + return max(torch.cuda.device_count()-1, 1) + +if __name__ == "__main__": + common_utils.run_tests() + diff --git a/apex/apex/contrib/test/peer_memory/__init__.py b/apex/apex/contrib/test/peer_memory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/peer_memory/test_peer_halo_exchange_module.py b/apex/apex/contrib/test/peer_memory/test_peer_halo_exchange_module.py new file mode 100644 index 00000000..1bc6264d --- /dev/null +++ b/apex/apex/contrib/test/peer_memory/test_peer_halo_exchange_module.py @@ -0,0 +1,328 @@ +import unittest + +import torch +from torch.testing._internal import common_utils + +SKIP_TEST = None +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +try: + from apex.contrib.peer_memory import PeerMemoryPool, PeerHaloExchanger1d +except ImportError as e: + SKIP_TEST = e + +# How to run: +# python /path/to/test_peer_halo_exchange_module.py + + +# Output of this function is used as ground truth in module tests. +def nccl_halo_ex(peer_rank, peer_group_size, y, half_halo, explicit_nhwc, H_split): + if explicit_nhwc: + if H_split: + _, Hp, _, _ = list(y.shape) + H = Hp - 2 * half_halo + top_out_halo = y[:, half_halo : 2 * half_halo, :, :] + top_inp_halo = y[:, :half_halo, :, :] + btm_out_halo = y[:, H : H + half_halo, :, :] + btm_inp_halo = y[:, H + half_halo : H + 2 * half_halo, :, :] + else: + _, _, Wp, _ = list(y.shape) + W = Wp - 2 * half_halo + top_out_halo = y[:, :, half_halo : 2 * half_halo, :] + top_inp_halo = y[:, :, :half_halo, :] + btm_out_halo = y[:, :, W : W + half_halo, :] + btm_inp_halo = y[:, :, W + half_halo : W + 2 * half_halo, :] + else: + if H_split: + _, _, Hp, _ = list(y.shape) + H = Hp - 2 * half_halo + top_out_halo = y[:, :, half_halo : 2 * half_halo, :] + top_inp_halo = y[:, :, :half_halo, :] + btm_out_halo = y[:, :, H : H + half_halo, :] + btm_inp_halo = y[:, :, H + half_halo : H + 2 * half_halo, :] + else: + _, _, _, Wp = list(y.shape) + W = Wp - 2 * half_halo + top_out_halo = y[:, :, :, half_halo : 2 * half_halo] + top_inp_halo = y[:, :, :, :half_halo] + btm_out_halo = y[:, :, :, W : W + half_halo] + btm_inp_halo = y[:, :, :, W + half_halo : W + 2 * half_halo] + + mf = torch.channels_last if y.is_contiguous(memory_format=torch.channels_last) else torch.contiguous_format + top_out_halo = top_out_halo.contiguous() + btm_out_halo = btm_out_halo.contiguous() + + top_inp_halos = [torch.empty_like(top_out_halo) for _ in range(peer_group_size)] + torch.distributed.all_gather(top_inp_halos, top_out_halo) + btm_inp_halos = [torch.empty_like(btm_out_halo) for _ in range(peer_group_size)] + torch.distributed.all_gather(btm_inp_halos, btm_out_halo) + top_rank = (peer_rank + peer_group_size - 1) % peer_group_size + btm_rank = (peer_rank + 1) % peer_group_size + if peer_rank == 0: + top_inp_halo.zero_() + else: + top_inp_halo.copy_(btm_inp_halos[top_rank].to(memory_format=mf)) + if peer_rank == peer_group_size - 1: + btm_inp_halo.zero_() + else: + btm_inp_halo.copy_(top_inp_halos[btm_rank].to(memory_format=mf)) + + +def single_test( + peer_rank, + peer_group_size, + halo_ex, + C, + H, + W, + half_halo, + dtype, + memory_format, + H_split, + num_steps, + numSM=1, +): + if memory_format == 1: + # 1 -> explicit nhwc + explicit_nhwc = True + if H_split: + y = torch.randn([1, H + 2 * half_halo, W, C], dtype=dtype, device="cuda") + ym = y[:, half_halo : H + half_halo, :, :] + else: + y = torch.randn([1, H, W + 2 * half_halo, C], dtype=dtype, device="cuda") + ym = y[:, :, half_halo : W + half_halo, :] + else: + # 2 -> native nhwc + # 3 -> nchw + explicit_nhwc = False + if H_split: + y = torch.randn([1, C, H + 2 * half_halo, W], dtype=dtype, device="cuda") + if memory_format == 2: + y = y.to(memory_format=torch.channels_last) + ym = y[:, :, half_halo : H + half_halo, :] + else: + y = torch.randn([1, C, H, W + 2 * half_halo], dtype=dtype, device="cuda") + if memory_format == 2: + y = y.to(memory_format=torch.channels_last) + ym = y[:, :, :, half_halo : W + half_halo] + y3 = y.clone() + list_y = [] + for step in range(num_steps): + halo_ex(y, H_split, explicit_nhwc, numSM) + list_y.append(y.clone()) + y.copy_(y3) + halo_ex.peer_pool.reset() + torch.distributed.barrier() + y2 = y3.clone() + list_y2 = [] + for step in range(num_steps): + nccl_halo_ex(peer_rank, peer_group_size, y2, half_halo, explicit_nhwc, H_split) + list_y2.append(y2.clone()) + y2.copy_(y3) + if memory_format == 1: + memory_format_str = "explicit_nhwc" + elif memory_format == 2: + memory_format_str = "native nhwc" + elif memory_format == 3: + memory_format_str = "nchw" + else: + memory_format_str = "???" + torch.testing.assert_close(list_y, list_y2, msg=memory_format_str) + # is_equal = [torch.all(torch.eq(yy, yy2)) for yy, yy2 in zip(list_y, list_y2)] + # is_equal = torch.tensor(is_equal, dtype=torch.bool) + # is_equal = torch.all(is_equal) + # if peer_rank == 0: + # if is_equal: + # print( + # "SUCCESS : N,C,H,W = 1,%d,%d,%d, half_halo=%d, %s, %s, %s" + # % ( + # C, + # H, + # W, + # half_halo, + # str(dtype), + # memory_format_str, + # "H-split" if H_split else "W-split", + # ) + # ) + # else: + # print( + # "FAILURE : N,C,H,W = 1,%d,%d,%d, half_halo=%d, %s, %s, %s" + # % ( + # C, + # H, + # W, + # half_halo, + # str(dtype), + # memory_format_str, + # "H-split" if H_split else "W-split", + # ) + # ) + # + # peer memory flag sync relies on there being at least one barrier per step + # torch.distributed.barrier() + + +def H_split_tests(N, C, H, W, half_halo, rank, world_size, halo_ex, num_steps): + Hr = 8 * world_size + Hp = ((H + Hr - 1) // Hr) * 8 + + for i in range(4): + div = int(pow(2, i)) + single_test( + rank, + world_size, + halo_ex, + C * div, + Hp // div, + W // div, + half_halo, + torch.float16, + 1, + True, + num_steps, + ) + single_test( + rank, + world_size, + halo_ex, + C * div, + Hp // div, + W // div, + half_halo, + torch.float16, + 2, + True, + num_steps, + ) + single_test( + rank, + world_size, + halo_ex, + C * div, + Hp // div, + W // div, + half_halo, + torch.float16, + 3, + True, + num_steps, + ) + + +def W_split_tests(N, C, H, W, half_halo, rank, world_size, halo_ex, num_steps): + Wr = 8 * world_size + Wp = ((W + Wr - 1) // Wr) * 8 + + for i in range(4): + div = int(pow(2, i)) + single_test( + rank, + world_size, + halo_ex, + C * div, + H // div, + Wp // div, + half_halo, + torch.float16, + 1, + False, + num_steps, + ) + single_test( + rank, + world_size, + halo_ex, + C * div, + H // div, + Wp // div, + half_halo, + torch.float16, + 2, + False, + num_steps, + ) + single_test( + rank, + world_size, + halo_ex, + C * div, + H // div, + Wp // div, + half_halo, + torch.float16, + 3, + False, + num_steps, + ) + + +def main(): + # for this trivial example peer_rank == rank and peer_group_size == world_size + + torch.distributed.init_process_group("nccl") + rank = torch.distributed.get_rank() + world_size = torch.distributed.get_world_size() + torch.cuda.set_device(rank) + peer_ranks = [i for i in range(world_size)] + pool = PeerMemoryPool(0, 2 * 1024 * 1024, peer_ranks) + + num_steps = 100 + + half_halo = 1 + halo_ex = PeerHaloExchanger1d(peer_ranks, rank, pool, half_halo) + + H_split_tests(1, 64, 336, 200, half_halo, rank, world_size, halo_ex, num_steps) + W_split_tests(1, 64, 200, 336, half_halo, rank, world_size, halo_ex, num_steps) + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TestPeerMemory(NcclDistributedTestBase): + HALF_HALO = 1 + NUM_STEPS = 100 + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 2) + + # TODO(crcrpar): Check if `world_size` being multiple of 2 is must. + def _check_world_size_and_may_skip(self) -> None: + if not (self.world_size >= 2 and self.world_size % 2 == 0): + self.skipTest(f"world_size is expected to be a multiple of 2 but, {self.world_size}") + + def get_halo_excnahger_1d(self): + peer_ranks = [i for i in range(self.world_size)] + pool = PeerMemoryPool(64 * 1024, 2 * 1024 * 1024, peer_ranks) + halo_exchanger_1d = PeerHaloExchanger1d(peer_ranks, self.rank, pool, TestPeerMemory.HALF_HALO) + return halo_exchanger_1d + + def test_height_split(self): + self._check_world_size_and_may_skip() + H_split_tests( + 1, + 64, + 336, + 200, + TestPeerMemory.HALF_HALO, + self.rank, + self.world_size, + self.get_halo_excnahger_1d(), + TestPeerMemory.NUM_STEPS, + ) + + def test_width_split(self): + self._check_world_size_and_may_skip() + W_split_tests( + 1, + 64, + 200, + 336, + TestPeerMemory.HALF_HALO, + self.rank, + self.world_size, + self.get_halo_excnahger_1d(), + TestPeerMemory.NUM_STEPS, + ) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/apex/contrib/test/transducer/__init__.py b/apex/apex/contrib/test/transducer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/transducer/test_transducer_joint.py b/apex/apex/contrib/test/transducer/test_transducer_joint.py new file mode 100755 index 00000000..a2046488 --- /dev/null +++ b/apex/apex/contrib/test/transducer/test_transducer_joint.py @@ -0,0 +1,167 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.transducer import TransducerJoint + from apex.contrib.transducer import _transducer_ref as transducer_ref +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TransducerJointTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + + def gen_input(self, for_vector_kernel): + self.B = 4 + T_min = 51 + T_max = 101 + U_min = 12 + U_max = 25 + if for_vector_kernel: + H = 512 + else: + H = 509 + dtype = torch.float16 + device = "cuda" + + self.f_tst = torch.randn((self.B, T_max, H), dtype=dtype, requires_grad=True, device=device) + self.g_tst = torch.randn((self.B, U_max, H), dtype=dtype, requires_grad=True, device=device) + self.h_grad = torch.randn(self.B, T_max, U_max, H, dtype=dtype, device=device) + self.f_len = torch.randint(T_min, T_max+1, (self.B,), dtype=torch.int, device=device) + self.g_len = torch.randint(U_min, U_max+1, (self.B,), dtype=torch.int, device=device) + self.f_len[torch.randint(0, self.B, (1,)).item()] = T_max + self.g_len[torch.randint(0, self.B, (1,)).item()] = U_max + self.dropout_prob = 0.5 + + # Make sure gradients from out-of-bound locations are zero. This should be guaranteed by + # the loss function + for b in range(self.B): + self.h_grad[b, self.f_len[b]:, :, :] = 0 + self.h_grad[b, :, self.g_len[b]:, :] = 0 + self.h_grad_packed = self._pack(self.h_grad, self.f_len, self.g_len) + + + def _pack(self, x, f_len, g_len): + B = x.size(0) + list_x = [] + for b in range(B): + list_x_row = [x[b, t, :g_len[b]] for t in range(f_len[b])] + x_row = torch.cat(list_x_row) + list_x.append(x_row) + x_packed = torch.cat(list_x).data.clone() + x_packed.requires_grad = True + batch_offset = torch.cumsum(f_len * g_len, dim=0) + return x_packed + + def _unpack(self, x, f_len, g_len): + batch_offset = torch.cumsum(f_len * g_len, dim=0) + x_unpacked = torch.zeros_like(self.h_grad, dtype=torch.uint8) + B = self.h_grad.size(0) + H = self.h_grad.size(-1) + for b in range(B): + my_batch_offset = 0 if b == 0 else batch_offset[b-1] + my_f_len = f_len[b] + my_g_len = g_len[b] + for t in range(my_f_len): + x_unpacked[b, t, :my_g_len] = x[my_batch_offset + t*my_g_len : + my_batch_offset + t*my_g_len + my_g_len] + return x_unpacked + + def run_transducer_joint(self, for_vector_kernel, pack_output, relu, dropout): + self.gen_input(for_vector_kernel=for_vector_kernel) + # Generate reference + f_ref = self.f_tst.data.clone() + g_ref = self.g_tst.data.clone() + f_ref.requires_grad = True + g_ref.requires_grad = True + + my_joint = TransducerJoint(pack_output=pack_output, relu=relu, dropout=dropout, + dropout_prob=self.dropout_prob, probe_mask=True) + if not pack_output: + h_tst = my_joint( f=self.f_tst, + g=self.g_tst, + f_len=self.f_len, + g_len=self.g_len) + h_tst.backward(self.h_grad) + if dropout: + mask = my_joint.mask_probe[0] + else: + batch_offset = torch.cumsum(self.f_len * self.g_len, dim=0) + h_tst = my_joint( f=self.f_tst, + g=self.g_tst, + f_len=self.f_len, + g_len=self.g_len, + batch_offset=batch_offset, + packed_batch=batch_offset[-1]) + h_tst.backward(self.h_grad_packed) + if dropout: + mask_packed = my_joint.mask_probe[0] + mask = self._unpack(mask_packed, self.f_len, self.g_len) + + # reference + h_ref, f_grad_ref, g_grad_ref \ + = transducer_ref.transducer_joint_reference(f=f_ref, + g=g_ref, + h_grad=self.h_grad, + f_len=self.f_len, + g_len=self.g_len, + pack_output=pack_output, + relu=relu, + dropout=dropout, + dropout_prob=self.dropout_prob, + mask=mask if dropout else None) + + f_grad_tst = self.f_tst.grad + g_grad_tst = self.g_tst.grad + + self.assertTrue(torch.allclose(h_ref, h_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(f_grad_ref, f_grad_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(g_grad_ref, g_grad_tst, atol=1e-4, rtol=1e-4)) + + def test_transducer_joint(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=False, dropout=False) + + def test_transducer_joint_vec(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=False, relu=False, dropout=False) + + def test_transducer_joint_pack(self): + self.run_transducer_joint(for_vector_kernel=False, pack_output=True, relu=False, dropout=False) + + def test_transducer_joint_vec_pack(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=False, dropout=False) + + def test_transducer_joint_relu(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=False) + + def test_transducer_joint_vec_relu(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=False, relu=True, dropout=False) + + def test_transducer_joint_pack_relu(self): + self.run_transducer_joint(for_vector_kernel=False, pack_output=True, relu=True, dropout=False) + + def test_transducer_joint_vec_pack_relu(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=False) + + @unittest.expectedFailure + def test_transducer_joint_relu_dropout(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=True) + + @unittest.expectedFailure + def test_transducer_joint_vec_relu_dropout(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=False, relu=True, dropout=True) + + @unittest.expectedFailure + def test_transducer_joint_pack_relu_dropout(self): + self.run_transducer_joint(for_vector_kernel=False, pack_output=True, relu=True, dropout=True) + + @unittest.expectedFailure + def test_transducer_joint_vec_pack_relu_dropout(self): + self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/apex/contrib/test/transducer/test_transducer_loss.py b/apex/apex/contrib/test/transducer/test_transducer_loss.py new file mode 100755 index 00000000..0e911732 --- /dev/null +++ b/apex/apex/contrib/test/transducer/test_transducer_loss.py @@ -0,0 +1,139 @@ +import unittest + +import torch + +SKIP_TEST = None +try: + from apex.contrib.transducer import TransducerLoss + from apex.contrib.transducer import _transducer_ref as transducer_ref +except ImportError as e: + SKIP_TEST = e + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class TransducerLossTest(unittest.TestCase): + def setUp(self, seed=1234): + torch.manual_seed(seed) + + def gen_input(self, scalar_t, for_vector_kernel): + self.B = 5 + T_min = 23 + T_max = 51 + U_min = 12 + U_max = 25 + V = 16 if for_vector_kernel else 14 + self.blank_idx = V - 1 + device = "cuda" + + self.x_tst = torch.randn((self.B, T_max, U_max, V), dtype=scalar_t, requires_grad=True, + device=device) + self.y = torch.randint(0, self.blank_idx, (self.B, U_max-1), dtype=torch.int, device=device) + self.f_len = torch.randint(T_min, T_max+1, (self.B,), dtype=torch.int, device=device) + self.y_len = torch.randint(U_min-1, U_max, (self.B,), dtype=torch.int, device=device) + self.f_len[torch.randint(0, self.B, (1,)).item()] = T_max + self.y_len[torch.randint(0, self.B, (1,)).item()] = U_max-1 + self.x_tst_packed, self.batch_offset = self._pack(self.x_tst) + # Generate reference + x_ref = self.x_tst.data.clone() + x_ref.requires_grad = True + loss_grad = torch.ones(x_ref.size(0), dtype=x_ref.dtype, device=x_ref.device)/x_ref.size(0) + _, _, self.grad_ref, self.loss_ref \ + = transducer_ref.transducer_loss_reference( x=x_ref, + label=self.y, + f_len=self.f_len, + y_len=self.y_len, + blank_idx=self.blank_idx, + loss_grad=loss_grad) + + def _pack(self, x): + list_x = [] + for b in range(self.B): + list_x_row = [x[b, t, : self.y_len[b]+1] for t in range(self.f_len[b])] + x_row = torch.cat(list_x_row) + list_x.append(x_row) + x_packed = torch.cat(list_x).data.clone() + x_packed.requires_grad = True + batch_offset = torch.cumsum(self.f_len * (self.y_len+1), dim=0) + return x_packed, batch_offset + + def _unpack(self, x): + x_unpacked = torch.zeros(self.B, self.f_len.max(), self.y_len.max()+1, x.size(-1), + dtype=x.dtype, device=x.device) + for b in range(self.B): + my_batch_offset = 0 if b == 0 else self.batch_offset[b-1] + my_f_len = self.f_len[b] + my_g_len = self.y_len[b] + 1 + for t in range(my_f_len): + for u in range(my_g_len): + x_unpacked[b, t, u] = x[my_batch_offset + t*my_g_len + u] + return x_unpacked + + def run_transducer_loss(self, scalar_t, fuse_softmax_backward, packed_input, for_vector_kernel): + self.gen_input(scalar_t, for_vector_kernel) + my_loss = TransducerLoss( fuse_softmax_backward=fuse_softmax_backward, + packed_input=packed_input) + if not packed_input: + loss_tst = my_loss( x=self.x_tst, + label=self.y, + f_len=self.f_len, + y_len=self.y_len, + blank_idx=self.blank_idx) + loss_tst.mean().backward() + grad_tst = self.x_tst.grad + else: + loss_tst = my_loss( x=self.x_tst_packed, + label=self.y, + f_len=self.f_len, + y_len=self.y_len, + blank_idx=self.blank_idx, + batch_offset=self.batch_offset, + max_f_len=max(self.f_len)) + loss_tst.mean().backward() + grad_tst_packed = self.x_tst_packed.grad + grad_tst = self._unpack(grad_tst_packed) + + return loss_tst, grad_tst + + def test_transducer_loss_fp32(self): + loss_tst, grad_tst = self.run_transducer_loss( scalar_t=torch.float32, + fuse_softmax_backward=False, + packed_input=False, + for_vector_kernel=False) + self.assertTrue(torch.allclose(self.loss_ref, loss_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(self.grad_ref, grad_tst, atol=1e-5, rtol=1e-5)) + + def test_transducer_loss_fp16(self): + loss_tst, grad_tst = self.run_transducer_loss( scalar_t=torch.float16, + fuse_softmax_backward=False, + packed_input=False, + for_vector_kernel=False) + self.assertTrue(torch.allclose(self.loss_ref, loss_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(self.grad_ref, grad_tst, atol=1e-4, rtol=1e-3)) + + def test_transducer_loss_fp16_backward_fusion(self): + loss_tst, grad_tst = self.run_transducer_loss( scalar_t=torch.float16, + fuse_softmax_backward=True, + packed_input=False, + for_vector_kernel=False) + self.assertTrue(torch.allclose(self.loss_ref, loss_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(self.grad_ref, grad_tst, atol=1e-4, rtol=1e-3)) + + def test_transducer_loss_fp16_backward_fusion_packed(self): + loss_tst, grad_tst = self.run_transducer_loss( scalar_t=torch.float16, + fuse_softmax_backward=True, + packed_input=True, + for_vector_kernel=False) + self.assertTrue(torch.allclose(self.loss_ref, loss_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(self.grad_ref, grad_tst, atol=1e-4, rtol=1e-3)) + + def test_transducer_loss_fp16_backward_fusion_packed_vec(self): + loss_tst, grad_tst = self.run_transducer_loss( scalar_t=torch.float16, + fuse_softmax_backward=True, + packed_input=True, + for_vector_kernel=True) + self.assertTrue(torch.allclose(self.loss_ref, loss_tst, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(self.grad_ref, grad_tst, atol=1e-4, rtol=1e-3)) + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/apex/contrib/test/xentropy/__init__.py b/apex/apex/contrib/test/xentropy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/contrib/test/xentropy/test_label_smoothing.py b/apex/apex/contrib/test/xentropy/test_label_smoothing.py new file mode 100644 index 00000000..e64f2981 --- /dev/null +++ b/apex/apex/contrib/test/xentropy/test_label_smoothing.py @@ -0,0 +1,136 @@ +import unittest +import random +import time + +import numpy as np + +import torch + +SKIP_TEST = None +try: + from apex.contrib import xentropy as label_smoothing +except ImportError as e: + SKIP_TEST = e + + +def label_smoothing_raw(x, target, padding_idx, smoothing): + logprobs = torch.nn.functional.log_softmax(x, dim=-1, dtype=torch.float32) + + non_pad_mask = (target != padding_idx) + nll_loss = -logprobs.gather(dim=-1, index=target.unsqueeze(1)) + nll_loss = nll_loss.squeeze(1)[non_pad_mask] + smooth_loss = -logprobs.mean(dim=-1)[non_pad_mask] + loss = (1.0 - smoothing) * nll_loss + smoothing * smooth_loss + return loss + +def label_smoothing_opt_1(x, target, padding_idx, smoothing): + logprobs = torch.nn.functional.log_softmax(x, dim=-1, dtype=torch.float32) + + pad_mask = (target == padding_idx) + ll_loss = logprobs.gather(dim=-1, index=target.unsqueeze(1)).squeeze(1) + smooth_loss = logprobs.mean(dim=-1) + loss = (smoothing - 1.0) * ll_loss - smoothing * smooth_loss + loss.masked_fill_(pad_mask, 0) + return loss + + +@unittest.skipIf(SKIP_TEST, f"{SKIP_TEST}") +class LabelSmoothingTest(unittest.TestCase): + def setUp(self, seed=1234): + super().setUp() + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + # Set pytorch print precision + torch.set_printoptions(precision=10) + + def gen_test_inputs(self, N, T, H, smoothing, padding_idx): + logits = torch.randn((N*T, H), dtype=torch.half, device='cuda', + requires_grad=True) + labels = torch.randint(0, H, [N*T], device='cuda') + for i in random.sample(range(N*T), N*T//6): + labels[i] = padding_idx + half_to_float = (logits.dtype == torch.half) + + return logits, labels, half_to_float + + def print_max_diff_elem(self, ref, tst): + ref, tst = ref.flatten(), tst.flatten() + diff = (ref - tst).abs().max() + idx = (ref - tst).abs().argmax() + print("Max atol idx: {}, diff: {:.6f}, ref: {:.6f}, tst: {:.6f}".format( + idx, diff, ref[idx], tst[idx])) + + def test_label_smoothing_function(self): + # Set label smoothing configuration + smoothing, padding_idx = 0.1, 0 + N, T, H = 128, 74, 32320 + iters = 10 + loss_func = label_smoothing.SoftmaxCrossEntropyLoss.apply + + for i in range(iters): + logits, labels, half_to_float = self.gen_test_inputs( + N, T, H, smoothing, padding_idx) + + # Run original softmax cross entropy with label smoothing + logits.grad = None + losses = label_smoothing_raw(logits, labels, padding_idx, smoothing) + loss = losses.sum() + loss.backward() + + ref_loss = loss.clone().detach() + ref_grad = logits.grad.clone().detach() + + # Run optimized softmax cross entropy with label smoothing + logits.grad = None + losses = loss_func(logits, labels, smoothing, padding_idx, half_to_float) + loss = losses.sum() + loss.backward() + + val_loss = loss.clone().detach() + val_grad = logits.grad.clone().detach() + + # Validate + self.print_max_diff_elem(ref_grad, val_grad) + self.assertTrue(torch.allclose(ref_loss, val_loss, atol=1e-5, rtol=1e-5)) + self.assertTrue(torch.allclose(ref_grad, val_grad, atol=1e-5, rtol=1e-5)) + + def test_label_smoothing_perf(self): + # Set label smoothing configuration + smoothing, padding_idx = 0.1, 0 + N, T, H = 128, 74, 32320 + iters = 1000 + loss_func = label_smoothing.SoftmaxCrossEntropyLoss.apply + print() + + logits, labels, half_to_float = self.gen_test_inputs( + N, T, H, smoothing, padding_idx) + + # Run original softmax cross entropy with label smoothing + torch.cuda.synchronize() + ts = time.time() + for i in range(iters): + logits.grad = None + losses = label_smoothing_raw(logits, labels, padding_idx, smoothing) + loss = losses.sum() / N + loss.backward() + torch.cuda.synchronize() + print("Raw time {:.2f} s elapsed for {} iterations, norm {:.4f}".format( + time.time() - ts, iters, logits.grad.norm())) + + # Run optimized softmax cross entropy with label smoothing + torch.cuda.synchronize() + ts = time.time() + for i in range(iters): + logits.grad = None + losses = loss_func(logits, labels, smoothing, padding_idx, half_to_float) + loss = losses.sum() / N + loss.backward() + torch.cuda.synchronize() + print("Opt time {:.2f} s elapsed for {} iterations, norm {:.4f}".format( + time.time() - ts, iters, logits.grad.norm())) + +if __name__ == '__main__': + unittest.main() + diff --git a/apex/apex/contrib/transducer/__init__.py b/apex/apex/contrib/transducer/__init__.py new file mode 100755 index 00000000..955ca180 --- /dev/null +++ b/apex/apex/contrib/transducer/__init__.py @@ -0,0 +1,3 @@ +from .transducer import TransducerJoint +from .transducer import TransducerLoss +from . import _transducer_ref diff --git a/apex/apex/contrib/transducer/_transducer_ref.py b/apex/apex/contrib/transducer/_transducer_ref.py new file mode 100755 index 00000000..baa71458 --- /dev/null +++ b/apex/apex/contrib/transducer/_transducer_ref.py @@ -0,0 +1,109 @@ +import torch + + +def transducer_loss_reference(x, label, f_len, y_len, blank_idx, loss_grad): + def log_sum_exp(a, b): + if (a >= b): + return a + torch.log(1 + torch.exp(b-a)) + else: + return b + torch.log(1 + torch.exp(a-b)) + + def forward_alpha(x, label, f_len, y_len, blank_idx): + B, T, U, V = x.size() + acc_t = torch.float32 if x.dtype in [torch.float16, torch.float32] else x.dtype + alpha = torch.zeros((B, T, U), dtype=acc_t, device=x.device) + for b in range(B): + alpha[b, 0, 0] = 0 + for t in range(1, f_len[b]): + alpha[b, t, 0] = alpha[b, t-1, 0] + x[b, t-1, 0, blank_idx] + for u in range(1, y_len[b]+1): + alpha[b, 0, u] = alpha[b, 0, u-1] + x[b, 0, u-1, label[b, u-1]] + for t in range(1, f_len[b]): + for u in range(1, y_len[b]+1): + curr_ = alpha[b, t-1, u] + x[b, t-1, u, blank_idx] + next_ = alpha[b, t, u-1] + x[b, t, u-1, label[b, u-1]] + alpha[b, t, u] = log_sum_exp(curr_, next_) + return alpha + + def forward_beta(x, label, f_len, y_len, blank_idx): + B, T, U, V = x.shape + acc_t = torch.float32 if x.dtype in [torch.float16, torch.float32] else x.dtype + beta = torch.zeros((B, T, U), dtype=acc_t, device=x.device) + for b in range(B): + beta[b, f_len[b]-1, y_len[b]] = x[b, f_len[b]-1, y_len[b], blank_idx] + for t in range(f_len[b]-2, -1, -1): + beta[b, t, y_len[b]] = beta[b, t+1, y_len[b]] + x[b, t, y_len[b], blank_idx] + for u in range(y_len[b]-1, -1, -1): + beta[b, f_len[b]-1, u] = beta[b, f_len[b]-1, u+1] + x[b, f_len[b]-1, u, label[b, u]] + for t in range(f_len[b]-2, -1, -1): + for u in range(y_len[b]-1, -1, -1): + curr_ = beta[b, t+1, u] + x[b, t, u, blank_idx] + next_ = beta[b, t, u+1] + x[b, t, u, label[b, u]] + beta[b, t, u] = log_sum_exp(curr_, next_) + return beta + + def backward(x, label, f_len, y_len, alpha, beta, loss_grad, blank_idx): + grad = torch.zeros_like(x) + B, T, U, V = x.size() + for b in range(B): + common_factor = torch.log(loss_grad[b]) + alpha - beta[b, 0, 0] + # next + for u in range(y_len[b]): + grad[b, :f_len[b], u, label[b, u]] = -torch.exp(common_factor[b, :f_len[b], u] + + beta[b, :f_len[b], u+1] + + x[b, :f_len[b], u, label[b, u]]) + + # current + grad[b, :f_len[b]-1, :y_len[b]+1, blank_idx] \ + = -torch.exp(common_factor[b, :f_len[b]-1, :y_len[b]+1] + + beta[b, 1:f_len[b], :y_len[b]+1] + + x[b, :f_len[b]-1, :y_len[b]+1, blank_idx]) + + grad[b, f_len[b]-1, y_len[b], blank_idx] = -torch.exp(common_factor[b, f_len[b]-1, y_len[b]] + + x[b, f_len[b]-1, y_len[b], blank_idx]) + + return grad + + x_log = torch.nn.functional.log_softmax(x, dim=-1) + alpha = forward_alpha(x_log, label, f_len, y_len, blank_idx) + beta = forward_beta(x_log, label, f_len, y_len, blank_idx) + grad = backward(x_log, label, f_len, y_len, alpha, beta, + loss_grad, blank_idx) + x_log.backward(grad) + loss = -beta[:, 0, 0] + loss = loss.to(x.dtype) + return alpha, beta, x.grad, loss + + +def transducer_joint_reference(f, g, h_grad, f_len, g_len, pack_output, relu, dropout, + dropout_prob=0, mask=None): + if dropout and mask == None: + raise NotImplementedError("mask needs to supplied to test dropout.") + B, T, H = f.size() + U = g.size(1) + f_expand = f.unsqueeze(dim=2) + g_expand = g.unsqueeze(dim=1) + h = f_expand + g_expand + if relu: + h = torch.nn.functional.relu(h) + if dropout: + h *= mask + scale = 1/(1-dropout_prob) + h *= scale + h.backward(h_grad) + + if pack_output == False: + # intentionally set don't-care region to -1 to test if transducer joint + # write these regions to avoid NaN and inf + for b in range(B): + h[b, f_len[b]:] = -1 + h[b, :, g_len[b]:] = -1 + + return h, f.grad, g.grad + + # packing + list_to_pack = [] + for b in range(B): + list_to_pack.append(h[b, :f_len[b], :g_len[b], :].reshape(-1, H)) + h_packed = torch.cat(list_to_pack) + return h_packed, f.grad, g.grad diff --git a/apex/apex/contrib/transducer/transducer.py b/apex/apex/contrib/transducer/transducer.py new file mode 100755 index 00000000..78439627 --- /dev/null +++ b/apex/apex/contrib/transducer/transducer.py @@ -0,0 +1,195 @@ +import torch +import transducer_loss_cuda +import transducer_joint_cuda + +class TransducerJoint(torch.nn.Module): + """Transducer joint + Detail of this loss function can be found in: Sequence Transduction with Recurrent Neural + Networks + + Arguments: + pack_output (bool, optional): whether to pack the output in a compact form with don't-care + data being removed. (default: False) + relu (bool, optional): apply ReLU to the output of the joint operation. Requires opt=1 + (default: False) + dropout (bool, optional): apply dropout to the output of the joint operation. Requires opt=1 + (default: False) + opt (int, optional): pick the optimization level in [0, 1]. opt=1 picks a tiled algorithm. + (default: 1) + fwd_tile_size (int, optional): tile size used in forward operation. This argument will be + ignored if opt != 1. (default: 4) + dropout_prob (float, optional): dropout probability. (default: 0.0) + probe_mask (bool, optional): a flag used to probe the mask generated by ReLU and/or dropout + operation. When this argument is set to True, the mask can be accessed through + self.mask_probe. (default: false) + """ + + def __init__(self, pack_output=False, relu=False, dropout=False, opt=1, fwd_tile_size=4, + dropout_prob=0, probe_mask=False): + super(TransducerJoint, self).__init__() + self.pack_output = pack_output + self.relu = relu + self.dropout = dropout + self.dropout_prob = dropout_prob + self.opt = opt + self.fwd_tile_size = fwd_tile_size + self.dummy_batch_offset = torch.empty(0) + masked = self.relu or self.dropout + self.mask_probe = [] if masked and probe_mask else None + if masked and opt != 1: + raise NotImplementedError("ReLU and dropout fusion is only supported with opt=1") + + + def forward(self, f, g, f_len, g_len, batch_offset=None, packed_batch=0): + """Forward operation of transducer joint + + Arguments: + f (tensor): transcription vector from encode block of shape (B, T, H). + g (tensor): prediction vector form predict block of shape (B, U, H). + f_len (tensor): length of transcription vector for each batch. + g_len (tensor): length of prediction vector minus 1 for each batch. + batch_offset (tensor, optional): tensor containing the offset of each batch + in the results. For example, batch offset can be obtained from: + batch_offset = torch.cumsum(f_len*g_len, dim=0) + This argument is required if pack_output == True, and is ignored if + pack_output == False. (default: None) + packed_batch (int, optional): the batch size after packing. This argument is + ignored if pack_output == False. (default: 0) + """ + my_batch_offset = batch_offset if self.pack_output else self.dummy_batch_offset + if self.pack_output and (batch_offset is None or packed_batch == 0): + raise Exception("Please specify batch_offset and packed_batch when packing is enabled") + dropout = self.dropout and self.training # only dropout for training + return TransducerJointFunc.apply(f, g, f_len, g_len, self.pack_output, self.relu, dropout, + my_batch_offset, packed_batch, self.opt, + self.fwd_tile_size, self.dropout_prob, self.mask_probe) + + +class TransducerLoss(torch.nn.Module): + """Transducer loss + Detail of this loss function can be found in: Sequence Transduction with Recurrent Neural + Networks + + Arguments: + fuse_softmax_backward (bool, optional) whether to fuse the backward of transducer loss with + softmax. (default: True) + opt (int, optional): pick the optimization level in [0, 1]. opt=1 picks a more optimized + algorithm. In some cases, opt=1 might fall back to opt=0. (default: 1) + packed_input (bool, optional): whether to pack the output in a compact form with don't-care + data being removed. (default: False) + """ + def __init__(self, fuse_softmax_backward=True, opt=1, packed_input=False): + super(TransducerLoss, self).__init__() + self.fuse_softmax_backward = fuse_softmax_backward + self.opt = opt + self.packed_input = packed_input + self.dummy_batch_offset = torch.empty(0) + + + def forward(self, x, label, f_len, y_len, blank_idx, batch_offset=None, max_f_len=None, + debug_list=None): + """Forward operation of transducer joint + + Arguments: + x (tensor): input tensor to the loss function with a shape of (B, T, U, H). + label (tensor): labels for the input data. + f_len (tensor): lengths of the inputs in the time dimension for each batch. + y_len (tensor): lengths of the labels for each batch. + blank_idx (int): index for the null symbol. + batch_offset (tensor, optional): tensor containing the offset of each batch + in the input. For example, batch offset can be obtained from: + batch_offset = torch.cumsum(f_len*(y_len+1), dim=0) + This argument is required if packed_input == True, and is ignored if + packed_input == False. (default: None) + max_f_len (int, optional): maximum length of the input in the time dimension. + For example, it can be obtained as + max_f_len = max(f_len) + This argument is required if packed_input == True, and is ignored if + packed_input == False. (default: None) + (default: None) + debug_list (list, optional): when an empty list is supplied, Alpha and Beta generated + in the forward operation will be attached to this list for debug purpose. + (default: None) + """ + if self.packed_input: + if batch_offset is None or max_f_len is None: + raise Exception("Please specify batch_offset and max_f_len when packing is \ + enabled") + my_batch_offset = batch_offset + my_max_f_len = max_f_len + else: + my_batch_offset = self.dummy_batch_offset + my_max_f_len = x.size(1) + return TransducerLossFunc.apply(x, label, f_len, y_len, my_batch_offset, my_max_f_len, + blank_idx, self.fuse_softmax_backward, debug_list, + self.opt, self.packed_input) + +class TransducerLossFunc(torch.autograd.Function): + @staticmethod + def forward(ctx, x, label, f_len, y_len, batch_offset, max_f_len, blank_idx, + fuse_softmax_backward, debug_list, opt, packed_input): + if fuse_softmax_backward == False: + with torch.enable_grad(): + x = torch.nn.functional.log_softmax(x, dim=-1) + else: + x = torch.nn.functional.log_softmax(x, dim=-1) + alpha, beta, loss = transducer_loss_cuda.forward( x, label, f_len, y_len, batch_offset, + max_f_len, blank_idx, opt, packed_input) + if debug_list == []: + debug_list += [alpha, beta] + ctx.save_for_backward(x, alpha, beta, f_len, y_len, label, batch_offset) + ctx.blank_idx = blank_idx + ctx.fuse_softmax_backward = fuse_softmax_backward + ctx.opt = opt + ctx.packed_input = packed_input + ctx.max_f_len = max_f_len + return loss + + @staticmethod + def backward(ctx, loss_grad): + x, alpha, beta, f_len, y_len, label, batch_offset = ctx.saved_tensors + x_grad = transducer_loss_cuda.backward( x, loss_grad, alpha, beta, f_len, y_len, label, + batch_offset, ctx.max_f_len, ctx.blank_idx, ctx.opt, + ctx.fuse_softmax_backward, ctx.packed_input) + if ctx.fuse_softmax_backward == False: + x_grad = x.backward(x_grad) + return x_grad, None, None, None, None, None, None, None, None, None, None + +class TransducerJointFunc(torch.autograd.Function): + @staticmethod + def forward(ctx, f, g, f_len, g_len, pack_output, relu, dropout, batch_offset, packed_batch, + opt, fwd_tile_size, dropout_prob, mask_probe): + h = transducer_joint_cuda.forward(f, g, f_len, g_len, batch_offset, packed_batch, opt, + pack_output, relu, dropout, dropout_prob, fwd_tile_size) + masked = relu or dropout + if masked: + ctx.save_for_backward(h[1], f_len, g_len, batch_offset) + if mask_probe is not None: + mask_probe.append(h[1]) + else: + ctx.save_for_backward(f_len, g_len, batch_offset) + + ctx.pack_output = pack_output + ctx.masked = relu or dropout + ctx.max_f_len = f.size(1) + ctx.max_g_len = g.size(1) + ctx.scale = 1 / (1-dropout_prob) if dropout and dropout_prob != 1 else 1 + return h[0] + + @staticmethod + def backward(ctx, loss_grad): + if ctx.masked: + mask, f_len, g_len, batch_offset = ctx.saved_tensors + inp = [loss_grad, mask] + else: + f_len, g_len, batch_offset = ctx.saved_tensors + inp = [loss_grad] + + f_grad, g_grad = transducer_joint_cuda.backward( inp, f_len, g_len, batch_offset, + ctx.max_f_len, ctx.max_g_len, + ctx.pack_output, ctx.scale) + + return f_grad, g_grad, None, None, None, None, None, None, None, None, None, None, None, \ + None, None, None + + diff --git a/apex/apex/contrib/xentropy/__init__.py b/apex/apex/contrib/xentropy/__init__.py new file mode 100644 index 00000000..4c8cbeee --- /dev/null +++ b/apex/apex/contrib/xentropy/__init__.py @@ -0,0 +1,6 @@ +from .softmax_xentropy import SoftmaxCrossEntropyLoss + + +__all__ = [ + "SoftmaxCrossEntropyLoss", +] diff --git a/apex/apex/contrib/xentropy/softmax_xentropy.py b/apex/apex/contrib/xentropy/softmax_xentropy.py new file mode 100644 index 00000000..4abc5b47 --- /dev/null +++ b/apex/apex/contrib/xentropy/softmax_xentropy.py @@ -0,0 +1,30 @@ +import torch + +import xentropy_cuda + + +class SoftmaxCrossEntropyLoss(torch.autograd.Function): + @staticmethod + def forward(ctx, logits, labels, smoothing=0.0, padding_idx=0, half_to_float=False): + losses, max_log_sum_exp = xentropy_cuda.forward( + logits, labels, smoothing, half_to_float) + losses.masked_fill_(labels==padding_idx, 0) + + ctx.save_for_backward(logits, max_log_sum_exp, labels, + torch.FloatTensor([smoothing]), + torch.LongTensor([padding_idx])) + + return losses + + @staticmethod + def backward(ctx, grad_loss): + logits, max_log_sum_exp, labels, smoothing, padding_idx = ctx.saved_tensors + + if not grad_loss.is_contiguous(): + grad_loss = grad_loss.contiguous() + grad_loss.masked_fill_(labels==padding_idx.item(), 0) + grad_logits = xentropy_cuda.backward( + grad_loss.contiguous(), logits, max_log_sum_exp, + labels, smoothing.item()) + + return grad_logits, None, None, None, None diff --git a/apex/apex/fp16_utils/README.md b/apex/apex/fp16_utils/README.md new file mode 100644 index 00000000..941de179 --- /dev/null +++ b/apex/apex/fp16_utils/README.md @@ -0,0 +1,16 @@ +fp16_optimizer.py contains `FP16_Optimizer`, a Python class designed to wrap an existing Pytorch optimizer and automatically enable master parameters and loss scaling in a manner transparent to the user. To use `FP16_Optimizer`, only two lines of one's Python model need to change. + +#### [FP16_Optimizer API documentation](https://nvidia.github.io/apex/fp16_utils.html#automatic-management-of-master-params-loss-scaling) + +#### [Simple examples with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple) + +#### [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) + +#### [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) + + +fp16_util.py contains a number of utilities to manually manage master parameters and loss scaling, if the user chooses. + +#### [Manual management documentation](https://nvidia.github.io/apex/fp16_utils.html#manual-master-parameter-management) + +The [Imagenet with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) and [word_language_model with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/word_language_model) directories also contain `main.py` files that demonstrate manual management of master parameters and static loss scaling. These examples illustrate what sort of operations `FP16_Optimizer` is performing automatically. diff --git a/apex/apex/fp16_utils/__init__.py b/apex/apex/fp16_utils/__init__.py new file mode 100644 index 00000000..c7bb1f53 --- /dev/null +++ b/apex/apex/fp16_utils/__init__.py @@ -0,0 +1,16 @@ +from .fp16util import ( + BN_convert_float, + network_to_half, + prep_param_lists, + model_grads_to_master_grads, + master_params_to_model_params, + tofp16, + to_python_float, + clip_grad_norm, + convert_module, + convert_network, + FP16Model, +) + +from .fp16_optimizer import FP16_Optimizer +from .loss_scaler import LossScaler, DynamicLossScaler diff --git a/apex/apex/fp16_utils/fp16_optimizer.py b/apex/apex/fp16_utils/fp16_optimizer.py new file mode 100755 index 00000000..15873c97 --- /dev/null +++ b/apex/apex/fp16_utils/fp16_optimizer.py @@ -0,0 +1,557 @@ +import torch +from torch import nn +from torch.autograd import Variable +from torch.nn.parameter import Parameter +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from ..amp._amp_state import _amp_state, maybe_print +from ..amp.scaler import LossScaler +from ..multi_tensor_apply import multi_tensor_applier +from .fp16util import model_grads_to_master_grads, master_params_to_model_params, clip_grad_norm + +# TODO: Update overflow check + downscale to use Carl's fused kernel. +class FP16_Optimizer(object): + def __init__(self, + init_optimizer, + static_loss_scale=1.0, + dynamic_loss_scale=False, + dynamic_loss_args=None, + verbose=True): + print("Warning: FP16_Optimizer is deprecated and dangerous, and will be deleted soon. " + "If it still works, you're probably getting lucky. " + "For mixed precision, use the documented API https://nvidia.github.io/apex/amp.html, with opt_level=O1.") + + from apex import deprecated_warning + deprecated_warning("apex.fp16_utils is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + + if not torch.cuda.is_available: + raise SystemError("Cannot use fp16 without CUDA.") + + self.verbose = verbose + + self.optimizer = init_optimizer + # init_state_dict sets up an alternative way to cast per-param state tensors. + # Stashing here in case https://github.com/pytorch/pytorch/issues/7733 makes it necessary. + # init_state_dict = init_optimizer.state_dict() + + self.fp16_groups = [] + self.fp32_from_fp16_groups = [] + self.fp32_from_fp32_groups = [] + for i, param_group in enumerate(self.optimizer.param_groups): + self.maybe_print("FP16_Optimizer processing param group {}:".format(i)) + fp16_params_this_group = [] + fp32_params_this_group = [] + fp32_from_fp16_params_this_group = [] + for i, param in enumerate(param_group['params']): + if param.requires_grad: + if param.type() == 'torch.cuda.HalfTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.HalfTensor with {}" + .format(param.size())) + fp16_params_this_group.append(param) + master_param = param.detach().clone().float() + master_param.requires_grad = True + param_group['params'][i] = master_param + fp32_from_fp16_params_this_group.append(master_param) + # Reset existing state dict key to the new master param. + # We still need to recast per-param state tensors, if any, to FP32. + if param in self.optimizer.state: + self.optimizer.state[master_param] = self.optimizer.state.pop(param) + elif param.type() == 'torch.cuda.FloatTensor': + self.maybe_print("FP16_Optimizer received torch.cuda.FloatTensor with {}" + .format(param.size())) + fp32_params_this_group.append(param) + param_group['params'][i] = param + else: + raise TypeError("Wrapped parameters must be either " + "torch.cuda.FloatTensor or torch.cuda.HalfTensor. " + "Received {}".format(param.type())) + + self.fp16_groups.append(fp16_params_this_group) + self.fp32_from_fp16_groups.append(fp32_from_fp16_params_this_group) + self.fp32_from_fp32_groups.append(fp32_params_this_group) + + self.all_fp16_params = [] + for group in self.fp16_groups: + self.all_fp16_params += group + + self.all_fp32_from_fp16_params = [] + for group in self.fp32_from_fp16_groups: + self.all_fp32_from_fp16_params += group + + self.all_fp32_from_fp32_params = [] + for group in self.fp32_from_fp32_groups: + self.all_fp32_from_fp32_params += group + + # Leverage state_dict() and load_state_dict() to recast preexisting per-param state tensors + self.optimizer.load_state_dict(self.optimizer.state_dict()) + # alternative way to cast per-param state tensors: + # self.optimizer.load_state_dict(init_state_dict) + + if dynamic_loss_scale: + self.dynamic_loss_scale = True + if dynamic_loss_args is not None: + self.loss_scaler = LossScaler("dynamic", **dynamic_loss_args) + else: + self.loss_scaler = LossScaler("dynamic") + else: + self.dynamic_loss_scale = False + self.loss_scaler = LossScaler(static_loss_scale) + + self.overflow = False + self.first_closure_call_this_step = True + + self.clip_grad_norm = clip_grad_norm + + # TODO: Centralize exposure and import error checking for the C backend. + if multi_tensor_applier.available: + import amp_C + self.multi_tensor_scale = amp_C.multi_tensor_scale + self._dummy_overflow_buf = torch.cuda.IntTensor([0]); + + # Having self.maybe_print distinct from _amp_state.maybe_print is another artifact + # of having to support FP16_Optimizer separately, for the time being. + def maybe_print(self, msg): + if self.verbose: + print(msg) + + def __getstate__(self): + raise RuntimeError("FP16_Optimizer should be serialized using state_dict().") + + def __setstate__(self, state): + raise RuntimeError("FP16_Optimizer should be deserialized using load_state_dict().") + + def zero_grad(self, set_grads_to_None=False): + """ + Zero fp32 and fp16 parameter grads. + """ + # In principle, only the .grad attributes of the model params need to be zeroed, + # because gradients are copied into the FP32 master params. However, we zero + # all gradients owned by the optimizer, just to be safe: + for group in self.optimizer.param_groups: + for p in group['params']: + if set_grads_to_None: + p.grad = None + else: + if p.grad is not None: + p.grad.detach_() + p.grad.zero_() + + # Zero fp16 gradients owned by the model: + for fp16_group in self.fp16_groups: + for param in fp16_group: + if set_grads_to_None: + param.grad = None + else: + if param.grad is not None: + param.grad.detach_() # as in torch.optim.optimizer.zero_grad() + param.grad.zero_() + + # Should not be used anymore. + # def _check_overflow(self): + # params = [] + # for group in self.fp16_groups: + # for param in group: + # params.append(param) + # for group in self.fp32_from_fp32_groups: + # for param in group: + # params.append(param) + # self.overflow = self.loss_scaler.has_overflow(params) + + # def _update_scale(self, has_overflow=False): + # self.loss_scaler.update_scale(has_overflow) + + def _master_params_to_model_params(self): + if multi_tensor_applier.available: + if len(self.all_fp16_params) > 0: + multi_tensor_applier( + self.multi_tensor_scale, + self._dummy_overflow_buf, + [self.all_fp32_from_fp16_params, self.all_fp16_params], + 1.0) + else: + for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + master_params_to_model_params(fp16_group, fp32_from_fp16_group) + + # To consider: Integrate distributed with this wrapper by registering a hook on each variable + # that does the overflow check, gradient copy + downscale, and fp32 allreduce in a different stream. + # def _model_grads_to_master_grads(self): + # for fp16_group, fp32_from_fp16_group in zip(self.fp16_groups, self.fp32_from_fp16_groups): + # model_grads_to_master_grads(fp16_group, fp32_from_fp16_group) + + # def _downscale_master(self): + # if self.loss_scale != 1.0: + # for group in self.optimizer.param_groups: + # for param in group['params']: + # if param.grad is not None: + # param.grad.data.mul_(1./self.loss_scale) + + def clip_master_grads(self, max_norm, norm_type=2): + """ + Clips fp32 master gradients via ``torch.nn.utils.clip_grad_norm``. + + Args: + max_norm (float or int): max norm of the gradients + norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for + infinity norm. + + Returns: + Total norm of the current fp32 gradients (viewed as a single vector). + + .. warning:: + Returns -1 if the most recently computed fp16 gradients overflowed (that is, if ``self.overflow`` is ``True``). + """ + if not self.overflow: + fp32_params = [] + for param_group in self.optimizer.param_groups: + for param in param_group['params']: + fp32_params.append(param) + return self.clip_grad_norm(fp32_params, max_norm, norm_type) + else: + return -1 + + def state_dict(self): + """ + Returns a dict containing the current state of this :class:`FP16_Optimizer` instance. + This dict contains attributes of :class:`FP16_Optimizer`, as well as the state_dict + of the contained Pytorch optimizer. + Example:: + + checkpoint = {} + checkpoint['model'] = model.state_dict() + checkpoint['optimizer'] = optimizer.state_dict() + torch.save(checkpoint, "saved.pth") + """ + state_dict = {} + state_dict['loss_scaler'] = self.loss_scaler + state_dict['dynamic_loss_scale'] = self.dynamic_loss_scale + state_dict['overflow'] = self.overflow + state_dict['first_closure_call_this_step'] = self.first_closure_call_this_step + state_dict['optimizer_state_dict'] = self.optimizer.state_dict() + state_dict['fp32_from_fp16'] = self.fp32_from_fp16_groups + return state_dict + + def load_state_dict(self, state_dict): + """ + Loads a state_dict created by an earlier call to state_dict(). + If ``fp16_optimizer_instance`` was constructed from some ``init_optimizer``, + whose parameters in turn came from ``model``, it is expected that the user + will call ``model.load_state_dict()`` before + ``fp16_optimizer_instance.load_state_dict()`` is called. + + Example:: + + model = torch.nn.Linear(D_in, D_out).cuda().half() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0) + ... + checkpoint = torch.load("saved.pth") + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + """ + # I think it should actually be ok to reload the optimizer before the model. + self.loss_scaler = state_dict['loss_scaler'] + self.dynamic_loss_scale = state_dict['dynamic_loss_scale'] + self.overflow = state_dict['overflow'] + self.first_closure_call_this_step = state_dict['first_closure_call_this_step'] + self.optimizer.load_state_dict(state_dict['optimizer_state_dict']) + # At this point, the optimizer's references to the model's fp32 parameters are up to date. + # The optimizer's hyperparameters and internal buffers are also up to date. + # However, the fp32 master copies of the model's fp16 params stored by the optimizer are still + # out of date. There are two options. + # 1: Refresh the master params from the model's fp16 params. + # This requires less storage but incurs precision loss. + # 2: Save and restore the fp32 master copies separately. + # We choose option 2. + # + # Pytorch Optimizer.load_state_dict casts saved buffers (e.g. momentum) to the type and device + # of their associated parameters, because it's possible those buffers might not exist yet in + # the current optimizer instance. In our case, as long as the current FP16_Optimizer has been + # constructed in the same way as the one whose state_dict we are loading, the same master params + # are guaranteed to exist, so we can just copy_() from the saved master params. + for current_group, saved_group in zip(self.fp32_from_fp16_groups, state_dict['fp32_from_fp16']): + for current, saved in zip(current_group, saved_group): + current.data.copy_(saved.data) + + def step(self, closure=None): # could add clip option. + """ + If no closure is supplied, :attr:`step` should be called after + ``fp16_optimizer_obj.backward(loss)``. + :attr:`step` updates the fp32 master copy of parameters using the optimizer supplied to + :class:`FP16_Optimizer`'s constructor, then copies the updated fp32 params into the fp16 params + originally referenced by :class:`FP16_Optimizer`'s constructor, so the user may immediately run + another forward pass using their model. + + If a closure is supplied, :attr:`step` may be called without a prior call to + :attr:`backward(loss)`. + This control flow is identical to `ordinary Pytorch optimizer use`_ with closures. + However, the user should take care that any ``loss.backward()`` call within the closure + has been replaced by ``fp16_optimizer_obj.backward(loss)``. + + Args: + closure (optional): Closure that will be supplied to the underlying optimizer originally passed to :class:`FP16_Optimizer`'s constructor. closure should call :attr:`zero_grad()` on the :class:`FP16_Optimizer` object, compute the loss, call :attr:`backward(loss)`, and return the loss. + + Example with closure:: + + # optimizer is assumed to be an FP16_Optimizer object, previously constructed from an + # existing pytorch optimizer. + for input, target in dataset: + def closure(): + optimizer.zero_grad() + output = model(input) + loss = loss_fn(output, target) + # loss.backward() becomes: + optimizer.backward(loss) + return loss + optimizer.step(closure) + + .. warning:: + Currently, calling :attr:`step` with a closure is not compatible with dynamic loss scaling. + + .. _`ordinary Pytorch optimizer use`: + http://pytorch.org/docs/master/optim.html#optimizer-step-closure + """ + + scale = self.loss_scaler.loss_scale() + # To consider: Should this be in step(), or update_master_grads? It works either way, + # but I should make it consistent with the Amp control flow, which updates the scale + # during backward context manager exit. + # self._update_scale(self.overflow) + + if self.overflow: + # Using _amp_state.maybe_print instead of self.print here is intentional. + maybe_print("Gradient overflow. Skipping step, reducing " + + "loss scale to {}".format(self.loss_scaler.loss_scale())) + return + + if closure is not None: + retval = self._step_with_closure(closure) + else: + # torch.cuda.nvtx.range_push("pytorch optimizer step") + retval = self.optimizer.step() + # torch.cuda.nvtx.range_pop() + + self._master_params_to_model_params() + + return retval + + def _step_with_closure(self, closure): + def wrapped_closure(): + # helpful for debugging + # print("Calling wrapped_closure, first_closure_call_this_step = {}" + # .format(self.first_closure_call_this_step)) + if self.first_closure_call_this_step: + # We expect that the fp16 params are initially fresh on entering self.step(), + # so _master_params_to_model_params() is unnecessary the first time wrapped_closure() + # is called within self.optimizer.step(). + self.first_closure_call_this_step = False + else: + # If self.optimizer.step() internally calls wrapped_closure more than once, + # it may update the fp32 params after each call. However, self.optimizer + # doesn't know about the fp16 params at all. If the fp32 params get updated, + # we can't rely on self.optimizer to refresh the fp16 params. We need + # to handle that manually: + self._master_params_to_model_params() + # Our API expects the user to give us ownership of the backward() call by + # replacing all calls to loss.backward() with optimizer.backward(loss). + # This requirement holds whether or not the call to backward() is made within a closure. + # If the user is properly calling optimizer.backward(loss) within "closure," + # calling closure() here will give the fp32 master params fresh gradients + # for the optimizer to play with, so all wrapped_closure needs to do is call + # closure() and return the loss. + temp_loss = closure() + while(self.overflow): + scale = self.loss_scaler.loss_scale() + # self._update_scale(self.overflow) # now done at the end of backward + print("OVERFLOW within closure! Skipping step, reducing loss scale to {}".format( + self.loss_scaler.loss_scale())) + temp_loss = closure() + return temp_loss + + retval = self.optimizer.step(wrapped_closure) + + self.first_closure_call_this_step = True + + return retval + + def backward(self, loss, update_master_grads=True, retain_graph=False): + """ + :attr:`backward` performs the following conceptual steps: + + 1. fp32_loss = loss.float() (see first Note below) + 2. scaled_loss = fp32_loss*loss_scale + 3. scaled_loss.backward(), which accumulates scaled gradients into the ``.grad`` attributes of the model's leaves (which may be fp16, fp32, or a mixture, depending how your model was defined). + 4. fp16 grads are then copied to the master params' ``.grad`` attributes (see second Note), which are guaranteed to be fp32. + 5. Finally, master grads are divided by loss_scale. + + In this way, after :attr:`backward`, the master params have fresh gradients, + and :attr:`step` may be called. + + .. note:: + :attr:`backward` internally converts the loss to fp32 before applying the loss scale. + This provides some additional safety against overflow if the user has supplied an + fp16 loss value. + However, for maximum overflow safety, the user should + compute the loss criterion (MSE, cross entropy, etc) in fp32 before supplying it to + :attr:`backward`. + + .. warning:: + The gradients found in a model's leaves after the call to + :attr:`backward` should not be regarded as valid in general, + because it's possible + they have been scaled (and in the case of dynamic loss scaling, + the scale factor may change over time). + If the user wants to inspect gradients after a call to :attr:`backward`, + only the master gradients should be regarded as valid. These can be retrieved via + :attr:`inspect_master_grad_data()`. + + Args: + loss: The loss output by the user's model. loss may be either float or half (but see first Note above). + update_master_grads (bool, optional, default=True): Option to copy fp16 grads to fp32 grads on this call. By setting this to False, the user can delay the copy, which is useful to eliminate redundant fp16->fp32 grad copies if :attr:`backward` is being called on multiple losses in one iteration. If set to False, the user becomes responsible for calling :attr:`update_master_grads` before calling :attr:`step`. + retain_graph (bool, optional, default=False): Forwards the usual ``retain_graph=True`` option to the internal call to ``loss.backward``. If ``retain_graph`` is being used to accumulate gradient values from multiple backward passes before calling ``optimizer.step``, passing ``update_master_grads=False`` is also recommended (see Example below). + + Example:: + + # Ordinary operation: + optimizer.backward(loss) + + # Naive operation with multiple losses (technically valid, but less efficient): + # fp32 grads will be correct after the second call, but + # the first call incurs an unnecessary fp16->fp32 grad copy. + optimizer.backward(loss1) + optimizer.backward(loss2) + + # More efficient way to handle multiple losses: + # The fp16->fp32 grad copy is delayed until fp16 grads from all + # losses have been accumulated. + optimizer.backward(loss1, update_master_grads=False) + optimizer.backward(loss2, update_master_grads=False) + optimizer.update_master_grads() + """ + # To consider: try multiple backward passes using retain_grad=True to find + # a loss scale that works. After you find a loss scale that works, do a final dummy + # backward pass with retain_graph=False to tear down the graph. Doing this would avoid + # discarding the iteration, but probably wouldn't improve overall efficiency. + scaled_loss = loss.float()*self.loss_scaler.loss_scale() + scaled_loss.backward(retain_graph=retain_graph) + if update_master_grads: + self.update_master_grads() + + def update_master_grads(self): + # torch.cuda.nvtx.range_push("update_master_grads") + """ + Copy the ``.grad`` attribute from stored references to fp16 parameters to + the ``.grad`` attribute of the fp32 master parameters that are directly + updated by the optimizer. :attr:`update_master_grads` only needs to be called if + ``fp16_optimizer_obj.backward`` was called with ``update_master_grads=False``. + """ + # if self.dynamic_loss_scale: + # self._check_overflow() + # if self.overflow: return + # self._model_grads_to_master_grads() + # self._downscale_master() + # Use the one-shot multi-tensor apply kernel + self.loss_scaler.clear_overflow_state() + if len(self.all_fp16_params) > 0: + # print("Model grads before") + # print([param.grad.data for param in self.all_fp16_params]) + # I'm ONLY writing this as an incremental way to make some tests pass until + # I can refactor the tests as well. + # FP16_Optimizer should not be used by anyone. + model_grads = [] + master_grads = [] + for model_param, master_param in zip(self.all_fp16_params, + self.all_fp32_from_fp16_params): + if model_param.grad is not None: + model_grads.append(model_param.grad) + if master_param.grad is None: + master_param.grad = torch.empty_like(master_param) + master_grads.append(master_param.grad) + self.loss_scaler.unscale( + model_grads, + master_grads, + self.loss_scaler.loss_scale()) + # print("Master grads after") + # print([param.grad.data for param in self.all_fp32_from_fp16_params]) + if len(self.all_fp32_from_fp32_params) > 0: + model_grads = [] + master_grads = [] + for model_param, master_param in zip(self.all_fp32_from_fp32_params, + self.all_fp32_from_fp32_params): + if model_param.grad is not None: + model_grads.append(model_param.grad) + master_grads.append(master_param.grad) + # print("Model grads before") + # print([param.grad.data for param in self.all_fp32_from_fp32_params]) + self.loss_scaler.unscale( + model_grads, + master_grads, + self.loss_scaler.loss_scale()) + # print("Master grads after") + # print([param.grad.data for param in self.all_fp32_from_fp32_params]) + # quit() + self.overflow = self.loss_scaler.update_scale() + # torch.cuda.nvtx.range_pop() + + + def inspect_master_grad_data(self): + """ + When running with :class:`FP16_Optimizer`, + ``.grad`` attributes of a model's fp16 leaves should not be + regarded as truthful, because they might be scaled. + After a call to :attr:`fp16_optimizer_obj.backward(loss)`, if no overflow was encountered, + the fp32 master params' ``.grad`` + attributes will contain valid gradients properly divided by the loss scale. However, + because :class:`FP16_Optimizer` flattens some parameters, accessing them may be + nonintuitive. :attr:`inspect_master_grad_data` + allows those gradients to be viewed with shapes corresponding to their associated model leaves. + + Returns: + List of lists (one list for each parameter group). The list for each parameter group + is a list of the ``.grad.data`` attributes of the fp32 master params belonging to that group. + """ + if self.overflow: + print("Warning: calling FP16_Optimizer.inspect_master_grad_data while in an overflow state. " + "Gradients are currently invalid (may be inf, nan, or stale). Returning None.") + return None + else: + # The optimizer owns only references to master params. + master_grads_data = [] + for param_group in self.optimizer.param_groups: + master_grads_this_group = [] + for param in param_group['params']: + if param.grad is not None: + master_grads_this_group.append(param.grad.data) + else: + master_grads_this_group.append(None) + master_grads_data.append(master_grads_this_group) + return master_grads_data + + + # Promote loss scale so it can be retrieved or set via "fp16_optimizer_instance.loss_scale" + def _get_loss_scale(self): + return self.loss_scaler.loss_scale() + + def _set_loss_scale(self, value): + self.loss_scaler._loss_scale = value + + loss_scale = property(_get_loss_scale, _set_loss_scale) + + # Promote state so it can be retrieved or set via "fp16_optimizer_instance.state" + def _get_state(self): + return self.optimizer.state + + def _set_state(self, value): + self.optimizer.state = value + + state = property(_get_state, _set_state) + + # Promote param_groups so it can be retrieved or set via "fp16_optimizer_instance.param_groups" + # (for example, to adjust the learning rate) + def _get_param_groups(self): + return self.optimizer.param_groups + + def _set_param_groups(self, value): + self.optimizer.param_groups = value + + param_groups = property(_get_param_groups, _set_param_groups) + diff --git a/apex/apex/fp16_utils/fp16util.py b/apex/apex/fp16_utils/fp16util.py new file mode 100644 index 00000000..325abd1b --- /dev/null +++ b/apex/apex/fp16_utils/fp16util.py @@ -0,0 +1,189 @@ +import torch +import torch.nn as nn +from torch.autograd import Variable +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + + +class tofp16(nn.Module): + """ + Utility module that implements:: + + def forward(self, input): + return input.half() + """ + + def __init__(self): + super(tofp16, self).__init__() + + def forward(self, input): + return input.half() + + +def BN_convert_float(module): + """ + Utility function for network_to_half(). + + Retained for legacy purposes. + """ + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + module.float() + for child in module.children(): + BN_convert_float(child) + return module + + +def network_to_half(network): + """ + Convert model to half precision in a batchnorm-safe way. + + Retained for legacy purposes. It is recommended to use FP16Model. + """ + return nn.Sequential(tofp16(), BN_convert_float(network.half())) + + +def convert_module(module, dtype): + """ + Converts a module's immediate parameters and buffers to dtype. + """ + for param in module.parameters(recurse=False): + if param is not None: + if param.data.dtype.is_floating_point: + param.data = param.data.to(dtype=dtype) + if param._grad is not None and param._grad.data.dtype.is_floating_point: + param._grad.data = param._grad.data.to(dtype=dtype) + + for buf in module.buffers(recurse=False): + if buf is not None and buf.data.dtype.is_floating_point: + buf.data = buf.data.to(dtype=dtype) + + +def convert_network(network, dtype): + """ + Converts a network's parameters and buffers to dtype. + """ + for module in network.modules(): + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm) and module.affine is True: + continue + convert_module(module, dtype) + if isinstance(module, torch.nn.RNNBase) or isinstance(module, torch.nn.modules.rnn.RNNBase): + module.flatten_parameters() + return network + + +class FP16Model(nn.Module): + """ + Convert model to half precision in a batchnorm-safe way. + """ + + def __init__(self, network): + from apex import deprecated_warning + deprecated_warning("apex.fp16_utils is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + super(FP16Model, self).__init__() + self.network = convert_network(network, dtype=torch.half) + + def forward(self, *inputs): + inputs = tuple(t.half() for t in inputs) + return self.network(*inputs) + + +def backwards_debug_hook(grad): + raise RuntimeError("master_params recieved a gradient in the backward pass!") + +def prep_param_lists(model, flat_master=False): + """ + Creates a list of FP32 master parameters for a given model, as in + `Training Neural Networks with Mixed Precision: Real Examples`_. + + Args: + model (torch.nn.Module): Existing Pytorch model + flat_master (bool, optional, default=False): Flatten the master parameters into a single tensor, as a performance optimization. + Returns: + A tuple (``model_params``, ``master_params``). ``model_params`` is a list of the model's parameters for later use with :func:`model_grads_to_master_grads` and :func:`master_params_to_model_params`. ``master_params`` is a list of FP32 master gradients. If ``flat_master=True``, ``master_params`` will be a list with one element. + + Example:: + + model_params, master_params = prep_param_lists(model) + + .. warning:: + Currently, if ``flat_master=True``, all the model's parameters must be the same type. If the model has parameters of different types, use ``flat_master=False``, or use :class:`FP16_Optimizer`. + + .. _`Training Neural Networks with Mixed Precision: Real Examples`: + http://on-demand.gputechconf.com/gtc/2018/video/S81012/ + """ + model_params = [param for param in model.parameters() if param.requires_grad] + + if flat_master: + # Give the user some more useful error messages + try: + # flatten_dense_tensors returns a contiguous flat array. + # http://pytorch.org/docs/master/_modules/torch/_utils.html + master_params = _flatten_dense_tensors([param.data for param in model_params]).float() + except: + print("Error in prep_param_lists: model may contain a mixture of parameters " + "of different types. Use flat_master=False, or use F16_Optimizer.") + raise + master_params = torch.nn.Parameter(master_params) + master_params.requires_grad = True + # master_params.register_hook(backwards_debug_hook) + if master_params.grad is None: + master_params.grad = master_params.new(*master_params.size()) + return model_params, [master_params] + else: + master_params = [param.clone().float().detach() for param in model_params] + for param in master_params: + param.requires_grad = True + return model_params, master_params + + +def model_grads_to_master_grads(model_params, master_params, flat_master=False): + """ + Copy model gradients to master gradients. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`model_grads_to_master_grads`. + """ + if flat_master: + # The flattening may incur one more deep copy than is necessary. + master_params[0].grad.data.copy_( + _flatten_dense_tensors([p.grad.data for p in model_params])) + else: + for model, master in zip(model_params, master_params): + if model.grad is not None: + if master.grad is None: + master.grad = Variable(master.data.new(*master.data.size())) + master.grad.data.copy_(model.grad.data) + else: + master.grad = None + + +def master_params_to_model_params(model_params, master_params, flat_master=False): + """ + Copy master parameters to model parameters. + + Args: + model_params: List of model parameters created by :func:`prep_param_lists`. + master_params: List of FP32 master parameters created by :func:`prep_param_lists`. If ``master_params`` was created with ``flat_master=True``, ``flat_master=True`` should also be supplied to :func:`master_params_to_model_params`. + """ + if flat_master: + for model, master in zip(model_params, + _unflatten_dense_tensors(master_params[0].data, model_params)): + model.data.copy_(master) + else: + for model, master in zip(model_params, master_params): + model.data.copy_(master.data) + +# Backward compatibility fixes + +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +TORCH_MAJOR = int(torch.__version__.split('.')[0]) +TORCH_MINOR = int(torch.__version__.split('.')[1]) +if TORCH_MAJOR == 0 and TORCH_MINOR <= 4: + clip_grad_norm = torch.nn.utils.clip_grad_norm +else: + clip_grad_norm = torch.nn.utils.clip_grad_norm_ diff --git a/apex/apex/fp16_utils/loss_scaler.py b/apex/apex/fp16_utils/loss_scaler.py new file mode 100644 index 00000000..7c7ea241 --- /dev/null +++ b/apex/apex/fp16_utils/loss_scaler.py @@ -0,0 +1,188 @@ +import torch + +# item() is a recent addition, so this helps with backward compatibility. +def to_python_float(t): + if hasattr(t, 'item'): + return t.item() + else: + return t[0] + +class LossScaler: + """ + Class that manages a static loss scale. This class is intended to interact with + :class:`FP16_Optimizer`, and should not be directly manipulated by the user. + + Use of :class:`LossScaler` is enabled via the ``static_loss_scale`` argument to + :class:`FP16_Optimizer`'s constructor. + + Args: + scale (float, optional, default=1.0): The loss scale. + """ + + def __init__(self, scale=1): + from apex import deprecated_warning + deprecated_warning("apex.fp16_utils is deprecated and will be removed by the end of February 2023. Use [PyTorch AMP](https://pytorch.org/docs/stable/amp.html)") + self.cur_scale = scale + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + return False + + def update_scale(self, overflow): + pass + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +class DynamicLossScaler: + """ + Class that manages dynamic loss scaling. It is recommended to use :class:`DynamicLossScaler` + indirectly, by supplying ``dynamic_loss_scale=True`` to the constructor of + :class:`FP16_Optimizer`. However, it's important to understand how :class:`DynamicLossScaler` + operates, because the default options can be changed using the + the ``dynamic_loss_args`` argument to :class:`FP16_Optimizer`'s constructor. + + Loss scaling is designed to combat the problem of underflowing gradients encountered at long + times when training fp16 networks. Dynamic loss scaling begins by attempting a very high loss + scale. Ironically, this may result in OVERflowing gradients. If overflowing gradients are + encountered, :class:`DynamicLossScaler` informs :class:`FP16_Optimizer` that an overflow has + occurred. + :class:`FP16_Optimizer` then skips the update step for this particular iteration/minibatch, + and :class:`DynamicLossScaler` adjusts the loss scale to a lower value. + If a certain number of iterations occur without overflowing gradients detected, + :class:`DynamicLossScaler` increases the loss scale once more. + In this way :class:`DynamicLossScaler` attempts to "ride the edge" of + always using the highest loss scale possible without incurring overflow. + + Args: + init_scale (float, optional, default=2**32): Initial loss scale attempted by :class:`DynamicLossScaler.` + scale_factor (float, optional, default=2.0): Factor used when adjusting the loss scale. If an overflow is encountered, the loss scale is readjusted to loss scale/``scale_factor``. If ``scale_window`` consecutive iterations take place without an overflow, the loss scale is readjusted to loss_scale*``scale_factor``. + scale_window (int, optional, default=1000): Number of consecutive iterations without an overflow to wait before increasing the loss scale. + """ + + def __init__(self, + init_scale=2**32, + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + # `params` is a list / generator of torch.Variable + def has_overflow(self, params): + for p in params: + if p.grad is not None and DynamicLossScaler._has_inf_or_nan(p.grad.data): + return True + + return False + + # `x` is a torch.Tensor + def _has_inf_or_nan(x): + try: + # if x is half, the .float() incurs an additional deep copy, but it's necessary if + # Pytorch's .sum() creates a one-element tensor of the same type as x + # (which is true for some recent version of pytorch). + cpu_sum = float(x.float().sum()) + # More efficient version that can be used if .sum() returns a Python scalar + # cpu_sum = float(x.sum()) + except RuntimeError as instance: + # We want to check if inst is actually an overflow exception. + # RuntimeError could come from a different error. + # If so, we still want the exception to propagate. + if "value cannot be converted" not in instance.args[0]: + raise + return True + else: + if cpu_sum == float('inf') or cpu_sum == -float('inf') or cpu_sum != cpu_sum: + return True + return False + + # `overflow` is boolean indicating whether the gradient overflowed + def update_scale(self, overflow): + if overflow: + # self.cur_scale /= self.scale_factor + self.cur_scale = max(self.cur_scale/self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % self.scale_window == 0: + self.cur_scale *= self.scale_factor + self.cur_iter += 1 + + @property + def loss_scale(self): + return self.cur_scale + + def scale_gradient(self, module, grad_in, grad_out): + return tuple(self.loss_scale * g for g in grad_in) + + def backward(self, loss, retain_graph=False): + scaled_loss = loss*self.loss_scale + scaled_loss.backward(retain_graph=retain_graph) + +############################################################## +# Example usage below here -- assuming it's in a separate file +############################################################## +""" +TO-DO separate out into an example. +if __name__ == "__main__": + import torch + from torch.autograd import Variable + from dynamic_loss_scaler import DynamicLossScaler + + # N is batch size; D_in is input dimension; + # H is hidden dimension; D_out is output dimension. + N, D_in, H, D_out = 64, 1000, 100, 10 + + # Create random Tensors to hold inputs and outputs, and wrap them in Variables. + x = Variable(torch.randn(N, D_in), requires_grad=False) + y = Variable(torch.randn(N, D_out), requires_grad=False) + + w1 = Variable(torch.randn(D_in, H), requires_grad=True) + w2 = Variable(torch.randn(H, D_out), requires_grad=True) + parameters = [w1, w2] + + learning_rate = 1e-6 + optimizer = torch.optim.SGD(parameters, lr=learning_rate) + loss_scaler = DynamicLossScaler() + + for t in range(500): + y_pred = x.mm(w1).clamp(min=0).mm(w2) + loss = (y_pred - y).pow(2).sum() * loss_scaler.loss_scale + print('Iter {} loss scale: {}'.format(t, loss_scaler.loss_scale)) + print('Iter {} scaled loss: {}'.format(t, loss.data[0])) + print('Iter {} unscaled loss: {}'.format(t, loss.data[0] / loss_scaler.loss_scale)) + + # Run backprop + optimizer.zero_grad() + loss.backward() + + # Check for overflow + has_overflow = DynamicLossScaler.has_overflow(parameters) + + # If no overflow, unscale grad and update as usual + if not has_overflow: + for param in parameters: + param.grad.data.mul_(1. / loss_scaler.loss_scale) + optimizer.step() + # Otherwise, don't do anything -- ie, skip iteration + else: + print('OVERFLOW!') + + # Update loss scale for next iteration + loss_scaler.update_scale(has_overflow) + +""" diff --git a/apex/apex/fused_dense/__init__.py b/apex/apex/fused_dense/__init__.py new file mode 100644 index 00000000..83d12cab --- /dev/null +++ b/apex/apex/fused_dense/__init__.py @@ -0,0 +1 @@ +from .fused_dense import * diff --git a/apex/apex/fused_dense/fused_dense.py b/apex/apex/fused_dense/fused_dense.py new file mode 100644 index 00000000..02930317 --- /dev/null +++ b/apex/apex/fused_dense/fused_dense.py @@ -0,0 +1,95 @@ +import torch +from torch import nn +import fused_dense_cuda +from apex._autocast_utils import _cast_if_autocast_enabled + +#implements fused GEMM+bias in forward pass using mlp_cuda from apex +class FusedDenseFunc(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weight, bias): + ctx.save_for_backward(input, weight) + output = fused_dense_cuda.linear_bias_forward(input, weight, bias) + return output + + @staticmethod + def backward(ctx, grad_output): + input, weight = ctx.saved_tensors + grad_input, grad_weight, grad_bias = fused_dense_cuda.linear_bias_backward(input, weight, grad_output) + return grad_input, grad_weight, grad_bias + +class DenseNoBiasFunc(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weight): + ctx.save_for_backward(input, weight) + output = torch.matmul(input, weight.t()) + return output + + @staticmethod + def backward(ctx, grad_output): + input, weight = ctx.saved_tensors + grad_input = grad_output.mm(weight) + grad_weight = grad_output.t().mm(input) + return grad_input, grad_weight + + +class FusedDenseGeluDenseFunc(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weight1, bias1, weight2, bias2): + ctx.save_for_backward(input, weight1, weight2) + output1, output2, gelu_in = fused_dense_cuda.linear_gelu_linear_forward(input, weight1, bias1, weight2, bias2) + ctx.save_for_backward(input, weight1, weight2, gelu_in, output1) + return output2 + + @staticmethod + def backward(ctx, grad_output): + input, weight1, weight2, gelu_in, output1 = ctx.saved_tensors + grad_input, grad_weight1, grad_bias1, grad_weight2, grad_bias2 = fused_dense_cuda.linear_gelu_linear_backward(input, gelu_in, output1, weight1, weight2, grad_output) + return grad_input, grad_weight1, grad_bias1, grad_weight2, grad_bias2 + +def _fused_dense(input, weight, bias): + args = _cast_if_autocast_enabled(input, weight, bias) + with torch.cuda.amp.autocast(enabled=False): + return FusedDenseFunc.apply(*args) + +def _dense_no_bias(input, weight): + args = _cast_if_autocast_enabled(input, weight) + with torch.cuda.amp.autocast(enabled=False): + return DenseNoBiasFunc.apply(*args) + +def _fused_dense_gelu_dense(input, weight1, bias1, weight2, bias2): + args = _cast_if_autocast_enabled(input, weight1, bias1, weight2, bias2) + with torch.cuda.amp.autocast(enabled=False): + return FusedDenseGeluDenseFunc.apply(*args) + +class FusedDense(nn.Module): + def __init__(self, in_features, out_features, bias=True): + super(FusedDense, self).__init__() + self.in_features = in_features + self.out_features = out_features + self.weight = nn.Parameter(torch.empty(out_features, in_features)) + if bias: + self.bias = nn.Parameter(torch.empty(out_features)) + else: + #assert False, "no-bias option not added yet" + self.register_parameter('bias', None) + + def forward(self, input): + if self.bias is not None: + return _fused_dense(input, self.weight, self.bias) + else: + return _dense_no_bias(input, self.weight) + +class FusedDenseGeluDense(nn.Module): + def __init__(self, in_features, intermediate_features, out_features, bias=True): + super(FusedDenseGeluDense, self).__init__() + assert bias == True, "DenseGeluDense module without bias is currently not supported" + self.in_features = in_features + self.intermediate_features = intermediate_features + self.out_features = out_features + self.weight1 = nn.Parameter(torch.empty(intermediate_features, in_features)) + self.bias1 = nn.Parameter(torch.empty(intermediate_features)) + self.weight2 = nn.Parameter(torch.empty(out_features, intermediate_features)) + self.bias2 = nn.Parameter(torch.empty(out_features)) + + def forward(self, input): + return _fused_dense_gelu_dense(input, self.weight1, self.bias1, self.weight2, self.bias2) diff --git a/apex/apex/mlp/__init__.py b/apex/apex/mlp/__init__.py new file mode 100644 index 00000000..f2f30f7d --- /dev/null +++ b/apex/apex/mlp/__init__.py @@ -0,0 +1 @@ +from .mlp import * diff --git a/apex/apex/mlp/mlp.py b/apex/apex/mlp/mlp.py new file mode 100644 index 00000000..be0901eb --- /dev/null +++ b/apex/apex/mlp/mlp.py @@ -0,0 +1,86 @@ +from copy import copy +import math + +import torch +from torch import nn + +from apex._autocast_utils import _cast_if_autocast_enabled +import mlp_cuda + + +class MlpFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, bias, activation, *args): + output = mlp_cuda.forward(bias, activation, args) + ctx.save_for_backward(*args) + ctx.outputs = output + ctx.bias = bias + ctx.activation = activation + return output[0] + + @staticmethod + def backward(ctx, grad_o): + grads = mlp_cuda.backward(ctx.bias, ctx.activation, grad_o, ctx.outputs, ctx.saved_tensors) + del ctx.outputs + return (None, None, *grads) + + +def mlp_function(bias, activation, *args): + autocast_args = _cast_if_autocast_enabled(bias, activation, *args) + return MlpFunction.apply(*autocast_args) + + +class MLP(torch.nn.Module): + """Launch MLP in C++ + + Args: + mlp_sizes (list of int): MLP sizes. Example: [1024,1024,1024] will create 2 MLP layers with shape 1024x1024 + bias (bool): Default True: + relu (bool): Default True + """ + def __init__(self, mlp_sizes, bias=True, activation='relu'): + super().__init__() + self.num_layers = len(mlp_sizes) - 1 + self.mlp_sizes = copy(mlp_sizes) + self.bias = 1 if bias else 0 + + if activation == 'none': + self.activation = 0 + elif activation == 'relu': + self.activation = 1 + elif activation == 'sigmoid': + self.activation = 2 + else: + raise TypeError("activation must be relu or none.") + + self.weights = [] + self.biases = [] + for i in range(self.num_layers): + w = torch.nn.Parameter(torch.empty(mlp_sizes[i+1], mlp_sizes[i])) + self.weights.append(w) + name = 'weight_{}'.format(i) + setattr(self, name, w) + if self.bias: + b = torch.nn.Parameter(torch.empty(mlp_sizes[i+1])) + self.biases.append(b) + name = 'bias_{}'.format(i) + setattr(self, name, b) + + self.reset_parameters() + + def reset_parameters(self): + for weight in self.weights: + dimsum = weight.size(0) + weight.size(1) + std = math.sqrt(2. / float(dimsum)) + nn.init.normal_(weight, 0., std) + if self.bias: + for bias in self.biases: + std = math.sqrt(1. / float(bias.size(0))) + nn.init.normal_(bias, 0., std) + + def forward(self, input): + return mlp_function(self.bias, self.activation, input, *self.weights, *self.biases) + + def extra_repr(self): + s = F"MLP sizes: {self.mlp_sizes}, Bias={self.bias}, activation={self.activation}" + return s diff --git a/apex/apex/multi_tensor_apply/__init__.py b/apex/apex/multi_tensor_apply/__init__.py new file mode 100644 index 00000000..0a80e3c5 --- /dev/null +++ b/apex/apex/multi_tensor_apply/__init__.py @@ -0,0 +1,4 @@ +from .multi_tensor_apply import MultiTensorApply + +multi_tensor_applier = MultiTensorApply(2048*32) + diff --git a/apex/apex/multi_tensor_apply/multi_tensor_apply.py b/apex/apex/multi_tensor_apply/multi_tensor_apply.py new file mode 100644 index 00000000..346c6e50 --- /dev/null +++ b/apex/apex/multi_tensor_apply/multi_tensor_apply.py @@ -0,0 +1,30 @@ +import torch + +class MultiTensorApply(object): + available = False + warned = False + + def __init__(self, chunk_size): + try: + import amp_C + MultiTensorApply.available = True + self.chunk_size = chunk_size + except ImportError as err: + MultiTensorApply.available = False + MultiTensorApply.import_err = err + + def check_avail(self): + if MultiTensorApply.available == False: + raise RuntimeError( + "Attempted to call MultiTensorApply method, but MultiTensorApply " + "is not available, possibly because Apex was installed without " + "--cpp_ext --cuda_ext. Original import error message:", + MultiTensorApply.import_err) + + def __call__(self, op, noop_flag_buffer, tensor_lists, *args): + self.check_avail() + + return op(self.chunk_size, + noop_flag_buffer, + tensor_lists, + *args) diff --git a/apex/apex/normalization/__init__.py b/apex/apex/normalization/__init__.py new file mode 100644 index 00000000..8bd4f159 --- /dev/null +++ b/apex/apex/normalization/__init__.py @@ -0,0 +1,2 @@ +from .fused_layer_norm import FusedLayerNorm, MixedFusedLayerNorm, FusedRMSNorm, MixedFusedRMSNorm +from .instance_norm import InstanceNorm3dNVFuser diff --git a/apex/apex/normalization/fused_layer_norm.py b/apex/apex/normalization/fused_layer_norm.py new file mode 100644 index 00000000..c528614f --- /dev/null +++ b/apex/apex/normalization/fused_layer_norm.py @@ -0,0 +1,437 @@ +import importlib +import numbers + +import torch +from torch.nn.parameter import Parameter +from torch.nn import init +from torch.nn import functional as F + +from apex._autocast_utils import _cast_if_autocast_enabled + +global fused_layer_norm_cuda +fused_layer_norm_cuda = None + + +# Reference implementation from Huggingface +def manual_rms_norm(input, normalized_shape, weight, eps): + # layer norm should always be calculated in float32 + dims = tuple(i for i in range(-1, -len(normalized_shape)-1, -1)) + variance = input.to(torch.float32).pow(2).mean(dims, keepdim=True) + input = input * torch.rsqrt(variance + eps) + + if weight is None: + return input + + # convert into half-precision if necessary + if weight.dtype in [torch.float16, torch.bfloat16]: + input = input.to(weight.dtype) + + return weight * input + + +class FusedLayerNormAffineFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weight, bias, normalized_shape, eps): + global fused_layer_norm_cuda + if fused_layer_norm_cuda is None: + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + ctx.normalized_shape = normalized_shape + ctx.eps = eps + input_ = input.contiguous() + weight_ = weight.contiguous() + bias_ = bias.contiguous() + output, mean, invvar = fused_layer_norm_cuda.forward_affine( + input_, ctx.normalized_shape, weight_, bias_, ctx.eps + ) + ctx.save_for_backward(input_, weight_, bias_, mean, invvar) + return output + + @staticmethod + def backward(ctx, grad_output): + input_, weight_, bias_, mean, invvar = ctx.saved_tensors + grad_input = grad_weight = grad_bias = None + grad_input, grad_weight, grad_bias = fused_layer_norm_cuda.backward_affine( + grad_output.contiguous(), mean, invvar, input_, ctx.normalized_shape, weight_, bias_, ctx.eps + ) + return grad_input, grad_weight, grad_bias, None, None + + +class FusedRMSNormAffineFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weight, normalized_shape, eps): + global fused_layer_norm_cuda + if fused_layer_norm_cuda is None: + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + ctx.normalized_shape = normalized_shape + ctx.eps = eps + input_ = input.contiguous() + weight_ = weight.contiguous() + output, invvar = fused_layer_norm_cuda.rms_forward_affine( + input_, ctx.normalized_shape, weight_, ctx.eps) + ctx.save_for_backward(input_, weight_, invvar) + return output + + @staticmethod + def backward(ctx, grad_output): + input_, weight_, invvar = ctx.saved_tensors + grad_input = grad_weight = None + grad_input, grad_weight = fused_layer_norm_cuda.rms_backward_affine( + grad_output.contiguous(), invvar, input_, ctx.normalized_shape, weight_, ctx.eps + ) + return grad_input, grad_weight, None, None + + +class FusedLayerNormAffineMixedDtypesFunction(FusedLayerNormAffineFunction): + + @staticmethod + def forward(ctx, input, weight, bias, normalized_shape, eps): + global fused_layer_norm_cuda + if fused_layer_norm_cuda is None: + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + ctx.normalized_shape = normalized_shape + ctx.eps = eps + input_ = input.contiguous() + weight_ = weight.contiguous() + bias_ = bias.contiguous() + output, mean, invvar = fused_layer_norm_cuda.forward_affine_mixed_dtypes( + input_, ctx.normalized_shape, weight_, bias_, ctx.eps + ) + ctx.save_for_backward(input_, weight_, bias_, mean, invvar) + return output + + +class FusedRMSNormAffineMixedDtypesFunction(FusedRMSNormAffineFunction): + + @staticmethod + def forward(ctx, input, weight, normalized_shape, eps): + global fused_layer_norm_cuda + if fused_layer_norm_cuda is None: + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + ctx.normalized_shape = normalized_shape + ctx.eps = eps + input_ = input.contiguous() + weight_ = weight.contiguous() + output, invvar = fused_layer_norm_cuda.rms_forward_affine_mixed_dtypes( + input_, ctx.normalized_shape, weight_, ctx.eps + ) + + ctx.save_for_backward(input_, weight_, invvar) + return output + + +class FusedLayerNormFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, normalized_shape, eps): + global fused_layer_norm_cuda + if fused_layer_norm_cuda is None: + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + ctx.normalized_shape = normalized_shape + ctx.eps = eps + input_ = input.contiguous() + output, mean, invvar = fused_layer_norm_cuda.forward(input_, ctx.normalized_shape, ctx.eps) + ctx.save_for_backward(input_, mean, invvar) + return output + + @staticmethod + def backward(ctx, grad_output): + input_, mean, invvar = ctx.saved_tensors + grad_input = None + grad_input = fused_layer_norm_cuda.backward( + grad_output.contiguous(), mean, invvar, input_, ctx.normalized_shape, ctx.eps + ) + return grad_input, None, None + + +class FusedRMSNormFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, normalized_shape, eps): + global fused_layer_norm_cuda + if fused_layer_norm_cuda is None: + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + ctx.normalized_shape = normalized_shape + ctx.eps = eps + input_ = input.contiguous() + output, invvar = fused_layer_norm_cuda.rms_forward(input_, ctx.normalized_shape, ctx.eps) + ctx.save_for_backward(input_, invvar) + return output + + @staticmethod + def backward(ctx, grad_output): + input_, invvar = ctx.saved_tensors + grad_input = None + grad_input = fused_layer_norm_cuda.rms_backward( + grad_output.contiguous(), invvar, input_, ctx.normalized_shape, ctx.eps + ) + return grad_input, None, None + + +def fused_layer_norm_affine(input, weight, bias, normalized_shape, eps=1e-6): + args = _cast_if_autocast_enabled(input, weight, bias, normalized_shape, eps) + with torch.cuda.amp.autocast(enabled=False): + return FusedLayerNormAffineFunction.apply(*args) + + +def fused_layer_norm(input, normalized_shape, eps=1e-6): + args = _cast_if_autocast_enabled(input, normalized_shape, eps) + with torch.cuda.amp.autocast(enabled=False): + return FusedLayerNormFunction.apply(*args) + + +def mixed_dtype_fused_layer_norm_affine(input, weight, bias, normalized_shape, eps=1e-6): + args = _cast_if_autocast_enabled(input, weight, bias, normalized_shape, eps) + with torch.cuda.amp.autocast(enabled=False): + return FusedLayerNormAffineMixedDtypesFunction.apply(*args) + + +def fused_rms_norm_affine(input, weight, normalized_shape, eps=1e-6): + args = _cast_if_autocast_enabled(input, weight, normalized_shape, eps) + with torch.cuda.amp.autocast(enabled=False): + return FusedRMSNormAffineFunction.apply(*args) + + +def fused_rms_norm(input, normalized_shape, eps=1e-6): + args = _cast_if_autocast_enabled(input, normalized_shape, eps) + with torch.cuda.amp.autocast(enabled=False): + return FusedRMSNormFunction.apply(*args) + + +def mixed_dtype_fused_rms_norm_affine(input, weight, normalized_shape, eps=1e-6): + args = _cast_if_autocast_enabled(input, weight, normalized_shape, eps) + with torch.cuda.amp.autocast(enabled=False): + return FusedRMSNormAffineMixedDtypesFunction.apply(*args) + + +class FusedLayerNorm(torch.nn.Module): + r"""Applies Layer Normalization over a mini-batch of inputs as described in + the paper `Layer Normalization`_ . + + Currently only runs on cuda() tensors. + + .. math:: + y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta + + The mean and standard-deviation are calculated separately over the last + certain number dimensions which have to be of the shape specified by + :attr:`normalized_shape`. + :math:`\gamma` and :math:`\beta` are learnable affine transform parameters of + :attr:`normalized_shape` if :attr:`elementwise_affine` is ``True``. + + .. note:: + Unlike Batch Normalization and Instance Normalization, which applies + scalar scale and bias for each entire channel/plane with the + :attr:`affine` option, Layer Normalization applies per-element scale and + bias with :attr:`elementwise_affine`. + + This layer uses statistics computed from input data in both training and + evaluation modes. + + Args: + normalized_shape (int or list or torch.Size): input shape from an expected input + of size + + .. math:: + [* \times \text{normalized}\_\text{shape}[0] \times \text{normalized}\_\text{shape}[1] + \times \ldots \times \text{normalized}\_\text{shape}[-1]] + + If a single integer is used, it is treated as a singleton list, and this module will + normalize over the last dimension which is expected to be of that specific size. + eps: a value added to the denominator for numerical stability. Default: 1e-5 + elementwise_affine: a boolean value that when set to ``True``, this module + has learnable per-element affine parameters initialized to ones (for weights) + and zeros (for biases). Default: ``True``. + + Shape: + - Input: :math:`(N, *)` + - Output: :math:`(N, *)` (same shape as input) + + Examples:: + + >>> input = torch.randn(20, 5, 10, 10) + >>> # With Learnable Parameters + >>> m = apex.normalization.FusedLayerNorm(input.size()[1:]) + >>> # Without Learnable Parameters + >>> m = apex.normalization.FusedLayerNorm(input.size()[1:], elementwise_affine=False) + >>> # Normalize over last two dimensions + >>> m = apex.normalization.FusedLayerNorm([10, 10]) + >>> # Normalize over last dimension of size 10 + >>> m = apex.normalization.FusedLayerNorm(10) + >>> # Activating the module + >>> output = m(input) + + .. _`Layer Normalization`: https://arxiv.org/abs/1607.06450 + """ + + def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True): + super().__init__() + + global fused_layer_norm_cuda + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + if self.elementwise_affine: + self.weight = Parameter(torch.empty(*normalized_shape)) + self.bias = Parameter(torch.empty(*normalized_shape)) + else: + self.register_parameter("weight", None) + self.register_parameter("bias", None) + self.reset_parameters() + + def reset_parameters(self): + if self.elementwise_affine: + init.ones_(self.weight) + init.zeros_(self.bias) + + def forward(self, input): + if not input.is_cuda: + return F.layer_norm(input, self.normalized_shape, self.weight, self.bias, self.eps) + if self.elementwise_affine: + return fused_layer_norm_affine(input, self.weight, self.bias, self.normalized_shape, self.eps) + else: + return fused_layer_norm(input, self.normalized_shape, self.eps) + + def extra_repr(self): + return "{normalized_shape}, eps={eps}, " "elementwise_affine={elementwise_affine}".format(**self.__dict__) + + +class FusedRMSNorm(torch.nn.Module): + r"""Applies RMS Normalization over a mini-batch of inputs + + Currently only runs on cuda() tensors. + + .. math:: + y = \frac{x}{\mathrm{RMS}[x]} * \gamma + + The root-mean-square is calculated separately over the last + certain number dimensions which have to be of the shape specified by + :attr:`normalized_shape`. + :math:`\gamma` is a learnable affine transform parameter of + :attr:`normalized_shape` if :attr:`elementwise_affine` is ``True``. + `epsilon` is added to the mean-square, then the root of the sum is taken. + + .. note:: + Unlike Batch Normalization and Instance Normalization, which applies + scalar scale and bias for each entire channel/plane with the + :attr:`affine` option, RMS Normalization applies per-element scale + with :attr:`elementwise_affine`. + + This layer uses statistics computed from input data in both training and + evaluation modes. + + Args: + normalized_shape (int or list or torch.Size): input shape from an expected input + of size + + .. math:: + [* \times \text{normalized}\_\text{shape}[0] \times \text{normalized}\_\text{shape}[1] + \times \ldots \times \text{normalized}\_\text{shape}[-1]] + + If a single integer is used, it is treated as a singleton list, and this module will + normalize over the last dimension which is expected to be of that specific size. + eps: a value added to the denominator for numerical stability. Default: 1e-5 + elementwise_affine: a boolean value that when set to ``True``, this module + has learnable per-element affine parameters initialized to ones (for weights) + and zeros (for biases). Default: ``True``. + + Shape: + - Input: :math:`(N, *)` + - Output: :math:`(N, *)` (same shape as input) + + Examples:: + + >>> input = torch.randn(20, 5, 10, 10) + >>> # With Learnable Parameters + >>> m = apex.normalization.FusedRMSNorm(input.size()[1:]) + >>> # Without Learnable Parameters + >>> m = apex.normalization.FusedRMSNorm(input.size()[1:], elementwise_affine=False) + >>> # Normalize over last two dimensions + >>> m = apex.normalization.FusedRMSNorm([10, 10]) + >>> # Normalize over last dimension of size 10 + >>> m = apex.normalization.FusedRMSNorm(10) + >>> # Activating the module + >>> output = m(input) + + .. _`Root Mean Square Layer Normalization`: https://arxiv.org/pdf/1910.07467.pdf + """ + + def __init__(self, normalized_shape, eps=1e-5, elementwise_affine=True): + super().__init__() + + global fused_layer_norm_cuda + fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda") + + if isinstance(normalized_shape, numbers.Integral): + normalized_shape = (normalized_shape,) + self.normalized_shape = torch.Size(normalized_shape) + self.eps = eps + self.elementwise_affine = elementwise_affine + if self.elementwise_affine: + self.weight = Parameter(torch.empty(*normalized_shape)) + else: + self.register_parameter("weight", None) + self.reset_parameters() + + def reset_parameters(self): + if self.elementwise_affine: + init.ones_(self.weight) + + def forward(self, input): + if not input.is_cuda: + return manual_rms_norm(input, self.normalized_shape, self.weight, self.eps) + + if self.elementwise_affine: + return fused_rms_norm_affine(input, self.weight, self.normalized_shape, self.eps) + else: + return fused_rms_norm(input, self.normalized_shape, self.eps) + + def extra_repr(self): + return "{normalized_shape}, eps={eps}, " "elementwise_affine={elementwise_affine}".format(**self.__dict__) + + +# NOTE (mkozuki): Why "mixed"? +# MixedFusedLayerNorm differs from FusedLayerNorm in that this layer norm uses parameter's dtype +# as output tensor's dtype while FusedLayerNorm uses input tensor's dtype for output tensor's dtype. +# See: `layer_norm_affine` and `layer_norm_affine_mixed_dtypes` in "csrc/layer_norm_cuda.cpp" +class MixedFusedLayerNorm(FusedLayerNorm): + + def __init__(self, normalized_shape, eps=1e-5, **kwargs): + if "elementwise_affine" in kwargs: + import warnings + warnings.warn("MixedFusedLayerNorm does not support `elementwise_affine` argument") + elementwise_affine = kwargs.pop("elementwise_affine") + if not elementwise_affine: + raise RuntimeError("MixedFusedLayerNorm does not support `elementwise_affine = False`") + + super().__init__(normalized_shape=normalized_shape, eps=eps, elementwise_affine=True) + + def forward(self, input: torch.Tensor): + # NOTE (mkozuki): CPU path is here mainly for unittest sake. + if not input.is_cuda: + return F.layer_norm(input, self.normalized_shape, self.weight, self.bias, self.eps) + return mixed_dtype_fused_layer_norm_affine(input, self.weight, self.bias, self.normalized_shape, self.eps) + + +# MixedFusedLayerNorm differs from FusedLayerNorm in that this layer norm uses parameter's dtype +# as output tensor's dtype while FusedLayerNorm uses input tensor's dtype for output tensor's dtype. +# See: `layer_norm_affine` and `layer_norm_affine_mixed_dtypes` in "csrc/layer_norm_cuda.cpp" +class MixedFusedRMSNorm(FusedRMSNorm): + + def __init__(self, normalized_shape, eps=1e-5, **kwargs): + if "elementwise_affine" in kwargs: + import warnings + warnings.warn("MixedFusedRMSNorm does not support `elementwise_affine` argument") + elementwise_affine = kwargs.pop("elementwise_affine") + if not elementwise_affine: + raise RuntimeError("MixedFusedRMSNorm does not support `elementwise_affine = False`") + + super().__init__(normalized_shape=normalized_shape, eps=eps, elementwise_affine=True) + + def forward(self, input: torch.Tensor): + # NOTE (mkozuki): CPU path is here mainly for unittest sake. + # TODO Manual RMS Norm Implementation Here + if not input.is_cuda: + return manual_rms_norm(input, self.normalized_shape, self.weight, self.eps) + return mixed_dtype_fused_rms_norm_affine(input, self.weight, self.normalized_shape, self.eps) diff --git a/apex/apex/normalization/instance_norm.py b/apex/apex/normalization/instance_norm.py new file mode 100644 index 00000000..ce76ea6d --- /dev/null +++ b/apex/apex/normalization/instance_norm.py @@ -0,0 +1,151 @@ +import importlib + +import torch +from torch import Tensor +from torch.nn.modules.batchnorm import _NormBase + +global instance_norm_nvfuser_cuda +instance_norm_nvfuser_cuda = None + +class InstanceNormNVFuserFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, input, weight, bias, running_mean, running_var, + use_input_stats, momentum, eps): + global instance_norm_nvfuser_cuda + if instance_norm_nvfuser_cuda is None: + instance_norm_nvfuser_cuda = importlib.import_module("instance_norm_nvfuser_cuda") + + channels_last = input.is_contiguous(memory_format=torch.channels_last) or input.is_contiguous(memory_format=torch.channels_last_3d) + if channels_last: + order = [0] + [i for i in range(2, len(input.shape))] + [1] + _input = input.permute(order) + else: + _input = input + assert _input.is_contiguous() + result = instance_norm_nvfuser_cuda.forward(_input, weight, bias, running_mean, running_var, + use_input_stats, momentum, eps, channels_last) + if len(result) == 3: + out, mean, invstd = result + else: + running_mean, running_var, out, mean, invstd = result + ctx.use_input_stats = use_input_stats + ctx.eps = eps + ctx.channels_last = channels_last + # saving for backward in "explicit channels-last format" + ctx.save_for_backward(_input, weight, running_mean, running_var, mean, invstd) + if channels_last: + order = [0, len(_input.shape) - 1] + [i for i in range(1, len(_input.shape) - 1)] + out = out.permute(order) + if len(out.shape) == 4: + assert out.is_contiguous(memory_format=torch.channels_last) + assert input.is_contiguous(memory_format=torch.channels_last) + elif len(out.shape) == 5: + assert out.is_contiguous(memory_format=torch.channels_last_3d) + assert input.is_contiguous(memory_format=torch.channels_last_3d) + else: + assert False, "unhandled channels_last format variation in forward" + return out + + @staticmethod + def backward(ctx, grad_output): + global instance_norm_nvfuser_cuda + if instance_norm_nvfuser_cuda is None: + instance_norm_nvfuser_cuda = importlib.import_module("instance_norm_nvfuser_cuda") + + if ctx.channels_last: + order = [0] + [i for i in range(2, len(grad_output.shape))] + [1] + grad_output = grad_output.permute(order) + # input was saved in "explicit channels-last format" + assert ctx.saved_tensors[0].is_contiguous() + grad_output = grad_output.contiguous() + saved = list(ctx.saved_tensors) + saved.insert(1, grad_output) + running_mean = saved[3] + running_var = saved[4] + mean = saved[-2] + var = saved[-1] + grad_input, grad_weight, grad_bias = instance_norm_nvfuser_cuda.backward(*saved, ctx.use_input_stats, ctx.eps, ctx.channels_last) + if ctx.channels_last: + order = [0, len(grad_input.shape) - 1] + [i for i in range(1, len(grad_input.shape) - 1)] + grad_input = grad_input.permute(order) + if len(grad_input.shape) == 4: + assert grad_input.is_contiguous(memory_format=torch.channels_last) + elif len(grad_input.shape) == 5: + assert grad_input.is_contiguous(memory_format=torch.channels_last_3d) + else: + assert False, "unhandled channels_last format variation in backward" + return grad_input, grad_weight, grad_bias, None, None, None, None, None, None + + +class _InstanceNormNVFuser(_NormBase): + def __init__( + self, + num_features: int, + eps: float = 1e-5, + momentum: float = 0.1, + affine: bool = False, + track_running_stats: bool = False, + device=None, + dtype=None + ) -> None: + factory_kwargs = {'device': device, 'dtype': dtype} + super(_InstanceNormNVFuser, self).__init__( + num_features, eps, momentum, affine, track_running_stats, **factory_kwargs) + self.dummy = torch.empty([], device=device) + + def _check_input_dim(self, input): + raise NotImplementedError + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + version = local_metadata.get('version', None) + # at version 1: removed running_mean and running_var when + # track_running_stats=False (default) + if version is None and not self.track_running_stats: + running_stats_keys = [] + for name in ('running_mean', 'running_var'): + key = prefix + name + if key in state_dict: + running_stats_keys.append(key) + if len(running_stats_keys) > 0: + error_msgs.append( + 'Unexpected running stats buffer(s) {names} for {klass} ' + 'with track_running_stats=False. If state_dict is a ' + 'checkpoint saved before 0.4.0, this may be expected ' + 'because {klass} does not track running stats by default ' + 'since 0.4.0. Please remove these keys from state_dict. If ' + 'the running stats are actually needed, instead set ' + 'track_running_stats=True in {klass} to enable them. See ' + 'the documentation of {klass} for details.' + .format(names=" and ".join('"{}"'.format(k) for k in running_stats_keys), + klass=self.__class__.__name__)) + for key in running_stats_keys: + state_dict.pop(key) + + super(_InstanceNormNVFuser, self)._load_from_state_dict( + state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs) + + def forward(self, input: Tensor) -> Tensor: + assert input.is_cuda, "NVFuser InstanceNorm is CUDA only" + self._check_input_dim(input) + if self.dummy.device != input.device: + self.dummy = torch.empty([], device=input.device) + if self.running_mean is not None: + out = InstanceNormNVFuserFunction.apply( + input, self.weight if self.weight is not None else self.dummy, + self.bias if self.bias is not None else self.dummy, self.running_mean, self.running_var, + self.training or not self.track_running_stats, self.momentum, self.eps) + else: + out = InstanceNormNVFuserFunction.apply( + input, self.weight if self.weight is not None else self.dummy, + self.bias if self.bias is not None else self.dummy, self.dummy, self.dummy, + self.training or not self.track_running_stats, self.momentum, self.eps) + return out + +class InstanceNorm3dNVFuser(_InstanceNormNVFuser): + def _check_input_dim(self, input): + if input.dim() != 5: + raise ValueError('expected 5D input (got {}D input)' + .format(input.dim())) + diff --git a/apex/apex/optimizers/__init__.py b/apex/apex/optimizers/__init__.py new file mode 100644 index 00000000..25c178c5 --- /dev/null +++ b/apex/apex/optimizers/__init__.py @@ -0,0 +1,6 @@ +from .fused_sgd import FusedSGD +from .fused_adam import FusedAdam +from .fused_novograd import FusedNovoGrad +from .fused_lamb import FusedLAMB +from .fused_adagrad import FusedAdagrad +from .fused_mixed_precision_lamb import FusedMixedPrecisionLamb diff --git a/apex/apex/optimizers/fused_adagrad.py b/apex/apex/optimizers/fused_adagrad.py new file mode 100644 index 00000000..d72a68c5 --- /dev/null +++ b/apex/apex/optimizers/fused_adagrad.py @@ -0,0 +1,122 @@ +import torch +from apex.multi_tensor_apply import multi_tensor_applier + + +class FusedAdagrad(torch.optim.Optimizer): + """Implements Adagrad algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``. + + This version of fused Adagrad implements 2 fusions. + * Fusion of the Adagrad update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.optimizers.FusedAdagrad`'s usage is identical to any ordinary Pytorch optimizer:: + opt = apex.optimizers.FusedAdagrad(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedAdagrad` may be used with or without Amp. If you wish to use :class:`FusedAdagrad` with Amp, + you may choose any ``opt_level``:: + opt = apex.optimizers.FusedAdagrad(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + In general, ``opt_level="O1"`` is recommended. + + It has been proposed in `Adaptive Subgradient Methods for Online Learning + and Stochastic Optimization`_. + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-2) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-10) + adagrad_w_mode (boolean, optional): Apply L2 regularization or weight decay + True for decoupled weight decay (also known as AdamW) (default: False) + + .. _Adaptive Subgradient Methods for Online Learning and Stochastic + Optimization: http://jmlr.org/papers/v12/duchi11a.html + """ + def __init__(self, params, lr=1e-2, eps=1e-10, + weight_decay=0., set_grad_none=True, adagrad_w_mode=False): + + defaults = dict(lr=lr, eps=eps, weight_decay=weight_decay) + super(FusedAdagrad, self).__init__(params, defaults) + self.adagrad_w_mode = 1 if adagrad_w_mode else 0 + self.set_grad_none = set_grad_none + + if multi_tensor_applier.available: + import amp_C + # Skip buffer + self._dummy_overflow_buf = torch.cuda.IntTensor([0]) + self.multi_tensor_adagrad = amp_C.multi_tensor_adagrad + else: + raise RuntimeError('apex.optimizers.FusedAdagrad requires cuda extensions') + + def zero_grad(self): + if self.set_grad_none: + for group in self.param_groups: + for p in group['params']: + p.grad = None + else: + super(FusedAdagrad, self).zero_grad() + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + # create lists for multi-tensor apply + g_16, p_16, h_16 = [], [], [] + g_32, p_32, h_32 = [], [], [] + + for p in group['params']: + if p.grad is None: + continue + if p.grad.data.is_sparse: + raise RuntimeError('FusedAdagrad does not support sparse gradients') + + state = self.state[p] + # State initialization + if len(state) == 0: + # Exponential moving average of gradient values + state['sum'] = torch.zeros_like(p.data) + if p.dtype == torch.float16: + g_16.append(p.grad.data) + p_16.append(p.data) + h_16.append(state['sum']) + elif p.dtype == torch.float32: + g_32.append(p.grad.data) + p_32.append(p.data) + h_32.append(state['sum']) + else: + raise RuntimeError('FusedAdagrad only support fp16 and fp32.') + + if(len(g_16) > 0): + multi_tensor_applier(self.multi_tensor_adagrad, + self._dummy_overflow_buf, + [g_16, p_16, h_16], + group['lr'], + group['eps'], + self.adagrad_w_mode, + group['weight_decay']) + if(len(g_32) > 0): + multi_tensor_applier(self.multi_tensor_adagrad, + self._dummy_overflow_buf, + [g_32, p_32, h_32], + group['lr'], + group['eps'], + self.adagrad_w_mode, + group['weight_decay']) + + return loss \ No newline at end of file diff --git a/apex/apex/optimizers/fused_adam.py b/apex/apex/optimizers/fused_adam.py new file mode 100644 index 00000000..841d5634 --- /dev/null +++ b/apex/apex/optimizers/fused_adam.py @@ -0,0 +1,305 @@ +import torch +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedAdam(torch.optim.Optimizer): + + """Implements Adam algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``. + + This version of fused Adam implements 2 fusions. + + * Fusion of the Adam update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.optimizers.FusedAdam` may be used as a drop-in replacement for ``torch.optim.AdamW``, + or ``torch.optim.Adam`` with ``adam_w_mode=False``:: + + opt = apex.optimizers.FusedAdam(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedAdam` may be used with or without Amp. If you wish to use :class:`FusedAdam` with Amp, + you may choose any ``opt_level``:: + + opt = apex.optimizers.FusedAdam(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + + In general, ``opt_level="O1"`` is recommended. + + + .. warning:: + A previous version of :class:`FusedAdam` allowed a number of additional arguments to ``step``. These additional arguments + are now deprecated and unnecessary. + + Adam was been proposed in `Adam: A Method for Stochastic Optimization`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) NOT SUPPORTED in FusedAdam! + adam_w_mode (boolean, optional): Apply L2 regularization or weight decay + True for decoupled weight decay(also known as AdamW) (default: True) + set_grad_none (bool, optional): whether set grad to None when zero_grad() + method is called. (default: True) + capturable (bool, optional): whether to use the version of the optimizer + that can be used with CUDA Graphs. (default: False) + master_weights (bool, optional): whether to maintain FP32 master weights + in the optimizer with FP16 mixed precision training, currently can + only be used with capturable set to True. (default: False) + + .. _Adam - A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__(self, params, lr=1e-3, bias_correction=True, + betas=(0.9, 0.999), eps=1e-8, adam_w_mode=True, + weight_decay=0., amsgrad=False, set_grad_none=True, + capturable=False, master_weights=False): + + if amsgrad: + raise RuntimeError('FusedAdam does not support the AMSGrad variant.') + if master_weights and not capturable: + raise RuntimeError('Master weights is currently only supported with the capturable version.') + # If the optimizer is capturable then LR should be a tensor (on GPU) + lr = torch.tensor(lr, dtype=torch.float32) if capturable else lr + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay) + super(FusedAdam, self).__init__(params, defaults) + self.adam_w_mode = 1 if adam_w_mode else 0 + self.set_grad_none = set_grad_none + + self.capturable = capturable + self.master_weights = master_weights + + # Create full precision master weights + self.param_groups_master = [] + for i, pg in enumerate(self.param_groups): + param_list = pg['params'] + self.param_groups_master.append({ + 'params': [ + p.clone().detach().float() if self.master_weights else None + for p in param_list + ], + }) + + if capturable: + for idx, group in enumerate(self.param_groups): + if len(group['params']) == 0: + continue + device = group['params'][0].device + for item in ['lr']: + self.param_groups[idx][item] = group[item].to(device=device) + + self._step_supports_amp_scaling = True + + if multi_tensor_applier.available: + import amp_C + # Skip buffer + self._dummy_overflow_buf = torch.cuda.IntTensor([0]) + self.multi_tensor_adam = amp_C.multi_tensor_adam + self.multi_tensor_adam_capturable = amp_C.multi_tensor_adam_capturable + self.multi_tensor_adam_capturable_master = amp_C.multi_tensor_adam_capturable_master + else: + raise RuntimeError('apex.optimizers.FusedAdam requires cuda extensions') + + def zero_grad(self): + if self.set_grad_none: + for group in self.param_groups: + for p in group['params']: + p.grad = None + else: + super(FusedAdam, self).zero_grad() + + def step(self, closure=None, grads=None, output_params=None, scale=None, grad_norms=None, grad_scaler=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + + The remaining arguments are deprecated, and are only retained (for the moment) for error-checking purposes. + """ + if any(p is not None for p in [grads, output_params, scale, grad_norms]): + raise RuntimeError('FusedAdam has been updated. Simply initialize it identically to torch.optim.Adam, and call step() with no arguments.') + loss = None + if closure is not None: + loss = closure() + + for group, group_master in zip(self.param_groups, self.param_groups_master): + if len(group['params']) == 0: + continue + device = group['params'][0].device + bias_correction = 1 if group['bias_correction'] else 0 + beta1, beta2 = group['betas'] + + # assume same step across group now to simplify things + # per parameter step can be easily support by making it tensor, or pass list into kernel + if 'step' in group: + group['step'] += 1 if not self.capturable else (self._dummy_overflow_buf != 1).to(torch.int) + else: + group['step'] = 1 if not self.capturable else torch.tensor([1], dtype=torch.int, device=device) + + # create lists for multi-tensor apply + g_16, p_16, m_16, v_16 = [], [], [], [] + g_bf, p_bf, m_bf, v_bf = [], [], [], [] + g_32, p_32, m_32, v_32 = [], [], [], [] + p_16_master = [] + p_32_master = [] + + for p, p_master in zip(group['params'], group_master['params']): + if p.grad is None: + continue + if p.grad.data.is_sparse: + raise RuntimeError('FusedAdam does not support sparse gradients, please consider SparseAdam instead') + + state = self.state[p] + # State initialization + if len(state) == 0: + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data).float() + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data).float() + + if p.dtype == torch.float16: + if self.master_weights: + p_16_master.append(p_master.data) + g_16.append(p.grad.data) + p_16.append(p.data) + m_16.append(state['exp_avg']) + v_16.append(state['exp_avg_sq']) + elif p.dtype == torch.bfloat16: + g_bf.append(p.grad) + p_bf.append(p) + m_bf.append(state['exp_avg']) + v_bf.append(state['exp_avg_sq']) + elif p.dtype == torch.float32: + if self.master_weights: + p_32_master.append(p_master.data) + g_32.append(p.grad.data) + p_32.append(p.data) + m_32.append(state['exp_avg']) + v_32.append(state['exp_avg_sq']) + else: + raise RuntimeError('FusedAdam only support fp16 and fp32.') + + # If the optimizer is capturable, then if there's a grad scaler it works + # on the GPU + a different multi_tensor_applier should be called + if self.capturable: + # overflow check of gradients + found_inf = ( + grad_scaler._check_inf_per_device(self)[device] + if grad_scaler is not None else torch.zeros((1,), device=device) + ) + self._dummy_overflow_buf.copy_(found_inf) + + # get unscale scale factor + scale, inv_scale = None, None + if grad_scaler: + scale = grad_scaler._get_scale_async() + inv_scale = scale.double().reciprocal().float() + else: + scale = torch.ones((1,), device=device) + inv_scale = torch.ones((1,), device=device) + + if len(g_16) > 0: + multi_tensor_applier(self.multi_tensor_adam_capturable_master if self.master_weights + else self.multi_tensor_adam_capturable, + self._dummy_overflow_buf, + [g_16, p_16, m_16, v_16, p_16_master] if self.master_weights + else [g_16, p_16, m_16, v_16], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + self.adam_w_mode, + bias_correction, + group['weight_decay'], + inv_scale) + + if len(g_bf) > 0: + multi_tensor_applier( + self.multi_tensor_adam_capturable, + self._dummy_overflow_buf, + [g_bf, p_bf, m_bf, v_bf], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + self.adam_w_mode, + bias_correction, + group['weight_decay'], + inv_scale) + + if len(g_32) > 0: + multi_tensor_applier(self.multi_tensor_adam_capturable_master if self.master_weights + else self.multi_tensor_adam_capturable, + self._dummy_overflow_buf, + [g_32, p_32, m_32, v_32, p_32_master] if self.master_weights + else [g_32, p_32, m_32, v_32], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + self.adam_w_mode, + bias_correction, + group['weight_decay'], + inv_scale) + else: + if len(g_16) > 0: + multi_tensor_applier(self.multi_tensor_adam, + self._dummy_overflow_buf, + [g_16, p_16, m_16, v_16], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + self.adam_w_mode, + bias_correction, + group['weight_decay']) + + if len(g_bf) > 0: + multi_tensor_applier( + self.multi_tensor_adam, + self._dummy_overflow_buf, + [g_bf, p_bf, m_bf, v_bf], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + self.adam_w_mode, + bias_correction, + group['weight_decay']) + + if len(g_32) > 0: + multi_tensor_applier(self.multi_tensor_adam, + self._dummy_overflow_buf, + [g_32, p_32, m_32, v_32], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + self.adam_w_mode, + bias_correction, + group['weight_decay']) + + return loss diff --git a/apex/apex/optimizers/fused_lamb.py b/apex/apex/optimizers/fused_lamb.py new file mode 100644 index 00000000..854525dc --- /dev/null +++ b/apex/apex/optimizers/fused_lamb.py @@ -0,0 +1,215 @@ +import torch +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedLAMB(torch.optim.Optimizer): + + """Implements LAMB algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``. + + This version of fused LAMB implements 2 fusions. + + * Fusion of the LAMB update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.optimizers.FusedLAMB`'s usage is identical to any ordinary Pytorch optimizer:: + + opt = apex.optimizers.FusedLAMB(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedLAMB` may be used with or without Amp. If you wish to use :class:`FusedLAMB` with Amp, + you may choose any ``opt_level``:: + + opt = apex.optimizers.FusedLAMB(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + + In general, ``opt_level="O1"`` is recommended. + + LAMB was proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its norm. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + NOT SUPPORTED now! (default: False) + adam_w_mode (boolean, optional): Apply L2 regularization or weight decay + True for decoupled weight decay(also known as AdamW) (default: True) + grad_averaging (bool, optional): whether apply (1-beta2) to grad when + calculating running averages of gradient. (default: True) + set_grad_none (bool, optional): whether set grad to None when zero_grad() + method is called. (default: True) + max_grad_norm (float, optional): value used to clip global grad norm + (default: 1.0) + use_nvlamb (boolean, optional): Apply adaptive learning rate to 0.0 + weight decay parameter (default: False) + + .. _Large Batch Optimization for Deep Learning - Training BERT in 76 minutes: + https://arxiv.org/abs/1904.00962 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__(self, params, lr=1e-3, bias_correction=True, + betas=(0.9, 0.999), eps=1e-6, weight_decay=0.01, + amsgrad=False, adam_w_mode=True, + grad_averaging=True, set_grad_none=True, + max_grad_norm=1.0, use_nvlamb=False): + if amsgrad: + raise RuntimeError('FusedLAMB does not support the AMSGrad variant.') + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay, + grad_averaging=grad_averaging, + max_grad_norm=max_grad_norm) + super(FusedLAMB, self).__init__(params, defaults) + if multi_tensor_applier.available: + import amp_C + self.multi_tensor_l2norm=amp_C.multi_tensor_l2norm + # Skip buffer + self._dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device=self.param_groups[0]["params"][0].device) + self.multi_tensor_lamb = amp_C.multi_tensor_lamb + else: + raise RuntimeError('apex.optimizers.FusedLAMB requires cuda extensions') + + self.adam_w_mode = 1 if adam_w_mode else 0 + self.set_grad_none = set_grad_none + self.use_nvlamb = use_nvlamb + + def zero_grad(self): + if self.set_grad_none: + for group in self.param_groups: + for p in group['params']: + p.grad = None + else: + super(FusedLAMB, self).zero_grad() + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + # create separate grad lists for fp32 and fp16 params + g_all_32, g_all_16 = [], [] + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + if p.dtype == torch.float32: + g_all_32.append(p.grad.data) + elif p.dtype == torch.float16: + g_all_16.append(p.grad.data) + else: + raise RuntimeError('FusedLAMB only support fp16 and fp32.') + + device = self.param_groups[0]["params"][0].device + g_norm_32, g_norm_16 = torch.zeros(1, device=device), torch.zeros(1, device=device) + # compute grad norm for two lists + if len(g_all_32) > 0: + g_norm_32 = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [g_all_32], False)[0] + if len(g_all_16) > 0: + g_norm_16 = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [g_all_16], False)[0] + + # blend two grad norms to get global grad norm + global_grad_norm = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [[g_norm_32, g_norm_16]], + False)[0] + max_grad_norm = self.defaults['max_grad_norm'] + + for group in self.param_groups: + bias_correction = 1 if group['bias_correction'] else 0 + beta1, beta2 = group['betas'] + grad_averaging = 1 if group['grad_averaging'] else 0 + + # assume same step across group now to simplify things + # per parameter step can be easily support by making it tensor, or pass list into kernel + if 'step' in group: + group['step'] += 1 + else: + group['step'] = 1 + + # create lists for multi-tensor apply + g_16, p_16, m_16, v_16 = [], [], [], [] + g_32, p_32, m_32, v_32 = [], [], [], [] + + for p in group['params']: + if p.grad is None: + continue + if p.grad.data.is_sparse: + raise RuntimeError('FusedLAMB does not support sparse gradients, please consider SparseAdam instead') + + state = self.state[p] + # State initialization + if len(state) == 0: + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + if p.dtype == torch.float16: + g_16.append(p.grad.data) + p_16.append(p.data) + m_16.append(state['exp_avg']) + v_16.append(state['exp_avg_sq']) + elif p.dtype == torch.float32: + g_32.append(p.grad.data) + p_32.append(p.data) + m_32.append(state['exp_avg']) + v_32.append(state['exp_avg_sq']) + else: + raise RuntimeError('FusedLAMB only support fp16 and fp32.') + + if(len(g_16) > 0): + multi_tensor_applier(self.multi_tensor_lamb, + self._dummy_overflow_buf, + [g_16, p_16, m_16, v_16], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.adam_w_mode, + global_grad_norm, + max_grad_norm, + self.use_nvlamb) + if(len(g_32) > 0): + multi_tensor_applier(self.multi_tensor_lamb, + self._dummy_overflow_buf, + [g_32, p_32, m_32, v_32], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.adam_w_mode, + global_grad_norm, + max_grad_norm, + self.use_nvlamb) + + return loss diff --git a/apex/apex/optimizers/fused_mixed_precision_lamb.py b/apex/apex/optimizers/fused_mixed_precision_lamb.py new file mode 100644 index 00000000..f1b2902c --- /dev/null +++ b/apex/apex/optimizers/fused_mixed_precision_lamb.py @@ -0,0 +1,256 @@ +import torch +from copy import deepcopy +from itertools import chain +from collections import defaultdict, abc as container_abcs + +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedMixedPrecisionLamb(torch.optim.Optimizer): + + def __init__(self, params, lr=1e-3, step=0, bias_correction=True, + betas=(0.9, 0.999), eps=1e-6, weight_decay=0.01, + amsgrad=False, adam_w_mode=True, + grad_averaging=True, max_grad_norm=1.0, use_nvlamb=False, + reduced_precision_dtype=None): + if amsgrad: + raise RuntimeError('FusedLAMB does not support the AMSGrad variant.') + + # The learning rate (lr) and optimizer step (step) should be located on device + # in order to faciliated device sync free execution + defaults = dict(lr=torch.tensor(lr, dtype=torch.float32), + step=torch.tensor([step], dtype=torch.int), + bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay, + grad_averaging=grad_averaging, + max_grad_norm=max_grad_norm) + tensor_state = ['lr', 'step'] + super(FusedMixedPrecisionLamb, self).__init__(params, defaults) + + device = self.param_groups[0]['params'][0].device + + for idx,group in enumerate(self.param_groups): + for item in tensor_state: + self.param_groups[idx][item] = group[item].to(device=device) + + if multi_tensor_applier.available: + import amp_C + self.multi_tensor_l2norm=amp_C.multi_tensor_l2norm_mp + # Skip buffer + self._dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device=device) + self.multi_tensor_lamb = amp_C.multi_tensor_lamb_mp + else: + raise RuntimeError('apex.optimizers.FusedLAMB requires cuda extensions') + + # Mixed Precision support + self.reduced_precision_dtype = reduced_precision_dtype + self.param_groups_full_precision = [] + + self._step_supports_amp_scaling = True + self.adam_w_mode = 1 if adam_w_mode else 0 + self.use_nvlamb = use_nvlamb + + # This method is overridden from the parent class because there is not a way to override + # the nested function cast() that copies a saved piece of state to the device without + # redundantly doing the copy. + def load_state_dict(self, state_dict): + r"""Loads the optimizer state. + + Args: + state_dict (dict): optimizer state. Should be an object returned + from a call to :meth:`state_dict`. + """ + # deepcopy, to be consistent with module API + state_dict = deepcopy(state_dict) + # Validate the state_dict + groups = self.param_groups + saved_groups = state_dict['param_groups'] + + if len(groups) != len(saved_groups): + raise ValueError("loaded state dict has a different number of " + "parameter groups") + param_lens = (len(g['params']) for g in groups) + saved_lens = (len(g['params']) for g in saved_groups) + if any(p_len != s_len for p_len, s_len in zip(param_lens, saved_lens)): + raise ValueError("loaded state dict contains a parameter group " + "that doesn't match the size of optimizer's group") + + # Update the state + id_map = {old_id: p for old_id, p in + zip(chain.from_iterable((g['params'] for g in saved_groups)), + chain.from_iterable((g['params'] for g in groups)))} + + def cast(param, value): + r"""Make a deep copy of value, casting all tensors to device of param.""" + if isinstance(value, torch.Tensor): + # The original version casted the saved value to the params dtype + # This doesn't work for mixed precision Lamb where the momentum and + # velocity are expected to be in full precision while the params are + # in reduced precision + value = value.to(value.device) + return value + elif isinstance(value, dict): + return {k: cast(param, v) for k, v in value.items()} + elif isinstance(value, container_abcs.Iterable): + return type(value)(cast(param, v) for v in value) + else: + return value + + # Copy state assigned to params (and cast tensors to appropriate types). + # State that is not assigned to params is copied as is (needed for + # backward compatibility). + state = defaultdict(dict) + for k, v in state_dict['state'].items(): + if k in id_map: + param = id_map[k] + state[param] = cast(param, v) + else: + state[k] = v + + # Update parameter groups, setting their 'params' value + def update_group(group, new_group): + new_group['params'] = group['params'] + return new_group + param_groups = [ + update_group(g, ng) for g, ng in zip(groups, saved_groups)] + self.__setstate__({'state': state, 'param_groups': param_groups}) + + def _setup_full_precision_params(self): + for i, pg in enumerate(self.param_groups): + param_list = pg['params'] + self.param_groups_full_precision.append({ + 'params': [ + p.clone().detach().to(dtype=torch.float32) + if (self.reduced_precision_dtype is not None) and (p.dtype == self.reduced_precision_dtype) + else None + for p in param_list + ], + }) + + # add_param_groups() is overridden because default items can be tensors. The + # parent version does not clone the default item, so two param groups can + # accidentally point to the same default item value where they can differ + # given they are in separate groups. + def add_param_group(self, param_group): + super().add_param_group(param_group) + for name, default in self.defaults.items(): + if isinstance(default, torch.Tensor): + self.param_groups[len(self.param_groups) - 1][name] = default.clone() + + @torch.no_grad() + def step(self, closure=None, grad_scaler=None): + loss = None + if closure is not None: + loss = closure() + + # The full precision params are set up in the first step of the optimizer + # instead of in the constructor because the full precision params will get out + # out of sync with the model params if DDP syncs the model params across devices + # after the optimizer is constructed. + if len(self.param_groups_full_precision) == 0 : + self._setup_full_precision_params() + + # create separate grad lists for params + grad_list = [] + for gid,group in enumerate(self.param_groups): + for pid,p in enumerate(group['params']): + assert group['params'][0].dtype == p.dtype, \ + "Error: Parameters are not of the identical type: {} != {}".format( + group['params'][0].dtype, p.dtype) + if p.grad is None: + continue + grad_list.append(p.grad) + + # Overflow check of gradients + device = self.param_groups[0]["params"][0].device + found_inf = ( + grad_scaler._check_inf_per_device(self)[device] + if grad_scaler is not None else torch.zeros((1,), device=device) + ) + self._dummy_overflow_buf.copy_(found_inf) + + # Get unscale scale factor + scale, inv_scale = None, None + if grad_scaler: + scale = grad_scaler._get_scale_async() + inv_scale = scale.double().reciprocal().float() + else: + scale = torch.ones((1,), device=device) + inv_scale = torch.ones((1,), device=device) + + # grad_norm is of scaled gradients. + # So, multiply `max_grad_norm` by scale. + max_grad_norm = self.defaults['max_grad_norm'] * scale + grad_norm = multi_tensor_applier( + self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [grad_list], + False, + )[0] + + # Run LAMB optimization math + for gid, (group, group_full) in enumerate(zip(self.param_groups, self.param_groups_full_precision)): + bias_correction = 1 if group['bias_correction'] else 0 + beta1, beta2 = group['betas'] + grad_averaging = 1 if group['grad_averaging'] else 0 + + # assume same step across group now to simplify things + # per parameter step can be easily support by making it tensor, or pass list into kernel + group['step'] += (self._dummy_overflow_buf != 1).to(torch.int) + + state_lists = [ [], # (0) grads + [], # (1) params + [], # (2) momentum state + [], # (3) velocity state + ] + if self.reduced_precision_dtype is not None: + state_lists.append([]) # (4) params reduced_dtype + + + for p, p_full in zip(group['params'], group_full['params']): + if p.grad is None: + continue + assert not p.grad.is_sparse + + state = self.state[p] + # State initialization + if len(state) == 0: + dtype = p.dtype + if self.reduced_precision_dtype is not None and p.dtype == self.reduced_precision_dtype : + dtype = torch.float32 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data, dtype=dtype) + # Exponential moving average of gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data, dtype=dtype) + + if self.reduced_precision_dtype is not None : + state_lists[0].append(p.grad.data) + state_lists[1].append(p_full.data) + state_lists[2].append(state['exp_avg']) + state_lists[3].append(state['exp_avg_sq']) + state_lists[4].append(p.data) + else : + state_lists[0].append(p.grad.data) + state_lists[1].append(p.data) + state_lists[2].append(state['exp_avg']) + state_lists[3].append(state['exp_avg_sq']) + + multi_tensor_applier( + self.multi_tensor_lamb, + self._dummy_overflow_buf, + state_lists, + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.adam_w_mode, + grad_norm, + max_grad_norm, + self.use_nvlamb, + found_inf, + inv_scale) + + return loss diff --git a/apex/apex/optimizers/fused_novograd.py b/apex/apex/optimizers/fused_novograd.py new file mode 100644 index 00000000..2820ae36 --- /dev/null +++ b/apex/apex/optimizers/fused_novograd.py @@ -0,0 +1,214 @@ +import torch +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedNovoGrad(torch.optim.Optimizer): + + """Implements NovoGrad algorithm. + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``. + + This version of fused NovoGrad implements 2 fusions. + + * Fusion of the NovoGrad update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.optimizers.FusedNovoGrad`'s usage is identical to any Pytorch optimizer:: + + opt = apex.optimizers.FusedNovoGrad(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedNovoGrad` may be used with or without Amp. If you wish to use :class:`FusedNovoGrad` with Amp, + you may choose any ``opt_level``:: + + opt = apex.optimizers.FusedNovoGrad(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + + In general, ``opt_level="O1"`` is recommended. + + It has been proposed in `Jasper: An End-to-End Convolutional Neural Acoustic Model`_. + More info: https://nvidia.github.io/OpenSeq2Seq/html/optimizers.html#novograd + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its norm. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + NOT SUPPORTED now! (default: False) + reg_inside_moment (bool, optional): whether do regularization (norm and L2) + in momentum calculation. True for include, False for not include and + only do it on update term. (default: False) + grad_averaging (bool, optional): whether apply (1-beta1) to grad when + calculating running averages of gradient. (default: True) + norm_type (int, optional): which norm to calculate for each layer. + 2 for L2 norm, and 0 for infinite norm. These 2 are only supported + type now. (default: 2) + init_zero (bool, optional): whether init norm with 0 (start averaging on + 1st step) or first step norm (start averaging on 2nd step). True for + init with 0. (default: False) + set_grad_none (bool, optional): whether set grad to None when zero_grad() + method is called. (default: True) + + .. _Jasper - An End-to-End Convolutional Neural Acoustic Model: + https://arxiv.org/abs/1904.03288 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__(self, params, lr=1e-3, bias_correction=True, + betas=(0.9, 0.999), eps=1e-8, weight_decay=0., + amsgrad=False, reg_inside_moment=False, + grad_averaging=True, norm_type=2, init_zero=False, + set_grad_none=True): + if amsgrad: + raise RuntimeError('FusedNovoGrad does not support the AMSGrad variant.') + defaults = dict(lr=lr, bias_correction=bias_correction, + betas=betas, eps=eps, weight_decay=weight_decay, + grad_averaging=grad_averaging, norm_type=norm_type, + init_zero=init_zero) + super(FusedNovoGrad, self).__init__(params, defaults) + if multi_tensor_applier.available: + import amp_C + # Skip buffer + + # Creating the overflow buffer on the same device as the params tensors. + self._dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device=self.param_groups[0]["params"][0].device) + self.multi_tensor_novograd = amp_C.multi_tensor_novograd + else: + raise RuntimeError('apex.optimizers.FusedNovoGrad requires cuda extensions') + + self.moment_mode = 0 if reg_inside_moment else 1 + self.set_grad_none = set_grad_none + + def zero_grad(self): + if self.set_grad_none: + for group in self.param_groups: + for p in group['params']: + p.grad = None + else: + super(FusedNovoGrad, self).zero_grad() + + def load_state_dict(self, state_dict): + super(FusedNovoGrad, self).load_state_dict(state_dict) + # in case exp_avg_sq is not on the same device as params, move it there + for group in self.param_groups: + if len(group['params']) > 0: + group['exp_avg_sq'][0] = group['exp_avg_sq'][0].to(group['params'][0].device) + group['exp_avg_sq'][1] = group['exp_avg_sq'][1].to(group['params'][0].device) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + bias_correction = 1 if group['bias_correction'] else 0 + beta1, beta2 = group['betas'] + grad_averaging = 1 if group['grad_averaging'] else 0 + + # assume same step across group now to simplify things + # per parameter step can be easily support by making it tensor, or pass list into kernel + if 'step' in group: + group['step'] += 1 + else: + group['step'] = 1 + + # create lists for multi-tensor apply + g_16, p_16, m_16 = [], [], [] + g_32, p_32, m_32 = [], [], [] + + for p in group['params']: + if p.grad is None: + continue + if p.grad.data.is_sparse: + raise RuntimeError('FusedNovoGrad does not support sparse gradients, please consider SparseAdam instead') + + state = self.state[p] + # State initialization + if len(state) == 0: + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + + if p.dtype == torch.float16: + g_16.append(p.grad.data) + p_16.append(p.data) + m_16.append(state['exp_avg']) + elif p.dtype == torch.float32: + g_32.append(p.grad.data) + p_32.append(p.data) + m_32.append(state['exp_avg']) + else: + raise RuntimeError('FusedNovoGrad only support fp16 and fp32.') + + # we store per weight norm as one tensor for one group/precision combination + # different from optim.Adam, we store norm here(not ^2) so we can unify calculation for norm types + if 'exp_avg_sq' not in group: + group['exp_avg_sq'] = [None, None] + if group['init_zero']: + # Creating the following parameters on the same device as the params tensors. + group['exp_avg_sq'][0] = torch.cuda.FloatTensor(len(g_16), device=self.param_groups[0]["params"][0].device).contiguous().fill_(0) + group['exp_avg_sq'][1] = torch.cuda.FloatTensor(len(g_32), device=self.param_groups[0]["params"][0].device).contiguous().fill_(0) + else: # init with first step norm, so first blend have no effect + if group['norm_type'] == 0: + v_16 = [torch.max(torch.abs(g.to(torch.float32))).item() for g in g_16] + v_32 = [torch.max(torch.abs(g)).item() for g in g_32] + elif group['norm_type'] == 2: + v_16 = [torch.sum(torch.pow(g.to(torch.float32), 2)).sqrt().item() for g in g_16] + v_32 = [torch.sum(torch.pow(g, 2)).sqrt().item() for g in g_32] + else: + raise RuntimeError('FusedNovoGrad only support l2/inf norm now.') + # Creating the following parameters on the same device as the params tensors. + group['exp_avg_sq'][0] = torch.cuda.FloatTensor(v_16, device=self.param_groups[0]["params"][0].device) + group['exp_avg_sq'][1] = torch.cuda.FloatTensor(v_32, device=self.param_groups[0]["params"][0].device) + else: + assert(len(g_16) == group['exp_avg_sq'][0].numel()) + assert(len(g_32) == group['exp_avg_sq'][1].numel()) + + if(len(g_16) > 0): + multi_tensor_applier(self.multi_tensor_novograd, + self._dummy_overflow_buf, + [g_16, p_16, m_16], + group['exp_avg_sq'][0], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.moment_mode, + group['norm_type']) + if(len(g_32) > 0): + multi_tensor_applier(self.multi_tensor_novograd, + self._dummy_overflow_buf, + [g_32, p_32, m_32], + group['exp_avg_sq'][1], + group['lr'], + beta1, + beta2, + group['eps'], + group['step'], + bias_correction, + group['weight_decay'], + grad_averaging, + self.moment_mode, + group['norm_type']) + + + return loss diff --git a/apex/apex/optimizers/fused_sgd.py b/apex/apex/optimizers/fused_sgd.py new file mode 100644 index 00000000..e7bdcb2b --- /dev/null +++ b/apex/apex/optimizers/fused_sgd.py @@ -0,0 +1,227 @@ +import torch +from torch.optim.optimizer import Optimizer, required + +from apex.multi_tensor_apply import multi_tensor_applier + +class FusedSGD(Optimizer): + r"""Implements stochastic gradient descent (optionally with momentum). + + Currently GPU-only. Requires Apex to be installed via + ``pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./``. + + This version of fused SGD implements 2 fusions. + + * Fusion of the SGD update's elementwise operations + * A multi-tensor apply launch that batches the elementwise updates applied to all the model's parameters into one or a few kernel launches. + + :class:`apex.optimizers.FusedSGD` may be used as a drop-in replacement for ``torch.optim.SGD``:: + + opt = apex.optimizers.FusedSGD(model.parameters(), lr = ....) + ... + opt.step() + + :class:`apex.optimizers.FusedSGD` may be used with or without Amp. If you wish to use :class:`FusedSGD` with Amp, + you may choose any ``opt_level``:: + + opt = apex.optimizers.FusedSGD(model.parameters(), lr = ....) + model, opt = amp.initialize(model, opt, opt_level="O0" or "O1 or "O2") + ... + opt.step() + + In general, ``opt_level="O1"`` is recommended. + + Nesterov momentum is based on the formula from + `On the importance of initialization and momentum in deep learning`__. + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float): learning rate + momentum (float, optional): momentum factor (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + dampening (float, optional): dampening for momentum (default: 0) + nesterov (bool, optional): enables Nesterov momentum (default: False) + + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + + __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + + .. note:: + The implementation of SGD with Momentum/Nesterov subtly differs from + Sutskever et. al. and implementations in some other frameworks. + + Considering the specific case of Momentum, the update can be written as + + .. math:: + v = \rho * v + g \\ + p = p - lr * v + + where p, g, v and :math:`\rho` denote the parameters, gradient, + velocity, and momentum respectively. + + This is in contrast to Sutskever et. al. and + other frameworks which employ an update of the form + + .. math:: + v = \rho * v + lr * g \\ + p = p - v + + The Nesterov version is analogously modified. + """ + + def __init__(self, params, lr=required, momentum=0, dampening=0, + weight_decay=0, nesterov=False, + wd_after_momentum=False, + materialize_master_grads=True, + set_grad_none=False): + if lr is not required and lr < 0.0: + raise ValueError("Invalid learning rate: {}".format(lr)) + if momentum < 0.0: + raise ValueError("Invalid momentum value: {}".format(momentum)) + if weight_decay < 0.0: + raise ValueError("Invalid weight_decay value: {}".format(weight_decay)) + + defaults = dict(lr=lr, momentum=momentum, dampening=dampening, + weight_decay=weight_decay, nesterov=nesterov) + if nesterov and (momentum <= 0 or dampening != 0): + raise ValueError("Nesterov momentum requires a momentum and zero dampening") + super(FusedSGD, self).__init__(params, defaults) + + self.wd_after_momentum = wd_after_momentum + self.materialize_master_grads = materialize_master_grads + self.most_recent_scale = 1.0 + self.scale_set_by_backward = False + self.set_grad_none = set_grad_none + + if multi_tensor_applier.available: + import amp_C + # Skip buffer + self._dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device=self.param_groups[0]["params"][0].device) + self.multi_tensor_sgd = amp_C.multi_tensor_sgd + else: + raise RuntimeError('apex.optimizers.FusedSGD requires cuda extensions') + + def __setstate__(self, state): + super(FusedSGD, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('nesterov', False) + + def zero_grad(self): + if self.set_grad_none: + for group in self.param_groups: + for p in group['params']: + p.grad = None + else: + super(FusedSGD, self).zero_grad() + + def get_momentums(self, params): + momentums = [] + first_run = True + for p in params: + param_state = self.state[p] + # torch.optim.SGD initializes momentum in the main loop, we have + # to do it here, and track whether or not we've done so, so that + # momentum application can be skipped in the main kernel. + if 'momentum_buffer' not in param_state: + first_run = True + buf = param_state['momentum_buffer'] = torch.zeros_like(p.data) + momentums.append(buf) + else: + first_run = False + momentums.append(param_state['momentum_buffer']) + return momentums, first_run + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + explicit_master_params = (hasattr(self, "_amp_stash") and + hasattr(self._amp_stash, "fp32_from_fp16_groups")) + + for gid, group in enumerate(self.param_groups): + weight_decay = group['weight_decay'] + momentum = group['momentum'] + dampening = group['dampening'] + nesterov = group['nesterov'] + + + # For each group, there are 3 possible combinations we need to consider: + # grad_type, param_to_update_type, momentum_type, requires_fp16_model_copy + # 1. fp16, fp16, fp16, No + # 2. fp32, fp32, fp32, No + # 3. fp16, fp32, fp32, Yes + + first_runs = [True, True] + + # I think a bit of code divergence in exchange for naming clarity is worthwhile + if explicit_master_params: + stash = self._amp_stash + + fp32_params = [p for p in stash.fp32_from_fp32_groups[gid] if p.grad is not None] + fp32_grads = [p.grad for p in stash.fp32_from_fp32_groups[gid] if p.grad is not None] + fp32_momentums, first_runs[1] = self.get_momentums(fp32_params) + + if self.materialize_master_grads: + fp16_model_params = [p for i, p in enumerate( + stash.fp16_groups[gid]) if stash.fp32_from_fp16_groups[gid][i].grad is not None] + fp32_from_fp16_grads = [p.grad for p in stash.fp32_from_fp16_groups[gid] if p.grad is not None] + fp32_from_fp16_params = [p for p in stash.fp32_from_fp16_groups[gid] if p.grad is not None] + fp32_from_fp16_momentums, first_runs[0] = self.get_momentums(fp32_from_fp16_params) + + fp16_set = [fp32_from_fp16_grads, fp32_from_fp16_params, + fp32_from_fp16_momentums, fp16_model_params] + else: + fp16_model_params = [p for p in stash.fp16_groups[gid] if p.grad is not None] + fp16_model_grads = [p.grad for p in stash.fp16_groups[gid] if p.grad is not None] + fp32_from_fp16_params = [p for i, p in enumerate( + stash.fp32_from_fp16_groups[gid]) if stash.fp16_groups[gid][i].grad is not None] + fp32_from_fp16_momentums, first_runs[0] = self.get_momentums(fp32_from_fp16_params) + + fp16_set = [fp16_model_grads, fp32_from_fp16_params, + fp32_from_fp16_momentums, fp16_model_params] + + launch_sets= [fp16_set, [fp32_grads, fp32_params, fp32_momentums]] + else: + fp16_params = [p for p in group['params'] if (p.dtype == torch.float16 and p.grad is not None)] + fp16_grads = [p.grad for p in group['params'] if (p.dtype == torch.float16 and p.grad is not None)] + fp16_momentums, first_runs[0] = self.get_momentums(fp16_params) + + fp32_params = [p for p in group['params'] if (p.dtype == torch.float32 and p.grad is not None)] + fp32_grads = [p.grad for p in group['params'] if (p.dtype == torch.float32 and p.grad is not None)] + fp32_momentums, first_runs[1] = self.get_momentums(fp32_params) + + launch_sets = [[fp16_grads, fp16_params, fp16_momentums], + [fp32_grads, fp32_params, fp32_momentums]] + + for s, (launch_set, first_run) in enumerate(zip(launch_sets, first_runs)): + assert len(launch_set[0]) == len(launch_set[1]) + assert len(launch_set[0]) == len(launch_set[2]) + if len(launch_set[0]) > 0: + multi_tensor_applier( + self.multi_tensor_sgd, + self._dummy_overflow_buf, + launch_set, + weight_decay, + momentum, + dampening, + group['lr'], + nesterov, + first_run, + self.wd_after_momentum, + 1.0/self.most_recent_scale) + + self.most_recent_scale = 1.0 + self.scale_set_by_backward = False + + return loss diff --git a/apex/apex/parallel/LARC.py b/apex/apex/parallel/LARC.py new file mode 100644 index 00000000..4a93fcd6 --- /dev/null +++ b/apex/apex/parallel/LARC.py @@ -0,0 +1,107 @@ +import torch +from torch import nn +from torch.nn.parameter import Parameter + +class LARC(object): + """ + :class:`LARC` is a pytorch implementation of both the scaling and clipping variants of LARC, + in which the ratio between gradient and parameter magnitudes is used to calculate an adaptive + local learning rate for each individual parameter. The algorithm is designed to improve + convergence of large batch training. + + See https://arxiv.org/abs/1708.03888 for calculation of the local learning rate. + + In practice it modifies the gradients of parameters as a proxy for modifying the learning rate + of the parameters. This design allows it to be used as a wrapper around any torch.optim Optimizer. + + ``` + model = ... + optim = torch.optim.Adam(model.parameters(), lr=...) + optim = LARC(optim) + ``` + + It can even be used in conjunction with apex.fp16_utils.FP16_optimizer. + + ``` + model = ... + optim = torch.optim.Adam(model.parameters(), lr=...) + optim = LARC(optim) + optim = apex.fp16_utils.FP16_Optimizer(optim) + ``` + + Args: + optimizer: Pytorch optimizer to wrap and modify learning rate for. + trust_coefficient: Trust coefficient for calculating the lr. See https://arxiv.org/abs/1708.03888 + clip: Decides between clipping or scaling mode of LARC. If `clip=True` the learning rate is set to `min(optimizer_lr, local_lr)` for each parameter. If `clip=False` the learning rate is set to `local_lr*optimizer_lr`. + eps: epsilon kludge to help with numerical stability while calculating adaptive_lr + """ + + def __init__(self, optimizer, trust_coefficient=0.02, clip=True, eps=1e-8): + self.optim = optimizer + self.trust_coefficient = trust_coefficient + self.eps = eps + self.clip = clip + + def __getstate__(self): + return self.optim.__getstate__() + + def __setstate__(self, state): + self.optim.__setstate__(state) + + @property + def state(self): + return self.optim.state + + def __repr__(self): + return self.optim.__repr__() + + @property + def param_groups(self): + return self.optim.param_groups + + @param_groups.setter + def param_groups(self, value): + self.optim.param_groups = value + + def state_dict(self): + return self.optim.state_dict() + + def load_state_dict(self, state_dict): + self.optim.load_state_dict(state_dict) + + def zero_grad(self): + self.optim.zero_grad() + + def add_param_group(self, param_group): + self.optim.add_param_group( param_group) + + def step(self): + with torch.no_grad(): + weight_decays = [] + for group in self.optim.param_groups: + # absorb weight decay control from optimizer + weight_decay = group['weight_decay'] if 'weight_decay' in group else 0 + weight_decays.append(weight_decay) + group['weight_decay'] = 0 + for p in group['params']: + if p.grad is None: + continue + param_norm = torch.norm(p.data) + grad_norm = torch.norm(p.grad.data) + + if param_norm != 0 and grad_norm != 0: + # calculate adaptive lr + weight decay + adaptive_lr = self.trust_coefficient * (param_norm) / (grad_norm + param_norm * weight_decay + self.eps) + + # clip learning rate for LARC + if self.clip: + # calculation of adaptive_lr so that when multiplied by lr it equals `min(adaptive_lr, lr)` + adaptive_lr = min(adaptive_lr/group['lr'], 1) + + p.grad.data += weight_decay * p.data + p.grad.data *= adaptive_lr + + self.optim.step() + # return weight decay control to optimizer + for i, group in enumerate(self.optim.param_groups): + group['weight_decay'] = weight_decays[i] diff --git a/apex/apex/parallel/README.md b/apex/apex/parallel/README.md new file mode 100644 index 00000000..e7910d82 --- /dev/null +++ b/apex/apex/parallel/README.md @@ -0,0 +1,66 @@ +## Distributed Data Parallel + +distributed.py contains the source code for `apex.parallel.DistributedDataParallel`, a module wrapper that enables multi-process multi-GPU data parallel training optimized for NVIDIA's NCCL communication library. + +`apex.parallel.DistributedDataParallel` achieves high performance by overlapping communication with +computation in the backward pass and bucketing smaller transfers to reduce the total number of +transfers required. + +multiproc.py contains the source code for `apex.parallel.multiproc`, a launch utility that places one process on each of the node's available GPUs. + +#### [API Documentation](https://nvidia.github.io/apex/parallel.html) + +#### [Example/Walkthrough](https://github.com/NVIDIA/apex/tree/master/examples/distributed) + +#### [Imagenet example with Mixed Precision](https://github.com/NVIDIA/apex/tree/master/examples/imagenet) + +#### [Simple example with FP16_Optimizer](https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple/distributed_apex) + +### Synchronized Batch Normalization + +`apex.parallel.SyncBatchNorm` has similar APIs as with `torch.nn.BatchNorm*N*d`. +It reduces stats on the first (channel) dimension of the Tensor and accepts +arbitrary spatial dimensions. + +#### Installation + +Apex provides two sync BN implementation: + +1. There is the Python-only implementation, which is the default implementation +when install with `python setup.py install`. +It uses PyTorch primitive operations and distributed communication package from +`torch.distributed`. + + - _Python-only implementation requires input tensor to be of same data type as +layer_ + +2. We also provide implementation with kernels through CUDA/C++ extension with +improved performance. We are experimenting with Welford and Kahan for reduction +hoping to get better accuracy. + To use the kernel implementation, user need to install Apex with CUDA extension +enabled `python setup.py install --cuda_ext`. + + - _Custom kernel implementation supports fp16 input with fp32 layer as cudnn. +This is required to run imagenet example in fp16._ + + - _Currently kernel implementation only supports GPU._ + +#### HowTo + +1. User could use `apex.parallel.SyncBatchNorm` by building their module with +the layer explicitly. + +``` +import apex +input_t = torch.randn(3, 5, 20).cuda() +sbn = apex.parallel.SyncBatchNorm(5).cuda() +output_t = sbn(input) +``` + +2. User could also take a constructed `torch.nn.Model` and replace all its `torch.nn.BatchNorm*N*d` modules with `apex.parallel.SyncBatchNorm` through utility function `apex.parallel.convert_syncbn_model`. + +``` +# model is an instance of torch.nn.Module +import apex +sync_bn_model = apex.parallel.convert_syncbn_model(model) +``` diff --git a/apex/apex/parallel/__init__.py b/apex/apex/parallel/__init__.py new file mode 100644 index 00000000..d6c8b0f0 --- /dev/null +++ b/apex/apex/parallel/__init__.py @@ -0,0 +1,97 @@ +import torch + +if hasattr(torch.distributed, 'ReduceOp'): + ReduceOp = torch.distributed.ReduceOp +elif hasattr(torch.distributed, 'reduce_op'): + ReduceOp = torch.distributed.reduce_op +else: + ReduceOp = torch.distributed.deprecated.reduce_op + +from .distributed import DistributedDataParallel, Reducer +# This is tricky because I'd like SyncBatchNorm to be exposed the same way +# for both the cuda-enabled and python-fallback versions, and I don't want +# to suppress the error information. +try: + import syncbn + from .optimized_sync_batchnorm import SyncBatchNorm +except ImportError as err: + from .sync_batchnorm import SyncBatchNorm + SyncBatchNorm.syncbn_import_error = err + +def convert_syncbn_model(module, process_group=None, channel_last=False): + ''' + Recursively traverse module and its children to replace all instances of + ``torch.nn.modules.batchnorm._BatchNorm`` with :class:`apex.parallel.SyncBatchNorm`. + + All ``torch.nn.BatchNorm*N*d`` wrap around + ``torch.nn.modules.batchnorm._BatchNorm``, so this function lets you easily switch + to use sync BN. + + Args: + module (torch.nn.Module): input module + + Example:: + + >>> # model is an instance of torch.nn.Module + >>> import apex + >>> sync_bn_model = apex.parallel.convert_syncbn_model(model) + ''' + from apex import deprecated_warning + deprecated_warning("apex.parallel.convert_syncbn_model is deprecated and will be removed by the end of February 2023. Use `torch.nn.SyncBatchNorm.convert_sync_batchnorm`.") + mod = module + if isinstance(module, torch.nn.modules.instancenorm._InstanceNorm): + return module + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm): + mod = SyncBatchNorm(module.num_features, module.eps, module.momentum, module.affine, module.track_running_stats, process_group, channel_last=channel_last) + mod.running_mean = module.running_mean + mod.running_var = module.running_var + mod.num_batches_tracked = module.num_batches_tracked + if module.affine: + mod.weight.data = module.weight.data.clone().detach() + mod.bias.data = module.bias.data.clone().detach() + for name, child in module.named_children(): + mod.add_module(name, convert_syncbn_model(child, + process_group=process_group, + channel_last=channel_last)) + # TODO(jie) should I delete model explicitly? + del module + return mod + +def create_syncbn_process_group(group_size): + ''' + Creates process groups to be used for syncbn of a give ``group_size`` and returns + process group that current GPU participates in. + + ``group_size`` must divide the total number of GPUs (world_size). + + ``group_size`` of 0 would be considered as =world_size. In this case ``None`` will be returned. + + ``group_size`` of 1 would be equivalent to using non-sync bn, but will still carry the overhead. + + Args: + group_size (int): number of GPU's to collaborate for sync bn + + Example:: + + >>> # model is an instance of torch.nn.Module + >>> import apex + >>> group = apex.parallel.create_syncbn_process_group(group_size) + ''' + + if group_size==0: + return None + + world_size = torch.distributed.get_world_size() + assert(world_size >= group_size) + assert(world_size % group_size == 0) + + group=None + for group_num in (range(world_size//group_size)): + group_ids = range(group_num*group_size, (group_num+1)*group_size) + cur_group = torch.distributed.new_group(ranks=group_ids) + if (torch.distributed.get_rank()//group_size == group_num): + group = cur_group + #can not drop out and return here, every process must go through creation of all subgroups + + assert(group is not None) + return group diff --git a/apex/apex/parallel/distributed.py b/apex/apex/parallel/distributed.py new file mode 100644 index 00000000..1c530d51 --- /dev/null +++ b/apex/apex/parallel/distributed.py @@ -0,0 +1,643 @@ +from collections import OrderedDict +import copy +import importlib +from itertools import chain + +import torch +import torch.distributed as dist +from torch.nn.modules import Module +from torch.autograd import Variable + +from ..multi_tensor_apply import multi_tensor_applier + +imported_flatten_impl = False + +def import_flatten_impl(): + global flatten_impl, unflatten_impl, imported_flatten_impl + try: + import apex_C + flatten_impl = apex_C.flatten + unflatten_impl = apex_C.unflatten + except ImportError: + print("Warning: apex was installed without --cpp_ext. Falling back to Python flatten and unflatten.") + flatten_impl = torch._utils._flatten_dense_tensors + unflatten_impl = torch._utils._unflatten_dense_tensors + imported_flatten_impl = True + +def flatten(bucket): + if not imported_flatten_impl: + import_flatten_impl() + return flatten_impl(bucket) + +def unflatten(coalesced, bucket): + if not imported_flatten_impl: + import_flatten_impl() + return unflatten_impl(coalesced, bucket) + +# apply_dist_call requires that tensors in 'bucket' are all the same type. +def apply_flat_dist_call(bucket, call, extra_args=None): + + coalesced = flatten(bucket) + + if extra_args is not None: + call(coalesced, *extra_args) + else: + call(coalesced) + + if call is dist.all_reduce: + coalesced /= dist.get_world_size() + + for buf, synced in zip(bucket, unflatten(coalesced, bucket)): + buf.copy_(synced) + +def split_half_float_double(tensors): + dtypes = ["torch.cuda.HalfTensor", "torch.cuda.FloatTensor", "torch.cuda.DoubleTensor"] + buckets = [] + for i, dtype in enumerate(dtypes): + bucket = [t for t in tensors if t.type() == dtype] + if bucket: + buckets.append(bucket) + return buckets + +def split_by_type(tensors): + buckets = OrderedDict() + for tensor in tensors: + tp = tensor.type() + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(tensor) + return buckets + +# flat_dist_call organizes 'tensors' by type. +def flat_dist_call(tensors, call, extra_args=None): + buckets = split_by_type(tensors) + + for tp in buckets: + bucket = buckets[tp] + apply_flat_dist_call(bucket, call, extra_args) + + +def extract_tensors(maybe_tensor, tensor_list): + if torch.is_tensor(maybe_tensor): + tensor_list.append(maybe_tensor) + else: + try: + for item in maybe_tensor: + extract_tensors(item, tensor_list) + except TypeError: + return + + +class Reducer(object): + """ + :class:`apex.parallel.Reducer` is a simple class that helps allreduce a module's parameters + across processes. :class:`Reducer` is intended to give the user additional control: + Unlike :class:`DistributedDataParallel`, :class:`Reducer` will not automatically allreduce + parameters during ``backward()``. + Instead, :class:`Reducer` waits for the user to call ``.reduce()`` manually. + This enables, for example, delaying the allreduce to be carried out every + several iterations instead of every single iteration. + + Like :class:`DistributedDataParallel`, :class:`Reducer` averages any tensors it allreduces + over the number of participating processes. + + :class:`Reducer` is designed to work with the upstream launch utility script + ``torch.distributed.launch`` with ``--nproc_per_node <= number of gpus per node``. + When used with this launcher, :class:`Reducer` assumes 1:1 mapping of processes to GPUs. + It also assumes that your script calls ``torch.cuda.set_device(args.rank)`` before creating the model. + + Args: + module_or_grads_list: Either a network definition (module) being run in multi-gpu/distributed mode, or an iterable of gradients to be reduced. If a module is passed in, the Reducer constructor will sync the parameters across processes (broadcasting from rank 0) to make sure they're all initialized with the same values. If a list of gradients (that came from some module) is passed in, the user is responsible for manually syncing that module's parameters at the beginning of training. + """ + + def __init__(self, module_or_grads_list): + if isinstance(module_or_grads_list, Module): + self.module = module_or_grads_list + flat_dist_call([param.data for param in self.module.parameters()], dist.broadcast, (0,) ) + + else: + self.module = None + self.grads = [] + extract_tensors(module_or_grads_list, self.grads) + + def reduce(self): + if self.module: + grads = [param.grad.data for param in self.module.parameters() if param.grad is not None] + flat_dist_call(grads, dist.all_reduce) + else: + flat_dist_call(self.grads, dist.all_reduce) + + +class DistributedDataParallel(Module): + """ + :class:`apex.parallel.DistributedDataParallel` is a module wrapper that enables + easy multiprocess distributed data parallel training, similar to ``torch.nn.parallel.DistributedDataParallel``. Parameters are broadcast across participating processes on initialization, and gradients are + allreduced and averaged over processes during ``backward()``. + + :class:`DistributedDataParallel` is optimized for use with NCCL. It achieves high performance by + overlapping communication with computation during ``backward()`` and bucketing smaller gradient + transfers to reduce the total number of transfers required. + + :class:`DistributedDataParallel` is designed to work with the upstream launch utility script + ``torch.distributed.launch`` with ``--nproc_per_node <= number of gpus per node``. + When used with this launcher, :class:`DistributedDataParallel` assumes 1:1 mapping of processes to GPUs. + It also assumes that your script calls ``torch.cuda.set_device(args.rank)`` before creating the model. + + https://github.com/NVIDIA/apex/tree/master/examples/simple/distributed shows detailed usage. + https://github.com/NVIDIA/apex/tree/master/examples/imagenet shows another example + that combines :class:`DistributedDataParallel` with mixed precision training. + + Args: + module: Network definition to be run in multi-gpu/distributed mode. + message_size (int, default=1e7): Minimum number of elements in a communication bucket. + delay_allreduce (bool, default=False): Delay all communication to the end of the backward pass. This disables overlapping communication with computation. + allreduce_trigger_params (list, optional, default=None): If supplied, should contain a list of parameters drawn from the model. Allreduces will be kicked off whenever one of these parameters receives its gradient (as opposed to when a bucket of size message_size is full). At the end of backward(), a cleanup allreduce to catch any remaining gradients will also be performed automatically. If allreduce_trigger_params is supplied, the message_size argument will be ignored. + allreduce_always_fp32 (bool, default=False): Convert any FP16 gradients to FP32 before allreducing. This can improve stability for widely scaled-out runs. + gradient_average (bool, default=True): Option to toggle whether or not DDP averages the allreduced gradients over processes. For proper scaling, the default value of True is recommended. + gradient_predivide_factor (float, default=1.0): Allows perfoming the average of gradients over processes partially before and partially after the allreduce. Before allreduce: ``grads.mul_(1.0/gradient_predivide_factor)``. After allreduce: ``grads.mul_(gradient_predivide_factor/world size)``. This can reduce the stress on the dynamic range of FP16 allreduces for widely scaled-out runs. + + .. warning:: + If ``gradient_average=False``, the pre-allreduce division (``grads.mul_(1.0/gradient_predivide_factor)``) will still be applied, but the post-allreduce gradient averaging (``grads.mul_(gradient_predivide_factor/world size)``) will be omitted. + + """ + + def __init__(self, + module, + message_size=10000000, + delay_allreduce=False, + shared_param=None, + allreduce_trigger_params=None, + retain_allreduce_buffers=False, + allreduce_always_fp32=False, + num_allreduce_streams=1, + allreduce_communicators=None, + gradient_average=True, + gradient_predivide_factor=1.0, + gradient_average_split_factor=None, + prof=False): + super(DistributedDataParallel, self).__init__() + from apex import deprecated_warning + deprecated_warning("apex.parallel.DistributedDataParallel is deprecated and will be removed by the end of February 2023.") + + # Backward/forward compatibility around + # https://github.com/pytorch/pytorch/commit/540ef9b1fc5506369a48491af8a285a686689b36 and + # https://github.com/pytorch/pytorch/commit/044d00516ccd6572c0d6ab6d54587155b02a3b86 + if hasattr(dist, "get_backend"): + self._backend = dist.get_backend() + if hasattr(dist, "DistBackend"): + self.backend_enum_holder = dist.DistBackend + else: + self.backend_enum_holder = dist.Backend + else: + self._backend = dist._backend + self.backend_enum_holder = dist.dist_backend + + self.warn_on_half = True if self._backend == self.backend_enum_holder.GLOO else False + + self.prof = prof + + self.allreduce_different_streams = (num_allreduce_streams > 1) + self.num_allreduce_streams = num_allreduce_streams + self.allreduce_communicators = allreduce_communicators + if self.allreduce_communicators: + assert len(allreduce_communicators[0]) == num_allreduce_streams + assert len(allreduce_communicators[0]) == len(allreduce_communicators[1]) + assert self.allreduce_different_streams + + if self.allreduce_different_streams and delay_allreduce: + raise ValueError("self.allreduce_different_streams may only be used if delay_allreduce=False.") + + if shared_param is not None: + raise ValueError("shared_param is no longer supported as an option. It was misleadingly named from the start. It turns out overlapping communication with computation should work fine with shared parameters. If you still wish to delay communication to the end of the backward pass, use delay_allreduce=True|False instead.") + + self.world_size = float(dist.get_world_size()) + + self.retain_allreduce_buffers = retain_allreduce_buffers + self.allreduce_always_fp32 = allreduce_always_fp32 + self.gradient_average = gradient_average + self.gradient_predivide_factor = gradient_predivide_factor + + self.custom_allreduce_triggers = False + if allreduce_trigger_params is not None: + if delay_allreduce: + raise ValueError("Setting allreduce_trigger_params is only valid if delay_allreduce=False.") + self.custom_allreduce_triggers = True + self.allreduce_trigger_params = set([id(param) for param in allreduce_trigger_params]) + + self.delay_allreduce = delay_allreduce + self.message_size = message_size + + self.main_stream = torch.cuda.current_stream() + + self.bucket_streams = [] + self.bucket_events = [] + + self.module = module + + self._disable_allreduce = False + + if self._backend == self.backend_enum_holder.NCCL: + for param in self.module.parameters(): + assert param.is_cuda, "NCCL backend only supports model parameters to be on GPU." + + self.active_params = [] + + self.param_type_to_tmp_i = {"torch.cuda.HalfTensor" : 0, + "torch.cuda.FloatTensor" : 1, + "torch.cuda.DoubleTensor" : 2} + + if multi_tensor_applier.available: + # TODO: I really need to centralize the C++ backed imports + import amp_C + self.multi_tensor_scale = amp_C.multi_tensor_scale + self._overflow_buf = torch.cuda.IntTensor([0]) + + self.create_hooks() + + flat_dist_call([param.data for param in self.module.parameters()], dist.broadcast, (0,) ) + + + def __setstate__(self, state): + super(DistributedDataParallel, self).__setstate__(state) + if self.allreduce_different_streams and delay_allreduce: + raise ValueError("self.allreduce_different_streams may only be used if delay_allreduce=False.") + + if self.delay_allreduce: + self.needs_refresh = True + + self.bucket_streams = [] + self.bucket_events = [] + + + def __getstate__(self): + attrs = copy.copy(self.__dict__) + if self._backend != self.backend_enum_holder.NCCL: + del attrs['self.bucket_streams'] + del attrs['self.bucket_events'] + return attrs + + def enable_allreduce(self): + self._disable_allreduce = False + + def disable_allreduce(self): + self._disable_allreduce = True + + # Broadcast rank 0's bucket structure across all processes, and have all processes + # regenerate their bucket structures to match. + def sync_bucket_structure(self): + # Append leftover buckets + for tmp_bucket in self.tmp_buckets: + if len(tmp_bucket) > 0: + self.active_i_buckets.append(tmp_bucket) + + self.num_buckets = len(self.active_i_buckets) + self.bucket_sizes = [len(bucket) for bucket in self.active_i_buckets] + + info_tensor = torch.cuda.IntTensor([self.num_buckets] + + self.bucket_sizes + + list(chain(*self.active_i_buckets))) + + dist.broadcast(info_tensor, 0) + + info = [int(entry) for entry in info_tensor] + + self.num_buckets = info[0] + self.bucket_sizes = info[1:self.num_buckets + 1] + self.buckets = [[None for _ in range(self.bucket_sizes[i])] + for i in range(self.num_buckets)] + # Technically, active_i_buckets' work is done. But the information is still useful to + # keep around. Therefore, refresh active_i_buckets based on rank 0 as well. + self.active_i_buckets = [[None for _ in range(self.bucket_sizes[i])] + for i in range(self.num_buckets)] + + flattened_buckets = info[self.num_buckets + 1:] + flat_i = 0 + for bucket_idx in range(self.num_buckets): + for bucket_loc in range(self.bucket_sizes[bucket_idx]): + param_i = flattened_buckets[flat_i] + self.active_i_buckets[bucket_idx][bucket_loc] = param_i + self.param_id_to_bucket[id(self.active_params[param_i])] = (bucket_idx, bucket_loc) + flat_i += 1 + + + def create_hooks(self): + # Fallback hook that's only called at the end of backward. + # Used if you deliberately want to delay allreduces to the end, or to refresh the + # bucket structure that will be used to overlap communication with computation in later + # iterations. + def allreduce_params(): + # Bucket record refresh + if not self.delay_allreduce: + if self.needs_refresh: + self.sync_bucket_structure() + + self.needs_refresh = False + + self.allreduce_fallback() + + + def overlapping_backward_epilogue(): + for stream, event in zip(self.bucket_streams, self.bucket_events): + stream.record_event(event) + torch.cuda.current_stream().wait_event(event) + + # Sanity checks that all the buckets were kicked off + if self.next_bucket != self.num_buckets: + raise RuntimeError("In epilogue, next_bucket ({}) != num_buckets ({}). ".format( + self.next_bucket, self.num_buckets), + "This probably indicates some buckets were not allreduced.") + + for actual, expected in zip(self.buckets_ready_size, self.bucket_sizes): + if actual != expected: + raise RuntimeError("Some param buckets were not allreduced.") + + + self.grad_accs = [] + for param in self.module.parameters(): + if param.requires_grad: + def wrapper(param): + param_tmp = param.expand_as(param) + grad_acc = param_tmp.grad_fn.next_functions[0][0] + + def allreduce_hook(*unused): + if self.prof: + torch.cuda.nvtx.range_push("allreduce_hook") + + if not self._disable_allreduce: + if self.delay_allreduce or self.needs_refresh: + # TODO: How do we want to handle multiple backward passes between + # each forward, e.g., backward passes with retain_graph=True? + # needs_refresh and callback_queued are both vulnerable states. + if not self.delay_allreduce and self.needs_refresh: + # Use the backward pass to build the bucket structure on the fly. + active_i = self.param_id_to_active_i[id(param)] + + # Float, half, and double tensors are grouped into buckets separately. + current_type = self.param_type_to_tmp_i[param.type()] + + self.tmp_buckets[current_type].append(active_i) + + ship_tmp_bucket = False + if self.custom_allreduce_triggers: + if id(param) in self.allreduce_trigger_params: + ship_tmp_bucket = True + else: + self.tmp_numels[current_type] += param.numel() + if self.tmp_numels[current_type] >= self.message_size: + ship_tmp_bucket = True + + # To consider: If custom_allreduce_triggers are in use, ship all + # tmp_buckets, not just tmp_buckets[current_type]. + if ship_tmp_bucket: + self.active_i_buckets.append(self.tmp_buckets[current_type]) + self.tmp_buckets[current_type] = [] + self.tmp_numels[current_type] = 0 + + if not self.callback_queued: + Variable._execution_engine.queue_callback(allreduce_params) + self.callback_queued = True + else: + if not self.callback_queued: + Variable._execution_engine.queue_callback(overlapping_backward_epilogue) + self.callback_queued = True + + self.comm_ready_buckets(param) + + if self.prof: + torch.cuda.nvtx.range_pop() + + grad_acc.register_hook(allreduce_hook) + self.grad_accs.append(grad_acc) + + wrapper(param) + + + def _stream_this_bucket(self, bucket_idx): + if self.allreduce_different_streams: + return self.bucket_streams[bucket_idx%self.num_allreduce_streams] + else: + return self.bucket_streams[0] + + + def _event_this_bucket(self, bucket_idx): + if self.allreduce_different_streams: + return self.bucket_events[bucket_idx%self.num_allreduce_streams] + else: + return self.bucket_events[0] + + + def allreduce_bucket(self, bucket, bucket_idx, force_default_stream): + tensor = flatten(bucket) + + if force_default_stream: + bucket_stream = self.main_stream + else: + bucket_stream = self._stream_this_bucket(bucket_idx) + bucket_event = self._event_this_bucket(bucket_idx) + torch.cuda.current_stream().record_event(bucket_event) + bucket_stream.wait_event(bucket_event) + + with torch.cuda.stream(bucket_stream): + # self.main_stream.wait_stream(torch.cuda.current_stream()) + # torch.cuda.synchronize() + + tensor_to_allreduce = tensor + + if self.allreduce_always_fp32: + tensor_to_allreduce = tensor.float() + + if self.gradient_predivide_factor != 1.0: + tensor_to_allreduce.mul_(1./self.gradient_predivide_factor) + + if self.allreduce_different_streams and not force_default_stream: + dist.all_reduce(tensor_to_allreduce, group=self.bucket_pgs[bucket_idx%self.num_allreduce_streams]) + else: + dist.all_reduce(tensor_to_allreduce) + + if self.gradient_average: + tensor_to_allreduce.mul_(self.gradient_predivide_factor/self.world_size) + + if self.allreduce_always_fp32 and tensor is not tensor_to_allreduce: + tensor.copy_(tensor_to_allreduce) + + if not self.retain_allreduce_buffers: + if multi_tensor_applier.available: + multi_tensor_applier( + self.multi_tensor_scale, + self._overflow_buf, + [unflatten(tensor, bucket), bucket], + 1.0) + else: + for buf, synced in zip(bucket, unflatten(tensor, bucket)): + buf.copy_(synced) + + # I think we actually do need this here. After allreduce_bucket returns, tensor will + # eventually go out of scope and die, at which point it could otherwise be freed for + # further reuse by the main stream while the allreduce/div/unflatten are underway in bucket_stream. + tensor.record_stream(bucket_stream) + + return tensor + + + def allreduce_maybe_retain(self, bucket, bucket_idx, force_default_stream=False): + allreduced = self.allreduce_bucket(bucket, bucket_idx, force_default_stream) + if self.retain_allreduce_buffers: + if self.allreduce_buffers[bucket_idx] is not None: + raise RuntimeError("The backward pass is attempting to replace an already-filled " + "allreduce buffer. This is almost certainly an error.") + self.allreduce_buffers[bucket_idx] = allreduced + for view, grad in zip(unflatten(allreduced, bucket), bucket): + grad.data = view + # for buf, synced in zip(bucket, unflatten(allreduced, bucket)): + # buf.copy_(synced) + + + def allreduce_fallback(self): + for stream, event in zip(self.bucket_streams, self.bucket_events): + stream.record_event(event) + torch.cuda.current_stream().wait_event(event) + + if self.retain_allreduce_buffers: + grads = [param.grad for param in self.module.parameters() if param.grad is not None] + else: + grads = [param.grad.data for param in self.module.parameters() if param.grad is not None] + + split_buckets = split_half_float_double(grads) + + # If retain_allreduce_buffers is True and delay_allreduce is False, + # this will only be done during the first backward pass, ignored by the + # training script, and overwritten in the next forward pass. So it's harmless. + if self.retain_allreduce_buffers: + self.allreduce_buffers = [None for _ in range(len(split_buckets))] + + for i, bucket in enumerate(split_buckets): + allreduced = self.allreduce_maybe_retain(bucket, i, force_default_stream=True) + + + def comm_ready_buckets(self, param): + # Need to do this in every hook for compatibility with Ruberry's streaming backward PR. + # self.reduction_stream.wait_stream(torch.cuda.current_stream()) + if self.prof: + torch.cuda.nvtx.range_push("comm_ready_buckets") + + bucket_idx, bucket_loc = self.param_id_to_bucket[id(param)] + + if self.buckets[bucket_idx][bucket_loc] is not None: + raise RuntimeError("The backward pass is attempting to replace an already-filled " + "bucket slot. This is almost certainly an error.") + + if self.retain_allreduce_buffers: + self.buckets[bucket_idx][bucket_loc] = param.grad + else: + self.buckets[bucket_idx][bucket_loc] = param.grad.data + + self.buckets_ready_size[bucket_idx] += 1 + + if self.buckets_ready_size[bucket_idx] == self.bucket_sizes[bucket_idx]: + if bucket_idx == self.next_bucket: + self.allreduce_maybe_retain(self.buckets[bucket_idx], bucket_idx) + + self.next_bucket += 1 + + # Reversing upstream's logic here, because we constructed our buckets based on + # the order things were received during backward. + if len(self.ready_buckets_not_reduced) > 0: + sorted_todo = sorted(self.ready_buckets_not_reduced) + for i in sorted_todo: + # Nothing can be reduced now + if i > self.next_bucket: + break + elif i == self.next_bucket: + self.allreduce_maybe_retain(self.buckets[i], i) + self.ready_buckets_not_reduced.remove(i) + self.next_bucket += 1 + else: + raise ValueError("i should always be >= next_bucket") + else: + self.ready_buckets_not_reduced.add(bucket_idx) + + if self.prof: + torch.cuda.nvtx.range_pop() + + + def forward(self, *inputs, **kwargs): + result = self.module(*inputs, **kwargs) + + if self.prof: + torch.cuda.nvtx.range_push("forward pass DDP logic") + + if not self._disable_allreduce: + if not self.delay_allreduce: + param_list = [param for param in self.module.parameters() if param.requires_grad] + + # Conditions under which to refresh self.record + # Forward has the authority to set needs_refresh to True, but only allreduce_params + # in backward has the authority to set needs_refresh to False. + # Parentheses are not necessary for correct order of operations, but make the intent clearer. + if ((not self.active_params) or + (len(param_list) != len(self.active_params)) or + any([param1 is not param2 for param1, param2 in zip(param_list, self.active_params)])): + self.needs_refresh = True + + if self.needs_refresh: + self.active_i_buckets = [] + self.buckets = [] + self.tmp_buckets = [[], [], []] # [running half, float, double buckets] + self.tmp_numels = [0, 0, 0] + self.bucket_sizes = [] + self.param_id_to_active_i = {id(param) : i for i, param in enumerate(param_list)} + self.param_id_to_bucket = {} + self.bucket_pgs = [] + self.bucket_streams = [] + self.bucket_events = [] + else: + # self.buckets = [[None for _ in range(self.bucket_sizes[i])] + # for i in range(self.num_buckets)] + if not self.buckets: + self.buckets = [[None for _ in range(self.bucket_sizes[i])] + for i in range(self.num_buckets)] + else: + assert len(self.buckets) == self.num_buckets, "len(buckets) = {}, expected {}".format( + len(self.buckets), self.num_buckets) + for b, bucket in enumerate(self.buckets): + assert len(bucket) == self.bucket_sizes[b], "len(buckets[{}]) = {}, expected {})".format( + b, len(buckets[b]), self.bucket_sizes[b]) + for i in range(len(bucket)): + bucket[i] = None + + if self.allreduce_communicators: + self.bucket_pgs = self.allreduce_communicators[0] + self.bucket_streams = self.allreduce_communicators[1] + self.bucket_events = [torch.cuda.Event(enable_timing=False, + blocking=False) for _ in range(self.num_allreduce_streams)] + else: + if self.allreduce_different_streams: + if not self.bucket_pgs: + self.bucket_pgs = [dist.new_group() for _ in range(self.num_allreduce_streams)] + for i, bg in enumerate(self.bucket_pgs): + print("rank {} created group {} with backend {}".format( + dist.get_rank(), i, dist.get_backend(bg))) + if self.allreduce_different_streams: + if not self.bucket_streams: + self.bucket_streams = [torch.cuda.Stream() for _ in range(self.num_allreduce_streams)] + self.bucket_events = [torch.cuda.Event(enable_timing=False, + blocking=False) for _ in range(self.num_allreduce_streams)] + else: + if not self.bucket_streams: + self.bucket_streams = [torch.cuda.Stream()] + self.bucket_events = [torch.cuda.Event(enable_timing=False, blocking=False)] + + self.buckets_ready_size = [0 for i in range(self.num_buckets)] + if(self.retain_allreduce_buffers): + self.allreduce_buffers = [None for _ in range(self.num_buckets)] + self.next_bucket = 0 + self.ready_buckets_not_reduced = set() + + self.active_params = param_list + + self.callback_queued = False + + if self.prof: + torch.cuda.nvtx.range_pop() + + return result diff --git a/apex/apex/parallel/multiproc.py b/apex/apex/parallel/multiproc.py new file mode 100644 index 00000000..ff743df2 --- /dev/null +++ b/apex/apex/parallel/multiproc.py @@ -0,0 +1,35 @@ +import torch +import sys +import subprocess + +def docstring_hack(): + """ + Multiproc file which will launch a set of processes locally for multi-gpu + usage: python -m apex.parallel.multiproc main.py ... + """ + pass + +argslist = list(sys.argv)[1:] +world_size = torch.cuda.device_count() + +if '--world-size' in argslist: + world_size = int(argslist[argslist.index('--world-size')+1]) +else: + argslist.append('--world-size') + argslist.append(str(world_size)) + +workers = [] + +for i in range(world_size): + if '--rank' in argslist: + argslist[argslist.index('--rank')+1] = str(i) + else: + argslist.append('--rank') + argslist.append(str(i)) + stdout = None if i == 0 else open("GPU_"+str(i)+".log", "w") + print(argslist) + p = subprocess.Popen([str(sys.executable)]+argslist, stdout=stdout) + workers.append(p) + +for p in workers: + p.wait() diff --git a/apex/apex/parallel/optimized_sync_batchnorm.py b/apex/apex/parallel/optimized_sync_batchnorm.py new file mode 100644 index 00000000..65cf5eab --- /dev/null +++ b/apex/apex/parallel/optimized_sync_batchnorm.py @@ -0,0 +1,85 @@ +import torch +from torch.nn.modules.batchnorm import _BatchNorm +from torch.nn import functional as F + +import syncbn +from .optimized_sync_batchnorm_kernel import SyncBatchnormFunction + + +class SyncBatchNorm(_BatchNorm): + """ + synchronized batch normalization module extented from `torch.nn.BatchNormNd` + with the added stats reduction across multiple processes. + :class:`apex.parallel.SyncBatchNorm` is designed to work with + `DistributedDataParallel`. + + When running in training mode, the layer reduces stats across all processes + to increase the effective batchsize for normalization layer. This is useful + in applications where batch size is small on a given process that would + diminish converged accuracy of the model. The model uses collective + communication package from `torch.distributed`. + + When running in evaluation mode, the layer falls back to + `torch.nn.functional.batch_norm` + + Args: + num_features: :math:`C` from an expected input of size + :math:`(N, C, L)` or :math:`L` from input of size :math:`(N, L)` + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Can be set to ``None`` for cumulative moving average + (i.e. simple average). Default: 0.1 + affine: a boolean value that when set to ``True``, this module has + learnable affine parameters. Default: ``True`` + track_running_stats: a boolean value that when set to ``True``, this + module tracks the running mean and variance, and when set to ``False``, + this module does not track such statistics and always uses batch + statistics in both training and eval modes. Default: ``True`` + process_group: pass in a process group within which the stats of the + mini-batch is being synchronized. ``None`` for using default process + group + channel_last: a boolean value that when set to ``True``, this module + take the last dimension of the input tensor to be the channel + dimension. Default: False + + Examples:: + >>> # channel first tensor + >>> sbn = apex.parallel.SyncBatchNorm(100).cuda() + >>> inp = torch.randn(10, 100, 14, 14).cuda() + >>> out = sbn(inp) + >>> inp = torch.randn(3, 100, 20).cuda() + >>> out = sbn(inp) + >>> # channel last tensor + >>> sbn = apex.parallel.SyncBatchNorm(100, channel_last=True).cuda() + >>> inp = torch.randn(10, 14, 14, 100).cuda() + """ + + def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True, process_group=None, channel_last=False, fuse_relu=False): + super(SyncBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine, track_running_stats=track_running_stats) + self.process_group = process_group + self.channel_last = channel_last + self.fuse_relu = fuse_relu + + def _specify_process_group(self, process_group): + self.process_group = process_group + + def _specify_channel_last(self, channel_last): + self.channel_last = channel_last + + def forward(self, input, z = None): + # if input.dim() == 2, we switch to channel_last for efficient memory accessing + channel_last = self.channel_last if input.dim() != 2 else True + + if not self.training and self.track_running_stats and not channel_last and not self.fuse_relu and z == None: + # fall back to pytorch implementation for inference + return F.batch_norm(input, self.running_mean, self.running_var, self.weight, self.bias, False, 0.0, self.eps) + else: + exponential_average_factor = 0.0 + if self.training and self.track_running_stats: + self.num_batches_tracked += 1 + if self.momentum is None: + exponential_average_factor = 1.0 / float(self.num_batches_tracked) + else: + exponential_average_factor = self.momentum + return SyncBatchnormFunction.apply(input, z, self.weight, self.bias, self.running_mean, self.running_var, self.eps, self.training or not self.track_running_stats, exponential_average_factor, self.process_group, channel_last, self.fuse_relu) diff --git a/apex/apex/parallel/optimized_sync_batchnorm_kernel.py b/apex/apex/parallel/optimized_sync_batchnorm_kernel.py new file mode 100644 index 00000000..61684714 --- /dev/null +++ b/apex/apex/parallel/optimized_sync_batchnorm_kernel.py @@ -0,0 +1,119 @@ +import torch +from torch.autograd.function import Function + +import syncbn +from apex.parallel import ReduceOp + +class SyncBatchnormFunction(Function): + + @staticmethod + def forward(ctx, input, z, weight, bias, running_mean, running_variance, eps, track_running_stats = True, momentum = 1.0, process_group = None, channel_last = False, fuse_relu = False): + input = input.contiguous() + world_size = 0 + + mean = None + var_biased = None + inv_std = None + var = None + out = None + count = None + if track_running_stats: + if channel_last: + count = int(input.numel()/input.size(-1)) + mean, var_biased = syncbn.welford_mean_var_c_last(input) + num_channels = input.size(-1) + else: + count = int(input.numel()/input.size(1)) + mean, var_biased = syncbn.welford_mean_var(input) + num_channels = input.size(1) + + if torch.distributed.is_initialized(): + if not process_group: + process_group = torch.distributed.group.WORLD + device = mean.device + world_size = torch.distributed.get_world_size(process_group) + + count_t = torch.empty(1, dtype=mean.dtype, device=mean.device).fill_(count) + combined = torch.cat([mean.view(-1), var_biased.view(-1), count_t], dim=0) + combined_list = [torch.empty_like(combined) for k in range(world_size)] + torch.distributed.all_gather(combined_list, combined, process_group) + combined = torch.stack(combined_list, dim=0) + mean_all, invstd_all, count_all = torch.split(combined, num_channels, dim=1) + count_all = count_all.view(-1) + mean, var, inv_std = syncbn.welford_parallel(mean_all, invstd_all, count_all.to(torch.int32), eps) + else: + device = mean.device + count_all = torch.cuda.IntTensor([count], device=device) + inv_std = 1.0 / torch.sqrt(var_biased + eps) + var = var_biased * (count) / (count-1) + + if count == 1 and world_size < 2: + raise ValueError('Expected more than 1 value per channel when training, got input size{}'.format(input.size())) + + r_m_inc = mean if running_mean.dtype != torch.float16 else mean.half() + r_v_inc = var if running_variance.dtype != torch.float16 else var.half() + running_mean.data = running_mean.data * (1-momentum) + momentum*r_m_inc + running_variance.data = running_variance.data * (1-momentum) + momentum*r_v_inc + else: + mean = running_mean.data + inv_std = 1.0 / torch.sqrt(running_variance.data + eps) + + ctx.save_for_backward(input, weight, mean, inv_std, z, bias, count_all.to(torch.int32)) + ctx.process_group = process_group + ctx.channel_last = channel_last + ctx.world_size = world_size + ctx.fuse_relu = fuse_relu + + if channel_last: + out = syncbn.batchnorm_forward_c_last(input, z, mean, inv_std, weight, bias, fuse_relu) + else: + out = syncbn.batchnorm_forward(input, mean, inv_std, weight, bias) + + return out + + @staticmethod + def backward(ctx, grad_output): + grad_output = grad_output.contiguous() + # mini batch mean & var are calculated by forward path. + # mu = 1./N*np.sum(h, axis = 0) + # var = 1./N*np.sum((h-mu)**2, axis = 0) + saved_input, weight, mean, inv_std, z, bias, count = ctx.saved_tensors + process_group = ctx.process_group + channel_last = ctx.channel_last + world_size = ctx.world_size + fuse_relu = ctx.fuse_relu + grad_input = grad_z = grad_weight = grad_bias = None + + if fuse_relu: + grad_output = syncbn.relu_bw_c_last(grad_output, saved_input, z, mean, inv_std, weight, bias) + if isinstance(z, torch.Tensor) and ctx.needs_input_grad[1]: + grad_z = grad_output.clone() + + # TODO: update kernel to not pre_divide by item_num + if channel_last: + sum_dy, sum_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn_c_last(grad_output, saved_input, mean, inv_std, weight) + else: + sum_dy, sum_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output, saved_input, mean, inv_std, weight) + + # calculate grad_input + if ctx.needs_input_grad[0]: + + if torch.distributed.is_initialized(): + num_channels = sum_dy.shape[0] + combined = torch.cat([sum_dy, sum_dy_xmu], dim=0) + torch.distributed.all_reduce( + combined, torch.distributed.ReduceOp.SUM, process_group, async_op=False) + sum_dy, sum_dy_xmu = torch.split(combined, num_channels) + + if channel_last: + grad_input = syncbn.batchnorm_backward_c_last(grad_output, saved_input, mean, inv_std, weight, sum_dy, sum_dy_xmu, count) + else: + grad_input = syncbn.batchnorm_backward(grad_output, saved_input, mean, inv_std, weight, sum_dy, sum_dy_xmu, count) + + if weight is None or not ctx.needs_input_grad[2]: + grad_weight = None + + if weight is None or not ctx.needs_input_grad[3]: + grad_bias = None + + return grad_input, grad_z, grad_weight, grad_bias, None, None, None, None, None, None, None, None diff --git a/apex/apex/parallel/sync_batchnorm.py b/apex/apex/parallel/sync_batchnorm.py new file mode 100644 index 00000000..fd184436 --- /dev/null +++ b/apex/apex/parallel/sync_batchnorm.py @@ -0,0 +1,136 @@ +import torch +from torch.nn.modules.batchnorm import _BatchNorm +from torch.nn import functional as F + +from .sync_batchnorm_kernel import SyncBatchnormFunction +from apex.parallel import ReduceOp + + +class SyncBatchNorm(_BatchNorm): + """ + synchronized batch normalization module extented from ``torch.nn.BatchNormNd`` + with the added stats reduction across multiple processes. + :class:`apex.parallel.SyncBatchNorm` is designed to work with + ``DistributedDataParallel``. + + When running in training mode, the layer reduces stats across all processes + to increase the effective batchsize for normalization layer. This is useful + in applications where batch size is small on a given process that would + diminish converged accuracy of the model. The model uses collective + communication package from ``torch.distributed``. + + When running in evaluation mode, the layer falls back to + ``torch.nn.functional.batch_norm``. + + Args: + num_features: :math:`C` from an expected input of size + :math:`(N, C, L)` or :math:`L` from input of size :math:`(N, L)` + eps: a value added to the denominator for numerical stability. + Default: 1e-5 + momentum: the value used for the running_mean and running_var + computation. Can be set to ``None`` for cumulative moving average + (i.e. simple average). Default: 0.1 + affine: a boolean value that when set to ``True``, this module has + learnable affine parameters. Default: ``True`` + track_running_stats: a boolean value that when set to ``True``, this + module tracks the running mean and variance, and when set to ``False``, + this module does not track such statistics and always uses batch + statistics in both training and eval modes. Default: ``True`` + + Example:: + + >>> sbn = apex.parallel.SyncBatchNorm(100).cuda() + >>> inp = torch.randn(10, 100, 14, 14).cuda() + >>> out = sbn(inp) + >>> inp = torch.randn(3, 100, 20).cuda() + >>> out = sbn(inp) + """ + + warned = False + + def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, track_running_stats=True, process_group=None, channel_last=False): + from apex import deprecated_warning + deprecated_warning("apex.parallel.SyncBatchNorm is deprecated and will be removed by the end of February 2023. Use `torch.nn.SyncBatchNorm`.") + if channel_last == True: + raise AttributeError("channel_last is not supported by primitive SyncBatchNorm implementation. Try install apex with `--cuda_ext` if channel_last is desired.") + + if not SyncBatchNorm.warned: + if hasattr(self, "syncbn_import_error"): + print("Warning: using Python fallback for SyncBatchNorm, possibly because apex was installed without --cuda_ext. The exception raised when attempting to import the cuda backend was: ", self.syncbn_import_error) + else: + print("Warning: using Python fallback for SyncBatchNorm") + SyncBatchNorm.warned = True + + super(SyncBatchNorm, self).__init__(num_features, eps=eps, momentum=momentum, affine=affine, track_running_stats=track_running_stats) + self.process_group = process_group + + def _specify_process_group(self, process_group): + self.process_group = process_group + + def forward(self, input): + torch.cuda.nvtx.range_push("sync_bn_fw_with_mean_var") + mean = None + var = None + cast = None + out = None + + # casting to handle mismatch input type to layer type + if self.running_mean is not None: + if self.running_mean.dtype != input.dtype: + input = input.to(self.running_mean.dtype) + cast = input.dtype + elif self.weight is not None: + if self.weight.dtype != input.dtype: + input = input.to(self.weight.dtype) + cast = input.dtype + + if not self.training and self.track_running_stats: + # fall back to pytorch implementation for inference + torch.cuda.nvtx.range_pop() + out = F.batch_norm(input, self.running_mean, self.running_var, self.weight, self.bias, False, 0.0, self.eps) + else: + process_group = self.process_group + world_size = 1 + if not self.process_group: + process_group = torch.distributed.group.WORLD + self.num_batches_tracked += 1 + with torch.no_grad(): + channel_first_input = input.transpose(0, 1).contiguous() + squashed_input_tensor_view = channel_first_input.view( + channel_first_input.size(0), -1) + # total number of data points for each variance entry. Used to calculate unbiased variance estimate + m = None + local_m = float(squashed_input_tensor_view.size()[1]) + local_mean = torch.mean(squashed_input_tensor_view, 1) + local_sqr_mean = torch.pow( + squashed_input_tensor_view, 2).mean(1) + if torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size(process_group) + torch.distributed.all_reduce( + local_mean, ReduceOp.SUM, process_group) + mean = local_mean / world_size + torch.distributed.all_reduce( + local_sqr_mean, ReduceOp.SUM, process_group) + sqr_mean = local_sqr_mean / world_size + m = local_m * world_size + else: + m = local_m + mean = local_mean + sqr_mean = local_sqr_mean + # var(x) = E (( x - mean_x ) ** 2) + # = 1 / N * sum ( x - mean_x ) ** 2 + # = 1 / N * sum (x**2) - mean_x**2 + var = sqr_mean - mean.pow(2) + + if self.running_mean is not None: + self.running_mean = self.momentum * mean + \ + (1 - self.momentum) * self.running_mean + if self.running_var is not None: + # as noted by the paper, we used unbiased variance estimate of the mini-batch + # Var[x] = m / (m-1) * Eb (sample_variance) + self.running_var = m / \ + (m-1) * self.momentum * var + \ + (1 - self.momentum) * self.running_var + torch.cuda.nvtx.range_pop() + out = SyncBatchnormFunction.apply(input, self.weight, self.bias, mean, var, self.eps, process_group, world_size) + return out.to(cast) diff --git a/apex/apex/parallel/sync_batchnorm_kernel.py b/apex/apex/parallel/sync_batchnorm_kernel.py new file mode 100644 index 00000000..e407a63d --- /dev/null +++ b/apex/apex/parallel/sync_batchnorm_kernel.py @@ -0,0 +1,87 @@ +import torch +from torch.autograd.function import Function + +from apex.parallel import ReduceOp + + +class SyncBatchnormFunction(Function): + + @staticmethod + def forward(ctx, input, weight, bias, running_mean, running_variance, eps, process_group, world_size): + torch.cuda.nvtx.range_push("sync_BN_fw") + # transpose it to channel last to support broadcasting for input with different rank + c_last_input = input.transpose(1, -1).contiguous().clone() + + ctx.save_for_backward(c_last_input, weight, bias, + running_mean, running_variance) + ctx.eps = eps + ctx.process_group = process_group + ctx.world_size = world_size + + c_last_input = (c_last_input - running_mean) / \ + torch.sqrt(running_variance + eps) + + if weight is not None: + c_last_input = c_last_input * weight + if bias is not None: + c_last_input = c_last_input + bias + + torch.cuda.nvtx.range_pop() + return c_last_input.transpose(1, -1).contiguous().clone() + + @staticmethod + def backward(ctx, grad_output): + torch.cuda.nvtx.range_push("sync_BN_bw") + # mini batch mean & var are calculated by forward path. + # mu = 1./N*np.sum(h, axis = 0) + # var = 1./N*np.sum((h-mu)**2, axis = 0) + c_last_input, weight, bias, running_mean, running_variance = ctx.saved_tensors + + eps = ctx.eps + process_group = ctx.process_group + world_size = ctx.world_size + grad_input = grad_weight = grad_bias = None + num_features = running_mean.size()[0] + + # transpose it to channel last to support broadcasting for input with different rank + torch.cuda.nvtx.range_push("carilli field") + c_last_grad = grad_output.transpose(1, -1).contiguous() + # squash non-channel dimension so we can easily calculate mean + c_grad = c_last_grad.view(-1, num_features).contiguous() + torch.cuda.nvtx.range_pop() + + # calculate grad_input + if ctx.needs_input_grad[0]: + # dh = gamma * (var + eps)**(-1. / 2.) * (dy - np.mean(dy, axis=0) + # - (h - mu) * (var + eps)**(-1.0) * np.mean(dy * (h - mu), axis=0)) + mean_dy = c_grad.mean(0) + mean_dy_xmu = (c_last_grad * (c_last_input - + running_mean)).view(-1, num_features).mean(0) + if torch.distributed.is_initialized(): + torch.distributed.all_reduce( + mean_dy, ReduceOp.SUM, process_group) + mean_dy = mean_dy / world_size + torch.distributed.all_reduce( + mean_dy_xmu, ReduceOp.SUM, process_group) + mean_dy_xmu = mean_dy_xmu / world_size + c_last_grad_input = (c_last_grad - mean_dy - (c_last_input - running_mean) / ( + running_variance + eps) * mean_dy_xmu) / torch.sqrt(running_variance + eps) + if weight is not None: + c_last_grad_input.mul_(weight) + grad_input = c_last_grad_input.transpose(1, -1).contiguous() + + # calculate grad_weight + grad_weight = None + if weight is not None and ctx.needs_input_grad[1]: + # dgamma = np.sum((h - mu) * (var + eps)**(-1. / 2.) * dy, axis=0) + grad_weight = ((c_last_input - running_mean) / torch.sqrt( + running_variance + eps) * c_last_grad).view(-1, num_features).sum(0) + + # calculate grad_bias + grad_bias = None + if bias is not None and ctx.needs_input_grad[2]: + # dbeta = np.sum(dy, axis=0) + grad_bias = c_grad.sum(0) + + torch.cuda.nvtx.range_pop() + return grad_input, grad_weight, grad_bias, None, None, None, None, None diff --git a/apex/apex/transformer/README.md b/apex/apex/transformer/README.md new file mode 100644 index 00000000..7383f65e --- /dev/null +++ b/apex/apex/transformer/README.md @@ -0,0 +1,81 @@ +# apex.transformer + +`apex.transformer` is a module which enables efficient large Transformer models at scale. + +`apex.transformer.tensor_parallel` and `apex.transformer.pipeline_parallel` are both based on [NVIDIA/Megatron-LM](https://github.com/NVIDIA/Megatron-LM)'s module. +The former is based on `megatron.mpu` and the latter is on `megatron.schedules` and `megatron.p2p_communication`. + +## Tensor Model Parallel (TP) + +APEX's tensor model parallel utilities provides some `torch.nn.Module`'s, custom fused kernels, and PRNG state handling. +See Appendix B.2 of [Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism](https://arxiv.org/abs/1909.08053) for the details of +PRNG state handling. + +## Pipeline Model Parallel (PP) +APEX's pipeline model parallel functions require models to have `.set_input_tensor` because +the input tensor for `.forward` method can be `None`. + +The following is a really casual sketch of training script with apex pp. + +```python +import torch +import torch.nn as nn +import torch.nn.functional as F + +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel import get_forward_backward_func + + +class Model(nn.Module): + + ... + + def __init__(self, *args, **kwargs): + super().__init__() + pre_process = kwargs.pop("pre_process") + post_process = kwargs.pop("post_process") + + def set_input_tensor(self, tensor): + self.input_tensor = tensor + + def forward(self, x, ...): + if parallel_state.is_pipeline_first_stage(): + input = x + else: + input = self.input_tensor + ... + + +def model_provider_func(*args, **kwargs): + return Model(*args, **kwargs) + + +def loss_func(pred, label): + loss = ... + averaged_loss = average_losses_across_data_parallel_group([loss]) + return loss, {'nice_loss': averaged_loss} + + +def forward_step_func(batch, model): + input, label = process_batch(batch) + out = model(input) + return out, partial(loss_func, label) + + +forward_backward_func = get_forward_backward_func(virtual_pipeline_model_parallel_size, pipeline_model_parallel_size) + + +parallel_state.initialize_model_parallel( + tensor_model_parallel_size, + pipeline_model_parallel_size, + virtual_pipeline_model_parallel_size, +) +# The following line basically is equivalent to `build_model(Model, wrap_with_ddp, virtual_pipeline_model_parallel_size, *model_args, **model_kwargs)` +model = build_model(model_provider_func, wrap_with_ddp, virtual_pipeline_model_parallel_size, *model_args, **model_kwargs) +optimizer = ... +data_loader = ... +for epoch in range(num_epochs): + for batch in data_loader: + forward_backward_func(forward_step_func, batch, model, forward_only=False, tensor_shape) + optimizer.step() +``` diff --git a/apex/apex/transformer/__init__.py b/apex/apex/transformer/__init__.py new file mode 100644 index 00000000..ff9c7b95 --- /dev/null +++ b/apex/apex/transformer/__init__.py @@ -0,0 +1,23 @@ +from apex.transformer import amp +from apex.transformer import functional +from apex.transformer import parallel_state +from apex.transformer import pipeline_parallel +from apex.transformer import tensor_parallel +from apex.transformer import utils +from apex.transformer.enums import LayerType +from apex.transformer.enums import AttnType +from apex.transformer.enums import AttnMaskType + + +__all__ = [ + "amp", + "functional", + "parallel_state", + "pipeline_parallel", + "tensor_parallel", + "utils", + # enums.py + "LayerType", + "AttnType", + "AttnMaskType", +] diff --git a/apex/apex/transformer/_data/__init__.py b/apex/apex/transformer/_data/__init__.py new file mode 100644 index 00000000..2831dfb1 --- /dev/null +++ b/apex/apex/transformer/_data/__init__.py @@ -0,0 +1,8 @@ +from apex.transformer._data._batchsampler import MegatronPretrainingRandomSampler +from apex.transformer._data._batchsampler import MegatronPretrainingSampler + + +__all__ = [ + "MegatronPretrainingRandomSampler", + "MegatronPretrainingSampler", +] diff --git a/apex/apex/transformer/_data/_batchsampler.py b/apex/apex/transformer/_data/_batchsampler.py new file mode 100644 index 00000000..b2e96a9e --- /dev/null +++ b/apex/apex/transformer/_data/_batchsampler.py @@ -0,0 +1,180 @@ +"""BatchSampler implementations for POC of dynamic batch size or rampup_batch_size support. + +Implementations are based on https://github.com/NVIDIA/Megatron-LM/blob/bcd605f8570ebeeb0436c115ebbfafc3c5a40ae5/megatron/data/data_samplers.py. +""" # NOQA +import abc + +import torch + + +__all__ = [ + "MegatronPretrainingSampler", + "MegatronPretrainingRandomSampler", +] + + +class _Base: + """Base class for Megatron style BatchSampler.""" + + @abc.abstractmethod + def __len__(self) -> int: + ... + + @abc.abstractmethod + def __iter__(self): + ... + + @property + @abc.abstractmethod + def local_minibatch_size(self) -> int: + ... + + @local_minibatch_size.setter + @abc.abstractclassmethod + def local_minibatch_size(self) -> None: + ... + + +class MegatronPretrainingSampler(_Base): + + def __init__( + self, + total_samples: int, + consumed_samples: int, + local_minibatch_size: int, + data_parallel_rank: int, + data_parallel_size: int, + drop_last: bool = True, + ): + # Sanity checks. + if total_samples <= 0: + raise RuntimeError('no sample to consume: {}'.format(self.total_samples)) + if consumed_samples >= total_samples: + raise RuntimeError('no samples left to consume: {}, {}'.format(self.consumed_samples, self.total_samples)) + if local_minibatch_size <= 0: + raise RuntimeError(f"local minibatch size must be greater than 0: {local_minibatch_size}") + if data_parallel_size <= 0: + raise RuntimeError(f"data parallel size must be greater than 0: {data_parallel_size}") + if data_parallel_rank >= data_parallel_size: + raise RuntimeError('data_parallel_rank should be smaller than data size: {}, {}'.format(self.data_parallel_rank, data_parallel_size)) + # Keep a copy of input params for later use. + self.total_samples = total_samples + self.consumed_samples = consumed_samples + self._local_minibatch_size = local_minibatch_size + self.data_parallel_rank = data_parallel_rank + self.data_parallel_size = data_parallel_size + self.local_minibatch_times_data_parallel_size = self._local_minibatch_size * data_parallel_size + self.drop_last = drop_last + + def __len__(self): + return self.total_samples + + def get_start_end_idx(self): + start_idx = self.data_parallel_rank * self.local_minibatch_size + end_idx = start_idx + self.local_minibatch_size + return start_idx, end_idx + + @property + def local_minibatch_size(self) -> int: + return self._local_minibatch_size + + @local_minibatch_size.setter + def local_minibatch_size(self, new_local_minibatch_size) -> None: + self._local_minibatch_size = new_local_minibatch_size + self.local_minibatch_times_data_parallel_size = self._local_minibatch_size * self.data_parallel_size + + def __iter__(self): + batch = [] + # Last batch will be dropped if drop_last is not set False + for idx in range(self.consumed_samples, self.total_samples): + batch.append(idx) + if len(batch) == self.local_minibatch_size: + start_idx, end_idx = self.get_start_end_idx() + yield batch[start_idx:end_idx] + batch = [] + + # Check the last partial batch and see drop_last is set + if len(batch) > 0 and not self.drop_last: + start_idx, end_idx = self.get_start_end_idx() + yield batch[start_idx:end_idx] + + +class MegatronPretrainingRandomSampler(_Base): + """Megatron style Random Batch Sampler. + + Major difference is that `__iter__` yields a local minibatch, not a microbatch. + A local minibatch consists of `global_batch_size / data_parallel_size` + + Args: + total_samples: The number of data samples, i.e. ``len(dataset)``. + consumed_samples: The number of samples already consumed in pretraining. + local_minibatch_size: The number of data in each batch returned from `__iter__`. Basically + `local_minibatch_size = global_batch_size / data_parallel_size`. + data_parallel_rank: + data_parallel_size: + """ + + def __init__( + self, + total_samples: int, + consumed_samples: int, + local_minibatch_size: int, + data_parallel_rank: int, + data_parallel_size: int, + ) -> None: + if total_samples <= 0: + raise ValueError(f"no sample to consume: total_samples of {total_samples}") + if local_minibatch_size <= 0: + raise ValueError(f"Invalid local_minibatch_size: {local_minibatch_size}") + if data_parallel_size <= 0: + raise ValueError(f"Invalid data_parallel_size: {data_parallel_size}") + if data_parallel_rank >= data_parallel_size: + raise ValueError( + f"data_parallel_rank should be smaller than data parallel size: {data_parallel_rank} < {data_parallel_size}" + ) + # Keep a copy of input params for later use. + self.total_samples = total_samples + self.consumed_samples = consumed_samples + self._local_minibatch_size = local_minibatch_size + self.data_parallel_rank = data_parallel_rank + self.data_parallel_size = data_parallel_size + self.local_minibatch_times_data_parallel_size = self._local_minibatch_size * self.data_parallel_size + self.last_batch_size = self.total_samples % self.local_minibatch_times_data_parallel_size + + def __len__(self) -> int: + return self.total_samples + + @property + def local_minibatch_size(self) -> int: + return self._local_minibatch_size + + @local_minibatch_size.setter + def local_minibatch_size(self, new_local_minibatch_size) -> None: + self._local_minibatch_size = new_local_minibatch_size + self.local_minibatch_times_data_parallel_size = self._local_minibatch_size * self.data_parallel_size + + def __iter__(self): + active_total_samples = self.total_samples - self.last_batch_size + self.epoch = self.consumed_samples // active_total_samples + current_epoch_samples = self.consumed_samples % active_total_samples + # note(mkozuki): might be better to uncomment + # assert current_epoch_samples % (self.data_parallel_size * apex.transformer.pipeline_parallel.utils.get_micro_batch_size()) == 0 + + # data sharding and random sampling + bucket_size = (self.total_samples // self.local_minibatch_times_data_parallel_size) * self.local_minibatch_size + bucket_offset = current_epoch_samples // self.data_parallel_size + start_idx = self.data_parallel_rank * bucket_size + + g = torch.Generator() + g.manual_seed(self.epoch) + random_idx = torch.randperm(bucket_size, generator=g).tolist() + idx_range = [start_idx + x for x in random_idx[bucket_offset:]] + + batch = [] + # Last batch if not complete will be dropped. + for idx in idx_range: + batch.append(idx) + if len(batch) == self.local_minibatch_size: + self.consumed_samples += self.local_minibatch_times_data_parallel_size + yield batch + batch = [] diff --git a/apex/apex/transformer/_ucc_util.py b/apex/apex/transformer/_ucc_util.py new file mode 100644 index 00000000..a286e2ab --- /dev/null +++ b/apex/apex/transformer/_ucc_util.py @@ -0,0 +1,9 @@ +from torch import distributed as dist + +HAS_UCC = hasattr(dist, "is_ucc_available") and dist.is_ucc_available() +if not HAS_UCC: + try: + import torch_ucc + HAS_UCC = True + except ImportError: + HAS_UCC = False diff --git a/apex/apex/transformer/amp/__init__.py b/apex/apex/transformer/amp/__init__.py new file mode 100644 index 00000000..dbef36a3 --- /dev/null +++ b/apex/apex/transformer/amp/__init__.py @@ -0,0 +1,6 @@ +from apex.transformer.amp.grad_scaler import GradScaler + + +__all__ = [ + "GradScaler", +] diff --git a/apex/apex/transformer/amp/grad_scaler.py b/apex/apex/transformer/amp/grad_scaler.py new file mode 100644 index 00000000..931110af --- /dev/null +++ b/apex/apex/transformer/amp/grad_scaler.py @@ -0,0 +1,125 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict + +import torch + +from apex.transformer import parallel_state + + +class GradScaler(torch.cuda.amp.GradScaler): + """ + Gradient scaler for model-parallel inf check. The inf in gradients are checked across tensor-parallel + ranks in (1) executing optimizer step and (2) gradient scaler update. + """ + + def __init__( + self, init_scale=2.0 ** 16, growth_factor=2.0, backoff_factor=0.5, growth_interval=2000, enabled=True + ): + super().__init__( + init_scale=init_scale, + growth_factor=growth_factor, + backoff_factor=backoff_factor, + growth_interval=growth_interval, + enabled=enabled, + ) + + def _unscale_grads_(self, optimizer, *args): + if getattr(optimizer, "_custom_amp_unscale_grads", False): + return optimizer.unscale_grads(*args) + else: + return super()._unscale_grads_(optimizer, *args) + + def _maybe_opt_step(self, optimizer, optimizer_state, *args, **kwargs): + retval = None + found_inf = torch.cuda.FloatTensor([sum(v.item() for v in optimizer_state["found_inf_per_device"].values())]) + + # Update across all model parallel instances. + torch.distributed.all_reduce( + found_inf, op=torch.distributed.ReduceOp.MAX, group=parallel_state.get_model_parallel_group() + ) + + if found_inf.item() == 0: + retval = optimizer.step(*args, **kwargs) + return retval + + def update(self, new_scale=None): + """ + Updates the scale factor. + If any optimizer steps were skipped the scale is multiplied by ``backoff_factor`` + to reduce it. If ``growth_interval`` unskipped iterations occurred consecutively, + the scale is multiplied by ``growth_factor`` to increase it. + Passing ``new_scale`` sets the new scale value manually. (``new_scale`` is not + used directly, it's used to fill GradScaler's internal scale tensor. So if + ``new_scale`` was a tensor, later in-place changes to that tensor will not further + affect the scale GradScaler uses internally.) + Args: + new_scale (float or :class:`torch.cuda.FloatTensor`, optional, default=None): New scale factor. + .. warning:: + :meth:`update` should only be called at the end of the iteration, after ``scaler.step(optimizer)`` has + been invoked for all optimizers used this iteration. + """ + if not self._enabled: + return + + _scale, _growth_tracker = self._check_scale_growth_tracker("update") + + if new_scale is not None: + # Accept a new user-defined scale. + if isinstance(new_scale, float): + self._scale.fill_(new_scale) # type: ignore[union-attr] + else: + reason = "new_scale should be a float or a 1-element torch.cuda.FloatTensor with requires_grad=False." + assert isinstance(new_scale, torch.cuda.FloatTensor), reason # type: ignore[attr-defined] + assert new_scale.numel() == 1, reason + assert new_scale.requires_grad is False, reason + self._scale.copy_(new_scale) # type: ignore[union-attr] + else: + # Consume shared inf/nan data collected from optimizers to update the scale. + # If all found_inf tensors are on the same device as self._scale, this operation is asynchronous. + found_infs = [ + found_inf.to(device=_scale.device, non_blocking=True) + for state in self._per_optimizer_states.values() + for found_inf in state["found_inf_per_device"].values() + ] + + assert len(found_infs) > 0, "No inf checks were recorded prior to update." + + found_inf_combined = found_infs[0] + + # Update across all model parallel instances. + torch.distributed.all_reduce( + found_inf_combined, op=torch.distributed.ReduceOp.MAX, group=parallel_state.get_model_parallel_group() + ) + + if len(found_infs) > 1: + for i in range(1, len(found_infs)): + found_inf = found_infs[i] + # Update across all model parallel instances. + torch.distributed.all_reduce( + found_inf, op=torch.distributed.ReduceOp.MAX, group=parallel_state.get_model_parallel_group() + ) + found_inf_combined += found_inf + + torch._amp_update_scale_( + _scale, + _growth_tracker, + found_inf_combined, + self._growth_factor, + self._backoff_factor, + self._growth_interval, + ) + + # To prepare for next iteration, clear the data collected from optimizers this iteration. + self._per_optimizer_states = defaultdict(torch.cuda.amp.grad_scaler._refresh_per_optimizer_state) diff --git a/apex/apex/transformer/enums.py b/apex/apex/transformer/enums.py new file mode 100644 index 00000000..78da6c99 --- /dev/null +++ b/apex/apex/transformer/enums.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import enum + + +class LayerType(enum.Enum): + encoder = 1 + decoder = 2 + + +class AttnType(enum.Enum): + self_attn = 1 + cross_attn = 2 + + +class AttnMaskType(enum.Enum): + padding = 1 + causal = 2 + + +class ModelType(enum.Enum): + encoder_or_decoder = 1 + encoder_and_decoder = 2 diff --git a/apex/apex/transformer/functional/__init__.py b/apex/apex/transformer/functional/__init__.py new file mode 100644 index 00000000..d770c885 --- /dev/null +++ b/apex/apex/transformer/functional/__init__.py @@ -0,0 +1,5 @@ +from apex.transformer.functional.fused_softmax import FusedScaleMaskSoftmax + +__all__ = [ + "FusedScaleMaskSoftmax", +] diff --git a/apex/apex/transformer/functional/fused_softmax.py b/apex/apex/transformer/functional/fused_softmax.py new file mode 100644 index 00000000..8d29337a --- /dev/null +++ b/apex/apex/transformer/functional/fused_softmax.py @@ -0,0 +1,301 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from apex._autocast_utils import _cast_if_autocast_enabled +from apex.transformer.enums import AttnMaskType + + +class ScaledUpperTriangMaskedSoftmax(torch.autograd.Function): + """ + Fused operation which performs following three operations in sequence + 1. Scale the tensor. + 2. Apply upper triangular mask (typically used in gpt models). + 3. Perform softmax. + """ + + @staticmethod + def forward(ctx, inputs, scale): + import scaled_upper_triang_masked_softmax_cuda + + scale_t = torch.tensor([scale]) + softmax_results = scaled_upper_triang_masked_softmax_cuda.forward( + inputs, scale_t[0] + ) + + ctx.save_for_backward(softmax_results, scale_t) + return softmax_results + + @staticmethod + def backward(ctx, output_grads): + import scaled_upper_triang_masked_softmax_cuda + + softmax_results, scale_t = ctx.saved_tensors + input_grads = scaled_upper_triang_masked_softmax_cuda.backward( + output_grads, softmax_results, scale_t[0] + ) + + return input_grads, None + + +def scaled_upper_triang_masked_softmax(inputs, _, scale): + b, np, sq, sk = inputs.size() + assert sq == sk, "causal mask is only for self attention" + # Reshaping input to 3D tensor (attn_batches, sq, sk) + inputs = inputs.view(-1, sq, sk) + args = _cast_if_autocast_enabled(inputs, scale) + with torch.cuda.amp.autocast(enabled=False): + probs = ScaledUpperTriangMaskedSoftmax.apply(*args) + return probs.view(b, np, sq, sk) + + +# NOTE (mkozuki): `ScaledMaskedSoftmax` somehow doesn't work well with `torch.cuda.amp.custom_fwd`. +# Without `cast_inputs` kwarg, somehow inputs are not cast to dtype used in the autocast context. +# So I needed to manually write two `torch.autograd.Function` inheritances. +# Fused operation which performs following three operations in sequence +# 1. Scale the tensor. +# 2. Apply the mask. +# 3. Perform softmax. +class ScaledMaskedSoftmax(torch.autograd.Function): + @staticmethod + def forward(ctx, inputs, mask, scale): + import scaled_masked_softmax_cuda + + scale_t = torch.tensor([scale]) + + softmax_results = scaled_masked_softmax_cuda.forward(inputs, mask, scale_t[0]) + ctx.save_for_backward(softmax_results, scale_t) + return softmax_results + + @staticmethod + def backward(ctx, output_grads): + import scaled_masked_softmax_cuda + + softmax_results, scale_t = ctx.saved_tensors + + input_grads = scaled_masked_softmax_cuda.backward( + output_grads, softmax_results, scale_t[0] + ) + return input_grads, None, None + + +def scaled_masked_softmax(inputs, mask, scale): + # input is 4D tensor (b, np, sq, sk) + if mask is not None: + args = _cast_if_autocast_enabled(inputs, mask, scale) + with torch.cuda.amp.autocast(enabled=False): + return ScaledMaskedSoftmax.apply(*args) + else: + args = _cast_if_autocast_enabled(inputs, scale) + with torch.cuda.amp.autocast(enabled=False): + return ScaledSoftmax.apply(*args) + + +class GenericScaledMaskedSoftmax(torch.autograd.Function): + @staticmethod + def forward(ctx, inputs, mask, scale): + import generic_scaled_masked_softmax_cuda + + scale_t = torch.tensor([scale]) + softmax_results = generic_scaled_masked_softmax_cuda.forward(inputs, mask, scale_t[0]) + ctx.save_for_backward(softmax_results, scale_t) + return softmax_results + + @staticmethod + def backward(ctx, output_grads): + import generic_scaled_masked_softmax_cuda_new + + softmax_results, scale_t = ctx.saved_tensors + + input_grads = generic_scaled_masked_softmax_cuda.backward(output_grads, softmax_results, scale_t[0]) + return input_grads, None, None + + +def generic_scaled_masked_softmax(inputs, mask, scale): + # input is 4D tensor (b, np, sq, sk) + args = _cast_if_autocast_enabled(inputs, mask, scale) + with torch.cuda.amp.autocast(enabled=False): + return GenericScaledMaskedSoftmax.apply(*args) + + +class ScaledSoftmax(torch.autograd.Function): + """ + Fused operation which performs following two operations in sequence + 1. Scale the tensor. + 2. Perform softmax. + """ + + @staticmethod + def forward(ctx, inputs, scale): + import scaled_softmax_cuda + + scale_t = torch.tensor([scale]) + + softmax_results = scaled_softmax_cuda.forward( + inputs, scale_t[0] + ) + ctx.save_for_backward(softmax_results, scale_t) + return softmax_results + + @staticmethod + def backward(ctx, output_grads): + import scaled_softmax_cuda + + softmax_results, scale_t = ctx.saved_tensors + + input_grads = scaled_softmax_cuda.backward( + output_grads, softmax_results, scale_t[0] + ) + return input_grads, None, None + + +class FusedScaleMaskSoftmax(torch.nn.Module): + """ + fused operation: scaling + mask + softmax + + Arguments: + input_in_fp16: flag to indicate if input in fp16 data format. + input_in_bf16: flag to indicate if input in bf16 data format. + attn_mask_type: attention mask type (pad or causal) + scaled_masked_softmax_fusion: flag to indicate user want to use softmax fusion + mask_func: mask function to be applied. + softmax_in_fp32: if true, softmax in performed at fp32 precision. + scale: scaling factor used in input tensor scaling. + """ + + def __init__( + self, + input_in_fp16, + input_in_bf16, + attn_mask_type, + scaled_masked_softmax_fusion, + mask_func, + softmax_in_fp32, + scale, + ): + super().__init__() + self.input_in_fp16 = input_in_fp16 + self.input_in_bf16 = input_in_bf16 + if self.input_in_fp16 and self.input_in_bf16: + raise RuntimeError( + "both fp16 and bf16 flags cannot be active at the same time." + ) + self.input_in_float16 = self.input_in_fp16 or self.input_in_bf16 + self.attn_mask_type = attn_mask_type + self.scaled_masked_softmax_fusion = scaled_masked_softmax_fusion + self.mask_func = mask_func + self.softmax_in_fp32 = softmax_in_fp32 + self.scale = scale + + if not (self.scale is None or softmax_in_fp32): + raise RuntimeError("softmax should be in fp32 when scaled") + + if self.scaled_masked_softmax_fusion: + if self.attn_mask_type == AttnMaskType.causal: + self.fused_softmax_func = scaled_upper_triang_masked_softmax + elif self.attn_mask_type == AttnMaskType.padding: + self.fused_softmax_func = scaled_masked_softmax + else: + raise ValueError("Invalid attn_mask_type.") + + def forward(self, input, mask): + # [b, np, sq, sk] + assert input.dim() == 4 + + if self.is_kernel_available(mask, *input.size()): + return self.forward_fused_softmax(input, mask) + else: + return self.forward_torch_softmax(input, mask) + + def is_kernel_available(self, mask, b, np, sq, sk): + attn_batches = b * np + + if ( + self.scaled_masked_softmax_fusion # user want to fuse + and self.input_in_float16 # input must be fp16 + and ( + self.attn_mask_type == AttnMaskType.causal + or self.attn_mask_type == AttnMaskType.padding + ) + and 16 < sk <= 16384 # sk must be 16 ~ 16384 + and sq % 4 == 0 # sq must be divisor of 4 + and sk % 4 == 0 # sk must be divisor of 4 + and attn_batches % 4 == 0 # np * b must be divisor of 4 + ): + if 0 <= sk <= 16384: + batch_per_block = self.get_batch_per_block(sq, sk, b, np) + + if self.attn_mask_type == AttnMaskType.causal: + if attn_batches % batch_per_block == 0: + return True + else: + if sq % batch_per_block == 0: + return True + return False + + def forward_fused_softmax(self, input, mask): + # input.shape = [b, np, sq, sk] + scale = self.scale if self.scale is not None else 1.0 + return self.fused_softmax_func(input, mask, scale) + + def forward_torch_softmax(self, input, mask): + if self.input_in_float16 and self.softmax_in_fp32: + input = input.float() + + if self.scale is not None: + input = input * self.scale + mask_output = self.mask_func(input, mask) if mask is not None else input + probs = torch.nn.Softmax(dim=-1)(mask_output) + + if self.input_in_float16 and self.softmax_in_fp32: + if self.input_in_fp16: + probs = probs.half() + else: + probs = probs.bfloat16() + + return probs + + @staticmethod + def get_batch_per_block(sq, sk, b, np): + import scaled_masked_softmax_cuda + + return scaled_masked_softmax_cuda.get_batch_per_block(sq, sk, b, np) + +class GenericFusedScaleMaskSoftmax(FusedScaleMaskSoftmax): + """ + Generic version of FusedSacleMaskSoftmax. + It removes the seq-len limitations and has slight performance degragation compared with FusedScaleMaskSoftmax + + fused operation: scaling + mask + softmax + + Arguments: + input_in_fp16: flag to indicate if input in fp16 data format. + input_in_bf16: flag to indicate if input in bf16 data format. + scaled_masked_softmax_fusion: flag to indicate user want to use softmax fusion + mask_func: mask function to be applied. + softmax_in_fp32: if true, softmax in performed at fp32 precision. + scale: scaling factor used in input tensor scaling. + """ + + def __init__( + self, input_in_fp16, input_in_bf16, scaled_masked_softmax_fusion, mask_func, softmax_in_fp32, scale, + ): + super().__init__(input_in_fp16, input_in_bf16, AttnMaskType.padding, scaled_masked_softmax_fusion, mask_func, softmax_in_fp32, scale) + self.scaled_masked_softmax_fusion = generic_scaled_masked_softmax + + def is_kernel_available(self, mask, b, np, sq, sk): + if self.scaled_masked_softmax_fusion and 0 < sk: # user want to fuse # sk must be 1 ~ + return True + return False \ No newline at end of file diff --git a/apex/apex/transformer/layers/__init__.py b/apex/apex/transformer/layers/__init__.py new file mode 100644 index 00000000..bc247d3c --- /dev/null +++ b/apex/apex/transformer/layers/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +from apex.transformer.layers.layer_norm import FastLayerNorm +from apex.transformer.layers.layer_norm import FusedLayerNorm +from apex.transformer.layers.layer_norm import MixedFusedLayerNorm + + +__all__ = [ + "FastLayerNorm", + "FusedLayerNorm", + "MixedFusedLayerNorm", +] diff --git a/apex/apex/transformer/layers/layer_norm.py b/apex/apex/transformer/layers/layer_norm.py new file mode 100644 index 00000000..81cc2397 --- /dev/null +++ b/apex/apex/transformer/layers/layer_norm.py @@ -0,0 +1,99 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# NOTE(mkozuki): This file defines two LayerNorm that are compatible with Megatron-LM. +# while avoiding introducing the breaking change of `"sequence_parallel_enabled"` attribute into apex.normalization.FusedLayerNorm +# and apex.contrib.layer_norm.FastLayerNorm. +import warnings + +import torch + +from apex.normalization import FusedLayerNorm as OrigFusedLayerNorm +from apex.normalization import MixedFusedLayerNorm as OrigMixedFusedLayerNorm +try: + from apex.contrib.layer_norm import FastLayerNorm as OrigFastLayerNorm +except ImportError: + HAS_FAST_LAYER_NORM = False +else: + HAS_FAST_LAYER_NORM = True + + +__all__ = [ + "FusedLayerNorm", + "FastLayerNorm", + "MixedFusedLayerNorm", +] + + +def _set_sequence_parallel_enabled( + param: torch.Tensor, + sequence_parallel_enabled: bool, +) -> None: + setattr(param, "sequence_parallel_enabled", sequence_parallel_enabled) + + +class FusedLayerNorm(OrigFusedLayerNorm): + def __init__( + self, + normalized_shape, + eps: float = 1e-5, + elementwise_affine: bool = True, + *, + sequence_parallel_enabled: bool = False, + ): + super().__init__( + normalized_shape=normalized_shape, + eps=eps, + elementwise_affine=elementwise_affine, + ) + self.sequence_parallel_enabled = sequence_parallel_enabled + if self.elementwise_affine: + _set_sequence_parallel_enabled(self.weight, self.sequence_parallel_enabled) + _set_sequence_parallel_enabled(self.bias, self.sequence_parallel_enabled) + + +# note: MixedFusedLayerNorm is no different from FusedLayerNorm if it's used in `torch.cuda.amp`. +class MixedFusedLayerNorm(OrigMixedFusedLayerNorm): + def __init__( + self, + normalized_shape, + eps: float = 1e-5, + **kwargs, + ) -> None: + self.sequence_parallel_enabled = kwargs.get("sequence_parallel_enabled", False) + super().__init__(normalized_shape=normalized_shape, eps=eps, **kwargs) + if self.sequence_parallel_enabled: + _set_sequence_parallel_enabled(self.weight, self.sequence_parallel_enabled) + _set_sequence_parallel_enabled(self.bias, self.sequence_parallel_enabled) + + +if HAS_FAST_LAYER_NORM: + class FastLayerNorm(OrigFastLayerNorm): + def __init__( + self, + hidden_size, + eps: float = 1e-5, + *, + sequence_parallel_enabled: bool = False, + ): + super().__init__( + hidden_size=hidden_size, + eps=eps + ) + self.sequence_parallel_enabled = sequence_parallel_enabled + _set_sequence_parallel_enabled(self.weight, self.sequence_parallel_enabled) + _set_sequence_parallel_enabled(self.bias, self.sequence_parallel_enabled) +else: + class FastLayerNorm(FusedLayerNorm): + def __init__( + self, + hidden_size, + eps: float = 1e-5, + *, + sequence_parallel_enabled: bool = False, + ): + warnings.warn("`apex.contrib.layer_norm.FastLayerNorm` isn't available thus falling back to `apex.normalization.FusedLayerNorm`") + super().__init__( + normalized_shape=hidden_size, + eps=eps, + elementwise_affine=True, + sequence_parallel_enabled=sequence_parallel_enabled, + ) diff --git a/apex/apex/transformer/log_util.py b/apex/apex/transformer/log_util.py new file mode 100644 index 00000000..7eaafee2 --- /dev/null +++ b/apex/apex/transformer/log_util.py @@ -0,0 +1,18 @@ +import logging +import os + + +def get_transformer_logger(name: str) -> logging.Logger: + name_wo_ext = os.path.splitext(name)[0] + return logging.getLogger(name_wo_ext) + + +def set_logging_level(verbosity) -> None: + """Change logging severity. + + Args: + verbosity + """ + from apex import _library_root_logger + + _library_root_logger.setLevel(verbosity) diff --git a/apex/apex/transformer/microbatches.py b/apex/apex/transformer/microbatches.py new file mode 100644 index 00000000..69673bc1 --- /dev/null +++ b/apex/apex/transformer/microbatches.py @@ -0,0 +1,195 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Megatron number of micro-batches calculators.""" +from abc import ABC +from abc import abstractmethod +from typing import Optional, List + +from apex.transformer.log_util import get_transformer_logger + + +_logger = get_transformer_logger(__name__) + + +def build_num_microbatches_calculator( + rank: int, + rampup_batch_size: Optional[List[int]], + global_batch_size: int, + micro_batch_size: int, + data_parallel_size: int, +): + # Constant num micro-batches. + if rampup_batch_size is None: + num_microbatches_calculator = ConstantNumMicroBatches( + global_batch_size, micro_batch_size, data_parallel_size + ) + if rank == 0: + _logger.info( + "setting number of micro-batches to constant {}".format( + num_microbatches_calculator.get() + ) + ) + + else: + assert len(rampup_batch_size) == 3, ( + "expected the following " + "format: --rampup-batch-size " + " " + ) + start_batch_size = int(rampup_batch_size[0]) + batch_size_increment = int(rampup_batch_size[1]) + ramup_samples = int(rampup_batch_size[2]) + if rank == 0: + _logger.info( + "will use batch size rampup starting from global batch " + "size {} to global batch size {} with batch size increments " + "{} over {} samples.".format( + start_batch_size, + global_batch_size, + batch_size_increment, + ramup_samples, + ) + ) + num_microbatches_calculator = RampupBatchsizeNumMicroBatches( + start_batch_size, + batch_size_increment, + ramup_samples, + global_batch_size, + micro_batch_size, + data_parallel_size, + ) + + return num_microbatches_calculator + + +class NumMicroBatchesCalculator(ABC): + def __init__(self): + self.num_micro_batches = None + self.current_global_batch_size = None + + def get(self): + return self.num_micro_batches + + def get_current_global_batch_size(self): + return self.current_global_batch_size + + @abstractmethod + def update(self, consumed_samples, consistency_check): + pass + + +class ConstantNumMicroBatches(NumMicroBatchesCalculator): + def __init__(self, global_batch_size, micro_batch_size, data_parallel_size): + micro_batch_times_data_parallel = micro_batch_size * data_parallel_size + assert global_batch_size % micro_batch_times_data_parallel == 0, ( + "global batch size ({}) is not divisible by micro batch size ({})" + " times data parallel size ({})".format( + global_batch_size, micro_batch_size, data_parallel_size + ) + ) + self.num_micro_batches = global_batch_size // micro_batch_times_data_parallel + assert self.num_micro_batches >= 1 + self.current_global_batch_size = global_batch_size + + self.micro_batch_size = micro_batch_size + + def update(self, consumed_samples, consistency_check): + pass + + +class RampupBatchsizeNumMicroBatches(NumMicroBatchesCalculator): + def __init__( + self, + start_batch_size, + batch_size_increment, + ramup_samples, + global_batch_size, + micro_batch_size, + data_parallel_size, + ): + """Batch size ramp up. + Over + steps = (global-batch-size - start-batch-size) / batch_size_increment + increment batch size from start-batch-size to global-batch-size using + rampup-samples / steps + samples. + Arguments: + start_batch_size: global batch size to start with + batch_size_increment: global batch size increments + ramup_samples: number of samples to use ramp up global + batch size from `start_batch_size` to `global_batch_size` + global_batch_size: global batch size post rampup + micro_batch_size: micro batch size + data_parallel_size: data parallel size. + """ + + self.micro_batch_size = micro_batch_size + self.data_parallel_size = data_parallel_size + self.micro_batch_times_data_parallel_size = ( + self.micro_batch_size * self.data_parallel_size + ) + assert self.micro_batch_times_data_parallel_size > 0 + + assert start_batch_size > 0 + self.start_batch_size = start_batch_size + + assert global_batch_size > 0 + self.global_batch_size = global_batch_size + diff_batch_size = self.global_batch_size - self.start_batch_size + assert diff_batch_size >= 0 + assert batch_size_increment > 0 + self.batch_size_increment = batch_size_increment + assert diff_batch_size % batch_size_increment == 0, ( + "expected " + "global batch size interval ({}) to be divisible by global batch " + "size increment ({})".format(diff_batch_size, batch_size_increment) + ) + + num_increments = diff_batch_size // self.batch_size_increment + self.ramup_samples = ramup_samples + assert self.ramup_samples >= 0 + self.rampup_samples_per_increment = self.ramup_samples / num_increments + + # Initialize number of microbatches. + self.update(0, False) + + def update(self, consumed_samples, consistency_check): + + if consumed_samples > self.ramup_samples: + self.current_global_batch_size = self.global_batch_size + else: + steps = int(consumed_samples / self.rampup_samples_per_increment) + self.current_global_batch_size = ( + self.start_batch_size + steps * self.batch_size_increment + ) + assert self.current_global_batch_size <= self.global_batch_size + + if consistency_check: + assert ( + self.current_global_batch_size + % self.micro_batch_times_data_parallel_size + == 0 + ), ( + "current global " + "batch size ({}) is not divisible by micro-batch-size ({}) times" + "data parallel size ({})".format( + self.current_global_batch_size, + self.micro_batch_size, + self.data_parallel_size, + ) + ) + self.num_micro_batches = ( + self.current_global_batch_size // self.micro_batch_times_data_parallel_size + ) diff --git a/apex/apex/transformer/parallel_state.py b/apex/apex/transformer/parallel_state.py new file mode 100644 index 00000000..3d555d65 --- /dev/null +++ b/apex/apex/transformer/parallel_state.py @@ -0,0 +1,796 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# TODO (mkozuki): Replace assert with RuntimeError. +# TODO (mkozuki): Sort the functions in the same order of megatron/mpu/initialize.py +"""Model and data parallel groups.""" +from typing import Tuple, Optional +import warnings +import os +import torch + +from apex.transformer.log_util import get_transformer_logger +from apex.transformer._ucc_util import HAS_UCC + + +_logger = get_transformer_logger(__name__) + +# N.B. (mkozuki): Diff btwn Megatron-LM & apex parallel_state +# set(megatron_mpu_initialize_funcs) - set(apex.transformer.parallel_state) = +# { +# 'get_num_layers', +# } + + +# Intra-layer model parallel group that the current rank belongs to. +_TENSOR_MODEL_PARALLEL_GROUP = None +# Inter-layer model parallel group that the current rank belongs to. +_PIPELINE_MODEL_PARALLEL_GROUP = None +# Model parallel group (both intra- and pipeline) that the current rank belongs to. +_MODEL_PARALLEL_GROUP = None +# Embedding group. +_EMBEDDING_GROUP = None +# Position embedding group. +_POSITION_EMBEDDING_GROUP = None +# Relative position embedding group. +_ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP = None +_DECODER_RELATIVE_POSITION_EMBEDDING_GROUP = None +# Data parallel group that the current rank belongs to. +_DATA_PARALLEL_GROUP = None +# Data parallel AMAX reduction group that the current rank belongs to. +_AMAX_REDUCTION_GROUP = None + +_VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None +_VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None +_PIPELINE_MODEL_PARALLEL_SPLIT_RANK = None + +# These values enable us to change the mpu sizes on the fly. +_MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None +_MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None +_MPU_TENSOR_MODEL_PARALLEL_RANK = None +_MPU_PIPELINE_MODEL_PARALLEL_RANK = None + +# A list of ranks that have a copy of the embedding. +_EMBEDDING_GLOBAL_RANKS = None + +# A list of ranks that have a copy of the position embedding. +_POSITION_EMBEDDING_GLOBAL_RANKS = None + +# A list of ranks that have a copy of the relative position embedding. +_ENCODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS = None +_DECODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS = None + +# A list of global ranks for each pipeline group to ease calculation of the source +# rank when broadcasting from the first or last pipeline stage +_PIPELINE_GLOBAL_RANKS = None + + +def is_unitialized(): + """Useful for code segments that may be accessed with or without mpu initialization""" + return _DATA_PARALLEL_GROUP is None + +def set_nccl_socket_envs(): + if os.getenv("NCCL_SOCKET_IFNAME") is None: + raise RuntimeError("NCCL_SOCKET_IFNAME was not set") + os.environ["NCCL_NET"] = "Socket" + +def set_nccl_ib_envs(): + os.environ["NCCL_NET"] = "IB" + +def init_nccl_net(group): + temp = torch.ones(1, device="cuda") + torch.distributed.all_reduce(temp, group=group) + torch.cuda.synchronize() + +def new_nccl_socket_group(ranks): + set_nccl_socket_envs() + group = torch.distributed.new_group(ranks, backend="nccl") + init_nccl_net(group=group) + return group + +def new_nccl_ib_group(ranks): + set_nccl_ib_envs() + group = torch.distributed.new_group(ranks, backend="nccl") + init_nccl_net(group=group) + return group + +def new_process_group(ranks, backend): + """ + This function creates process groups. + + In addition to simply creating the process groups, it initializes NCCL + for hybrid IB/Socket network like in the following diagram: + + ____________ + [GPU Node 0]---TCP---| |---TCP---[GPU Node 2] + | | | | + | | | | + IB | IP Network | IB + | | | | + | | | | + [GPU Node 1]---TCP---|____________|---TCP---[GPU Node 3] + + + If an environment variable NUM_GPUS_PER_IB_BLOCK is defined it looks up the ranks + and determines whether the list of ranks belong to the same computational block where + GPUs nodes are interconnected via IB type of connection or not. + If all ranks are in the same block, the process group will use NCCL_NET=IB for + communication, otherwise it will use NCCL_NET=Socket. + + If NCCL_NET=Socket is ever to be used, the user must set NCCL_SOCKET_IFNAME. + Additionally, it is recommended to set NCCL_SOCKET_NTHREADS and + NCCL_NSOCKS_PERTHREAD before running the job. + See: https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html + for more info + + The core assumption for this functionality is that the ranks are evenly divided + into IB blocks and all these IB blocks are of the same size. + """ + if backend is None: + backend = "nccl" + + compute_block_size = os.getenv("NUM_GPUS_PER_IB_BLOCK") + if backend == "nccl" and compute_block_size is not None: + compute_block_size = int(compute_block_size) + blocks = [rank // compute_block_size for rank in ranks] + use_ib = all(block == blocks[0] for block in blocks) + if use_ib: + return new_nccl_ib_group(ranks) + else: + return new_nccl_socket_group(ranks) + else: + return torch.distributed.new_group(ranks, backend=backend) + +def initialize_model_parallel( + tensor_model_parallel_size_: int = 1, + pipeline_model_parallel_size_: int = 1, + virtual_pipeline_model_parallel_size_: Optional[int] = None, + pipeline_model_parallel_split_rank_: Optional[int] = None, + use_fp8_: bool = False, + init_mpi_proc_group: bool = False, + *, + default_backend: Optional[str] = None, + p2p_backend: Optional[str] = None, +) -> None: + """ + Initialize model data parallel groups. + + Arguments: + tensor_model_parallel_size: number of GPUs used to parallelize model tensor. + pipeline_model_parallel_size: number of GPUs used to parallelize model pipeline. + virtual_pipeline_model_parallel_size: number of virtual stages (interleaved pipeline). + pipeline_model_parallel_split_rank: for models with both encoder and decoder, rank in pipeline with split point. + use_fp8_: FP8 training that needs AMAX reduction across data-parallel ranks. + init_mpi_proc_group: Create a MPI process group, which is used for UCX-based communication APIs. + Keyword Arguments: + default_backend: Backend of process groups except for pipeline parallel ones. + If :obj:`None`, the backend specified in `torch.distributed.init_process_group` will be used. + p2p_backend: Backend of process groups for pipeline model parallel. + If :obj:`None`, the backend specified in `torch.distributed.init_process_group` will be used. + + .. note:: + `torch_ucc `_ is + necessary for "ucc" backend. + + Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we + use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize + the model pipeline. The present function will + create 8 tensor model-parallel groups, 4 pipeline model-parallel groups + and 8 data-parallel groups as: + 8 data_parallel groups: + [g0, g2], [g1, g3], [g4, g6], [g5, g7], [g8, g10], [g9, g11], [g12, g14], [g13, g15] + 8 tensor model-parallel groups: + [g0, g1], [g2, g3], [g4, g5], [g6, g7], [g8, g9], [g10, g11], [g12, g13], [g14, g15] + 4 pipeline model-parallel groups: + [g0, g4, g8, g12], [g1, g5, g9, g13], [g2, g6, g10, g14], [g3, g7, g11, g15] + Note that for efficiency, the caller should make sure adjacent ranks + are on the same DGX box. For example if we are using 2 DGX-1 boxes + with a total of 16 GPUs, rank 0 to 7 belong to the first box and + ranks 8 to 15 belong to the second box. + """ + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + assert default_backend is None or default_backend in ("nccl", "ucc") + assert p2p_backend is None or p2p_backend in ("nccl", "ucc") + if "ucc" in (default_backend, p2p_backend): + if not HAS_UCC: + raise ImportError("UCC backend requires pytorch source build with UCC installed and enabled") + warnings.warn("`ucc` backend support is experimental", ExperimentalWarning) + if default_backend == "ucc": + warnings.warn("The UCC's functionality as `default_backend` is not well verified", ExperimentalWarning) + + # Saving the NCCL_NET type for reusing it at the epilogue + default_nccl_net = os.getenv("NCCL_NET") + + world_size: int = torch.distributed.get_world_size() + tensor_model_parallel_size: int = min(tensor_model_parallel_size_, world_size) + pipeline_model_parallel_size: int = min(pipeline_model_parallel_size_, world_size) + if world_size % (tensor_model_parallel_size * pipeline_model_parallel_size) != 0: + raise RuntimeError( + f"`world_size` ({world_size}) is not divisible by tensor_model_parallel_size ({tensor_model_parallel_size}) x pipeline_model_parallel_size ({pipeline_model_parallel_size})" + ) + data_parallel_size: int = world_size // ( + tensor_model_parallel_size * pipeline_model_parallel_size + ) + if torch.distributed.get_rank() == 0: + _logger.info( + "> initializing tensor model parallel with size {}".format( + tensor_model_parallel_size + ) + ) + _logger.info( + "> initializing pipeline model parallel with size {}".format( + pipeline_model_parallel_size + ) + ) + _logger.info( + "> initializing data parallel with size {}".format(data_parallel_size) + ) + + num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size + num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size + num_data_parallel_groups: int = world_size // data_parallel_size + + if virtual_pipeline_model_parallel_size_ is not None: + # n.b. (eqy) This check was inherited from Megatron-LM, need to revisit + # the root cause as we do see numerical mismatches with 2 stages and + # the interleaved schedule + assert pipeline_model_parallel_size_ > 2, ( + "pipeline-model-parallel size should be greater than 2 with " + "interleaved schedule" + ) + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = 0 + _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = ( + virtual_pipeline_model_parallel_size_ + ) + + if pipeline_model_parallel_split_rank_ is not None: + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK + _PIPELINE_MODEL_PARALLEL_SPLIT_RANK = pipeline_model_parallel_split_rank_ + + rank = torch.distributed.get_rank() + + # Build the data-parallel groups. + global _DATA_PARALLEL_GROUP + assert _DATA_PARALLEL_GROUP is None, "data parallel group is already initialized" + all_data_parallel_group_ranks = [] + for i in range(pipeline_model_parallel_size): + start_rank = i * num_pipeline_model_parallel_groups + end_rank = (i + 1) * num_pipeline_model_parallel_groups + for j in range(tensor_model_parallel_size): + ranks = range(start_rank + j, end_rank, tensor_model_parallel_size) + all_data_parallel_group_ranks.append(list(ranks)) + group = new_process_group(ranks, backend=default_backend) + if rank in ranks: + _DATA_PARALLEL_GROUP = group + + # Build the amax-reduction groups for fp8 precision conversion. + if use_fp8_: + global _AMAX_REDUCTION_GROUP + assert _AMAX_REDUCTION_GROUP is None, "amax reduction group is already initialized" + amax_group_size: int = tensor_model_parallel_size * data_parallel_size + num_amax_groups: int = world_size // amax_group_size + for i in range(num_amax_groups): + start_rank = i * amax_group_size + end_rank = (i + 1) * amax_group_size + ranks = range(start_rank, end_rank) + group = torch.distributed.new_group(ranks, backend=default_backend) + if rank in ranks: + _AMAX_REDUCTION_GROUP = group + + # Build the model-parallel groups. + global _MODEL_PARALLEL_GROUP + assert _MODEL_PARALLEL_GROUP is None, "model parallel group is already initialized" + for i in range(data_parallel_size): + ranks = [ + data_parallel_group_ranks[i] + for data_parallel_group_ranks in all_data_parallel_group_ranks + ] + group = new_process_group(ranks, backend=default_backend) + if rank in ranks: + _MODEL_PARALLEL_GROUP = group + + # Build the tensor model-parallel groups. + global _TENSOR_MODEL_PARALLEL_GROUP + assert ( + _TENSOR_MODEL_PARALLEL_GROUP is None + ), "tensor model parallel group is already initialized" + for i in range(num_tensor_model_parallel_groups): + ranks = list( + range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) + ) + group = new_process_group(ranks, backend=default_backend) + if rank in ranks: + _TENSOR_MODEL_PARALLEL_GROUP = group + + # Build the pipeline model-parallel groups and embedding groups + # (first and last rank in each pipeline model-parallel group). + global _PIPELINE_MODEL_PARALLEL_GROUP + global _PIPELINE_GLOBAL_RANKS + assert ( + _PIPELINE_MODEL_PARALLEL_GROUP is None + ), "pipeline model parallel group is already initialized" + global _EMBEDDING_GROUP + global _EMBEDDING_GLOBAL_RANKS + assert _EMBEDDING_GROUP is None, "embedding group is already initialized" + global _POSITION_EMBEDDING_GROUP + global _POSITION_EMBEDDING_GLOBAL_RANKS + assert ( + _POSITION_EMBEDDING_GROUP is None + ), "position embedding group is already initialized" + global _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP + global _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP + global _ENCODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS + global _DECODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS + assert _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP is None or \ + _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP is None, \ + 'relative position embedding group is already initialized' + for i in range(num_pipeline_model_parallel_groups): + ranks = range(i, world_size, num_pipeline_model_parallel_groups) + group = new_process_group(ranks, backend=p2p_backend) + if rank in ranks: + _PIPELINE_MODEL_PARALLEL_GROUP = group + _PIPELINE_GLOBAL_RANKS = ranks + # Setup embedding group (to exchange gradients between + # first and last stages). + encoder_relative_position_embedding_ranks = None + decoder_relative_position_embedding_ranks = None + if len(ranks) > 1: + embedding_ranks = [ranks[0], ranks[-1]] + position_embedding_ranks = [ranks[0]] + encoder_relative_position_embedding_ranks = [ranks[0]] + decoder_relative_position_embedding_ranks = [ranks[0]] + if pipeline_model_parallel_split_rank_ is not None: + encoder_relative_position_embedding_ranks = \ + ranks[:pipeline_model_parallel_split_rank_] + decoder_relative_position_embedding_ranks = \ + ranks[pipeline_model_parallel_split_rank_:] + if ranks[pipeline_model_parallel_split_rank_] not in embedding_ranks: + embedding_ranks = [ + ranks[0], + ranks[pipeline_model_parallel_split_rank_], + ranks[-1], + ] + if ( + ranks[pipeline_model_parallel_split_rank_] + not in position_embedding_ranks + ): + position_embedding_ranks = [ + ranks[0], + ranks[pipeline_model_parallel_split_rank_], + ] + else: + embedding_ranks = ranks + position_embedding_ranks = ranks + encoder_relative_position_embedding_ranks = ranks + decoder_relative_position_embedding_ranks = ranks + + group = new_process_group(embedding_ranks, backend=p2p_backend) + if rank in embedding_ranks: + _EMBEDDING_GROUP = group + if rank in ranks: + _EMBEDDING_GLOBAL_RANKS = embedding_ranks + + group = new_process_group(position_embedding_ranks, backend=p2p_backend) + if rank in position_embedding_ranks: + _POSITION_EMBEDDING_GROUP = group + if rank in ranks: + _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks + + if encoder_relative_position_embedding_ranks: + group = new_process_group(encoder_relative_position_embedding_ranks, backend=p2p_backend) + if rank in encoder_relative_position_embedding_ranks: + _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP = group + if rank in ranks: + _ENCODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS = \ + encoder_relative_position_embedding_ranks + + if decoder_relative_position_embedding_ranks: + group = new_process_group(decoder_relative_position_embedding_ranks, backend=p2p_backend) + if rank in decoder_relative_position_embedding_ranks: + _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP = group + if rank in ranks: + _DECODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS = \ + decoder_relative_position_embedding_ranks + + if init_mpi_proc_group: + torch.distributed.new_group(backend='mpi') + + if default_nccl_net == "Socket": + set_nccl_socket_envs() + elif default_nccl_net == "IB": + set_nccl_ib_envs() + elif default_nccl_net is None: + os.unsetenv("NCCL_NET") + else: + os.environ["NCCL_NET"] = default_nccl_net + +def get_rank_info() -> Tuple[int, int, int]: + """Returns a tuple of (data, tensor, pipeline, virtual pipeline)-parallel-rank for logger.""" + if model_parallel_is_initialized(): + return ( + get_data_parallel_rank(), + get_tensor_model_parallel_rank(), + get_pipeline_model_parallel_rank(), + get_virtual_pipeline_model_parallel_rank(), + ) + return (0, 0, 0, 0) + + +def model_parallel_is_initialized(): + """Check if model and data parallel groups are initialized.""" + if ( + _TENSOR_MODEL_PARALLEL_GROUP is None + or _PIPELINE_MODEL_PARALLEL_GROUP is None + or _DATA_PARALLEL_GROUP is None + ): + return False + return True + + +def get_model_parallel_group(): + """Get the model parallel group the caller rank belongs to.""" + assert _MODEL_PARALLEL_GROUP is not None, "model parallel group is not initialized" + return _MODEL_PARALLEL_GROUP + + +def get_tensor_model_parallel_group(): + """Get the tensor model parallel group the caller rank belongs to.""" + assert ( + _TENSOR_MODEL_PARALLEL_GROUP is not None + ), "intra_layer_model parallel group is not initialized" + return _TENSOR_MODEL_PARALLEL_GROUP + + +def get_pipeline_model_parallel_group(): + """Get the pipeline model parallel group the caller rank belongs to.""" + assert ( + _PIPELINE_MODEL_PARALLEL_GROUP is not None + ), "pipeline_model parallel group is not initialized" + return _PIPELINE_MODEL_PARALLEL_GROUP + + +def get_data_parallel_group(): + """Get the data parallel group the caller rank belongs to.""" + assert _DATA_PARALLEL_GROUP is not None, "data parallel group is not initialized" + return _DATA_PARALLEL_GROUP + + +def get_amax_reduction_group(): + """Get the amax reduction group the caller rank belongs to.""" + assert _AMAX_REDUCTION_GROUP is not None, \ + "AMAX reduction group is not initialized" + return _AMAX_REDUCTION_GROUP + + +def get_embedding_group(): + """Get the embedding group the caller rank belongs to.""" + assert _EMBEDDING_GROUP is not None, "embedding group is not initialized" + return _EMBEDDING_GROUP + + +def get_position_embedding_group(): + """Get the position embedding group the caller rank belongs to.""" + assert ( + _POSITION_EMBEDDING_GROUP is not None + ), "position embedding group is not initialized" + return _POSITION_EMBEDDING_GROUP + +def get_encoder_relative_position_embedding_group(): + """Get the encoder relative position embedding group the caller rank belongs to.""" + assert _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP is not None, \ + 'encoder relative position embedding group is not initialized' + return _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP + +def get_decoder_relative_position_embedding_group(): + """Get the decoder relative position embedding group the caller rank belongs to.""" + assert _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP is not None, \ + 'decoder relative position embedding group is not initialized' + return _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP + +def is_rank_in_embedding_group(ignore_virtual=False): + """Return true if current rank is in embedding group, False otherwise.""" + rank = torch.distributed.get_rank() + global _EMBEDDING_GLOBAL_RANKS + if ignore_virtual: + return rank in _EMBEDDING_GLOBAL_RANKS + if rank in _EMBEDDING_GLOBAL_RANKS: + if rank == _EMBEDDING_GLOBAL_RANKS[0]: + return is_pipeline_first_stage(ignore_virtual=False) + elif rank == _EMBEDDING_GLOBAL_RANKS[-1]: + return is_pipeline_last_stage(ignore_virtual=False) + else: + return True + return False + + +def is_rank_in_position_embedding_group(): + """Return whether the current rank is in position embedding group.""" + rank = torch.distributed.get_rank() + global _POSITION_EMBEDDING_GLOBAL_RANKS + return rank in _POSITION_EMBEDDING_GLOBAL_RANKS + +def is_rank_in_encoder_relative_position_embedding_group(): + """Return true if current rank is in encoder relative position embedding group, False otherwise.""" + rank = torch.distributed.get_rank() + global _ENCODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS + return rank in _ENCODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS + +def is_rank_in_decoder_relative_position_embedding_group(): + """Return true if current rank is in decoder relative position embedding group, False otherwise.""" + rank = torch.distributed.get_rank() + global _DECODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS + return rank in _DECODER_RELATIVE_POSITION_EMBEDDING_GLOBAL_RANKS + +def is_pipeline_stage_before_split(rank=None): + """Return True if pipeline stage executes encoder block for a model + with both encoder and decoder.""" + if get_pipeline_model_parallel_world_size() == 1: + return True + if rank is None: + rank = get_pipeline_model_parallel_rank() + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK + if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None: + return True + if rank < _PIPELINE_MODEL_PARALLEL_SPLIT_RANK: + return True + return False + + +def is_pipeline_stage_after_split(rank=None): + """Return True if pipeline stage executes decoder block for a model + with both encoder and decoder.""" + if get_pipeline_model_parallel_world_size() == 1: + return True + if rank is None: + rank = get_pipeline_model_parallel_rank() + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK + if _PIPELINE_MODEL_PARALLEL_SPLIT_RANK is None: + return True + if rank >= _PIPELINE_MODEL_PARALLEL_SPLIT_RANK: + return True + return False + + +def is_pipeline_stage_at_split(): + """Return true if pipeline stage executes decoder block and next + stage executes encoder block for a model with both encoder and + decoder.""" + rank = get_pipeline_model_parallel_rank() + return is_pipeline_stage_before_split(rank) and is_pipeline_stage_after_split( + rank + 1 + ) + + +def set_tensor_model_parallel_world_size(world_size): + """Set the tensor model parallel size""" + global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE + _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = world_size + + +def set_pipeline_model_parallel_world_size(world_size): + """Set the pipeline model parallel size""" + global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = world_size + + +def get_tensor_model_parallel_world_size(): + """Return world size for the tensor model parallel group.""" + global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE + if _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE is not None: + return _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE + return torch.distributed.get_world_size(group=get_tensor_model_parallel_group()) + + +def get_pipeline_model_parallel_world_size(): + """Return world size for the pipeline model parallel group.""" + global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + if _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE is not None: + return _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + return torch.distributed.get_world_size(group=get_pipeline_model_parallel_group()) + + +def set_tensor_model_parallel_rank(rank): + """Set tensor model parallel rank.""" + global _MPU_TENSOR_MODEL_PARALLEL_RANK + _MPU_TENSOR_MODEL_PARALLEL_RANK = rank + + +def set_pipeline_model_parallel_rank(rank): + """Set pipeline model parallel rank.""" + global _MPU_PIPELINE_MODEL_PARALLEL_RANK + _MPU_PIPELINE_MODEL_PARALLEL_RANK = rank + + +def get_tensor_model_parallel_rank(): + """Return my rank for the tensor model parallel group.""" + global _MPU_TENSOR_MODEL_PARALLEL_RANK + if _MPU_TENSOR_MODEL_PARALLEL_RANK is not None: + return _MPU_TENSOR_MODEL_PARALLEL_RANK + return torch.distributed.get_rank(group=get_tensor_model_parallel_group()) + + +def get_pipeline_model_parallel_rank(): + """Return my rank for the pipeline model parallel group.""" + global _MPU_PIPELINE_MODEL_PARALLEL_RANK + if _MPU_PIPELINE_MODEL_PARALLEL_RANK is not None: + return _MPU_PIPELINE_MODEL_PARALLEL_RANK + return torch.distributed.get_rank(group=get_pipeline_model_parallel_group()) + + +# TODO (mkozuki): Add [`get_num_layers`](https://github.com/NVIDIA/Megatron-LM/blob/e156d2fea7fc5c98e645f7742eb86b643956d840/megatron/mpu/initialize.py#L321) here, maybe? + + +def get_pipeline_model_parallel_split_rank(): + """Return my rank for the pipeline model parallel split rank.""" + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK + return _PIPELINE_MODEL_PARALLEL_SPLIT_RANK + + +def set_pipeline_model_parallel_split_rank(pipeline_model_parallel_split_rank: int): + """Set my rank for the pipeline model parallel split rank.""" + global _PIPELINE_MODEL_PARALLEL_SPLIT_RANK + _PIPELINE_MODEL_PARALLEL_SPLIT_RANK = pipeline_model_parallel_split_rank + + +def is_pipeline_first_stage(ignore_virtual=False): + """Return True if in the first pipeline model-parallel stage, False otherwise.""" + if not ignore_virtual: + if ( + get_virtual_pipeline_model_parallel_world_size() is not None + and get_virtual_pipeline_model_parallel_rank() != 0 + ): + return False + return get_pipeline_model_parallel_rank() == 0 + + +def is_pipeline_last_stage(ignore_virtual=False): + """Return True if in the last pipeline model-parallel stage, False otherwise.""" + if not ignore_virtual: + virtual_pipeline_model_parallel_world_size = ( + get_virtual_pipeline_model_parallel_world_size() + ) + if virtual_pipeline_model_parallel_world_size is not None and get_virtual_pipeline_model_parallel_rank() != ( + virtual_pipeline_model_parallel_world_size - 1 + ): + return False + return get_pipeline_model_parallel_rank() == ( + get_pipeline_model_parallel_world_size() - 1 + ) + + +def get_virtual_pipeline_model_parallel_rank(): + """Return the virtual pipeline-parallel rank.""" + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK + return _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK + + +def set_virtual_pipeline_model_parallel_rank(rank): + """Set the virtual pipeline-parallel rank.""" + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK + _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = rank + + +def get_virtual_pipeline_model_parallel_world_size(): + """Return the virtual pipeline-parallel world size.""" + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + return _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + + +def set_virtual_pipeline_model_parallel_world_size(size): + """Return the virtual pipeline-parallel world size.""" + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = size + + +def get_tensor_model_parallel_src_rank(): + """Calculate the global rank corresponding to the first local rank + in the tensor model parallel group.""" + global_rank = torch.distributed.get_rank() + local_world_size = get_tensor_model_parallel_world_size() + return (global_rank // local_world_size) * local_world_size + + +def get_data_parallel_src_rank(): + """Calculate the global rank corresponding to the first local rank in the data parallel group.""" + global_rank = torch.distributed.get_rank() + data_parallel_size: int = get_data_parallel_world_size() + num_data_parallel_groups = torch.distributed.get_world_size() // data_parallel_size + return global_rank % num_data_parallel_groups + + +def get_pipeline_model_parallel_first_rank(): + assert ( + _PIPELINE_GLOBAL_RANKS is not None + ), "Pipeline parallel group is not initialized" + return _PIPELINE_GLOBAL_RANKS[0] + + +def get_pipeline_model_parallel_last_rank(): + assert ( + _PIPELINE_GLOBAL_RANKS is not None + ), "Pipeline parallel group is not initialized" + last_rank_local = get_pipeline_model_parallel_world_size() - 1 + return _PIPELINE_GLOBAL_RANKS[last_rank_local] + + +def get_pipeline_model_parallel_next_rank(): + assert ( + _PIPELINE_GLOBAL_RANKS is not None + ), "Pipeline parallel group is not initialized" + rank_in_pipeline = get_pipeline_model_parallel_rank() + world_size = get_pipeline_model_parallel_world_size() + return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline + 1) % world_size] + + +def get_pipeline_model_parallel_prev_rank(): + assert ( + _PIPELINE_GLOBAL_RANKS is not None + ), "Pipeline parallel group is not initialized" + rank_in_pipeline = get_pipeline_model_parallel_rank() + world_size = get_pipeline_model_parallel_world_size() + return _PIPELINE_GLOBAL_RANKS[(rank_in_pipeline - 1) % world_size] + + +def get_data_parallel_world_size(): + """Return world size for the data parallel group.""" + return torch.distributed.get_world_size(group=get_data_parallel_group()) + + +def get_data_parallel_rank(): + """Return my rank for the data parallel group.""" + return torch.distributed.get_rank(group=get_data_parallel_group()) + + +# note (mkozuki): `destroy_model_parallel` voids more global variables than Megatron-LM. +# Otherwise pipeline parallel forward_backward functions test hangs possibly because +# the clean-up of the original is NOT enough. +def destroy_model_parallel(): + """Set the groups to none.""" + global _MODEL_PARALLEL_GROUP + _MODEL_PARALLEL_GROUP = None + global _TENSOR_MODEL_PARALLEL_GROUP + _TENSOR_MODEL_PARALLEL_GROUP = None + global _PIPELINE_MODEL_PARALLEL_GROUP + _PIPELINE_MODEL_PARALLEL_GROUP = None + global _DATA_PARALLEL_GROUP + _DATA_PARALLEL_GROUP = None + global _AMAX_REDUCTION_GROUP + _AMAX_REDUCTION_GROUP = None + global _EMBEDDING_GROUP + _EMBEDDING_GROUP = None + global _POSITION_EMBEDDING_GROUP + _POSITION_EMBEDDING_GROUP = None + global _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP + _ENCODER_RELATIVE_POSITION_EMBEDDING_GROUP = None + global _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP + _DECODER_RELATIVE_POSITION_EMBEDDING_GROUP = None + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK + _VIRTUAL_PIPELINE_MODEL_PARALLEL_RANK = None + global _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + _VIRTUAL_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None + global _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE + _MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = None + global _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE + _MPU_PIPELINE_MODEL_PARALLEL_WORLD_SIZE = None + global _MPU_TENSOR_MODEL_PARALLEL_RANK + _MPU_TENSOR_MODEL_PARALLEL_RANK = None + global _MPU_PIPELINE_MODEL_PARALLEL_RANK + _MPU_PIPELINE_MODEL_PARALLEL_RANK = None + + +# Used to warn when the UCC is specified. +class ExperimentalWarning(Warning): pass diff --git a/apex/apex/transformer/pipeline_parallel/__init__.py b/apex/apex/transformer/pipeline_parallel/__init__.py new file mode 100644 index 00000000..98bb9602 --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/__init__.py @@ -0,0 +1,8 @@ +from apex.transformer.pipeline_parallel.schedules import get_forward_backward_func +from apex.transformer.pipeline_parallel.schedules.common import build_model + + +__all__ = [ + "get_forward_backward_func", + "build_model", +] diff --git a/apex/apex/transformer/pipeline_parallel/_timers.py b/apex/apex/transformer/pipeline_parallel/_timers.py new file mode 100644 index 00000000..55d89f35 --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/_timers.py @@ -0,0 +1,83 @@ +import time + +import torch + + +class _Timer: + """Timer.""" + + def __init__(self, name): + self.name_ = name + self.elapsed_ = 0.0 + self.started_ = False + self.start_time = time.time() + + def start(self): + """Start the timer.""" + assert not self.started_, "timer has already been started" + torch.cuda.synchronize() + self.start_time = time.time() + self.started_ = True + + def stop(self): + """Stop the timer.""" + assert self.started_, "timer is not started" + torch.cuda.synchronize() + self.elapsed_ += time.time() - self.start_time + self.started_ = False + + def reset(self): + """Reset timer.""" + self.elapsed_ = 0.0 + self.started_ = False + + def elapsed(self, reset=True): + """Calculate the elapsed time.""" + started_ = self.started_ + # If the timing in progress, end it first. + if self.started_: + self.stop() + # Get the elapsed time. + elapsed_ = self.elapsed_ + # Reset the elapsed time + if reset: + self.reset() + # If timing was in progress, set it back. + if started_: + self.start() + return elapsed_ + + +class _Timers: + """Group of timers.""" + + def __init__(self): + self.timers = {} + + def __call__(self, name): + if name not in self.timers: + self.timers[name] = _Timer(name) + return self.timers[name] + + def write(self, names, writer, iteration, normalizer=1.0, reset=False): + """Write timers to a tensorboard writer""" + # currently when using add_scalars, + # torch.utils.add_scalars makes each timer its own run, which + # polutes the runs list, so we just add each as a scalar + assert normalizer > 0.0 + for name in names: + value = self.timers[name].elapsed(reset=reset) / normalizer + writer.add_scalar(name + "-time", value, iteration) + + def log(self, names, normalizer=1.0, reset=True): + """Log a group of timers.""" + assert normalizer > 0.0 + string = "time (ms)" + for name in names: + elapsed_time = self.timers[name].elapsed(reset=reset) * 1000.0 / normalizer + string += " | {}: {:.2f}".format(name, elapsed_time) + if torch.distributed.is_initialized(): + if torch.distributed.get_rank() == (torch.distributed.get_world_size() - 1): + print(string, flush=True) + else: + print(string, flush=True) diff --git a/apex/apex/transformer/pipeline_parallel/p2p_communication.py b/apex/apex/transformer/pipeline_parallel/p2p_communication.py new file mode 100644 index 00000000..6c279ff6 --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/p2p_communication.py @@ -0,0 +1,684 @@ +# coding=utf-8 +# Copyright (c) 2021-22, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# TODO(mkozuki): Consider removing `timers`. + +from functools import reduce +import operator +from typing import Union, Optional, Tuple + +import torch + +from apex.transformer import parallel_state +from apex.transformer.log_util import get_transformer_logger +from apex.transformer.utils import split_tensor_into_1d_equal_chunks +from apex.transformer.utils import gather_split_1d_tensor +from apex.transformer.pipeline_parallel.utils import Shape +from apex.transformer.pipeline_parallel._timers import _Timers + + +_logger = get_transformer_logger(__name__) + + +class FutureTensor: + def __init__(self, tensor: torch.Tensor, waitfunc): + self.tensor = tensor + self.waitfunc = waitfunc + + def get(self): + if self.waitfunc is not None: + res = self.waitfunc() + if isinstance(res, torch.Tensor): + self.tensor = res + self.waitfunc = None + return self.tensor + + +def _run_p2pops( + tensor_send_prev: Union[torch.Tensor, None], + tensor_send_next: Union[torch.Tensor, None], + tensor_recv_prev: Union[torch.Tensor, None], + tensor_recv_next: Union[torch.Tensor, None], + async_comm: bool = False, + overlap_p2p_comm: bool = False, + batch_p2p_comm: bool = True, +): + p2p_group = parallel_state.get_pipeline_model_parallel_group() + default_group = parallel_state.get_model_parallel_group() + + need_to_sync = p2p_group.name() != default_group.name() + reqs = [] + + if batch_p2p_comm: + ops = [] + if tensor_send_prev is not None: + send_prev_op = torch.distributed.P2POp( + op=torch.distributed.isend, + tensor=tensor_send_prev, + peer=parallel_state.get_pipeline_model_parallel_prev_rank(), + group=p2p_group, + ) + ops.append(send_prev_op) + if tensor_recv_prev is not None: + recv_prev_op = torch.distributed.P2POp( + op=torch.distributed.irecv, + tensor=tensor_recv_prev, + peer=parallel_state.get_pipeline_model_parallel_prev_rank(), + group=p2p_group, + ) + ops.append(recv_prev_op) + if tensor_send_next is not None: + send_next_op = torch.distributed.P2POp( + op=torch.distributed.isend, + tensor=tensor_send_next, + peer=parallel_state.get_pipeline_model_parallel_next_rank(), + group=p2p_group, + ) + ops.append(send_next_op) + if tensor_recv_next is not None: + recv_next_op = torch.distributed.P2POp( + op=torch.distributed.irecv, + tensor=tensor_recv_next, + peer=parallel_state.get_pipeline_model_parallel_next_rank(), + group=p2p_group, + ) + ops.append(recv_next_op) + if len(ops) > 0: + # sync before communication if needed + if need_to_sync: + torch.cuda.synchronize() + reqs = torch.distributed.batch_isend_irecv(ops) + else: + # sync before communication if needed + if need_to_sync and any( + tensor_send_prev is not None, tensor_recv_prev is not None, + tensor_send_next is not None, tensor_recv_next is not None): + torch.cuda.synchronize() + + if tensor_send_prev is not None: + send_prev_req = torch.distributed.isend( + tensor=tensor_send_prev, + dst=parallel_state.get_pipeline_model_parallel_prev_rank(), + group=p2p_group, + ) + reqs.append(send_prev_req) + if tensor_recv_prev is not None: + recv_prev_req = torch.distributed.irecv( + tensor=tensor_recv_prev, + src=parallel_state.get_pipeline_model_parallel_prev_rank(), + group=p2p_group, + ) + reqs.append(recv_prev_req) + if tensor_send_next is not None: + send_next_req = torch.distributed.isend( + tensor=tensor_send_next, + dst=parallel_state.get_pipeline_model_parallel_next_rank(), + group=p2p_group, + ) + reqs.append(send_next_req) + if tensor_recv_next is not None: + recv_next_op = torch.distributed.irecv( + tensor=tensor_recv_next, + src=parallel_state.get_pipeline_model_parallel_next_rank(), + group=p2p_group, + ) + reqs.append(recv_next_op) + + if len(reqs) > 0: + if overlap_p2p_comm: + return (None, None, None, None, reqs) + + if async_comm: + if batch_p2p_comm: + assert len(reqs) == len(ops) + tensor_send_prev_req = None if tensor_send_prev is None else reqs.pop(0) + tensor_recv_prev_req = None if tensor_recv_prev is None else reqs.pop(0) + tensor_send_next_req = None if tensor_send_next is None else reqs.pop(0) + tensor_recv_next_req = None if tensor_recv_next is None else reqs.pop(0) + return (tensor_send_prev_req, tensor_recv_prev_req, tensor_send_next_req, tensor_recv_next_req, None) + else: + for req in reqs: + req.wait() + return (None, None, None, None, None) + return (None, None, None, None, None) + + +# TODO(mkozuki): Check if it's possible to sunset `override_scatter_gather_tensors_in_pipeline`. +# TODO(mkozuki): Think about if it's possible to push some logic and arguments e.g. +# `scatter_gather_tensors_in_pipeline`, `sequence_parallel_enabled`, and +# `override_scatter_gather_tensors_in_pipeline` # to the user of +# apex.transformer forward_backwardfunctions. +def _communicate( + tensor_send_next: Optional[torch.Tensor], + tensor_send_prev: Optional[torch.Tensor], + recv_prev: bool, + recv_next: bool, + tensor_shape: Optional[Shape] = None, + override_scatter_gather_tensors_in_pipeline: bool = False, + dtype_: Optional[torch.dtype] = None, + *, + scatter_gather_tensors_in_pipeline: bool = True, + params_dtype: Optional[torch.dtype] = None, + fp32_residual_connection: bool = False, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + overlap_p2p_comm: bool = False, + batch_p2p_comm: bool = True, +) -> Tuple[Union[torch.Tensor, FutureTensor, None], Union[torch.Tensor, FutureTensor, None]]: + """Base function for communication of tensors between stages. + + + .. note:: + Reference https://gitlab-master.nvidia.com/ADLR/megatron-lm/-/blob/cfd2e2160700b7f2c1bf35298ac14bc341f4c759/megatron/p2p_communication.py#L24-L159 + + dtype logic: If none of ``dtype_``, ``params_dtype``, ``fp32_residual_connection`` is specified, + torch.float32 is used. + + See https://github.com/NVIDIA/Megatron-LM/blob/d41696840ed0a7edb7e0499eb82a48ae112d9bb3/megatron/arguments.py#L145-L159 + for the details of arguments of ``dtype_``, ``params_dtype``, ``fp32_residual_connection``. + + Args: + tensor_send_next: tensor to send to next rank (no tensor sent if set to None). + tensor_send_prev: tensor to send to prev rank (no tensor sent if set to None). + recv_prev: boolean for whether tensor should be received from previous rank. + recv_next: boolean for whether tensor should be received from next rank. + tensor_shape: optional, use when the input sequence contains less tokens than the default sequence length + override_scatter_gather_tensors_in_pipeline: + optional, this is used when tensor_shape is provided to override scatter gather tensors + dtype_: This is used when tensor_shape is provided and what is the type of tensor_shape + + Keyword args: + scatter_gather_tensors_in_pipeline: Optional. If :obj:`True`, use scatter/gather to optimize communication of tensors. + params_dtype: Optional and legacy. Defaults to torch.float. If you manually call `.half()` or `.bfloat16()` on + your model deliberately, pass this argument. + fp32_residual_connection: Optional. If :obj:`True`, move residual connections to fp32. + sequence_parallel_enabled: Set to :obj:`True` if sequence parallel is enabled. + This argument is here for consistency with Megatron-LM. + This argument has an effect on the communication optimization, not on tensor_shape update. + sync_batch_comm: If :obj:`False`, disable cuda synchronization after the batched communication. + To disable, https://github.com/pytorch/pytorch/pull/82450 would be required. + overlap_p2p_comm: If :obj:`True`, returns cuda wait handles to scheduler instead of completing + the communication within the p2p transfer API instance. The scheduler manages the communication completion + to overlap with computation. + batch_p2p_comm: If :obj:`True`, use the batched send and receive api to conduct the communication of + a collection of send and receive operations between peer. If :obj:`False`, conduct each send and recv operation + individually. + + Returns: + tuple containing + + - tensor_recv_prev: `torch.Tensor` if `recv_prev` is :obj:`True`, `None` otherwise. + - tensor_recv_next: `torch.Tensor` if `recv_next` is :obj:`True`, `None` otherwise. + """ + if async_comm and sequence_parallel_enabled: + import warnings # NOQA + class ExperimentalWarning(UserWarning): pass # NOQA + warnings.warn( + "The combination of `async_comm` and `sequence_parallel_enabled` is not well tested.", + ExperimentalWarning, + ) + # Create placeholder tensors for receive in forward and backward directions if needed. + tensor_recv_prev = None + tensor_recv_next = None + if tensor_shape is None: + # In megatron, `tensor_shape` is set to `(args.seq_length, args.micro_batch_size, args.hidden_size)` + raise RuntimeError( + "`tensor_shape` must be specified. Common `tensor_shape` is `(seq_length, micro_batch_size, hidden_size)`") + + tensor_parallel_size = parallel_state.get_tensor_model_parallel_world_size() + override_scatter_gather_tensors_in_pipeline_ = False + # TODO(mkozuki): Demystify hardcode False of `scatter_gather_tensors_in_pipeline` and add a testcase if possible. + # NOTE(mkozuki): This is super strange and doesn't make sense to me. I have no idea what is happening here. + # However, I can say that this hardcoding override is necessary for sequence parallel in nemo megatron to work. + # I've not managed to reproduce the hang using standalone GPT with sequence parallel. + # The hang in NeMo Megatron happens in the 3rd iteration, the last iteration of stead phase inside + # forward_backward_pipelining_without_interleaving, pipeline parallel rank of 0 (tensor model parallel world + # size of 2 and pipeline model parallel world size of 2). The commit then of APEX and NeMo were + # https://github.com/NVIDIA/apex/pull/1396/commits/3060c98dd8ba42abf7702ea9d2cff0f39ea74f45 and + # https://github.com/NVIDIA/NeMo/pull/4232/commits/1cb32dfca2ab9b20f53ebdb84476c34cb42f0205. + # The PyTorch version was 1.13.0a0+git2d354cd, for what is worth. + # Currently, indiscriminately this is set to `False`, which can lead to an unexpected performance regression + # for non sequence parallel case. + scatter_gather_tensors_in_pipeline = False + if scatter_gather_tensors_in_pipeline and not sequence_parallel_enabled: + tensor_chunk_size = int(reduce(operator.mul, tensor_shape, 1)) + if tensor_chunk_size % tensor_parallel_size == 0: + tensor_chunk_shape = [tensor_chunk_size // tensor_parallel_size] + else: + tensor_chunk_shape = tensor_shape + override_scatter_gather_tensors_in_pipeline_ = True + else: + tensor_chunk_shape = tensor_shape + + # The dtype logic below is copied from NVIDIA/Megatron-LM repo: + # https://github.com/NVIDIA/Megatron-LM/blob/d41696840ed0a7edb7e0499eb82a48ae112d9bb3/megatron/p2p_communication.py#L74-L81 + dtype = params_dtype or torch.float + if fp32_residual_connection: + dtype = torch.float + requires_grad = True + if dtype_ is not None: + dtype = dtype_ + # TODO(mkozuki): Figure out why this logic of requires_grad isn't working + # when sequence_parallel_enabled=True. Otherwise, `x.retain_grad()` of + # https://github.com/crcrpar/apex/blob/069832078a652b4bd8a99db84faf953a81415ab3/apex/transformer/pipeline_parallel/schedules/common.py#L360 + # fails. + # requires_grad = False + + if recv_prev: + tensor_recv_prev = torch.empty( + tensor_chunk_shape, + requires_grad=requires_grad, + device=torch.cuda.current_device(), + dtype=dtype, + ) + if recv_next: + tensor_recv_next = torch.empty( + tensor_chunk_shape, + requires_grad=requires_grad, + device=torch.cuda.current_device(), + dtype=dtype, + ) + + # Split tensor into smaller chunks if using scatter-gather optimization. + scatter_gather_optimization_doable = ( + not override_scatter_gather_tensors_in_pipeline_ + and scatter_gather_tensors_in_pipeline + and not sequence_parallel_enabled + ) + if scatter_gather_optimization_doable: + if tensor_send_next is not None: + tensor_send_next = split_tensor_into_1d_equal_chunks(tensor_send_next) + + if tensor_send_prev is not None: + tensor_send_prev = split_tensor_into_1d_equal_chunks(tensor_send_prev) + + # Send tensors in both the forward and backward directions as appropriate. + tensor_send_prev_req, tensor_recv_prev_req, tensor_send_next_req, tensor_recv_next_req, wait_handles = _run_p2pops( + tensor_send_prev, tensor_send_next, tensor_recv_prev, tensor_recv_next, async_comm, overlap_p2p_comm, batch_p2p_comm) + + if async_comm: + tensor_recv_prev_waitfunc = None + tensor_recv_next_waitfunc = None + # TODO: investigate whether this is necessary for correctness (ref: https://github.com/pytorch/pytorch/issues/38642) + # see also: sync added for async_comm callbacks below in gather_recv_prev_wait and gather_recv_next_wait + if tensor_recv_prev_req is not None: + def tensor_recv_prev_wait(): + tensor_recv_prev_req.wait() + torch.cuda.synchronize() + tensor_recv_prev_waitfunc = tensor_recv_prev_wait + if tensor_recv_next_req is not None: + def tensor_recv_next_wait(): + tensor_recv_next_req.wait() + torch.cuda.synchronize() + tensor_recv_next_waitfunc = tensor_recv_next_wait + else: + if sync_batch_comm: + # To protect against race condition when using batch_isend_irecv(). + torch.cuda.synchronize() + + # If using scatter-gather optimization, gather smaller chunks. + if scatter_gather_optimization_doable: + if not async_comm: + if recv_prev: + tensor_recv_prev = ( + gather_split_1d_tensor(tensor_recv_prev) + .view(tensor_shape) + .requires_grad_() + ) + + if recv_next: + tensor_recv_next = ( + gather_split_1d_tensor(tensor_recv_next) + .view(tensor_shape) + .requires_grad_() + ) + else: + def gather_recv_prev_wait(): + tensor_recv_prev_req.wait() + # From @Deepak's PR https://github.com/NVIDIA/Megatron-LM/commit/27fc468964064eeb33b703c9a0b2af938d80dd14 + # A sync seems to be needed before gather otherwise losses jump around e.g., in run_gpt_minimal_test + torch.cuda.synchronize() + return ( + gather_split_1d_tensor(tensor_recv_prev) + .view(tensor_shape) + .requires_grad_() + ) + def gather_recv_next_wait(): + tensor_recv_next_req.wait() + torch.cuda.synchronize() + return ( + gather_split_1d_tensor(tensor_recv_next) + .view(tensor_shape) + .requires_grad_() + ) + tensor_recv_prev_waitfunc = gather_recv_prev_wait + tensor_recv_next_waitfunc = gather_recv_next_wait + if async_comm: + future_tensor_recv_prev = None + future_tensor_recv_next = None + if tensor_recv_prev is not None: + future_tensor_recv_prev = FutureTensor(tensor_recv_prev, tensor_recv_prev_waitfunc) + if tensor_recv_next is not None: + future_tensor_recv_next = FutureTensor(tensor_recv_next, tensor_recv_next_waitfunc) + return future_tensor_recv_prev, future_tensor_recv_next, None + return tensor_recv_prev, tensor_recv_next, wait_handles + + +def recv_forward( + tensor_shape: Shape, + override_scatter_gather_tensors_in_pipeline: bool = False, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Union[torch.Tensor, FutureTensor, None]: + """Receive tensor from previous rank in pipeline (forward receive).""" + if parallel_state.is_pipeline_first_stage(): + return None + # if timers is not None: + # timers("forward-recv").start() + input_tensor, _, _ = _communicate( + tensor_send_next=None, + tensor_send_prev=None, + recv_prev=True, + recv_next=False, + tensor_shape=tensor_shape, + override_scatter_gather_tensors_in_pipeline=override_scatter_gather_tensors_in_pipeline, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("forward-recv").stop() + return input_tensor + + +def recv_backward( + tensor_shape: Shape = None, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Union[torch.Tensor, FutureTensor, None]: + """Receive tensor from next rank in pipeline (backward receive).""" + if parallel_state.is_pipeline_last_stage(): + return None + # if timers is not None: + # timers("backward-recv").start() + _, output_tensor_grad, _ = _communicate( + tensor_send_next=None, + tensor_send_prev=None, + recv_prev=False, + recv_next=True, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("backward-recv").stop() + return output_tensor_grad + + +def send_forward( + output_tensor: torch.Tensor, + override_scatter_gather_tensors_in_pipeline: bool = False, + tensor_shape: Shape = None, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> None: + """Send tensor to next rank in pipeline (forward send).""" + if parallel_state.is_pipeline_last_stage(): + return + # if timers is not None: + # timers("forward-send").start() + _communicate( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=False, + recv_next=False, + override_scatter_gather_tensors_in_pipeline=override_scatter_gather_tensors_in_pipeline, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("forward-send").stop() + + +def send_backward( + input_tensor_grad: torch.Tensor, + tensor_shape: Shape, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> None: + """Send tensor to previous rank in pipeline (backward send).""" + if parallel_state.is_pipeline_first_stage(): + return + # if timers is not None: + # timers("backward-send").start() + _communicate( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=False, + recv_next=False, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("backward-send").stop() + + +def send_forward_recv_backward( + output_tensor: torch.Tensor, + tensor_shape: Shape, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Union[torch.Tensor, FutureTensor, None]: + """Batched send and recv with next rank in pipeline.""" + if parallel_state.is_pipeline_last_stage(): + return None + # if timers is not None: + # timers("forward-send-backward-recv").start() + _, output_tensor_grad, _ = _communicate( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=False, + recv_next=True, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("forward-send-backward-recv").stop() + return output_tensor_grad + + +def send_backward_recv_forward( + input_tensor_grad: torch.Tensor, + tensor_shape: Shape, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Union[torch.Tensor, FutureTensor, None]: + """Batched send and recv with previous rank in pipeline.""" + if parallel_state.is_pipeline_first_stage(): + return None + # if timers is not None: + # timers("backward-send-forward-recv").start() + input_tensor, _, _ = _communicate( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=True, + recv_next=False, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("backward-send-forward-recv").stop() + return input_tensor + + +def send_forward_recv_forward( + output_tensor: torch.Tensor, + recv_prev: bool, + tensor_shape: Shape, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + overlap_p2p_comm: bool = False, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Union[torch.Tensor, FutureTensor]: + """Batched recv from previous rank and send to next rank in pipeline.""" + # if timers is not None: + # timers("forward-send-forward-recv").start() + input_tensor, _, wait_handles = _communicate( + tensor_send_next=output_tensor, + tensor_send_prev=None, + recv_prev=recv_prev, + recv_next=False, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=overlap_p2p_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("forward-send-forward-recv").stop() + if overlap_p2p_comm: + return input_tensor, wait_handles + return input_tensor + + +def send_backward_recv_backward( + input_tensor_grad: torch.Tensor, + recv_next: bool, + tensor_shape: Shape, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + overlap_p2p_comm: bool = False, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Union[torch.Tensor, FutureTensor]: + """Batched recv from next rank and send to previous rank in pipeline.""" + # if timers is not None: + # timers("backward-send-backward-recv").start() + _, output_tensor_grad, wait_handles = _communicate( + tensor_send_next=None, + tensor_send_prev=input_tensor_grad, + recv_prev=False, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=overlap_p2p_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("backward-send-backward-recv").stop() + if overlap_p2p_comm: + return output_tensor_grad, wait_handles + return output_tensor_grad + + +def send_forward_backward_recv_forward_backward( + output_tensor: torch.Tensor, + input_tensor_grad: torch.Tensor, + recv_prev: bool, + recv_next: bool, + tensor_shape: Shape, + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, + overlap_p2p_comm: bool = False, + batch_p2p_comm: bool = True, + timers: _Timers = None, +) -> Tuple[Union[torch.Tensor, FutureTensor], Union[torch.Tensor, FutureTensor]]: + """Batched send and recv with previous and next ranks in pipeline.""" + # if timers is not None: + # timers("forward-backward-send-forward-backward-recv").start() + input_tensor, output_tensor_grad, wait_handles = _communicate( + tensor_send_next=output_tensor, + tensor_send_prev=input_tensor_grad, + recv_prev=recv_prev, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype_=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=overlap_p2p_comm, + batch_p2p_comm=batch_p2p_comm, + ) + # if timers is not None: + # timers("forward-backward-send-forward-backward-recv").stop() + if overlap_p2p_comm: + return input_tensor, output_tensor_grad, wait_handles + return input_tensor, output_tensor_grad diff --git a/apex/apex/transformer/pipeline_parallel/schedules/__init__.py b/apex/apex/transformer/pipeline_parallel/schedules/__init__.py new file mode 100644 index 00000000..7e131928 --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/schedules/__init__.py @@ -0,0 +1,35 @@ +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel.utils import get_num_microbatches +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_no_pipelining import ( + forward_backward_no_pipelining, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_with_interleaving import ( + _forward_backward_pipelining_with_interleaving, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_without_interleaving import ( + forward_backward_pipelining_without_interleaving, +) + +__all__ = [ + "get_forward_backward_func", +] + + +class ExperimentalWarning(Warning): + pass + + +def get_forward_backward_func( + virtual_pipeline_model_parallel_size, pipeline_model_parallel_size, +): + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + if virtual_pipeline_model_parallel_size is not None: + if get_num_microbatches() % pipeline_model_parallel_size != 0: + msg = "number of microbatches is not divisible by pipeline-parallel size when using interleaved schedule" + raise RuntimeError(msg) + forward_backward_func = _forward_backward_pipelining_with_interleaving + else: + forward_backward_func = forward_backward_pipelining_without_interleaving + else: + forward_backward_func = forward_backward_no_pipelining + return forward_backward_func diff --git a/apex/apex/transformer/pipeline_parallel/schedules/common.py b/apex/apex/transformer/pipeline_parallel/schedules/common.py new file mode 100644 index 00000000..7f4b15fe --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/schedules/common.py @@ -0,0 +1,403 @@ +from typing import Any, Callable, Dict, List, Tuple, Union, Optional, Sequence + +import torch +from torch.autograd.variable import Variable + +from apex.normalization.fused_layer_norm import FusedLayerNorm +from apex.transformer import parallel_state +from apex.transformer.enums import ModelType +from apex.transformer.pipeline_parallel.p2p_communication import FutureTensor +from apex.transformer.pipeline_parallel.utils import get_num_microbatches +from apex.transformer.pipeline_parallel.utils import listify_model +from apex.transformer.pipeline_parallel.utils import unwrap_model +from apex.transformer.pipeline_parallel.utils import get_model_type +from apex.transformer.tensor_parallel.layers import ( + set_defaults_if_not_set_tensor_model_parallel_attributes, +) +from apex.transformer.log_util import get_transformer_logger + + +_logger = get_transformer_logger(__name__) + + +Batch = Union[torch.Tensor, FutureTensor, List[Union[torch.Tensor, FutureTensor]], Tuple[Union[torch.Tensor, FutureTensor], ...]] +LossFunc = Callable[[torch.Tensor], torch.Tensor] +FwdStepFunc = Callable[ + [Optional[Batch], torch.nn.Module], Tuple[torch.Tensor, LossFunc] +] + + +def build_model( + model_provider_func: Callable[[Any, Dict[str, Any]], torch.nn.Module], + wrap_with_ddp: bool = True, + virtual_pipeline_model_parallel_size: Optional[int] = None, + model_type: ModelType = ModelType.encoder_or_decoder, + *args: Any, + **kwargs: Any, +) -> List[torch.nn.Module]: + """Build the model satisfying pipeline model parallel requirements. + + This function sets `pre_process` and `post_process` to `**kwargs` and pass `*args` and `**kwargs` to + `model_provider_func`. + + Args: + model_provider_func: A function which takes `*args` and `**kwargs` and returns a `nn.Module`. + wrap_with_ddp: If :obj:`True`, wrap the instantiated model + with `torch.nn.parallel.distributed.DistributedDataParallel`, a.k.a. `DDP`. + virtual_pipeline_model_parallel_size: Specify when using interleaving scheduling pipeline model parallel. + model_type: + *args: arguments for model provider func + **kwargs: Keyword arguments for model provider func + + Returns: + a list of `nn.Module`(s). If `virtual_pipeline_model_parallel_size` is not None, + the list has multiple models, otherwise one. + """ + if ( + parallel_state.get_pipeline_model_parallel_world_size() > 1 + and virtual_pipeline_model_parallel_size is not None + ): + model = [] + for i in range(virtual_pipeline_model_parallel_size): + cur_args = args + cur_kwargs = kwargs + parallel_state.set_virtual_pipeline_model_parallel_rank(i) + # Set pre_process and post_process only after virtual rank is set. + pre_process = parallel_state.is_pipeline_first_stage() + post_process = parallel_state.is_pipeline_last_stage() + cur_kwargs.update( + {"pre_process": pre_process, "post_process": post_process,} + ) + this_model = model_provider_func(*cur_args, **cur_kwargs) + model.append(this_model) + else: + cur_args = args + cur_kwargs = kwargs + if model_type == ModelType.encoder_or_decoder: + pre_process = parallel_state.is_pipeline_first_stage() + post_process = parallel_state.is_pipeline_last_stage() + cur_kwargs.update( + {"pre_process": pre_process, "post_process": post_process,} + ) + model = model_provider_func(*cur_args, **cur_kwargs) + elif model_type == ModelType.encoder_and_decoder: + pre_process = parallel_state.is_pipeline_first_stage() + post_process = parallel_state.is_pipeline_last_stage() + # `add_encoder` & `add_decoder` logic. + add_encoder, add_decoder = True, True + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + split_rank = parallel_state.get_pipeline_model_parallel_split_rank() + if split_rank is None: + raise RuntimeError( + "Split rank needs to be specified for model with both encoder and decoder." + ) + rank = parallel_state.get_pipeline_model_parallel_rank() + world_size = parallel_state.get_pipeline_model_parallel_world_size() + pre_process = rank == 0 or rank == split_rank + post_process = rank == (split_rank - 1) or rank == (world_size - 1) + add_encoder = parallel_state.is_pipeline_stage_before_split() + add_decoder = parallel_state.is_pipeline_stage_after_split() + cur_kwargs.update( + { + "pre_process": pre_process, + "post_process": post_process, + "add_encoder": add_encoder, + "add_decoder": add_decoder, + } + ) + model = model_provider_func(*cur_args, **cur_kwargs) + model.model_type = model_type + + if not isinstance(model, list): + model = [model] + + # Set tensor model parallel attributes if not set. + # Only parameters that are already tensor model parallel have these + # attributes set for them. We should make sure the default attributes + # are set for all params so the optimizer can use them. + for model_module in model: + for param in model_module.parameters(): + set_defaults_if_not_set_tensor_model_parallel_attributes(param) + + # Print number of parameters. + if ( + parallel_state.model_parallel_is_initialized() + and parallel_state.get_data_parallel_rank() == 0 + ): + msg = " > number of parameters on (tensor, pipeline) model parallel rank ({}, {}): {}".format( + parallel_state.get_tensor_model_parallel_rank(), + parallel_state.get_pipeline_model_parallel_rank(), + _calc_number_of_params(model), + ) + print(msg, flush=True) + + # GPU allocation. + for model_module in model: + model_module.cuda(torch.cuda.current_device()) + + if wrap_with_ddp: + i = torch.cuda.current_device() + model = [ + torch.nn.parallel.distributed.DistributedDataParallel( + model_module, + device_ids=[i], + output_device=i, + process_group=parallel_state.get_data_parallel_group(), + ) + for model_module in model + ] + return model + + +def _calc_number_of_params(model: List[torch.nn.Module]) -> int: + assert isinstance(model, list) + return sum( + [ + sum([p.nelement() for p in model_module.parameters()]) + for model_module in model + ] + ) + + +def _get_params_for_weight_decay_optimization( + model: Union[torch.nn.Module, List[torch.nn.Module]], + *, + no_weight_decay_modules=(FusedLayerNorm,), +) -> Dict[str, torch.nn.Parameter]: + """Divide params into with-weight-decay and without-weight-decay groups. + + Layernorms and biases will have no weight decay but the rest will. + """ + modules = listify_model(model) + weight_decay_params = {"params": []} + no_weight_decay_params = {"params": [], "weight_decay": 0.0} + for module in modules: + for module_ in module.modules(): + if isinstance(module_, no_weight_decay_modules): + no_weight_decay_params["params"].extend( + [p for p in list(module_._parameters.values()) if p is not None] + ) + else: + weight_decay_params["params"].extend( + [ + p + for n, p in list(module_._parameters.items()) + if p is not None and n != "bias" + ] + ) + no_weight_decay_params["params"].extend( + [ + p + for n, p in list(module_._parameters.items()) + if p is not None and n == "bias" + ] + ) + + return weight_decay_params, no_weight_decay_params + + +def free_output_tensor( + output_tensors: Optional[Union[torch.Tensor, Sequence[torch.Tensor]]], + deallocate_pipeline_outputs: bool = False, +) -> None: + """Pseudo-free the output tensor's `.data` field. + + This method should be called right after the output tensor has been sent to the next + pipeline stage. At this point, the output tensor is only useful for its `.grad_fn` field, + and not its `.data`. + """ + if not deallocate_pipeline_outputs: + return + if output_tensors is None: + return + if isinstance(output_tensors, torch.Tensor): + output_tensors = [output_tensors] + for output_tensor in output_tensors: + output_tensor.data = torch.cuda.FloatTensor([0]) + + +def custom_backward(output: torch.Tensor, grad_output: Optional[torch.Tensor]) -> None: + """Directly call C++ autograd engine. + + To make the `free_output_tensor` optimization work, the C++ autograd engine must be called + directly, bypassing PyTorch's `torch.autograd.backward`. PyTorch's `backward` checks that the + output and grad have the same shape, while C++ `backward` does not. + """ + assert ( + output.numel() == 1 + ), "output should be pseudo-freed in schedule, to optimize memory consumption" + assert isinstance(output, torch.Tensor), "output == {}.".format( + type(output).__name__ + ) + assert isinstance( + grad_output, (torch.Tensor, type(None)) + ), "grad_outptu == {}.".format(type(grad_output).__name__) + + # Handle scalar output + if grad_output is None: + assert output.numel() == 1, "Implicit grad requires scalar output." + grad_output = torch.ones_like(output, memory_format=torch.preserve_format) + + # Call C++ engine [ see torch/csrc/autograd/python_engine.cpp ] + Variable._execution_engine.run_backward( + tensors=(output,), + grad_tensors=(grad_output,), + keep_graph=False, + create_graph=False, + inputs=(), + allow_unreachable=True, + accumulate_grad=True, + ) + + +def forward_step( + forward_step_func: FwdStepFunc, + batch: Optional[Batch], + model: torch.nn.Module, + input_tensor: Optional[Union[torch.Tensor, List[torch.Tensor]]], + losses_reduced: List[torch.Tensor], + dtype: torch.dtype, + disable_autocast: bool = False, + checkpoint_activations_micro_batch: Optional[bool] = None, +) -> Union[torch.Tensor, Sequence[torch.Tensor]]: + """Forward step for passed-in model. + + If first stage, input tensor is obtained from batch, otherwise passed-in input_tensor is used. + + Returns output tensor. + + Args: + forward_step_func: Model specific function. This takes a minibatch and model as its arguments and + returns the model's output and the loss function. + batch: minibatch + model: unwrappable model + input_tensor: + losses_reduced: + dtype: + disable_autocast: + checkpoint_activations_micro_batch: + + Returns: + output_tensor + """ + # timers = get_timers() + # timers("forward-compute").start() + unwrapped_model = unwrap_model(model) + model_type = get_model_type(unwrapped_model) + # NOTE (mkozuki): The passed `model` is expected to implement `set_input_tensor`. + # See https://github.com/NVIDIA/Megatron-LM/blob/5ac5571ba0265af4c491ee0af1508ca7589450c6/megatron/model/transformer.py#L679 # NOQA + # for the details of `set_input_tensor`. + unwrap_output_tensor = not isinstance(input_tensor, list) + if unwrap_output_tensor: + input_tensor = [input_tensor] + + input_tensor = [inp.get() if isinstance(inp, FutureTensor) else inp for inp in input_tensor] + + unwrapped_model.set_input_tensor(input_tensor) + with torch.cuda.amp.autocast( + enabled=not disable_autocast and dtype in (torch.half, torch.bfloat16), + dtype=dtype, + ): + if checkpoint_activations_micro_batch is None: + output_tensor, loss_func = forward_step_func(batch, model) + else: + output_tensor, loss_func = forward_step_func(batch, model, checkpoint_activations_micro_batch) + if parallel_state.is_pipeline_last_stage(): + output_tensor = loss_func(output_tensor) + loss, loss_reduced = output_tensor + output_tensor = loss / get_num_microbatches() + losses_reduced.append(loss_reduced) + # timers("forward-compute").stop() + + # If T5 model (or other model with encoder and decoder) + # and in decoder stack, then send encoder_hidden_state + # downstream as well. + if ( + parallel_state.is_pipeline_stage_after_split() + and model_type == ModelType.encoder_and_decoder + ): + return [output_tensor, input_tensor[-1]] + if unwrap_output_tensor: + return output_tensor + return [output_tensor] + + +def backward_step( + input_tensor: Optional[torch.Tensor], + output_tensor: torch.Tensor, + output_tensor_grad: Optional[torch.Tensor], + model_type: ModelType, + *, + grad_scaler: Optional[torch.cuda.amp.GradScaler] = None, + deallocate_pipeline_outputs: bool = False, +) -> Union[None, torch.Tensor, Sequence[torch.Tensor]]: + """Backward step through passed-in output tensor. + + If last stage, output_tensor_grad is None, otherwise gradient of loss + with respect to stage's output tensor. + + Returns gradient of loss with respect to input tensor (None if first + stage). + + Args: + input_tensor: + output_tensor: + output_tensor_grad: + Keyword Arguments: + grad_scaler: + deallocate_pipeline_outputs: Experimental. + Returns: + input_tensor_grad + """ + + # timers = get_timers() + # timers("backward-compute").start() + + # Retain the grad on the input_tensor. + unwrap_input_tensor_grad = not isinstance(input_tensor, list) + if unwrap_input_tensor_grad: + input_tensor = [input_tensor] + + input_tensor = [inp.get() if isinstance(inp, FutureTensor) else inp for inp in input_tensor] + + for x in input_tensor: + if x is not None: + x.retain_grad() + + if not isinstance(output_tensor, list): + output_tensor = [output_tensor] + + output_tensor = [out.get() if isinstance(out, FutureTensor) else out for out in output_tensor] + + if not isinstance(output_tensor_grad, list): + output_tensor_grad = [output_tensor_grad] + + output_tensor_grad = [ogr.get() if isinstance(ogr, FutureTensor) else ogr for ogr in output_tensor_grad] + + # Backward pass. + if grad_scaler is not None and output_tensor_grad[0] is None: + output_tensor[0] = grad_scaler.scale(output_tensor[0]) + if deallocate_pipeline_outputs: + custom_backward(output_tensor[0], output_tensor_grad[0]) + else: + torch.autograd.backward(output_tensor[0], grad_tensors=output_tensor_grad[0]) + + # Collect the grad of the input_tensor. + input_tensor_grad = [None] + if input_tensor is not None: + input_tensor_grad = [] + for x in input_tensor: + input_tensor_grad.append(None if x is None else x.grad) + + # Handle single skip connection if it exists (encoder_hidden_state in model with encoder and decoder). + if ( + parallel_state.get_pipeline_model_parallel_world_size() > 1 + and parallel_state.is_pipeline_stage_after_split() + and model_type == ModelType.encoder_and_decoder + ): + if output_tensor_grad[1] is not None: + # todo (mkozuki): Replace the inplace add with `+= output_tensor_grad[1]`? + input_tensor_grad[-1].add_(output_tensor_grad[1]) + + # timers("backward-compute").stop() + return input_tensor_grad[0] if unwrap_input_tensor_grad else input_tensor_grad diff --git a/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_no_pipelining.py b/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_no_pipelining.py new file mode 100644 index 00000000..6bd3ff13 --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_no_pipelining.py @@ -0,0 +1,124 @@ +import contextlib +from typing import List, Union, Optional + +import torch + +from apex.transformer.pipeline_parallel.utils import listify_model +from apex.transformer.pipeline_parallel.utils import get_num_microbatches +from apex.transformer.pipeline_parallel.utils import get_kth_microbatch +from apex.transformer.pipeline_parallel.utils import get_model_type +from apex.transformer.pipeline_parallel.schedules.common import Batch +from apex.transformer.pipeline_parallel.schedules.common import FwdStepFunc +from apex.transformer.pipeline_parallel.schedules.common import forward_step +from apex.transformer.pipeline_parallel.schedules.common import backward_step +from apex.transformer.log_util import get_transformer_logger + + +_all__ = ["forward_backward_no_pipelining"] + + +_logger = get_transformer_logger(__name__) + + +def forward_backward_no_pipelining( + forward_step_func: FwdStepFunc, + batch: Batch, + model: Union[torch.nn.Module, List[torch.nn.Module]], + *, + forward_only: bool, + dtype: Optional[torch.dtype] = None, + grad_scaler: Optional[torch.cuda.amp.GradScaler] = None, + disable_autocast: bool = False, + custom_sync_context_handler=None, + **kwargs, +): + """Run forward and backward passes with no pipeline parallelism (no inter-stage communication). + + This pipeline parallel scheduling handles the last microbatch differently to synchronize gradients. + + Args: + forward_step_func: A function which takes a minibatch and model as its arguments and + returns model's forward output and the loss function. + The loss function is supposed to take one `torch.Tensor` and + return a `torch.Tensor` of loss and a dictionary of `str` and `torch.Tensor`. + batch: A List of torch.Tensors + model: A `torch.nn.Module` or a list of `torch.nn.Module`. + + Keyword args: + forward_only: + grad_scaler: + dtype: + disable_autocast: Turn off `enabled` flag of `torch.cuda.amp.autocast` if :obj:`True`. + Should be used when your forward and loss computation is in the autocast context to + avoid unnecesarily nest autocast context. + custom_sync_context_handler: Context manager to disable asynchronous gradient reductions. + **kwargs: Added to handle `tensor_shape` which has no effect on this function. + + Returns: + a list of dictionaries of loss `torch.Tensor`s if the last stage, empty list otherwise. + """ + model = listify_model(model) + if len(model) != 1: + msg = f"`model` is expected be a `nn.Module`, but {type(model)}" + raise RuntimeError(msg) + model = model[0] + model_type = get_model_type(model) + + if custom_sync_context_handler is not None: + context_handler = custom_sync_context_handler + elif isinstance(model, torch.nn.parallel.distributed.DistributedDataParallel): + context_handler = model.no_sync + else: + context_handler = contextlib.nullcontext + + losses_reduced = [] + input_tensor, output_tensor_grad = None, None + num_micro_batches = get_num_microbatches() + with context_handler(): + for i in range(num_micro_batches - 1): + _logger.info(f"Iter {i} of {num_micro_batches - 1}") + cur_micro_batch = get_kth_microbatch(batch, i) + _logger.debug("Call `forward_step`") + output_tensor = forward_step( + forward_step_func, + cur_micro_batch, + model, + input_tensor, + losses_reduced, + dtype=dtype, + disable_autocast=disable_autocast, + ) + if not forward_only: + _logger.debug("Call `backward_step`") + backward_step( + input_tensor, + output_tensor, + output_tensor_grad, + model_type=model_type, + grad_scaler=grad_scaler, + ) + + # Run computation for last microbatch out of context handler (want to + # synchronize gradients). + _logger.info("Cooldown") + _logger.debug("Call `forward_step`") + output_tensor = forward_step( + forward_step_func, + get_kth_microbatch(batch, num_micro_batches - 1), + model, + input_tensor, + losses_reduced, + dtype=dtype, + disable_autocast=disable_autocast, + ) + if not forward_only: + _logger.debug("Call `backward_step`") + backward_step( + input_tensor, + output_tensor, + output_tensor_grad, + model_type=model_type, + grad_scaler=grad_scaler, + ) + + return losses_reduced diff --git a/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_with_interleaving.py b/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_with_interleaving.py new file mode 100644 index 00000000..fea11182 --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_with_interleaving.py @@ -0,0 +1,744 @@ +import contextlib +from typing import Any, Callable, List, Optional, Sequence, Union +import warnings + +import torch + +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel import p2p_communication +from apex.transformer.pipeline_parallel.schedules.common import Batch +from apex.transformer.pipeline_parallel.schedules.common import FwdStepFunc +from apex.transformer.pipeline_parallel.schedules.common import backward_step +from apex.transformer.pipeline_parallel.schedules.common import forward_step +from apex.transformer.pipeline_parallel.schedules.common import free_output_tensor +from apex.transformer.pipeline_parallel.utils import get_kth_microbatch +from apex.transformer.pipeline_parallel.utils import get_num_microbatches +from apex.transformer.pipeline_parallel.utils import get_model_type +from apex.transformer.log_util import get_transformer_logger + + +__all__ = ["_forward_backward_pipelining_with_interleaving"] + + +_logger = get_transformer_logger(__name__) + + +# TODO(mkozuki): Reduce cyclomatic complexity +def _forward_backward_pipelining_with_interleaving( + forward_step_func: FwdStepFunc, + batch: List[Optional[Batch]], + model: List[torch.nn.Module], + *, + forward_only: bool, + tensor_shape: Optional[Union[List[int], torch.Size]] = None, + dtype: Optional[torch.dtype] = None, + grad_scaler: Optional[torch.cuda.amp.GradScaler] = None, + disable_autocast: bool = False, + deallocate_pipeline_outputs: bool = False, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + custom_sync_context_handler: Optional[Callable] = None, + custom_grad_sync_func: Optional[Callable] = None, + custom_param_sync_func: Optional[Callable] = None, + sync_batch_comm: bool = True, + num_micro_batches_with_partial_activation_checkpoints: Optional[int] = None, + overlap_p2p_comm: bool = False, + batch_p2p_comm: bool = True, + **kwargs, +) -> List[Union[torch.Tensor, Sequence[torch.Tensor]]]: + """Run interleaved 1F1B schedule with communication between pipeline stages as needed. + + This function assumes `batch` and `model` is a list of `Batch`'s and a list of `torch.nn.Module`, respectively. + This means that model is split into model chunks. + + This pipeline parallel scheduling consists of three steps: + 1. warmup + 2. 1F1B a.k.a. steady state + 3. cooldown + Note that if `forward_only` this scheduling consists of only warmup phase. + + Args: + forward_step_func: A function which takes a minibatch and model as its arguments and + returns model's forward output and the loss function. + The loss function is supposed to take one `torch.Tensor` and + return a `torch.Tensor` of loss and a dictionary of `str` and `torch.Tensor`. + batch: A minibatch, i.e., a list of `torch.Tensor`'s. + model: A `torch.nn.Module` or a list of `torch.nn.Module`. + + Keyword args: + forward_only: + tensor_shape: Shape of tensor. The tensor is expected to be 3D and its order of dimension + is supposed to be ``(sequence, batch, hidden)``. + dtype: dtype used in p2p communication. If ``None`` (default value), + torch.float32 will be used even if ``autocast`` is enabled. + grad_scaler: + disable_autocast: + deallocate_pipeline_outputs: If :obj:`True`, free the data of the output tensor of + each pipeline stage. Experimental. + sequence_parallel_enabled: Set to :obj:`True` for this function to handle sequence length. + When :obj:`True`, the sequence length on each tensor model parallel rank is updated + to :math:`original\_sequence\_length / tensor\_model\_parallel\_world\_size`. + custom_sync_context_handler: If provided, this is treated as a + function to construct a context manager to disable + asynchronous gradient reductions. Asynchronous gradient + reductions are only enabled in the final backward pass of + each model chunk. + custom_grad_sync_func: If provided, this is treated as a + function to launch asynchronous gradient reductions (e.g. + reduce-scatters with distributed optimizer). The function + should take one positional argument: a list of parameters + whose gradients should be synchronized. Asynchronous + gradient reductions are launched after the final backward + pass of each model chunk. + custom_param_sync_func: If provided, this is treated as a + function to launch asynchronous parameter synchronizations + (e.g. all-gathers with distributed optimizer). The + function should take one positional argument: a list of + parameters whose values should be synchronized. + Asynchronous parameter synchronizations are launched + before the first forward pass of each model chunk. + sync_batch_comm: If :obj:`False`, disable cuda synchronization after the batched communication. + To disable, https://github.com/pytorch/pytorch/pull/82450 would be required. + num_micro_batches_with_partial_activation_checkpoints: If :obj:`int`, set the number of + micro-batches checkpointing the activation of partial number of Transformer layers. + The rest of the micro-batch within the window of maximum outstanding micro-batch + backpropagations would checkpoint all Transformer layers. + overlap_p2p_comm: If :obj:`True`, returns cuda wait handles to scheduler instead of completing + the communication within the p2p transfer API instance. The scheduler manages the communication completion + to overlap with computation. + batch_p2p_comm: If :obj:`True`, use the batched send and receive api to conduct the communication of + a collection of send and receive operations between peer. If :obj:`False`, conduct each send and recv operation + individually. + + Returns: + a list of loss `torch.Tensor`s if the last stage, empty list otherwise. + + """ + if not isinstance(model, list): + raise RuntimeError("`model` must be a list of `nn.Module`'s'") + + if deallocate_pipeline_outputs: + warnings.warn( + "`deallocate_pipeline_outputs` is experimental and subject to change. " + "This option is not recommended." + ) + + # Construct helper functions for async grad reductions + if custom_sync_context_handler is not None: + sync_context_handler = custom_sync_context_handler + else: + sync_context_handler = contextlib.nullcontext + sync_context = None + def disable_grad_sync(): + """Disable asynchronous grad reductions""" + nonlocal sync_context + if sync_context is None: + sync_context = sync_context_handler() + sync_context.__enter__() + def enable_grad_sync(): + """Enable asynchronous grad reductions""" + nonlocal sync_context + if sync_context is not None: + sync_context.__exit__(None, None, None) + sync_context = None + disable_grad_sync() + + # mypy will blame the following if statement + if sequence_parallel_enabled: + seq_length, batch_size, hidden = tensor_shape + tensor_shape = ( + seq_length // parallel_state.get_tensor_model_parallel_world_size(), + batch_size, + hidden, + ) + + num_model_chunks: int = len(model) + input_tensors: List[List[Union[None, torch.Tensor]]] = [ + [] for _ in range(num_model_chunks) + ] + output_tensors: List[List[Union[None, torch.Tensor]]] = [ + [] for _ in range(num_model_chunks) + ] + curr_iters: List[int] = [0 for _ in range(num_model_chunks)] + losses_reduced: List[Union[None, torch.Tensor]] = [] + if not forward_only: + output_tensor_grads: List[List[Union[None, torch.Tensor]]] = [ + [] for _ in range(num_model_chunks) + ] + + pipeline_parallel_size: int = parallel_state.get_pipeline_model_parallel_world_size() + pipeline_parallel_rank: int = parallel_state.get_pipeline_model_parallel_rank() + + # Compute number of warmup and remaining microbatches. + num_microbatches: int = get_num_microbatches() * num_model_chunks + all_warmup_microbatches: bool = False + if forward_only: + num_warmup_microbatches: int = num_microbatches + else: + # Run all forward passes and then all backward passes if number of + # microbatches is just the number of pipeline stages. + # Otherwise, perform (num_model_chunks-1)*pipeline_parallel_size on + # all workers, followed by more microbatches after depending on + # stage ID (more forward passes for earlier stages, later stages can + # immediately start with 1F1B). + if get_num_microbatches() == pipeline_parallel_size: + num_warmup_microbatches = num_microbatches + all_warmup_microbatches = True + else: + num_warmup_microbatches = ( + pipeline_parallel_size - pipeline_parallel_rank - 1 + ) * 2 + num_warmup_microbatches += (num_model_chunks - 1) * pipeline_parallel_size + num_warmup_microbatches = min(num_warmup_microbatches, num_microbatches) + num_microbatches_remaining: int = num_microbatches - num_warmup_microbatches + + # Checkpoint the activations of partial Transformer layers in a number of micro-batches + # within the maximum outstanding micro-batch backpropagations. + # Micro-batches with the ids less than 'num_micro_batches_with_partial_activation_checkpoints' + # checkpoint partial Transformer layers (or skip checkpointing) and + # the rest of micro-batches within a window of micro-batches checkpoint + # all Transformer layers. The window of micro-batches is set by the maximum + # outstanding backpropagations and becomes smaller at later pipeline stages. + # Please refer the appendix C in https://arxiv.org/pdf/2205.05198.pdf + max_outstanding_backprops = None + if num_micro_batches_with_partial_activation_checkpoints is not None: + max_outstanding_backprops = num_warmup_microbatches + 1 + + _logger.info( + f"num_microbatches: {num_microbatches}, " + f"num_warmup_microbatches: {num_warmup_microbatches}, " + f"num_microbatches_remaining: {num_microbatches_remaining}" + ) + + # Synchronize params for first two model chunks + if custom_param_sync_func is not None: + custom_param_sync_func(model[0].parameters()) + custom_param_sync_func(model[1].parameters()) + + ################################################################################################################### + # Helper function definitions. + ################################################################################################################### + def get_model_chunk_id(microbatch_id: int, forward: bool) -> int: + """Helper function to get the model chunk ID given the iteration number. + + Each model chunk processes pipeline_parallel_size microbatches + at a time. We assume that the number of microbatches is a + multiple of pipeline_parallel_size*num_model_chunks. + """ + microbatch_group_size = pipeline_parallel_size * num_model_chunks + microbatch_id_in_group = microbatch_id % microbatch_group_size + model_chunk_id = microbatch_id_in_group // pipeline_parallel_size + if not forward: + model_chunk_id = num_model_chunks - model_chunk_id - 1 + return model_chunk_id + + def is_first_microbatch_for_model_chunk(microbatch_id: int) -> bool: + """Helper function to check if an iteration is the first for a model + chunk. + """ + microbatch_group_size = pipeline_parallel_size * num_model_chunks + num_microbatch_groups = num_microbatches // microbatch_group_size + microbatch_group_id = microbatch_id // microbatch_group_size + microbatch_id_in_group = microbatch_id % microbatch_group_size + if microbatch_group_id == 0: + return microbatch_id_in_group % pipeline_parallel_size == 0 + else: + return False + + def is_last_microbatch_for_model_chunk(microbatch_id: int) -> bool: + """Helper function to check if an iteration is the last for a model + chunk. + """ + microbatch_group_size = pipeline_parallel_size * num_model_chunks + num_microbatch_groups = num_microbatches // microbatch_group_size + microbatch_group_id = microbatch_id // microbatch_group_size + microbatch_id_in_group = microbatch_id % microbatch_group_size + if microbatch_group_id == num_microbatch_groups - 1: + return microbatch_id_in_group % pipeline_parallel_size == pipeline_parallel_size - 1 + else: + return False + + def forward_step_helper( + microbatch_id: int, + curr_iters: List[int], + checkpoint_activations_micro_batch: Optional[bool] = None, + ) -> torch.Tensor: + """Helper method to run forward step with model split into chunks + + (run set_virtual_pipeline_model_parallel_rank() before calling forward_step()). + """ + model_chunk_id = get_model_chunk_id(microbatch_id, forward=True) + parallel_state.set_virtual_pipeline_model_parallel_rank(model_chunk_id) + + # launch param synchronization for next model chunk + # Note: To achieve maximum performance, pipeline parallelism + # assumes all ranks have the same compute time. However, + # asynchronous communication tends to slow down compute. Thus, + # we launch asynchronous communication at the same time across + # the pipeline-parallel group. + if custom_param_sync_func is not None: + param_sync_microbatch_id = microbatch_id + pipeline_parallel_rank + if param_sync_microbatch_id < num_microbatches and is_first_microbatch_for_model_chunk(param_sync_microbatch_id): + param_sync_chunk_id = get_model_chunk_id(param_sync_microbatch_id, forward=True) + 1 + if 1 < param_sync_chunk_id < num_model_chunks: + custom_param_sync_func(model[param_sync_chunk_id].parameters()) + + # forward step + if parallel_state.is_pipeline_first_stage() and len( + input_tensors[model_chunk_id] + ) == len(output_tensors[model_chunk_id]): + input_tensors[model_chunk_id].append(None) + input_tensor = input_tensors[model_chunk_id][-1] + output_tensor = forward_step( + forward_step_func, + get_kth_microbatch(batch, curr_iters[model_chunk_id]), + model[model_chunk_id], + input_tensor, + losses_reduced, + dtype, + disable_autocast, + checkpoint_activations_micro_batch, + ) + curr_iters[model_chunk_id] += 1 + output_tensors[model_chunk_id].append(output_tensor) + + # if forward-only, no need to save tensors for a backward pass + if forward_only: + input_tensors[model_chunk_id].pop() + output_tensors[model_chunk_id].pop() + + return output_tensor + + def backward_step_helper(microbatch_id: int) -> torch.Tensor: + """Helper method to run backward step with model split into chunks + + (run set_virtual_pipeline_model_parallel_rank() before calling backward_step()). + """ + model_chunk_id = get_model_chunk_id(microbatch_id, forward=False) + model_type = get_model_type(model[model_chunk_id]) + parallel_state.set_virtual_pipeline_model_parallel_rank(model_chunk_id) + + # launch grad synchronization (default) + if custom_grad_sync_func is None and is_last_microbatch_for_model_chunk(microbatch_id): + enable_grad_sync() + + # backward step + if parallel_state.is_pipeline_last_stage(): + if len(output_tensor_grads[model_chunk_id]) == 0: + output_tensor_grads[model_chunk_id].append(None) + input_tensor = input_tensors[model_chunk_id].pop(0) + output_tensor = output_tensors[model_chunk_id].pop(0) + output_tensor_grad = output_tensor_grads[model_chunk_id].pop(0) + input_tensor_grad = backward_step( + input_tensor, + output_tensor, + output_tensor_grad, + model_type=model_type, + grad_scaler=grad_scaler, + deallocate_pipeline_outputs=deallocate_pipeline_outputs, + ) + + # launch grad synchronization (custom grad sync) + # Note: To achieve maximum performance, pipeline parallelism + # assumes all ranks have the same compute time. However, + # asynchronous communication tends to slow down compute. Thus, + # we launch asynchronous communication at the same time across + # the pipeline-parallel group. + if custom_grad_sync_func is not None: + grad_sync_microbatch_id = microbatch_id - pipeline_parallel_rank + if grad_sync_microbatch_id >= 0 and is_last_microbatch_for_model_chunk(grad_sync_microbatch_id): + grad_sync_chunk_id = get_model_chunk_id(grad_sync_microbatch_id, forward=False) + enable_grad_sync() + custom_grad_sync_func(model[grad_sync_chunk_id].parameters()) + disable_grad_sync() + + return input_tensor_grad + + ################################################################################################################### + # Run warmup forward passes. + ################################################################################################################### + fwd_wait_handles, bwd_wait_handles = None, None + parallel_state.set_virtual_pipeline_model_parallel_rank(0) + input_tensors[0].append( + p2p_communication.recv_forward( + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + ) + _logger.info("Warmup phase") + for k in range(num_warmup_microbatches): + _logger.debug(f"warmup iter: {k} / {num_warmup_microbatches}") + + # Decide to checkpoint all layers' activations of the current micro-batch + if max_outstanding_backprops is not None: + checkpoint_activations_micro_batch = k % max_outstanding_backprops >= \ + num_micro_batches_with_partial_activation_checkpoints + else: + checkpoint_activations_micro_batch = None + + if fwd_wait_handles is not None: + for wait_handle in fwd_wait_handles: + wait_handle.wait() + + output_tensor = forward_step_helper(k, curr_iters, checkpoint_activations_micro_batch) + + # Determine if tensor should be received from previous stage. + next_forward_model_chunk_id = get_model_chunk_id(k + 1, forward=True) + recv_prev = True + if parallel_state.is_pipeline_first_stage(ignore_virtual=True): + if next_forward_model_chunk_id == 0: + recv_prev = False + if k == (num_microbatches - 1): + recv_prev = False + _logger.debug( + f"next fwd model chunk ID: {next_forward_model_chunk_id}, recv_prev: {recv_prev}" + ) + + # Don't send tensor downstream if on last stage. + if parallel_state.is_pipeline_last_stage(): + _logger.debug("Pipeline last stage, not sending tensor downstream") + output_tensor = None + + if overlap_p2p_comm: + # P2P communications in warmup are not overlapped with computes. We split P2P + # communications for activation forward and activation_gradient backward in warmup, + # to match the send/recv API granularity in 1F1B in case of using batched send/recv API. + + # Send and receive tensors as appropriate (send tensors computed + # in this iteration; receive tensors for next iteration). + _logger.debug("send fwd and receive fwd") + input_tensor, fwd_wait_handles = p2p_communication.send_forward_recv_forward( + output_tensor, + recv_prev=recv_prev, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=True, + batch_p2p_comm=batch_p2p_comm, + ) + if ( + k == (num_warmup_microbatches - 1) + and not forward_only + and not all_warmup_microbatches + ): + input_tensor_grad = None + recv_next = True + if parallel_state.is_pipeline_last_stage(ignore_virtual=True): + recv_next = False + _logger.debug("send bwd and receive bwd") + output_tensor_grad, bwd_wait_handles = p2p_communication.send_backward_recv_backward( + input_tensor_grad, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=True, + batch_p2p_comm=batch_p2p_comm, + ) + output_tensor_grads[num_model_chunks - 1].append(output_tensor_grad) + input_tensors[next_forward_model_chunk_id].append(input_tensor) + else: + # Send and receive tensors as appropriate (send tensors computed + # in this iteration; receive tensors for next iteration). + if ( + k == (num_warmup_microbatches - 1) + and not forward_only + and not all_warmup_microbatches + ): + input_tensor_grad = None + recv_next = True + if parallel_state.is_pipeline_last_stage(ignore_virtual=True): + recv_next = False + _logger.debug("send fwd&bwd and receive fwd&bwd") + ( + input_tensor, + output_tensor_grad, + ) = p2p_communication.send_forward_backward_recv_forward_backward( + output_tensor, + input_tensor_grad, + recv_prev=recv_prev, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + output_tensor_grads[num_model_chunks - 1].append(output_tensor_grad) + else: + _logger.debug("send fwd and receive fwd") + input_tensor = p2p_communication.send_forward_recv_forward( + output_tensor, + recv_prev=recv_prev, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + input_tensors[next_forward_model_chunk_id].append(input_tensor) + free_output_tensor(output_tensor, deallocate_pipeline_outputs) + + ################################################################################################################### + # Run 1F1B in steady state. + ################################################################################################################### + _logger.info("Steady phase") + for k in range(num_microbatches_remaining): + # Forward pass. + _logger.debug(f" steady phase iter {k} / {num_microbatches_remaining}") + forward_k = k + num_warmup_microbatches + + # Decide to checkpoint all layers' activations of the current micro-batch + if max_outstanding_backprops is not None: + checkpoint_activations_micro_batch = ( + forward_k % max_outstanding_backprops >= num_micro_batches_with_partial_activation_checkpoints + ) + else: + checkpoint_activations_micro_batch = None + + if overlap_p2p_comm: + if fwd_wait_handles is not None: + for wait_handle in fwd_wait_handles: + wait_handle.wait() + + output_tensor = forward_step_helper(forward_k, curr_iters, checkpoint_activations_micro_batch) + + # Set forward model chunk id + forward_model_chunk_id = get_model_chunk_id(forward_k, forward=True) + parallel_state.set_virtual_pipeline_model_parallel_rank(forward_model_chunk_id) + + # Last virtual stage no activation tensor to send + if parallel_state.is_pipeline_last_stage(): + output_tensor = None + + # Determine if the current virtual stage has an activation tensor to receive + recv_prev = True + if parallel_state.is_pipeline_first_stage(ignore_virtual=True): + # First stage is ahead of last stage by (pipeline_parallel_size - 1). + next_forward_model_chunk_id = get_model_chunk_id( + forward_k - (pipeline_parallel_size - 1), forward=True + ) + if next_forward_model_chunk_id == (num_model_chunks - 1): + recv_prev = False + next_forward_model_chunk_id += 1 + else: + next_forward_model_chunk_id = get_model_chunk_id( + forward_k + 1, forward=True + ) + + # If last iteration, don't receive; we already received one extra + # before the start of the for loop. + if k == (num_microbatches_remaining - 1): + recv_prev = False + + # Send activation tensor to the next stage and receive activation tensor from the + # previous stage + _logger.debug("send fwd and receive fwd") + input_tensor, fwd_wait_handles = p2p_communication.send_forward_recv_forward( + output_tensor, + recv_prev=recv_prev, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=True, + batch_p2p_comm=batch_p2p_comm, + ) + + if bwd_wait_handles is not None: + for wait_handle in bwd_wait_handles: + wait_handle.wait() + + # Backward pass. + backward_k = k + input_tensor_grad = backward_step_helper(backward_k) + + # Set backward model chunk id + backward_model_chunk_id = get_model_chunk_id(backward_k, forward=False) + parallel_state.set_virtual_pipeline_model_parallel_rank(backward_model_chunk_id) + _logger.debug( + f"fwd/bwd model chunk id: {forward_model_chunk_id}/{backward_model_chunk_id}" + ) + + # First virtual stage no activation gradient tensor to send + if parallel_state.is_pipeline_first_stage(): + input_tensor_grad = None + + # Determine if the current virtual stage has an activation gradient tensor to receive + recv_next = True + if parallel_state.is_pipeline_last_stage(ignore_virtual=True): + # Last stage is ahead of first stage by (pipeline_parallel_size - 1). + next_backward_model_chunk_id = get_model_chunk_id( + backward_k - (pipeline_parallel_size - 1), forward=False + ) + if next_backward_model_chunk_id == 0: + recv_next = False + next_backward_model_chunk_id -= 1 + else: + next_backward_model_chunk_id = get_model_chunk_id( + backward_k + 1, forward=False + ) + + # Send activation grad tensor to the previous stage and receive activation grad tensor + # from the previous stage + _logger.debug("send bwd and receive bwd") + output_tensor_grad, bwd_wait_handles = p2p_communication.send_backward_recv_backward( + input_tensor_grad, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + overlap_p2p_comm=True, + batch_p2p_comm=batch_p2p_comm, + ) + else: + output_tensor = forward_step_helper(forward_k, curr_iters, checkpoint_activations_micro_batch) + + # Backward pass. + backward_k = k + input_tensor_grad = backward_step_helper(backward_k) + + # Send output_tensor and input_tensor_grad, receive input_tensor + # and output_tensor_grad. + + # Determine if current stage has anything to send in either direction, + # otherwise set tensor to None. + forward_model_chunk_id = get_model_chunk_id(forward_k, forward=True) + parallel_state.set_virtual_pipeline_model_parallel_rank(forward_model_chunk_id) + if parallel_state.is_pipeline_last_stage(): + output_tensor = None + + backward_model_chunk_id = get_model_chunk_id(backward_k, forward=False) + parallel_state.set_virtual_pipeline_model_parallel_rank(backward_model_chunk_id) + _logger.debug( + f"fwd/bwd model chunk id: {forward_model_chunk_id}/{backward_model_chunk_id}" + ) + if parallel_state.is_pipeline_first_stage(): + input_tensor_grad = None + + # Determine if peers are sending, and where in data structure to put + # received tensors. + recv_prev = True + if parallel_state.is_pipeline_first_stage(ignore_virtual=True): + # First stage is ahead of last stage by (pipeline_parallel_size - 1). + next_forward_model_chunk_id = get_model_chunk_id( + forward_k - (pipeline_parallel_size - 1), forward=True + ) + if next_forward_model_chunk_id == (num_model_chunks - 1): + recv_prev = False + next_forward_model_chunk_id += 1 + else: + next_forward_model_chunk_id = get_model_chunk_id( + forward_k + 1, forward=True + ) + + recv_next = True + if parallel_state.is_pipeline_last_stage(ignore_virtual=True): + # Last stage is ahead of first stage by (pipeline_parallel_size - 1). + next_backward_model_chunk_id = get_model_chunk_id( + backward_k - (pipeline_parallel_size - 1), forward=False + ) + if next_backward_model_chunk_id == 0: + recv_next = False + next_backward_model_chunk_id -= 1 + else: + next_backward_model_chunk_id = get_model_chunk_id( + backward_k + 1, forward=False + ) + + # If last iteration, don't receive; we already received one extra + # before the start of the for loop. + if k == (num_microbatches_remaining - 1): + recv_prev = False + + # Communicate tensors. + _logger.debug("send fwd&bwd and receive fwd&bwd") + ( + input_tensor, + output_tensor_grad, + ) = p2p_communication.send_forward_backward_recv_forward_backward( + output_tensor, + input_tensor_grad, + recv_prev=recv_prev, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + free_output_tensor(output_tensor, deallocate_pipeline_outputs) + + # Put input_tensor and output_tensor_grad in data structures in the + # right location. + if recv_prev: + input_tensors[next_forward_model_chunk_id].append(input_tensor) + if recv_next: + output_tensor_grads[next_backward_model_chunk_id].append(output_tensor_grad) + + ################################################################################################################### + # Run cooldown backward passes (flush out pipeline). + ################################################################################################################### + _logger.info("Cooldown phase") + if not forward_only: + if overlap_p2p_comm and bwd_wait_handles is not None: + for wait_handle in bwd_wait_handles: + wait_handle.wait() + + if all_warmup_microbatches: + output_tensor_grads[num_model_chunks - 1].append( + p2p_communication.recv_backward( + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + ) + + for k in range(num_microbatches_remaining, num_microbatches): + _logger.debug( + f"cooldown iter {k} in range({num_microbatches_remaining}, {num_microbatches})" + ) + input_tensor_grad = backward_step_helper(k) + next_backward_model_chunk_id = get_model_chunk_id(k + 1, forward=False) + + recv_next = True + if parallel_state.is_pipeline_last_stage(ignore_virtual=True): + if next_backward_model_chunk_id == (num_model_chunks - 1): + recv_next = False + if k == (num_microbatches - 1): + recv_next = False + + output_tensor_grads[next_backward_model_chunk_id].append( + p2p_communication.send_backward_recv_backward( + input_tensor_grad, + recv_next=recv_next, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + batch_p2p_comm=batch_p2p_comm, + ) + ) + + # Make sure to exit context handler for async grad reductions + enable_grad_sync() + + return losses_reduced diff --git a/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_without_interleaving.py b/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_without_interleaving.py new file mode 100644 index 00000000..b72710cc --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/schedules/fwd_bwd_pipelining_without_interleaving.py @@ -0,0 +1,597 @@ +import contextlib +from typing import Any, List, Optional, Sequence, Union +import warnings + +import torch + +from apex.transformer import parallel_state +from apex.transformer.enums import ModelType +from apex.transformer.pipeline_parallel import p2p_communication +from apex.transformer.pipeline_parallel.p2p_communication import FutureTensor +from apex.transformer.pipeline_parallel.utils import get_kth_microbatch +from apex.transformer.pipeline_parallel.utils import listify_model +from apex.transformer.pipeline_parallel.utils import get_num_microbatches +from apex.transformer.pipeline_parallel.utils import get_model_type +from apex.transformer.pipeline_parallel.schedules.common import Batch +from apex.transformer.pipeline_parallel.schedules.common import FwdStepFunc +from apex.transformer.pipeline_parallel.schedules.common import backward_step +from apex.transformer.pipeline_parallel.schedules.common import forward_step +from apex.transformer.pipeline_parallel.schedules.common import free_output_tensor +from apex.transformer.log_util import get_transformer_logger + + +__all__ = ["forward_backward_pipelining_without_interleaving"] + + +_logger = get_transformer_logger(__name__) + + +def get_tensor_shapes( + rank: int, + model_type: ModelType, + *, + tensor_shape: Union[List[int], torch.Size], + decoder_sequence_length: Optional[int] = None, + sequence_parallel_enabled: bool = False, +) -> Sequence[Sequence[int]]: + """Get tensors shapes + + Args: + rank: pipeline parallel rank + model_type: + + Keyword Args: + tensor_shape: + decoder_sequence_length: + sequence_parallel_enabled: + """ + # Determine right tensor sizes (based on position of rank with respect to split + # rank) and model size. + # Send two tensors if model is T5 and rank is in decoder stage: + # first tensor is decoder (pre-transpose), + # second tensor is encoder (post-transpose). + # If model is T5 and rank is at the boundary: + # send one tensor (post-transpose from encoder). + # Otherwise, send one tensor (pre-transpose). + assert ( + len(tensor_shape) == 3 + ), f"`tensor_shape` should be [sequence_length, micro_batch_size, hidden_size] but {tensor_shape}" + + sequence_length, micro_batch_size, hidden_size = tensor_shape + + tensor_shapes = [] + + if sequence_parallel_enabled: + seq_length = sequence_length // parallel_state.get_tensor_model_parallel_world_size() + else: + seq_length = sequence_length + + if model_type == ModelType.encoder_and_decoder: + + if sequence_parallel_enabled: + dec_seq_length = decoder_sequence_length // parallel_state.get_tensor_model_parallel_world_size() + else: + dec_seq_length = decoder_sequence_length + + if parallel_state.is_pipeline_stage_before_split(rank): + tensor_shapes.append((seq_length, micro_batch_size, hidden_size)) + else: + tensor_shapes.append((dec_seq_length, micro_batch_size, hidden_size)) + tensor_shapes.append((seq_length, micro_batch_size, hidden_size)) + else: + tensor_shapes.append((seq_length, micro_batch_size, hidden_size)) + + return tensor_shapes + + +def recv_forward( + tensor_shapes: List[Union[None, List[int]]], + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, +) -> List[Union[None, torch.Tensor, FutureTensor]]: + input_tensors = [] + for tensor_shape in tensor_shapes: + if tensor_shape is None: + input_tensors.append(None) + else: + input_tensors.append( + p2p_communication.recv_forward( + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + ) + return input_tensors + + +def recv_backward( + tensor_shapes: List[Union[None, List[int]]], + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, +) -> List[Union[None, torch.Tensor, FutureTensor]]: + output_tensor_grads = [] + for tensor_shape in tensor_shapes: + if tensor_shape is None: + output_tensor_grads.append(None) + else: + output_tensor_grads.append( + p2p_communication.recv_backward( + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + ) + return output_tensor_grads + + +def send_forward( + output_tensors: Union[torch.Tensor, List[Union[None, torch.Tensor]]], + tensor_shapes: List[Union[None, List[int]]], + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, +) -> None: + if not isinstance(output_tensors, list): + output_tensors = [output_tensors] + for (output_tensor, tensor_shape) in zip(output_tensors, tensor_shapes): + if tensor_shape is None: + continue + p2p_communication.send_forward( + output_tensor, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + +def send_backward( + input_tensor_grads: Union[torch.Tensor, List[Union[None, torch.Tensor]]], + tensor_shapes: List[Union[None, List[int]]], + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, +) -> None: + if not isinstance(input_tensor_grads, list): + input_tensor_grads = [input_tensor_grads] + for (input_tensor_grad, tensor_shape) in zip(input_tensor_grads, tensor_shapes): + if tensor_shape is None: + continue + p2p_communication.send_backward( + input_tensor_grad, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + +def send_forward_recv_backward( + output_tensors: Union[torch.Tensor, List[Union[None, torch.Tensor]]], + tensor_shapes: List[Union[None, List[int]]], + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, +) -> List[Union[None, torch.Tensor, FutureTensor]]: + if not isinstance(output_tensors, list): + output_tensors = [output_tensors] + output_tensor_grads = [] + for (output_tensor, tensor_shape) in zip(output_tensors, tensor_shapes): + if tensor_shape is None: + output_tensor_grads.append(None) + continue + output_tensor_grad = p2p_communication.send_forward_recv_backward( + output_tensor, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + output_tensor_grads.append(output_tensor_grad) + return output_tensor_grads + + +def send_backward_recv_forward( + input_tensor_grads: Union[torch.Tensor, List[Union[None, torch.Tensor]]], + tensor_shapes: List[Union[None, List[int]]], + *, + dtype: Optional[torch.dtype] = None, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + sync_batch_comm: bool = True, +) -> List[Union[None, torch.Tensor, FutureTensor]]: + if not isinstance(input_tensor_grads, list): + input_tensor_grads = [input_tensor_grads] + input_tensors = [] + for (input_tensor_grad, tensor_shape) in zip(input_tensor_grads, tensor_shapes): + if tensor_shape is None: + input_tensors.append(None) + continue + input_tensor = p2p_communication.send_backward_recv_forward( + input_tensor_grad, + tensor_shape=tensor_shape, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + input_tensors.append(input_tensor) + return input_tensors + + +def forward_backward_pipelining_without_interleaving( + forward_step_func: FwdStepFunc, + batch: Optional[Batch], + model: Union[torch.nn.Module, List[torch.nn.Module]], + *, + forward_only: bool, + tensor_shape: Optional[Union[List[int], torch.Size]] = None, + decoder_sequence_length: Optional[int] = None, + dtype: Optional[torch.dtype] = None, + grad_scaler: Optional[torch.cuda.amp.GradScaler] = None, + disable_autocast: bool = False, + deallocate_pipeline_outputs: bool = False, + async_comm: bool = False, + sequence_parallel_enabled: bool = False, + custom_sync_context_handler: Optional[Any] = None, + custom_grad_sync_func: Optional[Any] = None, + sync_batch_comm: bool = True, + num_micro_batches_with_partial_activation_checkpoints: Optional[int] = None, + **kwargs, +) -> List[Union[torch.Tensor, Sequence[torch.Tensor]]]: + """Run non-interleaved 1F1B schedule, with communication between pipeline stages. + + This pipeline parallel scheduling consists of three steps: + 1. warmup + 2. 1F1B a.k.a. steady state + 3. cooldown if not forward_only + + Args: + forward_step_func: A function which takes a minibatch and model as its arguments and + returns model's forward output and the loss function. + The loss function is supposed to take one `torch.Tensor` and + return a `torch.Tensor` of loss and a dictionary of `str` and `torch.Tensor`. + batch: A minibatch, i.e., a list of `torch.Tensor`'s. + model: A `torch.nn.Module` or a list of `torch.nn.Module`. + + Keyword args: + forward_only: + tensor_shape: Shape of tensor. The tensor is expected to be 3D and its order of dimension + is supposed to be ``(sequence, batch, hidden)``. + dtype: dtype used in p2p communication. If ``None`` (default value), + torch.float32 will be used even if ``autocast`` is enabled. + grad_scaler: + disable_autocast: + deallocate_pipeline_outputs: If :obj:`True`, free the data of the output tensor of + each pipeline stage. Experimental. + sequence_parallel_enabled: Set to :obj:`True` for this function to handle sequence length. + When :obj:`True`, the sequence length on each tensor model parallel rank is updated + to :math:`original\_sequence\_length / tensor\_model\_parallel\_world\_size`. + custom_sync_context_handler: Does nothing if ``None`` (default + value). Otherwise, a function to construct a context + manager that disable asynchronous gradient reductions. + Asynchronous gradient reductions are only enabled in the + first pipeline stage, during the last backward pass. + custom_grad_sync_func: Does nothing if ``None`` (default + value). Otherwise, a function to perform gradient + reductions. This is called in all pipeline stages except + the first, during the bubble overhead. + sync_batch_comm: If :obj:`False`, disable cuda synchronization after the batched communication. + To disable, https://github.com/pytorch/pytorch/pull/82450 would be required. + num_micro_batches_with_partial_activation_checkpoints: If :obj:`int`, set the number of + micro-batches checkpointing the activation of partial number of Transformer layers. + The rest of the micro-batch within the window of maximum outstanding micro-batch + backpropagations would checkpoint all Transformer layers. + + Returns: + a list of loss `torch.Tensor`s if the last stage, empty list otherwise. + + """ + # timers = get_timers() + + if deallocate_pipeline_outputs: + warnings.warn( + "`deallocate_pipeline_outputs` is experimental and subject to change. " + "This option is not recommended." + ) + + model: List[torch.nn.Module] = listify_model(model) + if len(model) != 1: + msg = f"`model` is expected be a `nn.Module`, but {type(model)}" + raise RuntimeError(msg) + model: torch.nn.Module = model[0] + + # Disable async grad reductions + if custom_sync_context_handler is not None: + sync_context_handler = custom_sync_context_handler + else: + sync_context_handler = contextlib.nullcontext + sync_context = None + def disable_grad_sync(): + """Disable asynchronous grad reductions""" + nonlocal sync_context + if sync_context is None: + sync_context = sync_context_handler() + sync_context.__enter__() + def enable_grad_sync(): + """Enable asynchronous grad reductions""" + nonlocal sync_context + if sync_context is not None: + sync_context.__exit__(None, None, None) + sync_context = None + disable_grad_sync() + + # Compute number of warmup microbatches. + num_microbatches: int = get_num_microbatches() + num_warmup_microbatches: int = ( + parallel_state.get_pipeline_model_parallel_world_size() - parallel_state.get_pipeline_model_parallel_rank() - 1 + ) + num_warmup_microbatches: int = min(num_warmup_microbatches, num_microbatches) + num_microbatches_remaining: int = num_microbatches - num_warmup_microbatches + + # Checkpoint the activations of partial Transformer layers in a number of micro-batches + # within the maximum outstanding micro-batch backpropagations. + # Micro-batches with the ids less than 'num_micro_batches_with_partial_activation_checkpoints' + # checkpoint partial Transformer layers (or skip checkpointing) and + # the rest of micro-batches within a window of micro-batches checkpoint + # all Transformer layers. The window of micro-batches is set by the maximum + # outstanding backpropagations and becomes smaller at later pipeline stages. + # Please refer the appendix C in https://arxiv.org/pdf/2205.05198.pdf + max_outstanding_backprops = None + if num_micro_batches_with_partial_activation_checkpoints is not None: + max_outstanding_backprops = num_warmup_microbatches + 1 + + model_type = get_model_type(model) + rank: int = parallel_state.get_pipeline_model_parallel_rank() + recv_tensor_shapes: List[List[int]] = get_tensor_shapes( + rank - 1, + model_type, + tensor_shape=tensor_shape, + decoder_sequence_length=decoder_sequence_length, + sequence_parallel_enabled=sequence_parallel_enabled, + ) + send_tensor_shapes: List[List[int]] = get_tensor_shapes( + rank, + model_type, + tensor_shape=tensor_shape, + decoder_sequence_length=decoder_sequence_length, + sequence_parallel_enabled=sequence_parallel_enabled, + ) + + _logger.info( + f"num_microbatches: {num_microbatches}, " + f"num_warmup_microbatches: {num_warmup_microbatches}, " + f"num_microbatches_remaining: {num_microbatches_remaining}" + ) + + # Input, output tensors only need to be saved when doing backward passes + input_tensors: List[Union[None, torch.Tensor]] = [] + output_tensors: List[Union[None, torch.Tensor]] = [] + losses_reduced: List[Union[None, torch.Tensor]] = [] + ################################################################################################################### + # Run warmup forward passes. + ################################################################################################################### + _logger.info("Warmup") + for i in range(num_warmup_microbatches): + _logger.debug(f"warmup iter: {i} / {num_warmup_microbatches}") + _logger.debug("receive fwd") + + # Decide to checkpoint all layers' activations of the current micro-batch + if max_outstanding_backprops is not None: + checkpoint_activations_micro_batch = ( + i % max_outstanding_backprops >= num_micro_batches_with_partial_activation_checkpoints + ) + else: + checkpoint_activations_micro_batch = None + input_tensor = recv_forward( + tensor_shapes=recv_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + cur_microbatch: Optional[torch.Tensor] = get_kth_microbatch(batch, i) + output_tensor = forward_step( + forward_step_func, + cur_microbatch, + model, + input_tensor, + losses_reduced, + dtype, + disable_autocast, + checkpoint_activations_micro_batch, + ) + _logger.debug("send fwd") + send_forward( + output_tensor, + tensor_shapes=send_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + if not forward_only: + input_tensors.append(input_tensor) + output_tensors.append(output_tensor) + free_output_tensor(output_tensor, deallocate_pipeline_outputs) + + # Before running 1F1B, need to receive first forward tensor. + # If all microbatches are run in warmup / cooldown phase, then no need to + # receive this tensor here. + if num_microbatches_remaining > 0: + _logger.debug("recv_forward before steady state start") + input_tensor: List[Union[None, torch.Tensor, FutureTensor]] = recv_forward( + tensor_shapes=recv_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sync_batch_comm=sync_batch_comm, + ) + + ################################################################################################################### + # Run 1F1B in steady state. + ################################################################################################################### + _logger.info("Steady phase") + for i in range(num_microbatches_remaining): + _logger.debug(f"steady iter: {i} / {num_microbatches_remaining}") + last_iteration: bool = i == (num_microbatches_remaining - 1) + + # Decide to checkpoint all layers' activations of the current micro-batch + if max_outstanding_backprops is not None: + checkpoint_activations_micro_batch = ( + ((i+num_warmup_microbatches) % max_outstanding_backprops) >= num_micro_batches_with_partial_activation_checkpoints + ) + else: + checkpoint_activations_micro_batch = None + cur_microbatch: Optional[torch.Tensor] = get_kth_microbatch(batch, i + num_warmup_microbatches) + output_tensor: Union[torch.Tensor, Sequence[torch.Tensor]] = forward_step( + forward_step_func, + cur_microbatch, + model, + input_tensor, + losses_reduced, + dtype, + disable_autocast, + checkpoint_activations_micro_batch, + ) + if forward_only: + _logger.debug("send fwd") + send_forward( + output_tensor, + tensor_shapes=send_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + if not last_iteration: + _logger.debug("receive fwd (last iteration)") + input_tensor = recv_forward( + tensor_shapes=recv_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + else: + _logger.debug("send fwd & receive bwd") + output_tensor_grad = send_forward_recv_backward( + output_tensor, + tensor_shapes=send_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + # Add input_tensor and output_tensor to end of list. + input_tensors.append(input_tensor) + output_tensors.append(output_tensor) + free_output_tensor(output_tensor, deallocate_pipeline_outputs) + + # Pop input_tensor and output_tensor from the start of the list for the backward pass. + input_tensor = input_tensors.pop(0) + output_tensor = output_tensors.pop(0) + + input_tensor_grad = backward_step( + input_tensor, + output_tensor, + output_tensor_grad, + model_type=model_type, + grad_scaler=grad_scaler, + deallocate_pipeline_outputs=deallocate_pipeline_outputs, + ) + + if last_iteration: + input_tensor = None + _logger.debug("send bwd") + send_backward( + input_tensor_grad, + tensor_shapes=recv_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + else: + _logger.debug("send bwd and receive fwd") + input_tensor = send_backward_recv_forward( + input_tensor_grad, + tensor_shapes=recv_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + ################################################################################################################### + # Run cooldown backward passes. + ################################################################################################################### + _logger.info("Cooldown phase") + if not forward_only: + for i in range(num_warmup_microbatches): + _logger.debug(f"cooldown iter: {i} / {num_warmup_microbatches}") + + if i == num_warmup_microbatches-1 and rank == 0: + # Async grad reduction in first pipeline stage, during + # last backward pass + enable_grad_sync() + + input_tensor = input_tensors.pop(0) + output_tensor = output_tensors.pop(0) + + _logger.debug("receive bwd") + output_tensor_grad = recv_backward( + tensor_shapes=send_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + input_tensor_grad = backward_step( + input_tensor, + output_tensor, + output_tensor_grad, + model_type=model_type, + grad_scaler=grad_scaler, + deallocate_pipeline_outputs=deallocate_pipeline_outputs, + ) + + _logger.debug("send bwd") + send_backward( + input_tensor_grad, + tensor_shapes=recv_tensor_shapes, + dtype=dtype, + async_comm=async_comm, + sequence_parallel_enabled=sequence_parallel_enabled, + sync_batch_comm=sync_batch_comm, + ) + + # Grad reduction in all pipeline stages except the first, during + # the bubble overhead + enable_grad_sync() + if rank != 0 and custom_grad_sync_func is not None: + custom_grad_sync_func() + + return losses_reduced diff --git a/apex/apex/transformer/pipeline_parallel/utils.py b/apex/apex/transformer/pipeline_parallel/utils.py new file mode 100644 index 00000000..7738934c --- /dev/null +++ b/apex/apex/transformer/pipeline_parallel/utils.py @@ -0,0 +1,357 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for pipeline model parallel.""" +from typing import Optional, List, Union, Tuple + +import torch +from torch.nn.parallel import DistributedDataParallel + +from apex.multi_tensor_apply import multi_tensor_applier +from apex.transformer import parallel_state +from apex.transformer.enums import ModelType +from apex.transformer.microbatches import build_num_microbatches_calculator +from apex.transformer.pipeline_parallel._timers import _Timers +if multi_tensor_applier.available: + import amp_C + + +_GLOBAL_ARGS = None +_GLOBAL_NUM_MICROBATCHES_CALCULATOR = None +_GLOBAL_TOKENIZER = None +_GLOBAL_TENSORBOARD_WRITER = None +_GLOBAL_AUTORESUME = None +_GLOBAL_TIMERS = None + + +Shape = Union[List[int], torch.Size] + + +def listify_model(model: Union[torch.nn.Module, List[torch.nn.Module]]) -> List[torch.nn.Module]: + if isinstance(model, list): + return model + return [model] + + +def _ensure_var_is_initialized(var, name): + """Make sure the input variable is not None.""" + assert var is not None, "{} is not initialized.".format(name) + + +def _ensure_var_is_not_initialized(var, name): + """Make sure the input variable is not None.""" + assert var is None, "{} is already initialized.".format(name) + + +def setup_microbatch_calculator( + rank: int, + rampup_batch_size: Optional[List[int]], + global_batch_size: int, + micro_batch_size: int, + data_parallel_size: int, +) -> None: + global _GLOBAL_NUM_MICROBATCHES_CALCULATOR + _ensure_var_is_not_initialized(_GLOBAL_NUM_MICROBATCHES_CALCULATOR, 'num microbatches calculator') + + _GLOBAL_NUM_MICROBATCHES_CALCULATOR = build_num_microbatches_calculator( + rank, rampup_batch_size, global_batch_size, micro_batch_size, data_parallel_size) + + +def _reconfigure_microbatch_calculator( + rank: int, + rampup_batch_size: Optional[List[int]], + global_batch_size: int, + micro_batch_size: int, + data_parallel_size: int, +) -> None: + if torch.distributed.get_rank() == 0: + import warnings + warnings.warn("This function is only for unittest") + global _GLOBAL_NUM_MICROBATCHES_CALCULATOR + + _GLOBAL_NUM_MICROBATCHES_CALCULATOR = build_num_microbatches_calculator( + rank, rampup_batch_size, global_batch_size, micro_batch_size, data_parallel_size) + + +def get_micro_batch_size(): + return _GLOBAL_NUM_MICROBATCHES_CALCULATOR.micro_batch_size + + +def get_num_microbatches(): + return _GLOBAL_NUM_MICROBATCHES_CALCULATOR.get() + + +def get_current_global_batch_size(): + return _GLOBAL_NUM_MICROBATCHES_CALCULATOR.get_current_global_batch_size() + + +def update_num_microbatches(consumed_samples, consistency_check=True): + _GLOBAL_NUM_MICROBATCHES_CALCULATOR.update(consumed_samples, consistency_check) + + +# note (mkozuki): Comment out in favor of `get_kth_microbatch` +def _split_batch_into_microbatch( + batch: List[torch.Tensor], + *, + _micro_batch_size: Optional[int] = None, + _global_batch_size: Optional[int] = None, +) -> List[List[torch.Tensor]]: + micro_batch_size = _micro_batch_size + global_batch_size = _global_batch_size + if micro_batch_size is None: + micro_batch_size = get_micro_batch_size() + if global_batch_size is None: + global_batch_size = get_current_global_batch_size() + for i in range(0, global_batch_size, micro_batch_size): + yield [x[i * micro_batch_size:(i + 1) * micro_batch_size] for x in batch] + + +# TODO(mkozuki): Support non-tensor local minibatches? +def get_kth_microbatch(batch: Optional[List[torch.Tensor]], k: int) -> List[torch.Tensor]: + """Create a list of microbatches from a list of local minibatches. + + This function creates a list of `k`th microbatches from a list of local minibatches. + `a local minibatch` consists of `global_batch_size / data_parallel_size` samples. + """ + if batch is None or not isinstance(batch, (List, Tuple)): + return batch + micro_batch_size = get_micro_batch_size() + start = k * micro_batch_size + end = start + micro_batch_size + microbatch = list() + for x in batch: + size = x.size(0) + assert size > start and size >= end + microbatch.append(x[start:end]) + assert len(microbatch) > 0 + return microbatch + + +def get_autoresume(): + return _GLOBAL_AUTORESUME + + +def _set_timers(): + """Initialize timers.""" + global _GLOBAL_TIMERS + _ensure_var_is_not_initialized(_GLOBAL_TIMERS, "timers") + _GLOBAL_TIMERS = _Timers() + + +def get_timers(): + """Return timers.""" + _ensure_var_is_initialized(_GLOBAL_TIMERS, "timers") + return _GLOBAL_TIMERS + + +def print_rank_0(message: str) -> None: + """If distributed is initialized, print only on rank 0.""" + if torch.distributed.is_initialized(): + if torch.distributed.get_rank() == 0: + print(message, flush=True) + else: + print(message, flush=True) + + +def is_last_rank(): + return torch.distributed.get_rank() == (torch.distributed.get_world_size() - 1) + + +def print_rank_last(message): + """If distributed is initialized, print only on last rank.""" + if torch.distributed.is_initialized(): + if is_last_rank(): + print(message, flush=True) + else: + print(message, flush=True) + + +def param_is_not_shared(param: torch.nn.Parameter) -> bool: + return getattr(param, "shared", False) + + +def unwrap_model(model, module_instances=(DistributedDataParallel,)): + return_list = True + if not isinstance(model, list): + model = [model] + return_list = False + unwrapped_model = [] + for model_module in model: + while isinstance(model_module, module_instances): + model_module = model_module.module + unwrapped_model.append(model_module) + if not return_list: + return unwrapped_model[0] + return unwrapped_model + + +def get_model_type( + model: torch.nn.Module, +) -> ModelType: + """Get `model_type` of `model`. + + If ``model`` doesn't have ``model_type`` attribute, return ``ModelType.encoder_or_decoder``. + + Args: + model + """ + return getattr(unwrap_model(model), "model_type", ModelType.encoder_or_decoder) + + +def calc_params_l2_norm(model: torch.nn.Module, bf16: bool): + """Calculate l2 norm of parameters """ + # args = get_args() + if not isinstance(model, list): + model = [model] + # Remove duplicate params. + params_data = [] + for model_ in model: + for param in model_.parameters(): + is_not_shared = param_is_not_shared(param) + is_not_tp_duplicate = parallel_state.param_is_not_tensor_parallel_duplicate(param) + if is_not_shared and is_not_tp_duplicate: + if bf16: + params_data.append(param.data.float()) + else: + params_data.append(param.data) + # Calculate norm + dummy_overflow_buf = torch.cuda.IntTensor([0]) + norm, _ = multi_tensor_applier( + amp_C.multi_tensor_l2norm, dummy_overflow_buf, [params_data], False # no per-parameter norm + ) + norm_2 = norm * norm + # Sum across all model-parallel GPUs. + torch.distributed.all_reduce( + norm_2, op=torch.distributed.ReduceOp.SUM, group=parallel_state.get_model_parallel_group() + ) + return norm_2.item() ** 0.5 + + +def average_losses_across_data_parallel_group(losses): + """Reduce a tensor of losses across all GPUs.""" + averaged_losses = torch.cat([loss.clone().detach().view(1) for loss in losses]) + torch.distributed.all_reduce(averaged_losses, group=parallel_state.get_data_parallel_group()) + averaged_losses = averaged_losses / torch.distributed.get_world_size( + group=parallel_state.get_data_parallel_group() + ) + + return averaged_losses + + +def report_memory(name): + """Simple GPU memory report.""" + mega_bytes = 1024.0 * 1024.0 + string = name + " memory (MB)" + string += " | allocated: {}".format(torch.cuda.memory_allocated() / mega_bytes) + string += " | max allocated: {}".format(torch.cuda.max_memory_allocated() / mega_bytes) + string += " | reserved: {}".format(torch.cuda.memory_reserved() / mega_bytes) + string += " | max reserved: {}".format(torch.cuda.max_memory_reserved() / mega_bytes) + if parallel_state.get_data_parallel_rank() == 0: + print("[Rank {}] {}".format(torch.distributed.get_rank(), string), flush=True) + + +def print_params_min_max_norm(optimizer, iteration): + """Print min, max, and norm of all parameters.""" + index = 0 + rank = torch.distributed.get_rank() + string = "iteration, rank, index, tensor-model-parallel, min, max, norm\n" + optimizer_ = optimizer.optimizer + for param_group in optimizer_.param_groups: + for param in param_group["params"]: + index += 1 + min_ = param.data.min() + max_ = param.data.max() + norm = torch.linalg.norm(param.data) + string += "{:7d}, {:4d}, {:4d}, {:2d}, ".format( + iteration, rank, index, int(param.tensor_model_parallel) + ) + string += "{:.6E}, {:.6E}, {:.6E}\n".format(min_, max_, norm) + print(string, flush=True) + + +# NOTE (mkozuki): APEX doesn't have anything equivalent for +# `_GLOBAL_ADLR_AUTORESUME` like Megatron-LM. +# def check_adlr_autoresume_termination(iteration, model, optimizer, lr_scheduler, save: bool): +# """Check for autoresume signal and exit if it is received.""" +# from apex.ppu.checkpointing import save_checkpoint +# +# autoresume = get_adlr_autoresume() +# # Add barrier to ensure consistency. +# torch.distributed.barrier() +# if autoresume.termination_requested(): +# if save: +# save_checkpoint(iteration, model, optimizer, lr_scheduler) +# print_rank_0(">>> autoresume termination request found!") +# if torch.distributed.get_rank() == 0: +# autoresume.request_resume() +# print_rank_0(">>> training terminated. Returning") +# sys.exit(0) + + +def get_ltor_masks_and_position_ids( + data, eod_token, reset_position_ids, reset_attention_mask, eod_mask_loss +): + """Build masks and position id for left to right model.""" + + # Extract batch size and sequence length. + micro_batch_size, seq_length = data.size() + + # Attention mask (lower triangular). + if reset_attention_mask: + att_mask_batch = micro_batch_size + else: + att_mask_batch = 1 + attention_mask = torch.tril( + torch.ones((att_mask_batch, seq_length, seq_length), device=data.device) + ).view(att_mask_batch, 1, seq_length, seq_length) + + # Loss mask. + loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device) + if eod_mask_loss: + loss_mask[data == eod_token] = 0.0 + + # Position ids. + position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device) + position_ids = position_ids.unsqueeze(0).expand_as(data) + # We need to clone as the ids will be modifed based on batch index. + if reset_position_ids: + position_ids = position_ids.clone() + + if reset_position_ids or reset_attention_mask: + # Loop through the batches: + for b in range(micro_batch_size): + + # Find indecies where EOD token is. + eod_index = position_ids[b, data[b] == eod_token] + # Detach indecies from positions if going to modify positions. + if reset_position_ids: + eod_index = eod_index.clone() + + # Loop through EOD indecies: + prev_index = 0 + for j in range(eod_index.size()[0]): + i = eod_index[j] + # Mask attention loss. + if reset_attention_mask: + attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0 + # Reset positions. + if reset_position_ids: + position_ids[b, (i + 1) :] -= i + 1 - prev_index + prev_index = i + 1 + + # Convert attention mask to binary: + attention_mask = attention_mask < 0.5 + + return attention_mask, loss_mask, position_ids diff --git a/apex/apex/transformer/tensor_parallel/__init__.py b/apex/apex/transformer/tensor_parallel/__init__.py new file mode 100644 index 00000000..ccad80e6 --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/__init__.py @@ -0,0 +1,75 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Model parallel utility interface.""" + +from apex.transformer.tensor_parallel.cross_entropy import vocab_parallel_cross_entropy + +from apex.transformer.tensor_parallel.data import broadcast_data + +from apex.transformer.tensor_parallel.layers import ( + ColumnParallelLinear, + RowParallelLinear, + VocabParallelEmbedding, + set_tensor_model_parallel_attributes, + set_defaults_if_not_set_tensor_model_parallel_attributes, + copy_tensor_model_parallel_attributes, +) + +from apex.transformer.tensor_parallel.mappings import ( + copy_to_tensor_model_parallel_region, + gather_from_tensor_model_parallel_region, + reduce_from_tensor_model_parallel_region, + scatter_to_tensor_model_parallel_region, + scatter_to_sequence_parallel_region, +) + +from .random import ( + checkpoint, + get_cuda_rng_tracker, + init_checkpointed_activations_memory_buffer, + model_parallel_cuda_manual_seed, + reset_checkpointed_activations_memory_buffer, +) + +from apex.transformer.tensor_parallel.utils import split_tensor_along_last_dim + + +__all__ = [ + # cross_entropy.py + "vocab_parallel_cross_entropy", + # data.py + "broadcast_data", + # layers.py + "ColumnParallelLinear", + "RowParallelLinear", + "VocabParallelEmbedding", + "set_tensor_model_parallel_attributes", + "set_defaults_if_not_set_tensor_model_parallel_attributes", + "copy_tensor_model_parallel_attributes", + # mappings.py + "copy_to_tensor_model_parallel_region", + "gather_from_tensor_model_parallel_region", + "reduce_from_tensor_model_parallel_region", + "scatter_to_tensor_model_parallel_region", + "scatter_to_sequence_parallel_region", + # random.py + "checkpoint", + "get_cuda_rng_tracker", + "init_checkpointed_activations_memory_buffer", + "model_parallel_cuda_manual_seed", + "reset_checkpointed_activations_memory_buffer", + # utils.py + "split_tensor_along_last_dim", +] diff --git a/apex/apex/transformer/tensor_parallel/cross_entropy.py b/apex/apex/transformer/tensor_parallel/cross_entropy.py new file mode 100644 index 00000000..b14d7c19 --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/cross_entropy.py @@ -0,0 +1,134 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from apex.transformer.parallel_state import get_tensor_model_parallel_group +from apex.transformer.parallel_state import get_tensor_model_parallel_rank +from apex.transformer.parallel_state import get_tensor_model_parallel_world_size +from apex.transformer.tensor_parallel.utils import VocabUtility + + +class _VocabParallelCrossEntropy(torch.autograd.Function): + @staticmethod + def forward(ctx, vocab_parallel_logits, target, label_smoothing=0.0): + + # Maximum value along vocab dimension across all GPUs. + logits_max = torch.max(vocab_parallel_logits, dim=-1)[0] + torch.distributed.all_reduce( + logits_max, op=torch.distributed.ReduceOp.MAX, group=get_tensor_model_parallel_group() + ) + # Subtract the maximum value. + vocab_parallel_logits = vocab_parallel_logits - logits_max.unsqueeze(dim=-1) + + # Get the partition's vocab indecies + get_vocab_range = VocabUtility.vocab_range_from_per_partition_vocab_size + partition_vocab_size = vocab_parallel_logits.size()[-1] + rank = get_tensor_model_parallel_rank() + world_size = get_tensor_model_parallel_world_size() + vocab_start_index, vocab_end_index = get_vocab_range(partition_vocab_size, rank, world_size) + + # Create a mask of valid vocab ids (1 means it needs to be masked). + target_mask = (target < vocab_start_index) | (target >= vocab_end_index) + masked_target = target.clone() - vocab_start_index + masked_target[target_mask] = 0 + + # Get predicted-logits = logits[target]. + # For Simplicity, we convert logits to a 2-D tensor with size + # [*, partition-vocab-size] and target to a 1-D tensor of size [*]. + logits_2d = vocab_parallel_logits.view(-1, partition_vocab_size) + masked_target_1d = masked_target.view(-1) + arange_1d = torch.arange(start=0, end=logits_2d.size()[0], device=logits_2d.device) + predicted_logits_1d = logits_2d[arange_1d, masked_target_1d] + predicted_logits_1d = predicted_logits_1d.clone().contiguous() + predicted_logits = predicted_logits_1d.view_as(target) + predicted_logits[target_mask] = 0.0 + # All reduce is needed to get the chunks from other GPUs. + torch.distributed.all_reduce( + predicted_logits, op=torch.distributed.ReduceOp.SUM, group=get_tensor_model_parallel_group() + ) + + # Sum of exponential of logits along vocab dimension across all GPUs. + exp_logits = vocab_parallel_logits + torch.exp(vocab_parallel_logits, out=exp_logits) + sum_exp_logits = exp_logits.sum(dim=-1) + torch.distributed.all_reduce( + sum_exp_logits, op=torch.distributed.ReduceOp.SUM, group=get_tensor_model_parallel_group() + ) + + # Loss = log(sum(exp(logits))) - predicted-logit. + loss = torch.log(sum_exp_logits) - predicted_logits + + # Store softmax, target-mask and masked-target for backward pass. + exp_logits.div_(sum_exp_logits.unsqueeze(dim=-1)) + + vocab_size = exp_logits.size(-1) + if label_smoothing > 0: + """ + We'd like to assign 1 / (K - 1) probability mass to every index that is not the ground truth. + = (1 - alpha) * y_gt + alpha * mean(y_{i for i != gt}) + = (1 - alpha) * y_gt + (alpha / (K - 1)) * \sum_{i != gt} y_i + = ((K - 1) * (1 - alpha) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_{i != gt} y_i + = (K * (1 - alpha) - 1) / (K - 1)) * y_gt + (alpha / (K - 1)) * \sum_{i} y_i + = (1 - (alpha * K) / (K - 1)) * y_gt + ( (alpha * K) / (K - 1) ) * \sum_{i} y_i / K + From: https://github.com/NVIDIA/NeMo/blob/main/nemo/collections/common/losses/smoothed_cross_entropy.py + """ + assert 1.0 > label_smoothing > 0.0 + smoothing = label_smoothing * vocab_size / (vocab_size - 1) + + # Exp logits at this point are normalized probabilities. So we can just take the log to get log-probs. + log_probs = torch.log(exp_logits) + mean_log_probs = log_probs.mean(dim=-1) + loss = (1.0 - smoothing) * loss - smoothing * mean_log_probs + + ctx.label_smoothing, ctx.vocab_size = label_smoothing, vocab_size + ctx.save_for_backward(exp_logits, target_mask, masked_target_1d) + + return loss + + @staticmethod + def backward(ctx, grad_output): + + # Retreive tensors from the forward path. + softmax, target_mask, masked_target_1d = ctx.saved_tensors + label_smoothing, vocab_size = ctx.label_smoothing, ctx.vocab_size + + # All the inputs have softmax as thier gradient. + grad_input = softmax + # For simplicity, work with the 2D gradient. + partition_vocab_size = softmax.size()[-1] + grad_2d = grad_input.view(-1, partition_vocab_size) + + # Add the gradient from matching classes. + arange_1d = torch.arange(start=0, end=grad_2d.size()[0], device=grad_2d.device) + + softmax_update = 1.0 - target_mask.view(-1).float() + + if label_smoothing > 0: + smoothing = label_smoothing * vocab_size / (vocab_size - 1) + grad_2d[arange_1d, masked_target_1d] -= (1.0 - smoothing) * softmax_update + average_grad = 1 / vocab_size + grad_2d[arange_1d, :] -= smoothing * average_grad + else: + grad_2d[arange_1d, masked_target_1d] -= softmax_update + + # Finally elementwise multiplication with the output gradients. + grad_input.mul_(grad_output.unsqueeze(dim=-1)) + + return grad_input, None, None + + +def vocab_parallel_cross_entropy(vocab_parallel_logits, target, label_smoothing=0.0): + """Helper function for the cross entropy.""" + return _VocabParallelCrossEntropy.apply(vocab_parallel_logits, target, label_smoothing) diff --git a/apex/apex/transformer/tensor_parallel/data.py b/apex/apex/transformer/tensor_parallel/data.py new file mode 100644 index 00000000..39d6ca8e --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/data.py @@ -0,0 +1,122 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from apex.transformer.parallel_state import get_tensor_model_parallel_group +from apex.transformer.parallel_state import get_tensor_model_parallel_rank +from apex.transformer.parallel_state import get_tensor_model_parallel_src_rank + + +_MAX_DATA_DIM = 5 + + +def _check_data_types(keys, data, target_dtype): + """Check that all the keys have the same target data type.""" + for key in keys: + assert data[key].dtype == target_dtype, ( + "{} has data type {} which " + "is different than {}".format(key, data[key].dtype, target_dtype) + ) + + +def _build_key_size_numel_dictionaries(keys, data): + """Build the size on rank 0 and broadcast.""" + max_dim = _MAX_DATA_DIM + sizes = [0 for _ in range(max_dim) for _ in keys] + + # Pack the sizes on rank zero. + if get_tensor_model_parallel_rank() == 0: + offset = 0 + for key in keys: + assert data[key].dim() < max_dim, "you should increase MAX_DATA_DIM" + size = data[key].size() + for i, s in enumerate(size): + sizes[i + offset] = s + offset += max_dim + + # Move to GPU and broadcast. + sizes_cuda = torch.cuda.LongTensor(sizes) + torch.distributed.broadcast( + sizes_cuda, + get_tensor_model_parallel_src_rank(), + group=get_tensor_model_parallel_group(), + ) + + # Move back to cpu and unpack. + sizes_cpu = sizes_cuda.cpu() + key_size = {} + key_numel = {} + total_numel = 0 + offset = 0 + for key in keys: + i = 0 + size = [] + numel = 1 + while sizes_cpu[offset + i] > 0: + this_size = sizes_cpu[offset + i] + size.append(this_size) + numel *= this_size + i += 1 + key_size[key] = size + key_numel[key] = numel + total_numel += numel + offset += max_dim + + return key_size, key_numel, total_numel + + +def broadcast_data(keys, data, datatype): + """Broadcast data from rank zero of each model parallel group to the + members of the same model parallel group. + + Arguments: + keys: list of keys in the data disctionary to be broadcasted + data: data dictionary of string keys and cpu tensor values. + datatype: torch data type of all tensors in data associated + with keys. + """ + # Build (key, size) and (key, number of elements) dictionaries along + # with the total number of elements on all ranks. + key_size, key_numel, total_numel = _build_key_size_numel_dictionaries(keys, data) + # Pack on rank zero. + if get_tensor_model_parallel_rank() == 0: + # Check that all keys have the same data type. + _check_data_types(keys, data, datatype) + # Flatten the data associated with the keys + flatten_data = torch.cat( + [data[key].contiguous().view(-1) for key in keys], dim=0 + ).cuda() + else: + flatten_data = torch.empty( + total_numel, device=torch.cuda.current_device(), dtype=datatype + ) + + # Broadcast + torch.distributed.broadcast( + flatten_data, + get_tensor_model_parallel_src_rank(), + group=get_tensor_model_parallel_group(), + ) + + # Unpack + output = {} + offset = 0 + for key in keys: + size = key_size[key] + numel = key_numel[key] + output[key] = flatten_data.narrow(0, offset, numel).view(size) + offset += numel + + return output diff --git a/apex/apex/transformer/tensor_parallel/layers.py b/apex/apex/transformer/tensor_parallel/layers.py new file mode 100644 index 00000000..6fee7c8c --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/layers.py @@ -0,0 +1,813 @@ +# coding=utf-8 +# Copyright (c) 2021-22, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Parts of the code here are adapted from PyTorch +# repo: https://github.com/pytorch/pytorch +from typing import Optional, Dict, Tuple, List +import warnings + +import torch +import torch.nn.functional as F +import torch.nn.init as init +from torch.nn.parameter import Parameter + +from apex._autocast_utils import _cast_if_autocast_enabled +from apex.transformer.parallel_state import get_tensor_model_parallel_group +from apex.transformer.parallel_state import get_tensor_model_parallel_rank +from apex.transformer.parallel_state import get_tensor_model_parallel_world_size +from apex.transformer.utils import divide +from apex.transformer.tensor_parallel.mappings import ( + copy_to_tensor_model_parallel_region, +) +from apex.transformer.tensor_parallel.mappings import ( + gather_from_tensor_model_parallel_region, +) +from apex.transformer.tensor_parallel.mappings import ( + reduce_from_tensor_model_parallel_region, +) +from apex.transformer.tensor_parallel.mappings import ( + scatter_to_tensor_model_parallel_region, +) +from apex.transformer.tensor_parallel.mappings import ( + reduce_scatter_to_sequence_parallel_region, +) +from apex.transformer.tensor_parallel.random import get_cuda_rng_tracker +from apex.transformer.tensor_parallel.utils import VocabUtility +from apex.transformer.log_util import get_transformer_logger + +# `all_gather_into_tensor` and `reduce_scatter_tensor` are new placeholders for +# `_all_gather_base` and `_reduce_scatter_base`. They require the most recent +# version of PyTorch. The following 4 lines are for backward comparability with +# older PyTorch. +if "reduce_scatter_tensor" not in dir(torch.distributed): + torch.distributed.reduce_scatter_tensor = torch.distributed._reduce_scatter_base +if "all_gather_into_tensor" not in dir(torch.distributed): + torch.distributed.all_gather_into_tensor = torch.distributed._all_gather_base + +_logger = get_transformer_logger(__name__) + + +_grad_accum_fusion_available = True +try: + import fused_weight_gradient_mlp_cuda +except ImportError: + _grad_accum_fusion_available = False + + +_MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = { + "tensor_model_parallel": False, + "partition_dim": -1, + "partition_stride": 1, +} + +def param_is_not_tensor_parallel_duplicate(param: torch.Tensor) -> bool: + return ( + hasattr(param, "tensor_model_parallel") and param.tensor_model_parallel + ) or (get_tensor_model_parallel_rank() == 0) + + +def set_tensor_model_parallel_attributes(tensor: torch.Tensor, is_parallel: bool, dim: int, stride: int) -> None: + # Make sure the attributes are not set. + for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS: + assert not hasattr(tensor, attribute) + # Set the attributes. + setattr(tensor, "tensor_model_parallel", is_parallel) + setattr(tensor, "partition_dim", dim) + setattr(tensor, "partition_stride", stride) + + +def set_defaults_if_not_set_tensor_model_parallel_attributes(tensor: torch.Tensor) -> None: + def maybe_set(attribute, value): + if not hasattr(tensor, attribute): + setattr(tensor, attribute, value) + + for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS: + maybe_set(attribute, _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS[attribute]) + + +def copy_tensor_model_parallel_attributes(destination_tensor: torch.Tensor, source_tensor: torch.Tensor) -> None: + def maybe_copy(attribute): + if hasattr(source_tensor, attribute): + setattr(destination_tensor, attribute, getattr(source_tensor, attribute)) + + for attribute in _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS: + maybe_copy(attribute) + + +def _initialize_affine_weight_gpu(weight, init_method, partition_dim, stride=1): + """Initialize affine weight for model parallel on GPU. + + Args: + weight (Parameter): + init_method (Callable[[Tensor], None]): Taking a Tensor and initialize its elements. + partition_dim (int): Dimension to apply partition. + stride (int): + """ + + set_tensor_model_parallel_attributes( + tensor=weight, is_parallel=True, dim=partition_dim, stride=stride + ) + + with get_cuda_rng_tracker().fork(): + init_method(weight) + + +# TODO (mkozuki): Re-consider removing params_dtype from arguments to make this +# more parallel with _initialize_affine_weight_gpu +def _initialize_affine_weight_cpu( + weight, + output_size, + input_size, + per_partition_size, + partition_dim, + init_method, + stride=1, + return_master_weight=False, + *, + params_dtype=torch.float32, +): + """Initialize affine weight for model parallel. + + Build the master weight on all processes and scatter + the relevant chunk.""" + + set_tensor_model_parallel_attributes( + tensor=weight, is_parallel=True, dim=partition_dim, stride=stride + ) + + # Initialize master weight + master_weight = torch.empty( + output_size, input_size, dtype=torch.float, requires_grad=False + ) + init_method(master_weight) + master_weight = master_weight.to(dtype=params_dtype) + + # Split and copy + per_partition_per_stride_size = divide(per_partition_size, stride) + weight_list = torch.split( + master_weight, per_partition_per_stride_size, dim=partition_dim + ) + rank = get_tensor_model_parallel_rank() + world_size = get_tensor_model_parallel_world_size() + my_weight_list = weight_list[rank::world_size] + + with torch.no_grad(): + torch.cat(my_weight_list, dim=partition_dim, out=weight) + if return_master_weight: + return master_weight + return None + + +class VocabParallelEmbedding(torch.nn.Module): + """Embedding parallelized in the vocabulary dimension. + + This is mainly adapted from torch.nn.Embedding and all the default + values are kept. + Arguments: + num_embeddings: vocabulary size. + embedding_dim: size of hidden state. + init_method: method to initialize weights. + """ + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + init_method=init.xavier_normal_, + *, + params_dtype: torch.dtype=torch.float32, + use_cpu_initialization: bool = False, + ): + super().__init__() + # Keep the input dimensions. + self.num_embeddings = num_embeddings + self.embedding_dim = embedding_dim + # Set the detauls for compatibility. + self.padding_idx = None + self.max_norm = None + self.norm_type = 2.0 + self.scale_grad_by_freq = False + self.sparse = False + self._weight = None + self.tensor_model_parallel_size = get_tensor_model_parallel_world_size() + # Divide the weight matrix along the vocabulary dimension. + ( + self.vocab_start_index, + self.vocab_end_index, + ) = VocabUtility.vocab_range_from_global_vocab_size( + self.num_embeddings, + get_tensor_model_parallel_rank(), + self.tensor_model_parallel_size, + ) + self.num_embeddings_per_partition = ( + self.vocab_end_index - self.vocab_start_index + ) + + # Allocate weights and initialize. + if use_cpu_initialization: + self.weight = Parameter( + torch.empty( + self.num_embeddings_per_partition, + self.embedding_dim, + dtype=params_dtype, + ) + ) + _initialize_affine_weight_cpu( + self.weight, + self.num_embeddings, + self.embedding_dim, + self.num_embeddings_per_partition, + 0, + init_method, + params_dtype=params_dtype, + ) + else: + self.weight = Parameter( + torch.empty( + self.num_embeddings_per_partition, + self.embedding_dim, + device=torch.cuda.current_device(), + dtype=params_dtype, + ) + ) + _initialize_affine_weight_gpu( + self.weight, init_method, partition_dim=0, stride=1 + ) + + def forward(self, input_): + if self.tensor_model_parallel_size > 1: + # Build the mask. + input_mask = (input_ < self.vocab_start_index) | ( + input_ >= self.vocab_end_index + ) + # Mask the input. + masked_input = input_.clone() - self.vocab_start_index + masked_input[input_mask] = 0 + else: + masked_input = input_ + # Get the embeddings. + output_parallel = F.embedding( + masked_input, + self.weight, + self.padding_idx, + self.max_norm, + self.norm_type, + self.scale_grad_by_freq, + self.sparse, + ) + # Mask the output embedding. + if self.tensor_model_parallel_size > 1: + output_parallel[input_mask, :] = 0.0 + # Reduce across all the model parallel GPUs. + output = reduce_from_tensor_model_parallel_region(output_parallel) + return output + + +class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function): + """Linear layer execution with asynchronous communication and gradient accumulation fusion in backprop.""" + + @staticmethod + def forward( + ctx, + input: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor], + gradient_accumulation_fusion: bool, + async_grad_allreduce: bool, + sequence_parallel_enabled: bool, + use_16bit_in_wgrad_accum_fusion: Optional[bool] = None, + ): + ctx.use_bias = bias is not None and weight.requires_grad + ctx.gradient_accumulation_fusion = gradient_accumulation_fusion + ctx.async_grad_allreduce = async_grad_allreduce + ctx.sequence_parallel_enabled = sequence_parallel_enabled + ctx.compute_weight_gradient = weight.requires_grad + + if use_16bit_in_wgrad_accum_fusion is not None: + warnings.warn( + "Deprecated option `use_16bit_in_wgrad_accum_fusion` " + f"is set to {use_16bit_in_wgrad_accum_fusion}" + ) + + if ctx.compute_weight_gradient: + ctx.save_for_backward(input, weight) + else: + ctx.save_for_backward(weight) + + + if ctx.sequence_parallel_enabled: + world_size = get_tensor_model_parallel_world_size() + # `input` is supposed to be 3D and its order of dimension is [sequence, batch, hidden] + shape = list(input.shape) + shape[0] *= world_size + + all_gather_buffer = torch.empty( + shape, + dtype=input.dtype, + device=torch.cuda.current_device(), + requires_grad=False, + ) + torch.distributed.all_gather_into_tensor(all_gather_buffer, input, group=get_tensor_model_parallel_group()) + total_input = all_gather_buffer + else: + total_input = input + output = torch.matmul(total_input, weight.t()) + if bias is not None: + output = output + bias + return output + + @staticmethod + def backward(ctx, grad_output): + if ctx.compute_weight_gradient: + input, weight = ctx.saved_tensors + else: + weight = ctx.saved_tensors[0] + input = None + + use_bias = ctx.use_bias + + #only get sequence parallel inputs if need to calculate weight grad + handle = None + if ctx.compute_weight_gradient: + if ctx.sequence_parallel_enabled: + world_size = get_tensor_model_parallel_world_size() + shape = list(input.shape) + shape[0] *= world_size + + all_gather_buffer = torch.empty( + shape, + dtype=input.dtype, + device=torch.cuda.current_device(), + requires_grad=False, + ) + handle = torch.distributed.all_gather_into_tensor( + all_gather_buffer, + input, + group=get_tensor_model_parallel_group(), + async_op=True, + ) + total_input = all_gather_buffer + else: + total_input = input + + grad_input = grad_output.matmul(weight) + + if handle is not None: + handle.wait() + + if ctx.async_grad_allreduce: + # Asynchronous all-reduce + handle = torch.distributed.all_reduce( + grad_input, group=get_tensor_model_parallel_group(), async_op=True + ) + + #if no weight gradient, immediately return + if not ctx.compute_weight_gradient: + if ctx.sequence_parallel_enabled: + assert not ctx.async_grad_allreduce + world_size = get_tensor_model_parallel_world_size() + shape = list(grad_input.shape) + shape[0] //= world_size + + sub_grad_input = torch.empty(torch.Size(shape), dtype=grad_input.dtype, device=torch.cuda.current_device(), requires_grad=False) + handle = torch.distributed.reduce_scatter_tensor( + sub_grad_input, + grad_input, + group=get_tensor_model_parallel_group(), + async_op=True + ) + handle.wait() + return sub_grad_input, None, None, None, None, None, None + if ctx.async_grad_allreduce: + handle.wait() + return grad_input, None, None, None, None, None, None + + # Convert the tensor shapes to 2D for execution compatibility + grad_output = grad_output.contiguous() + grad_output = grad_output.view( + grad_output.shape[0] * grad_output.shape[1], grad_output.shape[2] + ) + total_input = total_input.view(total_input.shape[0] * total_input.shape[1], total_input.shape[2]) + + if ctx.sequence_parallel_enabled: + assert not ctx.async_grad_allreduce + sub_grad_input = torch.empty(input.shape, dtype=input.dtype, device=torch.cuda.current_device(), requires_grad=False) + handle = torch.distributed.reduce_scatter_tensor( + sub_grad_input, + grad_input, + group=get_tensor_model_parallel_group(), + async_op=True + ) + + if ctx.gradient_accumulation_fusion: + if not hasattr(weight, "main_grad"): + raise RuntimeError("attempted to perform gradient accumulation fusion on param without setting main_grad") + if weight.main_grad.dtype == torch.float32: + fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp32( + total_input, grad_output, weight.main_grad + ) + elif weight.main_grad.dtype in (torch.float16, torch.bfloat16): + fused_weight_gradient_mlp_cuda.wgrad_gemm_accum_fp16( + total_input, grad_output, weight.main_grad + ) + else: + raise RuntimeError(f"unsupported dtype for main_grad ({weight.main_grad.dtype})") + grad_weight = None + else: + grad_weight = grad_output.t().matmul(total_input) + grad_bias = grad_output.sum(dim=0) if use_bias else None + if ctx.sequence_parallel_enabled: + handle.wait() + return sub_grad_input, grad_weight, grad_bias, None, None, None, None + if ctx.async_grad_allreduce: + handle.wait() + return grad_input, grad_weight, grad_bias, None, None, None, None + + +def linear_with_grad_accumulation_and_async_allreduce( + input: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor], + gradient_accumulation_fusion: bool, + async_grad_allreduce: bool, + sequence_parallel_enabled: bool, +) -> torch.Tensor: + args = _cast_if_autocast_enabled( + input, + weight, + bias, + gradient_accumulation_fusion, + async_grad_allreduce, + sequence_parallel_enabled, + ) + with torch.cuda.amp.autocast(enabled=False): + return LinearWithGradAccumulationAndAsyncCommunication.apply(*args) + + +class ColumnParallelLinear(torch.nn.Module): + """Linear layer with column parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its second dimension as A = [A_1, ..., A_p]. + + .. note:: + Input is supposed to be three dimensional and each dimension + is expected to be sequence, batch, and hidden feature, respectively. + + Arguments: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias + gather_output: If true, call all-gether on output and make Y avaiable + to all GPUs, otherwise, every GPU will have its output + which is Y_i = XA_i + init_method: method to initialize weights. Note that bias is always set + to zero. + stride: For the strided linear layers. + keep_master_weight_for_test: This was added for testing and should be + set to False. It returns the master weights + used for initialization. + skip_bias_add: This was added to enable performance optimations where bias + can be fused with other elementwise operations. we skip + adding bias but instead return it. + + Keyword Arguments: + no_async_tensor_model_parallel_allreduce: + params_dtype: + use_cpu_initialization: + gradient_accumulation_fusion: + sequence_parallel_enabled: + accumulation_in_fp16: Deprecated + """ + + def __init__( + self, + input_size, + output_size, + bias=True, + gather_output=True, + init_method=init.xavier_normal_, + stride=1, + keep_master_weight_for_test=False, + skip_bias_add=False, + *, + no_async_tensor_model_parallel_allreduce=False, + params_dtype=torch.float32, + use_cpu_initialization=False, + gradient_accumulation_fusion=False, + sequence_parallel_enabled: bool = False, + accumulation_in_fp16: Optional[bool] = None, + ): + super().__init__() + + # Keep input parameters + self.input_size = input_size + self.output_size = output_size + self.gather_output = gather_output + # Divide the weight matrix along the last dimension. + world_size = get_tensor_model_parallel_world_size() + self.output_size_per_partition = divide(output_size, world_size) + self.skip_bias_add = skip_bias_add + + if accumulation_in_fp16 is not None: + warnings.warn( + f"Deprecated option `accumulation_in_fp16` is set to {accumulation_in_fp16}" + ) + + # Parameters. + # Note: torch.nn.functional.linear performs XA^T + b and as a result + # we allocate the transpose. + # Initialize weight. + if use_cpu_initialization: + self.weight = Parameter( + torch.empty(self.output_size_per_partition, self.input_size, dtype=params_dtype) + ) + self.master_weight = _initialize_affine_weight_cpu( + self.weight, + self.output_size, + self.input_size, + self.output_size_per_partition, + 0, + init_method, + stride=stride, + return_master_weight=keep_master_weight_for_test, + params_dtype=params_dtype, + ) + else: + self.weight = Parameter( + torch.empty( + self.output_size_per_partition, + self.input_size, + device=torch.cuda.current_device(), + dtype=params_dtype, + ) + ) + _initialize_affine_weight_gpu(self.weight, init_method, partition_dim=0, stride=stride) + + if bias: + if use_cpu_initialization: + self.bias = Parameter(torch.empty(self.output_size_per_partition, dtype=params_dtype)) + else: + self.bias = Parameter( + torch.empty( + self.output_size_per_partition, + device=torch.cuda.current_device(), + dtype=params_dtype, + ) + ) + set_tensor_model_parallel_attributes(self.bias, True, 0, stride) + # Always initialize bias to zero. + with torch.no_grad(): + self.bias.zero_() + else: + self.register_parameter("bias", None) + + self.async_tensor_model_parallel_allreduce = ( + not no_async_tensor_model_parallel_allreduce and world_size > 1 + ) + if sequence_parallel_enabled: + if world_size <= 1: + warnings.warn( + f"`sequence_parallel_enabled` is set to `True`, but got world_size of {world_size}" + ) + # sequence_parallel_enabled = False + self.sequence_parallel_enabled = sequence_parallel_enabled + if gradient_accumulation_fusion: + if not _grad_accum_fusion_available: + # Basically, apex.transformer module users are expected to install APEX's + # `--cpp_ext` and `--cuda_ext`. The example installation command is as follows: + # `pip install --global-option="--cpp_ext" --global-option="--cuda_ext ." + # at the root of APEX repository. + warnings.warn( + "`gradient_accumulation_fusion` is set to `True` but " + "the custom CUDA extension of `fused_weight_gradient_mlp_cuda` module not " + "found. Thus `gradient_accumulation_fusion` set to `False`. " + "Note that the extension requires CUDA>=11." + ) + gradient_accumulation_fusion = False + self.gradient_accumulation_fusion = gradient_accumulation_fusion + + + if self.async_tensor_model_parallel_allreduce and self.sequence_parallel_enabled: + raise RuntimeError("`async_tensor_model_parallel_allreduce` and `sequence_parallel_enabled` cannot be enabled at the same time.") + + self._forward_impl = linear_with_grad_accumulation_and_async_allreduce + + def forward(self, input_: torch.Tensor) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Forward of ColumnParallelLinear + + Args: + input_: 3D tensor whose order of dimension is [sequence, batch, hidden] + + Returns: + - output + - bias + """ + bias = self.bias if not self.skip_bias_add else None + + if self.async_tensor_model_parallel_allreduce or self.sequence_parallel_enabled: + input_parallel = input_ + else: + input_parallel = copy_to_tensor_model_parallel_region(input_) + + # Matrix multiply. + output_parallel = self._forward_impl( + input=input_parallel, + weight=self.weight, + bias=bias, + gradient_accumulation_fusion=self.gradient_accumulation_fusion, + async_grad_allreduce=self.async_tensor_model_parallel_allreduce, + sequence_parallel_enabled=self.sequence_parallel_enabled, + ) + if self.gather_output: + # All-gather across the partitions. + assert not self.sequence_parallel_enabled + output = gather_from_tensor_model_parallel_region(output_parallel) + else: + output = output_parallel + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + +class RowParallelLinear(torch.nn.Module): + """Linear layer with row parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its first dimension and X along its second dimension as: + - - + | A_1 | + | . | + A = | . | X = [X_1, ..., X_p] + | . | + | A_p | + - - + + .. note:: + Input is supposed to be three dimensional and each dimension + is expected to be sequence, batch, and hidden feature, respectively. + + Arguments: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. Note that bias is not parallelized. + input_is_parallel: If true, we assume that the input is already + split across the GPUs and we do not split + again. + init_method: method to initialize weights. Note that bias is always set + to zero. + stride: For the strided linear layers. + keep_master_weight_for_test: This was added for testing and should be + set to False. It returns the master weights + used for initialization. + skip_bias_add: This was added to enable performance optimization where bias + can be fused with other elementwise operations. We skip + adding bias but instead return it. + Keyword Arguments: + params_dtype: + use_cpu_initialization: + gradient_accumulation_fusion: + sequence_parallel_enabled: + accumulation_in_fp16: Deprecated + """ + + def __init__( + self, + input_size, + output_size, + bias=True, + input_is_parallel=False, + init_method=init.xavier_normal_, + stride=1, + keep_master_weight_for_test=False, + skip_bias_add=False, + *, + params_dtype=torch.float32, + use_cpu_initialization=False, + gradient_accumulation_fusion=False, + sequence_parallel_enabled: bool = False, + accumulation_in_fp16: Optional[bool] = None, + ): + super().__init__() + + # Keep input parameters + self.input_size = input_size + self.output_size = output_size + self.input_is_parallel = input_is_parallel + # Divide the weight matrix along the last dimension. + world_size = get_tensor_model_parallel_world_size() + self.input_size_per_partition = divide(input_size, world_size) + self.skip_bias_add = skip_bias_add + self.gradient_accumulation_fusion = gradient_accumulation_fusion + self.sequence_parallel_enabled = sequence_parallel_enabled + if self.sequence_parallel_enabled and not self.input_is_parallel: + raise RuntimeError("To enable `sequence_parallel_enabled`, `input_is_parallel` must be `True`") + + if accumulation_in_fp16 is not None: + warnings.warn( + f"Deprecated option `accumulation_in_fp16` is set to {accumulation_in_fp16}" + ) + + # as an argument to this function? + # Parameters. + # Note: torch.nn.functional.linear performs XA^T + b and as a result + # we allocate the transpose. + # Initialize weight. + if use_cpu_initialization: + self.weight = Parameter( + torch.empty( + self.output_size, self.input_size_per_partition, dtype=params_dtype + ) + ) + self.master_weight = _initialize_affine_weight_cpu( + self.weight, + self.output_size, + self.input_size, + self.input_size_per_partition, + 1, + init_method, + stride=stride, + return_master_weight=keep_master_weight_for_test, + params_dtype=params_dtype, + ) + else: + self.weight = Parameter( + torch.empty( + self.output_size, + self.input_size_per_partition, + device=torch.cuda.current_device(), + dtype=params_dtype, + ) + ) + _initialize_affine_weight_gpu( + self.weight, init_method, partition_dim=1, stride=stride + ) + if bias: + if use_cpu_initialization: + self.bias = Parameter(torch.empty(self.output_size, dtype=params_dtype)) + else: + self.bias = Parameter( + torch.empty( + self.output_size, + device=torch.cuda.current_device(), + dtype=params_dtype, + ) + ) + # Always initialize bias to zero. + with torch.no_grad(): + self.bias.zero_() + setattr(self.bias, "sequence_parallel_enabled", sequence_parallel_enabled) + else: + self.register_parameter("bias", None) + + self._forward_impl = linear_with_grad_accumulation_and_async_allreduce + + def forward(self, input_: torch.Tensor) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Forward of RowParallelLinear + + Args: + input_: 3D tensor whose order of dimension is [sequence, batch, hidden] + + Returns: + - output + - bias + """ + # Set up backprop all-reduce. + if self.input_is_parallel: + input_parallel = input_ + else: + assert not self.sequence_parallel_enabled + input_parallel = scatter_to_tensor_model_parallel_region(input_) + # Matrix multiply. + output_parallel = self._forward_impl( + input=input_parallel, + weight=self.weight, + bias=None, + gradient_accumulation_fusion=self.gradient_accumulation_fusion, + async_grad_allreduce=False, + sequence_parallel_enabled=False, + ) + # All-reduce across all the partitions. + if self.sequence_parallel_enabled: + output_ = reduce_scatter_to_sequence_parallel_region(output_parallel) + else: + output_ = reduce_from_tensor_model_parallel_region(output_parallel) + if not self.skip_bias_add: + output = output_ + self.bias if self.bias is not None else output_ + output_bias = None + else: + output = output_ + output_bias = self.bias + return output, output_bias diff --git a/apex/apex/transformer/tensor_parallel/mappings.py b/apex/apex/transformer/tensor_parallel/mappings.py new file mode 100644 index 00000000..ae1f8c18 --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/mappings.py @@ -0,0 +1,312 @@ +# coding=utf-8 +# Copyright (c) 2021-22, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch + +from apex.transformer.parallel_state import get_tensor_model_parallel_group +from apex.transformer.parallel_state import get_tensor_model_parallel_world_size +from apex.transformer.parallel_state import get_tensor_model_parallel_rank +from apex.transformer.tensor_parallel.utils import split_tensor_along_last_dim + +# `all_gather_into_tensor` and `reduce_scatter_tensor` are new placeholders for +# `_all_gather_base` and `_reduce_scatter_base`. They require the most recent +# version of PyTorch. The following 4 lines are for backward comparability with +# older PyTorch. +if "all_gather_into_tensor" not in dir(torch.distributed): + torch.distributed.all_gather_into_tensor = torch.distributed._all_gather_base +if "reduce_scatter_tensor" not in dir(torch.distributed): + torch.distributed.reduce_scatter_tensor = torch.distributed._reduce_scatter_base + +def _reduce(input_: torch.Tensor) -> torch.Tensor: + """All-reduce the input tensor across model parallel group.""" + + # Bypass the function if we are using only 1 GPU. + if get_tensor_model_parallel_world_size() == 1: + return input_ + + # All-reduce. + torch.distributed.all_reduce(input_, group=get_tensor_model_parallel_group()) + + return input_ + + +def _split_along_last_dim(input_: torch.Tensor) -> torch.Tensor: + """Split the tensor along its last dimension and keep the + corresponding slice.""" + + world_size = get_tensor_model_parallel_world_size() + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + + # Split along last dimension. + input_list = split_tensor_along_last_dim(input_, world_size) + + # Note: torch.split does not create contiguous tensors by default. + rank = get_tensor_model_parallel_rank() + output = input_list[rank].contiguous() + + return output + + +def _split_along_first_dim(input_: torch.Tensor) -> torch.Tensor: + """Split the tensor along its first dimension and keep the corresponding slice.""" + world_size = get_tensor_model_parallel_world_size() + # Bypass the function if we are using only 1 GPU for tensor model parallel. + if world_size == 1: + return input_ + + # Split along first dimension. + dim_size = input_.size(0) + assert dim_size % world_size == 0 + local_dim_size = dim_size // world_size + dim_offset = get_tensor_model_parallel_rank() * local_dim_size + output = input_[dim_offset:dim_offset + local_dim_size].contiguous() + return output + + +def _gather_along_last_dim(input_: torch.Tensor) -> torch.Tensor: + """Gather tensors and concatenate along the last dimension.""" + + world_size = get_tensor_model_parallel_world_size() + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + + # Size and dimension. + last_dim = input_.dim() - 1 + rank = get_tensor_model_parallel_rank() + + tensor_list = [torch.empty_like(input_) for _ in range(world_size)] + tensor_list[rank] = input_ + torch.distributed.all_gather( + tensor_list, input_, group=get_tensor_model_parallel_group() + ) + + # Note: torch.cat already creates a contiguous tensor. + output = torch.cat(tensor_list, dim=last_dim).contiguous() + + return output + + +def _gather_along_first_dim(input_: torch.Tensor) -> torch.Tensor: + """Gather tensors and concatenate along the first dimension.""" + world_size = get_tensor_model_parallel_world_size() + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + + shape = list(input_.shape) + shape[0] *= world_size + + output = torch.empty(shape, dtype=input_.dtype, device=torch.cuda.current_device()) + torch.distributed.all_gather_into_tensor( + output, + input_.contiguous(), + group=get_tensor_model_parallel_group() + ) + return output + + +def _reduce_scatter_along_first_dim(input_: torch.Tensor) -> torch.Tensor: + """Reduce-scatter the input tensor across model parallel group.""" + world_size = get_tensor_model_parallel_world_size() + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + + shape = list(input_.shape) + assert shape[0] % world_size == 0 + shape[0] //= world_size + output = torch.empty(shape, dtype=input_.dtype, device=torch.cuda.current_device()) + torch.distributed.reduce_scatter_tensor( + output, + input_.contiguous(), + group=get_tensor_model_parallel_group() + ) + return output + + +class _CopyToModelParallelRegion(torch.autograd.Function): + """Pass the input to the tensor model parallel region.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_): + return input_ + + @staticmethod + def forward(ctx, input_): + return input_ + + @staticmethod + def backward(ctx, grad_output): + return _reduce(grad_output) + + +class _ReduceFromModelParallelRegion(torch.autograd.Function): + """All-reduce the input from the tensor model parallel region.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_): + return _reduce(input_) + + @staticmethod + def forward(ctx, input_): + return _reduce(input_) + + @staticmethod + def backward(ctx, grad_output): + return grad_output + + +class _ScatterToModelParallelRegion(torch.autograd.Function): + """Split the input and keep only the corresponding chuck to the rank.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_): + return _split_along_last_dim(input_) + + @staticmethod + def forward(ctx, input_): + return _split_along_last_dim(input_) + + @staticmethod + def backward(ctx, grad_output): + return _gather_along_last_dim(grad_output) + + +class _GatherFromModelParallelRegion(torch.autograd.Function): + """Gather the input from tensor model parallel region and concatenate.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_): + return _gather_along_last_dim(input_) + + @staticmethod + def forward(ctx, input_): + return _gather_along_last_dim(input_) + + @staticmethod + def backward(ctx, grad_output): + return _split_along_last_dim(grad_output) + + +class _ScatterToSequenceParallelRegion(torch.autograd.Function): + """Split the input and keep only the corresponding chunk to the rank.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_): + return _split_along_first_dim(input_) + + @staticmethod + def forward(ctx, input_): + return _split_along_first_dim(input_) + + @staticmethod + def backward(ctx, grad_output): + return _gather_along_first_dim(grad_output) + + +class _GatherFromSequenceParallelRegion(torch.autograd.Function): + """Gather the input from sequence parallel region and concatenate.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_, to_model_parallel: bool = True): + return _gather_along_first_dim(input_) + + @staticmethod + def forward(ctx, input_, to_model_parallel: bool = True): + ctx.to_model_parallel = to_model_parallel + return _gather_along_first_dim(input_) + + @staticmethod + def backward(ctx, grad_output): + if ctx.to_model_parallel: + return _reduce_scatter_along_first_dim(grad_output), None + else: + return _split_along_first_dim(grad_output), None + + +class _ReduceScatterToSequenceParallelRegion(torch.autograd.Function): + """Reduce scatter the input from the sequence parallel region and concatenate.""" + + # FIXME(mkozuki): Definition of static symbolic methods don't look correct according to + # https://pytorch.org/docs/stable/onnx.html#static-symbolic-method + @staticmethod + def symbolic(graph, input_): + return _reduce_scatter_along_first_dim(input_) + + @staticmethod + def forward(ctx, input_): + return _reduce_scatter_along_first_dim(input_) + + @staticmethod + def backward(ctx, grad_output): + return _gather_along_first_dim(grad_output) + + +# ----------------- +# Helper functions. +# ----------------- + + +def copy_to_tensor_model_parallel_region(input_: torch.Tensor) -> torch.Tensor: + return _CopyToModelParallelRegion.apply(input_) + + +def reduce_from_tensor_model_parallel_region(input_: torch.Tensor) -> torch.Tensor: + return _ReduceFromModelParallelRegion.apply(input_) + + +def scatter_to_tensor_model_parallel_region(input_: torch.Tensor) -> torch.Tensor: + return _ScatterToModelParallelRegion.apply(input_) + + +def gather_from_tensor_model_parallel_region(input_: torch.Tensor) -> torch.Tensor: + return _GatherFromModelParallelRegion.apply(input_) + + +def scatter_to_sequence_parallel_region(input_: torch.Tensor) -> torch.Tensor: + return _ScatterToSequenceParallelRegion.apply(input_) + + +def gather_from_sequence_parallel_region(input_: torch.Tensor, to_model_parallel: bool = True) -> torch.Tensor: + return _GatherFromSequenceParallelRegion.apply(input_, to_model_parallel) + + +def reduce_scatter_to_sequence_parallel_region(input_: torch.Tensor) -> torch.Tensor: + return _ReduceScatterToSequenceParallelRegion.apply(input_) + + +__all__ = [ + "copy_to_tensor_model_parallel_region", + "reduce_from_tensor_model_parallel_region", + "scatter_to_tensor_model_parallel_region", + "gather_from_tensor_model_parallel_region", + "scatter_to_sequence_parallel_region", + "gather_from_sequence_parallel_region", + "reduce_scatter_to_sequence_parallel_region", +] diff --git a/apex/apex/transformer/tensor_parallel/memory.py b/apex/apex/transformer/tensor_parallel/memory.py new file mode 100644 index 00000000..6df9a139 --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/memory.py @@ -0,0 +1,151 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO(mkozuki): Remove this file as Megatron-LM seems to have done so. +import torch + + +# A dictionary of all the memory buffers allocated. +_MEM_BUFFS = dict() + + +def allocate_mem_buff(name, numel, dtype, track_usage): + """Allocate a memory buffer.""" + assert name not in _MEM_BUFFS, "memory buffer {} already allocated.".format(name) + _MEM_BUFFS[name] = MemoryBuffer(name, numel, dtype, track_usage) + return _MEM_BUFFS[name] + + +def get_mem_buff(name): + """Get the memory buffer.""" + return _MEM_BUFFS[name] + + +class MemoryBuffer: + """Contiguous memory buffer. + Allocate a contiguous memory of type `dtype` and size `numel`. It is + used to reduce memory fragmentation. + + Usage: After the allocation, the `_start` index is set tot the first + index of the memory. A memory chunk starting from `_start` index + can be `allocated` for an input tensor, with the elements of the + tensor being coppied. The buffer can be reused by resetting the + `_start` index. + + """ + + def __init__(self, name, numel, dtype, track_usage): + if torch.distributed.get_rank() == 0: + element_size = torch.tensor([], dtype=dtype).element_size() + print( + "> building the {} memory buffer with {} num elements " + "and {} dtype ({:.1f} MB)...".format( + name, numel, dtype, numel * element_size / 1024 / 1024 + ), + flush=True, + ) + self.name = name + self.numel = numel + self.dtype = dtype + self.data = torch.empty( + self.numel, + dtype=self.dtype, + device=torch.cuda.current_device(), + requires_grad=False, + ) + + # Index tracking the start of the free memory. + self._start = 0 + + # Values used for tracking usage. + self.track_usage = track_usage + if self.track_usage: + self.in_use_value = 0.0 + self.total_value = 0.0 + + def reset(self): + """Reset the buffer start index to the beginning of the buffer.""" + self._start = 0 + + def is_in_use(self): + """Whether the current buffer hold on to any memory.""" + return self._start > 0 + + def numel_in_use(self): + """Return number of elements in use.""" + return self._start + + def add(self, tensor): + """Allocate a chunk of memory from the buffer to tensor and copy + the values.""" + assert ( + tensor.dtype == self.dtype + ), "Input tensor type {} different from buffer type {}".format( + tensor.dtype, self.dtype + ) + # Number of elements of the input tensor. + tensor_numel = torch.numel(tensor) + new_start = self._start + tensor_numel + assert ( + new_start <= self.numel + ), "Not enough memory left in the buffer ({} > {})".format( + tensor_numel, self.numel - self._start + ) + # New tensor is a view into the memory. + new_tensor = self.data[self._start : new_start] + self._start = new_start + new_tensor = new_tensor.view(tensor.shape) + new_tensor.copy_(tensor) + # Return a pointer to the new tensor. + return new_tensor + + def get_data(self): + """Return the data currently in use.""" + if self.track_usage: + self.in_use_value += float(self._start) + self.total_value += float(self.numel) + return self.data[: self._start] + + def print_average_usage(self): + """Print memory usage average over time. We would like this value + to be as high as possible.""" + assert self.track_usage, "You need to enable track usage." + if torch.distributed.get_rank() == 0: + print( + " > usage of {} memory buffer: {:.2f} %".format( + self.name, self.in_use_value * 100.0 / self.total_value + ), + flush=True, + ) + + +class RingMemBuffer: + """A ring of memory buffers.""" + + def __init__(self, name, num_buffers, numel, dtype, track_usage): + self.num_buffers = num_buffers + self.buffers = [ + allocate_mem_buff(name + " {}".format(i), numel, dtype, track_usage) + for i in range(num_buffers) + ] + self._index = -1 + + def get_next_buffer(self): + self._index += 1 + self._index = self._index % self.num_buffers + buff = self.buffers[self._index] + assert not buff.is_in_use(), "buffer is already in use." + return buff diff --git a/apex/apex/transformer/tensor_parallel/random.py b/apex/apex/transformer/tensor_parallel/random.py new file mode 100644 index 00000000..4bd64cb7 --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/random.py @@ -0,0 +1,311 @@ +# coding=utf-8 +# Copyright (c) 2021-22, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE(mkozuki): This file is based on megatron-lm/mpu/random.py with some differences: +# - Not using "viewless" tensor: +# - _kernel_make_viewless_tensor +# - MakeViewlessTensor +# - make_viewless_tensor +# - assert_viewless_tensor +# - safely_set_viewless_tensor_data + +# Parts of the code here are adapted from PyTorch +# repo: https://github.com/pytorch/pytorch +import contextlib + +import torch +from torch import _C +from torch.cuda import _lazy_call, device as device_ctx_manager +from torch.utils.checkpoint import detach_variable + +from apex.transformer.parallel_state import get_tensor_model_parallel_rank +from apex.transformer.tensor_parallel.memory import allocate_mem_buff +from apex.transformer.utils import split_tensor_into_1d_equal_chunks +from apex.transformer.utils import gather_split_1d_tensor + + +# Default name for the model parallel rng tracker. +_MODEL_PARALLEL_RNG_TRACKER_NAME = "model-parallel-rng" + +# TODO(mkozuki): Remove `_CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER` as megatron-lm doesn't seem to use. +# Whether apply model parallelism to checkpointed hidden states. +_CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER = None + + +# TODO(mkozuki): Remove `init_checkpointed_activations_memory_buffer` as megatron-lm doesn't seem to use. +def init_checkpointed_activations_memory_buffer( + micro_batch_size, + max_position_embeddings, + hidden_size, + num_layers, + tensor_model_parallel_size, + checkpoint_num_layers, + fp16, +): + """Initializ the memory buffer for the checkpointed activations.""" + + per_layer = ( + micro_batch_size + * max_position_embeddings + * hidden_size + // tensor_model_parallel_size + ) + assert ( + num_layers % checkpoint_num_layers == 0 + ), "number of layers is not divisible by checkpoint-num-layers" + num_checkpointer_layers = num_layers // checkpoint_num_layers + numel = per_layer * num_checkpointer_layers + dtype = torch.half + if not fp16: + dtype = torch.float + + global _CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER + assert ( + _CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER is None + ), "checkpointed activations memory buffer is already allocated." + _CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER = allocate_mem_buff( + "checkpointed activations", numel, dtype, track_usage=False + ) + + +# TODO(mkozuki): Remove `reset_checkpointed_activations_memory_buffer` as megatron-lm doesn't seem to use. +def reset_checkpointed_activations_memory_buffer(): + """Reset the memory used for checkpointing.""" + if _CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER is not None: + _CHECKPOINTED_ACTIVATIONS_MEMORY_BUFFER.reset() + + +def _set_cuda_rng_state(new_state, device=-1): + """Sets the random number generator state of the current GPU. + + Arguments: + new_state (torch.ByteTensor): The desired state + This function is adapted from PyTorch repo (torch.cuda.set_rng_state) + with a single change: the input state is not cloned. Cloning caused + major performance issues for +4 GPU cases. + """ + if hasattr(_C, "_cuda_setRNGState") and callable(_C._cuda_setRNGState): + # older PyTorch + def cb(): + with device_ctx_manager(device): + _C._cuda_setRNGState(new_state) + + else: + # newer PyTorch + if device == -1: + device = torch.device("cuda") + elif isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + + def cb(): + idx = device.index + if idx is None: + idx = torch.cuda.current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.set_state(new_state) + + _lazy_call(cb) + + +class CudaRNGStatesTracker: + """Tracker for the cuda RNG states. + + Using the `add` method, a cuda rng state is initialized based on + the input `seed` and is assigned to `name`. Later, by forking the + rng state, we can perform operations and return to our starting + cuda state. + """ + + def __init__(self): + # Map from a string name to the cuda rng state. + self.states_ = {} + # Seeds are just for book keeping and ensure no seed is set twice. + self.seeds_ = set() + + def reset(self): + """Set to the initial state (no tracker).""" + self.states_ = {} + self.seeds_ = set() + + def get_states(self): + """Get rng states. Copy the dictionary so we have direct + pointers to the states, not just a pointer to the dictionary.""" + states = {} + for name in self.states_: + states[name] = self.states_[name] + return states + + def set_states(self, states): + """Set the rng states. For efficiency purposes, we do not check + the size of seed for compatibility.""" + self.states_ = states + + def add(self, name, seed): + """Track the rng state.""" + # Check seed is not already used. + if seed in self.seeds_: + raise Exception("seed {} already exists".format(seed)) + self.seeds_.add(seed) + # Check that state is not already defined. + if name in self.states_: + raise Exception("cuda rng state {} already exists".format(name)) + # Get the current rng state. + orig_rng_state = torch.cuda.get_rng_state() + # Set the new state and store it. + torch.cuda.manual_seed(seed) + self.states_[name] = torch.cuda.get_rng_state() + # Reset rng state to what it was. + _set_cuda_rng_state(orig_rng_state) + + @contextlib.contextmanager + def fork(self, name=_MODEL_PARALLEL_RNG_TRACKER_NAME): + """Fork the cuda rng state, perform operations, and exit with + the original state.""" + # Check if we have added the state + if name not in self.states_: + raise Exception("cuda rng state {} is not added".format(name)) + # Store current rng state. + orig_cuda_rng_state = torch.cuda.get_rng_state() + # Set rng state to the desired one + _set_cuda_rng_state(self.states_[name]) + # Do the stuff we wanted to do. + try: + yield + finally: + # Update the current rng state for later use. + self.states_[name] = torch.cuda.get_rng_state() + # And set the state to the original state we started with. + _set_cuda_rng_state(orig_cuda_rng_state) + + +# RNG tracker object. +_CUDA_RNG_STATE_TRACKER = CudaRNGStatesTracker() + + +def get_cuda_rng_tracker(): + """Get cuda rng tracker.""" + return _CUDA_RNG_STATE_TRACKER + + +def model_parallel_cuda_manual_seed(seed): + """Initialize model parallel cuda seed. + + This function should be called after the model parallel is + initialized. Also, no torch.cuda.manual_seed should be called + after this function. Basically, this is replacement for that + function. + Two set of RNG states are tracked: + default state: This is for data parallelism and is the same among a + set of model parallel GPUs but different across + different model paralle groups. This is used for + example for dropout in the non-tensor-model-parallel regions. + tensor-model-parallel state: This state is different among a set of model + parallel GPUs, but the same across data parallel + groups. This is used for example for dropout in + model parallel regions. + """ + # 2718 is just for fun and any POSITIVE value will work. + offset = seed + 2718 + tensor_model_parallel_seed = offset + get_tensor_model_parallel_rank() + # Data parallel gets the original seed. + data_parallel_seed = seed + + _CUDA_RNG_STATE_TRACKER.reset() + # Set the default state. + torch.cuda.manual_seed(data_parallel_seed) + # and model parallel state. + _CUDA_RNG_STATE_TRACKER.add( + _MODEL_PARALLEL_RNG_TRACKER_NAME, tensor_model_parallel_seed + ) + + +# TODO (mkozuki): Move the below gradient checkpoint related features to another (new) file. +class CheckpointFunction(torch.autograd.Function): + """This function is adapted from torch.utils.checkpoint with + two main changes: + 1) torch.cuda.set_rng_state is replaced with `_set_cuda_rng_state` + 2) the states in the model parallel tracker are also properly + tracked/set/reset. + """ + + @staticmethod + def forward(ctx, run_function, distribute_saved_activations, *args): + ctx.run_function = run_function + ctx.distribute_saved_activations = distribute_saved_activations + + # Copy the rng states. + ctx.fwd_cpu_rng_state = torch.get_rng_state() + ctx.fwd_cuda_rng_state = torch.cuda.get_rng_state() + ctx.fwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states() + + with torch.no_grad(): + outputs = run_function(*args) + + # Divide hidden states across model parallel group and only keep + # the chunk corresponding to the current rank. + if ctx.distribute_saved_activations: + ctx.input_0_shape = args[0].shape + + # Store everything. + ctx.save_for_backward(*args) + return outputs + + @staticmethod + def backward(ctx, *args): + if not torch.autograd._is_checkpoint_valid(): + raise RuntimeError( + "Checkpointing is not compatible with .grad(), " + "please use .backward() if possible" + ) + inputs = ctx.saved_tensors + + # Store the current states. + bwd_cpu_rng_state = torch.get_rng_state() + bwd_cuda_rng_state = torch.cuda.get_rng_state() + bwd_cuda_rng_state_tracker = get_cuda_rng_tracker().get_states() + + # Set the states to what it used to be before the forward pass. + torch.set_rng_state(ctx.fwd_cpu_rng_state) + _set_cuda_rng_state(ctx.fwd_cuda_rng_state) + get_cuda_rng_tracker().set_states(ctx.fwd_cuda_rng_state_tracker) + + # Compute the forward pass. + detached_inputs = detach_variable(inputs) + with torch.enable_grad(): + outputs = ctx.run_function(*detached_inputs) + + # Set the states back to what it was at the start of this function. + torch.set_rng_state(bwd_cpu_rng_state) + _set_cuda_rng_state(bwd_cuda_rng_state) + get_cuda_rng_tracker().set_states(bwd_cuda_rng_state_tracker) + + if isinstance(outputs, torch.Tensor): + outputs = (outputs,) + torch.autograd.backward(outputs, args) + grads = tuple( + inp.grad if isinstance(inp, torch.Tensor) else inp + for inp in detached_inputs + ) + return (None, None) + grads + + +# NOTE(mkozuki): It doesn't look like `distribute_saved_activations` is used in apex.transformer +# but I added this change to reduce the superficial difference from Megatron-LM. +def checkpoint(function, distribute_saved_activations, *args): + """Checkpoint a model or part of the model. + This has been directly copied from torch.utils.checkpoint.""" + return CheckpointFunction.apply(function, distribute_saved_activations, *args) diff --git a/apex/apex/transformer/tensor_parallel/utils.py b/apex/apex/transformer/tensor_parallel/utils.py new file mode 100644 index 00000000..85ea413e --- /dev/null +++ b/apex/apex/transformer/tensor_parallel/utils.py @@ -0,0 +1,64 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Sequence + +import torch + +from apex.transformer.utils import divide + + +def split_tensor_along_last_dim( + tensor: torch.Tensor, + num_partitions: int, + contiguous_split_chunks: bool = False, +) -> List[torch.Tensor]: + """Split a tensor along its last dimension. + Arguments: + tensor: input tensor. + num_partitions: number of partitions to split the tensor + contiguous_split_chunks: If True, make each chunk contiguous + in memory. + """ + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = divide(tensor.size()[last_dim], num_partitions) + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # Note: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + +class VocabUtility: + """Split the vocabulary into `world_size` chunks and return the + first and last index of the vocabulary belonging to the `rank` + partition: Note that indices in [fist, last)""" + + @staticmethod + def vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size: int, rank, world_size: int + ) -> Sequence[int]: + index_f = rank * per_partition_vocab_size + index_l = index_f + per_partition_vocab_size + return index_f, index_l + + @staticmethod + def vocab_range_from_global_vocab_size(global_vocab_size: int, rank: int, world_size: int) -> Sequence[int]: + per_partition_vocab_size = divide(global_vocab_size, world_size) + return VocabUtility.vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size, rank, world_size + ) diff --git a/apex/apex/transformer/testing/__init__.py b/apex/apex/transformer/testing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/apex/transformer/testing/arguments.py b/apex/apex/transformer/testing/arguments.py new file mode 100644 index 00000000..5c74054c --- /dev/null +++ b/apex/apex/transformer/testing/arguments.py @@ -0,0 +1,977 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Megatron arguments.""" + +import argparse +import os + +import torch + +def parse_args(extra_args_provider=None, defaults={}, override_args={}, + ignore_unknown_args=False): + """Parse all arguments.""" + parser = argparse.ArgumentParser(description='Megatron-LM Arguments', + allow_abbrev=False) + + # Standard arguments. + parser = _add_network_size_args(parser) + parser = _add_regularization_args(parser) + parser = _add_training_args(parser) + parser = _add_initialization_args(parser) + parser = _add_learning_rate_args(parser) + parser = _add_checkpointing_args(parser) + parser = _add_mixed_precision_args(parser) + parser = _add_distributed_args(parser) + parser = _add_validation_args(parser) + parser = _add_data_args(parser) + parser = _add_autoresume_args(parser) + parser = _add_biencoder_args(parser) + parser = _add_vision_args(parser) + parser = _add_logging_args(parser) + + # NOTE(mkozuki): This option is added to investigate the potential of `torch.autograd.graph.save_on_cpu()`. + # ref: https://pytorch.org/docs/stable/autograd.html#torch.autograd.graph.save_on_cpu. + parser.add_argument('--cpu-offload', action='store_true', default=False, help='Turns on CPU offloading') + + # Custom arguments. + if extra_args_provider is not None: + parser = extra_args_provider(parser) + + # Parse. + if ignore_unknown_args: + args, _ = parser.parse_known_args() + else: + args = parser.parse_args() + + # Distributed args. + args.rank = int(os.getenv('RANK', '0')) + args.world_size = int(os.getenv("WORLD_SIZE", '1')) + + for key in override_args: + setattr(args, key, override_args[key]) + + # Tensor model parallel size. + args.tensor_model_parallel_size = min( + args.tensor_model_parallel_size, args.world_size) + assert args.world_size % args.tensor_model_parallel_size == 0, 'world size'\ + ' ({}) is not divisible by tensor model parallel size ({})'.format( + args.world_size, args.tensor_model_parallel_size) + + # Pipeline model parallel size. + args.pipeline_model_parallel_size = min( + args.pipeline_model_parallel_size, + (args.world_size // args.tensor_model_parallel_size)) + + args.transformer_pipeline_model_parallel_size = ( + args.pipeline_model_parallel_size - 1 + if args.standalone_embedding_stage else + args.pipeline_model_parallel_size + ) + # Checks. + model_parallel_size = args.pipeline_model_parallel_size * \ + args.tensor_model_parallel_size + assert args.world_size % model_parallel_size == 0, 'world size is not'\ + ' divisible by tensor parallel size ({}) times pipeline parallel ' \ + 'size ({})'.format(args.world_size, args.tensor_model_parallel_size, + args.pipeline_model_parallel_size) + args.data_parallel_size = args.world_size // model_parallel_size + if args.rank == 0: + print('using world size: {}, data-parallel-size: {}, ' + 'tensor-model-parallel size: {}, ' + 'pipeline-model-parallel size: {} '.format( + args.world_size, args.data_parallel_size, + args.tensor_model_parallel_size, + args.pipeline_model_parallel_size), flush=True) + if args.pipeline_model_parallel_size > 1: + if args.pipeline_model_parallel_split_rank is not None: + assert args.pipeline_model_parallel_split_rank < \ + args.pipeline_model_parallel_size, 'split rank needs'\ + ' to be less than pipeline model parallel size ({})'.format( + args.pipeline_model_parallel_size) + + # Deprecated arguments + assert args.batch_size is None, '--batch-size argument is no longer ' \ + 'valid, use --micro-batch-size instead' + del args.batch_size + assert args.warmup is None, '--warmup argument is no longer valid, use ' \ + '--lr-warmup-fraction instead' + del args.warmup + assert args.model_parallel_size is None, '--model-parallel-size is no ' \ + 'longer valid, use --tensor-model-parallel-size instead' + del args.model_parallel_size + if args.checkpoint_activations: + args.recompute_granularity = 'full' + args.recompute_method = 'uniform' + if args.rank == 0: + print('--checkpoint-activations is no longer valid, ' + 'use --recompute-granularity and --recompute-method instead. ' + 'Defaulting to recompute-granularity=full and recompute-method=uniform.') + del args.checkpoint_activations + + if args.recompute_activations: + args.recompute_granularity = 'selective' + del args.recompute_activations + + # Set input defaults. + for key in defaults: + # For default to be valid, it should not be provided in the + # arguments that are passed to the program. We check this by + # ensuring the arg is set to None. + if getattr(args, key) is not None: + if args.rank == 0: + print('WARNING: overriding default arguments for {key}:{v} \ + with {key}:{v2}'.format(key=key, v=defaults[key], + v2=getattr(args, key)), + flush=True) + else: + setattr(args, key, defaults[key]) + + # Batch size. + assert args.micro_batch_size is not None + assert args.micro_batch_size > 0 + if args.global_batch_size is None: + args.global_batch_size = args.micro_batch_size * args.data_parallel_size + if args.rank == 0: + print('setting global batch size to {}'.format( + args.global_batch_size), flush=True) + assert args.global_batch_size > 0 + if args.num_layers_per_virtual_pipeline_stage is not None: + assert args.pipeline_model_parallel_size > 2, \ + 'pipeline-model-parallel size should be greater than 2 with ' \ + 'interleaved schedule' + assert args.num_layers % args.num_layers_per_virtual_pipeline_stage == 0, \ + 'number of layers is not divisible by number of layers per virtual ' \ + 'pipeline stage' + args.virtual_pipeline_model_parallel_size = \ + (args.num_layers // args.pipeline_model_parallel_size) // \ + args.num_layers_per_virtual_pipeline_stage + else: + args.virtual_pipeline_model_parallel_size = None + + # Parameters dtype. + args.params_dtype = torch.float + if args.fp16: + assert not args.bf16 + args.params_dtype = torch.half + if args.bf16: + assert not args.fp16 + args.params_dtype = torch.bfloat16 + # bfloat16 requires gradient accumulation and all-reduce to + # be done in fp32. + if not args.accumulate_allreduce_grads_in_fp32: + args.accumulate_allreduce_grads_in_fp32 = True + if args.rank == 0: + print('accumulate and all-reduce gradients in fp32 for ' + 'bfloat16 data type.', flush=True) + + if args.rank == 0: + print('using {} for parameters ...'.format(args.params_dtype), + flush=True) + + # If we do accumulation and all-reduces in fp32, we need to have local DDP + # and we should make sure use-contiguous-buffers-in-local-ddp is not off. + if args.accumulate_allreduce_grads_in_fp32: + assert args.DDP_impl == 'local' + assert args.use_contiguous_buffers_in_local_ddp + else: + if args.gradient_accumulation_fusion: + args.gradient_accumulation_fusion = False + if args.rank == 0: + print('Gradient accumulation fusion to linear layer weight ' + 'gradient computation is supported only with fp32 ' + 'gradient accumulation. Setting gradient_accumulation_fusion ' + 'to False', flush=True) + + # For torch DDP, we do not use contiguous buffer + if args.DDP_impl == 'torch': + args.use_contiguous_buffers_in_local_ddp = False + + if args.dataloader_type is None: + args.dataloader_type = 'single' + + # Consumed tokens. + args.consumed_train_samples = 0 + args.consumed_valid_samples = 0 + + # Iteration-based training. + if args.train_iters: + # If we use iteration-based training, make sure the + # sample-based options are off. + assert args.train_samples is None, \ + 'expected iteration-based training' + assert args.lr_decay_samples is None, \ + 'expected iteration-based learning rate decay' + assert args.lr_warmup_samples == 0, \ + 'expected iteration-based learning rate warmup' + assert args.rampup_batch_size is None, \ + 'expected no batch-size rampup for iteration-based training' + if args.lr_warmup_fraction is not None: + assert args.lr_warmup_iters == 0, \ + 'can only specify one of lr-warmup-fraction and lr-warmup-iters' + + # Sample-based training. + if args.train_samples: + # If we use sample-based training, make sure the + # iteration-based options are off. + assert args.train_iters is None, \ + 'expected sample-based training' + assert args.lr_decay_iters is None, \ + 'expected sample-based learning rate decay' + assert args.lr_warmup_iters == 0, \ + 'expected sample-based learnig rate warmup' + if args.lr_warmup_fraction is not None: + assert args.lr_warmup_samples == 0, \ + 'can only specify one of lr-warmup-fraction ' \ + 'and lr-warmup-samples' + + # Check required arguments. + required_args = ['num_layers', 'hidden_size', 'num_attention_heads', + 'max_position_embeddings'] + for req_arg in required_args: + _check_arg_is_not_none(args, req_arg) + + # Checks. + if args.ffn_hidden_size is None: + args.ffn_hidden_size = 4 * args.hidden_size + + if args.kv_channels is None: + assert args.hidden_size % args.num_attention_heads == 0 + args.kv_channels = args.hidden_size // args.num_attention_heads + + if args.seq_length is not None: + assert args.encoder_seq_length is None + args.encoder_seq_length = args.seq_length + else: + assert args.encoder_seq_length is not None + args.seq_length = args.encoder_seq_length + + if args.seq_length is not None: + assert args.max_position_embeddings >= args.seq_length + if args.decoder_seq_length is not None: + assert args.max_position_embeddings >= args.decoder_seq_length + if args.lr is not None: + assert args.min_lr <= args.lr + if args.save is not None: + assert args.save_interval is not None + # Mixed precision checks. + if args.fp16_lm_cross_entropy: + assert args.fp16, 'lm cross entropy in fp16 only support in fp16 mode.' + if args.fp32_residual_connection: + assert args.fp16 or args.bf16, \ + 'residual connection in fp32 only supported when using fp16 or bf16.' + + if args.weight_decay_incr_style == 'constant': + assert args.start_weight_decay is None + assert args.end_weight_decay is None + args.start_weight_decay = args.weight_decay + args.end_weight_decay = args.weight_decay + else: + assert args.start_weight_decay is not None + assert args.end_weight_decay is not None + + TORCH_MAJOR = int(torch.__version__.split('.')[0]) + TORCH_MINOR = int(torch.__version__.split('.')[1]) + # Persistent fused layer norm. + if TORCH_MAJOR < 1 or (TORCH_MAJOR == 1 and TORCH_MINOR < 11): + args.no_persist_layer_norm = True + if args.rank == 0: + print('Persistent fused layer norm kernel is supported from ' + 'pytorch v1.11 (nvidia pytorch container paired with v1.11). ' + 'Defaulting to no_persist_layer_norm=True') + + # Activation recomputing. + if args.distribute_saved_activations: + assert args.tensor_model_parallel_size > 1, 'can distribute ' \ + 'recomputed activations only across tensor model ' \ + 'parallel groups' + assert args.recompute_granularity == 'full', \ + 'distributed recompute activations is only '\ + 'application to full recompute granularity' + assert args.recompute_method is not None, \ + 'for distributed recompute activations to work you '\ + 'need to use a recompute method ' + assert TORCH_MAJOR >= 1 and TORCH_MINOR >= 10, \ + 'distributed recompute activations are supported for pytorch ' \ + 'v1.10 and above (Nvidia Pytorch container >= 21.07). Current ' \ + 'pytorch version is v%s.%s.' % (TORCH_MAJOR, TORCH_MINOR) + + if args.recompute_granularity == 'selective': + assert args.recompute_method is None, \ + 'recompute method is not yet supported for ' \ + 'selective recomputing granularity' + + # disable async_tensor_model_parallel_allreduce when + # model parallel memory optimization is enabled + if args.sequence_parallel: + args.async_tensor_model_parallel_allreduce = False + + _print_args(args) + return args + + +def _print_args(args): + """Print arguments.""" + if args.rank == 0: + print('------------------------ arguments ------------------------', + flush=True) + str_list = [] + for arg in vars(args): + dots = '.' * (48 - len(arg)) + str_list.append(' {} {} {}'.format(arg, dots, getattr(args, arg))) + for arg in sorted(str_list, key=lambda x: x.lower()): + print(arg, flush=True) + print('-------------------- end of arguments ---------------------', + flush=True) + + +def _check_arg_is_not_none(args, arg): + assert getattr(args, arg) is not None, '{} argument is None'.format(arg) + + +def _add_inference_args(parser): + group = parser.add_argument_group(title='inference') + + group.add_argument('--inference-batch-times-seqlen-threshold', + type=int, default=512, + help='During inference, if batch-size times ' + 'sequence-length is smaller than this threshold ' + 'then we will not use pipelining, otherwise we will.') + + return parser + + +def _add_network_size_args(parser): + group = parser.add_argument_group(title='network size') + + group.add_argument('--num-layers', type=int, default=None, + help='Number of transformer layers.') + group.add_argument('--hidden-size', type=int, default=None, + help='Tansformer hidden size.') + group.add_argument('--ffn-hidden-size', type=int, default=None, + help='Transformer Feed-Forward Network hidden size. ' + 'This is set to 4*hidden-size if not provided') + group.add_argument('--num-attention-heads', type=int, default=None, + help='Number of transformer attention heads.') + group.add_argument('--kv-channels', type=int, default=None, + help='Projection weights dimension in multi-head ' + 'attention. This is set to ' + ' args.hidden_size // args.num_attention_heads ' + 'if not provided.') + group.add_argument('--max-position-embeddings', type=int, default=None, + help='Maximum number of position embeddings to use. ' + 'This is the size of position embedding.') + group.add_argument('--make-vocab-size-divisible-by', type=int, default=128, + help='Pad the vocab size to be divisible by this value.' + 'This is added for computational efficieny reasons.') + group.add_argument('--layernorm-epsilon', type=float, default=1e-5, + help='Layer norm epsilon.') + group.add_argument('--apply-residual-connection-post-layernorm', + action='store_true', + help='If set, use original BERT residula connection ' + 'ordering.') + group.add_argument('--openai-gelu', action='store_true', + help='Use OpenAIs GeLU implementation. This option' + 'should not be used unless for backward compatibility' + 'reasons.') + group.add_argument('--onnx-safe', type=bool, required=False, + help='Use workarounds for known problems with ' + 'Torch ONNX exporter') + group.add_argument('--bert-no-binary-head', action='store_false', + help='Disable BERT binary head.', + dest='bert_binary_head') + group.add_argument('--num-experts', type=int, default=None, + help='Number of Experts in Switch Transformer (None means no Switch)') + + return parser + + +def _add_logging_args(parser): + group = parser.add_argument_group(title='logging') + + group.add_argument('--log-params-norm', action='store_true', + help='If set, calculate and log parameters norm.') + group.add_argument('--log-num-zeros-in-grad', action='store_true', + help='If set, calculate and log the number of zeros in gradient.') + group.add_argument('--tensorboard-log-interval', type=int, default=1, + help='Report to tensorboard interval.') + group.add_argument('--tensorboard-queue-size', type=int, default=1000, + help='Size of the tensorboard queue for pending events ' + 'and summaries before one of the ‘add’ calls forces a ' + 'flush to disk.') + group.add_argument('--log-timers-to-tensorboard', action='store_true', + help='If set, write timers to tensorboard.') + group.add_argument('--log-batch-size-to-tensorboard', action='store_true', + help='If set, write batch-size to tensorboard.') + group.add_argument('--no-log-learnig-rate-to-tensorboard', + action='store_false', + help='Disable learning rate logging to tensorboard.', + dest='log_learning_rate_to_tensorboard') + group.add_argument('--no-log-loss-scale-to-tensorboard', + action='store_false', + help='Disable loss-scale logging to tensorboard.', + dest='log_loss_scale_to_tensorboard') + group.add_argument('--log-validation-ppl-to-tensorboard', + action='store_true', + help='If set, write validation perplexity to ' + 'tensorboard.') + group.add_argument('--log-memory-to-tensorboard', + action='store_true', + help='Enable memory logging to tensorboard.') + group.add_argument('--log-world-size-to-tensorboard', + action='store_true', + help='Enable world size logging to tensorboard.') + + return parser + + +def _add_regularization_args(parser): + group = parser.add_argument_group(title='regularization') + + group.add_argument('--attention-dropout', type=float, default=0.1, + help='Post attention dropout probability.') + group.add_argument('--hidden-dropout', type=float, default=0.1, + help='Dropout probability for hidden state transformer.') + group.add_argument('--weight-decay', type=float, default=0.01, + help='Weight decay coefficient for L2 regularization.') + group.add_argument('--start-weight-decay', type=float, + help='Initial weight decay coefficient for L2 regularization.') + group.add_argument('--end-weight-decay', type=float, + help='End of run weight decay coefficient for L2 regularization.') + group.add_argument('--weight-decay-incr-style', type=str, default='constant', + choices=['constant', 'linear', 'cosine'], + help='Weight decay increment function.') + group.add_argument('--clip-grad', type=float, default=1.0, + help='Gradient clipping based on global L2 norm.') + group.add_argument('--adam-beta1', type=float, default=0.9, + help='First coefficient for computing running averages ' + 'of gradient and its square') + group.add_argument('--adam-beta2', type=float, default=0.999, + help='Second coefficient for computing running averages ' + 'of gradient and its square') + group.add_argument('--adam-eps', type=float, default=1e-08, + help='Term added to the denominator to improve' + 'numerical stability') + group.add_argument('--sgd-momentum', type=float, default=0.9, + help='Momentum factor for sgd') + + return parser + + +def _add_training_args(parser): + group = parser.add_argument_group(title='training') + + group.add_argument('--micro-batch-size', type=int, default=None, + help='Batch size per model instance (local batch size). ' + 'Global batch size is local batch size times data ' + 'parallel size times number of micro batches.') + group.add_argument('--batch-size', type=int, default=None, + help='Old batch size parameter, do not use. ' + 'Use --micro-batch-size instead') + group.add_argument('--global-batch-size', type=int, default=None, + help='Training batch size. If set, it should be a ' + 'multiple of micro-batch-size times data-parallel-size. ' + 'If this value is None, then ' + 'use micro-batch-size * data-parallel-size as the ' + 'global batch size. This choice will result in 1 for ' + 'number of micro-batches.') + group.add_argument('--rampup-batch-size', nargs='*', default=None, + help='Batch size ramp up with the following values:' + ' --rampup-batch-size ' + ' ' + ' ' + 'For example:' + ' --rampup-batch-size 16 8 300000 \ ' + ' --global-batch-size 1024' + 'will start with global batch size 16 and over ' + ' (1024 - 16) / 8 = 126 intervals will increase' + 'the batch size linearly to 1024. In each interval' + 'we will use approximately 300000 / 126 = 2380 samples.') + group.add_argument('--recompute-activations', action='store_true', + help='recompute activation to allow for training ' + 'with larger models, sequences, and batch sizes.') + group.add_argument('--recompute-granularity', type=str, default=None, + choices=['full', 'selective'], + help='Checkpoint activations to allow for training ' + 'with larger models, sequences, and batch sizes. ' + 'It is supported at two granularities 1) full: ' + 'whole transformer layer is recomputed, ' + '2) selective: core attention part of the transformer ' + 'layer is recomputed.') + group.add_argument('--distribute-saved-activations', + action='store_true', + help='If set, distribute recomputed activations ' + 'across model parallel group.') + group.add_argument('--recompute-method', type=str, default=None, + choices=['uniform', 'block'], + help='1) uniform: uniformly divide the total number of ' + 'Transformer layers and recompute the input activation of ' + 'each divided chunk at specified granularity, ' + '2) recompute the input activations of only a set number of ' + 'individual Transformer layers per pipeline stage and do the ' + 'rest without any recomputing at specified granularity' + 'default) do not apply activations recompute to any layers') + group.add_argument('--recompute-num-layers', type=int, default=1, + help='1) uniform: the number of Transformer layers in each ' + 'uniformly divided recompute unit, ' + '2) block: the number of individual Transformer layers ' + 'to recompute within each pipeline stage.') + + # deprecated + group.add_argument('--checkpoint-activations', action='store_true', + help='Checkpoint activation to allow for training ' + 'with larger models, sequences, and batch sizes.') + group.add_argument('--train-iters', type=int, default=None, + help='Total number of iterations to train over all ' + 'training runs. Note that either train-iters or ' + 'train-samples should be provided.') + group.add_argument('--train-samples', type=int, default=None, + help='Total number of samples to train over all ' + 'training runs. Note that either train-iters or ' + 'train-samples should be provided.') + group.add_argument('--log-interval', type=int, default=100, + help='Report loss and timing interval.') + group.add_argument('--exit-interval', type=int, default=None, + help='Exit the program after the iteration is divisible ' + 'by this value.') + group.add_argument('--exit-duration-in-mins', type=int, default=None, + help='Exit the program after this many minutes.') + group.add_argument('--tensorboard-dir', type=str, default=None, + help='Write TensorBoard logs to this directory.') + group.add_argument('--no-masked-softmax-fusion', + action='store_false', + help='Disable fusion of query_key_value scaling, ' + 'masking, and softmax.', + dest='masked_softmax_fusion') + group.add_argument('--no-bias-gelu-fusion', action='store_false', + help='Disable bias and gelu fusion.', + dest='bias_gelu_fusion') + group.add_argument('--no-bias-dropout-fusion', action='store_false', + help='Disable bias and dropout fusion.', + dest='bias_dropout_fusion') + group.add_argument('--optimizer', type=str, default='adam', + choices=['adam', 'sgd'], + help='Optimizer function') + group.add_argument('--dataloader-type', type=str, default=None, + choices=['single', 'cyclic'], + help='Single pass vs multiple pass data loader') + group.add_argument('--no-async-tensor-model-parallel-allreduce', + action='store_true', + help='Disable asynchronous execution of ' + 'tensor-model-parallel all-reduce with weight ' + 'gradient compuation of a column-linear layer.', + dest='async_tensor_model_parallel_allreduce') + group.add_argument('--no-persist-layer-norm', action='store_true', + help='Disable using persistent fused layer norm kernel. ' + 'This kernel supports only a set of hidden sizes. Please ' + 'check persist_ln_hidden_sizes if your hidden ' + 'size is supported.') + group.add_argument('--sequence-parallel', action='store_true', + help='Enable sequence parallel optimization.') + group.add_argument('--no-gradient-accumulation-fusion', + action='store_false', + help='Disable fusing gradient accumulation to weight ' + 'gradient computation of linear layers', + dest='gradient_accumulation_fusion') + return parser + + +def _add_initialization_args(parser): + group = parser.add_argument_group(title='initialization') + + group.add_argument('--seed', type=int, default=1234, + help='Random seed used for python, numpy, ' + 'pytorch, and cuda.') + group.add_argument('--init-method-std', type=float, default=0.02, + help='Standard deviation of the zero mean normal ' + 'distribution used for weight initialization.') + group.add_argument('--init-method-xavier-uniform', action='store_true', + help='Enable Xavier uniform parameter initialization') + + return parser + + +def _add_learning_rate_args(parser): + group = parser.add_argument_group(title='learning rate') + + group.add_argument('--lr', type=float, default=None, + help='Initial learning rate. Depending on decay style ' + 'and initial warmup, the learing rate at each ' + 'iteration would be different.') + group.add_argument('--lr-decay-style', type=str, default='linear', + choices=['constant', 'linear', 'cosine'], + help='Learning rate decay function.') + group.add_argument('--lr-decay-iters', type=int, default=None, + help='number of iterations to decay learning rate over,' + ' If None defaults to `--train-iters`') + group.add_argument('--lr-decay-samples', type=int, default=None, + help='number of samples to decay learning rate over,' + ' If None defaults to `--train-samples`') + group.add_argument('--lr-warmup-fraction', type=float, default=None, + help='fraction of lr-warmup-(iters/samples) to use ' + 'for warmup (as a float)') + group.add_argument('--lr-warmup-iters', type=int, default=0, + help='number of iterations to linearly warmup ' + 'learning rate over.') + group.add_argument('--lr-warmup-samples', type=int, default=0, + help='number of samples to linearly warmup ' + 'learning rate over.') + group.add_argument('--warmup', type=int, default=None, + help='Old lr warmup argument, do not use. Use one of the' + '--lr-warmup-* arguments above') + group.add_argument('--min-lr', type=float, default=0.0, + help='Minumum value for learning rate. The scheduler' + 'clip values below this threshold.') + group.add_argument('--override-lr-scheduler', action='store_true', + help='Reset the values of the scheduler (learning rate,' + 'warmup iterations, minimum learning rate, maximum ' + 'number of iterations, and decay style from input ' + 'arguments and ignore values from checkpoints. Note' + 'that all the above values will be reset.') + group.add_argument('--use-checkpoint-lr-scheduler', action='store_true', + help='Use checkpoint to set the values of the scheduler ' + '(learning rate, warmup iterations, minimum learning ' + 'rate, maximum number of iterations, and decay style ' + 'from checkpoint and ignore input arguments.') + + return parser + + +def _add_checkpointing_args(parser): + group = parser.add_argument_group(title='checkpointing') + + group.add_argument('--save', type=str, default=None, + help='Output directory to save checkpoints to.') + group.add_argument('--save-interval', type=int, default=None, + help='Number of iterations between checkpoint saves.') + group.add_argument('--no-save-optim', action='store_true', default=None, + help='Do not save current optimizer.') + group.add_argument('--no-save-rng', action='store_true', default=None, + help='Do not save current rng state.') + group.add_argument('--load', type=str, default=None, + help='Directory containing a model checkpoint.') + group.add_argument('--no-load-optim', action='store_true', default=None, + help='Do not load optimizer when loading checkpoint.') + group.add_argument('--no-load-rng', action='store_true', default=None, + help='Do not load rng state when loading checkpoint.') + group.add_argument('--finetune', action='store_true', + help='Load model for finetuning. Do not load optimizer ' + 'or rng state from checkpoint and set iteration to 0. ' + 'Assumed when loading a release checkpoint.') + + return parser + + +def _add_mixed_precision_args(parser): + group = parser.add_argument_group(title='mixed precision') + + group.add_argument('--fp16', action='store_true', + help='Run model in fp16 mode.') + group.add_argument('--bf16', action='store_true', + help='Run model in bfloat16 mode.') + group.add_argument('--loss-scale', type=float, default=None, + help='Static loss scaling, positive power of 2 ' + 'values can improve fp16 convergence. If None, dynamic' + 'loss scaling is used.') + group.add_argument('--initial-loss-scale', type=float, default=2**32, + help='Initial loss-scale for dynamic loss scaling.') + group.add_argument('--min-loss-scale', type=float, default=1.0, + help='Minimum loss scale for dynamic loss scale.') + group.add_argument('--loss-scale-window', type=float, default=1000, + help='Window over which to raise/lower dynamic scale.') + group.add_argument('--hysteresis', type=int, default=2, + help='hysteresis for dynamic loss scaling') + group.add_argument('--fp32-residual-connection', action='store_true', + help='Move residual connections to fp32.') + group.add_argument('--no-query-key-layer-scaling', action='store_false', + help='Do not scale Q * K^T by 1 / layer-number.', + dest='apply_query_key_layer_scaling') + group.add_argument('--attention-softmax-in-fp32', action='store_true', + help='Run attention masking and softmax in fp32. ' + 'This flag is ignored unless ' + '--no-query-key-layer-scaling is specified.') + group.add_argument('--accumulate-allreduce-grads-in-fp32', + action='store_true', + help='Gradient accumulation and all-reduce in fp32.') + group.add_argument('--fp16-lm-cross-entropy', action='store_true', + help='Move the cross entropy unreduced loss calculation' + 'for lm head to fp16.') + + return parser + + +def _add_distributed_args(parser): + group = parser.add_argument_group(title='distributed') + + group.add_argument('--tensor-model-parallel-size', type=int, default=1, + help='Degree of tensor model parallelism.') + group.add_argument('--pipeline-model-parallel-size', type=int, default=1, + help='Degree of pipeline model parallelism.') + group.add_argument('--pipeline-model-parallel-split-rank', + type=int, default=None, + help='Rank where encoder and decoder should be split.') + group.add_argument('--model-parallel-size', type=int, default=None, + help='Old model parallel argument, do not use. Use ' + '--tensor-model-parallel-size instead.') + group.add_argument('--num-layers-per-virtual-pipeline-stage', type=int, default=None, + help='Number of layers per virtual pipeline stage') + group.add_argument('--distributed-backend', default='nccl', + choices=['nccl', 'gloo'], + help='Which backend to use for distributed training.') + group.add_argument('--DDP-impl', default='local', + choices=['local', 'torch'], + help='which DistributedDataParallel implementation ' + 'to use.') + group.add_argument('--no-contiguous-buffers-in-local-ddp', + action='store_false', help='If set, dont use ' + 'contiguous buffer in local DDP.', + dest='use_contiguous_buffers_in_local_ddp') + group.add_argument('--no-scatter-gather-tensors-in-pipeline', action='store_false', + help='Use scatter/gather to optimize communication of tensors in pipeline', + dest='scatter_gather_tensors_in_pipeline') + group.add_argument('--local_rank', type=int, default=None, + help='local rank passed from distributed launcher.') + group.add_argument('--lazy-mpu-init', type=bool, required=False, + help='If set to True, initialize_megatron() ' + 'skips DDP initialization and returns function to ' + 'complete it instead.Also turns on ' + '--use-cpu-initialization flag. This is for ' + 'external DDP manager.' ) + group.add_argument('--use-cpu-initialization', action='store_true', + default=None, help='If set, affine parallel weights ' + 'initialization uses CPU' ) + group.add_argument('--empty-unused-memory-level', default=0, type=int, + choices=[0, 1, 2], + help='Call torch.cuda.empty_cache() each iteration ' + '(training and eval), to reduce fragmentation.' + '0=off, 1=moderate, 2=aggressive.') + group.add_argument('--standalone-embedding-stage', action='store_true', + default=False, help='If set, *input* embedding layer ' + 'is placed on its own pipeline stage, without any ' + 'transformer layers. (For T5, this flag currently only ' + 'affects the encoder embedding.)') + return parser + + +def _add_validation_args(parser): + group = parser.add_argument_group(title='validation') + + group.add_argument('--eval-iters', type=int, default=100, + help='Number of iterations to run for evaluation' + 'validation/test for.') + group.add_argument('--eval-interval', type=int, default=1000, + help='Interval between running evaluation on ' + 'validation set.') + + return parser + + +def _add_data_args(parser): + group = parser.add_argument_group(title='data and dataloader') + + group.add_argument('--data-path', nargs='*', default=None, + help='Path to the training dataset. Accepted format:' + '1) a single data path, 2) multiple datasets in the' + 'form: dataset1-weight dataset1-path dataset2-weight ' + 'dataset2-path ...') + group.add_argument('--split', type=str, default='969, 30, 1', + help='Comma-separated list of proportions for training,' + ' validation, and test split. For example the split ' + '`90,5,5` will use 90%% of data for training, 5%% for ' + 'validation and 5%% for test.') + group.add_argument('--vocab-file', type=str, default=None, + help='Path to the vocab file.') + group.add_argument('--merge-file', type=str, default=None, + help='Path to the BPE merge file.') + group.add_argument('--vocab-extra-ids', type=int, default=0, + help='Number of additional vocabulary tokens. ' + 'They are used for span masking in the T5 model') + group.add_argument('--seq-length', type=int, default=None, + help='Maximum sequence length to process.') + group.add_argument('--encoder-seq-length', type=int, default=None, + help='Maximum encoder sequence length to process.' + 'This should be exclusive of --seq-length') + group.add_argument('--decoder-seq-length', type=int, default=None, + help="Maximum decoder sequence length to process.") + group.add_argument('--retriever-seq-length', type=int, default=256, + help='Maximum sequence length for the biencoder model ' + ' for retriever') + group.add_argument('--sample-rate', type=float, default=1.0, + help='sample rate for training data. Supposed to be 0 ' + ' < sample_rate < 1') + group.add_argument('--mask-prob', type=float, default=0.15, + help='Probability of replacing a token with mask.') + group.add_argument('--short-seq-prob', type=float, default=0.1, + help='Probability of producing a short sequence.') + group.add_argument('--mmap-warmup', action='store_true', + help='Warm up mmap files.') + group.add_argument('--num-workers', type=int, default=2, + help="Dataloader number of workers.") + group.add_argument('--tokenizer-type', type=str, + default=None, + choices=['BertWordPieceLowerCase', + 'BertWordPieceCase', + 'GPT2BPETokenizer'], + help='What type of tokenizer to use.') + group.add_argument('--data-impl', type=str, default='infer', + choices=['lazy', 'cached', 'mmap', 'infer'], + help='Implementation of indexed datasets.') + group.add_argument('--reset-position-ids', action='store_true', + help='Reset posistion ids after end-of-document token.') + group.add_argument('--reset-attention-mask', action='store_true', + help='Reset self attention maske after ' + 'end-of-document token.') + group.add_argument('--eod-mask-loss', action='store_true', + help='Mask loss for the end of document tokens.') + + return parser + + +def _add_autoresume_args(parser): + group = parser.add_argument_group(title='autoresume') + + group.add_argument('--adlr-autoresume', action='store_true', + help='Enable autoresume on adlr cluster.') + group.add_argument('--adlr-autoresume-interval', type=int, default=1000, + help='Intervals over which check for autoresume' + 'termination signal') + + return parser + + +def _add_biencoder_args(parser): + group = parser.add_argument_group(title='biencoder') + + # network size + group.add_argument('--ict-head-size', type=int, default=None, + help='Size of block embeddings to be used in ICT and ' + 'REALM (paper default: 128)') + group.add_argument('--biencoder-projection-dim', type=int, default=0, + help='Size of projection head used in biencoder (paper' + ' default: 128)') + group.add_argument('--biencoder-shared-query-context-model', action='store_true', + help='Whether to share the parameters of the query ' + 'and context models or not') + + # checkpointing + group.add_argument('--ict-load', type=str, default=None, + help='Directory containing an ICTBertModel checkpoint') + group.add_argument('--bert-load', type=str, default=None, + help='Directory containing an BertModel checkpoint ' + '(needed to start ICT and REALM)') + + # data + group.add_argument('--titles-data-path', type=str, default=None, + help='Path to titles dataset used for ICT') + group.add_argument('--query-in-block-prob', type=float, default=0.1, + help='Probability of keeping query in block for ' + 'ICT dataset') + group.add_argument('--use-one-sent-docs', action='store_true', + help='Whether to use one sentence documents in ICT') + group.add_argument('--evidence-data-path', type=str, default=None, + help='Path to Wikipedia Evidence frm DPR paper') + + # training + group.add_argument('--retriever-report-topk-accuracies', nargs='+', type=int, + default=[], help="Which top-k accuracies to report " + "(e.g. '1 5 20')") + group.add_argument('--retriever-score-scaling', action='store_true', + help='Whether to scale retriever scores by inverse ' + 'square root of hidden size') + + # faiss index + group.add_argument('--block-data-path', type=str, default=None, + help='Where to save/load BlockData to/from') + group.add_argument('--embedding-path', type=str, default=None, + help='Where to save/load Open-Retrieval Embedding' + ' data to/from') + + # indexer + group.add_argument('--indexer-batch-size', type=int, default=128, + help='How large of batches to use when doing indexing ' + 'jobs') + group.add_argument('--indexer-log-interval', type=int, default=1000, + help='After how many batches should the indexer ' + 'report progress') + return parser + + +def _add_vision_args(parser): + group = parser.add_argument_group(title="vision") + + # general vision arguments + group.add_argument('--num-classes', type=int, default=1000, + help='num of classes in vision classificaiton task') + group.add_argument('--img-h', type=int, default=224, + help='Image height for vision classification task') + group.add_argument('--img-w', type=int, default=224, + help='Image height for vision classification task') + group.add_argument('--num-channels', type=int, default=3, + help='Number of channels in input image data') + group.add_argument('--patch-dim', type=int, default=16, + help='patch dimension') + group.add_argument('--classes-fraction', type=float, default=1.0, + help='training with fraction of classes.') + group.add_argument('--data-per-class-fraction', type=float, default=1.0, + help='training with fraction of data per class.') + group.add_argument('--no-data-sharding', action='store_false', + help='Disable data sharding.', + dest='data_sharding') + group.add_argument('--head-lr-mult', type=float, default=1.0, + help='learning rate multiplier for head during finetuning') + + # pretraining type and backbone selection` + group.add_argument('--vision-pretraining', action='store_true', + help='flag to indicate vision pretraining') + group.add_argument('--vision-pretraining-type', type=str, default='classify', + choices=['classify', 'inpaint', 'dino'], + help='pretraining objectives') + group.add_argument('--vision-backbone-type', type=str, default='vit', + choices=['vit', 'mit', 'swin'], + help='backbone types types') + group.add_argument('--swin-backbone-type', type=str, default='tiny', + choices=['tiny', 'base', 'h3'], + help='pretraining objectives') + + # inpainting arguments + group.add_argument('--mask-type', type=str, default='random', + choices=['random', 'row'], + help='mask types') + group.add_argument('--mask-factor', type=float, default=1.0, + help='mask size scaling parameter') + + # dino arguments + group.add_argument('--iter-per-epoch', type=int, default=1250, + help='iterations per epoch') + group.add_argument('--dino-local-img-size', type=int, default=96, + help='Image size for vision classification task') + group.add_argument('--dino-local-crops-number', type=int, default=10, + help='Number of local crops') + group.add_argument('--dino-head-hidden-size', type=int, default=2048, + help='Hidden dimension size in dino head') + group.add_argument('--dino-bottleneck-size', type=int, default=256, + help='Bottle neck dimension in dino head ') + group.add_argument('--dino-freeze-last-layer', type=float, default=1, + help='Freezing last layer weights') + group.add_argument('--dino-norm-last-layer', action='store_true', + help='Disable Norm in last layer.') + group.add_argument('--dino-warmup-teacher-temp', type=float, default=0.04, + help='warump teacher temperature') + group.add_argument('--dino-teacher-temp', type=float, default=0.07, + help='teacher temperature') + group.add_argument('--dino-warmup-teacher-temp-epochs', type=int, default=30, + help='warmup teacher temperaure epochs') + + return parser diff --git a/apex/apex/transformer/testing/commons.py b/apex/apex/transformer/testing/commons.py new file mode 100644 index 00000000..238e1902 --- /dev/null +++ b/apex/apex/transformer/testing/commons.py @@ -0,0 +1,296 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +import datetime +import os +import random +from typing import Optional, Union, List, Tuple, Callable, Dict + +import numpy +import torch +import torch.nn as nn + +from apex import transformer +from apex.transformer.tensor_parallel import( + ColumnParallelLinear, + RowParallelLinear, + scatter_to_sequence_parallel_region, +) +from apex.transformer.pipeline_parallel.utils import ( + average_losses_across_data_parallel_group, +) +from apex.transformer.pipeline_parallel.schedules.common import ( + Batch, +) +from apex.transformer.testing import global_vars +from apex.transformer._ucc_util import HAS_UCC + +TEST_SUCCESS_MESSAGE = ">> passed the test :-)" + + +# note (mkozuki): `pre_process` and `post_process` are a placeholder until interleaving schedule test comes. +class MyLayer(nn.Module): + def __init__(self, hidden_size: int, pre_process: bool, post_process: bool): + super().__init__() + self.pre_process = pre_process + self.post_process = post_process + self.layer = nn.Linear(hidden_size, hidden_size) + + def forward(self, x): + return self.layer(x) + + +class MyModel(nn.Module): + def __init__( + self, + hidden_size: int, pre_process: bool = False, post_process: bool = False, + *, + add_encoder: bool = False, add_decoder: bool = False, + ) -> None: + super().__init__() + self.pre_process = pre_process + self.post_process = post_process + self.layer = MyLayer( + hidden_size=hidden_size, pre_process=pre_process, post_process=post_process + ) + self.input_tensor = None + + def set_input_tensor( + self, input_tensor: Union[torch.Tensor, List[torch.Tensor]] + ) -> None: + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + self.input_tensor = input_tensor[0] + + def forward(self, x: Optional[torch.Tensor]) -> torch.Tensor: + if self.input_tensor is None: + return self.layer(x) + return self.layer(self.input_tensor) + + +class ToyParallelMLP(nn.Module): + def __init__( + self, + hidden_size: int, pre_process: bool = False, post_process: bool = False, + *, + sequence_parallel_enabled: bool = False, + # TODO(mkozuki): Support these two? + add_encoder: bool = False, add_decoder: bool = False, + ) -> None: + super().__init__() + self.pre_process = pre_process + self.post_process = post_process + self.sequence_parallel_enabled = sequence_parallel_enabled + + ffn_hidden_size = 4 * hidden_size + self.dense_h_to_4h = ColumnParallelLinear( + hidden_size, + ffn_hidden_size, + gather_output=False, + # init_method=init_method, + skip_bias_add=True, + # use_cpu_initialization=use_cpu_initialization, + bias=True, + sequence_parallel_enabled=sequence_parallel_enabled, + no_async_tensor_model_parallel_allreduce=True, + ) + self.dense_4h_to_h = RowParallelLinear( + ffn_hidden_size, + hidden_size, + input_is_parallel=True, + # init_method=output_layer_init_method, + skip_bias_add=False, + # use_cpu_initialization=use_cpu_initialization, + bias=True, + sequence_parallel_enabled=sequence_parallel_enabled, + ) + self.activation_func = torch.nn.GELU() + + def set_input_tensor( + self, + input_tensor: Union[torch.Tensor, List[torch.Tensor]], + ) -> None: + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + self.input_tensor = input_tensor[0] + + def forward( + self, + x: Optional[torch.Tensor], + ) -> torch.Tensor: + """Forward of Simplified ParallelMLP. + + Args: + x: :obj:`None` if pipeline rank != pippeline first rank. When :obj:`None`, + `self.input_tensor` is taken care of by `forward_step` defined in + apex/transformer/pipeline_parallel/schedules/common.py + """ + # [s, b, h] + if self.input_tensor is None: + input = x + else: + input = self.input_tensor + intermediate_parallel, bias_parallel = self.dense_h_to_4h(input) + + if bias_parallel is not None: + intermediate_parallel += bias_parallel + intermediate_parallel = self.activation_func(intermediate_parallel) + # [s, b, h] + output, output_bias = self.dense_4h_to_h(intermediate_parallel) + return output + + +def model_provider_func( + hidden_size: int, + pre_process: bool, + post_process: bool, + *, + add_encoder: bool = False, + add_decoder: bool = False) -> MyModel: + return MyModel(hidden_size, pre_process, post_process, add_encoder=add_encoder, add_decoder=add_decoder) + + +def mlp_provider_func( + hidden_size: int, + pre_process: bool, + post_process: bool, + *, + add_encoder: bool = False, + add_decoder: bool = False, + sequence_parallel_enabled: bool = False, +) -> ToyParallelMLP: + return ToyParallelMLP( + hidden_size, + pre_process, + post_process, + add_encoder=add_encoder, + add_decoder=add_decoder, + sequence_parallel_enabled=sequence_parallel_enabled, + ) + + +def process_batch(batch): + if isinstance(batch, list): + x = batch[0] + else: + x = batch + return x + + +def fwd_step_func(batch, model): + x = process_batch(batch) + y = model(x) + + # note (mkozuki): I don't think this function is nice but I do think this is enough for now + # just to check the sanity of ported pipeline functions. + def loss_func(x): + loss = torch.sum(x) + averaged_loss = average_losses_across_data_parallel_group([loss]) + return loss, {"avg": averaged_loss} + + return y, loss_func + + +@dataclass(frozen=True) +class ToyParallelMLPFwdBwdStepFunc: + + sequence_parallel_enabled: bool + + def __call__( + self, + batch: Batch, + model: torch.nn.Module, + ) -> Tuple[torch.Tensor, Callable[[torch.Tensor], Tuple[torch.Tensor, Dict[str, torch.Tensor]]]]: + x = batch[0] if isinstance(batch, list) else batch + if isinstance(x, torch.Tensor): + x = x.transpose(0, 1).contiguous() + if self.sequence_parallel_enabled: + x = scatter_to_sequence_parallel_region(x) + y = model(x) + + # note (mkozuki): I don't think this function is nice but I do think this is enough for now + # just to check the sanity of ported pipeline functions. + def loss_func(x): + loss = torch.sum(x) + averaged_loss = average_losses_across_data_parallel_group([loss]) + return loss, {"avg": averaged_loss} + + return y, loss_func + + +class IdentityLayer(torch.nn.Module): + def __init__(self, size, scale=1.0): + super(IdentityLayer, self).__init__() + self.weight = torch.nn.Parameter(scale * torch.randn(size)) + + def forward(self): + return self.weight + + +def set_random_seed(seed): + """Set random seed for reproducibility.""" + random.seed(seed) + numpy.random.seed(seed) + torch.manual_seed(seed) + transformer.tensor_parallel.model_parallel_cuda_manual_seed(seed) + + +def initialize_distributed(backend="nccl"): + """Initialize torch.distributed.""" + # Get local rank in case it is provided. + # parser = argparse.ArgumentParser() + # parser.add_argument('--local_rank', type=int, default=None, + # help='local rank passed from distributed launcher') + # args = parser.parse_args() + if backend not in ("nccl", "ucc"): + raise RuntimeError(f"Currently only nccl & ucc are supported but {backend}") + if backend == "ucc": + if not HAS_UCC: + raise ImportError("UCC backend requires pytorch source build with UCC installed and enabled") + args = global_vars.get_args() + local_rank = args.local_rank + + # Get rank and world size. + rank = int(os.getenv("RANK", "0")) + world_size = int(os.getenv("WORLD_SIZE", "1")) + + print( + "> initializing torch.distributed with local rank: {}, " + "rank: {}, world size: {}".format(local_rank, rank, world_size) + ) + + # Set the device id. + device = rank % torch.cuda.device_count() + if local_rank is not None: + device = local_rank + torch.cuda.set_device(device) + + # Call the init process. + init_method = "tcp://" + master_ip = os.getenv("MASTER_ADDR", "localhost") + master_port = os.getenv("MASTER_PORT", "6000") + init_method += master_ip + ":" + master_port + torch.distributed.init_process_group( + backend=backend, world_size=world_size, rank=rank, init_method=init_method, + timeout=datetime.timedelta(seconds=60), + ) + + +def print_separator(message): + filler_len = (78 - len(message)) // 2 + filler = "-" * filler_len + string = "\n" + filler + " {} ".format(message) + filler + if torch.distributed.get_rank() == 0: + print(string, flush=True) diff --git a/apex/apex/transformer/testing/distributed_test_base.py b/apex/apex/transformer/testing/distributed_test_base.py new file mode 100644 index 00000000..3bb1b14d --- /dev/null +++ b/apex/apex/transformer/testing/distributed_test_base.py @@ -0,0 +1,126 @@ +import os +import sys +import unittest +from packaging.version import Version, parse + +import torch +from torch import distributed as dist +from torch.utils import collect_env +from torch.testing._internal import common_utils +from torch.testing._internal import common_distributed + +from apex.transformer._ucc_util import HAS_UCC + +# NOTE(mkozuki): Version guard for ucc. ref: https://github.com/openucx/ucc/issues/496 +_TORCH_UCC_COMPAT_NVIDIA_DRIVER_VERSION = Version("470.42.01") +_driver_version = None +if torch.cuda.is_available(): + _driver_version = parse(collect_env.get_nvidia_driver_version(collect_env.run)) +HAS_TORCH_UCC_COMPAT_NVIDIA_DRIVER = _driver_version is not None and _driver_version >= _TORCH_UCC_COMPAT_NVIDIA_DRIVER_VERSION + + +class DistributedTestBase(common_distributed.MultiProcessTestCase): + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def setUp(self) -> None: + super().setUp() + self._setup_pre_spawn() + self._spawn_processes() + + def tearDown(self) -> None: + torch.cuda.empty_cache() + super().tearDown() + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 4) + + @property + def init_method(self): + return f"{common_utils.FILE_SCHEMA}{self.file_name}" + + @classmethod + def _run(cls, rank, test_name, file_name, pipe): + self = cls(test_name) + self.assertTrue(torch.cuda.is_available()) + self.assertTrue(hasattr(self, "DISTRIBUTED_BACKEND")) + self.rank = rank + self.file_name = file_name + + print(f"[dist init] rank = {self.rank}, world_size = {self.world_size}") + + try: + dist.init_process_group( + init_method=self.init_method, + backend=self.DISTRIBUTED_BACKEND, + world_size=int(self.world_size), + rank=self.rank, + ) + except RuntimeError as e: + if "recompile" in e.args[0]: + print(f"Backend of {self.DISTRIBUTED_BACKEND} not available") + sys.exit(0) + raise + + torch.cuda.set_device(self.rank % torch.cuda.device_count()) + + dist.barrier() + self.run_test(test_name, pipe) + dist.barrier() + + dist.destroy_process_group() + sys.exit(0) + + def _setup_pre_spawn(self): + pass + + +class NcclDistributedTestBase(DistributedTestBase): + + DISTRIBUTED_BACKEND = "nccl" + +@unittest.skipUnless( + HAS_UCC, + "Requires either torch ucc or pytorch build from source with native ucc installed and enabled", +) +@unittest.skipUnless( + HAS_TORCH_UCC_COMPAT_NVIDIA_DRIVER, + f"`torch_ucc` requires NVIDIA driver >= {_TORCH_UCC_COMPAT_NVIDIA_DRIVER_VERSION} but {_driver_version} found. " + "See https://github.com/openucx/ucc/issues/496", +) + +class UccDistributedTestBase(DistributedTestBase): + + DISTRIBUTED_BACKEND = "ucc" + + def _setup_pre_spawn(self) -> None: + self.master_addr = "localhost" + os.environ["MASTER_ADDR"] = "localhost" + self._has_master_port = "MASTER_PORT" in os.environ + if self._has_master_port: + self.master_port = os.environ["MASTER_PORT"] + else: + try: + from caffe2.torch.fb.common.utils import get_free_port + self.master_port = str(get_free_port()) + except ImportError: + self.master_port = "12375" + os.environ["MASTER_PORT"] = self.master_port + + self._has_ucx_tls = "UCX_TLS" in os.environ + if not self._has_ucx_tls: + os.environ["UCX_TLS"] = "tcp,cuda" + print('os.environ[\"UCX_TLS\"] = {}'.format(os.environ["UCX_TLS"])) + + def tearDown(self) -> None: + super().tearDown() + if not self._has_master_port: + del os.environ["MASTER_PORT"] + if not self._has_ucx_tls: + del os.environ["UCX_TLS"] + + @property + def init_method(self): + return "tcp://localhost:" + os.environ["MASTER_PORT"] diff --git a/apex/apex/transformer/testing/global_vars.py b/apex/apex/transformer/testing/global_vars.py new file mode 100644 index 00000000..fa68f2c1 --- /dev/null +++ b/apex/apex/transformer/testing/global_vars.py @@ -0,0 +1,272 @@ +# coding=utf-8 +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Megatron global variables.""" +import os +import sys +import time + +import torch + +from apex.transformer.microbatches import build_num_microbatches_calculator +from .arguments import parse_args + +_GLOBAL_ARGS = None +_GLOBAL_NUM_MICROBATCHES_CALCULATOR = None +_GLOBAL_TOKENIZER = None +_GLOBAL_TENSORBOARD_WRITER = None +_GLOBAL_ADLR_AUTORESUME = None +_GLOBAL_TIMERS = None + + +def get_args(): + """Return arguments.""" + _ensure_var_is_initialized(_GLOBAL_ARGS, 'args') + return _GLOBAL_ARGS + + +def get_num_microbatches() -> int: + return _GLOBAL_NUM_MICROBATCHES_CALCULATOR.get() + + +def get_current_global_batch_size() -> int: + return _GLOBAL_NUM_MICROBATCHES_CALCULATOR.get_current_global_batch_size() + + +def update_num_microbatches(consumed_samples: int, *, consistency_check: bool = True) -> None: + """Update the number of microbatches upon the number of consumed samples. + + .. note:: + This function has no effect unless ``rampup_batch_size`` is set. + + Args: + consumed_samples: The number of consumed samples so far. Basically this is equal to + :math:`num_iter * global_batch_size`. + consistency_check: If :obj:`True`, sanity checks the consumed samples, i.e., check if + ``consumed_samples`` is divisible by :math:`micro_batch_size \times data_parallel_size`. + """ + _GLOBAL_NUM_MICROBATCHES_CALCULATOR.update(consumed_samples, consistency_check) + + +# def get_tokenizer(): +# """Return tokenizer.""" +# _ensure_var_is_initialized(_GLOBAL_TOKENIZER, 'tokenizer') +# return _GLOBAL_TOKENIZER + + +def get_tensorboard_writer(): + """Return tensorboard writer. It can be None so no need + to check if it is initialized.""" + return _GLOBAL_TENSORBOARD_WRITER + + +def get_adlr_autoresume(): + """ADLR autoresume object. It can be None so no need + to check if it is initialized.""" + return _GLOBAL_ADLR_AUTORESUME + + +def get_timers(): + """Return timers.""" + _ensure_var_is_initialized(_GLOBAL_TIMERS, 'timers') + return _GLOBAL_TIMERS + + +def set_global_variables(extra_args_provider=None, args_defaults={}, override_args={}, + ignore_unknown_args=False): + """Set args, tokenizer, tensorboard-writer, adlr-autoresume, and timers.""" + args = _parse_args(extra_args_provider=extra_args_provider, + defaults=args_defaults, + override_args=override_args, + ignore_unknown_args=ignore_unknown_args) + # _build_num_microbatches_calculator(args) + # if args.vocab_file: + # _ = _build_tokenizer(args) + _set_tensorboard_writer(args) + _set_adlr_autoresume(args) + _set_timers() + + +def _parse_args(extra_args_provider=None, defaults={}, override_args={}, + ignore_unknown_args=False): + """Parse entire arguments.""" + global _GLOBAL_ARGS + _ensure_var_is_not_initialized(_GLOBAL_ARGS, 'args') + _GLOBAL_ARGS = parse_args(extra_args_provider=extra_args_provider, + defaults=defaults, + override_args=override_args, + ignore_unknown_args=ignore_unknown_args) + return _GLOBAL_ARGS + + +def _build_num_microbatches_calculator(args): + + global _GLOBAL_NUM_MICROBATCHES_CALCULATOR + _ensure_var_is_not_initialized(_GLOBAL_NUM_MICROBATCHES_CALCULATOR, + 'num microbatches calculator') + + _GLOBAL_NUM_MICROBATCHES_CALCULATOR = build_num_microbatches_calculator( + args) + + +# def _build_tokenizer(args): +# """Initialize tokenizer.""" +# global _GLOBAL_TOKENIZER +# _ensure_var_is_not_initialized(_GLOBAL_TOKENIZER, 'tokenizer') +# _GLOBAL_TOKENIZER = build_tokenizer(args) +# return _GLOBAL_TOKENIZER + + +# def rebuild_tokenizer(args): +# global _GLOBAL_TOKENIZER +# _GLOBAL_TOKENIZER = None +# return _build_tokenizer(args) + + +def _set_tensorboard_writer(args): + """Set tensorboard writer.""" + global _GLOBAL_TENSORBOARD_WRITER + _ensure_var_is_not_initialized(_GLOBAL_TENSORBOARD_WRITER, + 'tensorboard writer') + + if hasattr(args, 'tensorboard_dir') and \ + args.tensorboard_dir and args.rank == (args.world_size - 1): + try: + from torch.utils.tensorboard import SummaryWriter + print('> setting tensorboard ...') + _GLOBAL_TENSORBOARD_WRITER = SummaryWriter( + log_dir=args.tensorboard_dir, + max_queue=args.tensorboard_queue_size) + except ModuleNotFoundError: + print('WARNING: TensorBoard writing requested but is not ' + 'available (are you using PyTorch 1.1.0 or later?), ' + 'no TensorBoard logs will be written.', flush=True) + + +def _set_adlr_autoresume(args): + """Initialize ADLR autoresume.""" + global _GLOBAL_ADLR_AUTORESUME + _ensure_var_is_not_initialized(_GLOBAL_ADLR_AUTORESUME, 'adlr autoresume') + + if args.adlr_autoresume: + if args.rank == 0: + print('enabling autoresume ...', flush=True) + sys.path.append(os.environ.get('SUBMIT_SCRIPTS', '.')) + try: + from userlib.auto_resume import AutoResume + except BaseException: + print('ADLR autoresume is not available, exiting ...') + sys.exit() + + _GLOBAL_ADLR_AUTORESUME = AutoResume + + +def _set_timers(): + """Initialize timers.""" + global _GLOBAL_TIMERS + _ensure_var_is_not_initialized(_GLOBAL_TIMERS, 'timers') + _GLOBAL_TIMERS = Timers() + + +def _ensure_var_is_initialized(var, name): + """Make sure the input variable is not None.""" + assert var is not None, '{} is not initialized.'.format(name) + + +def _ensure_var_is_not_initialized(var, name): + """Make sure the input variable is not None.""" + assert var is None, '{} is already initialized.'.format(name) + + +class _Timer: + """Timer.""" + + def __init__(self, name): + self.name_ = name + self.elapsed_ = 0.0 + self.started_ = False + self.start_time = time.time() + + def start(self): + """Start the timer.""" + assert not self.started_, 'timer has already been started' + torch.cuda.synchronize() + self.start_time = time.time() + self.started_ = True + + def stop(self): + """Stop the timer.""" + assert self.started_, 'timer is not started' + torch.cuda.synchronize() + self.elapsed_ += (time.time() - self.start_time) + self.started_ = False + + def reset(self): + """Reset timer.""" + self.elapsed_ = 0.0 + self.started_ = False + + def elapsed(self, reset=True): + """Calculate the elapsed time.""" + started_ = self.started_ + # If the timing in progress, end it first. + if self.started_: + self.stop() + # Get the elapsed time. + elapsed_ = self.elapsed_ + # Reset the elapsed time + if reset: + self.reset() + # If timing was in progress, set it back. + if started_: + self.start() + return elapsed_ + + +class Timers: + """Group of timers.""" + + def __init__(self): + self.timers = {} + + def __call__(self, name): + if name not in self.timers: + self.timers[name] = _Timer(name) + return self.timers[name] + + def write(self, names, writer, iteration, normalizer=1.0, reset=False): + """Write timers to a tensorboard writer""" + # currently when using add_scalars, + # torch.utils.add_scalars makes each timer its own run, which + # polutes the runs list, so we just add each as a scalar + assert normalizer > 0.0 + for name in names: + value = self.timers[name].elapsed(reset=reset) / normalizer + writer.add_scalar(name + '-time', value, iteration) + + def log(self, names, normalizer=1.0, reset=True): + """Log a group of timers.""" + assert normalizer > 0.0 + string = 'time (ms)' + for name in names: + elapsed_time = self.timers[name].elapsed( + reset=reset) * 1000.0 / normalizer + string += ' | {}: {:.2f}'.format(name, elapsed_time) + if torch.distributed.is_initialized(): + if torch.distributed.get_rank() == ( + torch.distributed.get_world_size() - 1): + print(string, flush=True) + else: + print(string, flush=True) diff --git a/apex/apex/transformer/testing/standalone_bert.py b/apex/apex/transformer/testing/standalone_bert.py new file mode 100644 index 00000000..dd66abcf --- /dev/null +++ b/apex/apex/transformer/testing/standalone_bert.py @@ -0,0 +1,255 @@ +import contextlib + +import torch + +from apex.transformer import tensor_parallel +from apex.transformer.enums import AttnMaskType +from apex.transformer.enums import ModelType +from apex.transformer.layers import FusedLayerNorm as LayerNorm +from apex.transformer.testing.global_vars import get_args +from apex.transformer.testing.standalone_transformer_lm import ( + MegatronModule, + get_language_model, + get_linear_layer, + init_method_normal, + scaled_init_method_normal, + parallel_lm_logits, +) + + +def bert_extended_attention_mask(attention_mask): + # We create a 3D attention mask from a 2D tensor mask. + # [b, 1, s] + attention_mask_b1s = attention_mask.unsqueeze(1) + # [b, s, 1] + attention_mask_bs1 = attention_mask.unsqueeze(2) + # [b, s, s] + attention_mask_bss = attention_mask_b1s * attention_mask_bs1 + # [b, 1, s, s] + extended_attention_mask = attention_mask_bss.unsqueeze(1) + + # Convert attention mask to binary: + extended_attention_mask = (extended_attention_mask < 0.5) + + return extended_attention_mask + + +def bert_position_ids(token_ids): + # Create position ids + seq_length = token_ids.size(1) + position_ids = torch.arange(seq_length, dtype=torch.long, + device=token_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(token_ids) + + return position_ids + + +class BertLMHead(MegatronModule): + """Masked LM head for Bert + + Arguments: + mpu_vocab_size: model parallel size of vocabulary. + hidden_size: hidden size + init_method: init method for weight initialization + layernorm_epsilon: tolerance for layer norm divisions + parallel_output: whether output logits being distributed or not. + """ + + def __init__(self, mpu_vocab_size, hidden_size, init_method, + layernorm_epsilon, parallel_output): + + super(BertLMHead, self).__init__() + + args = get_args() + + self.bias = torch.nn.Parameter(torch.zeros(mpu_vocab_size)) + # TODO: do we need this? + # mpu.set_tensor_model_parallel_attributes(self.bias, True, 0, 1) + self.parallel_output = parallel_output + + self.dense = get_linear_layer(hidden_size, hidden_size, init_method) + setattr(self.dense.weight, 'sequence_parallel', args.sequence_parallel) + setattr(self.dense.bias, 'sequence_parallel', args.sequence_parallel) + + self.layernorm = LayerNorm( + hidden_size, eps=layernorm_epsilon, sequence_parallel_enabled=args.sequence_parallel) + self.gelu = torch.nn.functional.gelu + if args.openai_gelu: + self.gelu = openai_gelu + elif args.onnx_safe: + self.gelu = erf_gelu + + + def forward(self, hidden_states, word_embeddings_weight): + hidden_states = self.dense(hidden_states) + hidden_states = self.gelu(hidden_states) + hidden_states = self.layernorm(hidden_states) + output = parallel_lm_logits(hidden_states, + word_embeddings_weight, + self.parallel_output, + bias=self.bias) + return output + + +def post_language_model_processing(lm_output, pooled_output, + lm_head, binary_head, + lm_labels, + logit_weights, + fp16_lm_cross_entropy): + # Output. + lm_logits = lm_head( + lm_output, logit_weights) + + binary_logits = None + if binary_head is not None: + binary_logits = binary_head(pooled_output) + + if lm_labels is None: + # [s b h] => [b s h] + return lm_logits.transpose(0, 1).contiguous(), binary_logits + else: + # [b s] => [s b] + lm_labels = lm_labels.transpose(0, 1).contiguous() + # lm_logits: [s b h] lm_labels: [s b] + if fp16_lm_cross_entropy: + assert lm_logits.dtype == torch.half + lm_loss = tensor_parallel.vocab_parallel_cross_entropy(lm_logits, lm_labels) + else: + lm_loss = tensor_parallel.vocab_parallel_cross_entropy(lm_logits.float(), + lm_labels) + return lm_loss, binary_logits + + +class BertModel(MegatronModule): + """Bert Language model.""" + + def __init__(self, + num_tokentypes=2, + add_binary_head=True, + parallel_output=True, + pre_process=True, + post_process=True, + cpu_offload=False): + super(BertModel, self).__init__() + args = get_args() + + self.fp16_lm_cross_entropy = args.fp16_lm_cross_entropy + self.add_binary_head = add_binary_head + self.parallel_output = parallel_output + self.pre_process = pre_process + self.post_process = post_process + + init_method = init_method_normal(args.init_method_std) + scaled_init_method = scaled_init_method_normal(args.init_method_std, + args.num_layers) + + self.language_model, self._language_model_key = get_language_model( + num_tokentypes=num_tokentypes, + add_pooler=self.add_binary_head, + encoder_attn_mask_type=AttnMaskType.padding, + init_method=init_method, + scaled_init_method=scaled_init_method, + pre_process=self.pre_process, + post_process=self.post_process) + + self.initialize_word_embeddings(init_method_normal) + if self.post_process: + self.lm_head = BertLMHead( + self.word_embeddings_weight().size(0), + args.hidden_size, init_method, args.layernorm_epsilon, parallel_output) + self._lm_head_key = 'lm_head' + self.binary_head = None + if self.add_binary_head: + self.binary_head = get_linear_layer(args.hidden_size, 2, + init_method) + self._binary_head_key = 'binary_head' + + self.forward_context = contextlib.nullcontext + if cpu_offload: + self.forward_context = torch.autograd.graph.save_on_cpu + + def set_input_tensor(self, input_tensor): + """See megatron.model.transformer.set_input_tensor()""" + self.language_model.set_input_tensor(input_tensor) + + def forward(self, bert_model_input, attention_mask, + tokentype_ids=None, lm_labels=None): + with self.forward_context(): + extended_attention_mask = bert_extended_attention_mask(attention_mask) + input_ids = bert_model_input + position_ids = bert_position_ids(input_ids) + + lm_output = self.language_model( + input_ids, + position_ids, + extended_attention_mask, + tokentype_ids=tokentype_ids + ) + + if self.post_process and self.add_binary_head: + lm_output, pooled_output = lm_output + else: + pooled_output = None + + if self.post_process: + return post_language_model_processing(lm_output, pooled_output, + self.lm_head, self.binary_head, + lm_labels, + self.word_embeddings_weight(), + self.fp16_lm_cross_entropy) + else: + return lm_output + + # NOTE(mkozuki): This method is not maintained as apex only tests forward_backward with best effort. + def state_dict_for_save_checkpoint(self, destination=None, prefix='', + keep_vars=False): + """For easy load when model is combined with other heads, + add an extra key.""" + + state_dict_ = {} + state_dict_[self._language_model_key] \ + = self.language_model.state_dict_for_save_checkpoint( + destination, prefix, keep_vars) + if self.post_process: + state_dict_[self._lm_head_key] \ + = self.lm_head.state_dict_for_save_checkpoint( + destination, prefix, keep_vars) + if self.post_process and self.add_binary_head: + state_dict_[self._binary_head_key] \ + = self.binary_head.state_dict(destination, prefix, keep_vars) + # Save word_embeddings. + if self.post_process and not self.pre_process: + state_dict_[self._word_embeddings_for_head_key] \ + = self.word_embeddings.state_dict(destination, prefix, keep_vars) + return state_dict_ + + # NOTE(mkozuki): This method is not maintained as apex only tests forward_backward with best effort. + def load_state_dict(self, state_dict, strict=True): + """Customized load.""" + + self.language_model.load_state_dict( + state_dict[self._language_model_key], strict=strict) + if self.post_process: + self.lm_head.load_state_dict( + state_dict[self._lm_head_key], strict=strict) + if self.post_process and self.add_binary_head: + self.binary_head.load_state_dict( + state_dict[self._binary_head_key], strict=strict) + # Load word_embeddings. + if self.post_process and not self.pre_process: + self.word_embeddings.load_state_dict( + state_dict[self._word_embeddings_for_head_key], strict=strict) + + +def bert_model_provider(pre_process=True, post_process=True, cpu_offload=False): + args = get_args() + num_tokentypes = 2 if args.bert_binary_head else 0 + model = BertModel( + num_tokentypes=num_tokentypes, + add_binary_head=args.bert_binary_head, + parallel_output=True, + pre_process=pre_process, + post_process=post_process, + cpu_offload=cpu_offload, + ) + return model diff --git a/apex/apex/transformer/testing/standalone_gpt.py b/apex/apex/transformer/testing/standalone_gpt.py new file mode 100644 index 00000000..0e3d4645 --- /dev/null +++ b/apex/apex/transformer/testing/standalone_gpt.py @@ -0,0 +1,111 @@ +# Copyright (c) 2021-22, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import torch + +from apex.transformer.enums import AttnMaskType +from apex.transformer.enums import ModelType +from apex.transformer import tensor_parallel +from apex.transformer.testing.global_vars import get_args +from apex.transformer.testing.standalone_transformer_lm import MegatronModule +from apex.transformer.testing.standalone_transformer_lm import parallel_lm_logits +from apex.transformer.testing.standalone_transformer_lm import post_language_model_processing +from apex.transformer.testing.standalone_transformer_lm import get_language_model +from apex.transformer.testing.standalone_transformer_lm import init_method_normal +from apex.transformer.testing.standalone_transformer_lm import ( + scaled_init_method_normal, +) + + + +def gpt_model_provider(pre_process: bool = True, post_process: bool = True, cpu_offload: bool = False,) -> "GPTModel": + args = get_args() + model = GPTModel( + num_tokentypes=0, + parallel_output=True, + pre_process=pre_process, + post_process=post_process, + cpu_offload=args.cpu_offload, + ) + return model + + +class GPTModel(MegatronModule): + """GPT-2 Language model.""" + + def __init__( + self, + num_tokentypes:int = 0, + parallel_output: bool = True, + pre_process: bool = True, + post_process: bool = True, + cpu_offload: bool = False, + ): + super().__init__() + args = get_args() + + self.forward_context = contextlib.nullcontext + if cpu_offload: + self.forward_context = torch.autograd.graph.save_on_cpu + + self.parallel_output = parallel_output + self.pre_process = pre_process + self.post_process = post_process + self.fp16_lm_cross_entropy = args.fp16_lm_cross_entropy + + self.language_model, self._language_model_key = get_language_model( + num_tokentypes=num_tokentypes, + add_pooler=False, + encoder_attn_mask_type=AttnMaskType.causal, + init_method=init_method_normal(args.init_method_std), + scaled_init_method=scaled_init_method_normal( + args.init_method_std, args.num_layers + ), + pre_process=self.pre_process, + post_process=self.post_process, + ) + + self.initialize_word_embeddings(init_method_normal) + + def set_input_tensor(self, input_tensor): + """See megatron.model.transformer.set_input_tensor()""" + self.language_model.set_input_tensor(input_tensor) + + def forward( + self, + input_ids, + position_ids, + attention_mask, + labels=None, + tokentype_ids=None, + inference_params=None, + ): + + with self.forward_context(): + lm_output = self.language_model( + input_ids, position_ids, attention_mask, inference_params=inference_params + ) + + if self.post_process: + return post_language_model_processing( + lm_output, + # note(mkozuki): Am I overlooking some order of dim change? + labels.t().contiguous(), + self.word_embeddings_weight(), + self.parallel_output, + self.fp16_lm_cross_entropy, + ) + else: + return lm_output diff --git a/apex/apex/transformer/testing/standalone_transformer_lm.py b/apex/apex/transformer/testing/standalone_transformer_lm.py new file mode 100644 index 00000000..6cd90c74 --- /dev/null +++ b/apex/apex/transformer/testing/standalone_transformer_lm.py @@ -0,0 +1,1574 @@ +# coding=utf-8 +# Copyright (c) 2021-22, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""GPT-2 model.""" +import enum +import math +import contextlib +import json + +import torch +import torch.nn.functional as F + +import apex.transformer.utils +from apex.transformer.layers import FusedLayerNorm as LayerNorm +from apex.transformer.functional import FusedScaleMaskSoftmax +from apex.transformer import tensor_parallel +from apex.transformer.tensor_parallel.layers import ColumnParallelLinear +from apex.transformer.tensor_parallel.layers import RowParallelLinear +from apex.transformer.tensor_parallel.layers import VocabParallelEmbedding +from apex.transformer.tensor_parallel.mappings import scatter_to_sequence_parallel_region +from apex.transformer import parallel_state +from apex.transformer.testing.global_vars import get_args +from apex.transformer.enums import ModelType +from apex.transformer.enums import LayerType +from apex.transformer.enums import AttnType +from apex.transformer.enums import AttnMaskType +from apex.transformer.log_util import get_transformer_logger + + +_logger = get_transformer_logger(__name__) + + +def param_is_not_shared(param: torch.Tensor) -> bool: + return getattr(param, "shared", False) + + +class MegatronModule(torch.nn.Module): + """Megatron specific extensions of torch Module with support for pipelining.""" + + def __init__(self, share_word_embeddings: bool = True) -> None: + super().__init__() + self.share_word_embeddings = share_word_embeddings + + def word_embeddings_weight(self): + if self.pre_process: + return self.language_model.embedding.word_embeddings.weight + else: + if not self.share_word_embeddings: + raise Exception('word_embeddings_weight() called for last stage, but share_word_embeddings is false') + return self.word_embeddings.weight + + + def initialize_word_embeddings(self, init_method_normal): + args = get_args() + if not self.share_word_embeddings: + raise Exception("initialize_word_embeddings() was called but share_word_embeddings is false") + + # This function just initializes the word embeddings in the final stage + # when we are using pipeline parallelism. Nothing to do if we aren't + # using pipeline parallelism. + if args.pipeline_model_parallel_size == 1: + return + + # Parameters are shared between the word embeddings layers, and the + # heads at the end of the model. In a pipelined setup with more than + # one stage, the initial embedding layer and the head are on different + # workers, so we do the following: + # 1. Create a second copy of word_embeddings on the last stage, with + # initial parameters of 0.0. + # 2. Do an all-reduce between the first and last stage to ensure that + # the two copies of word_embeddings start off with the same + # parameter values. + # 3. In the training loop, before an all-reduce between the grads of + # the two word_embeddings layers to ensure that every applied weight + # update is the same on both stages. + if parallel_state.is_pipeline_last_stage() and not self.pre_process: + assert not parallel_state.is_pipeline_first_stage() + self._word_embeddings_for_head_key = 'word_embeddings_for_head' + # set word_embeddings weights to 0 here, then copy first + # stage's weights using all_reduce below. + self.word_embeddings = VocabParallelEmbedding( + args.padded_vocab_size, args.hidden_size, + init_method=init_method_normal(args.init_method_std)) + self.word_embeddings.weight.data.fill_(0) + self.word_embeddings.weight.shared = True + + # Zero out initial weights for decoder embedding. + # NOTE: We don't currently support T5 with the interleaved schedule. + if not parallel_state.is_pipeline_first_stage(ignore_virtual=True) and self.pre_process: + self.language_model.embedding.zero_parameters() + + # Ensure that first and last stages have the same initial parameter + # values. + if torch.distributed.is_initialized(): + if parallel_state.is_rank_in_embedding_group(): + torch.distributed.all_reduce(self.word_embeddings_weight(), + group=parallel_state.get_embedding_group()) + + # Ensure that encoder(first stage) and decoder(split stage) position + # embeddings have the same initial parameter values + # NOTE: We don't currently support T5 with the interleaved schedule. + if parallel_state.is_rank_in_position_embedding_group() and \ + args.pipeline_model_parallel_split_rank is not None: + # TODO: Support tokentype embedding. + self.language_model.embedding.cuda() + position_embeddings = self.language_model.embedding.position_embeddings + torch.distributed.all_reduce(position_embeddings.weight, + group=parallel_state.get_position_embedding_group()) + + else: + print("WARNING! Distributed processes aren't initialized, so " + "word embeddings in the last layer are not initialized. " + "If you are just manipulating a model this is fine, but " + "this needs to be handled manually. If you are training " + "something is definitely wrong.") + + +def get_linear_layer(rows, columns, init_method): + """Simple linear layer with weight initialization.""" + layer = torch.nn.Linear(rows, columns) + init_method(layer.weight) + with torch.no_grad(): + layer.bias.zero_() + return layer + + +# NOTE(mkozuki): Avoid inplace op. +def attention_mask_func(attention_scores: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor: + # attention_scores.masked_fill_(attention_mask, -10000.0) + # return attention_scores + return attention_scores.masked_fill(attention_mask, -10000.0) + + +def init_method_normal(sigma): + """Init method based on N(0, sigma).""" + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=0.0, std=sigma) + + return init_ + + +def scaled_init_method_normal(sigma, num_layers): + """Init method based on N(0, sigma/sqrt(2*num_layers).""" + std = sigma / math.sqrt(2.0 * num_layers) + + def init_(tensor): + return torch.nn.init.normal_(tensor, mean=0.0, std=std) + + return init_ + + +class ParallelMLP(MegatronModule): + """MLP. + + MLP will take the input with h hidden state, project it to 4*h + hidden dimension, perform nonlinear transformation, and project the + state back into h hidden dimension. + """ + + def __init__(self, init_method, output_layer_init_method): + super().__init__() + args = get_args() + + # Project to 4h. + self.dense_h_to_4h = ColumnParallelLinear( + args.hidden_size, + args.ffn_hidden_size, + gather_output=False, + init_method=init_method, + skip_bias_add=True, + no_async_tensor_model_parallel_allreduce=not args.async_tensor_model_parallel_allreduce, + sequence_parallel_enabled=args.sequence_parallel, + ) + + self.bias_gelu_fusion = args.bias_gelu_fusion + self.activation_func = F.gelu + + # Project back to h. + self.dense_4h_to_h = RowParallelLinear( + args.ffn_hidden_size, + args.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + skip_bias_add=True, + sequence_parallel_enabled=args.sequence_parallel, + ) + + def forward(self, hidden_states): + + # [s, b, 4hp] + intermediate_parallel, bias_parallel = self.dense_h_to_4h(hidden_states) + + intermediate_parallel = self.activation_func(intermediate_parallel + bias_parallel) + + # [s, b, h] + output, output_bias = self.dense_4h_to_h(intermediate_parallel) + return output, output_bias + + +class CoreAttention(MegatronModule): + + def __init__(self, layer_number, attn_mask_type=AttnMaskType.padding): + super().__init__() + args = get_args() + self.fp16 = args.fp16 + self.bf16 = args.bf16 + + self.apply_query_key_layer_scaling = args.apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = args.attention_softmax_in_fp32 + if self.apply_query_key_layer_scaling: + self.attention_softmax_in_fp32 = True + self.layer_number = max(1, layer_number) + self.attn_mask_type = attn_mask_type + self.sequence_parallel = args.sequence_parallel + + projection_size = args.kv_channels * args.num_attention_heads + + # Per attention head and per partition values. + world_size = parallel_state.get_tensor_model_parallel_world_size() + self.hidden_size_per_partition = apex.transformer.utils.divide( + projection_size, world_size + ) + self.hidden_size_per_attention_head = apex.transformer.utils.divide( + projection_size, args.num_attention_heads + ) + self.num_attention_heads_per_partition = apex.transformer.utils.divide( + args.num_attention_heads, world_size + ) + + coeff = None + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head) + if self.apply_query_key_layer_scaling: + coeff = self.layer_number + self.norm_factor *= coeff + + self.scale_mask_softmax = FusedScaleMaskSoftmax( + self.fp16, + self.bf16, + self.attn_mask_type, + args.masked_softmax_fusion, + attention_mask_func, + self.attention_softmax_in_fp32, + coeff, + ) + # Dropout. Note that for a single iteration, this layer will generate + # different outputs on different number of parallel partitions but + # on average it should not be partition dependent. + self.attention_dropout = torch.nn.Dropout(args.attention_dropout) + + def forward(self, query_layer, key_layer, value_layer, attention_mask): + # =================================== + # Raw attention scores. [b, np, s, s] + # =================================== + # [b, np, sq, sk] + output_size = ( + query_layer.size(1), + query_layer.size(2), + query_layer.size(0), + key_layer.size(0), + ) + # [sq, b, np, hn] -> [sq, b * np, hn] + query_layer = query_layer.view( + output_size[2], output_size[0] * output_size[1], -1 + ) + # [sk, b, np, hn] -> [sk, b * np, hn] + key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1) + + # preallocting input tensor: [b * np, sq, sk] + matmul_input_buffer = torch.empty( + output_size[0] * output_size[1], + output_size[2], + output_size[3], + dtype=query_layer.dtype, + device=torch.cuda.current_device(), + ) + # Raw attention scores. [b * np, sq, sk] + matmul_result = torch.baddbmm( + matmul_input_buffer, + query_layer.transpose(0, 1), # [b * np, sq, hn] + key_layer.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=(1.0 / self.norm_factor), + ) + + # change view to [b, np, sq, sk] + attention_scores = matmul_result.view(*output_size) + + # =========================== + # Attention probs and dropout + # =========================== + # attention scores and attention mask [b, np, sq, sk] + attention_probs = self.scale_mask_softmax(attention_scores, attention_mask) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + if not self.sequence_parallel: + with tensor_parallel.get_cuda_rng_tracker().fork(): + attention_probs = self.attention_dropout(attention_probs) + else: + attention_probs = self.attention_dropout(attention_probs) + + # ========================= + # Context layer. [sq, b, hp] + # ========================= + + # value_layer -> context layer. + # [sk, b, np, hn] --> [b, np, sq, hn] + + # context layer shape: [b, np, sq, hn] + output_size = ( + value_layer.size(1), + value_layer.size(2), + query_layer.size(0), + value_layer.size(3), + ) + + # change view [sk, b * np, hn] + value_layer = value_layer.view( + value_layer.size(0), output_size[0] * output_size[1], -1 + ) + + # change view [b * np, sq, sk] + attention_probs = attention_probs.view( + output_size[0] * output_size[1], output_size[2], -1 + ) + + # matmul: [b * np, sq, hn] + context_layer = torch.bmm(attention_probs, value_layer.transpose(0, 1)) + + # change view [b, np, sq, hn] + context_layer = context_layer.view(*output_size) + + # [b, np, sq, hn] --> [sq, b, np, hn] + context_layer = context_layer.permute(2, 0, 1, 3).contiguous() + + # [sq, b, np, hn] --> [sq, b, hp] + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size_per_partition, + ) + context_layer = context_layer.view(*new_context_layer_shape) + + return context_layer + + +class ParallelAttention(MegatronModule): + """Parallel self-attention layer abstract class. + + Self-attention layer takes input with size [b, s, h] + and returns output of the same size. + """ + + def __init__( + self, + init_method, + output_layer_init_method, + layer_number, + attention_type=AttnType.self_attn, + attn_mask_type=AttnMaskType.padding, + ): + super().__init__() + args = get_args() + self.layer_number = max(1, layer_number) + self.attention_type = attention_type + self.attn_mask_type = attn_mask_type + self.params_dtype = args.params_dtype + + projection_size = args.kv_channels * args.num_attention_heads + + # Per attention head and per partition values. + world_size = parallel_state.get_tensor_model_parallel_world_size() + self.hidden_size_per_attention_head = apex.transformer.utils.divide( + projection_size, args.num_attention_heads + ) + self.num_attention_heads_per_partition = apex.transformer.utils.divide( + args.num_attention_heads, world_size + ) + + # Strided linear layer. + if attention_type == AttnType.self_attn: + self.query_key_value = ColumnParallelLinear( + args.hidden_size, + 3 * projection_size, + gather_output=False, + init_method=init_method, + no_async_tensor_model_parallel_allreduce=not args.async_tensor_model_parallel_allreduce, + sequence_parallel_enabled=args.sequence_parallel, + ) + else: + assert attention_type == AttnType.cross_attn + self.query = ColumnParallelLinear( + args.hidden_size, + projection_size, + gather_output=False, + init_method=init_method, + no_async_tensor_model_parallel_allreduce=not args.async_tensor_model_parallel_allreduce, + sequence_parallel_enabled=args.sequence_parallel, + ) + + self.key_value = ColumnParallelLinear( + args.hidden_size, + 2 * projection_size, + gather_output=False, + init_method=init_method, + no_async_tensor_model_parallel_allreduce=not args.async_tensor_model_parallel_allreduce, + sequence_parallel_enabled=args.sequence_parallel, + ) + + self.core_attention = CoreAttention(self.layer_number, self.attn_mask_type) + self.checkpoint_core_attention = args.recompute_granularity == "selective" + + # Output. + self.dense = RowParallelLinear( + projection_size, + args.hidden_size, + input_is_parallel=True, + init_method=output_layer_init_method, + skip_bias_add=True, + sequence_parallel_enabled=args.sequence_parallel, + ) + + def _checkpointed_attention_forward( + self, query_layer, key_layer, value_layer, attention_mask + ): + """Forward method with activation checkpointing.""" + + def custom_forward(*inputs): + query_layer = inputs[0] + key_layer = inputs[1] + value_layer = inputs[2] + attention_mask = inputs[3] + output_ = self.core_attention( + query_layer, key_layer, value_layer, attention_mask + ) + return output_ + + hidden_states = tensor_parallel.checkpoint( + custom_forward, False, query_layer, key_layer, value_layer, attention_mask + ) + + return hidden_states + + def _allocate_memory(self, inference_max_sequence_len, batch_size): + return torch.empty( + inference_max_sequence_len, + batch_size, + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + dtype=self.params_dtype, + device=torch.cuda.current_device(), + ) + + def forward( + self, hidden_states, attention_mask, encoder_output=None, inference_params=None + ): + # hidden_states: [sq, b, h] + + # ================================================= + # Pre-allocate memory for key-values for inference. + # ================================================= + if inference_params: + if self.layer_number not in inference_params.key_value_memory_dict: + inf_max_seq_len = inference_params.max_sequence_len + inf_max_batch_size = inference_params.max_batch_size + inference_key_memory = self._allocate_memory( + inf_max_seq_len, inf_max_batch_size + ) + inference_value_memory = self._allocate_memory( + inf_max_seq_len, inf_max_batch_size + ) + inference_params.key_value_memory_dict[self.layer_number] = ( + inference_key_memory, + inference_value_memory, + ) + else: + ( + inference_key_memory, + inference_value_memory, + ) = inference_params.key_value_memory_dict[self.layer_number] + + # ===================== + # Query, Key, and Value + # ===================== + + if self.attention_type == AttnType.self_attn: + # Attention heads [sq, b, h] --> [sq, b, (np * 3 * hn)] + mixed_x_layer, _ = self.query_key_value(hidden_states) + + # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn] + new_tensor_shape = mixed_x_layer.size()[:-1] + ( + self.num_attention_heads_per_partition, + 3 * self.hidden_size_per_attention_head, + ) + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape) + + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn] + ( + query_layer, + key_layer, + value_layer, + ) = tensor_parallel.utils.split_tensor_along_last_dim(mixed_x_layer, 3) + else: + # Attention heads [sk, b, h] --> [sk, b, (np * 2 * hn)] + mixed_kv_layer, _ = self.key_value(encoder_output) + + # [sk, b, (np * 2 * hn)] --> [sk, b, np, 2 * hn] + new_tensor_shape = mixed_kv_layer.size()[:-1] + ( + self.num_attention_heads_per_partition, + 2 * self.hidden_size_per_attention_head, + ) + mixed_kv_layer = mixed_kv_layer.view(*new_tensor_shape) + + # [sk, b, np, 2 * hn] --> 2 [sk, b, np, hn] + ( + key_layer, + value_layer, + ) = tensor_parallel.utils.split_tensor_along_last_dim(mixed_kv_layer, 2) + + # Attention head [sq, b, h] --> [sq, b, hp] + query_layer, _ = self.query(hidden_states) + # [sq, b, hp] --> [sq, b, np, hn] + new_tensor_shape = query_layer.size()[:-1] + ( + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + ) + query_layer = query_layer.view(*new_tensor_shape) + + # ================================== + # Adjust key and value for inference + # ================================== + + if inference_params: + batch_start = inference_params.batch_size_offset + batch_end = batch_start + key_layer.size(1) + assert batch_end <= inference_key_memory.size(1) + sequence_start = inference_params.sequence_len_offset + sequence_end = sequence_start + key_layer.size(0) + assert sequence_end <= inference_key_memory.size(0) + # Copy key and values. + inference_key_memory[ + sequence_start:sequence_end, batch_start:batch_end, ... + ] = key_layer + inference_value_memory[ + sequence_start:sequence_end, batch_start:batch_end, ... + ] = value_layer + key_layer = inference_key_memory[:sequence_end, batch_start:batch_end, ...] + value_layer = inference_value_memory[ + :sequence_end, batch_start:batch_end, ... + ] + + # ================================== + # core attention computation + # ================================== + + if self.checkpoint_core_attention: + context_layer = self._checkpointed_attention_forward( + query_layer, key_layer, value_layer, attention_mask + ) + else: + context_layer = self.core_attention( + query_layer, key_layer, value_layer, attention_mask + ) + + # ================= + # Output. [sq, b, h] + # ================= + + output, bias = self.dense(context_layer) + + return output, bias + + +def bias_dropout_add(x: torch.Tensor, bias: torch.Tensor, residual: torch.Tensor, prob: float, training: bool) -> torch.Tensor: + out = torch.nn.functional.dropout(x + bias, p=prob, training=training) + out = residual + out + return out + + +def get_bias_dropout_add(training): + def _bias_dropout_add(x, bias, residual, prob): + return bias_dropout_add(x, bias, residual, prob, training) + + return _bias_dropout_add + + +class ParallelTransformerLayer(MegatronModule): + """A single transformer layer. + + Transformer layer takes input with size [s, b, h] and returns an + output of the same size. + """ + + def __init__( + self, + init_method, + output_layer_init_method, + layer_number, + layer_type=LayerType.encoder, + self_attn_mask_type=AttnMaskType.padding, + drop_path_rate=0.0, + ): + args = get_args() + + super().__init__() + self.layer_number = layer_number + self.layer_type = layer_type + + self.apply_residual_connection_post_layernorm = ( + args.apply_residual_connection_post_layernorm + ) + + self.bf16 = args.bf16 + self.fp32_residual_connection = args.fp32_residual_connection + + # Layernorm on the input data. + self.input_layernorm = LayerNorm( + args.hidden_size, + eps=args.layernorm_epsilon, + # no_persist_layer_norm=args.no_persist_layer_norm, + sequence_parallel_enabled=args.sequence_parallel, + ) + + # Self attention. + self.self_attention = ParallelAttention( + init_method, + output_layer_init_method, + layer_number, + attention_type=AttnType.self_attn, + attn_mask_type=self_attn_mask_type, + ) + self.hidden_dropout = args.hidden_dropout + self.bias_dropout_fusion = args.bias_dropout_fusion + # note(mkozuki) + # self.drop_path = DropPath(drop_path_rate) if drop_path_rate > 0.0 else None + assert drop_path_rate <= 0.0 + self.drop_path = None + + # Layernorm on the attention output + self.post_attention_layernorm = LayerNorm( + args.hidden_size, + eps=args.layernorm_epsilon, + # no_persist_layer_norm=args.no_persist_layer_norm, + sequence_parallel_enabled=args.sequence_parallel, + ) + + if self.layer_type == LayerType.decoder: + self.inter_attention = ParallelAttention( + init_method, + output_layer_init_method, + layer_number, + attention_type=AttnType.cross_attn, + ) + # Layernorm on the attention output. + self.post_inter_attention_layernorm = LayerNorm( + args.hidden_size, + eps=args.layernorm_epsilon, + # no_persist_layer_norm=args.no_persist_layer_norm, + sequence_parallel_enabled=args.sequence_parallel, + ) + + # MLP + # note(mkozuki) + assert args.num_experts is None + # if args.num_experts is not None: + # self.mlp = SwitchMLP(init_method, output_layer_init_method) + # else: + # self.mlp = ParallelMLP(init_method, output_layer_init_method) + self.mlp = ParallelMLP(init_method, output_layer_init_method) + + # Set bias+dropout+add fusion grad_enable execution handler. + TORCH_MAJOR = int(torch.__version__.split(".")[0]) + TORCH_MINOR = int(torch.__version__.split(".")[1]) + use_nvfuser = TORCH_MAJOR > 1 or (TORCH_MAJOR == 1 and TORCH_MINOR >= 10) + self.bias_dropout_add_exec_handler = ( + contextlib.nullcontext if use_nvfuser else torch.enable_grad + ) + + def forward( + self, + hidden_states, + attention_mask, + encoder_output=None, + enc_dec_attn_mask=None, + inference_params=None, + ): + # hidden_states: [s, b, h] + # Layer norm at the beginning of the transformer layer. + layernorm_output = self.input_layernorm(hidden_states) + # Self attention. + attention_output, attention_bias = self.self_attention( + layernorm_output, attention_mask, inference_params=inference_params + ) + + # Residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = hidden_states + + if self.drop_path is None: + bias_dropout_add_func = get_bias_dropout_add(self.training) + + with self.bias_dropout_add_exec_handler(): + layernorm_input = bias_dropout_add_func( + attention_output, + attention_bias.expand_as(residual), + residual, + self.hidden_dropout, + ) + else: + out = torch.nn.functional.dropout( + attention_output + attention_bias, + p=self.hidden_dropout, + training=self.training, + ) + layernorm_input = residual + self.drop_path(out) + + # Layer norm post the self attention. + layernorm_output = self.post_attention_layernorm(layernorm_input) + + if self.layer_type == LayerType.decoder: + attention_output, attention_bias = self.inter_attention( + layernorm_output, enc_dec_attn_mask, encoder_output=encoder_output + ) + # residual connection + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + with self.bias_dropout_add_exec_handler(): + layernorm_input = bias_dropout_add_func( + attention_output, + attention_bias.expand_as(residual), + residual, + self.hidden_dropout, + ) + + # Layer norm post the decoder attention + layernorm_output = self.post_inter_attention_layernorm(layernorm_input) + + # MLP. + mlp_output, mlp_bias = self.mlp(layernorm_output) + + # Second residual connection. + if self.apply_residual_connection_post_layernorm: + residual = layernorm_output + else: + residual = layernorm_input + + if self.drop_path is None: + with self.bias_dropout_add_exec_handler(): + output = bias_dropout_add_func( + mlp_output, + mlp_bias.expand_as(residual), + residual, + self.hidden_dropout, + ) + else: + out = torch.nn.functional.dropout( + mlp_output + mlp_bias, p=self.hidden_dropout, training=self.training + ) + output = residual + self.drop_path(out) + + return output + + +class ParallelTransformer(MegatronModule): + """Transformer class.""" + + def __init__( + self, + init_method, + output_layer_init_method, + layer_type=LayerType.encoder, + self_attn_mask_type=AttnMaskType.padding, + post_layer_norm=True, + pre_process=True, + post_process=True, + drop_path_rate=0.0, + ): + super().__init__() + args = get_args() + + self.layer_type = layer_type + self.model_type = args.model_type + self.bf16 = args.bf16 + self.fp32_residual_connection = args.fp32_residual_connection + self.post_layer_norm = post_layer_norm + self.pre_process = pre_process + self.post_process = post_process + self.input_tensor = None + self.drop_path_rate = drop_path_rate + + # Store activation checkpoiting flag. + self.recompute_granularity = args.recompute_granularity + self.recompute_method = args.recompute_method + self.recompute_num_layers = args.recompute_num_layers + self.distribute_saved_activations = ( + args.distribute_saved_activations and not args.sequence_parallel + ) + + self.sequence_parallel = args.sequence_parallel + + # Number of layers. + self.num_layers = get_num_layers( + args, args.model_type == ModelType.encoder_and_decoder + ) + + self.drop_path_rates = [ + rate.item() + for rate in torch.linspace(0, self.drop_path_rate, args.num_layers) + ] + + # Transformer layers. + def build_layer(layer_number): + return ParallelTransformerLayer( + init_method, + output_layer_init_method, + layer_number, + layer_type=layer_type, + self_attn_mask_type=self_attn_mask_type, + drop_path_rate=self.drop_path_rates[layer_number - 1], + ) + + if args.virtual_pipeline_model_parallel_size is not None: + assert args.num_layers % args.virtual_pipeline_model_parallel_size == 0, ( + "num_layers_per_stage must be divisible by " + "virtual_pipeline_model_parallel_size" + ) + assert args.model_type != ModelType.encoder_and_decoder + # Number of layers in each model chunk is the number of layers in the stage, + # divided by the number of model chunks in a stage. + self.num_layers = ( + self.num_layers // args.virtual_pipeline_model_parallel_size + ) + # With 8 layers, 2 stages, and 4 model chunks, we want an assignment of + # layers to stages like (each list is a model chunk): + # Stage 0: [0] [2] [4] [6] + # Stage 1: [1] [3] [5] [7] + # With 8 layers, 2 stages, and 2 virtual stages, we want an assignment of + # layers to stages like (each list is a model chunk): + # Stage 0: [0, 1] [4, 5] + # Stage 1: [2, 3] [6, 7] + offset = parallel_state.get_virtual_pipeline_model_parallel_rank() * ( + args.num_layers // args.virtual_pipeline_model_parallel_size + ) + (parallel_state.get_pipeline_model_parallel_rank() * self.num_layers) + else: + # Each stage gets a contiguous set of layers. + if ( + args.model_type == ModelType.encoder_and_decoder + and parallel_state.get_pipeline_model_parallel_world_size() > 1 + ): + pipeline_rank = parallel_state.get_pipeline_model_parallel_rank() + if layer_type == LayerType.encoder: + offset = pipeline_rank * self.num_layers + else: + num_ranks_in_enc = args.pipeline_model_parallel_split_rank + offset = (pipeline_rank - num_ranks_in_enc) * self.num_layers + else: + offset = ( + parallel_state.get_pipeline_model_parallel_rank() * self.num_layers + ) + + if self.num_layers == 0: + # When a standalone embedding stage is used (e.g., + # args.standalone_embedding_stage == True), virtual pipeline ranks + # on pipeline rank 0 will have zero transformer layers assigned to + # them. This results in the model's input and output tensors to be + # the same, which will cause failure for certain output tensor + # optimizations (e.g., pipeline output deallocation). To remedy + # this, we assign a 'no-op' layer on these ranks, which will + # disconnect the input tensor from the output tensor. + self.num_layers = 1 + self.layers = torch.nn.ModuleList([NoopTransformerLayer(1)]) + else: + self.layers = torch.nn.ModuleList( + [build_layer(i + 1 + offset) for i in range(self.num_layers)] + ) + + if self.post_process and self.post_layer_norm: + # Final layer norm before output. + self.final_layernorm = LayerNorm( + args.hidden_size, + eps=args.layernorm_epsilon, + # no_persist_layer_norm=args.no_persist_layer_norm, + sequence_parallel_enabled=args.sequence_parallel, + ) + + def _get_layer(self, layer_number): + return self.layers[layer_number] + + def _checkpointed_forward( + self, hidden_states, attention_mask, encoder_output, enc_dec_attn_mask + ): + """Forward method with activation checkpointing.""" + + def custom(start, end): + def custom_forward(*inputs): + x_ = inputs[0] + attention_mask = inputs[1] + encoder_output = inputs[2] + enc_dec_attn_mask = inputs[3] + for index in range(start, end): + layer = self._get_layer(index) + x_ = layer(x_, attention_mask, encoder_output, enc_dec_attn_mask) + return x_ + + return custom_forward + + if self.recompute_method == "uniform": + # Uniformly divide the total number of Transformer layers and checkpoint + # the input activation of each divided chunk. + # A method to further reduce memory usage reducing checkpoints. + l = 0 + while l < self.num_layers: + hidden_states = tensor_parallel.random.checkpoint( + custom(l, l + self.recompute_num_layers), + self.distribute_saved_activations, + hidden_states, + attention_mask, + encoder_output, + enc_dec_attn_mask, + ) + l += self.recompute_num_layers + + elif self.recompute_method == "block": + # Checkpoint the input activation of only a set number of individual + # Transformer layers and skip the rest. + # A method fully use the device memory removing redundant re-computation. + for l in range(self.num_layers): + if l < self.recompute_num_layers: + hidden_states = tensor_parallel.random.checkpoint( + custom(l, l + 1), + self.distribute_saved_activations, + hidden_states, + attention_mask, + encoder_output, + enc_dec_attn_mask, + ) + else: + hidden_states = custom(l, l + 1)( + hidden_states, attention_mask, encoder_output, enc_dec_attn_mask + ) + else: + raise ValueError("Invalid activation recompute method.") + + return hidden_states + + def set_input_tensor(self, input_tensor): + """Set input tensor to be used instead of forward()'s input. + + When doing pipeline parallelism the input from the previous + stage comes from communication, not from the input, so the + model's forward_step_func won't have it. This function is thus + used by internal code to bypass the input provided by the + forward_step_func""" + self.input_tensor = input_tensor + + def forward( + self, + hidden_states, + attention_mask, + encoder_output=None, + enc_dec_attn_mask=None, + inference_params=None, + ): + # hidden_states: [s, b, h] + + # Checks. + if inference_params: + assert ( + self.recompute_granularity is None + ), "inference does not work with activation checkpointing" + + if not self.pre_process: + # See set_input_tensor() + hidden_states = self.input_tensor + + # Viewless tensor. + # - We only need to create a viewless tensor in the case of micro batch + # size (mbs) == 1, since in this case, 'hidden_states.transpose()' + # above creates a view tensor, and '.contiguous()' is a pass-through. + # For mbs >= 2, '.contiguous()' creates a new tensor, eliminating + # the need to make it viewless. + # + # However, we don't explicitly check mbs == 1 here because + # make_viewless_tensor() has negligible overhead when its input + # is already viewless. + # + # - For the 'else' case above, calling make_viewless_tensor() here is + # likely redundant, since p2p_communication.py (likely originator) + # already creates viewless tensors. That said, make_viewless_tensor() + # is called here to be future-proof and corner-case-proof. + # hidden_states = mpu.make_viewless_tensor(hidden_states, requires_grad=True, keep_graph=True) + + if self.sequence_parallel: + rng_context = tensor_parallel.get_cuda_rng_tracker().fork() + else: + rng_context = contextlib.nullcontext() + + with rng_context: + # Forward pass. + if self.recompute_granularity == "full": + hidden_states = self._checkpointed_forward( + hidden_states, attention_mask, encoder_output, enc_dec_attn_mask + ) + else: + for index in range(self.num_layers): + layer = self._get_layer(index) + hidden_states = layer( + hidden_states, + attention_mask, + encoder_output=encoder_output, + enc_dec_attn_mask=enc_dec_attn_mask, + inference_params=inference_params, + ) + + # Final layer norm. + if self.post_process and self.post_layer_norm: + hidden_states = self.final_layernorm(hidden_states) + + return hidden_states + + +def get_num_layers(args, is_encoder_and_decoder_model): + """Compute the number of transformer layers resident on the current rank.""" + if parallel_state.get_pipeline_model_parallel_world_size() > 1: + if is_encoder_and_decoder_model: + assert args.pipeline_model_parallel_split_rank is not None + + # When a standalone embedding stage is used, a rank is taken from + # the encoder's ranks, to be used for the encoder's embedding + # layer. This way, the rank referenced by the 'split rank' remains + # the same whether or not a standalone embedding stage is used. + num_ranks_in_encoder = ( + args.pipeline_model_parallel_split_rank - 1 + if args.standalone_embedding_stage + else args.pipeline_model_parallel_split_rank + ) + num_ranks_in_decoder = ( + args.transformer_pipeline_model_parallel_size - num_ranks_in_encoder + ) + assert args.num_layers % num_ranks_in_encoder == 0, ( + "num_layers (%d) must be divisible by number of ranks given to encoder (%d)" + % ( + args.num_layers, + num_ranks_in_encoder, + ) + ) + assert args.num_layers % num_ranks_in_decoder == 0, ( + "num_layers (%d) must be divisible by number of ranks given to decoder (%d)" + % ( + args.num_layers, + num_ranks_in_decoder, + ) + ) + if parallel_state.is_pipeline_stage_before_split(): + num_layers = ( + 0 + if args.standalone_embedding_stage + and parallel_state.get_pipeline_model_parallel_rank() == 0 + else args.num_layers // num_ranks_in_encoder + ) + else: + num_layers = args.num_layers // num_ranks_in_decoder + else: + assert ( + args.num_layers % args.transformer_pipeline_model_parallel_size == 0 + ), "num_layers must be divisible by transformer_pipeline_model_parallel_size" + + # When a standalone embedding stage is used, all transformer layers + # are divided among pipeline rank >= 1, while on pipeline rank 0, + # ranks either contain the input embedding layer (virtual pp rank 0), + # or no layers at all (virtual pp rank >= 1). + num_layers = ( + 0 + if args.standalone_embedding_stage + and parallel_state.get_pipeline_model_parallel_rank() == 0 + else args.num_layers // args.transformer_pipeline_model_parallel_size + ) + else: + num_layers = args.num_layers + return num_layers + + +class NoopTransformerLayer(MegatronModule): + """A single 'no-op' transformer layer. + + The sole purpose of this layer is for when a standalone embedding layer + is used (i.e., args.standalone_embedding_stage == True). In this case, + zero transformer layers are assigned when pipeline rank == 0. Additionally, + when virtual pipeline rank >= 1, zero total model parameters are created + (virtual rank 0 contains the input embedding). This results in the model's + input and output tensors being the same, which causes an error when + performing certain memory optimiations on the output tensor (e.g., + deallocating it). Thus, this layer disconnects the input from the output + via a clone. Since ranks containing a no-op layer are generally under- + utilized (both compute and memory), there's no worry of any performance + degredation. + """ + + def __init__(self, layer_number): + super().__init__() + self.layer_number = layer_number + + def forward( + self, + hidden_states, + attention_mask, + encoder_output=None, + enc_dec_attn_mask=None, + inference_params=None, + ): + return hidden_states.clone() + + +def parallel_lm_logits(input_, word_embeddings_weight, parallel_output, bias=None): + """LM logits using word embedding weights.""" + args = get_args() + # Parallel logits. + if args.async_tensor_model_parallel_allreduce or args.sequence_parallel: + input_parallel = input_ + model_parallel = parallel_state.get_tensor_model_parallel_world_size() > 1 + async_grad_allreduce = ( + args.async_tensor_model_parallel_allreduce + and model_parallel + and not args.sequence_parallel + ) + else: + input_parallel = tensor_parallel.copy_to_tensor_model_parallel_region(input_) + async_grad_allreduce = False + + # Matrix multiply. + # logits_parallel = tensor_parallel.layers.LinearWithGradAccumulationAndAsyncCommunication.apply( + # input_parallel, word_embeddings_weight, bias, args.gradient_accumulation_fusion, async_grad_allreduce, args.sequence_parallel) + logits_parallel = ( + tensor_parallel.layers.linear_with_grad_accumulation_and_async_allreduce( + input_parallel, + word_embeddings_weight, + bias, + args.gradient_accumulation_fusion, + async_grad_allreduce, + args.sequence_parallel, + ) + ) + # Gather if needed. + + if parallel_output: + return logits_parallel + + return tensor_parallel.gather_from_tensor_model_parallel_region(logits_parallel) + + +def get_language_model( + num_tokentypes, + add_pooler, + encoder_attn_mask_type, + init_method=None, + scaled_init_method=None, + add_encoder=True, + add_decoder=False, + decoder_attn_mask_type=AttnMaskType.causal, + pre_process=True, + post_process=True, +): + """Build language model and return along with the key to save.""" + args = get_args() + + if init_method is None: + init_method = init_method_normal(args.init_method_std) + if scaled_init_method is None: + scaled_init_method = scaled_init_method_normal( + args.init_method_std, args.num_layers + ) + + # Language model. + language_model = TransformerLanguageModel( + init_method, + scaled_init_method, + encoder_attn_mask_type, + num_tokentypes=num_tokentypes, + add_encoder=add_encoder, + add_decoder=add_decoder, + decoder_attn_mask_type=decoder_attn_mask_type, + add_pooler=add_pooler, + pre_process=pre_process, + post_process=post_process, + ) + # key used for checkpoints. + language_model_key = "language_model" + + return language_model, language_model_key + + +class Pooler(MegatronModule): + """Pooler layer. + + Pool hidden states of a specific token (for example start of the + sequence) and add a linear transformation followed by a tanh. + + Arguments: + hidden_size: hidden size + init_method: weight initialization method for the linear layer. + bias is set to zero. + """ + + def __init__(self, hidden_size, init_method): + super().__init__() + args = get_args() + self.dense = get_linear_layer(hidden_size, hidden_size, init_method) + self.sequence_parallel = args.sequence_parallel + + def forward(self, hidden_states, sequence_index=0): + # hidden_states: [s, b, h] + # sequence_index: index of the token to pool. + # gather data along sequence dimensions + # same pooler is run on all tensor parallel nodes + if self.sequence_parallel: + hidden_states = tensor_parallel.mappings.gather_from_sequence_parallel_region(hidden_states) + pooled = hidden_states[sequence_index, :, :] + pooled = self.dense(pooled) + pooled = torch.tanh(pooled) + return pooled + + +class Embedding(MegatronModule): + """Language model embeddings. + + Arguments: + hidden_size: hidden size + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + init_method: weight initialization method + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__( + self, + hidden_size, + vocab_size, + max_sequence_length, + embedding_dropout_prob, + init_method, + num_tokentypes=0, + ): + super().__init__() + + self.hidden_size = hidden_size + self.init_method = init_method + self.num_tokentypes = num_tokentypes + + args = get_args() + + # Word embeddings (parallel). + self.word_embeddings = VocabParallelEmbedding( + vocab_size, self.hidden_size, init_method=self.init_method + ) + self._word_embeddings_key = "word_embeddings" + + # Position embedding (serial). + self.position_embeddings = torch.nn.Embedding( + max_sequence_length, self.hidden_size + ) + self._position_embeddings_key = "position_embeddings" + # Initialize the position embeddings. + self.init_method(self.position_embeddings.weight) + + # Token type embedding. + # Add this as an optional field that can be added through + # method call so we can load a pretrain model without + # token types and add them as needed. + self._tokentype_embeddings_key = "tokentype_embeddings" + if self.num_tokentypes > 0: + self.tokentype_embeddings = torch.nn.Embedding( + self.num_tokentypes, self.hidden_size + ) + # Initialize the token-type embeddings. + self.init_method(self.tokentype_embeddings.weight) + else: + self.tokentype_embeddings = None + + self.fp32_residual_connection = args.fp32_residual_connection + self.sequence_parallel = args.sequence_parallel + # Embeddings dropout + self.embedding_dropout = torch.nn.Dropout(embedding_dropout_prob) + + def zero_parameters(self): + """Zero out all parameters in embedding.""" + self.word_embeddings.weight.data.fill_(0) + self.word_embeddings.weight.shared = True + self.position_embeddings.weight.data.fill_(0) + self.position_embeddings.weight.shared = True + if self.num_tokentypes > 0: + self.tokentype_embeddings.weight.fill_(0) + self.tokentype_embeddings.weight.shared = True + + def add_tokentype_embeddings(self, num_tokentypes): + """Add token-type embedding. This function is provided so we can add + token-type embeddings in case the pretrained model does not have it. + This allows us to load the model normally and then add this embedding. + """ + if self.tokentype_embeddings is not None: + raise Exception("tokentype embeddings is already initialized") + if torch.distributed.get_rank() == 0: + print( + "adding embedding for {} tokentypes".format(num_tokentypes), flush=True + ) + self.num_tokentypes = num_tokentypes + self.tokentype_embeddings = torch.nn.Embedding(num_tokentypes, self.hidden_size) + # Initialize the token-type embeddings. + self.init_method(self.tokentype_embeddings.weight) + + def forward(self, input_ids, position_ids, tokentype_ids=None): + # Embeddings. + words_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + embeddings = words_embeddings + position_embeddings + if tokentype_ids is not None: + assert self.tokentype_embeddings is not None + embeddings = embeddings + self.tokentype_embeddings(tokentype_ids) + else: + assert self.tokentype_embeddings is None + + # Data format change to avoid explicit tranposes : [b s h] --> [s b h]. + embeddings = embeddings.transpose(0, 1).contiguous() + + # If the input flag for fp32 residual connection is set, convert for float. + if self.fp32_residual_connection: + embeddings = embeddings.float() + + # Dropout. + if self.sequence_parallel: + embeddings = scatter_to_sequence_parallel_region(embeddings) + with tensor_parallel.get_cuda_rng_tracker().fork(): + embeddings = self.embedding_dropout(embeddings) + else: + embeddings = self.embedding_dropout(embeddings) + + return embeddings + + +class TransformerLanguageModel(MegatronModule): + """Transformer language model. + + Arguments: + transformer_hparams: transformer hyperparameters + vocab_size: vocabulary size + max_sequence_length: maximum size of sequence. This + is used for positional embedding + embedding_dropout_prob: dropout probability for embeddings + num_tokentypes: size of the token-type embeddings. 0 value + will ignore this embedding + """ + + def __init__( + self, + init_method, + output_layer_init_method, + encoder_attn_mask_type, + num_tokentypes=0, + add_encoder=True, + add_decoder=False, + decoder_attn_mask_type=AttnMaskType.causal, + add_pooler=False, + pre_process=True, + post_process=True, + ): + super().__init__() + args = get_args() + + self.pre_process = pre_process + self.post_process = post_process + self.hidden_size = args.hidden_size + self.num_tokentypes = num_tokentypes + self.init_method = init_method + self.add_encoder = add_encoder + self.encoder_attn_mask_type = encoder_attn_mask_type + self.add_decoder = add_decoder + self.decoder_attn_mask_type = decoder_attn_mask_type + self.add_pooler = add_pooler + self.encoder_hidden_state = None + + # Embeddings. + if self.pre_process: + self.embedding = Embedding( + self.hidden_size, + args.padded_vocab_size, + args.max_position_embeddings, + args.hidden_dropout, + self.init_method, + self.num_tokentypes, + ) + self._embedding_key = "embedding" + + # Transformer. + # Encoder (usually set to True, False if part of an encoder-decoder + # architecture and in encoder-only stage). + if self.add_encoder: + self.encoder = ParallelTransformer( + self.init_method, + output_layer_init_method, + self_attn_mask_type=self.encoder_attn_mask_type, + pre_process=self.pre_process, + post_process=self.post_process, + ) + self._encoder_key = "encoder" + else: + self.encoder = None + + # Decoder (usually set to False, True if part of an encoder-decoder + # architecture and in decoder-only stage). + if self.add_decoder: + self.decoder = ParallelTransformer( + self.init_method, + output_layer_init_method, + layer_type=LayerType.decoder, + self_attn_mask_type=self.decoder_attn_mask_type, + pre_process=self.pre_process, + post_process=self.post_process, + ) + self._decoder_key = "decoder" + else: + self.decoder = None + + if self.post_process: + # Pooler. + if self.add_pooler: + self.pooler = Pooler(self.hidden_size, self.init_method) + self._pooler_key = "pooler" + + def set_input_tensor(self, input_tensor): + """See megatron.model.transformer.set_input_tensor()""" + + # This is usually handled in schedules.py but some inference code still + # gives us non-lists or None + if not isinstance(input_tensor, list): + input_tensor = [input_tensor] + + if self.add_encoder and self.add_decoder: + assert ( + len(input_tensor) == 1 + ), "input_tensor should only be length 1 for stage with both encoder and decoder" + self.encoder.set_input_tensor(input_tensor[0]) + elif self.add_encoder: + assert ( + len(input_tensor) == 1 + ), "input_tensor should only be length 1 for stage with only encoder" + self.encoder.set_input_tensor(input_tensor[0]) + elif self.add_decoder: + if len(input_tensor) == 2: + self.decoder.set_input_tensor(input_tensor[0]) + self.encoder_hidden_state = input_tensor[1] + elif len(input_tensor) == 1: + self.decoder.set_input_tensor(None) + self.encoder_hidden_state = input_tensor[0] + else: + raise Exception("input_tensor must have either length 1 or 2") + else: + raise Exception("Stage must have at least either encoder or decoder") + + def forward( + self, + enc_input_ids, + enc_position_ids, + enc_attn_mask, + dec_input_ids=None, + dec_position_ids=None, + dec_attn_mask=None, + enc_dec_attn_mask=None, + tokentype_ids=None, + inference_params=None, + pooling_sequence_index=0, + enc_hidden_states=None, + output_enc_hidden=False, + ): + + args = get_args() + # Encoder embedding. + if self.pre_process: + encoder_input = self.embedding( + enc_input_ids, enc_position_ids, tokentype_ids=tokentype_ids + ) + else: + encoder_input = None + + # Run encoder. + if enc_hidden_states is None: + if self.encoder is not None: + encoder_output = self.encoder( + encoder_input, enc_attn_mask, inference_params=inference_params + ) + else: + encoder_output = self.encoder_hidden_state + else: + encoder_output = enc_hidden_states.to(encoder_input.dtype) + + if self.post_process: + if self.add_pooler: + pooled_output = self.pooler(encoder_output, pooling_sequence_index) + + # output_enc_hidden refers to when we just need the encoder's + # output. For example, it is helpful to compute + # similarity between two sequences by average pooling + if not self.add_decoder or output_enc_hidden: + if self.add_pooler and self.post_process: + return encoder_output, pooled_output + else: + return encoder_output + + # Decoder embedding. + if self.pre_process: + decoder_input = self.embedding(dec_input_ids, dec_position_ids) + else: + decoder_input = None + + # Run decoder. + decoder_output = self.decoder( + decoder_input, + dec_attn_mask, + encoder_output=encoder_output, + enc_dec_attn_mask=enc_dec_attn_mask, + inference_params=inference_params, + ) + + if self.add_pooler and self.post_process: + return decoder_output, encoder_output, pooled_output + else: + return decoder_output, encoder_output + + +def post_language_model_processing( + lm_output, labels, logit_weights, parallel_output, fp16_lm_cross_entropy +): + # Output. + output = parallel_lm_logits(lm_output, logit_weights, parallel_output) + + if labels is None: + return output + else: + if fp16_lm_cross_entropy: + assert output.dtype == torch.half + loss = tensor_parallel.vocab_parallel_cross_entropy(output, labels) + else: + loss = tensor_parallel.vocab_parallel_cross_entropy(output.float(), labels) + return loss + + +def module_size(m: torch.nn.Module, only_trainable: bool = False): + """ + returns the total number of parameters used by `m` (only counting + shared parameters once); if `only_trainable` is True, then only + includes parameters with `requires_grad = True` + """ + parameters = list(m.parameters()) + if only_trainable: + parameters = [p for p in parameters if p.requires_grad] + unique = {p.data_ptr(): p for p in parameters}.values() + return sum(p.numel() for p in unique) diff --git a/apex/apex/transformer/utils.py b/apex/apex/transformer/utils.py new file mode 100644 index 00000000..4434e360 --- /dev/null +++ b/apex/apex/transformer/utils.py @@ -0,0 +1,54 @@ +"""Utility functions used by both `pipeline_parallel` and `tensor_parallel`""" +import torch + +from apex.transformer import parallel_state + +# `all_gather_into_tensor` is new placeholders for `_all_gather_base`. +# It requires the most recent version of PyTorch. +# The following 4 lines are for backward comparability with +# older PyTorch. +if "all_gather_into_tensor" not in dir(torch.distributed): + torch.distributed.all_gather_into_tensor = torch.distributed._all_gather_base + +def ensure_divisibility(numerator, denominator): + """Ensure that numerator is divisible by the denominator.""" + assert numerator % denominator == 0, "{} is not divisible by {}".format( + numerator, denominator + ) + + +def divide(numerator, denominator): + """Ensure that numerator is divisible by the denominator and return + the division value.""" + ensure_divisibility(numerator, denominator) + return numerator // denominator + + +def split_tensor_into_1d_equal_chunks(tensor): + """Break a tensor into equal 1D chunks.""" + data = tensor.view(-1) + partition_size = ( + torch.numel(data) // parallel_state.get_tensor_model_parallel_world_size() + ) + start_index = partition_size * parallel_state.get_tensor_model_parallel_rank() + end_index = start_index + partition_size + return data[start_index:end_index] + + +def gather_split_1d_tensor(tensor): + """Opposite of above function, gather values from model parallel ranks.""" + world_size = parallel_state.get_tensor_model_parallel_world_size() + numel = torch.numel(tensor) + numel_gathered = world_size * numel + gathered = torch.empty( + numel_gathered, + dtype=tensor.dtype, + device=torch.cuda.current_device(), + requires_grad=False, + ) + torch.distributed.all_gather_into_tensor( + gathered, + tensor, + group=parallel_state.get_tensor_model_parallel_group() + ) + return gathered diff --git a/apex/csrc/amp_C_frontend.cpp b/apex/csrc/amp_C_frontend.cpp new file mode 100644 index 00000000..32b58b5f --- /dev/null +++ b/apex/csrc/amp_C_frontend.cpp @@ -0,0 +1,205 @@ +#include + +void multi_tensor_scale_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float scale); + +void multi_tensor_sgd_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float wd, + float momentum, + float dampening, + float lr, + bool nesterov, + bool first_run, + bool wd_after_momentum, + float scale); + +void multi_tensor_axpby_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float a, + float b, + int arg_to_check); + +std::tuple multi_tensor_l2norm_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python); + +std::tuple multi_tensor_l2norm_mp_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python); + +std::tuple multi_tensor_l2norm_scale_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float scale, + at::optional per_tensor_python); + +void multi_tensor_lamb_stage1_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_decay, + const int step, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor global_grad_norm, + const float max_global_grad_norm); + +void multi_tensor_lamb_stage2_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_param_norm, + at::Tensor per_tensor_update_norm, + const float lr, + const float weight_decay, + at::optional use_nvlamb_python); + +void multi_tensor_adam_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int mode, + const int bias_correction, + const float weight_decay); + +void multi_tensor_adam_capturable_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor lr, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor step, + const int mode, + const int bias_correction, + const float weight_decay, + at::Tensor inv_scale); + +void multi_tensor_adam_capturable_master_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor lr, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor step, + const int mode, + const int bias_correction, + const float weight_decay, + at::Tensor inv_scale); + +void multi_tensor_adagrad_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float epsilon, + const int mode, + const float weight_decay); + + +void multi_tensor_novograd_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor grad_norms, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + const int norm_type); + +void multi_tensor_lamb_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + at::Tensor global_grad_norm, + const float max_grad_norm, + at::optional use_nvlamb_python); + +void multi_tensor_lamb_mp_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor lr, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + at::Tensor global_grad_norm, + at::Tensor max_grad_norm, + at::optional use_nvlamb_python, + at::Tensor found_inf, + at::Tensor inv_scale); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("multi_tensor_scale", &multi_tensor_scale_cuda, + "Fused overflow check + scale for a list of contiguous tensors"); + m.def("multi_tensor_sgd", &multi_tensor_sgd_cuda, + "Fused SGD optimizer for list of contiguous tensors"); + m.def("multi_tensor_axpby", &multi_tensor_axpby_cuda, + "out = a*x + b*y for a list of contiguous tensors"); + m.def("multi_tensor_l2norm", &multi_tensor_l2norm_cuda, + "Computes L2 norm for a list of contiguous tensors"); + m.def("multi_tensor_l2norm_mp", &multi_tensor_l2norm_mp_cuda, + "Computes L2 norm for a list of contiguous tensors"); + m.def("multi_tensor_l2norm_scale", &multi_tensor_l2norm_scale_cuda, + "Computes L2 norm for a list of contiguous tensors and does scaling"); + m.def("multi_tensor_lamb_stage1_cuda", &multi_tensor_lamb_stage1_cuda, + "Computes update part of LAMB optimizer"); + m.def("multi_tensor_lamb_stage2_cuda", &multi_tensor_lamb_stage2_cuda, + "Completes application of gradient to parameters for LAMB optimizer"); + m.def("multi_tensor_adam", &multi_tensor_adam_cuda, + "Compute and apply gradient update to parameters for Adam optimizer"); + m.def("multi_tensor_adam_capturable", &multi_tensor_adam_capturable_cuda, + "Compute and apply gradient update to parameters for Adam optimizer with CUDA graph support and LR scheduling"); + m.def("multi_tensor_adam_capturable_master", &multi_tensor_adam_capturable_master_cuda, + "Compute and apply gradient update to parameters for Adam optimizer with CUDA graph support, LR scheduling and FP32 master weights"); + m.def("multi_tensor_adagrad", &multi_tensor_adagrad_cuda, + "Compute and apply gradient update to parameters for Adam optimizer"); + m.def("multi_tensor_novograd", &multi_tensor_novograd_cuda, + "Compute and apply gradient update to parameters for Adam optimizer"); + m.def("multi_tensor_lamb", &multi_tensor_lamb_cuda, + "Computes and apply update for LAMB optimizer"); + m.def("multi_tensor_lamb_mp", &multi_tensor_lamb_mp_cuda, + "Computes and apply update for LAMB optimizer"); +} diff --git a/apex/csrc/compat.h b/apex/csrc/compat.h new file mode 100644 index 00000000..acafb056 --- /dev/null +++ b/apex/csrc/compat.h @@ -0,0 +1,9 @@ +#ifndef TORCH_CHECK +#define TORCH_CHECK AT_CHECK +#endif + +#ifdef VERSION_GE_1_3 +#define DATA_PTR data_ptr +#else +#define DATA_PTR data +#endif diff --git a/apex/csrc/flatten_unflatten.cpp b/apex/csrc/flatten_unflatten.cpp new file mode 100644 index 00000000..d49ce759 --- /dev/null +++ b/apex/csrc/flatten_unflatten.cpp @@ -0,0 +1,18 @@ +#include +#include +// https://github.com/pytorch/pytorch/blob/master/torch/csrc/utils/tensor_flatten.h + +at::Tensor flatten(std::vector tensors) +{ + return torch::utils::flatten_dense_tensors(tensors); +} + +std::vector unflatten(at::Tensor flat, std::vector tensors) +{ + return torch::utils::unflatten_dense_tensors(flat, tensors); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("flatten", &flatten, "Flatten dense tensors"); + m.def("unflatten", &unflatten, "Unflatten dense tensors"); +} diff --git a/apex/csrc/fused_dense.cpp b/apex/csrc/fused_dense.cpp new file mode 100644 index 00000000..7e99b460 --- /dev/null +++ b/apex/csrc/fused_dense.cpp @@ -0,0 +1,193 @@ +#include +#include +#include + +#include + + +template +int linear_bias_forward_cuda(at::Tensor input, T *weight, at::Tensor bias, int in_features, int batch_size, int out_features, at::Tensor output, void *lt_workspace); + +template +int linear_bias_backward_cuda(T *input, T *weight, T *d_output, int in_features, int batch_size, int out_features, T *d_weight, T *d_bias, T *d_input, void *lt_workspace); + +template +int linear_gelu_linear_forward_cuda(T *input, T *weight1, T *bias1, T *weight2, T *bias2, int in_features, int hidden_features, int batch_size, int out_features, T *output1, T *output2, T *gelu_in, void *lt_workspace) ; + +template +int linear_gelu_linear_backward_cuda(T *input, T *gelu_in, T *output1, T *weight1, T *weight2, T *d_output1, T *d_output2, int in_features, int batch_size, int hidden_features, int out_features, T *d_weight1, T *d_weight2, T *d_bias1, T *d_bias2, T *d_input, void *lt_workspace); + +at::Tensor linear_bias_forward(at::Tensor input, at::Tensor weight, at::Tensor bias) { + + auto batch_size = input.size(0); + auto in_features = input.size(1); + + int out_features = weight.size(0); + + //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); + + // create output/workspace tensor + auto out = at::empty({batch_size, out_features}, input.type()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB + auto lt_workspace = at::empty({1 << 22}, input.type()); + + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_bias_forward", [&] { + scalar_t* w_ptr = weight.data_ptr(); + scalar_t* b_ptr = bias.data_ptr(); + auto result = linear_bias_forward_cuda( + input, + w_ptr, + bias, + in_features, + batch_size, + out_features, + out, + //out.data_ptr(), + // reserved_space.data_ptr(), + (void*) (lt_workspace.data_ptr())); + }); + + return {out}; +} + +std::vector linear_bias_backward(at::Tensor input, at::Tensor weight, at::Tensor d_output) { + + auto batch_size = input.size(0); + auto in_features = input.size(1); + + int out_features = weight.size(0); + + //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); + + // create output/workspace tensor + auto d_weight = at::empty({out_features, in_features}, input.type()); +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION < 11600 + auto d_bias = d_output.view({-1, out_features}).sum(0, false); +#else + auto d_bias = at::empty({out_features}, input.type()); +#endif + auto d_input = at::empty({batch_size, in_features}, input.type()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB + auto lt_workspace = at::empty({1 << 22}, input.type()); + + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_bias_backward", [&] { + scalar_t* w_ptr = weight.data_ptr(); + scalar_t* d_b_ptr = d_bias.data_ptr(); + auto result = linear_bias_backward_cuda( + input.data_ptr(), + w_ptr, + d_output.data_ptr(), + in_features, + batch_size, + out_features, + d_weight.data_ptr(), + d_bias.data_ptr(), + d_input.data_ptr(), + // reserved_space.data_ptr(), + (void*) (lt_workspace.data_ptr())); + }); + + return {d_input, d_weight, d_bias}; +} + +std::vector linear_gelu_linear_forward(at::Tensor input, at::Tensor weight1, at::Tensor bias1, at::Tensor weight2, at::Tensor bias2) { + + auto batch_size = input.size(0); + auto in_features = input.size(1); + + int hidden_features = weight1.size(0); + int out_features = weight2.size(0); + + //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); + + // create output/workspace tensor + auto output1 = at::empty({batch_size, hidden_features}, input.type()); + auto gelu_in = at::empty({batch_size, hidden_features}, input.type()); + auto output2 = at::empty({batch_size, out_features}, input.type()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB + auto lt_workspace = at::empty({1 << 22}, input.type()); + + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_gelu_linear_forward", [&] { + scalar_t* w1_ptr = weight1.data_ptr(); + scalar_t* b1_ptr = bias1.data_ptr(); + scalar_t* w2_ptr = weight2.data_ptr(); + scalar_t* b2_ptr = bias2.data_ptr(); + auto result = linear_gelu_linear_forward_cuda( + input.data_ptr(), + w1_ptr, + b1_ptr, + w2_ptr, + b2_ptr, + in_features, + hidden_features, + batch_size, + out_features, + output1.data_ptr(), + output2.data_ptr(), + gelu_in.data_ptr(), + // reserved_space.data_ptr(), + (void*) (lt_workspace.data_ptr())); + }); + + return {output1, output2, gelu_in}; +} + +std::vector linear_gelu_linear_backward(at::Tensor input, at::Tensor gelu_in, at::Tensor output1, at::Tensor weight1, at::Tensor weight2, at::Tensor d_output2) { + + auto batch_size = input.size(0); + auto in_features = input.size(1); + + int hidden_features = weight1.size(0); + int out_features = weight2.size(0); + + //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); + + // create output/workspace tensor + auto d_weight1 = at::empty({hidden_features, in_features}, input.type()); + auto d_weight2 = at::empty({out_features, hidden_features}, input.type()); + auto d_bias1 = at::empty({hidden_features}, input.type()); + auto d_bias2 = at::empty({out_features}, input.type()); + auto d_input = at::empty({batch_size, in_features}, input.type()); + auto d_output1 = at::empty({batch_size, hidden_features}, input.type()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB + auto lt_workspace = at::empty({1 << 22}, input.type()); + + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_bias_backward", [&] { + + //scalar_t* w_ptr = weight.data_ptr(); + //scalar_t* d_b_ptr = d_bias.data_ptr(); + auto result = linear_gelu_linear_backward_cuda( + input.data_ptr(), + gelu_in.data_ptr(), + output1.data_ptr(), + weight1.data_ptr(), + weight2.data_ptr(), + d_output1.data_ptr(), + d_output2.data_ptr(), + in_features, + batch_size, + hidden_features, + out_features, + d_weight1.data_ptr(), + d_weight2.data_ptr(), + d_bias1.data_ptr(), + d_bias2.data_ptr(), + d_input.data_ptr(), + // reserved_space.data_ptr(), + (void*) (lt_workspace.data_ptr())); + }); + + return {d_input, d_weight1, d_bias1, d_weight2, d_bias2}; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("linear_bias_forward", &linear_bias_forward, "linear bias forward"); + m.def("linear_bias_backward", &linear_bias_backward, "linear bias backward"); + m.def("linear_gelu_linear_forward", &linear_gelu_linear_forward, "linear gelu linear forward"); + m.def("linear_gelu_linear_backward", &linear_gelu_linear_backward, "linear gelu linear backward"); +} + diff --git a/apex/csrc/fused_dense_cuda.cu b/apex/csrc/fused_dense_cuda.cu new file mode 100644 index 00000000..ff754fdb --- /dev/null +++ b/apex/csrc/fused_dense_cuda.cu @@ -0,0 +1,1930 @@ +#include +#include +#include +#include +#include +#include +#include + +/* Includes, cuda */ +#include +#include + +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11000 +// includes cublaslt +#include +#endif +// FP64 Wrapper around cublas GEMMEx +cublasStatus_t gemm_bias( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + double* A, + int lda, + double* B, + int ldb, + const float* beta, + double* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_64F, + lda, + B, + CUDA_R_64F, + ldb, + beta, + C, + CUDA_R_64F, + ldc, + CUDA_R_64F, + CUBLAS_GEMM_DEFAULT); +} + +// FP32 Wrapper around cublas GEMMEx +cublasStatus_t gemm_bias( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + float* A, + int lda, + float* B, + int ldb, + const float* beta, + float* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_32F, + lda, + B, + CUDA_R_32F, + ldb, + beta, + C, + CUDA_R_32F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT); +} + +// FP16 Tensor core wrapper around cublas GEMMEx +cublasStatus_t gemm_bias( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float* beta, + at::Half* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16F, + lda, + B, + CUDA_R_16F, + ldb, + beta, + C, + CUDA_R_16F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP); +} + +// BF16 Tensor core wrapper around cublas GEMMEx +cublasStatus_t gemm_bias( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float* beta, + at::BFloat16* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16BF, + lda, + B, + CUDA_R_16BF, + ldb, + beta, + C, + CUDA_R_16BF, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP); +} + +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11600 + + +int gemm_bias_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float *beta, /* host pointer */ + at::Half* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_BIAS; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + + +int gemm_bias_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float *beta, /* host pointer */ + at::BFloat16* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_BIAS; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16BF, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16BF, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16BF, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + + +int gemm_bias_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + double* A, + int lda, + double* B, + int ldb, + const float *beta, /* host pointer */ + double* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bias) { + return 1; +} + +int gemm_bias_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + float *A, + int lda, + float *B, + int ldb, + const float *beta, /* host pointer */ + float *C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_BIAS; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_32F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_32F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_32F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + &heuristicResult.algo, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + + +int gemm_bias_gelu_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float *beta, /* host pointer */ + at::Half* C, + int64_t ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* gelu_in, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_GELU_AUX; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER, &gelu_in, sizeof(gelu_in)); + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &ldc, sizeof(ldc)); + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_GELU_AUX_BIAS; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + +int gemm_bias_gelu_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float *beta, /* host pointer */ + at::BFloat16* C, + int64_t ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* gelu_in, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_GELU_AUX; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER, &gelu_in, sizeof(gelu_in)); + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &ldc, sizeof(ldc)); + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_GELU_AUX_BIAS; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16BF, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16BF, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16BF, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + +int gemm_bias_gelu_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + double* A, + int lda, + double* B, + int ldb, + const float *beta, /* host pointer */ + double* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void *gelu_in, + const void* bias) { + return 1; +} + + +int gemm_bias_gelu_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + float *A, + int lda, + float *B, + int ldb, + const float *beta, /* host pointer */ + float *C, + int64_t ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* gelu_in, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_GELU_AUX; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER, &gelu_in, sizeof(gelu_in)); + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &ldc, sizeof(ldc)); + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_GELU_AUX_BIAS; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_32F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_32F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_32F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + + +int gemm_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float *beta, /* host pointer */ + at::Half* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bgrad) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bgrad, sizeof(bgrad)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_BGRADB; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + +int gemm_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float *beta, /* host pointer */ + at::BFloat16* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bgrad) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bgrad, sizeof(bgrad)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_BGRADB; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16BF, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16BF, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16BF, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + + + + +int gemm_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + double* A, + int lda, + double* B, + int ldb, + const float *beta, /* host pointer */ + double* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bgrad) { + return 1; +} + +int gemm_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + float *A, + int lda, + float *B, + int ldb, + const float *beta, /* host pointer */ + float *C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + const void* bgrad) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bgrad, sizeof(bgrad)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + epilogue = CUBLASLT_EPILOGUE_BGRADB; + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_32F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_32F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_32F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + &heuristicResult.algo, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + +int gemm_dgelu_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float *beta, /* host pointer */ + at::Half* C, + int64_t ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + const void *gelu_in, + const void *bgrad) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DGELU_BGRAD; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bgrad, sizeof(bgrad)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER, &gelu_in, sizeof(gelu_in)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &ldc, sizeof(ldc)); + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + +int gemm_dgelu_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float *beta, /* host pointer */ + at::BFloat16* C, + int64_t ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + const void *gelu_in, + const void *bgrad) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DGELU_BGRAD; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bgrad, sizeof(bgrad)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER, &gelu_in, sizeof(gelu_in)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &ldc, sizeof(ldc)); + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16BF, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16BF, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16BF, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + + +int gemm_dgelu_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + double *A, + int lda, + double *B, + int ldb, + const float *beta, /* host pointer */ + double *C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + const void *gelu_in, + const void *bgrad) { + return 1; +} + +int gemm_dgelu_bgradb_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, /* host pointer */ + float *A, + int lda, + float *B, + int ldb, + const float *beta, /* host pointer */ + float *C, + int64_t ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + const void *gelu_in, + const void *bgrad) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DGELU_BGRAD; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bgrad, sizeof(bgrad)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_POINTER, &gelu_in, sizeof(gelu_in)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &ldc, sizeof(ldc)); + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_32F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_32F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_32F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + //&heuristicResult.algo, + NULL, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + +#endif + +template +int linear_bias_forward_cuda(at::Tensor input, T *weight, at::Tensor bias, int in_features, int batch_size, int out_features, at::Tensor output, void *lt_workspace) { + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + // Get the stream from cublas handle to reuse for biasReLU kernel. + cudaStream_t stream; + cublasGetStream(handle, &stream); + const float alpha = 1.0; + const float beta_zero = 0.0; + const float beta_one = 1.0; + int status = 1; +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11600 + status = gemm_bias_lt( + (cublasLtHandle_t)handle, + CUBLAS_OP_T, + CUBLAS_OP_N, + out_features, + batch_size, + in_features, + &alpha, /* host pointer */ + weight, + in_features, + input.data_ptr(), + in_features, + &beta_zero, /* host pointer */ + output.data_ptr(), + out_features, + lt_workspace, + 1 << 22, + stream, + true, + static_cast(bias.data_ptr())); +#endif + if (status != 0){ + output.copy_(bias); + status = gemm_bias( + handle, + CUBLAS_OP_T, + CUBLAS_OP_N, + out_features, + batch_size, + in_features, + &alpha, + weight, + in_features, + input.data_ptr(), + in_features, + &beta_one, + output.data_ptr(), + out_features); + } + return status; +} + + +template +int linear_bias_backward_cuda(T *input, T *weight, T *d_output, int in_features, int batch_size, int out_features, T *d_weight, T *d_bias, T *d_input, void *lt_workspace) { + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + // Get the stream from cublas handle to reuse for biasReLU kernel. + cudaStream_t stream; + cublasGetStream(handle, &stream); + const float alpha = 1.0; + const float beta_zero = 0.0; + int status = 1; +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11600 + status = gemm_bgradb_lt( + (cublasLtHandle_t)handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + in_features, + out_features, + batch_size, + &alpha, /* host pointer */ + input, + in_features, + d_output, + out_features, + &beta_zero, /* host pointer */ + d_weight, + in_features, + lt_workspace, + 1 << 22, + stream, + true, + static_cast(d_bias)); +#endif + + + if (status != 0){ + + status = gemm_bias( + handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + in_features, + out_features, + batch_size, + &alpha, + input, + in_features, + d_output, + out_features, + &beta_zero, + d_weight, + in_features); + } + + status = gemm_bias( + handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + in_features, + batch_size, + out_features, + &alpha, + weight, + in_features, + d_output, + out_features, + &beta_zero, + d_input, + in_features); + return status; + +} + +template +int linear_gelu_linear_forward_cuda(T *input, T *weight1, T *bias1, T *weight2, T *bias2, int in_features, int hidden_features, int batch_size, int out_features, T *output1, T *output2, T *gelu_in, void *lt_workspace) { + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + // Get the stream from cublas handle to reuse for biasReLU kernel. + cudaStream_t stream; + cublasGetStream(handle, &stream); + const float alpha = 1.0; + const float beta_zero = 0.0; + int status = 1; +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11600 + status = gemm_bias_gelu_lt( + (cublasLtHandle_t)handle, + CUBLAS_OP_T, + CUBLAS_OP_N, + hidden_features, + batch_size, + in_features, + &alpha, /* host pointer */ + weight1, + in_features, + input, + in_features, + &beta_zero, /* host pointer */ + output1, + hidden_features, + lt_workspace, + 1 << 22, + stream, + true, + static_cast(gelu_in), + static_cast(bias1)); + status = gemm_bias_lt( + (cublasLtHandle_t)handle, + CUBLAS_OP_T, + CUBLAS_OP_N, + out_features, + batch_size, + hidden_features, + &alpha, /* host pointer */ + weight2, + hidden_features, + output1, + hidden_features, + &beta_zero, /* host pointer */ + output2, + out_features, + lt_workspace, + 1 << 22, + stream, + true, + static_cast(bias2)); + return status; +#else + return 1; +#endif +} + +template +int linear_gelu_linear_backward_cuda(T *input, T *gelu_in, T *output1, T *weight1, T *weight2, T *d_output1, T *d_output2, int in_features, int batch_size, int hidden_features, int out_features, T *d_weight1, T *d_weight2, T *d_bias1, T *d_bias2, T *d_input, void *lt_workspace) { + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + // Get the stream from cublas handle to reuse for biasReLU kernel. + cudaStream_t stream; + cublasGetStream(handle, &stream); + const float alpha = 1.0; + const float beta_zero = 0.0; + int status = 1; +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11600 +//wgrad for first gemm + status = gemm_bgradb_lt( + (cublasLtHandle_t)handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + hidden_features, + out_features, + batch_size, + &alpha, /* host pointer */ + output1, + hidden_features, + d_output2, + out_features, + &beta_zero, /* host pointer */ + d_weight2, + hidden_features, + lt_workspace, + 1 << 22, + stream, + true, + static_cast(d_bias2)); +//dgrad for second GEMM + status = gemm_dgelu_bgradb_lt( + (cublasLtHandle_t)handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + hidden_features, + batch_size, + out_features, + &alpha, /* host pointer */ + weight2, + hidden_features, + d_output2, + out_features, + &beta_zero, /* host pointer */ + d_output1, + hidden_features, + lt_workspace, + 1 << 22, + stream, + static_cast(gelu_in), + static_cast(d_bias1)); +//wgrad for the first GEMM + status = gemm_bias( + handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + in_features, + hidden_features, + batch_size, + &alpha, + input, + in_features, + d_output1, + hidden_features, + &beta_zero, + d_weight1, + in_features); + +//dgrad for the first GEMM + status = gemm_bias( + handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + in_features, + batch_size, + hidden_features, + &alpha, + weight1, + in_features, + d_output1, + hidden_features, + &beta_zero, + d_input, + in_features); +#endif + return status; + +} + + +template int linear_bias_forward_cuda(at::Tensor input, at::Half *weight, at::Tensor bias, int in_features, int batch_size, int out_features, at::Tensor output, void *lt_workspace); + +template int linear_bias_forward_cuda(at::Tensor input, float *weight, at::Tensor bias, int in_features, int batch_size, int out_features, at::Tensor output, void *lt_workspace); + +template int linear_bias_forward_cuda(at::Tensor input, double *weight, at::Tensor bias, int in_features, int batch_size, int out_features, at::Tensor output, void *lt_workspace); + +template int linear_bias_backward_cuda(at::Half *input, at::Half *weight, at::Half *d_output, int in_features, int batch_size, int out_features, at::Half *d_weight, at::Half *d_bias, at::Half *d_input, void *lt_workspace) ; + +template int linear_bias_backward_cuda(float *input, float *weight, float *d_output, int in_features, int batch_size, int out_features, float *d_weight, float *d_bias, float *d_input, void *lt_workspace) ; + +template int linear_bias_backward_cuda(double *input, double *weight, double *d_output, int in_features, int batch_size, int out_features, double *d_weight, double *d_bias, double *d_input, void *lt_workspace) ; + + +template int linear_gelu_linear_forward_cuda(at::Half *input, at::Half *weight1, at::Half *bias1, at::Half *weight2, at::Half *bias2, int in_features, int hidden_features, int batch_size, int out_features, at::Half *output1, at::Half *output2, at::Half *gelu_in, void *lt_workspace) ; + +template int linear_gelu_linear_forward_cuda(float *input, float *weight1, float *bias1, float *weight2, float *bias2, int in_features, int hidden_features, int batch_size, int out_features, float *output1, float *output2, float *gelu_in, void *lt_workspace); + +template int linear_gelu_linear_forward_cuda(double *input, double *weight1, double *bias1, double *weight2, double *bias2, int in_features, int hidden_features, int batch_size, int out_features, double *output1, double *output2, double *gelu_in, void *lt_workspace) ; + +template int linear_gelu_linear_backward_cuda(at::Half *input, at::Half *gelu_in, at::Half *output1, at::Half *weight1, at::Half *weight2, at::Half *d_output1, at::Half *d_output2, int in_features, int batch_size, int hidden_features, int out_features, at::Half *d_weight1, at::Half *d_weight2, at::Half *d_bias1, at::Half *d_bias2, at::Half *d_input, void *lt_workspace); + +template int linear_gelu_linear_backward_cuda(float *input, float *gelu_in, float *output1, float *weight1, float *weight2, float *d_output1, float *d_output2, int in_features, int batch_size, int hidden_features, int out_features, float *d_weight1, float *d_weight2, float *d_bias1, float *d_bias2, float *d_input, void *lt_workspace); + +template int linear_gelu_linear_backward_cuda(double *input, double *gelu_in, double *output1, double *weight1, double *weight2, double *d_output1, double *d_output2, int in_features, int batch_size, int hidden_features, int out_features, double *d_weight1, double *d_weight2, double *d_bias1, double *d_bias2, double *d_input, void *lt_workspace); + + +template int linear_bias_forward_cuda(at::Tensor input, at::BFloat16 *weight, at::Tensor bias, int in_features, int batch_size, int out_features, at::Tensor output, void *lt_workspace); + +template int linear_bias_backward_cuda(at::BFloat16 *input, at::BFloat16 *weight, at::BFloat16 *d_output, int in_features, int batch_size, int out_features, at::BFloat16 *d_weight, at::BFloat16 *d_bias, at::BFloat16 *d_input, void *lt_workspace) ; + +template int linear_gelu_linear_forward_cuda(at::BFloat16 *input, at::BFloat16 *weight1, at::BFloat16 *bias1, at::BFloat16 *weight2, at::BFloat16 *bias2, int in_features, int hidden_features, int batch_size, int out_features, at::BFloat16 *output1, at::BFloat16 *output2, at::BFloat16 *gelu_in, void *lt_workspace) ; + +template int linear_gelu_linear_backward_cuda(at::BFloat16 *input, at::BFloat16 *gelu_in, at::BFloat16 *output1, at::BFloat16 *weight1, at::BFloat16 *weight2, at::BFloat16 *d_output1, at::BFloat16 *d_output2, int in_features, int batch_size, int hidden_features, int out_features, at::BFloat16 *d_weight1, at::BFloat16 *d_weight2, at::BFloat16 *d_bias1, at::BFloat16 *d_bias2, at::BFloat16 *d_input, void *lt_workspace); + diff --git a/apex/csrc/instance_norm_nvfuser.cpp b/apex/csrc/instance_norm_nvfuser.cpp new file mode 100644 index 00000000..3af0f28b --- /dev/null +++ b/apex/csrc/instance_norm_nvfuser.cpp @@ -0,0 +1,35 @@ +#include +#include + +#include + +std::vector instance_norm_nvfuser_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor run_mean, + at::Tensor run_var, + const bool use_input_stats, + const float momentum, + const float eps, + const bool channels_last = false + ); + +std::vector instance_norm_nvfuser_backward( + at::Tensor input, + at::Tensor grad_output, + at::Tensor weight, + at::Tensor running_mean, + at::Tensor running_var, + at::Tensor save_mean, + at::Tensor save_invstd, + const bool use_input_stats, + const float eps, + // const std::vector& output_mask, + bool channels_last = false + ); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &instance_norm_nvfuser_forward, "instance_norm forward (CUDA)"); + m.def("backward", &instance_norm_nvfuser_backward, "instance_norm backward (CUDA)"); +} diff --git a/apex/csrc/instance_norm_nvfuser_kernel.cu b/apex/csrc/instance_norm_nvfuser_kernel.cu new file mode 100644 index 00000000..077232e6 --- /dev/null +++ b/apex/csrc/instance_norm_nvfuser_kernel.cu @@ -0,0 +1,286 @@ +#include +#include +#include +#include + +#include + +// The following header file is found in `PYTORCH_HOME` +#include + +#if NVFUSER_THIRDPARTY +#include +#include +#include +#else +#include +#include +#endif + +using namespace torch::jit::fuser::cuda; +using namespace at::indexing; + +std::chrono::time_point t1; +std::chrono::time_point t2; +std::chrono::time_point t3; + +bool profile() { + static bool should_profile = std::getenv("APEX_NVFUSER_PROFILE") != nullptr; + return should_profile; +} + +// Make a tensor that is known to be fully contiguous of dimensionality=ndims, +// but unknown sizes +TensorView* makeContigTensor(size_t ndims, DataType dtype = DataType::Float) { + return TensorViewBuilder() + .ndims(ndims) + .dtype(dtype) + .contiguity(std::vector(ndims, true)) + .build(); +} + +struct InstanceNormKey { + c10::ScalarType input_dtype;//int8_t dtype; + c10::ScalarType weight_dtype; + c10::ScalarType mean_dtype; + size_t dim; + bool channels_last; + bool running_mean; + bool affine; +}; + +auto get_dtype(c10::ScalarType dtype) { + auto ret_dtype = DataType::Float; + if (dtype == c10::ScalarType::Double) { + ret_dtype = DataType::Double; + } else if (dtype == c10::ScalarType::Half) { + ret_dtype = DataType::Half; + } else if (dtype == c10::ScalarType::BFloat16) { + ret_dtype = DataType::BFloat16; + } + return ret_dtype; +} + +// TODO: doesn't support all combinations of dtype e.g., bias, run_var, .. +// bias is assumed to match weight, run_var is assumed to match run_mean +void setKey(const at::Tensor& input, const at::Tensor& weight, const at::Tensor& run_mean, const bool channels_last, InstanceNormKey& key) { + memset(&key, 0, sizeof(InstanceNormKey)); + key.input_dtype = input.scalar_type();// static_cast(input.scalar_type()); + key.weight_dtype = weight.scalar_type(); + key.mean_dtype = run_mean.scalar_type(); + key.dim = input.sizes().size(); + key.channels_last = channels_last; + key.running_mean = run_mean.sizes().size() > 0; + key.affine = weight.sizes().size() ? true : false; +} + +std::unordered_map, at::native::ParamsHash, at::native::ParamsEqual > forward_fusion_cache; +std::unordered_map, at::native::ParamsHash, at::native::ParamsEqual > backward_fusion_cache; + +std::vector instance_norm_nvfuser_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor run_mean, + at::Tensor run_var, + const bool use_input_stats, + const float momentum, + const float eps, + const bool channels_last) { + if (profile()) { + t1 = std::chrono::steady_clock::now(); + } + InstanceNormKey forward_key; + setKey(input, weight, run_mean, channels_last, forward_key); + if (forward_fusion_cache.find(forward_key) == forward_fusion_cache.end()) { + auto fusion = std::make_unique(); + FusionGuard fg(fusion.get()); + + const auto _input_dtype = get_dtype(input.scalar_type()); + const auto _weight_dtype = get_dtype(weight.scalar_type()); + const auto _bias_dtype = get_dtype(bias.scalar_type()); + const auto _running_mean_dtype = get_dtype(run_mean.scalar_type()); + const auto _running_var_dtype = get_dtype(run_var.scalar_type()); + auto _input = makeContigTensor(input.sizes().size(), _input_dtype); + auto _weight = makeContigTensor(weight.sizes().size(), _weight_dtype); + auto _bias = makeContigTensor(bias.sizes().size(), _bias_dtype); + auto _running_mean = makeContigTensor(run_mean.sizes().size(), get_dtype(run_mean.scalar_type())); + auto _running_var = makeContigTensor(run_var.sizes().size(), get_dtype(run_var.scalar_type())); + + fusion->addInput(_input); + fusion->addInput(_weight); + fusion->addInput(_bias); + + if (_input_dtype == DataType::Half || _input_dtype == DataType::BFloat16) { + _input = castOp(DataType::Float, _input); + } + if (_weight_dtype == DataType::Half || _weight_dtype == DataType::BFloat16) { + _weight = castOp(DataType::Float, _weight); + } + if (_bias_dtype == DataType::Half || _bias_dtype == DataType::BFloat16) { + _bias = castOp(DataType::Float, _bias); + } + + // TODO: decide if passing an empty tensor is the best way to signal no running mean/var + if (run_mean.sizes().size()) { + fusion->addInput(_running_mean); + fusion->addInput(_running_var); + // casting is done by Forward for running mean/var as it needs original inputs for aliasing + } + + Double* _momentum = IrBuilder::create(); + Double* _eps = IrBuilder::create(); + fusion->addInput(_momentum); + fusion->addInput(_eps); + + ForwardNormResult result; + if (!run_mean.sizes().size()) { + _running_mean = nullptr; + _running_var = nullptr; + } + if (!weight.sizes().size()) { + _weight = nullptr; + _bias = nullptr; + } + result = instance_norm( + _input, _weight, _bias, _running_mean, _running_var, use_input_stats, _momentum, _eps, channels_last); + + if (_input_dtype == DataType::Half || _input_dtype == DataType::BFloat16) { + fusion->addOutput(castOp(_input_dtype, result.output)); + fusion->addOutput(castOp(_input_dtype, result.mean)); + fusion->addOutput(castOp(_input_dtype, result.invstd)); + } else { + fusion->addOutput(result.output); + fusion->addOutput(result.mean); + fusion->addOutput(result.invstd); + } + forward_fusion_cache.emplace(forward_key, std::make_unique(std::move(fusion))); + } + std::vector aten_inputs = {input, weight, bias}; + if (run_mean.sizes().size()) { + aten_inputs.push_back(run_mean); + aten_inputs.push_back(run_var); + } + aten_inputs.push_back(momentum); + aten_inputs.push_back(eps); + if (profile()) { + t2 = std::chrono::steady_clock::now(); + } + auto r = forward_fusion_cache[forward_key].get()->runFusionWithInputs(aten_inputs); + if (profile()) { + t3 = std::chrono::steady_clock::now(); + std::chrono::duration full = t3 - t1; + std::chrono::duration pre = t2 - t1; + std::chrono::duration exec = t3 - t2; + std::cout << "NVFuserInstanceNorm Forward (full, pre-exec, exec) (" << full.count() + << ", " << pre.count() << ", " << exec.count() << ")" << std::endl; + } + return r; +} + +std::vector instance_norm_nvfuser_backward( + at::Tensor input, + at::Tensor grad_output, + at::Tensor weight, + at::Tensor run_mean, + at::Tensor run_var, + at::Tensor save_mean, + at::Tensor save_invstd, + const bool use_input_stats, + const float eps, + // const std::vector& output_mask, + bool channels_last + ) { + if (profile()) { + t1 = std::chrono::steady_clock::now(); + } + InstanceNormKey backward_key; + memset(&backward_key, 0, sizeof(InstanceNormKey)); + setKey(input, weight, run_mean, channels_last, backward_key); + if (backward_fusion_cache.find(backward_key) == backward_fusion_cache.end()) { + auto fusion = std::make_unique(); + FusionGuard fg(fusion.get()); + const auto _input_dtype = get_dtype(input.scalar_type()); + const auto _grad_output_dtype = get_dtype(grad_output.scalar_type()); + const auto _weight_dtype = get_dtype(weight.scalar_type()); + const auto _running_mean_dtype = get_dtype(run_mean.scalar_type()); + const auto _running_var_dtype = get_dtype(run_var.scalar_type()); + auto _input = makeContigTensor(input.sizes().size(), _input_dtype); + auto _grad_output = makeContigTensor(grad_output.sizes().size(), _grad_output_dtype); + auto _weight = makeContigTensor(weight.sizes().size(), _weight_dtype); + auto _running_mean = makeContigTensor(run_mean.sizes().size(), get_dtype(run_mean.scalar_type())); + auto _running_var = makeContigTensor(run_var.sizes().size(), get_dtype(run_var.scalar_type())); + auto _save_mean = makeContigTensor(save_mean.sizes().size(), get_dtype(save_mean.scalar_type())); + auto _save_invstd = makeContigTensor(save_invstd.sizes().size(), get_dtype(save_invstd.scalar_type())); + + fusion->addInput(_input); + fusion->addInput(_grad_output); + fusion->addInput(_weight); + fusion->addInput(_running_mean); + fusion->addInput(_running_var); + fusion->addInput(_save_mean); + fusion->addInput(_save_invstd); + + if (_input_dtype == DataType::Half || _input_dtype == DataType::BFloat16) { + _input = castOp(DataType::Float, _input); + } + if (_grad_output_dtype == DataType::Half || _grad_output_dtype == DataType::BFloat16) { + _grad_output = castOp(DataType::Float, _grad_output); + } + if (_weight_dtype == DataType::Half || _weight_dtype == DataType::BFloat16) { + _weight = castOp(DataType::Float, _weight); + } + if (_running_mean_dtype == DataType::Half || _running_mean_dtype == DataType::BFloat16) { + _running_mean = castOp(DataType::Float, _running_mean); + } + if (_running_var_dtype == DataType::Half || _running_var_dtype == DataType::BFloat16) { + _running_var = castOp(DataType::Float, _running_var); + } + + Double* _eps = IrBuilder::create(); + fusion->addInput(_eps); + if (!run_mean.sizes().size()) { + _running_mean = nullptr; + _running_var = nullptr; + } + if (!weight.sizes().size()) { + _weight = nullptr; + } + auto result = instance_norm_backward(_input, + _grad_output, + _weight, + _running_mean, + _running_var, + _save_mean, + _save_invstd, + use_input_stats, + _eps, + {true, true, true}, // TODO: is output mask useful? + channels_last); + if (_input_dtype == DataType::Half || _input_dtype == DataType::BFloat16) { + fusion->addOutput(castOp(_input_dtype, result.grad_input)); + fusion->addOutput(castOp(_input_dtype, result.grad_weight)); + fusion->addOutput(castOp(_input_dtype, result.grad_bias)); + } else { + fusion->addOutput(result.grad_input); + fusion->addOutput(result.grad_weight); + fusion->addOutput(result.grad_bias); + } + backward_fusion_cache.emplace(backward_key, std::make_unique(std::move(fusion))); + } + std::vector aten_inputs = { + input, grad_output, weight, run_mean, run_var, save_mean, save_invstd, eps}; + if (profile()) { + t2 = std::chrono::steady_clock::now(); + } + auto r = backward_fusion_cache[backward_key].get()->runFusionWithInputs(aten_inputs); + if (profile()) { + t3 = std::chrono::steady_clock::now(); + std::chrono::duration full = t3 - t1; + std::chrono::duration pre = t2 - t1; + std::chrono::duration exec = t3 - t2; + std::cout << "NVFuserInstanceNorm Backward (full, pre-exec, exec) (" << full.count() + << ", " << pre.count() << ", " << exec.count() << ")" << std::endl; + } + return r; + } diff --git a/apex/csrc/layer_norm_cuda.cpp b/apex/csrc/layer_norm_cuda.cpp new file mode 100644 index 00000000..00590610 --- /dev/null +++ b/apex/csrc/layer_norm_cuda.cpp @@ -0,0 +1,442 @@ +#include +#include +#include +#include "compat.h" + +namespace { +void compute_n1_n2( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + int& n1, + int& n2) +{ + int idiff = input.ndimension() - normalized_shape.size(); + n2 = 1; + for (int i = 0; i < (int)normalized_shape.size(); ++i) { + assert( input.sizes()[i+idiff] == normalized_shape[i] ); + n2 *= normalized_shape[i]; + } + n1 = 1; + for (int i = 0; i < idiff; ++i) { + n1 *= input.sizes()[i]; + } +} + +void check_args( + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + at::Tensor beta + ) +{ + TORCH_CHECK(!gamma.defined() || gamma.sizes().equals(normalized_shape)); + TORCH_CHECK(!beta.defined() || beta.sizes().equals(normalized_shape)); +} + +void check_args( + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma + ) +{ + TORCH_CHECK(!gamma.defined() || gamma.sizes().equals(normalized_shape)); +} + + +void check_args( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + int& n1, + int& n2 + ) +{ + int64_t normalized_ndim = normalized_shape.size(); + + if (normalized_ndim < 1) { + std::stringstream ss; + ss << "Expected normalized_shape to be at least 1-dimensional, i.e., " + << "containing at least one element, but got normalized_shape=" + << normalized_shape; + throw std::runtime_error(ss.str()); + } + + auto input_shape = input.sizes(); + auto input_ndim = input.dim(); + + if (input_ndim < normalized_ndim || + !input_shape.slice(input_ndim - normalized_ndim).equals(normalized_shape)) { + std::stringstream ss; + ss << "Given normalized_shape=" << normalized_shape + << ", expected input with shape [*"; + for (auto size : normalized_shape) { + ss << ", " << size; + } + ss << "], but got input of size" << input_shape; + throw std::runtime_error(ss.str()); + } + + compute_n1_n2(input,normalized_shape,n1,n2); +} + +void check_args( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + at::Tensor beta, + int& n1, + int& n2 + ) +{ + check_args(input,normalized_shape,n1,n2); + check_args(normalized_shape,gamma,beta); +} + +void check_args( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + int& n1, + int& n2 + ) +{ + check_args(input,normalized_shape,n1,n2); + check_args(normalized_shape,gamma); +} +} + +void cuda_layer_norm( + at::Tensor* output, + at::Tensor* mean, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + at::Tensor* beta, + double epsilon); + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::vector layer_norm( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + double epsilon) { + CHECK_INPUT(input); + int n1,n2; + check_args(input,normalized_shape,n1,n2); + at::Tensor output = at::empty_like(input); + at::Tensor mean = at::empty({n1}, input.options().dtype(input.scalar_type()==at::ScalarType::Half || input.scalar_type()==at::ScalarType::BFloat16 ? at::ScalarType::Float : input.scalar_type())); + at::Tensor invvar = at::empty_like(mean); + cuda_layer_norm(&output,&mean,&invvar,&input,n1,n2, + normalized_shape,NULL,NULL,epsilon); + return {output, mean, invvar}; +} + +std::vector layer_norm_affine( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + at::Tensor beta, + double epsilon) { + CHECK_INPUT(input); + CHECK_INPUT(gamma); + CHECK_INPUT(beta); + int n1,n2; + check_args(input,normalized_shape,gamma,beta,n1,n2); + at::Tensor output = at::empty_like(input); + const auto stats_dtype = (input.scalar_type() == at::ScalarType::Half || input.scalar_type() == at::ScalarType::BFloat16) ? at::ScalarType::Float : input.scalar_type(); + at::Tensor mean = at::empty({n1}, input.options().dtype(stats_dtype)); + at::Tensor invvar = at::empty_like(mean); + cuda_layer_norm(&output,&mean,&invvar,&input,n1,n2, + normalized_shape,&gamma,&beta,epsilon); + return {output, mean, invvar}; +} + +std::vector layer_norm_affine_mixed_dtypes( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + at::Tensor beta, + double epsilon) { + CHECK_INPUT(input); + int n1, n2; + check_args(input, normalized_shape, n1, n2); + at::Tensor output = at::empty_like(input, gamma.options().dtype(gamma.scalar_type())); + at::Tensor mean = at::empty({n1}, input.options().dtype(input.scalar_type() == at::ScalarType::Half || input.scalar_type() == at::ScalarType::BFloat16 ? at::ScalarType::Float : input.scalar_type())); + at::Tensor invvar = at::empty_like(mean); + cuda_layer_norm(&output, &mean, &invvar, &input, n1, n2, + normalized_shape, &gamma, &beta, epsilon); + return {output, mean, invvar}; +} + +void cuda_layer_norm_gradient( + at::Tensor* dout, + at::Tensor* mean, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + at::Tensor* beta, + double epsilon, + at::Tensor* grad_input, + at::Tensor* grad_gamma, + at::Tensor* grad_beta + ); + +at::Tensor layer_norm_gradient( + at::Tensor dout, + at::Tensor mean, + at::Tensor invvar, + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + double epsilon) { + CHECK_INPUT(dout); + CHECK_INPUT(mean); + CHECK_INPUT(invvar); + CHECK_INPUT(input); + int n1,n2; + check_args(input,normalized_shape,n1,n2); + at::Tensor grad_input = at::empty_like(input); + cuda_layer_norm_gradient(&dout,&mean,&invvar,&input,n1,n2, + normalized_shape,NULL,NULL,epsilon, + &grad_input,NULL,NULL); + return grad_input; +} + +std::vector layer_norm_gradient_affine( + at::Tensor dout, + at::Tensor mean, + at::Tensor invvar, + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + at::Tensor beta, + double epsilon) { + CHECK_INPUT(dout); + CHECK_INPUT(mean); + CHECK_INPUT(invvar); + CHECK_INPUT(input); + CHECK_INPUT(gamma); + CHECK_INPUT(beta); + int n1,n2; + check_args(input,normalized_shape,gamma,beta,n1,n2); + at::Tensor grad_input = at::empty_like(input); + at::Tensor grad_gamma = at::empty_like(gamma); + at::Tensor grad_beta = at::empty_like(beta); + cuda_layer_norm_gradient(&dout,&mean,&invvar,&input,n1,n2, + normalized_shape,&gamma,&beta,epsilon, + &grad_input,&grad_gamma,&grad_beta); + return {grad_input, grad_gamma, grad_beta}; +} + +void cuda_rms_norm( + at::Tensor* output, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + double epsilon); + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_INPUT(x) CHECK_CUDA(x); CHECK_CONTIGUOUS(x) + +std::vector rms_norm( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + double epsilon) { + CHECK_INPUT(input); + int n1,n2; + check_args(input,normalized_shape,n1,n2); + at::Tensor output = at::empty_like(input); + at::Tensor invvar = at::empty({n1}, input.options().dtype(input.scalar_type()==at::ScalarType::Half || input.scalar_type()==at::ScalarType::BFloat16 ? at::ScalarType::Float : input.scalar_type())); + cuda_rms_norm(&output,&invvar,&input,n1,n2, + normalized_shape,NULL,epsilon); + return {output, invvar}; +} + +std::vector rms_norm_affine( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + double epsilon) { + CHECK_INPUT(input); + CHECK_INPUT(gamma); + int n1,n2; + check_args(input,normalized_shape,gamma,n1,n2); + at::Tensor output = at::empty_like(input); + const auto stats_dtype = (input.scalar_type() == at::ScalarType::Half || input.scalar_type() == at::ScalarType::BFloat16) ? at::ScalarType::Float : input.scalar_type(); + at::Tensor invvar = at::empty({n1}, input.options().dtype(stats_dtype)); + cuda_rms_norm(&output,&invvar,&input,n1,n2, + normalized_shape,&gamma,epsilon); + return {output, invvar}; +} + +std::vector rms_norm_affine_mixed_dtypes( + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + double epsilon) { + CHECK_INPUT(input); + int n1, n2; + check_args(input, normalized_shape, n1, n2); + at::Tensor output = at::empty_like(input, gamma.options().dtype(gamma.scalar_type())); + at::Tensor invvar = at::empty({n1}, input.options().dtype(input.scalar_type() == at::ScalarType::Half || input.scalar_type() == at::ScalarType::BFloat16 ? at::ScalarType::Float : input.scalar_type())); + + cuda_rms_norm(&output,&invvar, &input, n1, n2, + normalized_shape, &gamma,epsilon); + return {output,invvar}; +} + +void cuda_rms_norm_gradient( + at::Tensor* dout, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + double epsilon, + at::Tensor* grad_input, + at::Tensor* grad_gamma); + +at::Tensor rms_norm_gradient( + at::Tensor dout, + at::Tensor invvar, + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + double epsilon) { + CHECK_INPUT(dout); + CHECK_INPUT(invvar); + CHECK_INPUT(input); + int n1,n2; + check_args(input,normalized_shape,n1,n2); + at::Tensor grad_input = at::empty_like(input); + cuda_rms_norm_gradient(&dout,&invvar,&input,n1,n2, + normalized_shape,NULL,epsilon, + &grad_input,NULL); + return grad_input; +} + +std::vector rms_norm_gradient_affine( + at::Tensor dout, + at::Tensor invvar, + at::Tensor input, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor gamma, + double epsilon) { + CHECK_INPUT(dout); + CHECK_INPUT(invvar); + CHECK_INPUT(input); + CHECK_INPUT(gamma); + int n1,n2; + check_args(input,normalized_shape,gamma,n1,n2); + at::Tensor grad_input = at::empty_like(input); + at::Tensor grad_gamma = at::empty_like(gamma); + cuda_rms_norm_gradient(&dout,&invvar,&input,n1,n2, + normalized_shape,&gamma,epsilon, + &grad_input,&grad_gamma); + return {grad_input, grad_gamma}; +} + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward_affine", &layer_norm_affine, "LayerNorm forward (CUDA)"); + m.def("forward", &layer_norm, "LayerNorm forward (CUDA)"); + m.def("backward_affine", &layer_norm_gradient_affine, "LayerNorm backward (CUDA)"); + m.def("backward", &layer_norm_gradient, "LayerNorm backward (CUDA)"); + + m.def("forward_affine_mixed_dtypes", &layer_norm_affine_mixed_dtypes, "LayerNorm forward with mixed dtypes (CUDA) compatible with Megatron's implementation"); + + m.def("rms_forward_affine", &rms_norm_affine, "RMSNorm forward (CUDA)"); + m.def("rms_forward", &rms_norm, "RMSNorm forward (CUDA)"); + m.def("rms_backward_affine", &rms_norm_gradient_affine, "RMSNorm backward (CUDA)"); + m.def("rms_backward", &rms_norm_gradient, "RMSNorm backward (CUDA)"); + + m.def("rms_forward_affine_mixed_dtypes", &rms_norm_affine_mixed_dtypes, "RMSNorm forward with mixed dtypes (CUDA) compatible with Megatron's implementation"); +} diff --git a/apex/csrc/layer_norm_cuda_kernel.cu b/apex/csrc/layer_norm_cuda_kernel.cu new file mode 100644 index 00000000..21366772 --- /dev/null +++ b/apex/csrc/layer_norm_cuda_kernel.cu @@ -0,0 +1,1170 @@ +#include "ATen/ATen.h" +#include "ATen/AccumulateType.h" +#include "ATen/cuda/CUDAContext.h" +#include "ATen/cuda/DeviceUtils.cuh" + +#include +#include + +#include "type_shim.h" + +template __device__ +void cuWelfordOnlineSum( + const U curr, + U& mu, + U& sigma2, + U& count) +{ + count = count + U(1); + U delta = curr - mu; + U lmean = mu + delta / count; + mu = lmean; + U delta2 = curr - lmean; + sigma2 = sigma2 + delta * delta2; +} + +template __device__ +void cuChanOnlineSum( + const U muB, + const U sigma2B, + const U countB, + U& mu, + U& sigma2, + U& count) +{ + U delta = muB - mu; + U nA = count; + U nB = countB; + count = count + countB; + U nX = count; + if (nX > U(0)) { + nA = nA / nX; + nB = nB / nX; + mu = nA*mu + nB*muB; + sigma2 = sigma2 + sigma2B + delta * delta * nA * nB * nX; + } else { + mu = U(0); + sigma2 = U(0); + } +} + +template __device__ +void cuRMSOnlineSum( + const U curr, + U& sigma2) +{ + sigma2 = sigma2 + curr * curr; +} + +template __device__ +void cuChanRMSOnlineSum( + const U sigma2B, + U& sigma2) +{ + sigma2 = sigma2 + sigma2B; +} + + +template __device__ +void cuWelfordMuSigma2( + const T* __restrict__ vals, + const int n1, + const int n2, + const int i1, + U& mu, + U& sigma2, + U* buf, + bool rms_only) +{ + // Assumptions: + // 1) blockDim.x == warpSize + // 2) Tensor is contiguous + // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available. + // + // compute variance and mean over n2 + U count = U(0); + mu= U(0); + sigma2 = U(0); + if (i1 < n1) { + // one warp normalizes one n1 index, + // synchronization is implicit + // initialize with standard Welford algorithm + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + const T* lvals = vals + i1*n2; + int l = 4*thrx; + for (; l+3 < n2; l+=4*numx) { + for (int k = 0; k < 4; ++k) { + U curr = static_cast(lvals[l+k]); + if (!rms_only) { + cuWelfordOnlineSum(curr,mu,sigma2,count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + } + for (; l < n2; ++l) { + U curr = static_cast(lvals[l]); + if (!rms_only) { + cuWelfordOnlineSum(curr,mu,sigma2,count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + // intra-warp reductions + for (int l = 0; l <= 4; ++l) { + int srcLaneB = (threadIdx.x+(1<(muB,sigma2B,countB,mu,sigma2,count); + } else { + cuChanRMSOnlineSum(sigma2B, sigma2); + } + } + // threadIdx.x == 0 has correct values for each warp + // inter-warp reductions + if (blockDim.y > 1) { + U* ubuf = (U*)buf; + U* ibuf = (U*)(ubuf + blockDim.y); + for (int offset = blockDim.y/2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.x == 0 && threadIdx.y >= offset && threadIdx.y < 2*offset) { + const int wrt_y = threadIdx.y - offset; + if (!rms_only) { + ubuf[2*wrt_y] = mu; + ibuf[wrt_y] = count; + } + ubuf[2*wrt_y+1] = sigma2; + } + __syncthreads(); + // lower half merges + if (threadIdx.x == 0 && threadIdx.y < offset) { + U sigma2B = ubuf[2*threadIdx.y+1]; + if (!rms_only) { + U muB = ubuf[2*threadIdx.y]; + U countB = ibuf[threadIdx.y]; + cuChanOnlineSum(muB,sigma2B,countB,mu,sigma2,count); + } else { + cuChanRMSOnlineSum(sigma2B,sigma2); + } + } + __syncthreads(); + } + // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct values + if (threadIdx.x == 0 && threadIdx.y == 0) { + if (!rms_only) { + ubuf[0] = mu; + } + ubuf[1] = sigma2; + } + __syncthreads(); + if (!rms_only) { + mu = ubuf[0]; + } + sigma2 = ubuf[1]/U(n2); + // don't care about final value of count, we know count == n2 + } else { + if (!rms_only) { + mu = WARP_SHFL(mu, 0); + } + sigma2 = WARP_SHFL(sigma2/U(n2), 0); + } + } +} + +template<> __device__ +void cuWelfordMuSigma2( + const at::Half* __restrict__ vals, + const int n1, + const int n2, + const int i1, + float& mu, + float& sigma2, + float* buf, + bool rms_only) +{ + // Assumptions: + // 1) blockDim.x == warpSize + // 2) Tensor is contiguous + // 3) 2*blockDim.y*sizeof(U)+blockDim.y*sizeof(int) shared memory available. + // + // compute variance and mean over n2 + float count = 0.0f; + mu= float(0); + sigma2 = float(0); + if (i1 < n1) { + // one warp normalizes one n1 index, + // synchronization is implicit + // initialize with standard Welford algorithm + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + const at::Half* lvals = vals + i1*n2; + int l = 8*thrx; + if ((((size_t)lvals)&3) != 0) { + // 16 bit alignment + // first thread consumes first point + if (thrx == 0) { + float curr = static_cast(lvals[0]); + if (!rms_only) { + cuWelfordOnlineSum(curr,mu,sigma2,count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + + } + ++l; + } + // at this point, lvals[l] are 32 bit aligned for all threads. + for (; l+7 < n2; l+=8*numx) { + for (int k = 0; k < 8; k+=2) { + float2 curr = __half22float2(*((__half2*)(lvals+l+k))); + if (!rms_only) { + cuWelfordOnlineSum(curr.x,mu,sigma2,count); + cuWelfordOnlineSum(curr.y,mu,sigma2,count); + } else { + cuRMSOnlineSum(curr.x, sigma2); + cuRMSOnlineSum(curr.y, sigma2); + } + } + } + for (; l < n2; ++l) { + float curr = static_cast(lvals[l]); + if (!rms_only) { + cuWelfordOnlineSum(curr,mu,sigma2,count); + } else { + cuRMSOnlineSum(curr, sigma2); + } + } + // intra-warp reductions + for (int l = 0; l <= 4; ++l) { + int srcLaneB = (threadIdx.x+(1< 1) { + float* ubuf = (float*)buf; + float* ibuf = (float*)(ubuf + blockDim.y); + for (int offset = blockDim.y/2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.x == 0 && threadIdx.y >= offset && threadIdx.y < 2*offset) { + const int wrt_y = threadIdx.y - offset; + ubuf[2*wrt_y+1] = sigma2; + if (!rms_only) { + ubuf[2*wrt_y] = mu; + ibuf[wrt_y] = count; + } + } + __syncthreads(); + // lower half merges + if (threadIdx.x == 0 && threadIdx.y < offset) { + float sigma2B = ubuf[2*threadIdx.y+1]; + if (!rms_only) { + float muB = ubuf[2*threadIdx.y]; + float countB = ibuf[threadIdx.y]; + cuChanOnlineSum(muB,sigma2B,countB,mu,sigma2,count); + } else { + cuChanRMSOnlineSum(sigma2B, sigma2); + } + } + __syncthreads(); + } + // threadIdx.x = 0 && threadIdx.y == 0 only thread that has correct values + if (threadIdx.x == 0 && threadIdx.y == 0) { + if (!rms_only) { + ubuf[0] = mu; + } + ubuf[1] = sigma2; + } + __syncthreads(); + if (!rms_only) { + mu = ubuf[0]; + } + sigma2 = ubuf[1]/float(n2); + // don't care about final value of count, we know count == n2 + } else { + if (!rms_only) { + mu = WARP_SHFL(mu, 0); + } + sigma2 = WARP_SHFL(sigma2/float(n2), 0); + } + } +} + +template U rsqrt(U v) { + return U(1) / sqrt(v); +} +template<> float rsqrt(float v) { + return rsqrtf(v); +} +template<> double rsqrt(double v) { + return rsqrt(v); +} + +namespace { +// This is the un-specialized struct. Note that we prevent instantiation of this +// struct by putting an undefined symbol in the function body so it won't compile. +// template +// struct SharedMemory +// { +// // Ensure that we won't compile any un-specialized types +// __device__ T *getPointer() +// { +// extern __device__ void error(void); +// error(); +// return NULL; +// } +// }; +// https://github.com/NVIDIA/apex/issues/246 +template +struct SharedMemory; + +template <> +struct SharedMemory +{ + __device__ float *getPointer() + { + extern __shared__ float s_float[]; + return s_float; + } +}; + +template <> +struct SharedMemory +{ + __device__ double *getPointer() + { + extern __shared__ double s_double[]; + return s_double; + } +}; +} + +template __device__ +void cuApplyLayerNorm_( + V* __restrict__ output_vals, + U* __restrict__ mean, + U* __restrict__ invvar, + const T* __restrict__ vals, + const int n1, + const int n2, + const U epsilon, + const V* __restrict__ gamma, + const V* __restrict__ beta, + bool rms_only + ) +{ + // Assumptions: + // 1) blockDim.x == warpSize + // 2) Tensors are contiguous + // + for (auto i1=blockIdx.y; i1 < n1; i1 += gridDim.y) { + SharedMemory shared; + U* buf = shared.getPointer(); + U mu,sigma2; + cuWelfordMuSigma2(vals,n1,n2,i1,mu,sigma2,buf,rms_only); + + const T* lvals = vals + i1*n2; + V* ovals = output_vals + i1*n2; + U c_invvar = rsqrt(sigma2 + epsilon); + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + if (gamma != NULL && (beta != NULL || rms_only)) { + for (int i = thrx; i < n2; i+=numx) { + U curr = static_cast(lvals[i]); + if (!rms_only) { + ovals[i] = gamma[i] * static_cast(c_invvar * (curr - mu)) + beta[i]; + } else { + ovals[i] = gamma[i] * static_cast(c_invvar * curr); + } + + } + } else { + for (int i = thrx; i < n2; i+=numx) { + U curr = static_cast(lvals[i]); + if (!rms_only) { + ovals[i] = static_cast(c_invvar * (curr - mu)); + } else { + ovals[i] = static_cast(c_invvar * curr); + } + } + } + if (threadIdx.x == 0 && threadIdx.y == 0) { + if (!rms_only) { + mean[i1] = mu; + } + invvar[i1] = c_invvar; + } + __syncthreads(); + } +} + +template __global__ +void cuApplyLayerNorm( + V* __restrict__ output_vals, + U* __restrict__ mean, + U* __restrict__ invvar, + const T* __restrict__ vals, + const int n1, + const int n2, + const U epsilon, + const V* __restrict__ gamma, + const V* __restrict__ beta + ) +{ + cuApplyLayerNorm_(output_vals, mean, invvar, vals, n1, n2, epsilon, gamma, beta, false); +} + +template __global__ +void cuApplyRMSNorm( + V* __restrict__ output_vals, + U* __restrict__ invvar, + const T* __restrict__ vals, + const int n1, + const int n2, + const U epsilon, + const V* __restrict__ gamma) +{ + cuApplyLayerNorm_(output_vals, NULL, invvar, vals, n1, n2, epsilon, gamma, NULL, true); +} + +template __device__ +void cuLoadWriteStridedInputs( + const int i1_block, + const int thr_load_row_off, + const int thr_load_col_off, + const int i2_off, + const int row_stride, + U* warp_buf1, + U* warp_buf2, + const T* input, + const V* dout, + const int i1_end, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + bool rms_only + ) +{ + int i1 = i1_block+thr_load_row_off; + if (i1 < i1_end) { + U curr_mean; + if (!rms_only) { + curr_mean = mean[i1]; + } + U curr_invvar = invvar[i1]; + for (int k = 0; k < blockDim.y; ++k) { + int i2 = i2_off + k; + int load_idx = i1*n2+i2; + int write_idx = thr_load_row_off*row_stride+thr_load_col_off+k; + if (i2(input[load_idx]); + U curr_dout = static_cast(dout[load_idx]); + if (!rms_only) { + warp_buf1[write_idx] = curr_dout; + warp_buf2[write_idx] = curr_dout * (curr_input - curr_mean) * curr_invvar; + } else { + warp_buf2[write_idx] = curr_dout * (curr_input) * curr_invvar; + } + } else { + if (!rms_only) { + warp_buf1[write_idx] = U(0); + } + warp_buf2[write_idx] = U(0); + } + } + } else { + for (int k = 0; k < blockDim.y; ++k) { + int write_idx = thr_load_row_off*row_stride+thr_load_col_off+k; + if (!rms_only) { + warp_buf1[write_idx] = U(0); + } + warp_buf2[write_idx] = U(0); + } + } +} + +template __device__ +void cuLoadAddStridedInputs( + const int i1_block, + const int thr_load_row_off, + const int thr_load_col_off, + const int i2_off, + const int row_stride, + U* warp_buf1, + U* warp_buf2, + const T* input, + const V* dout, + const int i1_end, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + bool rms_only + ) +{ + int i1 = i1_block+thr_load_row_off; + if (i1 < i1_end) { + U curr_mean; + if (!rms_only) { + curr_mean = mean[i1]; + } + U curr_invvar = invvar[i1]; + for (int k = 0; k < blockDim.y; ++k) { + int i2 = i2_off + k; + int load_idx = i1*n2+i2; + int write_idx = thr_load_row_off*row_stride+thr_load_col_off+k; + if (i2(input[load_idx]); + U curr_dout = static_cast(dout[load_idx]); + if (!rms_only) { + warp_buf1[write_idx] += curr_dout; + warp_buf2[write_idx] += curr_dout * (curr_input - curr_mean) * curr_invvar; + } else { + warp_buf2[write_idx] += curr_dout * (curr_input) * curr_invvar; + } + } + } + } +} + + +template __global__ +void cuComputePartGradGammaBeta( + const V* __restrict__ dout, + const T* __restrict__ input, + const int n1, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + U epsilon, + U* part_grad_gamma, + U* part_grad_beta, + bool rms_only) +{ + const int numsegs_n1 = (n1+blockDim.y*blockDim.y-1) / (blockDim.y*blockDim.y); + const int segs_per_block = (numsegs_n1 + gridDim.y - 1) / gridDim.y; + const int i1_beg = blockIdx.y * segs_per_block * blockDim.y*blockDim.y; + const int i1_beg_plus_one = (blockIdx.y+1) * segs_per_block * blockDim.y*blockDim.y; + const int i1_end = i1_beg_plus_one < n1 ? i1_beg_plus_one : n1; + const int row_stride = blockDim.x+1; + const int thr_load_col_off = (threadIdx.x*blockDim.y)&(blockDim.x-1); + const int thr_load_row_off = (threadIdx.x*blockDim.y)/blockDim.x + threadIdx.y*blockDim.y; + const int i2_off = blockIdx.x * blockDim.x + thr_load_col_off; + SharedMemory shared; + U* buf = shared.getPointer(); // buf has at least blockDim.x * blockDim.y * blockDim.y + (blockDim.y - 1)*(blockDim.x/blockDim.y) elements + U* warp_buf1 = (U*)buf; + U* warp_buf2 = warp_buf1 + blockDim.y * blockDim.y * row_stride; + // compute partial sums from strided inputs + // do this to increase number of loads in flight + cuLoadWriteStridedInputs(i1_beg,thr_load_row_off,thr_load_col_off,i2_off,row_stride,warp_buf1,warp_buf2,input,dout,i1_end,n2,mean,invvar, rms_only); + for (int i1_block = i1_beg+blockDim.y*blockDim.y; i1_block < i1_end; i1_block+=blockDim.y*blockDim.y) { + cuLoadAddStridedInputs(i1_block,thr_load_row_off,thr_load_col_off,i2_off,row_stride,warp_buf1,warp_buf2,input,dout,i1_end,n2,mean,invvar, rms_only); + } + __syncthreads(); + // inter-warp reductions + // sum within each warp + U acc1 = U(0); + U acc2 = U(0); + for (int k = 0; k < blockDim.y; ++k) { + int row1 = threadIdx.y + k*blockDim.y; + int idx1 = row1*row_stride + threadIdx.x; + if (!rms_only) { + acc1 += warp_buf1[idx1]; + } + acc2 += warp_buf2[idx1]; + } + if (!rms_only) { + warp_buf1[threadIdx.y*row_stride+threadIdx.x] = acc1; + } + warp_buf2[threadIdx.y*row_stride+threadIdx.x] = acc2; + __syncthreads(); + // sum all warps + for (int offset = blockDim.y/2; offset > 1; offset /= 2) { + if (threadIdx.y < offset) { + int row1 = threadIdx.y; + int row2 = threadIdx.y + offset; + int idx1 = row1*row_stride + threadIdx.x; + int idx2 = row2*row_stride + threadIdx.x; + if (!rms_only) { + warp_buf1[idx1] += warp_buf1[idx2]; + } + warp_buf2[idx1] += warp_buf2[idx2]; + } + __syncthreads(); + } + int i2 = blockIdx.x * blockDim.x + threadIdx.x; + if (threadIdx.y == 0 && i2 < n2) { + int row1 = threadIdx.y; + int row2 = threadIdx.y + 1; + int idx1 = row1*row_stride + threadIdx.x; + int idx2 = row2*row_stride + threadIdx.x; + if (!rms_only) { + part_grad_beta[blockIdx.y*n2+i2] = warp_buf1[idx1] + warp_buf1[idx2]; + } + part_grad_gamma[blockIdx.y*n2+i2] = warp_buf2[idx1] + warp_buf2[idx2]; + } +} + +template __global__ +void cuComputeGradGammaBeta( + const U* part_grad_gamma, + const U* part_grad_beta, + const int part_size, + const int n1, + const int n2, + V* grad_gamma, + V* grad_beta, + bool rms_only) +{ + // sum partial gradients for gamma and beta + SharedMemory shared; + U* buf = shared.getPointer(); + int i2 = blockIdx.x * blockDim.x + threadIdx.x; + if (i2 < n2) { + // each warp does sequential reductions until reduced part_size is num_warps + int num_warp_reductions = part_size / blockDim.y; + U sum_gamma = U(0); + U sum_beta = U(0); + const U* part_grad_gamma_ptr = part_grad_gamma + threadIdx.y * num_warp_reductions * n2 + i2; + const U* part_grad_beta_ptr = part_grad_beta + threadIdx.y * num_warp_reductions * n2 + i2; + for (int warp_offset = 0; warp_offset < num_warp_reductions; ++warp_offset) { + sum_gamma += part_grad_gamma_ptr[warp_offset*n2]; + if (!rms_only) { + sum_beta += part_grad_beta_ptr[warp_offset*n2]; + } + } + // inter-warp reductions + const int nbsize3 = blockDim.x * blockDim.y / 2; + for (int offset = blockDim.y/2; offset >= 1; offset /= 2) { + // top half write to shared memory + if (threadIdx.y >= offset && threadIdx.y < 2*offset) { + const int write_idx = (threadIdx.y - offset) * blockDim.x + threadIdx.x; + buf[write_idx] = sum_gamma; + if (!rms_only) { + buf[write_idx+nbsize3] = sum_beta; + } + } + __syncthreads(); + // bottom half sums + if (threadIdx.y < offset) { + const int read_idx = threadIdx.y * blockDim.x + threadIdx.x; + sum_gamma += buf[read_idx]; + if (!rms_only) { + sum_beta += buf[read_idx+nbsize3]; + } + } + __syncthreads(); + } + // write out fully summed gradients + if (threadIdx.y == 0) { + grad_gamma[i2] = sum_gamma; + if (!rms_only) { + grad_beta[i2] = sum_beta; + } + } + } +} + + +template __global__ +void cuComputeGradInput( + const V* __restrict__ dout, + const T* __restrict__ input, + const int n1, + const int n2, + const U* __restrict__ mean, + const U* __restrict__ invvar, + U epsilon, + const V* gamma, + T* grad_input, + bool rms_only) +{ + for (auto i1=blockIdx.y; i1 < n1; i1 += gridDim.y) { + U sum_loss1 = U(0); + U sum_loss2 = U(0); + U c_mean; + if (!rms_only) { + c_mean = mean[i1]; + } + const U c_invvar = invvar[i1]; + const T* k_input = input + i1*n2; + const V* k_dout = dout + i1*n2; + const int numx = blockDim.x * blockDim.y; + const int thrx = threadIdx.x + threadIdx.y * blockDim.x; + if (gamma != NULL) { + int l = 4*thrx; + for (; l+3 < n2; l+=4*numx) { + for (int k = 0; k < 4; ++k) { + const U c_h = static_cast(k_input[l+k]); + const U c_loss = static_cast(k_dout[l+k]); + if (!rms_only) { + sum_loss1 += c_loss * gamma[l+k]; + sum_loss2 += c_loss * gamma[l+k] * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * gamma[l+k] * (c_h) * c_invvar; + } + } + } + for (; l < n2; ++l) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + if (!rms_only) { + sum_loss1 += c_loss * gamma[l]; + sum_loss2 += c_loss * gamma[l] * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * gamma[l] * (c_h) * c_invvar; + } + + } + } else { + int l = 4*thrx; + for (; l+3 < n2; l+=4*numx) { + for (int k = 0; k < 4; ++k) { + const U c_h = static_cast(k_input[l+k]); + const U c_loss = static_cast(k_dout[l+k]); + if (!rms_only) { + sum_loss1 += c_loss; + sum_loss2 += c_loss * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * (c_h) * c_invvar; + } + } + } + for (; l < n2; ++l) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + if (!rms_only) { + sum_loss1 += c_loss; + sum_loss2 += c_loss * (c_h - c_mean) * c_invvar; + } else { + sum_loss2 += c_loss * (c_h) * c_invvar; + } + } + } + // intra-warp reductions + for (int mask = blockDim.x/2; mask > 0; mask /= 2) { + if (!rms_only) { + sum_loss1 += WARP_SHFL_XOR(sum_loss1, mask); + } + sum_loss2 += WARP_SHFL_XOR(sum_loss2, mask); + } + // inter-warp reductions + if (blockDim.y > 1) { + SharedMemory shared; + U* buf = shared.getPointer(); + for (int offset = blockDim.y/2; offset > 0; offset /= 2) { + // upper half of warps write to shared + if (threadIdx.y >= offset && threadIdx.y < 2*offset) { + const int wrt_i = (threadIdx.y - offset) * blockDim.x + threadIdx.x; + if (!rms_only) { + buf[2*wrt_i] = sum_loss1; + } + buf[2*wrt_i+1] = sum_loss2; + } + __syncthreads(); + // lower half merges + if (threadIdx.y < offset) { + const int read_i = threadIdx.y * blockDim.x + threadIdx.x; + if (!rms_only) { + sum_loss1 += buf[2*read_i]; + } + sum_loss2 += buf[2*read_i+1]; + } + __syncthreads(); + } + if (threadIdx.y == 0) { + if (!rms_only) { + buf[2*threadIdx.x] = sum_loss1; + } + buf[2*threadIdx.x+1] = sum_loss2; + } + __syncthreads(); + if (threadIdx.y !=0) { + if (!rms_only) { + sum_loss1 = buf[2*threadIdx.x]; + } + sum_loss2 = buf[2*threadIdx.x+1]; + } + } + // all threads now have the two sums over l + U fH = (U)n2; + U term1 = (U(1) / fH) * c_invvar; + T* k_grad_input = grad_input + i1*n2; + if (gamma != NULL) { + for (int l = thrx; l < n2; l+=numx) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + U f_grad_input = fH * c_loss * gamma[l]; + if (!rms_only) { + f_grad_input -= sum_loss1; + f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2; + } else { + f_grad_input -= (c_h) * c_invvar * sum_loss2; + } + f_grad_input *= term1; + k_grad_input[l] = static_cast(f_grad_input); + } + } else { + for (int l = thrx; l < n2; l+=numx) { + const U c_h = static_cast(k_input[l]); + const U c_loss = static_cast(k_dout[l]); + U f_grad_input = fH * c_loss; + if (!rms_only) { + f_grad_input -= sum_loss1; + f_grad_input -= (c_h - c_mean) * c_invvar * sum_loss2; + } else { + f_grad_input -= (c_h) * c_invvar * sum_loss2; + } + f_grad_input *= term1; + k_grad_input[l] = static_cast(f_grad_input); + } + } + // prevent race where buf is written again before reads are done + __syncthreads(); + } +} + + +template +void HostApplyLayerNorm( + V* output, + U* mean, + U* invvar, + const T* input, + int n1, + int n2, + double epsilon, + const V* gamma, + const V* beta + ) +{ + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const dim3 threads(32,4,1); + const uint64_t maxGridY = at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + const dim3 blocks(1, std::min((uint64_t)n1, maxGridY), 1); + int nshared = + threads.y > 1 ? + threads.y*sizeof(U)+(threads.y/2)*sizeof(U) : + 0; + cuApplyLayerNorm<<>>( + output, mean, invvar, input, n1, n2, U(epsilon), gamma, beta); +} + +template +void HostApplyRMSNorm( + V* output, + U* invvar, + const T* input, + int n1, + int n2, + double epsilon, + const V* gamma) +{ + auto stream = at::cuda::getCurrentCUDAStream().stream(); + const dim3 threads(32,4,1); + const uint64_t maxGridY = at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + const dim3 blocks(1, std::min((uint64_t)n1, maxGridY), 1); + int nshared = + threads.y > 1 ? + threads.y*sizeof(U)+(threads.y/2)*sizeof(U) : + 0; + cuApplyRMSNorm<<>>( + output, invvar, input, n1, n2, U(epsilon), gamma); +} + +void cuda_layer_norm( + at::Tensor* output, + at::Tensor* mean, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + at::Tensor* beta, + double epsilon) +{ + using namespace at; + DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES( + input->scalar_type(), output->scalar_type(), "layer_norm_cuda_kernel", + using accscalar_t = at::acc_type; + HostApplyLayerNorm( + output->DATA_PTR(), + mean->DATA_PTR(), + invvar->DATA_PTR(), + input->DATA_PTR(), + n1,n2, + epsilon, + gamma != NULL ? gamma->DATA_PTR() : NULL, + beta != NULL ? beta->DATA_PTR() : NULL); + ) +} + +void cuda_rms_norm( + at::Tensor* output, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + double epsilon) +{ + using namespace at; + DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES( + input->scalar_type(), output->scalar_type(), "rms_norm_cuda_kernel", + using accscalar_t = at::acc_type; + HostApplyRMSNorm( + output->DATA_PTR(), + invvar->DATA_PTR(), + input->DATA_PTR(), + n1,n2, + epsilon, + gamma != NULL ? gamma->DATA_PTR() : NULL); + ) +} + + +template +void HostLayerNormGradient( + const V* dout, + const U* mean, + const U* invvar, + at::Tensor* input, + int n1, + int n2, + const V* gamma, + const V* beta, + double epsilon, + T* grad_input, + V* grad_gamma, + V* grad_beta + ) +{ + auto stream = at::cuda::getCurrentCUDAStream().stream(); + + if (gamma != NULL && beta != NULL) { + // compute grad_gamma(j) and grad_beta(j) + const int part_size = 16; + const dim3 threads2(32,4,1); + const dim3 blocks2((n2+threads2.x-1)/threads2.x,part_size,1); + const int nshared2_a = 2 * sizeof(U) * threads2.y * threads2.y * (threads2.x + 1); + const int nshared2_b = threads2.x * threads2.y * sizeof(U); + const int nshared2 = nshared2_a > nshared2_b ? nshared2_a : nshared2_b; + // note (mkozuki): I can hard code part_grad_gamma's dtype as float given that + // the `cuda_layer_norm_gradient` doesn't support double. + const auto part_grad_dtype = + (input->scalar_type() == at::ScalarType::Half || input->scalar_type() == at::ScalarType::BFloat16) ? + at::ScalarType::Float : + input->scalar_type(); + at::Tensor part_grad_gamma = at::empty({part_size,n2}, input->options().dtype(part_grad_dtype)); + at::Tensor part_grad_beta = at::empty_like(part_grad_gamma); + cuComputePartGradGammaBeta<<>>( + dout, + input->DATA_PTR(), + n1,n2, + mean, + invvar, + U(epsilon), + part_grad_gamma.DATA_PTR(), + part_grad_beta.DATA_PTR(), + false); + + const dim3 threads3(32,8,1); + const dim3 blocks3((n2+threads2.x-1)/threads2.x,1,1); + const int nshared3 = threads3.x * threads3.y * sizeof(U); + cuComputeGradGammaBeta<<>>( + part_grad_gamma.DATA_PTR(), + part_grad_beta.DATA_PTR(), + part_size, + n1,n2, + grad_gamma, + grad_beta, + false); + } + + // compute grad_input + const uint64_t maxGridY = at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + const dim3 blocks1(1, std::min((uint64_t)n1, maxGridY), 1); + const dim3 threads1(32,4,1); + int nshared = + threads1.y > 1 ? + threads1.y*threads1.x*sizeof(U) : + 0; + cuComputeGradInput<<>>( + dout, + input->DATA_PTR(), + n1,n2, + mean, + invvar, + U(epsilon), + gamma, + grad_input, + false); +} + +template +void HostRMSNormGradient( + const V* dout, + const U* invvar, + at::Tensor* input, + int n1, + int n2, + const V* gamma, + double epsilon, + T* grad_input, + V* grad_gamma) +{ + auto stream = at::cuda::getCurrentCUDAStream().stream(); + + if (gamma != NULL) { + const int part_size = 16; + const dim3 threads2(32,4,1); + const dim3 blocks2((n2+threads2.x-1)/threads2.x,part_size,1); + const int nshared2_a = 2 * sizeof(U) * threads2.y * threads2.y * (threads2.x + 1); + const int nshared2_b = threads2.x * threads2.y * sizeof(U); + const int nshared2 = nshared2_a > nshared2_b ? nshared2_a : nshared2_b; + // note (mkozuki): I can hard code part_grad_gamma's dtype as float given that + // the `cuda_layer_norm_gradient` doesn't support double. + const auto part_grad_dtype = + (input->scalar_type() == at::ScalarType::Half || input->scalar_type() == at::ScalarType::BFloat16) ? + at::ScalarType::Float : + input->scalar_type(); + at::Tensor part_grad_gamma = at::empty({part_size,n2}, input->options().dtype(part_grad_dtype)); + cuComputePartGradGammaBeta<<>>( + dout, + input->DATA_PTR(), + n1,n2, + invvar, // unused + invvar, + U(epsilon), + part_grad_gamma.DATA_PTR(), + part_grad_gamma.DATA_PTR(), /* unused */ + true); + + const dim3 threads3(32,8,1); + const dim3 blocks3((n2+threads2.x-1)/threads2.x,1,1); + const int nshared3 = threads3.x * threads3.y * sizeof(U); + cuComputeGradGammaBeta<<>>( + part_grad_gamma.DATA_PTR(), + part_grad_gamma.DATA_PTR(), /* unused */ + part_size, + n1,n2, + grad_gamma, + grad_gamma, /* unused */ + true); + } + + // compute grad_input + const uint64_t maxGridY = at::cuda::getCurrentDeviceProperties()->maxGridSize[1]; + const dim3 blocks1(1, std::min((uint64_t)n1, maxGridY), 1); + const dim3 threads1(32,4,1); + int nshared = + threads1.y > 1 ? + threads1.y*threads1.x*sizeof(U) : + 0; + cuComputeGradInput<<>>( + dout, + input->DATA_PTR(), + n1,n2, + invvar, /* unused */ + invvar, + U(epsilon), + gamma, + grad_input, + true); +} + +void cuda_layer_norm_gradient( + at::Tensor* dout, + at::Tensor* mean, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + at::Tensor* beta, + double epsilon, + at::Tensor* grad_input, + at::Tensor* grad_gamma, + at::Tensor* grad_beta) +{ + using namespace at; + // we can do away with `accscalar_t` as there're only three dtypes: fp32, fp16, bf16 + DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES( + input->scalar_type(), gamma == NULL ? input->scalar_type() : gamma->scalar_type(), "cuComputeGradInput", + using accscalar_t = at::acc_type; + HostLayerNormGradient( + dout->DATA_PTR(), + mean->DATA_PTR(), + invvar->DATA_PTR(), + input, + n1,n2, + // TMJ pass NULL argument for gamma, beta, grad_gamma and grad_beta + // if gamma Tensor is NULL on input. + gamma != NULL ? gamma->DATA_PTR() : NULL, + gamma != NULL ? beta->DATA_PTR() : NULL, + epsilon, + grad_input->DATA_PTR(), + gamma != NULL ? grad_gamma->DATA_PTR() : NULL, + gamma != NULL ? grad_beta->DATA_PTR() : NULL); + ) +} + +void cuda_rms_norm_gradient( + at::Tensor* dout, + at::Tensor* invvar, + at::Tensor* input, + int n1, + int n2, + #ifdef VERSION_GE_1_1 + at::IntArrayRef normalized_shape, + #else + at::IntList normalized_shape, + #endif + at::Tensor* gamma, + double epsilon, + at::Tensor* grad_input, + at::Tensor* grad_gamma) +{ + using namespace at; + // we can do away with `accscalar_t` as there're only three dtypes: fp32, fp16, bf16 + // DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES( + DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES( + input->scalar_type(), gamma == NULL ? input->scalar_type() : gamma->scalar_type(), "cuComputeGradInputRMS", + using accscalar_t = at::acc_type; + HostRMSNormGradient( + dout->DATA_PTR(), + invvar->DATA_PTR(), + input, + n1,n2, + // TMJ pass NULL argument for gamma, beta, grad_gamma and grad_beta + // if gamma Tensor is NULL on input. + gamma != NULL ? gamma->DATA_PTR() : NULL, + epsilon, + grad_input->DATA_PTR(), + gamma != NULL ? grad_gamma->DATA_PTR() : NULL); + ) +} diff --git a/apex/csrc/megatron/fused_weight_gradient_dense.cpp b/apex/csrc/megatron/fused_weight_gradient_dense.cpp new file mode 100644 index 00000000..a14c2b21 --- /dev/null +++ b/apex/csrc/megatron/fused_weight_gradient_dense.cpp @@ -0,0 +1,21 @@ +#include + +#include +#include + +void wgrad_gemm_accum_fp32_cuda_stub( + at::Tensor &input_2d, + at::Tensor &d_output_2d, + at::Tensor &d_weight +); + +void wgrad_gemm_accum_fp16_cuda_stub( + at::Tensor &input_2d, + at::Tensor &d_output_2d, + at::Tensor &d_weight +); + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("wgrad_gemm_accum_fp32", &wgrad_gemm_accum_fp32_cuda_stub, "wgrad gemm accum in fp32"); + m.def("wgrad_gemm_accum_fp16", &wgrad_gemm_accum_fp16_cuda_stub, "wgrad gemm accum in fp16"); +} diff --git a/apex/csrc/megatron/fused_weight_gradient_dense_16bit_prec_cuda.cu b/apex/csrc/megatron/fused_weight_gradient_dense_16bit_prec_cuda.cu new file mode 100644 index 00000000..60d1e8d1 --- /dev/null +++ b/apex/csrc/megatron/fused_weight_gradient_dense_16bit_prec_cuda.cu @@ -0,0 +1,155 @@ +#include +#include +#include +#include + +#include +#include +#include + +/* Includes, cuda */ +#include +#include + +#include "type_shim.h" + + +// BF16 inputs and BF16 accumulation +void gemmex_wrapper_fp16( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float* beta, + at::BFloat16* C, + int ldc) { + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16BF, + lda, + B, + CUDA_R_16BF, + ldb, + beta, + C, + CUDA_R_16BF, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); +} + +// FP16 inputs and FP16 accumulation +void gemmex_wrapper_fp16( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float* beta, + at::Half* C, + int ldc) { + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16F, + lda, + B, + CUDA_R_16F, + ldb, + beta, + C, + CUDA_R_16F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); +} + +template +void wgrad_gemm_accum_fp16_cuda(T *input, T *d_output, T *d_weight, int in_dim, int hidden_dim, int out_dim) { + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream; + cublasGetStream(handle, &stream); + const float alpha = 1.0; + const float beta = 1.0; + + gemmex_wrapper_fp16( + handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + in_dim, + out_dim, + hidden_dim, + &alpha, + input, + in_dim, + d_output, + out_dim, + &beta, + d_weight, + in_dim); +} + +template void wgrad_gemm_accum_fp16_cuda(at::Half *input, at::Half *d_output, at::Half *d_weight, int in_dim, int hidden_dim, int out_dim); +template void wgrad_gemm_accum_fp16_cuda(at::BFloat16 *input, at::BFloat16 *d_output, at::BFloat16 *d_weight, int in_dim, int hidden_dim, int out_dim); + +void wgrad_gemm_accum_fp16_cuda_stub( + at::Tensor &input, + at::Tensor &d_output, + at::Tensor &d_weight +) { + at::Tensor input_2d, d_output_2d; + // input tensor: collapse to the first dim + auto in_sizes = input.sizes(); + if (input.dim() > 2) { + input_2d = input.view({-1, in_sizes[in_sizes.size() - 1]}); + } else { + input_2d = input; + } + // d_output tensor: collapse to the first dim + auto d_out_sizes = d_output.sizes(); + if (d_output.dim() > 2) { + d_output_2d = d_output.view({-1, d_out_sizes[d_out_sizes.size() - 1]}); + } else { + d_output_2d = d_output; + } + + const int hidden_dim = input_2d.size(0); + const int in_dim = input_2d.size(1); + const int out_dim = d_weight.size(0); + + DISPATCH_HALF_AND_BFLOAT(input_2d.scalar_type(), "wgrad_gemm_accum_fp16", + wgrad_gemm_accum_fp16_cuda( + input_2d.data_ptr(), + d_output_2d.data_ptr(), + d_weight.data_ptr(), + in_dim, + hidden_dim, + out_dim); + ); +} diff --git a/apex/csrc/megatron/fused_weight_gradient_dense_cuda.cu b/apex/csrc/megatron/fused_weight_gradient_dense_cuda.cu new file mode 100644 index 00000000..dfaa1345 --- /dev/null +++ b/apex/csrc/megatron/fused_weight_gradient_dense_cuda.cu @@ -0,0 +1,195 @@ +#include +#include +#include +#include + +#include +#include +#include + +/* Includes, cuda */ +#include +#include + +#include "type_shim.h" + + +// BF16 Tensor core wrapper around cublas GEMMEx +void gemmex_wrapper( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + at::BFloat16* A, + int lda, + at::BFloat16* B, + int ldb, + const float* beta, + float* C, + int ldc) { + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16BF, + lda, + B, + CUDA_R_16BF, + ldb, + beta, + C, + CUDA_R_32F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); +} + +// FP16 Tensor core wrapper around cublas GEMMEx +void gemmex_wrapper( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float* alpha, + at::Half* A, + int lda, + at::Half* B, + int ldb, + const float* beta, + float* C, + int ldc) { + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16F, + lda, + B, + CUDA_R_16F, + ldb, + beta, + C, + CUDA_R_32F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); +} + +// FP32 wrapper around cublas GEMMEx +void gemmex_wrapper( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + const float *alpha, + float *A, + int lda, + float *B, + int ldb, + const float *beta, + float *C, + int ldc) { + TORCH_CUDABLAS_CHECK(cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_32F, + lda, + B, + CUDA_R_32F, + ldb, + beta, + C, + CUDA_R_32F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP)); +} + +template +void wgrad_gemm_accum_fp32_cuda(T *input, T *d_output, float *d_weight, int in_dim, int hidden_dim, int out_dim) { + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + cudaStream_t stream; + cublasGetStream(handle, &stream); + const float alpha = 1.0; + const float beta = 1.0; + + gemmex_wrapper( + handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + in_dim, + out_dim, + hidden_dim, + &alpha, + input, + in_dim, + d_output, + out_dim, + &beta, + d_weight, + in_dim); +} + +template void wgrad_gemm_accum_fp32_cuda(at::Half *input, at::Half *d_output, float *d_weight, int in_dim, int hidden_dim, int out_dim); +template void wgrad_gemm_accum_fp32_cuda(at::BFloat16 *input, at::BFloat16 *d_output, float *d_weight, int in_dim, int hidden_dim, int out_dim); +template void wgrad_gemm_accum_fp32_cuda(float *input, float *d_output, float *d_weight, int in_dim, int hidden_dim, int out_dim); + + +void wgrad_gemm_accum_fp32_cuda_stub( + at::Tensor &input, + at::Tensor &d_output, + at::Tensor &d_weight +) { + at::Tensor input_2d, d_output_2d; + // input tensor: collapse to the first dim + auto in_sizes = input.sizes(); + if (input.dim() > 2) { + input_2d = input.view({-1, in_sizes[in_sizes.size() - 1]}); + } else { + input_2d = input; + } + // d_output tensor: collapse to the first dim + auto d_out_sizes = d_output.sizes(); + if (d_output.dim() > 2) { + d_output_2d = d_output.view({-1, d_out_sizes[d_out_sizes.size() - 1]}); + } else { + d_output_2d = d_output; + } + + const int hidden_dim = input_2d.size(0); + const int in_dim = input_2d.size(1); + const int out_dim = d_weight.size(0); + + DISPATCH_FLOAT_HALF_AND_BFLOAT(input_2d.scalar_type(), 0, "wgrad_gemm_accum_fp32", + wgrad_gemm_accum_fp32_cuda( + input_2d.data_ptr(), + d_output_2d.data_ptr(), + d_weight.data_ptr(), + in_dim, + hidden_dim, + out_dim); + ); +} diff --git a/apex/csrc/megatron/generic_scaled_masked_softmax.cpp b/apex/csrc/megatron/generic_scaled_masked_softmax.cpp new file mode 100644 index 00000000..190f6e7e --- /dev/null +++ b/apex/csrc/megatron/generic_scaled_masked_softmax.cpp @@ -0,0 +1,83 @@ +/* coding=utf-8 + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +namespace multihead_attn +{ + namespace fused_softmax + { + namespace generic_scaled_masked_softmax + { + + torch::Tensor fwd_cuda( + torch::Tensor const &input, + torch::Tensor const &mask, + float scale_factor); + + torch::Tensor bwd_cuda( + torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + float scale_factor); + + torch::Tensor fwd( + torch::Tensor const &input, + torch::Tensor const &mask, + float scale_factor) + { + TORCH_CHECK(input.dim() == 4, "expected 4D tensor"); + TORCH_CHECK((input.scalar_type() == at::ScalarType::Half) || + (input.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + TORCH_CHECK(mask.dim() == 4, "expected 4D tensor"); + + return fwd_cuda(input, mask, scale_factor); + } + + torch::Tensor bwd( + torch::Tensor const &output_grads, + torch::Tensor const &softmax_results, + float scale_factor) + { + + TORCH_CHECK(output_grads.dim() == 4, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 4, "expected 3D tensor"); + + TORCH_CHECK((output_grads.scalar_type() == at::ScalarType::Half) || + (output_grads.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + TORCH_CHECK((softmax_results.scalar_type() == at::ScalarType::Half) || + (softmax_results.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + + return bwd_cuda(output_grads, softmax_results, scale_factor); + } + + } // end namespace generic_scaled_masked_softmax + } // end namespace fused_softmax +} // end namespace multihead_attn + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", + &multihead_attn::fused_softmax::generic_scaled_masked_softmax::fwd, + "Self Multihead Attention scaled, time masked softmax -- Forward."); + + m.def("backward", + &multihead_attn::fused_softmax::generic_scaled_masked_softmax::bwd, + "Self Multihead Attention scaled, time masked softmax -- Backward."); +} diff --git a/apex/csrc/megatron/generic_scaled_masked_softmax.h b/apex/csrc/megatron/generic_scaled_masked_softmax.h new file mode 100644 index 00000000..4ff50feb --- /dev/null +++ b/apex/csrc/megatron/generic_scaled_masked_softmax.h @@ -0,0 +1,384 @@ +/* coding=utf-8 + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +template +struct Add { + __device__ __forceinline__ T operator()(T a, T b) const { + return a + b; + } +}; + +template +struct Max { + __device__ __forceinline__ T operator()(T a, T b) const { + return a < b ? b : a; + } +}; + +template +__device__ __forceinline__ T WARP_SHFL_DOWN_NATIVE(T value, int laneMask, int width = warpSize, unsigned int mask = 0xffffffff) +{ +#if CUDA_VERSION >= 9000 + return __shfl_down_sync(mask, value, laneMask, width); +#else + return __shfl_down(value, laneMask, width); +#endif +} + +template class ReduceOp> +__device__ __forceinline__ acc_t warp_reduce_new(acc_t val) { + ReduceOp r; + #pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) + { + val = r(val, WARP_SHFL_DOWN_NATIVE(val, offset, WARP_SIZE)); + } + return val; +} + + +template +__global__ void scaled_masked_softmax_warp_backward_new( + output_t *gradInput, //[batches, attn_heads, q_len, k_len] + input_t *grad, + const input_t *output, //[batches, attn_heads, q_len, k_len] + acc_t scale, + int element_count) +{ + int threads_per_block = blockDim.x; + //the first element_count*2 elements are used for cache, the last 128 is used for reduction + extern __shared__ acc_t shared_data[]; + input_t *local_data = (input_t *)shared_data; + input_t *output_data = &local_data[element_count]; + // maximum shared cached 128, enough for 4096 elements reduction into 4096/32= 128 elements + acc_t *shared = (acc_t *)(&(local_data[element_count*2])); + + int num_reductions = (element_count - 1) / threads_per_block + 1; + + int offset = blockIdx.x * element_count; + + int local_idx = threadIdx.x; + int lane = threadIdx.x % C10_WARP_SIZE; + int wid = threadIdx.x / C10_WARP_SIZE; + int warps_per_thread_block = threads_per_block / C10_WARP_SIZE; + + // load the data to local data + acc_t val = 0.0; + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < element_count){ + val = output[offset + i*threads_per_block + local_idx]; + output_data[i*threads_per_block + local_idx] = val; + local_data[i*threads_per_block + local_idx] = val * grad[offset + i*threads_per_block + local_idx]; + } + __syncthreads(); + } + + // find the sum + for (int i = local_idx; i < (element_count - 1) / C10_WARP_SIZE + 1; i += threads_per_block){ + shared[i] = 0.0; + } + __syncthreads(); + + #pragma unroll + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < element_count){ + val = local_data[i*threads_per_block + local_idx]; + } + else{ + val = 0.0; + } + __syncthreads(); + val = warp_reduce_new(val); + if (lane==0 && wid + warps_per_thread_block * i < (element_count - 1) / C10_WARP_SIZE + 1) { + shared[wid + warps_per_thread_block*i] = val; + } + __syncthreads(); + } + + // final shared reduction + + int shared_mem_len = (element_count - 1) / C10_WARP_SIZE + 1; + int num_warps = (shared_mem_len - 1) / C10_WARP_SIZE + 1; + while ( shared_mem_len > 1 ){ + #pragma unroll + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < shared_mem_len){ + val = shared[i*threads_per_block + local_idx]; + } + else{ + val = 0.0; + } + __syncthreads(); + val = warp_reduce_new(val); + if (lane==0) { + shared[wid + warps_per_thread_block * i] = val; + } + __syncthreads(); + } + shared_mem_len = num_warps; + num_warps = (shared_mem_len - 1) / C10_WARP_SIZE + 1; + } + val = shared[0]; + #pragma unroll + for (int i = local_idx; i < element_count; i += threads_per_block){ + gradInput[offset + i] = (output_t)(scale*(local_data[i] - output_data[i]*val)); + } +} + +} // end of anonymous namespace + +template +void dispatch_scaled_masked_softmax_backward_new( + output_t *grad_input, + input_t *grad, + const input_t *output, + const acc_t scale, + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads) +{ + if (key_seq_len == 0) + { + return; + } + else + { + int batch_count = batches * attn_heads * query_seq_len; + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + int num_warps = (key_seq_len - 1) / C10_WARP_SIZE + 1; + dim3 blocks(batch_count, 1, 1); + dim3 threads(threads_per_block, 1, 1); + + scaled_masked_softmax_warp_backward_new + <<>>(grad_input, grad, output, scale, key_seq_len); + } +} + +/* + * Extended softmax (from native aten pytorch) with following additional features + * 1) input scaling + * 2) Explicit masking + */ +template +__global__ void scaled_masked_softmax_warp_forward_new( + output_t *dst, + const input_t *src, + const uint8_t *mask, + const acc_t scale, + int query_len, // query_len + int attn_heads, + int element_count, // key_len + int pad_batches) // mask batch size +{ + // min threawds_per_block has to be bigger than 128 + int threads_per_block = blockDim.x; + // the first element_count is used for cache, the last 128 is used for reduction + extern __shared__ acc_t local_data[]; + // maximum shared cached 128, enough for 4096 elements reduction into 4096/32= 128 elements + acc_t *shared = &(local_data[element_count]); + // number of 1024 threads reductions + int num_reductions = (element_count - 1) / threads_per_block + 1; + + int offset = blockIdx.x * element_count; + int mask_offset; + int query_id = blockIdx.x % query_len; + if (pad_batches == 1){ + // broadcaste the mask tensor + mask_offset = query_id * element_count; + } + else{ + int mask_batch_id = blockIdx.x / attn_heads / query_len; + mask_offset = (mask_batch_id * query_len + query_id) * element_count; + } + + int local_idx = threadIdx.x; + int lane = threadIdx.x % C10_WARP_SIZE; + int wid = threadIdx.x / C10_WARP_SIZE; + int warps_per_thread_block = threads_per_block / C10_WARP_SIZE; + + // load the data to local data + for (int i = local_idx; i < element_count; i += threads_per_block) + { + // TODO, use the copy vector method + if (mask[mask_offset + i] == 1) + { + local_data[i] = -10000.0; + } + else + { + local_data[i] = src[offset + i] * scale; + } + } + + // first find the max value + for (int i = local_idx; i < (element_count - 1) / C10_WARP_SIZE + 1; i += threads_per_block){ + shared[i] = -10000.0; + } + __syncthreads(); + acc_t val = -10000.0; + #pragma unroll + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < element_count){ + val = local_data[i*threads_per_block + local_idx]; + } + else{ + val = -10000.0; + } + __syncthreads(); + val = warp_reduce_new(val); + + if (lane==0 && wid + warps_per_thread_block * i < (element_count - 1) / C10_WARP_SIZE + 1) { + shared[wid + warps_per_thread_block*i] = val; + } + __syncthreads(); + } + + // final shared reduction + int shared_mem_len = (element_count - 1) / C10_WARP_SIZE + 1; + int num_warps = (shared_mem_len - 1) / C10_WARP_SIZE + 1; + while ( shared_mem_len > 1 ){ + #pragma unroll + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < shared_mem_len){ + val = shared[i*threads_per_block + local_idx]; + } + else{ + val = -10000.0; + } + __syncthreads(); + val = warp_reduce_new(val); + if (lane==0) { + shared[wid + warps_per_thread_block * i] = val; + } + __syncthreads(); + } + shared_mem_len = num_warps; + num_warps = (shared_mem_len - 1) / C10_WARP_SIZE + 1; + } + + acc_t reduced_val = shared[0]; + if (reduced_val < -10000.0 + 0.1){ + // if everything is masked, pay attention to nothing + #pragma unroll + for (int i = local_idx; i < element_count; i += threads_per_block){ + dst[offset + i] = 0.0; + } + return; + } + + // update the values + #pragma unroll + for (int i = local_idx; i < element_count; i += threads_per_block){ + local_data[i] = std::exp(local_data[i] - reduced_val); + } + + // find the sum + for (int i = local_idx; i < (element_count - 1) / C10_WARP_SIZE + 1; i += threads_per_block){ + shared[i] = 0.0; + } + __syncthreads(); + + #pragma unroll + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < element_count){ + val = local_data[i*threads_per_block + local_idx]; + } + else{ + val = 0.0; + } + __syncthreads(); + + val = warp_reduce_new(val); + if (lane==0 && wid + warps_per_thread_block * i < (element_count - 1) / C10_WARP_SIZE + 1) { + shared[wid + warps_per_thread_block*i] = val; + } + __syncthreads(); + } + + shared_mem_len = (element_count - 1) / C10_WARP_SIZE + 1; + num_warps = (shared_mem_len - 1) / C10_WARP_SIZE + 1; + while ( shared_mem_len > 1 ){ + #pragma unroll + for (int i = 0; i < num_reductions; i++){ + if (i*threads_per_block + local_idx < shared_mem_len){ + val = shared[i*threads_per_block + local_idx]; + } + else{ + val = 0.0; + } + __syncthreads(); + val = warp_reduce_new(val); + if (lane==0) { + shared[wid + warps_per_thread_block * i] = val; + } + __syncthreads(); + } + shared_mem_len = num_warps; + num_warps = (shared_mem_len - 1) / C10_WARP_SIZE + 1; + } + + reduced_val = shared[0]; + + #pragma unroll + for (int i = local_idx; i < element_count; i += threads_per_block){ + dst[offset + i] = local_data[i] / reduced_val; + } +} + + +template +void dispatch_scaled_masked_softmax_forward_new( + output_t *dst, + const input_t *src, + const uint8_t *mask, + const input_t scale, + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads, + int pad_batches) +{ + if (key_seq_len == 0) { + return; + } else { + int batch_count = batches * attn_heads * query_seq_len; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + // calculate the needed shared memory + int num_warps = (key_seq_len - 1) / C10_WARP_SIZE + 1; + + dim3 blocks(batch_count, 1, 1); + dim3 threads(threads_per_block, 1, 1); + scaled_masked_softmax_warp_forward_new + <<>>(dst, src, mask, scale, query_seq_len, attn_heads, key_seq_len, pad_batches); + } +} diff --git a/apex/csrc/megatron/generic_scaled_masked_softmax_cuda.cu b/apex/csrc/megatron/generic_scaled_masked_softmax_cuda.cu new file mode 100644 index 00000000..93cd94b3 --- /dev/null +++ b/apex/csrc/megatron/generic_scaled_masked_softmax_cuda.cu @@ -0,0 +1,114 @@ +/* coding=utf-8 + * Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "generic_scaled_masked_softmax.h" +#include "type_shim.h" + +namespace multihead_attn { +namespace fused_softmax { +namespace generic_scaled_masked_softmax { + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + torch::Tensor const& mask, + float scale_factor) +{ + // input is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len] + const int batches = input.size(0); + const int pad_batches = mask.size(0); + const int attn_heads = input.size(1); + const int query_seq_len = input.size(2); + const int key_seq_len = input.size(3); + TORCH_INTERNAL_ASSERT(pad_batches == 1 || pad_batches == batches); + TORCH_INTERNAL_ASSERT(mask.size(1) == 1); + TORCH_INTERNAL_ASSERT(mask.size(2) == query_seq_len); + TORCH_INTERNAL_ASSERT(mask.size(3) == key_seq_len); + + // Output + auto act_options = input.options().requires_grad(false); + torch::Tensor softmax_results = + torch::empty({batches, attn_heads, query_seq_len, key_seq_len}, act_options); + + // Softmax Intermediate Result Ptr + void* input_ptr = static_cast(input.data_ptr()); + void* mask_ptr = static_cast(mask.data_ptr()); + void* softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + DISPATCH_HALF_AND_BFLOAT( + input.scalar_type(), + "dispatch_scaled_masked_softmax_forward", + dispatch_scaled_masked_softmax_forward_new( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), + reinterpret_cast(mask_ptr), + scale_factor, + query_seq_len, + key_seq_len, + batches, + attn_heads, + pad_batches); + ); + return softmax_results; +} + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads_, + torch::Tensor const& softmax_results_, + float scale_factor) { + + auto output_grads = output_grads_.contiguous(); + auto softmax_results = softmax_results_.contiguous(); + + //output grads is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len] + const int batches = output_grads.size(0); + const int attn_heads = output_grads.size(1); + const int query_seq_len = output_grads.size(2); + const int key_seq_len = output_grads.size(3); + + auto act_options = output_grads.options(); + torch::Tensor input_grad = + torch::empty({batches, attn_heads, query_seq_len, key_seq_len}, act_options); + + void* output_grads_ptr = static_cast(output_grads.data_ptr()); + + //Softmax Grad + DISPATCH_HALF_AND_BFLOAT( + output_grads_.scalar_type(), + "dispatch_scaled_masked_softmax_backward", + dispatch_scaled_masked_softmax_backward_new( + reinterpret_cast(static_cast(input_grad.data_ptr())), + reinterpret_cast(output_grads_ptr), + reinterpret_cast(softmax_results.data_ptr()), + scale_factor, + query_seq_len, + key_seq_len, + batches, + attn_heads); + ); + + //backward pass is completely in-place + return input_grad; +} +} +} +} diff --git a/apex/csrc/megatron/scaled_masked_softmax.cpp b/apex/csrc/megatron/scaled_masked_softmax.cpp new file mode 100644 index 00000000..3d914fae --- /dev/null +++ b/apex/csrc/megatron/scaled_masked_softmax.cpp @@ -0,0 +1,105 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +namespace multihead_attn { +namespace fused_softmax { +namespace scaled_masked_softmax { + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + torch::Tensor const& mask, + float scale_factor); + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads, + torch::Tensor const& softmax_results, + float scale_factor); + +int get_batch_per_block_cuda( + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads); + +torch::Tensor fwd( + torch::Tensor & input, + torch::Tensor & mask, + float scale_factor) { + TORCH_CHECK(input.dim() == 4, "expected 4D tensor"); + TORCH_CHECK((input.scalar_type() == at::ScalarType::Half) || + (input.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + TORCH_CHECK(mask.dim() == 4, "expected 4D tensor"); + if (!input.is_contiguous()) + input = input.contiguous(); + if (!mask.is_contiguous()) + mask = mask.contiguous(); + + return fwd_cuda(input, mask, scale_factor); +} + +torch::Tensor bwd( + torch::Tensor & output_grads, + torch::Tensor & softmax_results, + float scale_factor) { + + TORCH_CHECK(output_grads.dim() == 4, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 4, "expected 3D tensor"); + + TORCH_CHECK((output_grads.scalar_type() == at::ScalarType::Half) || + (output_grads.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + TORCH_CHECK((softmax_results.scalar_type() == at::ScalarType::Half) || + (softmax_results.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + if (!output_grads.is_contiguous()) + output_grads = output_grads.contiguous(); + if (!softmax_results.is_contiguous()) + softmax_results = softmax_results.contiguous(); + + return bwd_cuda(output_grads, softmax_results, scale_factor); +} + +int get_batch_per_block( + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads) { + return get_batch_per_block_cuda(query_seq_len, key_seq_len, batches, attn_heads); +} + +} // end namespace scaled_masked_softmax +} // end namespace fused_softmax +} // end namespace multihead_attn + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", + &multihead_attn::fused_softmax::scaled_masked_softmax::fwd, + "Self Multihead Attention scaled, time masked softmax -- Forward."); + + m.def("backward", + &multihead_attn::fused_softmax::scaled_masked_softmax::bwd, + "Self Multihead Attention scaled, time masked softmax -- Backward."); + + m.def("get_batch_per_block", + &multihead_attn::fused_softmax::scaled_masked_softmax::get_batch_per_block, + "Return Batch per block size." + ); +} diff --git a/apex/csrc/megatron/scaled_masked_softmax.h b/apex/csrc/megatron/scaled_masked_softmax.h new file mode 100644 index 00000000..d6123cc9 --- /dev/null +++ b/apex/csrc/megatron/scaled_masked_softmax.h @@ -0,0 +1,735 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +template +__device__ __inline__ void copy_vector(Datatype *dst, const Datatype *src); + +template <> +__device__ __inline__ void copy_vector(c10::BFloat16 *dst, const c10::BFloat16 *src) { *dst = *src; } + +template <> +__device__ __inline__ void copy_vector(c10::BFloat16 *dst, const c10::BFloat16 *src) { *((float2*) dst) = *((float2*) src); } + +template <> +__device__ __inline__ void copy_vector(c10::Half *dst, const c10::Half *src) { *dst = *src; } + +template <> +__device__ __inline__ void copy_vector(c10::Half *dst, const c10::Half *src) { *((float2*) dst) = *((float2*) src); } + +template <> +__device__ __inline__ void copy_vector(uint8_t *dst, const uint8_t *src) { *dst = *src; } + +template <> +__device__ __inline__ void copy_vector(uint8_t *dst, const uint8_t *src) {*((half2*) dst) = *((half2*) src); } + +int log2_ceil(int value) { + int log2_value = 0; + while ((1 << log2_value) < value) ++log2_value; + return log2_value; +} + +template +struct Add { + __device__ __forceinline__ T operator()(T a, T b) const { + return a + b; + } +}; + +template +struct Max { + __device__ __forceinline__ T operator()(T a, T b) const { + return a < b ? b : a; + } +}; + +template +__device__ __forceinline__ T WARP_SHFL_XOR_NATIVE(T value, int laneMask, int width = warpSize, unsigned int mask = 0xffffffff) +{ +#if CUDA_VERSION >= 9000 + return __shfl_xor_sync(mask, value, laneMask, width); +#else + return __shfl_xor(value, laneMask, width); +#endif +} + +template class ReduceOp> +__device__ __forceinline__ void warp_reduce(acc_t* sum) { + ReduceOp r; + #pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + acc_t b = WARP_SHFL_XOR_NATIVE(sum[i], offset, WARP_SIZE); + sum[i] = r(sum[i], b); + } + } +} + + +/* + * Extended softmax (from native aten pytorch) with following additional features + * 1) input scaling + */ +template +__global__ void scaled_softmax_warp_forward( + output_t *dst, + const input_t *src, + const acc_t scale, + int micro_batch_size, + int element_count) +{ + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_forward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + constexpr int ELEMENTS_PER_LDG_STG = (WARP_ITERATIONS < 4) ? 1 : 4; + + // blockDim/threadIdx = (WARP_SIZE, WARPS_PER_BLOCK, ) + // gridDim/blockIdx = (seq_len, attn_heads, batches) + long int first_batch = (blockDim.y * (blockIdx.x + gridDim.x * (blockIdx.y + gridDim.y * blockIdx.z))+ threadIdx.y) * WARP_BATCH; + + // micro_batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = micro_batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the batch + int local_idx = threadIdx.x; + + long int thread_offset = first_batch * element_count + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset; + dst += thread_offset; + + // load data from global memory + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + input_t temp_data[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + + if (element_index < batch_element_count) { + int itr_idx = i*element_count+it*WARP_SIZE; + copy_vector(temp_data, src + itr_idx); + + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements[i][it + element] = (acc_t)temp_data[element] * scale; + } + } else { + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements[i][it + element] = -std::numeric_limits::infinity(); + } + } + } + } + + // compute max_value + acc_t max_value[WARP_BATCH]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + #pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + max_value[i] = (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + warp_reduce(max_value); + + acc_t sum[WARP_BATCH] { 0.0f }; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = std::exp((elements[i][it] - max_value[i])); + sum[i] += elements[i][it]; + } + } + warp_reduce(sum); + + // store result + output_t out[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = elements[i][it + element] / sum[i]; + } + copy_vector(dst + i * element_count + it * WARP_SIZE, out); + } else { + break; + } + } + } +} + + +/* + * Extended softmax (from native aten pytorch) with following additional features + * 1) input scaling + * 2) Explicit masking + */ +template +__global__ void scaled_masked_softmax_warp_forward( + output_t *dst, + const input_t *src, + const uint8_t *mask, + const acc_t scale, + int micro_batch_size, + int element_count, + int pad_batches) +{ + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_forward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + constexpr int ELEMENTS_PER_LDG_STG = (WARP_ITERATIONS < 4) ? 1 : 4; + + // blockDim/threadIdx = (WARP_SIZE, WARPS_PER_BLOCK, ) + // gridDim/blockIdx = (seq_len, attn_heads, batches) + long int first_batch = (blockDim.y * (blockIdx.x + gridDim.x * (blockIdx.y + gridDim.y * blockIdx.z))+ threadIdx.y) * WARP_BATCH; + long int pad_first_batch = 0; + if (pad_batches != 1) { // bert style + pad_first_batch = (blockDim.y * (blockIdx.x + gridDim.x * blockIdx.z) + threadIdx.y) * WARP_BATCH; + } else { // gpt2 style + pad_first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + } + + // micro_batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = micro_batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the batch + int local_idx = threadIdx.x; + + long int thread_offset_src_dst = first_batch * element_count + ELEMENTS_PER_LDG_STG * local_idx; + long int thread_offset_mask = pad_first_batch * element_count + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset_src_dst; + dst += thread_offset_src_dst; + mask += thread_offset_mask; + + // load data from global memory + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + input_t temp_data[ELEMENTS_PER_LDG_STG]; + uint8_t temp_mask[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + + if (element_index < batch_element_count) { + int itr_idx = i*element_count+it*WARP_SIZE; + copy_vector(temp_data, src + itr_idx); + copy_vector(temp_mask, mask + itr_idx); + + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + if (temp_mask[element] != 1) { + elements[i][it + element] = (acc_t)temp_data[element] * scale; + } else { + elements[i][it + element] = -10000.0; + } + } + } else { + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements[i][it + element] = -std::numeric_limits::infinity(); + } + } + } + } + + // compute max_value + acc_t max_value[WARP_BATCH]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + #pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + max_value[i] = (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + warp_reduce(max_value); + + // compute scale value to account for full mask + acc_t scale_value[WARP_BATCH]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + scale_value[i] = (max_value[i] == -10000.0) ? 0.0 : 1.0; + } + + acc_t sum[WARP_BATCH] { 0.0f }; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + elements[i][it] = std::exp((elements[i][it] - max_value[i])); + sum[i] += elements[i][it]; + } + } + warp_reduce(sum); + + // store result + output_t out[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = elements[i][it + element] * scale_value[i]/ sum[i]; + } + copy_vector(dst + i * element_count + it * WARP_SIZE, out); + } else { + break; + } + } + } +} + +template +__global__ void scaled_masked_softmax_warp_backward( + output_t *gradInput, + input_t *grad, + const input_t *output, + acc_t scale, + int micro_batch_size, + int element_count) +{ + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_backward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + constexpr int ELEMENTS_PER_LDG_STG = (WARP_ITERATIONS < 4) ? 1 : 4; + + // blockDim/threadIdx = (WARP_SIZE, WARPS_PER_BLOCK, ) + // gridDim/blockIdx = (seq_len, attn_heads, batches) + long int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; + + // micro_batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = micro_batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the batch + int local_idx = threadIdx.x; + + // the first element to process by the current thread + long int thread_offset = first_batch * element_count + ELEMENTS_PER_LDG_STG * local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + + // load data from global memory + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS] { 0.0f }; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS] { 0.0f }; + input_t temp_grad[ELEMENTS_PER_LDG_STG]; + input_t temp_output[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : element_count; + + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + copy_vector(temp_grad, grad + i * element_count + it * WARP_SIZE); + copy_vector(temp_output, output + i * element_count + it * WARP_SIZE); + + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + output_reg[i][it + element] = (acc_t)temp_output[element]; + } + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + grad_reg[i][it + element] = (acc_t)temp_grad[element] * output_reg[i][it + element]; + } + } + } + } + + acc_t sum[WARP_BATCH]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] = grad_reg[i][0]; + #pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + sum[i] += grad_reg[i][it]; + } + } + warp_reduce(sum); + + // store result + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + output_t out[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = (output_t)(scale * (grad_reg[i][it + element] - output_reg[i][it + element] * sum[i])); + } + copy_vector(gradInput + i * element_count + it * WARP_SIZE, out); + } + } + } +} +} // end of anonymous namespace + +int get_batch_per_block(int query_seq_len, int key_seq_len, int batches, int attn_heads){ + int log2_elements = log2_ceil(key_seq_len); + const int next_power_of_two = 1 << log2_elements; + + int warp_size = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + constexpr int threads_per_block = 128; + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + + return batches_per_block; +} + +template +void dispatch_scaled_softmax_forward( + output_t *dst, + const input_t *src, + const input_t scale, + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads) +{ + TORCH_INTERNAL_ASSERT(key_seq_len >= 0 && key_seq_len <= 16384 ); + if (key_seq_len == 0) { + return; + } else { + int log2_elements = log2_ceil(key_seq_len); + const int next_power_of_two = 1 << log2_elements; + int batch_count = batches * attn_heads * query_seq_len; + + // This value must match the WARP_SIZE constexpr value computed inside softmax_warp_forward. + int warp_size = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_forward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + TORCH_INTERNAL_ASSERT(query_seq_len%batches_per_block == 0); + dim3 blocks(query_seq_len/batches_per_block, attn_heads, batches); + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 1: // 2 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 2: // 4 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 3: // 8 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 4: // 16 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 5: // 32 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 6: // 64 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 7: // 128 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 8: // 256 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 9: // 512 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 10: // 1024 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 11: // 2048 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 12: // 4096 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 13: // 8192 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + case 14: // 16384 + scaled_softmax_warp_forward + <<>>(dst, src, scale, batch_count, key_seq_len); + break; + default: + break; + } + } +} + +template +void dispatch_scaled_masked_softmax_forward( + output_t *dst, + const input_t *src, + const uint8_t *mask, + const input_t scale, + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads, + int pad_batches) +{ + TORCH_INTERNAL_ASSERT(key_seq_len >= 0 && key_seq_len <= 4096 ); + if (key_seq_len == 0) { + return; + } else { + int log2_elements = log2_ceil(key_seq_len); + const int next_power_of_two = 1 << log2_elements; + int batch_count = batches * attn_heads * query_seq_len; + + // This value must match the WARP_SIZE constexpr value computed inside softmax_warp_forward. + int warp_size = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_forward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + TORCH_INTERNAL_ASSERT(query_seq_len%batches_per_block == 0); + dim3 blocks(query_seq_len/batches_per_block, attn_heads, batches); + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 1: // 2 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 2: // 4 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 3: // 8 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 4: // 16 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 5: // 32 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 6: // 64 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 7: // 128 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 8: // 256 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 9: // 512 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 10: // 1024 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 11: // 2048 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + case 12: // 4096 + scaled_masked_softmax_warp_forward + <<>>(dst, src, mask, scale, batch_count, key_seq_len, pad_batches); + break; + default: + break; + } + } +} + +template +void dispatch_scaled_masked_softmax_backward( + output_t *grad_input, + input_t *grad, + const input_t *output, + const acc_t scale, + int query_seq_len, + int key_seq_len, + int batches, + int attn_heads) +{ + TORCH_INTERNAL_ASSERT( key_seq_len >= 0 && key_seq_len <= 4096 ); + if (key_seq_len == 0) { + return; + } else { + int log2_elements = log2_ceil(key_seq_len); + const int next_power_of_two = 1 << log2_elements; + int batch_count = batches * attn_heads * query_seq_len; + + // This value must match the WARP_SIZE constexpr value computed inside softmax_warp_backward. + int warp_size = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_backward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + int blocks = batch_count/batches_per_block; + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 1: // 2 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 2: // 4 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 3: // 8 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 4: // 16 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 5: // 32 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 6: // 64 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 7: // 128 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 8: // 256 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 9: // 512 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 10: // 1024 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 11: // 2048 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + case 12: // 4096 + scaled_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, key_seq_len); + break; + + default: + break; + } + } +} diff --git a/apex/csrc/megatron/scaled_masked_softmax_cuda.cu b/apex/csrc/megatron/scaled_masked_softmax_cuda.cu new file mode 100644 index 00000000..3a83b1a4 --- /dev/null +++ b/apex/csrc/megatron/scaled_masked_softmax_cuda.cu @@ -0,0 +1,121 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "scaled_masked_softmax.h" +#include "type_shim.h" + +namespace multihead_attn { +namespace fused_softmax { +namespace scaled_masked_softmax { + +int get_batch_per_block_cuda(int query_seq_len, int key_seq_len, int batches, int attn_heads){ + return get_batch_per_block(query_seq_len, key_seq_len, batches, attn_heads); +} + + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + torch::Tensor const& mask, + float scale_factor) +{ + // input is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len] + const int batches = input.size(0); + const int pad_batches = mask.size(0); + const int attn_heads = input.size(1); + const int query_seq_len = input.size(2); + const int key_seq_len = input.size(3); + TORCH_INTERNAL_ASSERT(key_seq_len <= 16384); + TORCH_INTERNAL_ASSERT(query_seq_len > 1); + TORCH_INTERNAL_ASSERT(pad_batches == 1 || pad_batches == batches); + TORCH_INTERNAL_ASSERT(mask.size(1) == 1); + TORCH_INTERNAL_ASSERT(mask.size(2) == query_seq_len); + TORCH_INTERNAL_ASSERT(mask.size(3) == key_seq_len); + + // Output + auto act_options = input.options().requires_grad(false); + torch::Tensor softmax_results = + torch::empty({batches, attn_heads, query_seq_len, key_seq_len}, act_options); + + // Softmax Intermediate Result Ptr + void* input_ptr = static_cast(input.data_ptr()); + void* mask_ptr = static_cast(mask.data_ptr()); + void* softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + DISPATCH_HALF_AND_BFLOAT( + input.scalar_type(), + "dispatch_scaled_masked_softmax_forward", + dispatch_scaled_masked_softmax_forward( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), + reinterpret_cast(mask_ptr), + scale_factor, + query_seq_len, + key_seq_len, + batches, + attn_heads, + pad_batches + ); + ); + return softmax_results; +} + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads_, + torch::Tensor const& softmax_results_, + float scale_factor) { + + auto output_grads = output_grads_.contiguous(); + auto softmax_results = softmax_results_.contiguous(); + + //output grads is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len] + const int batches = output_grads.size(0); + const int attn_heads = output_grads.size(1); + const int query_seq_len = output_grads.size(2); + const int key_seq_len = output_grads.size(3); + + auto act_options = output_grads.options().requires_grad(false); + torch::Tensor input_grads = + torch::empty({batches, attn_heads, query_seq_len, key_seq_len}, act_options); + void* input_grads_ptr = static_cast(input_grads.data_ptr()); + void* output_grads_ptr = static_cast(output_grads.data_ptr()); + + //Softmax Grad + DISPATCH_HALF_AND_BFLOAT( + output_grads_.scalar_type(), + "dispatch_scaled_masked_softmax_backward", + dispatch_scaled_masked_softmax_backward( + reinterpret_cast(input_grads_ptr), + reinterpret_cast(output_grads_ptr), + reinterpret_cast(softmax_results.data_ptr()), + scale_factor, + query_seq_len, + key_seq_len, + batches, + attn_heads + ); + ); + return input_grads; +} +} +} +} diff --git a/apex/csrc/megatron/scaled_softmax.cpp b/apex/csrc/megatron/scaled_softmax.cpp new file mode 100644 index 00000000..5d967857 --- /dev/null +++ b/apex/csrc/megatron/scaled_softmax.cpp @@ -0,0 +1,75 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +namespace multihead_attn { +namespace fused_softmax { +namespace scaled_softmax { + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + float scale_factor); + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads, + torch::Tensor const& softmax_results, + float scale_factor); + +torch::Tensor fwd( + torch::Tensor const& input, + float scale_factor) { + TORCH_CHECK(input.dim() == 4, "expected 4D tensor"); + TORCH_CHECK((input.scalar_type() == at::ScalarType::Half) || + (input.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + + return fwd_cuda(input, scale_factor); +} + +torch::Tensor bwd( + torch::Tensor const& output_grads, + torch::Tensor const& softmax_results, + float scale_factor) { + + TORCH_CHECK(output_grads.dim() == 4, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 4, "expected 3D tensor"); + + TORCH_CHECK((output_grads.scalar_type() == at::ScalarType::Half) || + (output_grads.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + TORCH_CHECK((softmax_results.scalar_type() == at::ScalarType::Half) || + (softmax_results.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + + return bwd_cuda(output_grads, softmax_results, scale_factor); +} + +} // end namespace scaled_softmax +} // end namespace fused_softmax +} // end namespace multihead_attn + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", + &multihead_attn::fused_softmax::scaled_softmax::fwd, + "Self Multihead Attention scaled, softmax -- Forward."); + m.def("backward", + &multihead_attn::fused_softmax::scaled_softmax::bwd, + "Self Multihead Attention scaled, softmax -- Backward."); +} + diff --git a/apex/csrc/megatron/scaled_softmax_cuda.cu b/apex/csrc/megatron/scaled_softmax_cuda.cu new file mode 100644 index 00000000..1bcaff36 --- /dev/null +++ b/apex/csrc/megatron/scaled_softmax_cuda.cu @@ -0,0 +1,104 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "scaled_masked_softmax.h" +#include "type_shim.h" + +namespace multihead_attn { +namespace fused_softmax { +namespace scaled_softmax { + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + float scale_factor) +{ + // input is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len] + const int batches = input.size(0); + const int attn_heads = input.size(1); + const int query_seq_len = input.size(2); + const int key_seq_len = input.size(3); + TORCH_INTERNAL_ASSERT(key_seq_len <= 16384); + TORCH_INTERNAL_ASSERT(query_seq_len > 1); + + // Output + auto act_options = input.options().requires_grad(false); + torch::Tensor softmax_results = + torch::empty({batches, attn_heads, query_seq_len, key_seq_len}, act_options); + + // Softmax Intermediate Result Ptr + void* input_ptr = static_cast(input.data_ptr()); + void* softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + DISPATCH_HALF_AND_BFLOAT( + input.scalar_type(), + "dispatch_scaled_softmax_forward", + dispatch_scaled_softmax_forward( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), + scale_factor, + query_seq_len, + key_seq_len, + batches, + attn_heads); + ); + return softmax_results; +} + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads_, + torch::Tensor const& softmax_results_, + float scale_factor) { + + auto output_grads = output_grads_.contiguous(); + auto softmax_results = softmax_results_.contiguous(); + + //output grads is a 4d tensor with dimensions [batches, attn_heads, seq_len, seq_len] + const int batches = output_grads.size(0); + const int attn_heads = output_grads.size(1); + const int query_seq_len = output_grads.size(2); + const int key_seq_len = output_grads.size(3); + + void* output_grads_ptr = static_cast(output_grads.data_ptr()); + + //Softmax Grad + DISPATCH_HALF_AND_BFLOAT( + output_grads_.scalar_type(), + "dispatch_scaled_masked_softmax_backward", + dispatch_scaled_masked_softmax_backward( + reinterpret_cast(output_grads_ptr), + reinterpret_cast(output_grads_ptr), + reinterpret_cast(softmax_results.data_ptr()), + scale_factor, + query_seq_len, + key_seq_len, + batches, + attn_heads); + ); + + //backward pass is completely in-place + return output_grads; +} +} +} +} + diff --git a/apex/csrc/megatron/scaled_upper_triang_masked_softmax.cpp b/apex/csrc/megatron/scaled_upper_triang_masked_softmax.cpp new file mode 100644 index 00000000..3d10dae3 --- /dev/null +++ b/apex/csrc/megatron/scaled_upper_triang_masked_softmax.cpp @@ -0,0 +1,72 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +namespace multihead_attn { +namespace fused_softmax { +namespace scaled_upper_triang_masked_softmax { + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + float scale_factor); + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads, + torch::Tensor const& softmax_results, + float scale_factor); + +torch::Tensor fwd(torch::Tensor const& input, float scale_factor) { + TORCH_CHECK(input.dim() == 3, "expected 3D tensor"); + TORCH_CHECK((input.scalar_type() == at::ScalarType::Half) || + (input.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + + return fwd_cuda(input, scale_factor); +} + +torch::Tensor bwd( + torch::Tensor const& output_grads, + torch::Tensor const& softmax_results, + float scale_factor) { + + TORCH_CHECK(output_grads.dim() == 3, "expected 3D tensor"); + TORCH_CHECK(softmax_results.dim() == 3, "expected 3D tensor"); + + TORCH_CHECK((output_grads.scalar_type() == at::ScalarType::Half) || + (output_grads.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + TORCH_CHECK((softmax_results.scalar_type() == at::ScalarType::Half) || + (softmax_results.scalar_type() == at::ScalarType::BFloat16), + "Only fp16 and bf16 are supported"); + + return bwd_cuda(output_grads, softmax_results, scale_factor); +} + +} // end namespace scaled_upper_triang_masked_softmax +} // end namespace fused_softmax +} // end namespace multihead_attn + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", + &multihead_attn::fused_softmax::scaled_upper_triang_masked_softmax::fwd, + "Self Multihead Attention scaled, time masked softmax -- Forward."); + m.def("backward", + &multihead_attn::fused_softmax::scaled_upper_triang_masked_softmax::bwd, + "Self Multihead Attention scaled, time masked softmax -- Backward."); +} diff --git a/apex/csrc/megatron/scaled_upper_triang_masked_softmax.h b/apex/csrc/megatron/scaled_upper_triang_masked_softmax.h new file mode 100644 index 00000000..2c495863 --- /dev/null +++ b/apex/csrc/megatron/scaled_upper_triang_masked_softmax.h @@ -0,0 +1,538 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace { + +template +__device__ __inline__ void copy_vector(Datatype *dst, const Datatype *src); + +template <> +__device__ __inline__ void copy_vector(c10::BFloat16 *dst, const c10::BFloat16 *src) { *dst = *src; } + +template <> +__device__ __inline__ void copy_vector(c10::BFloat16 *dst, const c10::BFloat16 *src) { *((float2*) dst) = *((float2*) src); } + +template <> +__device__ __inline__ void copy_vector(c10::Half *dst, const c10::Half *src) { *dst = *src; } + +template <> +__device__ __inline__ void copy_vector(c10::Half *dst, const c10::Half *src) { *((float2*) dst) = *((float2*) src); } + +template <> +__device__ __inline__ void copy_vector(uint8_t *dst, const uint8_t *src) { *dst = *src; } + +template <> +__device__ __inline__ void copy_vector(uint8_t *dst, const uint8_t *src) {*((half2*) dst) = *((half2*) src); } + +template +__device__ __inline__ void copy_zero_vector(Datatype *dst); + +template <> +__device__ __inline__ void copy_zero_vector(c10::BFloat16 *dst) { *dst = 0.0; } + +template <> +__device__ __inline__ void copy_zero_vector(c10::BFloat16 *dst) { *((float2*) dst) = make_float2(0.0f, 0.0f); } + +template <> +__device__ __inline__ void copy_zero_vector(c10::Half *dst) { *dst = 0.0; } + +template <> +__device__ __inline__ void copy_zero_vector(c10::Half *dst) { *((float2*) dst) = make_float2(0.0f, 0.0f); } + + +int log2_ceil(int value) { + int log2_value = 0; + while ((1 << log2_value) < value) ++log2_value; + return log2_value; +} + +template +struct Add { + __device__ __forceinline__ T operator()(T a, T b) const { + return a + b; + } +}; + +template +struct Max { + __device__ __forceinline__ T operator()(T a, T b) const { + return a < b ? b : a; + } +}; + +template +__device__ __forceinline__ T WARP_SHFL_XOR_NATIVE(T value, int laneMask, int width = warpSize, unsigned int mask = 0xffffffff) +{ +#if CUDA_VERSION >= 9000 + return __shfl_xor_sync(mask, value, laneMask, width); +#else + return __shfl_xor(value, laneMask, width); +#endif +} + +template class ReduceOp> +__device__ __forceinline__ void warp_reduce(acc_t* sum) { + ReduceOp r; + #pragma unroll + for (int offset = WARP_SIZE / 2; offset > 0; offset /= 2) { + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + acc_t b = WARP_SHFL_XOR_NATIVE(sum[i], offset, WARP_SIZE); + sum[i] = r(sum[i], b); + } + } +} + +/* + * Extended softmax (from native aten pytorch) with following additional features + * 1) input scaling + * 2) Implicit time (diagonal masking) + */ +template +__global__ void scaled_upper_triang_masked_softmax_warp_forward( + output_t *dst, + const input_t *src, + const acc_t scale, + int micro_batch_size, + int stride, + int element_count) +{ + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_forward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + constexpr int ELEMENTS_PER_LDG_STG = (WARP_ITERATIONS < 4) ? 1 : 4; + + long int first_batch = (blockDim.y * blockIdx.y + threadIdx.y) * gridDim.x * WARP_BATCH + blockIdx.x; + int local_seq = blockIdx.x + 1; + int warp_iteration_limit = (local_seq + ELEMENTS_PER_LDG_STG * WARP_SIZE - 1)/ WARP_SIZE; + + // micro_batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = micro_batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the batch + int local_idx = threadIdx.x; + + long int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + src += thread_offset; + dst += thread_offset; + + // load data from global memory + acc_t elements[WARP_BATCH][WARP_ITERATIONS]; + input_t temp_data[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : local_seq; + + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + + if (element_index < batch_element_count) { + copy_vector(temp_data, src + i*element_count*stride + it*WARP_SIZE); + + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + if ((element_index + element) < batch_element_count) { + elements[i][it+element] = (acc_t)temp_data[element] * scale; + } else { + elements[i][it + element] = -std::numeric_limits::infinity(); + } + } + } else { + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + elements[i][it + element] = -std::numeric_limits::infinity(); + } + } + } + } + + // compute max_value + acc_t max_value[WARP_BATCH]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + max_value[i] = elements[i][0]; + #pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + max_value[i] = (max_value[i] > elements[i][it]) ? max_value[i] : elements[i][it]; + } + } + warp_reduce(max_value); + + acc_t sum[WARP_BATCH] { 0.0f }; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; ++it) { + if (it < warp_iteration_limit) { + elements[i][it] = std::exp((elements[i][it] - max_value[i])); + sum[i] += elements[i][it]; + } + } + } + warp_reduce(sum); + + // store result + output_t out[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + + if (element_index < local_seq) { + + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + if (element_index + element < local_seq) { + out[element] = elements[i][it + element] / sum[i]; + } else { + out[element] = 0; + } + } + copy_vector(dst + i * element_count * stride + it * WARP_SIZE, out); + } else if (element_index < element_count) { + copy_zero_vector(dst + i * element_count * stride + it * WARP_SIZE); + } else { + break; + } + } + } +} + +template +__global__ void scaled_upper_triang_masked_softmax_warp_backward( + output_t *gradInput, + input_t *grad, + const input_t *output, + acc_t scale, + int micro_batch_size, + int stride, + int element_count) +{ + // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and + // warp_size of method warp_softmax_backward_kernel. + constexpr int next_power_of_two = 1 << log2_elements; + constexpr int WARP_SIZE = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; + constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; + constexpr int ELEMENTS_PER_LDG_STG = (WARP_ITERATIONS < 4) ? 1 : 4; + + long int first_batch = (blockDim.y * blockIdx.y + threadIdx.y) * gridDim.x * WARP_BATCH + blockIdx.x; + int local_seq = blockIdx.x + 1; + + // micro_batch_size might not be a multiple of WARP_BATCH. Check how + // many batches have to computed within this WARP. + int local_batches = micro_batch_size - first_batch; + if (local_batches > WARP_BATCH) + local_batches = WARP_BATCH; + + // there might be multiple batches per warp. compute the index within the batch + int local_idx = threadIdx.x; + + // the first element to process by the current thread + long int thread_offset = first_batch * stride + ELEMENTS_PER_LDG_STG * local_idx; + grad += thread_offset; + output += thread_offset; + gradInput += thread_offset; + + // load data from global memory + acc_t grad_reg[WARP_BATCH][WARP_ITERATIONS] { 0.0f }; + acc_t output_reg[WARP_BATCH][WARP_ITERATIONS] { 0.0f }; + input_t temp_grad[ELEMENTS_PER_LDG_STG]; + input_t temp_output[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + int batch_element_count = (i >= local_batches) ? 0 : local_seq; + + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < batch_element_count) { + copy_vector(temp_grad, grad + i * element_count * stride + it * WARP_SIZE); + copy_vector(temp_output, output + i * element_count * stride + it * WARP_SIZE); + + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + if (element_index + element < batch_element_count) { + output_reg[i][it + element] = (acc_t)temp_output[element]; + } + } + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + if (element_index + element < batch_element_count) { + grad_reg[i][it + element] = (acc_t)temp_grad[element] * output_reg[i][it + element]; + } + } + } + } + } + + acc_t sum[WARP_BATCH]; + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + sum[i] = grad_reg[i][0]; + #pragma unroll + for (int it = 1; it < WARP_ITERATIONS; ++it) { + sum[i] += grad_reg[i][it]; + } + } + warp_reduce(sum); + + // store result + #pragma unroll + for (int i = 0; i < WARP_BATCH; ++i) { + if (i >= local_batches) + break; + #pragma unroll + for (int it = 0; it < WARP_ITERATIONS; it+=ELEMENTS_PER_LDG_STG) { + int element_index = ELEMENTS_PER_LDG_STG * local_idx + it * WARP_SIZE; + if (element_index < element_count) { + // compute gradients + output_t out[ELEMENTS_PER_LDG_STG]; + #pragma unroll + for (int element = 0; element < ELEMENTS_PER_LDG_STG; ++element) { + out[element] = (output_t)(scale * (grad_reg[i][it + element] - output_reg[i][it + element] * sum[i])); + } + copy_vector(gradInput + i * element_count * stride + it * WARP_SIZE, out); + } + } + } +} + +} // end of anonymous namespace + +template +void dispatch_scaled_upper_triang_masked_softmax_forward( + output_t *dst, + const input_t *src, + const input_t scale, + int softmax_elements, + int softmax_elements_stride, + int attn_batches) +{ + TORCH_INTERNAL_ASSERT(softmax_elements >= 0 && softmax_elements <= 16384 ); + if (softmax_elements == 0) { + return; + } else { + int log2_elements = log2_ceil(softmax_elements); + const int next_power_of_two = 1 << log2_elements; + int seq_len = softmax_elements; + int batch_count = attn_batches * seq_len; + + // This value must match the WARP_SIZE constexpr value computed inside softmax_warp_forward. + int warp_size = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_forward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + TORCH_INTERNAL_ASSERT(attn_batches % batches_per_block == 0); + + int blocks_per_seq = attn_batches / batches_per_block; + dim3 blocks(seq_len, blocks_per_seq, 1); + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 1: // 2 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 2: // 4 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 3: // 8 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 4: // 16 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 5: // 32 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 6: // 64 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 7: // 128 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 8: // 256 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 9: // 512 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 10: // 1024 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 11: // 2048 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 12: // 4096 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 13: // 8192 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 14: // 16384 + scaled_upper_triang_masked_softmax_warp_forward + <<>>(dst, src, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + default: + break; + } + } +} + +template +void dispatch_scaled_upper_triang_masked_softmax_backward( + output_t *grad_input, + input_t *grad, + const input_t *output, + const acc_t scale, + int softmax_elements, + int softmax_elements_stride, + int attn_batches) +{ + TORCH_INTERNAL_ASSERT( softmax_elements >= 0 && softmax_elements <= 16384 ); + if (softmax_elements == 0) { + return; + } else { + int log2_elements = log2_ceil(softmax_elements); + const int next_power_of_two = 1 << log2_elements; + int seq_len = softmax_elements; + int batch_count = attn_batches * seq_len; + + // This value must match the WARP_SIZE constexpr value computed inside softmax_warp_backward. + int warp_size = (next_power_of_two < C10_WARP_SIZE) ? next_power_of_two : C10_WARP_SIZE; + + // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_backward. + int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; + + // use 128 threads per block to maximize gpu utilization + constexpr int threads_per_block = 128; + + int warps_per_block = (threads_per_block / warp_size); + int batches_per_block = warps_per_block * batches_per_warp; + TORCH_INTERNAL_ASSERT(attn_batches % batches_per_block == 0); + + int blocks_per_seq = attn_batches / batches_per_block; + dim3 blocks(seq_len, blocks_per_seq, 1); + dim3 threads(warp_size, warps_per_block, 1); + // Launch code would be more elegant if C++ supported FOR CONSTEXPR + switch (log2_elements) { + case 0: // 1 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 1: // 2 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 2: // 4 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 3: // 8 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 4: // 16 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 5: // 32 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 6: // 64 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 7: // 128 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 8: // 256 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 9: // 512 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 10: // 1024 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 11: // 2048 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 12: // 4096 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 13: // 8192 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + case 14: // 16384 + scaled_upper_triang_masked_softmax_warp_backward + <<>>(grad_input, grad, output, scale, batch_count, softmax_elements_stride, softmax_elements); + break; + default: + break; + } + } +} diff --git a/apex/csrc/megatron/scaled_upper_triang_masked_softmax_cuda.cu b/apex/csrc/megatron/scaled_upper_triang_masked_softmax_cuda.cu new file mode 100644 index 00000000..b92fbec9 --- /dev/null +++ b/apex/csrc/megatron/scaled_upper_triang_masked_softmax_cuda.cu @@ -0,0 +1,98 @@ +/* coding=utf-8 + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "scaled_upper_triang_masked_softmax.h" +#include "type_shim.h" + +namespace multihead_attn { +namespace fused_softmax { +namespace scaled_upper_triang_masked_softmax { + +torch::Tensor fwd_cuda( + torch::Tensor const& input, + float scale_factor) +{ + // input is a 3d tensor with dimensions [attn_batches, seq_len, seq_len] + const int attn_batches = input.size(0); + const int seq_len = input.size(1); + TORCH_INTERNAL_ASSERT(seq_len <= 16384); + + // Output + auto act_options = input.options().requires_grad(false); + torch::Tensor softmax_results = + torch::empty({attn_batches, seq_len, seq_len}, act_options); + + // Softmax Intermediate Result Ptr + void* input_ptr = static_cast(input.data_ptr()); + void* softmax_results_ptr = static_cast(softmax_results.data_ptr()); + + DISPATCH_HALF_AND_BFLOAT( + input.scalar_type(), + "dispatch_scaled_upper_triang_masked_softmax_forward", + dispatch_scaled_upper_triang_masked_softmax_forward( + reinterpret_cast(softmax_results_ptr), + reinterpret_cast(input_ptr), + scale_factor, + seq_len, + seq_len, + attn_batches); + ); + return softmax_results; +} + + +torch::Tensor bwd_cuda( + torch::Tensor const& output_grads_, + torch::Tensor const& softmax_results_, + float scale_factor) { + + auto output_grads = output_grads_.contiguous(); + auto softmax_results = softmax_results_.contiguous(); + + //output grads is a 3d tensor with dimensions [attn_batches, seq_len, seq_len] + const int attn_batches = output_grads.size(0); + const int seq_len = output_grads.size(1); + TORCH_INTERNAL_ASSERT(output_grads.size(1) == output_grads.size(2)); + + void* output_grads_ptr = static_cast(output_grads.data_ptr()); + + //Softmax Grad + DISPATCH_HALF_AND_BFLOAT( + output_grads_.scalar_type(), + "dispatch_scaled_upper_triang_masked_softmax_backward", + dispatch_scaled_upper_triang_masked_softmax_backward( + reinterpret_cast(output_grads_ptr), + reinterpret_cast(output_grads_ptr), + reinterpret_cast(softmax_results.data_ptr()), + scale_factor, + seq_len, + seq_len, + attn_batches); + ); + + //backward pass is completely in-place + return output_grads; +} +} +} +} diff --git a/apex/csrc/mlp.cpp b/apex/csrc/mlp.cpp new file mode 100644 index 00000000..830d6062 --- /dev/null +++ b/apex/csrc/mlp.cpp @@ -0,0 +1,166 @@ +#include +#include +#include + +#include + +size_t get_mlp_reserved_space(int64_t batch_size, int num_layers, const int* output_features); + +template +size_t get_mlp_bp_workspace_in_bytes(int batch_size, int num_layers, const int* output_features); + +template +int mlp_fp( + T* X, + int input_features, + int batch_size, + T** WPtr, + int num_layers, + int* output_features, + T** BPtr, + T* Y, + T* reserved_space, + int use_bias, + int activation, + void* lt_workspace); + +template +int mlp_bp( + T* X, + T* Y, + int input_features, + int batch_size, + T** WPtr, + int num_layers, + int* output_features, + T* dY, + T* reserved_space, + T* work_space, + T* dX, + T** dwPtr, + T** dbPtr, + bool requires_grad, + int use_bias, + int activation); + +std::vector mlp_forward(int use_bias, int activation, std::vector inputs) { + + auto num_layers = inputs.size() - 1; + if (use_bias) { + // inputs contains (input, weights, biases) + num_layers /= 2; + } + auto batch_size = inputs[0].size(0); + auto input_features = inputs[0].size(1); + + std::vector output_features; + for (int i = 0; i < num_layers; i++) { + output_features.push_back(inputs[i + 1].size(0)); + } + + auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); + + // create output/workspace tensor + auto out = at::empty({batch_size, output_features.back()}, inputs[0].type()); + auto reserved_space = at::empty({static_cast(reserved_size)}, inputs[0].type()); + // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB + auto lt_workspace = at::empty({1 << 22}, inputs[0].type()); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(inputs[0].type(), "mlp_forward", [&] { + std::vector w_ptr; + std::vector b_ptr; + for (int i = 0; i < num_layers; i++) { + w_ptr.push_back(inputs[i + 1].data_ptr()); + if (use_bias) { + b_ptr.push_back(inputs[i + 1 + num_layers].data_ptr()); + } + } + auto result = mlp_fp( + inputs[0].data_ptr(), + input_features, + batch_size, + w_ptr.data(), + num_layers, + output_features.data(), + b_ptr.data(), + out.data_ptr(), + reserved_space.data_ptr(), + use_bias, + activation, + (void*) (lt_workspace.data_ptr())); + }); + + return {out, reserved_space}; +} + +std::vector mlp_backward( + int use_bias, + int activation, + at::Tensor grad_o, + std::vector fprop_outputs, + std::vector inputs) { + + auto num_layers = inputs.size() - 1; + if (use_bias) { + // inputs contains (input, weights, biases) + num_layers /= 2; + } + + auto batch_size = inputs[0].size(0); + auto input_features = inputs[0].size(1); + + bool requires_grad = inputs[0].requires_grad(); + + std::vector output_features; + for (int i = 0; i < num_layers; i++) { + output_features.push_back(inputs[i + 1].size(0)); + } + // create outputs, length of inputs + std::vector outputs; + for (int i = 0; i < inputs.size(); i++) { + outputs.push_back(at::empty(inputs[i].sizes(), inputs[i].type())); // clone for testing now + } + + AT_DISPATCH_FLOATING_TYPES_AND_HALF(inputs[0].type(), "mlp_backward", [&] { + std::vector w_ptr; + for (int i = 0; i < num_layers; i++) { + w_ptr.push_back(inputs[i + 1].data_ptr()); + } + std::vector outputs_ptr; + for (int i = 0; i < inputs.size(); i++) { + outputs_ptr.push_back(outputs[i].data_ptr()); + } + + auto work_size = + get_mlp_bp_workspace_in_bytes(batch_size, num_layers, output_features.data()); + + // auto work_space = at::empty({work_size*4}, at::kByte); + auto work_space = at::empty({static_cast(work_size / sizeof(scalar_t))}, inputs[0].type()); + + auto result = mlp_bp( + inputs[0].data_ptr(), + fprop_outputs[0].data_ptr(), + input_features, + batch_size, + w_ptr.data(), + num_layers, + output_features.data(), + grad_o.contiguous().data_ptr(), + fprop_outputs[1].data_ptr(), + work_space.data_ptr(), + outputs_ptr[0], + outputs_ptr.data() + 1, + outputs_ptr.data() + 1 + num_layers, + requires_grad, + use_bias, + activation); + }); + + return outputs; +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("forward", &mlp_forward, "MLP forward"); + m.def("backward", &mlp_backward, "MLP backward"); +} + diff --git a/apex/csrc/mlp_cuda.cu b/apex/csrc/mlp_cuda.cu new file mode 100644 index 00000000..f93f1df1 --- /dev/null +++ b/apex/csrc/mlp_cuda.cu @@ -0,0 +1,1678 @@ +#include +#include +#include +#include +#include +#include +#include + +/* Includes, cuda */ +#include +#include + +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11000 +// includes cublaslt +#include +#endif +// constants for fused bias+relu kernel +#define BIAS_RELU_FW_NTHREADS 128 // forward number of thread per block +#define BIAS_RELU_BW_NTHREADS_X 32 // backward number of thread in feature dim +#define BIAS_RELU_BW_NTHREADS_Y 16 // backward number of thread in batch dim +#define BIAS_RELU_RED_PER_THREAD 16 // backward minimal reduction length per thread + +// move to a header later on +#define ILP 4 +template +__host__ __device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} +template +__device__ __forceinline__ void load_store(T* dst, volatile T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} +template +__device__ __forceinline__ void load_store(volatile T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +// Keep ReLU in float only. When using half, cast to float before calling. +__device__ __inline__ float relu(float a) { + float retf = max(a, 0.f); + return (retf); +} + +// Keep Sigmoid in float only. When using half, cast to float before calling. +__device__ __inline__ float sigmoid(float a) { + float retf = 1.f / (1.f + expf(-a)); + return (retf); +} + +// FP64 Wrapper around cublas GEMMEx +cublasStatus_t mlp_gemm( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + float* alpha, + const double* A, + int lda, + const double* B, + int ldb, + const float* beta, + double* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_64F, + lda, + B, + CUDA_R_64F, + ldb, + beta, + C, + CUDA_R_64F, + ldc, + CUDA_R_64F, + CUBLAS_GEMM_DEFAULT); +} + +// FP32 Wrapper around cublas GEMMEx +cublasStatus_t mlp_gemm( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + float* alpha, + const float* A, + int lda, + const float* B, + int ldb, + const float* beta, + float* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_32F, + lda, + B, + CUDA_R_32F, + ldb, + beta, + C, + CUDA_R_32F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT); +} + +// FP16 Tensor core wrapper around cublas GEMMEx +cublasStatus_t mlp_gemm( + cublasHandle_t handle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + float* alpha, + const at::Half* A, + int lda, + const at::Half* B, + int ldb, + float* beta, + at::Half* C, + int ldc) { + return cublasGemmEx( + handle, + transa, + transb, + m, + n, + k, + alpha, + A, + CUDA_R_16F, + lda, + B, + CUDA_R_16F, + ldb, + beta, + C, + CUDA_R_16F, + ldc, + CUDA_R_32F, + CUBLAS_GEMM_DEFAULT_TENSOR_OP); +} +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11000 +int mlp_gemm_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + float *alpha, /* host pointer */ + const at::Half* A, + int lda, + const at::Half* B, + int ldb, + float *beta, /* host pointer */ + at::Half* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + bool use_relu, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + if (use_relu) { + epilogue = CUBLASLT_EPILOGUE_RELU_BIAS; + } else { + epilogue = CUBLASLT_EPILOGUE_BIAS; + } + } else { + if (use_relu) { + epilogue = CUBLASLT_EPILOGUE_RELU; + } + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_16F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_16F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_16F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + &heuristicResult.algo, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} + +int mlp_gemm_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + float *alpha, /* host pointer */ + const double* A, + int lda, + const double* B, + int ldb, + float *beta, /* host pointer */ + double* C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + bool use_relu, + const void* bias) { + return 1; +} + +int mlp_gemm_lt( + cublasLtHandle_t ltHandle, + cublasOperation_t transa, + cublasOperation_t transb, + int m, + int n, + int k, + float *alpha, /* host pointer */ + const float *A, + int lda, + const float *B, + int ldb, + float *beta, /* host pointer */ + float *C, + int ldc, + void *workspace, + size_t workspaceSize, + cudaStream_t stream, + bool use_bias, + bool use_relu, + const void* bias) { + cublasStatus_t status = CUBLAS_STATUS_SUCCESS; + + cublasLtMatmulDescOpaque_t operationDesc = {}; + cublasLtMatrixLayoutOpaque_t Adesc = {}, Bdesc = {}, Cdesc = {}; + cublasLtMatmulPreferenceOpaque_t preference = {}; + + int returnedResults = 0; + cublasLtMatmulHeuristicResult_t heuristicResult = {}; + cublasLtEpilogue_t epilogue = CUBLASLT_EPILOGUE_DEFAULT; + + // Create operation descriptor; see cublasLtMatmulDescAttributes_t + // for details about defaults; here we just set the transforms for + // A and B. + status = cublasLtMatmulDescInit(&operationDesc, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSA, &transa, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_TRANSB, &transb, sizeof(transa)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (use_bias) { + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_BIAS_POINTER, &bias, sizeof(bias)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + if (use_relu) { + epilogue = CUBLASLT_EPILOGUE_RELU_BIAS; + } else { + epilogue = CUBLASLT_EPILOGUE_BIAS; + } + } else { + if (use_relu) { + epilogue = CUBLASLT_EPILOGUE_RELU; + } + } + + status = cublasLtMatmulDescSetAttribute(&operationDesc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue)); + if (status != CUBLAS_STATUS_SUCCESS) { + goto CLEANUP; + } + + // Create matrix descriptors. Not setting any extra attributes. + status = cublasLtMatrixLayoutInit( + &Adesc, CUDA_R_32F, transa == CUBLAS_OP_N ? m : k, transa == CUBLAS_OP_N ? k : m, lda); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit( + &Bdesc, CUDA_R_32F, transb == CUBLAS_OP_N ? k : n, transb == CUBLAS_OP_N ? n : k, ldb); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatrixLayoutInit(&Cdesc, CUDA_R_32F, m, n, ldc); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // Create preference handle; In general, extra attributes can be + // used here to disable tensor ops or to make sure algo selected + // will work with badly aligned A, B, C. However, for simplicity + // here we assume A,B,C are always well aligned (e.g., directly + // come from cudaMalloc) + status = cublasLtMatmulPreferenceInit(&preference); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + status = cublasLtMatmulPreferenceSetAttribute( + &preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &workspaceSize, sizeof(workspaceSize)); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + // We just need the best available heuristic to try and run matmul. + // There is no guarantee that this will work. For example, if A is + // badly aligned, you can request more (e.g. 32) algos and try to + // run them one by one until something works. + status = cublasLtMatmulAlgoGetHeuristic( + ltHandle, &operationDesc, &Adesc, &Bdesc, &Cdesc, &Cdesc, &preference, 1, &heuristicResult, &returnedResults); + if (status != CUBLAS_STATUS_SUCCESS) goto CLEANUP; + + if (returnedResults == 0) { + status = CUBLAS_STATUS_NOT_SUPPORTED; + goto CLEANUP; + } + + status = cublasLtMatmul(ltHandle, + &operationDesc, + alpha, + A, + &Adesc, + B, + &Bdesc, + beta, + C, + &Cdesc, + C, + &Cdesc, + &heuristicResult.algo, + workspace, + workspaceSize, + stream); + +CLEANUP: + // Descriptors are no longer needed as all GPU work was already + // enqueued. + return status == CUBLAS_STATUS_SUCCESS ? 0 : 1; +} +#endif + +// Bias ADD. Assume input X is [features x batch size], column major. +// Bias is one 'features' long vector, with implicit broadcast. +template +__global__ void biasAdd_fprop(T *X, T *b, uint batch_size, uint features) { + T r_x[ILP]; + T r_b[ILP]; + if(is_aligned(X) && is_aligned(b) && features % ILP ==0) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid*ILP < features * batch_size; tid += blockDim.x * gridDim.x) { + int row = tid % (features / ILP); + load_store(r_x, X, 0 , tid); + load_store(r_b, b, 0 , row); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + float bias_sum = static_cast(r_x[ii]) + static_cast(r_b[ii]); + r_x[ii] = bias_sum; + } + load_store(X, r_x, tid , 0); + } + } else { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid < features * batch_size; tid += ILP * blockDim.x * gridDim.x) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + int row = tid % features; + r_x[ii] = X[idx]; + r_b[ii] = b[row]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + float bias_sum = static_cast(r_x[ii]) + static_cast(r_b[ii]); + r_x[ii] = bias_sum; + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + X[idx] = r_x[ii]; + } + } + } + } +} + +// Bias ADD + ReLU. Assume input X is [features x batch size], column major. +// Activation support fuesed ReLU. Safe to call in-place. +template +__global__ void biasAddRelu_fprop(T *X, T *b, uint batch_size, uint features) { + T r_x[ILP]; + T r_b[ILP]; + if(is_aligned(X) && is_aligned(b) && features % ILP ==0) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid*ILP < features * batch_size; tid += blockDim.x * gridDim.x) { + int row = tid % (features / ILP); + load_store(r_x, X, 0 , tid); + load_store(r_b, b, 0 , row); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + float bias_sum = static_cast(r_x[ii]) + static_cast(r_b[ii]); + r_x[ii] = relu(bias_sum); + } + load_store(X, r_x, tid , 0); + } + } else { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid < features * batch_size; tid += ILP * blockDim.x * gridDim.x) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + int row = tid % features; + r_x[ii] = X[idx]; + r_b[ii] = b[row]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + float bias_sum = static_cast(r_x[ii]) + static_cast(r_b[ii]); + r_x[ii] = relu(bias_sum); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + X[idx] = r_x[ii]; + } + } + } + } +} + +// ReLU. Assume input X is [features x batch size], column major. +// Safe to call in-place. +template +__global__ void Relu_fprop(T *X, uint batch_size, uint features) { + T r_x[ILP]; + if(is_aligned(X) && features % ILP ==0) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid*ILP < features * batch_size; tid += blockDim.x * gridDim.x) { + load_store(r_x, X, 0 , tid); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + r_x[ii] = relu(static_cast(r_x[ii])); + } + load_store(X, r_x, tid , 0); + } + } else { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid < features * batch_size; tid += ILP * blockDim.x * gridDim.x) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + r_x[ii] = X[idx]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + r_x[ii] = relu(static_cast(r_x[ii])); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + X[idx] = r_x[ii]; + } + } + } + } +} + +// Sigmoid. Assume input X is [features x batch size], column major. +// Safe to call in-place. +template +__global__ void Sigmoid_fprop(T *X, uint batch_size, uint features) { + T r_x[ILP]; + if(is_aligned(X) && features % ILP ==0) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid*ILP < features * batch_size; tid += blockDim.x * gridDim.x) { + load_store(r_x, X, 0 , tid); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + r_x[ii] = sigmoid(static_cast(r_x[ii])); + } + load_store(X, r_x, tid , 0); + } + } else { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid < features * batch_size; tid += ILP * blockDim.x * gridDim.x) { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + r_x[ii] = X[idx]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + r_x[ii] = sigmoid(static_cast(r_x[ii])); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) { + int idx = tid + ii * blockDim.x * gridDim.x; + if(idx < features * batch_size) { + X[idx] = r_x[ii]; + } + } + } + } +} + +// ReLU. Assume input X is [features x batch size], column major. +// Safe to call in-place. +template +__global__ void Relu_bprop(T *dY, T *Y, uint batch_size, uint features, T *dX) { + T r_dy[ILP]; + T r_y[ILP]; + if(is_aligned(dY) && + is_aligned(Y) && + is_aligned(dX) && + features % ILP ==0) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid*ILP < features * batch_size; tid += blockDim.x * gridDim.x) { + load_store(r_dy, dY, 0 , tid); + load_store(r_y, Y, 0 , tid); +#pragma unroll + for(int ii=0;ii +__global__ void Sigmoid_bprop(T *dY, T *Y, uint batch_size, uint features, T *dX) { + T r_dy[ILP]; + T r_y[ILP]; + if(is_aligned(dY) && + is_aligned(Y) && + is_aligned(dX) && + features % ILP ==0) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + for (; tid*ILP < features * batch_size; tid += blockDim.x * gridDim.x) { + load_store(r_dy, dY, 0 , tid); + load_store(r_y, Y, 0 , tid); +#pragma unroll + for(int ii=0;iimultiProcessorCount; + // can switch to occupancy calculation. use 4 below now for sm_70 + int max_blocks_y = (num_SMs * 4+(*grid_x)-1) / (*grid_x); + // block_y should be from minimal work per thread + int nRedSplits = (batch_size + block_y - 1) / block_y; + // increase number of elem per thread redcution to not launch more than enough + // kernel adjust work, so here we just launch max block + *grid_y = std::min(nRedSplits, max_blocks_y); + return; +} + +// Addition done deterministically via a 2-pass approach. Each CTA writes out partial +// sum, and the last CTA in grid Y dimension accumulates partials serially and writes to result. +template +__global__ void biasAdd_bprop( + T* dY, + int features, + int batch_size, + volatile float* intermediate, + int* semaphores, + T* db) { + // The feature that this thread is responsible for + int f = blockIdx.x * blockDim.x + threadIdx.x; + + // Compute the span this thread is responsible for + // For this block + int b_chunkSize = (batch_size + gridDim.y - 1) / gridDim.y; + int b_nStart = blockIdx.y * b_chunkSize; + int b_nSpan = min(batch_size, b_nStart + b_chunkSize) - b_nStart; + // For this thread + int chunkSize = (b_chunkSize + blockDim.y - 1) / blockDim.y; + int nStart = threadIdx.y * chunkSize + b_nStart; + int nSpan = min(b_nStart + b_nSpan, nStart + chunkSize) - nStart; + + volatile float* out = intermediate + blockIdx.y * features; + + // Flag to trigger last reduction. + __shared__ bool isLastBlock; + // we know block size for now + __shared__ float smem[BIAS_RELU_BW_NTHREADS_X*BIAS_RELU_BW_NTHREADS_Y]; + + // Accumulate db in FP32 always + float db_local = 0; + if (f < features) { + int nidx = 0; + // Handle non-multiple of UNROLL_FACTOR residue + for (; nidx < nSpan % UNROLL_FACTOR; nidx++) { + int64_t row, col, flat_idx; + row = f; + col = nStart + nidx; + flat_idx = col * features + row; + db_local += (float)dY[flat_idx]; + } + + // Handle meat of work + for (; (nidx + UNROLL_FACTOR - 1) < nSpan; nidx += UNROLL_FACTOR) { + int64_t row, col, flat_idx; + row = f; + col = nStart + nidx; + flat_idx = col * features + row; +#pragma unroll 4 + for (int u = 0; u < UNROLL_FACTOR; u++) { + db_local += (float)dY[flat_idx]; + flat_idx += features; + } + } + + // naive block reduction on y-dim + int linear_idx = threadIdx.y * blockDim.x + threadIdx.x; + smem[linear_idx] = db_local; + } + __syncthreads(); + if (f < features) { + if(threadIdx.y == 0) { + for(int yidx = 1; yidx < blockDim.y; yidx++){ + db_local += smem[yidx * blockDim.x + threadIdx.x]; + } + + // block result is in db_local now for all threadIdx.y == 0 + // Write out partial result + out[f] = db_local; + } + } + __threadfence(); + __syncthreads(); + + // Increment semaphore and check if this is the last CTA in the grid_y dimension. + // Only thread (0,0) calls this + if (threadIdx.x == 0 && threadIdx.y == 0 && f < features) { + unsigned int sum_idx; + sum_idx = atomicAdd(&(semaphores[blockIdx.x]), 1); + isLastBlock = (sum_idx == (gridDim.y - 1)); + } + __syncthreads(); + + db_local = 0; + // No block reduction for now, only thread (*,0) do grid reduction + if (isLastBlock && f < features) { + if(threadIdx.y == 0) { + for (int n = 0; n < gridDim.y; n++) { + int row, col; + row = f; + col = n; + db_local += (float)(intermediate[col * features + row]); + } + db[f] = (T)db_local; + } + } +} + +// Addition done deterministically via a 2-pass approach. Each CTA writes out partial +// sum, and the last CTA in grid Y dimension accumulates partials serially and writes to result. +template +__global__ void biasAddRelu_bprop( + T* Y, + T* dY, + int features, + int batch_size, + T* dX, + volatile float* intermediate, + int* semaphores, + T* db) { + // The feature that this thread is responsible for + int f = blockIdx.x * blockDim.x + threadIdx.x; + + // Compute the span this thread is responsible for + // For this block + int b_chunkSize = (batch_size + gridDim.y - 1) / gridDim.y; + int b_nStart = blockIdx.y * b_chunkSize; + int b_nSpan = min(batch_size, b_nStart + b_chunkSize) - b_nStart; + // For this thread + int chunkSize = (b_chunkSize + blockDim.y - 1) / blockDim.y; + int nStart = threadIdx.y * chunkSize + b_nStart; + int nSpan = min(b_nStart + b_nSpan, nStart + chunkSize) - nStart; + + volatile float* out = intermediate + blockIdx.y * features; + + // Flag to trigger last reduction. + __shared__ bool isLastBlock; + // we know block size for now + __shared__ float smem[BIAS_RELU_BW_NTHREADS_X*BIAS_RELU_BW_NTHREADS_Y]; + + // Accumulate db in FP32 always + float db_local = 0; + if (f < features) { + int nidx = 0; + // Handle non-multiple of UNROLL_FACTOR residue + for (; nidx < nSpan % UNROLL_FACTOR; nidx++) { + int row, col, flat_idx; + row = f; + col = nStart + nidx; + flat_idx = col * features + row; + T y_val = Y[flat_idx]; + T dy_val = dY[flat_idx]; + T dx_val; + if ((float)y_val > 0.f) + dx_val = dy_val; + else + dx_val = 0; + dX[flat_idx] = dx_val; + db_local += (float)dx_val; + } + + // Handle meat of work + for (; (nidx + UNROLL_FACTOR - 1) < nSpan; nidx += UNROLL_FACTOR) { + int row, col, flat_idx; + row = f; + col = nStart + nidx; + flat_idx = col * features + row; +#pragma unroll 4 + for (int u = 0; u < UNROLL_FACTOR; u++) { + T y_val = Y[flat_idx]; + T dy_val = dY[flat_idx]; + T dx_val; + if ((float)y_val > 0.f) + dx_val = dy_val; + else + dx_val = 0; + dX[flat_idx] = dx_val; + db_local += (float)dx_val; + flat_idx += features; + } + } + + // naive block reduction on y-dim + int linear_idx = threadIdx.y * blockDim.x + threadIdx.x; + smem[linear_idx] = db_local; + } + __syncthreads(); + if (f < features) { + if(threadIdx.y == 0) { + for(int yidx = 1; yidx < blockDim.y; yidx++){ + db_local += smem[yidx * blockDim.x + threadIdx.x]; + } + + // block result is in db_local now for all threadIdx.y == 0 + // Write out partial result + out[f] = db_local; + } + } + __threadfence(); + __syncthreads(); + + // Increment semaphore and check if this is the last CTA in the grid_y dimension. + // Only thread (0,0) calls this + if (threadIdx.x == 0 && threadIdx.y == 0 && f < features) { + unsigned int sum_idx; + sum_idx = atomicAdd(&(semaphores[blockIdx.x]), 1); + isLastBlock = (sum_idx == (gridDim.y - 1)); + } + __syncthreads(); + + db_local = 0; + // No block reduction for now, only thread (*,0) do grid reduction + if (isLastBlock && f < features) { + if(threadIdx.y == 0) { + for (int n = 0; n < gridDim.y; n++) { + int row, col; + row = f; + col = n; + db_local += (float)(intermediate[col * features + row]); + } + db[f] = (T)db_local; + } + } +} + +// Addition done deterministically via a 2-pass approach. Each CTA writes out partial +// sum, and the last CTA in grid Y dimension accumulates partials serially and writes to result. +template +__global__ void biasAddRelu_bprop_aligned( + T* Y, + T* dY, + int features, + int batch_size, + T* dX, + volatile float* intermediate, + int* semaphores, + T* db) { + // The feature that this thread is responsible for + int f = blockIdx.x * blockDim.x + threadIdx.x; + + // Compute the span this thread is responsible for + // For this block + int b_chunkSize = (batch_size + gridDim.y - 1) / gridDim.y; + int b_nStart = blockIdx.y * b_chunkSize; + int b_nSpan = min(batch_size, b_nStart + b_chunkSize) - b_nStart; + // For this thread + int chunkSize = (b_chunkSize + blockDim.y - 1) / blockDim.y; + int nStart = threadIdx.y * chunkSize + b_nStart; + int nSpan = min(b_nStart + b_nSpan, nStart + chunkSize) - nStart; + + volatile float* out = intermediate + blockIdx.y * features; + + // Flag to trigger last reduction. + __shared__ bool isLastBlock; + + // Accumulate db in FP32 always + float db_local[ILP]; + T r_y[ILP]; + T r_dy[ILP]; +#pragma unroll + for(int ii=0;ii +size_t get_mlp_bp_workspace_in_bytes(int batch_size, int num_layers, const int* output_features) { + size_t work_space = 0; + + // Store each intermediate dY explicitly. Need 2 dYs per MLP layer (one for o/p + // of biasReLU_bp and one for o/p of dgrad GEMM). + work_space += 2 * get_all_activations_size(batch_size, num_layers, output_features) * sizeof(T); + work_space += + get_reduction_scratch_space(batch_size, num_layers, output_features) * sizeof(float); + work_space += get_semaphores_size(num_layers, output_features) * sizeof(int); + + return work_space; +} + +// Returns pointers to each segment of the workspace +template +void partition_mlp_bp_workspace( + int batch_size, + int num_layers, + const int* output_features, + void* work_space, + T** dy_gemms, + T** dx_gemms, + float** db_scratch, + int** semaphores) { + /* + Workspace is partitioned as + DY_GEMMs : DX_GEMMs : DB_SCRATCH : SEMAPHORES + */ + // Start address where dy_gemm tensors are stored + *dy_gemms = reinterpret_cast(work_space); + // Start address where dx_gemm tensors are stored + *dx_gemms = *dy_gemms + get_all_activations_size(batch_size, num_layers, output_features); + // Start address where db intermediate tensors are stored + *db_scratch = reinterpret_cast( + *dx_gemms + get_all_activations_size(batch_size, num_layers, output_features)); + // Start address of semaphores + *semaphores = reinterpret_cast( + *db_scratch + get_reduction_scratch_space(batch_size, num_layers, output_features)); + + return; +} + +// Does a simple MLP fprop (GEMM+bias+ReLU). +// Can handle num_layers number of layers, each with its own shape. Output of layer i is assumed +// to be input of layer i+1. output_features, WPtr and BPtr are arrays of length num_layers, and +// must be in the same order i.e. WPtr[i] and BPtr[i] are respectively the weight and bias of layer +// 'i'. +template +int mlp_fp( + T* X, + int input_features, + int batch_size, + T** WPtr, + int num_layers, + int* output_features, + T** BPtr, + T* Y, + T* reserved_space, + int use_bias, + int activation, + void* lt_workspace) { + T *weight, *input, *output, *bias; + T *reserved_space_x, *reserved_space_y; + reserved_space_x = NULL; + reserved_space_y = reserved_space; + + // Get cublas handle from Pytorch + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + // Get the stream from cublas handle to reuse for biasReLU kernel. + cudaStream_t stream; + cublasGetStream(handle, &stream); + + for (int layer = 0; layer < num_layers; layer++) { + weight = WPtr[layer]; + input = (layer == 0) ? X : reserved_space_x; + output = (layer == num_layers - 1) ? Y : reserved_space_y; + if (use_bias) { + bias = BPtr[layer]; + } + int ifeat = (layer == 0) ? input_features : output_features[layer - 1]; + int ofeat = output_features[layer]; + + float one = 1.f; + float zero = 0.f; + + // try with cublaslt first for supported case with valid handle + int cublaslt_status = 1; +#if defined(CUBLAS_VERSION) && CUBLAS_VERSION >= 11000 + if(activation < 1){ + cublaslt_status = mlp_gemm_lt( + //ltHandle, + (cublasLtHandle_t)handle, + CUBLAS_OP_T, + CUBLAS_OP_N, + ofeat, + batch_size, + ifeat, + &one, + weight, + ifeat, + input, + ifeat, + &zero, + output, + ofeat, + lt_workspace, + 1 << 22, + stream, + use_bias == 1, + activation == 1, + bias); + } +#endif + + // if cublaslt failed or not executed, fallback to cublas + if (cublaslt_status != 0) { + cublasStatus_t cublas_status; + // Call GEMM: fprop is Y = W'X + cublas_status = mlp_gemm( + handle, + CUBLAS_OP_T, + CUBLAS_OP_N, + ofeat, + batch_size, + ifeat, + &one, + weight, + ifeat, + input, + ifeat, + &zero, + output, + ofeat); + + if (cublas_status != CUBLAS_STATUS_SUCCESS) { + printf("GEMM fprop failed with %d\n", cublas_status); + return 1; + } + + const uint &input_size = ofeat; + int num_blocks = 0; + int num_SMs = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + // Call biasReLU + if(use_bias == 1) { + if (activation == 0) { // no activation + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, biasAdd_fprop, BIAS_RELU_FW_NTHREADS, 0); + biasAdd_fprop<<>>(output, bias, batch_size, input_size); + } else if (activation == 1) { // relu + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, biasAddRelu_fprop, BIAS_RELU_FW_NTHREADS, 0); + biasAddRelu_fprop<<>>(output, bias, batch_size, input_size); + } else if (activation == 2) { // sigmoid + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, biasAdd_fprop, BIAS_RELU_FW_NTHREADS, 0); + biasAdd_fprop<<>>(output, bias, batch_size, input_size); + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, Sigmoid_fprop, BIAS_RELU_FW_NTHREADS, 0); + Sigmoid_fprop<<>>(output, batch_size, input_size); + } + } else { + // don't need to do anything in case of no activation and no bias + if (activation == 1) { // relu + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, Relu_fprop, BIAS_RELU_FW_NTHREADS, 0); + Relu_fprop<<>>(output, batch_size, input_size); + } else if (activation == 2) { // sigmoid + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, Sigmoid_fprop, BIAS_RELU_FW_NTHREADS, 0); + Sigmoid_fprop<<>>(output, batch_size, input_size); + } + } + } + // Set current output as next layer input + reserved_space_x = reserved_space_y; + // Set next layer output + reserved_space_y += ofeat * batch_size; + } + + return 0; +} + +// Does a simple MLP bprop (GEMM+bias+ReLU). +// Needs reserved space to come back exactly as it was populated in fprop. +// Does dgrad and wgrad sequentially. +template +int mlp_bp( + T* X, + T* Y, + int input_features, + int batch_size, + T** WPtr, + int num_layers, + int* output_features, + T* dY, + T* reserved_space, + T* work_space, + T* dX, + T** dwPtr, + T** dbPtr, + bool requires_grad, + int use_bias, + int activation) { + T* weight; + T *dweight, *dx, *dy, *dbias; + T *x, *y; + + // Where the dx of the biasReLU (== dy of gemm) is stored. Can be thrown away + // after bp call. + T* dy_gemm_base; + // Where the dx after GEMM is stored. + T* dx_gemm_base; + // Where partial reduction results are stored. + float* db_scratch; + // Semaphores for reduction. + int* semaphores; + + partition_mlp_bp_workspace( + batch_size, + num_layers, + output_features, + work_space, + &dy_gemm_base, + &dx_gemm_base, + &db_scratch, + &semaphores); + + size_t semaphore_size = get_semaphores_size(num_layers, output_features) * sizeof(int); + + // Get cublas handle from Pytorch + cublasHandle_t handle = at::cuda::getCurrentCUDABlasHandle(); + // Get the stream from cublas handle to reuse for biasReLU kernel. + cudaStream_t stream; + cublasGetStream(handle, &stream); + + int* y_offsets = (int*)malloc(num_layers * sizeof(int)); + get_y_offsets(batch_size, num_layers, output_features, y_offsets); + + for (int layer = num_layers - 1; layer >= 0; layer--) { + weight = WPtr[layer]; + dweight = dwPtr[layer]; + + // x is read from reserved space + x = (layer == 0) ? X : reserved_space + y_offsets[layer - 1]; + // dx is written in workspace for all but layer==0 + dx = (layer == 0) ? dX : dx_gemm_base + y_offsets[layer - 1]; + + // y is read from reserved space + y = (layer == num_layers - 1) ? Y : reserved_space + y_offsets[layer]; + // dx from layer+1 + dy = (layer == num_layers - 1) ? dY : dx_gemm_base + y_offsets[layer]; + // dy_gemm is written to and read immediately + T* dy_gemm = dy_gemm_base + y_offsets[layer]; + + dbias = dbPtr[layer]; + int xfeat = (layer == 0) ? input_features : output_features[layer - 1]; + int yfeat = output_features[layer]; + + float one = 1.f; + float zero = 0.f; + + if (use_bias == 1) { + if (activation == 0) { // no acitvation + // bgrad + dim3 block(BIAS_RELU_BW_NTHREADS_X, BIAS_RELU_BW_NTHREADS_Y); + int grid_x, grid_y; + cudaMemsetAsync(semaphores, 0, semaphore_size, stream); + + int block_x = BIAS_RELU_BW_NTHREADS_X; + int block_y = BIAS_RELU_RED_PER_THREAD * BIAS_RELU_BW_NTHREADS_Y; + get_biasAddRelu_bprop_grid_size(yfeat, batch_size, block_x, block_y, &grid_x, &grid_y); + dim3 grid(grid_x, grid_y); + biasAdd_bprop<<>>( + dy, yfeat, batch_size, db_scratch, semaphores, dbias); + // bypass dgrad through reset pointer + dy_gemm = dy; + } else if (activation == 1) { // relu + dim3 block(BIAS_RELU_BW_NTHREADS_X, BIAS_RELU_BW_NTHREADS_Y); + int grid_x, grid_y; + cudaMemsetAsync(semaphores, 0, semaphore_size, stream); + + if(yfeat % (ILP * BIAS_RELU_BW_NTHREADS_X) == 0 && + is_aligned(y) && + is_aligned(dy) && + is_aligned(dy_gemm) && + is_aligned(dbias)){ + int block_x = ILP * BIAS_RELU_BW_NTHREADS_X; + int block_y = BIAS_RELU_RED_PER_THREAD * BIAS_RELU_BW_NTHREADS_Y; + get_biasAddRelu_bprop_grid_size(yfeat, batch_size, block_x, block_y, &grid_x, &grid_y); + dim3 grid(grid_x, grid_y); + biasAddRelu_bprop_aligned<<>>( + y, dy, yfeat, batch_size, dy_gemm, db_scratch, semaphores, dbias); + } else { + int block_x = BIAS_RELU_BW_NTHREADS_X; + int block_y = BIAS_RELU_RED_PER_THREAD * BIAS_RELU_BW_NTHREADS_Y; + get_biasAddRelu_bprop_grid_size(yfeat, batch_size, block_x, block_y, &grid_x, &grid_y); + dim3 grid(grid_x, grid_y); + biasAddRelu_bprop<<>>( + y, dy, yfeat, batch_size, dy_gemm, db_scratch, semaphores, dbias); + } + } else if (activation == 2) { // sigmoid + // activation backward + int num_blocks = 0; + int num_SMs = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, Sigmoid_bprop, BIAS_RELU_FW_NTHREADS, 0); + Sigmoid_bprop<<>>(dy, y, batch_size, yfeat, dy_gemm); + + // bgrad, from dy_gemm + dim3 block(BIAS_RELU_BW_NTHREADS_X, BIAS_RELU_BW_NTHREADS_Y); + int grid_x, grid_y; + cudaMemsetAsync(semaphores, 0, semaphore_size, stream); + + int block_x = BIAS_RELU_BW_NTHREADS_X; + int block_y = BIAS_RELU_RED_PER_THREAD * BIAS_RELU_BW_NTHREADS_Y; + get_biasAddRelu_bprop_grid_size(yfeat, batch_size, block_x, block_y, &grid_x, &grid_y); + dim3 grid(grid_x, grid_y); + biasAdd_bprop<<>>( + dy_gemm, yfeat, batch_size, db_scratch, semaphores, dbias); + } + } else { // no bias below + if (activation == 0) { + // bypass dgrad through reset pointer + dy_gemm = dy; + } else if (activation == 1) { // relu + int num_blocks = 0; + int num_SMs = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, Relu_bprop, BIAS_RELU_FW_NTHREADS, 0); + Relu_bprop<<>>(dy, y, batch_size, yfeat, dy_gemm); + } else if (activation == 2) { // sigmoid + int num_blocks = 0; + int num_SMs = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, Sigmoid_bprop, BIAS_RELU_FW_NTHREADS, 0); + Sigmoid_bprop<<>>(dy, y, batch_size, yfeat, dy_gemm); + } + } + + cublasStatus_t cublas_status; + // Call GEMM dgrad + if (layer > 0 || requires_grad == 1) { + cublas_status = mlp_gemm( + handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + xfeat, + batch_size, + yfeat, + &one, + weight, + xfeat, + dy_gemm, + yfeat, + &zero, + dx, + xfeat); + + if (cublas_status != CUBLAS_STATUS_SUCCESS) { + printf("GEMM dgrad failed with %d\n", cublas_status); + return 1; + } + } + + // Call GEMM wgrad + cublas_status = mlp_gemm( + handle, + CUBLAS_OP_N, + CUBLAS_OP_T, + xfeat, + yfeat, + batch_size, + &one, + x, + xfeat, + dy_gemm, + yfeat, + &zero, + dweight, + xfeat); + + if (cublas_status != CUBLAS_STATUS_SUCCESS) { + printf("GEMM wgrad failed with %d\n", cublas_status); + return 1; + } + } + + return 0; +} + +// Instantiate for floating point types +template int mlp_fp( + float* X, + int input_features, + int batch_size, + float** WPtr, + int num_layers, + int* output_features, + float** BPtr, + float* Y, + float* reserved_space, + int use_bias, + int activation, + void* lt_workspace); + +template int mlp_bp( + float* X, + float* Y, + int input_features, + int batch_size, + float** WPtr, + int num_layers, + int* output_features, + float* dY, + float* reserved_space, + float* work_space, + float* dX, + float** dwPtr, + float** dbPtr, + bool requires_grad, + int use_bias, + int activation); + +template int mlp_fp( + at::Half* X, + int input_features, + int batch_size, + at::Half** WPtr, + int num_layers, + int* output_features, + at::Half** BPtr, + at::Half* Y, + at::Half* reserved_space, + int use_bias, + int activation, + void* lt_workspace); + +template int mlp_bp( + at::Half* X, + at::Half* Y, + int input_features, + int batch_size, + at::Half** WPtr, + int num_layers, + int* output_features, + at::Half* dY, + at::Half* reserved_space, + at::Half* work_space, + at::Half* dX, + at::Half** dwPtr, + at::Half** dbPtr, + bool requires_grad, + int use_bias, + int activation); + +template int mlp_fp( + double* X, + int input_features, + int batch_size, + double** WPtr, + int num_layers, + int* output_features, + double** BPtr, + double* Y, + double* reserved_space, + int use_bias, + int activation, + void* lt_workspace); + +template int mlp_bp( + double* X, + double* Y, + int input_features, + int batch_size, + double** WPtr, + int num_layers, + int* output_features, + double* dY, + double* reserved_space, + double* work_space, + double* dX, + double** dwPtr, + double** dbPtr, + bool requires_grad, + int use_bias, + int activation); + +template size_t get_mlp_bp_workspace_in_bytes( + int batch_size, + int num_layers, + const int* output_features); +template size_t get_mlp_bp_workspace_in_bytes( + int batch_size, + int num_layers, + const int* output_features); +template size_t get_mlp_bp_workspace_in_bytes( + int batch_size, + int num_layers, + const int* output_features); + diff --git a/apex/csrc/multi_tensor_adagrad.cu b/apex/csrc/multi_tensor_adagrad.cu new file mode 100644 index 00000000..699681bc --- /dev/null +++ b/apex/csrc/multi_tensor_adagrad.cu @@ -0,0 +1,100 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "multi_tensor_apply.cuh" +#include "type_shim.h" + +#define BLOCK_SIZE 1024 +#define ILP 4 + +typedef enum { + ADAGRAD_MODE_0 = 0, // L2 regularization mode. + ADAGRAD_MODE_1 = 1, // AdamW-style weight decay. + +} adagradMode_t; + +using MATH_T = float; + +template struct AdagradFunctor { + __device__ __forceinline__ void + operator()(int chunk_size, volatile int *noop_gmem, TensorListMetadata<3> &tl, + const float epsilon, const float lr, adagradMode_t mode, + const float weight_decay) { + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + T *g = (T *)tl.addresses[0][tensor_loc]; + g += chunk_idx * chunk_size; + + T *p = (T *)tl.addresses[1][tensor_loc]; + p += chunk_idx * chunk_size; + + T *h = (T *)tl.addresses[2][tensor_loc]; + h += chunk_idx * chunk_size; + + n -= chunk_idx * chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for (int i_start = 0; i_start < n && i_start < chunk_size; + i_start += blockDim.x * ILP) { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_h[ILP]; +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + int i = i_start + threadIdx.x + ii * blockDim.x; + if (i < n && i < chunk_size) { + r_g[ii] = g[i]; + r_p[ii] = p[i]; + r_h[ii] = h[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_h[ii] = MATH_T(0); + } + } +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + if (mode == ADAGRAD_MODE_0) { // L2 + r_g[ii] = r_g[ii] + weight_decay * r_p[ii]; + r_h[ii] = r_h[ii] + r_g[ii] * r_g[ii]; + r_p[ii] = r_p[ii] - lr * (r_g[ii] / (sqrtf(r_h[ii]) + epsilon)); + } else { // AdamW-style + r_h[ii] = r_h[ii] + r_g[ii] * r_g[ii]; + r_p[ii] = r_p[ii] - lr * (r_g[ii] / (sqrtf(r_h[ii]) + epsilon) + weight_decay * r_p[ii]); + } + } +#pragma unroll + for (int ii = 0; ii < ILP; ii++) { + int i = i_start + threadIdx.x + ii * blockDim.x; + if (i < n && i < chunk_size) { + p[i] = r_p[ii]; + h[i] = r_h[ii]; + } + } + } + } +}; + +void multi_tensor_adagrad_cuda( + int chunk_size, at::Tensor noop_flag, + std::vector> tensor_lists, const float lr, + const float epsilon, const int mode, const float weight_decay) { + using namespace at; + + // Assume single type across p,g,h now + DISPATCH_DOUBLE_FLOAT_AND_HALF( + tensor_lists[0][0].scalar_type(), 0, "adagrad", + multi_tensor_apply<3>(BLOCK_SIZE, chunk_size, noop_flag, tensor_lists, + AdagradFunctor(), epsilon, lr, + (adagradMode_t)mode, weight_decay);) + + AT_CUDA_CHECK(cudaGetLastError()); +} diff --git a/apex/csrc/multi_tensor_adam.cu b/apex/csrc/multi_tensor_adam.cu new file mode 100644 index 00000000..e09cd8f0 --- /dev/null +++ b/apex/csrc/multi_tensor_adam.cu @@ -0,0 +1,478 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +typedef enum{ + ADAM_MODE_0 =0, // L2 regularization mode + ADAM_MODE_1 =1 // Decoupled weight decay mode(AdamW) +} adamMode_t; + +using MATH_T = float; + +template +struct AdamFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<4>& tl, + const float beta1, + const float beta2, + const float beta1_correction, + const float beta2_correction, + const float epsilon, + const float lr, + adamMode_t mode, + const float decay) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + + // potentially use to pass in list of scalar + // int tensor_num = tl.start_tensor_this_launch + tensor_loc; + + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + FULL_T* m = (FULL_T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + FULL_T* v = (FULL_T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i]; + r_p[ii] = p[i]; + r_m[ii] = m[i]; + r_v[ii] = v[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if(mode == ADAM_MODE_0) { // L2 + r_g[ii] = r_g[ii] + (decay * r_p[ii]); + r_m[ii] = beta1 * r_m[ii] + (1-beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1-beta2) * r_g[ii] * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + MATH_T update = next_m_unbiased / denom; + r_p[ii] = r_p[ii] - (lr * update); + } + else { // weight decay + r_m[ii] = beta1 * r_m[ii] + (1-beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1-beta2) * r_g[ii] * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + MATH_T update = (next_m_unbiased / denom) + (decay * r_p[ii]); + r_p[ii] = r_p[ii] - (lr * update); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + m[i] = r_m[ii]; + v[i] = r_v[ii]; + } + } + } + } +}; + +template +struct AdamCapturableFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<4>& tl, + const float beta1, + const float beta2, + const int* step, + const int bias_correction, + const float epsilon, + const float* lr, + adamMode_t mode, + const float decay, + const float* inv_scale) + { + if(*noop_gmem == 1) + return; + + float beta1_correction = 1.0f, beta2_correction = 1.0f; + if (bias_correction == 1) { + beta1_correction = 1 - pow(beta1, *step); + beta2_correction = 1 - pow(beta2, *step); + } + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + + // potentially use to pass in list of scalar + // int tensor_num = tl.start_tensor_this_launch + tensor_loc; + + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + FULL_T* m = (FULL_T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + FULL_T* v = (FULL_T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = static_cast(g[i]) * (*inv_scale); + g[i] = static_cast(r_g[ii]); + r_p[ii] = static_cast(p[i]); + r_m[ii] = static_cast(m[i]); + r_v[ii] = static_cast(v[i]); + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if(mode == ADAM_MODE_0) { // L2 + r_g[ii] = r_g[ii] + (decay * r_p[ii]); + r_m[ii] = beta1 * r_m[ii] + (1-beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1-beta2) * r_g[ii] * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + MATH_T update = next_m_unbiased / denom; + r_p[ii] = r_p[ii] - (*lr * update); + } + else { // weight decay + r_m[ii] = beta1 * r_m[ii] + (1-beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1-beta2) * r_g[ii] * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + MATH_T update = (next_m_unbiased / denom) + (decay * r_p[ii]); + r_p[ii] = r_p[ii] - (*lr * update); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = static_cast(r_p[ii]); + m[i] = static_cast(r_m[ii]); + v[i] = static_cast(r_v[ii]); + } + } + } + } +}; + +template +struct AdamCapturableMasterFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<5>& tl, + const float beta1, + const float beta2, + const int* step, + const int bias_correction, + const float epsilon, + const float* lr, + adamMode_t mode, + const float decay, + const float* inv_scale) + { + if(*noop_gmem == 1) + return; + + float beta1_correction = 1.0f, beta2_correction = 1.0f; + if (bias_correction == 1) { + beta1_correction = 1 - pow(beta1, *step); + beta2_correction = 1 - pow(beta2, *step); + } + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + + // potentially use to pass in list of scalar + // int tensor_num = tl.start_tensor_this_launch + tensor_loc; + + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + FULL_T* m = (FULL_T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + FULL_T* v = (FULL_T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + FULL_T* p_master = (FULL_T*)tl.addresses[4][tensor_loc]; + p_master += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = static_cast(g[i]) * (*inv_scale); + g[i] = static_cast(r_g[ii]); + r_p[ii] = static_cast(p_master[i]); + r_m[ii] = static_cast(m[i]); + r_v[ii] = static_cast(v[i]); + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if(mode == ADAM_MODE_0) { // L2 + r_g[ii] = r_g[ii] + (decay * r_p[ii]); + r_m[ii] = beta1 * r_m[ii] + (1-beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1-beta2) * r_g[ii] * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + MATH_T update = next_m_unbiased / denom; + r_p[ii] = r_p[ii] - (*lr * update); + } + else { // weight decay + r_m[ii] = beta1 * r_m[ii] + (1-beta1) * r_g[ii]; + r_v[ii] = beta2 * r_v[ii] + (1-beta2) * r_g[ii] * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + MATH_T update = (next_m_unbiased / denom) + (decay * r_p[ii]); + r_p[ii] = r_p[ii] - (*lr * update); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = static_cast(r_p[ii]); + p_master[i] = static_cast(r_p[ii]); + m[i] = static_cast(r_m[ii]); + v[i] = static_cast(r_v[ii]); + } + } + } + } +}; + +void multi_tensor_adam_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int mode, + const int bias_correction, + const float weight_decay) +{ + using namespace at; + + // Handle bias correction mode + float bias_correction1 = 1.0f, bias_correction2 = 1.0f; + if (bias_correction == 1) { + bias_correction1 = 1 - std::pow(beta1, step); + bias_correction2 = 1 - std::pow(beta2, step); + } + + // Assume single type across p,g,m1,m2 now + DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT( + tensor_lists[0][0].scalar_type(), 0, "adam", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamFunctor(), + beta1, + beta2, + bias_correction1, + bias_correction2, + epsilon, + lr, + (adamMode_t) mode, + weight_decay); ) + + AT_CUDA_CHECK(cudaGetLastError()); + +} + +void multi_tensor_adam_capturable_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor lr, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor step, + const int mode, + const int bias_correction, + const float weight_decay, + at::Tensor inv_scale) +{ + using namespace at; + + DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT( + tensor_lists[0][0].scalar_type(), 0, "adam", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamCapturableFunctor(), + beta1, + beta2, + step.data_ptr(), + bias_correction, + epsilon, + lr.data_ptr(), + (adamMode_t) mode, + weight_decay, + inv_scale.data_ptr()); ) + + AT_CUDA_CHECK(cudaGetLastError()); + +} + +void multi_tensor_adam_capturable_master_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor lr, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor step, + const int mode, + const int bias_correction, + const float weight_decay, + at::Tensor inv_scale) +{ + using namespace at; + + DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT( + tensor_lists[0][0].scalar_type(), 0, "adam", + multi_tensor_apply<5>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AdamCapturableMasterFunctor(), + beta1, + beta2, + step.data_ptr(), + bias_correction, + epsilon, + lr.data_ptr(), + (adamMode_t) mode, + weight_decay, + inv_scale.data_ptr()); ) + + AT_CUDA_CHECK(cudaGetLastError()); + +} + diff --git a/apex/csrc/multi_tensor_apply.cuh b/apex/csrc/multi_tensor_apply.cuh new file mode 100644 index 00000000..9261d707 --- /dev/null +++ b/apex/csrc/multi_tensor_apply.cuh @@ -0,0 +1,133 @@ +#include +#include +#include +#include +#include +#include "compat.h" + +#include + +// #include + +// This header is the one-stop shop for all your multi-tensor apply needs. + + +// TODO: Kernel arg size limit may be <4KB for some other cards (ie Jetson) +constexpr int depth_to_max_tensors[6] = {110, 64, 48, 36, 30, 24}; +constexpr int depth_to_max_blocks[6] = {320, 320, 320, 320, 320, 320}; + +template struct TensorListMetadata +{ + void* addresses[n][depth_to_max_tensors[n-1]]; + int sizes[depth_to_max_tensors[n-1]]; + unsigned char block_to_tensor[depth_to_max_blocks[n-1]]; + int block_to_chunk[depth_to_max_blocks[n-1]]; // I fear this needs to be a full int. + int start_tensor_this_launch; +}; + + +template +__global__ void multi_tensor_apply_kernel( + int chunk_size, + volatile int* noop_flag, + T tl, + U callable, + ArgTypes... args) +{ + // Hand the chunk information to the user-supplied functor to process however it likes. + callable(chunk_size, noop_flag, tl, args...); +} + +template +void multi_tensor_apply( + int block_size, + int chunk_size, + const at::Tensor& noop_flag, + const std::vector>& tensor_lists, + T callable, + ArgTypes... args) +{ + TORCH_CHECK(tensor_lists.size() == depth, "tensor_lists.size() != depth"); + int len0 = tensor_lists[0].size(); + TORCH_CHECK(len0 > 0, "tensor_lists[0].size() is not > 0"); + auto ref_device = tensor_lists[0][0].device(); + TORCH_CHECK(ref_device.type() == at::kCUDA, "expected input to be on cuda"); + for (int l = 0; l < tensor_lists.size(); l++) // No range-based for because I need indices + { + TORCH_CHECK(tensor_lists[l].size() == len0, "Size mismatch among tensor lists"); + for(int t = 0; t < tensor_lists[l].size(); t++) + { + // TODO: Print which tensor fails. + bool contiguous_memory = tensor_lists[l][t].is_contiguous(); +#ifdef VERSION_GE_1_5 + contiguous_memory = (contiguous_memory || tensor_lists[l][t].is_contiguous(at::MemoryFormat::ChannelsLast) || tensor_lists[l][t].is_contiguous(at::MemoryFormat::ChannelsLast3d)); +#endif + TORCH_CHECK(contiguous_memory, "A tensor was not contiguous."); + TORCH_CHECK(tensor_lists[l][t].device() == ref_device, "A tensor was not on the same device as the first tensor"); + TORCH_CHECK(tensor_lists[l][t].numel() == tensor_lists[0][t].numel(), "Size mismatch"); + } + } + + int ntensors = tensor_lists[0].size(); + + TensorListMetadata tl; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(tensor_lists[0][0])); + auto stream = at::cuda::getCurrentCUDAStream(); + + tl.start_tensor_this_launch = 0; + int loc_block_info = 0; + int loc_tensor_info = 0; + for(int t = 0; t < ntensors; t++) + { + tl.sizes[loc_tensor_info] = tensor_lists[0][t].numel(); + for(int d = 0; d < depth; d++) + tl.addresses[d][loc_tensor_info] = tensor_lists[d][t].data_ptr(); + loc_tensor_info++; + + int chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size; + + for(int chunk = 0; chunk < chunks_this_tensor; chunk++) + { + // std::cout << chunks_this_tensor << std::endl; + tl.block_to_tensor[loc_block_info] = loc_tensor_info - 1; + tl.block_to_chunk[loc_block_info] = chunk; + loc_block_info++; + + bool tensors_full = (loc_tensor_info == depth_to_max_tensors[depth-1] && + chunk == chunks_this_tensor - 1); + bool blocks_full = (loc_block_info == depth_to_max_blocks[depth-1]); + bool last_chunk = (t == ntensors - 1 && chunk == chunks_this_tensor - 1); + if(tensors_full || blocks_full || last_chunk) + { + // using accscalar_t = acc_type; + multi_tensor_apply_kernel<<>>( + chunk_size, + noop_flag.DATA_PTR(), + tl, + callable, + args...); + + AT_CUDA_CHECK(cudaGetLastError()); + + // Reset. The control flow possibilities here make my brain hurt. + loc_block_info = 0; + if(chunk == chunks_this_tensor - 1) + { + // std::cout << "Hit case 1 " << cond1 << " " << cond2 << " " << cond3 << std::endl; + loc_tensor_info = 0; + tl.start_tensor_this_launch = t + 1; + } + else + { + // std::cout << "Hit case 2 " << cond1 << " " << cond2 << " " << cond3 << std::endl; + tl.sizes[0] = tl.sizes[loc_tensor_info-1]; + for(int d = 0; d < depth; d++) + tl.addresses[d][0] = tl.addresses[d][loc_tensor_info-1]; + loc_tensor_info = 1; + tl.start_tensor_this_launch = t; + } + } + } + } +} diff --git a/apex/csrc/multi_tensor_axpby_kernel.cu b/apex/csrc/multi_tensor_axpby_kernel.cu new file mode 100644 index 00000000..021df27d --- /dev/null +++ b/apex/csrc/multi_tensor_axpby_kernel.cu @@ -0,0 +1,157 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +template +struct AxpbyFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<3>& tl, + float a, + float b, + int arg_to_check) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + x_t* x = (x_t*)tl.addresses[0][tensor_loc]; + x += chunk_idx*chunk_size; + + y_t* y = (y_t*)tl.addresses[1][tensor_loc]; + y += chunk_idx*chunk_size; + + out_t* out = (out_t*)tl.addresses[2][tensor_loc]; + out += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + bool finite = true; + x_t r_x[ILP]; + y_t r_y[ILP]; + out_t r_out[ILP]; + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(x) && is_aligned(y) && is_aligned(out)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_x, x, 0 , i_start); + load_store(r_y, y, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_out[ii] = a*static_cast(r_x[ii]) + b*static_cast(r_y[ii]); + if(arg_to_check == -1) + finite = finite && (isfinite(r_x[ii]) && isfinite(r_y[ii])); + if(arg_to_check == 0) + finite = finite && isfinite(r_x[ii]); + if(arg_to_check == 1) + finite = finite && isfinite(r_y[ii]); + } + // store + load_store(out, r_out, i_start , 0); + } + } + else + { + // Non-divergent exit condition for __syncthreads, not necessary here + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_x[ii] = 0; + r_y[ii] = 0; + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_x[ii] = x[i]; + r_y[ii] = y[i]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_out[ii] = a*static_cast(r_x[ii]) + b*static_cast(r_y[ii]); + if(arg_to_check == -1) + finite = finite && (isfinite(r_x[ii]) && isfinite(r_y[ii])); + if(arg_to_check == 0) + finite = finite && isfinite(r_x[ii]); + if(arg_to_check == 1) + finite = finite && isfinite(r_y[ii]); + } + // see note in multi_tensor_scale_kernel.cu +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + out[i] = r_out[ii]; + } + } + } + if(!finite) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + } +}; + +void multi_tensor_axpby_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float a, + float b, + int arg_to_check) +{ + using namespace at; + // The output (downscaled) type is always float. + // If build times suffer, think about where to put this dispatch, + // and what logic should be moved out of multi_tensor_apply. + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "multi_tensor_axpby_cuda", + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, "multi_tensor_axpby_cuda", + DISPATCH_FLOAT_AND_HALF(tensor_lists[2][0].scalar_type(), 2, "multi_tensor_axpby_cuda", + multi_tensor_apply<3>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + AxpbyFunctor(), + a, + b, + arg_to_check); ))) + + AT_CUDA_CHECK(cudaGetLastError()); + + // AT_CUDA_CHECK(cudaDeviceSynchronize()); +} diff --git a/apex/csrc/multi_tensor_l2norm_kernel.cu b/apex/csrc/multi_tensor_l2norm_kernel.cu new file mode 100644 index 00000000..49bcd701 --- /dev/null +++ b/apex/csrc/multi_tensor_l2norm_kernel.cu @@ -0,0 +1,448 @@ +#include +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +template +struct L2NormFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<1>& tl, + float* output, + float* output_per_tensor, + bool per_tensor, + int max_chunks_per_tensor) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + x_t* x = (x_t*)tl.addresses[0][tensor_loc]; + x += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + __shared__ float s_vals[512]; + + float vals[ILP]; // = {0}; // this probably works too but I want to be sure... + x_t r_x[ILP]; + for(int i = 0; i < ILP; i++) + { + vals[i] = 0.f; + r_x[i] = 0; + } + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(x)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_x, x, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + float next = static_cast(r_x[ii]); + vals[ii] += next*next; + } + } + } + else + { + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + float next = static_cast(x[i]); + vals[ii] += next*next; + } + } + } + } + + float val = 0.f; + for(int i = 0; i < ILP; i++) + val += vals[i]; + + float final = reduce_block_into_lanes(s_vals, val); + + if(threadIdx.x == 0) + { + if(!isfinite(final)) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + output[blockIdx.x] += final; + if(per_tensor) + output_per_tensor[(tl.start_tensor_this_launch + tensor_loc)*max_chunks_per_tensor + chunk_idx] = final; + } + } +}; + +// Probably better to template, but since we are not likely to support other norm +template +struct MaxNormFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<1>& tl, + float* output, + float* output_per_tensor, + bool per_tensor, + int max_chunks_per_tensor) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + x_t* x = (x_t*)tl.addresses[0][tensor_loc]; + x += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + __shared__ float s_vals[512]; + + float vals[ILP]; // = {0}; // this probably works too but I want to be sure... + x_t r_x[ILP]; + for(int i = 0; i < ILP; i++) + { + vals[i] = 0.f; + r_x[i] = 0; + } + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(x)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_x, x, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + float next = static_cast(r_x[ii]); + vals[ii] = fmaxf(fabsf(vals[ii]), fabsf(next)); + } + } + } + else + { + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + float next = static_cast(x[i]); + vals[ii] = fmaxf(fabsf(vals[ii]), fabsf(next)); + } + } + } + } + + float val = 0.f; + for(int i = 0; i < ILP; i++) + val = fmaxf(fabsf(val), fabsf(vals[i])); + + float final = reduce_block_into_lanes_max_op(s_vals, val); + + if(threadIdx.x == 0) + { + if(!isfinite(final)) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + output[blockIdx.x] = fmaxf(fabsf(output[blockIdx.x]), fabsf(final)); + if(per_tensor) + output_per_tensor[(tl.start_tensor_this_launch + tensor_loc)*max_chunks_per_tensor + chunk_idx] = final; + } + } +}; + + +__global__ void cleanup( + float* output, + float* output_per_tensor, + float* ret, + float* ret_per_tensor, + bool per_tensor, + int max_chunks_per_tensor) +{ + __shared__ float vals[512]; + + if(blockIdx.x == 0) + { + float val = 0; + if(threadIdx.x < 320) + val = output[threadIdx.x]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + *ret = sqrt(final); + } + + if(per_tensor) + { + float* output_this_tensor = output_per_tensor + blockIdx.x*max_chunks_per_tensor; + + float val = 0; + for(int i = threadIdx.x; i < max_chunks_per_tensor; i += blockDim.x) + val += output_this_tensor[i]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + ret_per_tensor[blockIdx.x] = sqrt(final); + } +} + +__global__ void cleanup_v2( + float* output, + float* output_per_tensor, + float* ret, + float* ret_per_tensor, + bool per_tensor, + int max_chunks_per_tensor, + int norm_type, + float alpha, + float beta) +{ + __shared__ float vals[512]; + + if(blockIdx.x == 0) + { + float val = 0; + if(threadIdx.x < 320) + val = output[threadIdx.x]; + + if (norm_type == 0) { + float final = reduce_block_into_lanes_max_op(vals, val); + if(threadIdx.x == 0) + *ret = alpha * (*ret) + beta * final; + } + else { + float final = reduce_block_into_lanes(vals, val); + if(threadIdx.x == 0) + *ret = sqrt(alpha * (*ret) * (*ret) + beta * final); + } + } + + if(per_tensor) + { + float* output_this_tensor = output_per_tensor + blockIdx.x*max_chunks_per_tensor; + + if (norm_type == 0) { + float val = 0; + for(int i = threadIdx.x; i < max_chunks_per_tensor; i += blockDim.x) + val = fmaxf(fabsf(val), fabsf(output_this_tensor[i])); + + float final = reduce_block_into_lanes_max_op(vals, val); + + if(threadIdx.x == 0) + ret_per_tensor[blockIdx.x] = alpha * ret_per_tensor[blockIdx.x] + beta * final; + } + else { + float val = 0; + for(int i = threadIdx.x; i < max_chunks_per_tensor; i += blockDim.x) + val += output_this_tensor[i]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + ret_per_tensor[blockIdx.x] = sqrt(alpha * ret_per_tensor[blockIdx.x] * ret_per_tensor[blockIdx.x] + beta * final); + } + } +} + +std::tuple multi_tensor_l2norm_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python) +{ + bool per_tensor = per_tensor_python.has_value() ? per_tensor_python.value() : false; + + auto float_options = tensor_lists[0][0].options().dtype(at::kFloat); + auto output = at::zeros({320}, float_options); + + at::Tensor output_per_tensor; + at::Tensor ret_per_tensor; + + int ntensors = tensor_lists[0].size(); + int max_chunks_per_tensor = -1; + + if(per_tensor) + { + for(int t = 0; t < ntensors; t++) + { + int max_chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size; + if(max_chunks_this_tensor > max_chunks_per_tensor) + max_chunks_per_tensor = max_chunks_this_tensor; + } + output_per_tensor = at::zeros({ntensors*max_chunks_per_tensor}, float_options); + ret_per_tensor = at::empty({ntensors}, float_options); + } + else + { + ret_per_tensor = at::empty({0}, float_options); + } + + DISPATCH_FLOAT_HALF_AND_BFLOAT(tensor_lists[0][0].scalar_type(), 0, "multi_tensor_l2norm_cuda", + multi_tensor_apply<1>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + L2NormFunctor(), + output.DATA_PTR(), + per_tensor ? output_per_tensor.DATA_PTR() : nullptr, + per_tensor, + max_chunks_per_tensor);) + + AT_CUDA_CHECK(cudaGetLastError()); + // AT_CUDA_CHECK(cudaDeviceSynchronize()); + + // This involves one more small kernel launches, but will be negligible end to end. + // I could get rid of these by hacking the functor + multi tensor harness with persistence + // logic, but keeping it simple for now + auto ret = at::empty({1}, output.options()); + const at::cuda::OptionalCUDAGuard device_guard(device_of(output)); + auto stream = at::cuda::getCurrentCUDAStream(); + cleanup<<>>( + output.DATA_PTR(), + per_tensor ? output_per_tensor.DATA_PTR() : nullptr, + ret.DATA_PTR(), + per_tensor ? ret_per_tensor.DATA_PTR() : nullptr, + per_tensor, + max_chunks_per_tensor); + + return std::tuple(ret, ret_per_tensor); +} + + +// Compute and update grad norm +// Here use a per tensor norm, and blend new norm(n) and old norm(gn) by +// L-2: gn = sqrt(a * gn^2 + b * n^2) +// L-inf: gn = a * gn + b * n +void multi_tensor_norm_out_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor out, + const float alpha, + const float beta, + const int norm_type) +{ + auto float_options = tensor_lists[0][0].options().dtype(at::kFloat); + TORCH_CHECK(tensor_lists[0][0].device() == noop_flag.device(), "noop flag should be on the same device as tensors"); + // we don't need global thus uses empty here + auto output = at::empty({320}, float_options); + + at::Tensor output_per_tensor; + at::Tensor ret_per_tensor; + + int ntensors = tensor_lists[0].size(); + int max_chunks_per_tensor = -1; + + for(int t = 0; t < ntensors; t++) + { + int max_chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size; + if(max_chunks_this_tensor > max_chunks_per_tensor) + max_chunks_per_tensor = max_chunks_this_tensor; + } + + // Although it is single write then read, still need to be zero + // Since tailing element also participate cleanup + output_per_tensor = at::zeros({ntensors*max_chunks_per_tensor}, float_options); + + if (norm_type == 0) { + DISPATCH_FLOAT_AND_HALF( + tensor_lists[0][0].scalar_type(), 0, "multi_tensor_maxnorm_cuda", + multi_tensor_apply<1>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + MaxNormFunctor(), + output.DATA_PTR(), + output_per_tensor.DATA_PTR(), + true, + max_chunks_per_tensor);) + } + else { + DISPATCH_FLOAT_AND_HALF( + tensor_lists[0][0].scalar_type(), 0, "multi_tensor_l2norm_cuda", + multi_tensor_apply<1>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + L2NormFunctor(), + output.DATA_PTR(), + output_per_tensor.DATA_PTR(), + true, + max_chunks_per_tensor);) + } + AT_CUDA_CHECK(cudaGetLastError()); + + // AT_CUDA_CHECK(cudaDeviceSynchronize()); + + // This involves one more small kernel launches, but will be negligible end to end. + // I could get rid of these by hacking the functor + multi tensor harness with persistence + // logic, but keeping it simple for now + auto ret = at::empty({1}, output.options()); + + // Adding the following device guard since it happens sometimes that the + // tensors are on one device and the cuda stream is on another device which + // results in ILLEGAL MEM ACCESS error. + const at::cuda::OptionalCUDAGuard device_guard(device_of(output)); + auto stream = at::cuda::getCurrentCUDAStream(); + cleanup_v2<<>>( + output.DATA_PTR(), + output_per_tensor.DATA_PTR(), + ret.DATA_PTR(), + out.DATA_PTR(), + true, + max_chunks_per_tensor, + norm_type, + alpha, + beta); + + return ; +} diff --git a/apex/csrc/multi_tensor_l2norm_kernel_mp.cu b/apex/csrc/multi_tensor_l2norm_kernel_mp.cu new file mode 100644 index 00000000..987f76f5 --- /dev/null +++ b/apex/csrc/multi_tensor_l2norm_kernel_mp.cu @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +template +struct L2NormFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<1>& tl, + float* output, + float* output_per_tensor, + bool per_tensor, + int max_chunks_per_tensor) + { + if (*noop_gmem) { + return; + } + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + x_t* x = (x_t*)tl.addresses[0][tensor_loc]; + x += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + __shared__ float s_vals[512]; + + float vals[ILP]; // = {0}; // this probably works too but I want to be sure... + x_t r_x[ILP]; + for(int i = 0; i < ILP; i++) + { + vals[i] = 0.f; + r_x[i] = 0; + } + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(x)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_x, x, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + float next = static_cast(r_x[ii]); + vals[ii] += next*next; + } + } + } + else + { + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + float next = static_cast(x[i]); + vals[ii] += next*next; + } + } + } + } + + float val = 0.f; + for(int i = 0; i < ILP; i++) + val += vals[i]; + + float final = reduce_block_into_lanes(s_vals, val); + + if(threadIdx.x == 0) + { + if(!isfinite(final)) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + output[blockIdx.x] += final; + if(per_tensor) + output_per_tensor[(tl.start_tensor_this_launch + tensor_loc)*max_chunks_per_tensor + chunk_idx] = final; + } + } +}; + +__global__ void cleanup( + float* output, + float* output_per_tensor, + float* ret, + float* ret_per_tensor, + bool per_tensor, + int max_chunks_per_tensor, + volatile int* noop_gmem) +{ + if (*noop_gmem) { + return; + } + __shared__ float vals[512]; + + if(blockIdx.x == 0) + { + float val = 0; + if(threadIdx.x < 320) + val = output[threadIdx.x]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + *ret = sqrt(final); + } + + if(per_tensor) + { + float* output_this_tensor = output_per_tensor + blockIdx.x*max_chunks_per_tensor; + + float val = 0; + for(int i = threadIdx.x; i < max_chunks_per_tensor; i += blockDim.x) + val += output_this_tensor[i]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + ret_per_tensor[blockIdx.x] = sqrt(final); + } +} + +std::tuple multi_tensor_l2norm_mp_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python) +{ + bool per_tensor = per_tensor_python.has_value() ? per_tensor_python.value() : false; + + auto float_options = tensor_lists[0][0].options().dtype(at::kFloat); + auto output = at::zeros({320}, float_options); + + at::Tensor output_per_tensor; + at::Tensor ret_per_tensor; + + int ntensors = tensor_lists[0].size(); + int max_chunks_per_tensor = -1; + + if(per_tensor) + { + for(int t = 0; t < ntensors; t++) + { + int max_chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size; + if(max_chunks_this_tensor > max_chunks_per_tensor) + max_chunks_per_tensor = max_chunks_this_tensor; + } + output_per_tensor = at::zeros({ntensors*max_chunks_per_tensor}, float_options); + ret_per_tensor = at::empty({ntensors}, float_options); + } + else + { + ret_per_tensor = at::empty({0}, float_options); + } + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "multi_tensor_l2norm_mp_cuda", + multi_tensor_apply<1>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + L2NormFunctor(), + output.data_ptr(), + per_tensor ? output_per_tensor.data_ptr() : nullptr, + per_tensor, + max_chunks_per_tensor);) + + AT_CUDA_CHECK(cudaGetLastError()); + // AT_CUDA_CHECK(cudaDeviceSynchronize()); + + // This involves one more small kernel launches, but will be negligible end to end. + // I could get rid of these by hacking the functor + multi tensor harness with persistence + // logic, but keeping it simple for now + auto ret = at::empty({1}, output.options()); + const at::cuda::OptionalCUDAGuard device_guard(device_of(output)); + auto stream = at::cuda::getCurrentCUDAStream(); + cleanup<<>>( + output.data_ptr(), + per_tensor ? output_per_tensor.data_ptr() : nullptr, + ret.data_ptr(), + per_tensor ? ret_per_tensor.data_ptr() : nullptr, + per_tensor, + max_chunks_per_tensor, noop_flag.data_ptr()); + + return std::tuple(ret, ret_per_tensor); +} diff --git a/apex/csrc/multi_tensor_l2norm_scale_kernel.cu b/apex/csrc/multi_tensor_l2norm_scale_kernel.cu new file mode 100644 index 00000000..f60e9609 --- /dev/null +++ b/apex/csrc/multi_tensor_l2norm_scale_kernel.cu @@ -0,0 +1,326 @@ +#include +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +template +struct L2NormScaleFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<2>& tl, + float* output, + float* output_per_tensor, + float scale, + bool per_tensor, + int max_chunks_per_tensor) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + in_t* in = (in_t*)tl.addresses[0][tensor_loc]; + in += chunk_idx*chunk_size; + + out_t* out = (out_t*)tl.addresses[1][tensor_loc]; + out += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + __shared__ float s_vals[512]; + + float vals[ILP]; // = {0}; // this probably works too but I want to be sure... + in_t r_in[ILP]; + for(int i = 0; i < ILP; i++) + { + vals[i] = 0.f; + r_in[i] = 0; + } + //bool finite = true; + out_t r_out[ILP]; + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(in) && is_aligned(out)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_in, in, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + float next = static_cast(r_in[ii]); + r_out[ii] = next*scale; + vals[ii] += next*next; + //finite = finite && isfinite(r_in[ii]); + } + load_store(out, r_out, i_start, 0); + } + } + else + { + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_in[ii] = 0; + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_in[ii] = in[i]; + float next = static_cast(in[i]); + vals[ii] += next*next; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_out[ii] = static_cast(r_in[ii]) * scale; + // finite = finite && isfinite(r_in[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + out[i] = r_out[ii]; + } + } + } + + float val = 0.f; + for(int i = 0; i < ILP; i++) + val += vals[i]; + + float final = reduce_block_into_lanes(s_vals, val); + + if(threadIdx.x == 0) + { + if(!isfinite(final)) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + output[blockIdx.x] += final; + if(per_tensor) + output_per_tensor[(tl.start_tensor_this_launch + tensor_loc)*max_chunks_per_tensor + chunk_idx] = final; + } + } +}; +// Probably better to template, but since we are not likely to support other norm +template +struct MaxNormFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<1>& tl, + float* output, + float* output_per_tensor, + bool per_tensor, + int max_chunks_per_tensor) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + x_t* x = (x_t*)tl.addresses[0][tensor_loc]; + x += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + __shared__ float s_vals[512]; + + float vals[ILP]; // = {0}; // this probably works too but I want to be sure... + x_t r_x[ILP]; + for(int i = 0; i < ILP; i++) + { + vals[i] = 0.f; + r_x[i] = 0; + } + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(x)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_x, x, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + float next = static_cast(r_x[ii]); + vals[ii] = fmaxf(fabsf(vals[ii]), fabsf(next)); + } + } + } + else + { + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + float next = static_cast(x[i]); + vals[ii] = fmaxf(fabsf(vals[ii]), fabsf(next)); + } + } + } + } + + float val = 0.f; + for(int i = 0; i < ILP; i++) + val = fmaxf(fabsf(val), fabsf(vals[i])); + + float final = reduce_block_into_lanes_max_op(s_vals, val); + + if(threadIdx.x == 0) + { + if(!isfinite(final)) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + output[blockIdx.x] = fmaxf(fabsf(output[blockIdx.x]), fabsf(final)); + if(per_tensor) + output_per_tensor[(tl.start_tensor_this_launch + tensor_loc)*max_chunks_per_tensor + chunk_idx] = final; + } + } +}; + +__global__ void cleanup_v3( + float* output, + float* output_per_tensor, + float* ret, + float* ret_per_tensor, + bool per_tensor, + int max_chunks_per_tensor) +{ + __shared__ float vals[512]; + + if(blockIdx.x == 0) + { + float val = 0; + if(threadIdx.x < 320) + val = output[threadIdx.x]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + *ret = sqrt(final); + } + + if(per_tensor) + { + float* output_this_tensor = output_per_tensor + blockIdx.x*max_chunks_per_tensor; + + float val = 0; + for(int i = threadIdx.x; i < max_chunks_per_tensor; i += blockDim.x) + val += output_this_tensor[i]; + + float final = reduce_block_into_lanes(vals, val); + + if(threadIdx.x == 0) + ret_per_tensor[blockIdx.x] = sqrt(final); + } +} + + +std::tuple multi_tensor_l2norm_scale_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float scale, + at::optional per_tensor_python) +{ + bool per_tensor = per_tensor_python.has_value() ? per_tensor_python.value() : false; + + auto float_options = tensor_lists[0][0].options().dtype(at::kFloat); + auto output = at::zeros({320}, float_options); + + at::Tensor output_per_tensor; + at::Tensor ret_per_tensor; + + int ntensors = tensor_lists[0].size(); + int max_chunks_per_tensor = -1; + + if(per_tensor) + { + for(int t = 0; t < ntensors; t++) + { + int max_chunks_this_tensor = (tensor_lists[0][t].numel() + chunk_size - 1)/chunk_size; + if(max_chunks_this_tensor > max_chunks_per_tensor) + max_chunks_per_tensor = max_chunks_this_tensor; + } + output_per_tensor = at::zeros({ntensors*max_chunks_per_tensor}, float_options); + ret_per_tensor = at::empty({ntensors}, float_options); + } + else + { + ret_per_tensor = at::empty({0}, float_options); + } + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "multi_tensor_l2norm_scale_cuda", + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, "multi_tensor_l2norm_scale_cuda", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + L2NormScaleFunctor(), + output.DATA_PTR(), + per_tensor ? output_per_tensor.DATA_PTR() : nullptr, + scale, + per_tensor, + max_chunks_per_tensor);)) + + AT_CUDA_CHECK(cudaGetLastError()); + // AT_CUDA_CHECK(cudaDeviceSynchronize()); + + // This involves one more small kernel launches, but will be negligible end to end. + // I could get rid of these by hacking the functor + multi tensor harness with persistence + // logic, but keeping it simple for now + auto ret = at::empty({1}, output.options()); + const at::cuda::OptionalCUDAGuard device_guard(device_of(output)); + auto stream = at::cuda::getCurrentCUDAStream(); + cleanup_v3<<>>( + output.DATA_PTR(), + per_tensor ? output_per_tensor.DATA_PTR() : nullptr, + ret.DATA_PTR(), + per_tensor ? ret_per_tensor.DATA_PTR() : nullptr, + per_tensor, + max_chunks_per_tensor); + + return std::tuple(ret, ret_per_tensor); +} + + diff --git a/apex/csrc/multi_tensor_lamb.cu b/apex/csrc/multi_tensor_lamb.cu new file mode 100644 index 00000000..3137fcd2 --- /dev/null +++ b/apex/csrc/multi_tensor_lamb.cu @@ -0,0 +1,413 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +typedef enum{ + MOMENT_MODE_0 =0, // L2 regularization mode + MOMENT_MODE_1 =1 // Decoupled weight decay mode +} adamMode_t; + +std::tuple multi_tensor_l2norm_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python); + +using MATH_T = float; + +template +struct LAMBStage1Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<4>& tl, + const float beta1, + const float beta2, + const float beta3, + const float beta1_correction, + const float beta2_correction, + const float epsilon, + adamMode_t mode, + const float decay, + const float* global_grad_norm, + const float max_global_grad_norm) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + float clipped_global_grad_norm = (*global_grad_norm) > max_global_grad_norm ? (*global_grad_norm) / max_global_grad_norm : 1.0f; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + T* m = (T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + T* v = (T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && + chunk_size % ILP == 0 && + is_aligned(g) && + is_aligned(p) && + is_aligned(m) && + is_aligned(v)) + { + T l_g[ILP]; + T l_p[ILP]; + T l_m[ILP]; + T l_v[ILP]; + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(l_g, g, 0, i_start); + if (decay != 0) + load_store(l_p, p, 0, i_start); + load_store(l_m, m, 0, i_start); + load_store(l_v, v, 0, i_start); + // unpack +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_g[ii] = l_g[ii]; + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = l_p[ii]; + } + r_m[ii] = l_m[ii]; + r_v[ii] = l_v[ii]; + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + l_p[ii] = r_p[ii]; + l_m[ii] = r_m[ii]; + l_v[ii] = r_v[ii]; + } + // store + load_store(g, l_p, i_start, 0); + load_store(m, l_m, i_start, 0); + load_store(v, l_v, i_start, 0); + } + } + else + { + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i]; + // special ?optimization? for lamb stage 1 + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = p[i]; + } + r_m[ii] = m[i]; + r_v[ii] = v[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + g[i] = r_p[ii]; + m[i] = r_m[ii]; + v[i] = r_v[ii]; + } + } + } + } + } +}; + +// Step 2 reads in 'update' value and per-tensor param_norm and update_norm. +// It computes new parameter value. +template +struct LAMBStage2Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<2>& tl, + const float* per_tensor_param_norm, + const float* per_tensor_update_norm, + const float learning_rate, + const float decay, + bool use_nvlamb) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + MATH_T ratio = learning_rate; + // nvlamb: apply adaptive learning rate to all parameters + // otherwise, only apply to those with non-zero weight decay + if (use_nvlamb || (decay != 0.0)) + { + float param_norm = per_tensor_param_norm[tensor_num]; + float update_norm = per_tensor_update_norm[tensor_num]; + ratio = (update_norm != 0.0f && param_norm != 0.0f) ? learning_rate * (param_norm / update_norm) : learning_rate; + } + + T* update = (T*)tl.addresses[0][tensor_loc]; + update += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && + chunk_size % ILP == 0 && + is_aligned(p) && + is_aligned(update)) + { + T r_p[ILP]; + T r_update[ILP]; + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_p, p, 0, i_start); + load_store(r_update, update, 0, i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = static_cast(r_p[ii]) - (ratio * static_cast(r_update[ii])); + } + load_store(p, r_p, i_start, 0); + } + } + else + { + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_p[ILP]; + MATH_T r_update[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_p[ii] = p[i]; + r_update[ii] = update[i]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = r_p[ii] - (ratio * r_update[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + } + } + } + } + } +}; + + +void multi_tensor_lamb_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + at::Tensor global_grad_norm, + const float max_grad_norm, + at::optional use_nvlamb_python) +{ + using namespace at; + // Master weight and 32bit momentum(potentially changing) is not handled by this + // So we assume every tensor are all in the same type + + bool use_nvlamb = use_nvlamb_python.has_value() ? use_nvlamb_python.value() : false; + + // Handle bias correction mode + float bias_correction1 = 1.0f, bias_correction2 = 1.0f; + if (bias_correction == 1) { + bias_correction1 = 1 - std::pow(beta1, step); + bias_correction2 = 1 - std::pow(beta2, step); + } + + // Handle grad averaging mode + float beta3 = 1.0f; + if (grad_averaging == 1) beta3 = 1 - beta1; + + std::vector> grad_list(tensor_lists.begin(), tensor_lists.begin()+1); + std::vector> param_list(tensor_lists.begin()+1, tensor_lists.begin()+2); + + // Compute per tensor param norm + auto param_norm_tuple = multi_tensor_l2norm_cuda(chunk_size, noop_flag, param_list, true); + + // We now in-place modify grad to store update before compute its norm + // Generally this is not a issue since people modify grad in step() method all the time + // We can also grab list of empty tensor to avoid this, but I'd like to save space/cpu code + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_1", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + LAMBStage1Functor(), + beta1, + beta2, + beta3, // 1-beta1 or 1 depends on averaging mode + bias_correction1, + bias_correction2, + epsilon, + (adamMode_t) mode, + weight_decay, + global_grad_norm.DATA_PTR(), + max_grad_norm); ) + + // Compute update norms + auto update_norm_tuple = multi_tensor_l2norm_cuda(chunk_size, noop_flag, grad_list, true); + + std::vector> grad_param_list(tensor_lists.begin(), tensor_lists.begin()+2); + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_2", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + noop_flag, + grad_param_list, + LAMBStage2Functor(), + std::get<1>(param_norm_tuple).DATA_PTR(), + std::get<1>(update_norm_tuple).DATA_PTR(), + lr, + weight_decay, + use_nvlamb); ) + + AT_CUDA_CHECK(cudaGetLastError()); + +} diff --git a/apex/csrc/multi_tensor_lamb_mp.cu b/apex/csrc/multi_tensor_lamb_mp.cu new file mode 100644 index 00000000..b52ebd9c --- /dev/null +++ b/apex/csrc/multi_tensor_lamb_mp.cu @@ -0,0 +1,496 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +typedef enum{ + MOMENT_MODE_0 =0, // L2 regularization mode + MOMENT_MODE_1 =1 // Decoupled weight decay mode +} adamMode_t; + +std::tuple multi_tensor_l2norm_mp_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::optional per_tensor_python); + +using MATH_T = float; + +template +struct LAMBStage1Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<4>& tl, + const float beta1, + const float beta2, + const float beta3, + const int* step_ptr, + const int bias_correction, + const float epsilon, + adamMode_t mode, + const float decay, + const float* global_grad_norm, + const float* max_global_grad_norm, + const float* found_inf, + const float* inv_scale) + { + if (*noop_gmem) { + return; + } + + float beta1_correction = 1.0f; + float beta2_correction = 1.0f; + if (bias_correction == 1) { + int step = *step_ptr; + beta1_correction = 1 - std::pow(beta1, step); + beta2_correction = 1 - std::pow(beta2, step); + } + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + float clipped_global_grad_norm = (*global_grad_norm) > (*max_global_grad_norm) ? (*global_grad_norm) / (*max_global_grad_norm) : 1.0f; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + param_t* p = (param_t*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + param_t* m = (param_t*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + param_t* v = (param_t*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && + chunk_size % ILP == 0 && + is_aligned(g) && + is_aligned(p) && + is_aligned(m) && + is_aligned(v)) + { + T l_g[ILP]; + param_t l_p[ILP]; + param_t l_m[ILP]; + param_t l_v[ILP]; + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(l_g, g, 0, i_start); + if (decay != 0) + load_store(l_p, p, 0, i_start); + load_store(l_m, m, 0, i_start); + load_store(l_v, v, 0, i_start); + // unpack +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_g[ii] = l_g[ii] * (*inv_scale); + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = l_p[ii]; + } + r_m[ii] = l_m[ii]; + r_v[ii] = l_v[ii]; + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + l_p[ii] = r_p[ii]; + // Difference from APEX's LAMB kernel. `g` and `p` can be different dtypes. + l_g[ii] = r_p[ii]; + l_m[ii] = r_m[ii]; + l_v[ii] = r_v[ii]; + } + // store + load_store(g, l_g, i_start, 0); + load_store(m, l_m, i_start, 0); + load_store(v, l_v, i_start, 0); + } + } + else + { + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; + MATH_T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i] * (*inv_scale); + // special ?optimization? for lamb stage 1 + if (decay == 0) { + r_p[ii] = MATH_T(0); + } + else { + r_p[ii] = p[i]; + } + r_m[ii] = m[i]; + r_v[ii] = v[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + r_v[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (mode == MOMENT_MODE_0) { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + // L2 on scaled grad + scaled_grad = scaled_grad + decay*r_p[ii]; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = next_m_unbiased / denom; + } + else { + MATH_T scaled_grad = r_g[ii] / clipped_global_grad_norm; + r_m[ii] = r_m[ii] * beta1 + beta3 * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = r_v[ii] / beta2_correction; + MATH_T denom = sqrtf(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + g[i] = r_p[ii]; + m[i] = r_m[ii]; + v[i] = r_v[ii]; + } + } + } + } + } +}; + +// Step 2 reads in 'update' value and per-tensor param_norm and update_norm. +// It computes new parameter value. +// N == 2: FP32 params, no master params +// N == 3: FP16 params, FP32 master params. +template +struct LAMBStage2Functor +{ + static_assert((N == 2 && std::is_same::value) || (N == 3 && std::is_same::value), ""); + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata& tl, + const float* per_tensor_param_norm, + const float* per_tensor_update_norm, + const float* learning_rate, + const float decay, + bool use_nvlamb) + { + if (*noop_gmem) { + return; + } + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + MATH_T ratio = *learning_rate; + // nvlamb: apply adaptive learning rate to all parameters + // otherwise, only apply to those with non-zero weight decay + if (use_nvlamb || (decay != 0.0)) + { + float param_norm = per_tensor_param_norm[tensor_num]; + float update_norm = per_tensor_update_norm[tensor_num]; + ratio = (update_norm != 0.0f && param_norm != 0.0f) ? *learning_rate * (param_norm / update_norm) : *learning_rate; + } + + T* update = (T*)tl.addresses[0][tensor_loc]; + update += chunk_idx*chunk_size; + + param_t* p = (param_t*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + T* out_p; + if (N == 3) { + out_p = (T*)tl.addresses[2][tensor_loc]; + out_p += chunk_idx*chunk_size; + } + + n -= chunk_idx*chunk_size; + + // to make things simple, we put aligned case in a different code path + bool can_use_aligned_path = n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(p) && is_aligned(update); + if (N == 3) { + can_use_aligned_path = can_use_aligned_path && is_aligned(out_p); + } + if(can_use_aligned_path) + { + param_t r_p[ILP]; + T r_update[ILP]; + T r_out_p[ILP]; + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_p, p, 0, i_start); + load_store(r_update, update, 0, i_start); + if (N == 3) { + load_store(r_out_p, out_p, 0, i_start); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = static_cast(r_p[ii]) - (ratio * static_cast(r_update[ii])); + if (N == 3) { + r_out_p[ii] = r_p[ii]; + } + } + load_store(p, r_p, i_start, 0); + if (N == 3) { + load_store(out_p, r_out_p, i_start, 0); + } + } + } + else + { + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_p[ILP]; + MATH_T r_update[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_p[ii] = p[i]; + r_update[ii] = update[i]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = r_p[ii] - (ratio * r_update[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + if (N == 3) { + out_p[i] = r_p[ii]; + } + } + } + } + } + } +}; + + +void multi_tensor_lamb_mp_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor lr, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int mode, + at::Tensor global_grad_norm, + at::Tensor max_grad_norm, + at::optional use_nvlamb_python, + at::Tensor found_inf, + at::Tensor inv_scale) +{ + // n_tensors == 5: FP16 model params & FP32 master params + // n_tensors == 4: FP32 model params & NO FP32 master params + const auto n_tensors = tensor_lists.size(); + assert(n_tensors == 4 || n_tensors == 5); + using namespace at; + + bool use_nvlamb = use_nvlamb_python.has_value() ? use_nvlamb_python.value() : false; + + // note(mkozuki): move bias handling below to functor + // Handle bias correction mode + // float bias_correction1 = 1.0f, bias_correction2 = 1.0f; + // if (bias_correction == 1) { + // bias_correction1 = 1 - std::pow(beta1, step); + // bias_correction2 = 1 - std::pow(beta2, step); + // } + + // Handle grad averaging mode + float beta3 = 1.0f; + if (grad_averaging == 1) beta3 = 1 - beta1; + + std::vector> stage1_tensor_lists(tensor_lists.begin(), tensor_lists.begin() + 4); + std::vector> grad_list(tensor_lists.begin(), tensor_lists.begin()+1); + std::vector> param_list(tensor_lists.begin()+1, tensor_lists.begin()+2); + + // Compute per tensor param norm + auto param_norm_tuple = multi_tensor_l2norm_mp_cuda(chunk_size, noop_flag, param_list, true); + + // We now in-place modify grad to store update before compute its norm + // Generally this is not a issue since people modify grad in step() method all the time + // We can also grab list of empty tensor to avoid this, but I'd like to save space/cpu code + if (n_tensors == 4) { + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_1", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + stage1_tensor_lists, + LAMBStage1Functor(), + beta1, + beta2, + beta3, // 1-beta1 or 1 depends on averaging mode + // bias_correction1, + // bias_correction2, + step.data_ptr(), + bias_correction, + epsilon, + (adamMode_t) mode, + weight_decay, + global_grad_norm.data_ptr(), + max_grad_norm.data_ptr(), + found_inf.data_ptr(), + inv_scale.data_ptr()); ) + } else { + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_1", + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + stage1_tensor_lists, + LAMBStage1Functor(), + beta1, + beta2, + beta3, // 1-beta1 or 1 depends on averaging mode + // bias_correction1, + // bias_correction2, + step.data_ptr(), + bias_correction, + epsilon, + (adamMode_t) mode, + weight_decay, + global_grad_norm.data_ptr(), + max_grad_norm.data_ptr(), + found_inf.data_ptr(), + inv_scale.data_ptr()); ) + } + + // Compute update norms + auto update_norm_tuple = multi_tensor_l2norm_mp_cuda(chunk_size, noop_flag, grad_list, true); + + std::vector> grad_param_list(tensor_lists.begin(), tensor_lists.begin()+2); + if (n_tensors == 4) { + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_2", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + noop_flag, + grad_param_list, + LAMBStage2Functor(), + std::get<1>(param_norm_tuple).data_ptr(), + std::get<1>(update_norm_tuple).data_ptr(), + lr.data_ptr(), + weight_decay, + use_nvlamb); ) + } else { + grad_param_list.push_back(tensor_lists[4]); + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_2", + multi_tensor_apply<3>( + BLOCK_SIZE, + chunk_size, + noop_flag, + grad_param_list, + LAMBStage2Functor(), + std::get<1>(param_norm_tuple).data_ptr(), + std::get<1>(update_norm_tuple).data_ptr(), + lr.data_ptr(), + weight_decay, + use_nvlamb); ) + } + AT_CUDA_CHECK(cudaGetLastError()); + +} diff --git a/apex/csrc/multi_tensor_lamb_stage_1.cu b/apex/csrc/multi_tensor_lamb_stage_1.cu new file mode 100644 index 00000000..6ad7649b --- /dev/null +++ b/apex/csrc/multi_tensor_lamb_stage_1.cu @@ -0,0 +1,151 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +// Step 1 computes the 'update' value of regular Adam optimizer. +template +struct LAMBStage1Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<5>& tl, + const float* per_tensor_decay, + const float beta1, + const float beta2, + const float beta1_correction, + const float beta2_correction, + const float epsilon, + const float clipped_global_grad_norm) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + float decay = per_tensor_decay[tensor_num]; + + GRAD_T* g = (GRAD_T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + T* m = (T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + T* v = (T*)tl.addresses[3][tensor_loc]; + v += chunk_idx*chunk_size; + + UPD_T* update = (UPD_T*)tl.addresses[4][tensor_loc]; + update += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + GRAD_T r_g[ILP]; + T r_p[ILP]; + T r_m[ILP]; + T r_v[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i]; + r_p[ii] = p[i]; + r_m[ii] = m[i]; + r_v[ii] = v[i]; + } else { + r_g[ii] = GRAD_T(0); + r_p[ii] = T(0); + r_m[ii] = T(0); + r_v[ii] = T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + T scaled_grad = r_g[ii] / clipped_global_grad_norm; + r_m[ii] = r_m[ii] * beta1 + (1-beta1) * scaled_grad; + r_v[ii] = r_v[ii] * beta2 + (1-beta2) * scaled_grad * scaled_grad; + T next_m_unbiased = r_m[ii] / beta1_correction; + T next_v_unbiased = r_v[ii] / beta2_correction; + T denom = std::sqrt(next_v_unbiased) + epsilon; + r_p[ii] = (next_m_unbiased/denom) + (decay*r_p[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + update[i] = (UPD_T)r_p[ii]; + m[i] = r_m[ii]; + v[i] = r_v[ii]; + } + } + } + } +}; + +void multi_tensor_lamb_stage1_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_decay, + const int step, + const float beta1, + const float beta2, + const float epsilon, + at::Tensor global_grad_norm, + const float max_global_grad_norm) +{ + using namespace at; + + const float* g_grad_norm = global_grad_norm.DATA_PTR(); + float clipped_global_grad_norm = *(g_grad_norm) > max_global_grad_norm ? *(g_grad_norm) / max_global_grad_norm : 1.0f; + float next_step = float(step+1); + float beta1_correction = 1.0f - std::pow(beta1, next_step); + float beta2_correction = 1.0f - std::pow(beta2, next_step); + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_1", + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, "lamb_stage_1", + DISPATCH_FLOAT_AND_HALF(tensor_lists[4][0].scalar_type(), 2, "lamb_stage_1", + multi_tensor_apply<5>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + LAMBStage1Functor(), + per_tensor_decay.DATA_PTR(), + beta1, + beta2, + beta1_correction, + beta2_correction, + epsilon, + clipped_global_grad_norm); ))) + + AT_CUDA_CHECK(cudaGetLastError()); + + // AT_CUDA_CHECK(cudaDeviceSynchronize()); +} diff --git a/apex/csrc/multi_tensor_lamb_stage_2.cu b/apex/csrc/multi_tensor_lamb_stage_2.cu new file mode 100644 index 00000000..90970666 --- /dev/null +++ b/apex/csrc/multi_tensor_lamb_stage_2.cu @@ -0,0 +1,125 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +using MATH_T = float; + +// Step 2 reads in 'update' value and per-tensor param_norm and update_norm. +// It computes new parameter value. +template +struct LAMBStage2Functor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<2>& tl, + const float* per_tensor_param_norm, + const float* per_tensor_update_norm, + const float learning_rate, + const float decay, + bool use_nvlamb) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + MATH_T ratio = learning_rate; + // nvlamb: apply adaptive learning rate to all parameters + // otherwise, only apply to those with non-zero weight decay + if (use_nvlamb || (decay != 0.0)) + { + float param_norm = per_tensor_param_norm[tensor_num]; + float update_norm = per_tensor_update_norm[tensor_num]; + ratio = (update_norm != 0.0f && param_norm != 0.0f) ? learning_rate * (param_norm / update_norm) : learning_rate; + } + + T* p = (T*)tl.addresses[0][tensor_loc]; + p += chunk_idx*chunk_size; + + UPD_T* update = (UPD_T*)tl.addresses[1][tensor_loc]; + update += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + T r_p[ILP]; + UPD_T r_update[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_p[ii] = p[i]; + r_update[ii] = update[i]; + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_p[ii] = r_p[ii] - (ratio*(T)r_update[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + } + } + } + } +}; + +void multi_tensor_lamb_stage2_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor per_tensor_param_norm, + at::Tensor per_tensor_update_norm, + const float lr, + const float weight_decay, + at::optional use_nvlamb_python) +{ + bool use_nvlamb = use_nvlamb_python.has_value() ? use_nvlamb_python.value() : false; + + using namespace at; + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "lamb_stage_2", + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, "lamb_stage_2", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + LAMBStage2Functor(), + per_tensor_param_norm.DATA_PTR(), + per_tensor_update_norm.DATA_PTR(), + lr, + weight_decay, + use_nvlamb); )) + + AT_CUDA_CHECK(cudaGetLastError()); + + // AT_CUDA_CHECK(cudaDeviceSynchronize()); +} diff --git a/apex/csrc/multi_tensor_novograd.cu b/apex/csrc/multi_tensor_novograd.cu new file mode 100644 index 00000000..2decc06b --- /dev/null +++ b/apex/csrc/multi_tensor_novograd.cu @@ -0,0 +1,188 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +typedef enum{ + MOMENT_MODE_0 =0, // Novograd paper mode, momentum caculation with denom then decay inside + MOMENT_MODE_1 =1 // Decoupled weight decay mode +} momentMode_t; + +void multi_tensor_norm_out_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor out, + const float alpha, + const float beta, + const int norm_type); + +using MATH_T = float; + +template +struct NovoGradFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<3>& tl, + const float beta1, + const float beta2, + const float beta3, + const float beta1_correction, + const float beta2_correction, + const float epsilon, + const float lr, + momentMode_t m_mode, + const float decay, + const float* per_tensor_grad_norm) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int tensor_num = tl.start_tensor_this_launch + tensor_loc; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + float grad_norm = per_tensor_grad_norm[tensor_num]; + + T* g = (T*)tl.addresses[0][tensor_loc]; + g += chunk_idx*chunk_size; + + T* p = (T*)tl.addresses[1][tensor_loc]; + p += chunk_idx*chunk_size; + + T* m = (T*)tl.addresses[2][tensor_loc]; + m += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + // see note in multi_tensor_scale_kernel.cu + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + MATH_T r_g[ILP]; + MATH_T r_p[ILP]; + MATH_T r_m[ILP]; +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + r_g[ii] = g[i]; + r_p[ii] = p[i]; + r_m[ii] = m[i]; + } else { + r_g[ii] = MATH_T(0); + r_p[ii] = MATH_T(0); + r_m[ii] = MATH_T(0); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + if (m_mode == MOMENT_MODE_0) { + MATH_T next_v_unbiased = grad_norm / beta2_correction; + MATH_T denom = next_v_unbiased + epsilon; + r_g[ii] = (r_g[ii] / denom) + (decay * r_p[ii]); + r_m[ii] = beta1 * r_m[ii] + beta3 * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + r_p[ii] = r_p[ii] - (lr * next_m_unbiased); + } + else { + r_m[ii] = beta1 * r_m[ii] + beta3 * r_g[ii]; + MATH_T next_m_unbiased = r_m[ii] / beta1_correction; + MATH_T next_v_unbiased = grad_norm / beta2_correction; + MATH_T denom = next_v_unbiased + epsilon; + MATH_T update = (next_m_unbiased / denom) + (decay * r_p[ii]); + r_p[ii] = r_p[ii] - (lr * update); + } + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + p[i] = r_p[ii]; + m[i] = r_m[ii]; + } + } + } + } +}; + +void multi_tensor_novograd_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + at::Tensor grad_norms, + const float lr, + const float beta1, + const float beta2, + const float epsilon, + const int step, + const int bias_correction, + const float weight_decay, + const int grad_averaging, + const int moment_mode, + const int norm_type) +{ + using namespace at; + + // Handle bias correction mode + float bias_correction1 = 1.0f, bias_correction2 = 1.0f; + if (bias_correction == 1) { + bias_correction1 = 1 - std::pow(beta1, step); + bias_correction2 = std::sqrt(1 - std::pow(beta2, step)); + } + + // Handle grad averaging mode + float beta3 = 1; + if (grad_averaging == 1) beta3 = 1 - beta1; + + std::vector> grad_list(tensor_lists.begin(), tensor_lists.begin()+1); + + // Compute and update grad norm + // Here use a per tensor norm, and blend new norm(n) and old norm(gn) by + // L-2: gn = sqrt(a * gn^2 + b * n^2) + // L-inf: gn = a * gn + b * n + multi_tensor_norm_out_cuda(chunk_size, noop_flag, grad_list, grad_norms, beta2, (1.0f - beta2), norm_type); + + // Assume single type across p,g,m1,m2 now + DISPATCH_DOUBLE_FLOAT_AND_HALF( + tensor_lists[0][0].scalar_type(), 0, "novograd", + multi_tensor_apply<3>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + NovoGradFunctor(), + beta1, + beta2, + beta3, // 1-beta1 or 1 depends on averaging mode + bias_correction1, + bias_correction2, + epsilon, + lr, + (momentMode_t) moment_mode, + weight_decay, + grad_norms.DATA_PTR()); ) + + AT_CUDA_CHECK(cudaGetLastError()); + +} diff --git a/apex/csrc/multi_tensor_scale_kernel.cu b/apex/csrc/multi_tensor_scale_kernel.cu new file mode 100644 index 00000000..629ee942 --- /dev/null +++ b/apex/csrc/multi_tensor_scale_kernel.cu @@ -0,0 +1,136 @@ +#include +#include +#include +#include +// Another possibility: +// #include + +#include +// Stringstream is a big hammer, but I want to rely on operator<< for dtype. +#include + +#include "type_shim.h" +#include "multi_tensor_apply.cuh" + +#define BLOCK_SIZE 512 +#define ILP 4 + +template +__device__ __forceinline__ bool is_aligned(T* p){ + return ((uint64_t)p) % (ILP*sizeof(T)) == 0; +} + +template +__device__ __forceinline__ void load_store(T* dst, T* src, int dst_offset, int src_offset){ + typedef typename std::aligned_storage::type LT; + ((LT*)dst)[dst_offset] = ((LT*)src)[src_offset]; +} + +template +struct ScaleFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata<2>& tl, + float scale) + { + // I'd like this kernel to propagate infs/nans. + // if(*noop_gmem == 1) + // return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + in_t* in = (in_t*)tl.addresses[0][tensor_loc]; + in += chunk_idx*chunk_size; + + out_t* out = (out_t*)tl.addresses[1][tensor_loc]; + out += chunk_idx*chunk_size; + + n -= chunk_idx*chunk_size; + + bool finite = true; + in_t r_in[ILP]; + out_t r_out[ILP]; + + // to make things simple, we put aligned case in a different code path + if(n % ILP == 0 && chunk_size % ILP == 0 && is_aligned(in) && is_aligned(out)) + { + for(int i_start = threadIdx.x; i_start*ILP < n && i_start*ILP < chunk_size; i_start += blockDim.x) + { + // load + load_store(r_in, in, 0 , i_start); +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_out[ii] = static_cast(r_in[ii]) * scale; + finite = finite && isfinite(r_in[ii]); + } + // store + load_store(out, r_out, i_start, 0); + } + } + else + { + // Non-divergent exit condition for __syncthreads, not necessary here + for(int i_start = 0; i_start < n && i_start < chunk_size; i_start += blockDim.x*ILP) + { +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_in[ii] = 0; + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + r_in[ii] = in[i]; + } + // note for clarification to future michael: + // From a pure memory dependency perspective, there's likely no point unrolling + // the write loop, since writes just fire off once their LDGs arrive. + // Put another way, the STGs are dependent on the LDGs, but not on each other. + // There is still compute ILP benefit from unrolling the loop though. +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + r_out[ii] = static_cast(r_in[ii]) * scale; + finite = finite && isfinite(r_in[ii]); + } +#pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + out[i] = r_out[ii]; + } + } + } + if(!finite) + *noop_gmem = 1; // Blindly fire off a write. These will race but that's ok. + } +}; + +void multi_tensor_scale_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float scale) +{ + using namespace at; + // The output (downscaled) type is always float. + // If build times suffer, think about where to put this dispatch, + // and what logic should be moved out of multi_tensor_apply. + + DISPATCH_FLOAT_AND_HALF(tensor_lists[0][0].scalar_type(), 0, "multi_tensor_scale_cuda", + DISPATCH_FLOAT_AND_HALF(tensor_lists[1][0].scalar_type(), 1, "multi_tensor_scale_cuda", + multi_tensor_apply<2>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + ScaleFunctor(), + scale); )) + AT_CUDA_CHECK(cudaGetLastError()); + + // AT_CUDA_CHECK(cudaDeviceSynchronize()); +} diff --git a/apex/csrc/multi_tensor_sgd_kernel.cu b/apex/csrc/multi_tensor_sgd_kernel.cu new file mode 100644 index 00000000..42a7406b --- /dev/null +++ b/apex/csrc/multi_tensor_sgd_kernel.cu @@ -0,0 +1,280 @@ +#include +#include +#include +#include +#include "multi_tensor_apply.cuh" +#include "compat.h" + +#include +#include + +#define BLOCK_SIZE 512 +#define ILP 4 + +/** + * Perform fused SGD on multiple buffers + * N: number of tensors + * tl[0] : gradients + * tl[1] : weights + * tl[2] : momentum buffers + * tl[3] : fp16 weights (if appropriate) + * wd : weight_decay (scalar) + * momentum : momentum (scalar) + * dampening : momentum dampening (scalar) + * lr : learning rate (scalar) + * nesterov : enable nesterov (bool) + * first run : necessary for proper momentum handling & init + * wd_after_momentum : apply weight decay _after_ momentum instead of before + **/ +template +struct SGDFunctor +{ + __device__ __forceinline__ void operator()( + int chunk_size, + volatile int* noop_gmem, + TensorListMetadata& tl, + float wd, + float momentum, + float dampening, + float lr, + bool nesterov, + bool first_run, + bool wd_after_momentum, + float scale) + { + // Early exit if we don't need to do anything + if (*noop_gmem) return; + + int tensor_loc = tl.block_to_tensor[blockIdx.x]; + int chunk_idx = tl.block_to_chunk[blockIdx.x]; + int n = tl.sizes[tensor_loc]; + + T_grad* grad_in = (T_grad*)tl.addresses[0][tensor_loc]; + grad_in += chunk_idx*chunk_size; + + T_weight* weight_in = (T_weight*)tl.addresses[1][tensor_loc]; + weight_in += chunk_idx*chunk_size; + + T_weight* mom_in = (T_weight*)tl.addresses[2][tensor_loc]; + mom_in += chunk_idx*chunk_size; + + at::Half *model_weights_out = nullptr; + if(N == 4) + { + model_weights_out = (at::Half*)tl.addresses[3][tensor_loc]; + model_weights_out += chunk_idx*chunk_size; + } + + n -= chunk_idx*chunk_size; + + // Non-divergent exit condition for the __syncthreads + float incoming_grads[ILP]; + float incoming_weights[ILP]; + float incoming_moms[ILP]; + for(int i_start = 0; + i_start < n && i_start < chunk_size; + i_start += blockDim.x*ILP) + { + #pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + incoming_grads[ii] = 0; + incoming_weights[ii] = 0; + incoming_moms[ii] = 0; + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + incoming_grads[ii] = static_cast(grad_in[i])*scale; + incoming_weights[ii] = static_cast(weight_in[i]); + incoming_moms[ii] = static_cast(mom_in[i]); + } + } + + // note for clarification to future michael: + // From a pure memory dependency perspective, there's likely no point unrolling + // the write loop, since writes just fire off once their LDGs arrive. + // Put another way, the STGs are dependent on the LDGs, but not on each other. + // There is still compute ILP benefit from unrolling the loop though. + #pragma unroll + for(int ii = 0; ii < ILP; ii++) + { + int i = i_start + threadIdx.x + ii*blockDim.x; + if(i < n && i < chunk_size) + { + // apply weight decay before momentum if necessary + if(wd != 0.f && !wd_after_momentum) + incoming_grads[ii] += wd * incoming_weights[ii]; + + if(momentum != 0.f) + { + if(!first_run) + incoming_moms[ii] = incoming_moms[ii] * momentum + (1.f - dampening) * incoming_grads[ii]; + else // initialize momentums to current incoming grads + incoming_moms[ii] = incoming_grads[ii]; + + if(nesterov) + incoming_grads[ii] += momentum * incoming_moms[ii]; + else + incoming_grads[ii] = incoming_moms[ii]; + } + + // Apply WD after momentum if desired + if(wd != 0.f && wd_after_momentum) + incoming_grads[ii] += wd * incoming_weights[ii]; + + // adjust the weight and write out + weight_in[i] += (-lr * incoming_grads[ii]); + + // if necessary, write out an fp16 copy of the weights + if(N == 4) + model_weights_out[i] = static_cast(weight_in[i]); + + // also write out the new momentum + if(momentum != 0.f) + mom_in[i] = incoming_moms[ii]; + } + } + } + } +}; + +void multi_tensor_sgd_cuda( + int chunk_size, + at::Tensor noop_flag, + std::vector> tensor_lists, + float wd, + float momentum, + float dampening, + float lr, + bool nesterov, + bool first_run, + bool wd_after_momentum, + float scale) +{ + auto num_tensors = tensor_lists.size(); + auto grad_type = tensor_lists[0][0].scalar_type(); + auto weight_type = tensor_lists[1][0].scalar_type(); + + if(num_tensors == 4) + for(int i = 0; i < tensor_lists[3].size(); i++) + TORCH_CHECK(tensor_lists[3][i].scalar_type() == at::ScalarType::Half, + "Additional output tensors should always be fp16."); + + TORCH_CHECK(noop_flag.device() == tensor_lists[0][0].device(), "expected noop flag to be on the same device as tensors"); + + // We have 3 possibilities to handle here, in terms of + // grad_type, param_type, momentum_type, requires_fp16_copy + // 1. fp16, fp16, fp16, No + // 2. fp32, fp32, fp32, No + // 3. fp16, fp32, fp32, Yes + // 4. fp32, fp32, fp32, Yes // this is the materialize_master_grads=True case + // It's easier to hardcode these possibilities than to use + // switches etc. to handle the cross-product of cases where + // we don't want the majority of them. + + // Case 1. fp16, fp16, fp16, No + if(grad_type == at::ScalarType::Half && + weight_type == at::ScalarType::Half && + num_tensors == 3) + { + multi_tensor_apply<3>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + SGDFunctor<3, at::Half, at::Half>(), + wd, + momentum, + dampening, + lr, + nesterov, + first_run, + wd_after_momentum, + scale); + } + // Case 2. fp16, fp32, fp32, No + // else if (grad_type == at::ScalarType::Half && + // weight_type == at::ScalarType::Float && + // num_tensors == 3) { + // multi_tensor_apply<3>( + // BLOCK_SIZE, + // chunk_size, + // noop_flag, + // tensor_lists, + // SGDFunctor<3, at::Half, float>(), + // wd, + // momentum, + // dampening, + // lr, + // nesterov, + // first_run, + // wd_after_momentum); + // } + // Case 2. fp32, fp32, fp32, No + else if(grad_type == at::ScalarType::Float && + weight_type == at::ScalarType::Float && + num_tensors == 3) + { + multi_tensor_apply<3>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + SGDFunctor<3, float, float>(), + wd, + momentum, + dampening, + lr, + nesterov, + first_run, + wd_after_momentum, + scale); + } + // Case 3. fp16, fp32, fp32, Yes + else if(grad_type == at::ScalarType::Half && + weight_type == at::ScalarType::Float && + num_tensors == 4) + { + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + SGDFunctor<4, at::Half, float>(), + wd, + momentum, + dampening, + lr, + nesterov, + first_run, + wd_after_momentum, + scale); + } + // Case 4. fp32, fp32, fp32, Yes + else if(grad_type == at::ScalarType::Float && + weight_type == at::ScalarType::Float && + num_tensors == 4) + { + multi_tensor_apply<4>( + BLOCK_SIZE, + chunk_size, + noop_flag, + tensor_lists, + SGDFunctor<4, float, float>(), + wd, + momentum, + dampening, + lr, + nesterov, + first_run, + wd_after_momentum, + scale); + } + else + { + AT_ERROR("multi_tensor_sgd only supports some combinations of gradient & weight types. Given: ", + "gradient: ", grad_type, ", weight: ", weight_type, ", num_lists: ", num_tensors); + } + + AT_CUDA_CHECK(cudaGetLastError()); +} diff --git a/apex/csrc/syncbn.cpp b/apex/csrc/syncbn.cpp new file mode 100644 index 00000000..578a6e65 --- /dev/null +++ b/apex/csrc/syncbn.cpp @@ -0,0 +1,109 @@ +#include +#include + +#include + +// returns {mean,biased_var} +// implemented using welford +std::vector welford_mean_var_CUDA(const at::Tensor input); + +// reduces array of mean/var across processes +// returns global {mean,inv_std,biased_var} +// implemented using welford +std::vector welford_parallel_CUDA(const at::Tensor mean_feature_nodes, + const at::Tensor var_biased_feature_nodes, + const at::Tensor numel, + const float eps); + +// elementwise BN operation, returns output +// input/weight/shift should have identical data type; +// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype) +at::Tensor batchnorm_forward_CUDA(const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::optional shift); + +// backward BN operation, returns {sum_dy, sum_dy_xmu, grad_weight, grad_bias} +// grad_output/input should have identical data type; +// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype) +// implemented using kahan summation +std::vector reduce_bn_CUDA(const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight); + +// elementwise backward BN operation, returns grad_input +// grad_output/input/weight precision could be fp16/fp32; +// mean/inv_std/sum_dy/sum_dy_xmu precision is fp32 +at::Tensor batchnorm_backward_CUDA(const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::Tensor sum_dy, + const at::Tensor sum_dy_xmu, + const at::Tensor count); + +// returns {mean, biased_var} +// implemented using welford +// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL +std::vector welford_mean_var_c_last_CUDA(const at::Tensor input); + +// elementwise BN operation, returns output +// input/weight/shift should have identical data type; +// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype) +// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL +at::Tensor batchnorm_forward_c_last_CUDA(const at::Tensor input, + const at::optional z, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::optional shift, + const bool fuse_relu); + +// backward BN operation, returns {sum_dy, sum_dy_xmu, grad_weight, grad_bias} +// grad_output/input should have identical data type; +// mean/inv_std have promoted data type (dtype==fp16?fp32:dtype) +// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL +std::vector reduce_bn_c_last_CUDA(const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight); + +// elementwise backward BN operation, returns grad_input +// grad_output/input/weight precision could be fp16/fp32; +// mean/inv_std/sum_dy/sum_dy_xmu precision is fp32 +// expect data to be in n+c format (channel last) and applies CUDNN_BATCHNORM_SPATIAL +at::Tensor batchnorm_backward_c_last_CUDA(const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::Tensor sum_dy, + const at::Tensor sum_dy_xmu, + const at::Tensor count); + +at::Tensor relu_backward_c_last_CUDA(const at::Tensor grad_output, + const at::Tensor input, + const at::optional z, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::optional shift); + + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("welford_mean_var", &welford_mean_var_CUDA, "welford mean variance"); + m.def("welford_parallel", &welford_parallel_CUDA, "welford parallel reduce mean variance"); + m.def("batchnorm_forward", &batchnorm_forward_CUDA, "batchnorm forward"); + m.def("reduce_bn", &reduce_bn_CUDA, "batchnorm backward reduce grad sum and bias/weight grad"); + m.def("batchnorm_backward", &batchnorm_backward_CUDA, "batchnorm backward dgrad"); + m.def("welford_mean_var_c_last", &welford_mean_var_c_last_CUDA, "welford mean variance nhwc"); + m.def("batchnorm_forward_c_last", &batchnorm_forward_c_last_CUDA, "batchnorm forward nhwc"); + m.def("reduce_bn_c_last", &reduce_bn_c_last_CUDA, "batchnorm backwards reduce grad sum and bias/weight grad nhwc"); + m.def("batchnorm_backward_c_last", &batchnorm_backward_c_last_CUDA, "batchnorm backward dgrad nhwc"); + m.def("relu_bw_c_last", &relu_backward_c_last_CUDA, "relu_bw_c_last"); +} diff --git a/apex/csrc/type_shim.h b/apex/csrc/type_shim.h new file mode 100644 index 00000000..a805941c --- /dev/null +++ b/apex/csrc/type_shim.h @@ -0,0 +1,419 @@ +#include +#include "compat.h" + +// Forward/backward compatiblity hack around +// https://github.com/pytorch/pytorch/commit/3aeb78079bcd68282fe9117088e138b77318e288 +// pending more future-proof guidance from upstream. +// struct TypeShim +// { +// const at::Type& payload; +// TypeShim(const at::Type& type) : payload(type) {} +// // Enable trivial conversion to a const at::Type& for pre-3aeb78 +// operator const at::Type&(){ return payload; }; +// // Enable dispatch switch statements to take *this directly for post-3aeb78 +// //operator at::ScalarType(){ return payload.; }; +// }; + +#define DISPATCH_FLOAT_AND_HALF(TYPE, LEVEL, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Float: \ + { \ + using scalar_t_##LEVEL = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_##LEVEL = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + +#define DISPATCH_FLOAT_HALF_AND_BFLOAT(TYPE, LEVEL, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Float: \ + { \ + using scalar_t_##LEVEL = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_##LEVEL = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_##LEVEL = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + +#define DISPATCH_FLOAT_HALF_AND_BYTE(TYPE, LEVEL, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Float: \ + { \ + using scalar_t_##LEVEL = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_##LEVEL = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Byte: \ + { \ + using scalar_t_##LEVEL = uint8_t; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + +#define DISPATCH_DOUBLE_FLOAT_AND_HALF(TYPE, LEVEL, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Double: \ + { \ + using scalar_t_##LEVEL = double; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Float: \ + { \ + using scalar_t_##LEVEL = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_##LEVEL = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + +#define DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT(TYPE, LEVEL, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Double: \ + { \ + using scalar_t_##LEVEL = double; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Float: \ + { \ + using scalar_t_##LEVEL = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_##LEVEL = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_##LEVEL = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + + #define DISPATCH_DOUBLE_AND_FLOAT(TYPE, LEVEL, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Double: \ + { \ + using scalar_t_##LEVEL = double; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Float: \ + { \ + using scalar_t_##LEVEL = float; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + + #define DISPATCH_HALF_AND_BFLOAT(TYPE, NAME, ...) \ + switch(TYPE) \ + { \ + case at::ScalarType::Half: \ + { \ + using scalar_t = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \ + } + + + #define DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES(TYPEIN, TYPEOUT, NAME, ...) \ + switch(TYPEIN) \ + { \ + case at::ScalarType::Float: \ + { \ + using scalar_t_in = float; \ + switch(TYPEOUT) \ + { \ + case at::ScalarType::Float: \ + { \ + using scalar_t_out = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_out = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_out = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPEOUT), "'"); \ + } \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_in = at::Half; \ + using scalar_t_out = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_in = at::BFloat16; \ + using scalar_t_out = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPEIN), "'"); \ + } + + + #define DISPATCH_DOUBLE_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES(TYPEIN, TYPEOUT, NAME, ...) \ + switch(TYPEIN) \ + { \ + case at::ScalarType::Double: \ + { \ + using scalar_t_in = double; \ + switch(TYPEOUT) \ + { \ + case at::ScalarType::Double: \ + { \ + using scalar_t_out = double; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Float: \ + { \ + using scalar_t_out = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_out = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_out = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPEOUT), "'"); \ + } \ + break; \ + } \ + case at::ScalarType::Float: \ + { \ + using scalar_t_in = float; \ + switch(TYPEOUT) \ + { \ + case at::ScalarType::Float: \ + { \ + using scalar_t_out = float; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_out = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_out = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPEOUT), "'"); \ + } \ + break; \ + } \ + case at::ScalarType::Half: \ + { \ + using scalar_t_in = at::Half; \ + using scalar_t_out = at::Half; \ + __VA_ARGS__; \ + break; \ + } \ + case at::ScalarType::BFloat16: \ + { \ + using scalar_t_in = at::BFloat16; \ + using scalar_t_out = at::BFloat16; \ + __VA_ARGS__; \ + break; \ + } \ + default: \ + AT_ERROR(#NAME, " not implemented for '", toString(TYPEIN), "'"); \ + } + + +template +__device__ __forceinline__ T reduce_block_into_lanes + (T *x, + T val, + int lanes=1, + bool share_result=false) // lanes is intended to be <= 32. +{ + int tid = threadIdx.x + threadIdx.y*blockDim.x; + int blockSize = blockDim.x*blockDim.y; // blockSize is intended to be a multiple of 32. + + if(blockSize >= 64) + { + x[tid] = val; + __syncthreads(); + } + + #pragma unroll + for(int i = (blockSize >> 1); i >= 64; i >>= 1) + { + if(tid < i) + x[tid] = x[tid] + x[tid+i]; + __syncthreads(); + } + + T final; + + if(tid < 32) + { + if(blockSize >= 64) + final = x[tid] + x[tid+32]; + else + final = val; + // __SYNCWARP(); + + #pragma unroll + for(int i = 16; i >= lanes; i >>= 1) + final = final + __shfl_down_sync(0xffffffff, final, i); + } + + if(share_result) + { + if(tid < lanes) + x[tid] = final; // EpilogueOp + // Make sure the smem result is visible to all warps. + __syncthreads(); + } + + return final; +} + +template +__device__ __forceinline__ T reduce_block_into_lanes_max_op + (T *x, + T val, + int lanes=1, + bool share_result=false) // lanes is intended to be <= 32. +{ + int tid = threadIdx.x + threadIdx.y*blockDim.x; + int blockSize = blockDim.x*blockDim.y; // blockSize is intended to be a multiple of 32. + + if(blockSize >= 64) + { + x[tid] = val; + __syncthreads(); + } + + #pragma unroll + for(int i = (blockSize >> 1); i >= 64; i >>= 1) + { + if(tid < i) + x[tid] = fmaxf(fabsf(x[tid]), fabsf(x[tid+i])); + __syncthreads(); + } + + T final; + + if(tid < 32) + { + if(blockSize >= 64) + final = fmaxf(fabsf(x[tid]), fabsf(x[tid+32])); + else + final = val; + // __SYNCWARP(); + + #pragma unroll + for(int i = 16; i >= lanes; i >>= 1) + final = fmaxf(fabsf(final), fabsf(__shfl_down_sync(0xffffffff, final, i))); + } + + if(share_result) + { + if(tid < lanes) + x[tid] = final; // EpilogueOp + // Make sure the smem result is visible to all warps. + __syncthreads(); + } + + return final; +} diff --git a/apex/csrc/welford.cu b/apex/csrc/welford.cu new file mode 100644 index 00000000..374a3845 --- /dev/null +++ b/apex/csrc/welford.cu @@ -0,0 +1,1510 @@ +#include +#include +#include +#include + +#include +#include + +#include + +#include "type_shim.h" +#include "compat.h" + + +__device__ __forceinline__ int lastpow2(int n) +{ + int out = 1 << (31 - __clz(n)); + if(n == out) + out >>= 1; + return out; +} + +__host__ __forceinline__ int h_next_pow2(unsigned int n) { + n--; + n |= (n >> 1); + n |= (n >> 2); + n |= (n >> 4); + n |= (n >> 8); + n |= (n >> 16); + return ++n; +} + +__host__ __forceinline__ int h_last_pow2(unsigned int n) { + n |= (n >> 1); + n |= (n >> 2); + n |= (n >> 4); + n |= (n >> 8); + n |= (n >> 16); + return n - (n >> 1); +} + + +#define WARP_SIZE 32 + +template +__device__ __forceinline__ T warp_reduce_sum(T val) +{ + #pragma unroll + for(int i = WARP_SIZE/2; i > 0; i >>= 1) + val = val + __shfl_down_sync(0xffffffff, val, i); + return val; +} + +template +__device__ __forceinline__ T reduce_block(T *x, T val) +{ + int tid = threadIdx.y*blockDim.x + threadIdx.x; + int blockSize = blockDim.x * blockDim.y; + + if (blockSize > 32) { + val = warp_reduce_sum(val); + if (tid % WARP_SIZE == 0) + x[tid/WARP_SIZE] = val; + + __syncthreads(); + + val = (tid < blockSize / WARP_SIZE? x[tid%WARP_SIZE] : T(0)); + } + + if(tid/WARP_SIZE==0) val = warp_reduce_sum(val); + + return val; +} + +#define ELEMENTS_PER_ITER 4 // enables concurrency within each thread to hide latency +#define ELEMENTS_PER_THREAD 16 +#define OPTIMAL_TILE_W 32 +#define MAX_H_BLOCK 128 +#define MAX_BLOCK_SIZE 512 + +__host__ int div_ru(int x, int y) { + return h_last_pow2(1 + (x-1)/y); +} + +__host__ void flexible_launch_configs( + const int reduction, + const int stride, + dim3 &block, + dim3 &grid, + const bool coop_flag = false) { + int block_x = std::min(h_last_pow2(stride), OPTIMAL_TILE_W); + int block_y = std::min(h_last_pow2(div_ru(reduction , ELEMENTS_PER_THREAD)), + MAX_BLOCK_SIZE / block_x); + if (block_x * block_y != MAX_BLOCK_SIZE) { + block_x = std::min(h_last_pow2(stride), MAX_BLOCK_SIZE / block_y); + } + + int grid_x = div_ru(stride, block_x); + int grid_y = std::min(div_ru(reduction, block_y * ELEMENTS_PER_THREAD), MAX_H_BLOCK); + if (coop_flag) { + // it's not worth having a grid reduction if the reduction dimension is not big enough + grid_y = grid_y < 8 ? 1 : grid_y; + } + + block.x = block_x; + block.y = block_y; + block.z = 1; + grid.x = grid_x; + grid.y = grid_y; + grid.z = 1; +} + +template +__device__ __forceinline__ void welford_merge_element(C& count, + T& mean, + T& m2n, + const C& num_new, + const T& mean_new, + const T& m2n_new) { + T factor = T(1.0) / max(1, (count + num_new)); + T delta0 = mean - mean_new; + mean = (mean_new * num_new + mean * count) * factor; + m2n += m2n_new + delta0 * delta0 * num_new * count * factor; + count += num_new; +} + +template +__device__ __forceinline__ void warp_reduce_mean_m2n(T &mean, T &m2n, int &num) +{ + #pragma unroll + for(int i = WARP_SIZE/2; i > 0; i >>= 1) { + auto num_new = __shfl_down_sync(0xffffffff, num, i); + auto mean_new = __shfl_down_sync(0xffffffff, mean, i); + auto m2n_new = __shfl_down_sync(0xffffffff, m2n, i); + welford_merge_element(num, mean, m2n, num_new, mean_new, m2n_new); + } +} + +template +__device__ void welford_reduce_mean_m2n( + T* __restrict__ x, + int* __restrict__ count, + T &mean, + T &m2n, + int &num, + int block_size, + int thread_id) +{ + int lane = thread_id % WARP_SIZE; + int wid = thread_id / WARP_SIZE; + + if (block_size > 32) { + warp_reduce_mean_m2n(mean, m2n, num); + if (lane == 0) { + x[wid*2] = mean; + x[wid*2+1] = m2n; + count[wid] = num; + } + __syncthreads(); + + if (wid == 0) { + mean = (thread_id < block_size / WARP_SIZE)? x[lane*2] : T(0); + m2n = (thread_id < block_size / WARP_SIZE)? x[lane*2+1] : T(0); + num = (thread_id < block_size / WARP_SIZE)? count[lane] : int(0); + } + } + + if (wid==0) warp_reduce_mean_m2n(mean, m2n, num); + + return; +} + +// return spatial size for NC+ Tensors +__host__ int get_tensor_spatial_size(const at::Tensor& input) +{ + auto space_size = input.size(2); + for (int i = 3; i < input.ndimension(); i++) { + space_size *= input.size(i); + } + return space_size; +} + +// promote accumulation scalar type. promote half to float. +__host__ at::ScalarType promote_scalartype(const at::Tensor& input) +{ + return input.scalar_type() == at::ScalarType::Half ? + at::ScalarType::Float : input.scalar_type(); +} + +// return single element size, optional accumulation type promotion. +__host__ size_t get_element_data_size(const at::Tensor& input, bool accumulation = false) +{ + auto scalar_type = accumulation ? promote_scalartype(input) : input.scalar_type(); + return at::elementSize(scalar_type); +} + +template +__device__ __forceinline__ void welford_merge_block_vertical(C& count, + T& mean, + T& m2n, + C* shmem_count, + T* shmem_mean, + T* shmem_m2n) { + // write to shared memory + auto address_base = threadIdx.x + threadIdx.y * blockDim.x; + shmem_mean[address_base] = mean; + shmem_m2n[address_base] = m2n; + shmem_count[address_base] = count; + +#pragma unroll + for (int offset = blockDim.y/2; offset > 0; offset >>= 1) { + __syncthreads(); + if (threadIdx.y < offset && threadIdx.y + offset < blockDim.y) { + auto address = address_base + offset * blockDim.x; + // read shared memory back to register for reduction + auto num_new = shmem_count[address]; + auto mean_new = shmem_mean[address]; + auto m2n_new = shmem_m2n[address]; + + welford_merge_element(count, mean, m2n, num_new, mean_new, m2n_new); + + // last write is not necessary + shmem_mean[address_base] = mean; + shmem_m2n[address_base] = m2n; + shmem_count[address_base] = count; + } + } +} + +template +__device__ __forceinline__ void merge_block_vertical(T& sum_dy, + T& sum_dy_xmu, + T* shmem_sum_dy, + T* shmem_sum_dy_xmu) { + // write to shared memory + auto address_base = threadIdx.x + threadIdx.y * blockDim.x; + shmem_sum_dy[address_base] = sum_dy; + shmem_sum_dy_xmu[address_base] = sum_dy_xmu; + +#pragma unroll + for (int offset = blockDim.y/2; offset > 0; offset >>= 1) { + __syncthreads(); + if (threadIdx.y < offset && threadIdx.y + offset < blockDim.y) { + auto address = address_base + offset * blockDim.x; + + sum_dy += shmem_sum_dy[address]; + sum_dy_xmu += shmem_sum_dy_xmu[address]; + + // last write is not necessary + shmem_sum_dy[address_base] = sum_dy; + shmem_sum_dy_xmu[address_base] = sum_dy_xmu; + } + } +} + + +// welford kernel calculating mean/biased_variance/unbiased_variance +template +__global__ void welford_kernel( + const scalar_t* __restrict__ input, + outscalar_t* __restrict__ out_mean, + outscalar_t* __restrict__ out_var_biased, + const int bs, + const int fs, + const int ss) { + int block_size = blockDim.x * blockDim.y; + int count = 0; + accscalar_t x_mean = accscalar_t(0); + accscalar_t m_2_n = accscalar_t(0); + + int thread_id = threadIdx.y*blockDim.x + threadIdx.x; + + for (int batch_id = threadIdx.y; batch_id < bs; batch_id += blockDim.y) { + int input_base = blockIdx.x*ss + batch_id*ss*fs; + // sequential welford + for (int offset = threadIdx.x; offset < ss ; offset += blockDim.x) { + count++; + auto x_n = static_cast(input[offset+input_base]); + auto d = x_n - x_mean; + x_mean += d / count; + m_2_n += d * (x_n - x_mean); + } + } + + static __shared__ int s_mem[160]; + accscalar_t* s_mem_ac = (accscalar_t*) &s_mem[32]; + + welford_reduce_mean_m2n(s_mem_ac, s_mem, x_mean, m_2_n, count, block_size, thread_id); + + if (thread_id == 0) { + out_mean[blockIdx.x] = static_cast(x_mean); + out_var_biased[blockIdx.x] = static_cast(m_2_n/count); + } +} + +// elementwise BN kernel +template +__global__ void batchnorm_forward_kernel( + const scalar_t* __restrict__ input, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + const layerscalar_t* __restrict__ weight, + const layerscalar_t* __restrict__ shift, + scalar_t* __restrict__ out, + const int ss, + const int bs) { + auto m_c = mean[blockIdx.x]; + auto inv_std_c = inv_std[blockIdx.x]; + auto w_c = weight == NULL ? accscalar_t(1.0) : static_cast(weight[blockIdx.x]); + auto s_c = shift == NULL ? accscalar_t(0.0) : static_cast(shift[blockIdx.x]); + + for (int batch_offset = blockIdx.y*blockDim.y + threadIdx.y; batch_offset < bs; batch_offset += gridDim.y*blockDim.y) { + int address_base = blockIdx.x*ss + batch_offset*gridDim.x*ss; + for (int offset = threadIdx.x + blockIdx.z*blockDim.x; offset < ss ; offset+= gridDim.z*blockDim.x) { + out[address_base+offset] = static_cast(w_c * (static_cast(input[address_base+offset]) - m_c ) * inv_std_c + s_c); + } + } +} + +// Backward BN kernel, calculates grad_bias, grad_weight as well as intermediate +// results to calculating grad_input. +// Breaking the grad_input to two step to support sync BN, which requires all +// reduce of the intermediate results across processes. +template +__global__ void reduce_bn_kernel( + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ grad_output, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + accscalar_t* __restrict__ sum_dy_o, + accscalar_t* __restrict__ sum_dy_xmu_o, + layerscalar_t* __restrict__ grad_weight, + layerscalar_t* __restrict__ grad_bias, + const int bs, + const int fs, + const int ss) { + static __shared__ int s_mem[64]; + //int total_item_num = bs * ss; + + int thread_id = threadIdx.y*blockDim.x + threadIdx.x; + + auto r_mean = mean[blockIdx.x]; + auto factor = inv_std[blockIdx.x]; + + // Kahan sum + accscalar_t sum_dy = 0.0; + accscalar_t sum_dy_xmu = 0.0; + accscalar_t sum_dy_c = 0.0; + accscalar_t sum_dy_xmu_c = 0.0; + for (int batch_id = threadIdx.y; batch_id < bs; batch_id += blockDim.y) { + int input_base = blockIdx.x*ss + batch_id*ss*fs; + for (int offset = threadIdx.x; offset < ss ; offset += blockDim.x) { + auto e_grad = static_cast(grad_output[offset+input_base]); + auto e_input = static_cast(input[offset+input_base]); + // calculating sum_dy + auto sum_dy_y = e_grad - sum_dy_c; + auto sum_dy_t = sum_dy + sum_dy_y; + sum_dy_c = (sum_dy_t - sum_dy) - sum_dy_y; + sum_dy = sum_dy_t; + + // calculating sum_dy_xmu + auto sum_dy_xmu_y = e_grad * (e_input - r_mean) - sum_dy_xmu_c; + auto sum_dy_xmu_t = sum_dy_xmu + sum_dy_xmu_y; + sum_dy_xmu_c = (sum_dy_xmu_t - sum_dy_xmu) - sum_dy_xmu_y; + sum_dy_xmu = sum_dy_xmu_t; + } + } + + sum_dy = reduce_block((accscalar_t*)s_mem, sum_dy); + __syncthreads(); + sum_dy_xmu = reduce_block((accscalar_t*)s_mem, sum_dy_xmu); + + if (thread_id == 0) { + if (grad_bias != NULL) { + grad_bias[blockIdx.x] = static_cast(sum_dy); + } + if (grad_weight != NULL) { + grad_weight[blockIdx.x] = static_cast(sum_dy_xmu * factor); + } + //mean_dy[blockIdx.x] = sum_dy / total_item_num; + //mean_dy_xmu[blockIdx.x] = sum_dy_xmu / total_item_num; + sum_dy_o[blockIdx.x] = sum_dy; + sum_dy_xmu_o[blockIdx.x] = sum_dy_xmu; + } +} + +// elementwise backward BN kernel +template +__global__ void batchnorm_backward_kernel( + const scalar_t* __restrict__ grad_output, + const scalar_t* __restrict__ input, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + const layerscalar_t* __restrict__ weight, + const accscalar_t* __restrict__ sum_dy, + const accscalar_t* __restrict__ sum_dy_xmu, + const int* __restrict__ numel, + scalar_t* __restrict__ grad_input, + const int64_t world_size, + const int ss, + const int bs) { + int64_t div = 0; + for (int i = 0; i < world_size; i++) { + div += numel[i]; + } + auto m_c = static_cast(mean[blockIdx.x]); + //auto m_dy_c = static_cast(mean_dy[blockIdx.x]); + auto m_dy_c = static_cast(sum_dy[blockIdx.x]) / div; + auto factor_1_c = inv_std[blockIdx.x]; + auto factor_2_c = (weight == NULL ? accscalar_t(1.0) : static_cast(weight[blockIdx.x])) * factor_1_c; + //factor_1_c = factor_1_c * factor_1_c * mean_dy_xmu[blockIdx.x]; + factor_1_c = factor_1_c * factor_1_c * sum_dy_xmu[blockIdx.x] / div; + + for (int batch_offset = blockIdx.y*blockDim.y+threadIdx.y; batch_offset < bs; batch_offset += gridDim.y*blockDim.y) { + int address_base = blockIdx.x*ss + batch_offset*gridDim.x*ss; + for (int offset = threadIdx.x + blockIdx.z*blockDim.x; offset < ss ; offset+= gridDim.z*blockDim.x) { + grad_input[address_base+offset] = (static_cast(grad_output[address_base+offset]) - m_dy_c - (static_cast(input[address_base+offset]) - m_c) * factor_1_c) * factor_2_c; + } + } +} + +// welford kernel for c last tensor calculating mean/biased_variance/unbiased_variance +template + +__global__ void +welford_kernel_c_last( + const scalar_t* __restrict__ input, + outscalar_t* __restrict__ out_mean, + outscalar_t* __restrict__ out_var_biased, + volatile accscalar_t* staging_data, + int* semaphores, + const int reduction_size, + const int stride) { + // hide latency with concurrency + accscalar_t x_mean[PARALLEL_LOADS]; + accscalar_t m_2_n[PARALLEL_LOADS]; + int count[PARALLEL_LOADS]; + +#pragma unroll + for (int i = 0; i < PARALLEL_LOADS; i++) { + x_mean[i] = accscalar_t(0); + m_2_n[i] = accscalar_t(0); + count[i] = accscalar_t(0); + } + // tensor dimension (m,c) + + // loop along m dimension + int inner_loop_stride = blockDim.y * gridDim.y; + + // offset along m dimension + int m_offset = blockIdx.y * blockDim.y + threadIdx.y; + int c_offset = blockIdx.x * blockDim.x + threadIdx.x; + + int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS); + int address_base = m_offset * stride + c_offset; + int address_increment = inner_loop_stride * stride; + + for (int i = 0; i < loop_count; i++) { + accscalar_t x_math[PARALLEL_LOADS]; + accscalar_t x_count_inv[PARALLEL_LOADS]; + accscalar_t is_valid[PARALLEL_LOADS]; + + // load multiple data in +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + if (c_offset < stride && m_offset < reduction_size) { + x_math[j] = input[address_base]; + count[j]++; + x_count_inv[j] = accscalar_t(1) / count[j]; + is_valid[j] = accscalar_t(1); + } else { + x_math[j] = accscalar_t(0); + x_count_inv[j] = accscalar_t(0); + is_valid[j] = accscalar_t(0); + } + m_offset += inner_loop_stride; + address_base += address_increment; + } + + // calculate mean/m2n with welford +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + accscalar_t delta0 = x_math[j] - x_mean[j]; + x_mean[j] += delta0 * x_count_inv[j]; + accscalar_t delta1 = x_math[j] - x_mean[j]; + m_2_n[j] += delta0 * delta1 * is_valid[j]; + } + } + + // thread reduction to accumulate mean/m_2_n/count between PARALLEL_LOADS +#pragma unroll + for (int j = 1; j < PARALLEL_LOADS; j++) { + welford_merge_element(count[0], x_mean[0], m_2_n[0], count[j], x_mean[j], m_2_n[j]); + } + + // release x_mean / m_2_n + auto mean_th = x_mean[0]; + auto m2_th = m_2_n[0]; + auto count_th = count[0]; + + // block-wise reduction with shared memory (since reduction cannot be done within a warp) + static __shared__ accscalar_t shmem_mean[MAX_BLOCK_SIZE]; + static __shared__ accscalar_t shmem_m2n[MAX_BLOCK_SIZE]; + static __shared__ int shmem_count[MAX_BLOCK_SIZE]; + + welford_merge_block_vertical(count_th, mean_th, m2_th, shmem_count, shmem_mean, shmem_m2n); + + // grid reduction if needed (coop launch used at the first place) + if (gridDim.y > 1) { + volatile accscalar_t* staging_mean = staging_data; + volatile accscalar_t* staging_m2n = &staging_data[stride*gridDim.y]; + volatile int* staging_count = reinterpret_cast(&staging_m2n[stride*gridDim.y]); + + address_base = c_offset + blockIdx.y * stride; + // write data to staging_data; + if (threadIdx.y == 0 && c_offset < stride) { + staging_mean[address_base] = mean_th; + staging_m2n[address_base] = m2_th; + staging_count[address_base] = count_th; + } + + __threadfence(); + __syncthreads(); // ensuring writes to staging_ is visible to all blocks + + __shared__ bool is_last_block_done; + // mark block done + if (threadIdx.x == 0 && threadIdx.y == 0) { + int old = atomicAdd(&semaphores[blockIdx.x], 1); + is_last_block_done = (old == (gridDim.y-1)); + } + + __syncthreads(); + + // check that all data is now available in global memory + if (is_last_block_done) { + count_th = 0; + mean_th = accscalar_t(0.0); + m2_th = accscalar_t(0.0); + + for (int y = threadIdx.y; y < gridDim.y; y += blockDim.y) { + address_base = c_offset + y * stride; + int num_new = c_offset < stride ? staging_count[address_base] : 0; + accscalar_t mean_new = c_offset < stride ? staging_mean[address_base] : accscalar_t(0.0); + accscalar_t m2n_new = c_offset < stride ? staging_m2n[address_base] : accscalar_t(0.0); + + welford_merge_element(count_th, mean_th, m2_th, num_new, mean_new, m2n_new); + } + + welford_merge_block_vertical(count_th, mean_th, m2_th, shmem_count, shmem_mean, shmem_m2n); + if (threadIdx.y == 0 && c_offset < stride) { + out_mean[c_offset] = static_cast(mean_th); + out_var_biased[c_offset] = static_cast(m2_th / count_th); + } + } + } else { + if (blockIdx.y == 0 && threadIdx.y == 0 && c_offset < stride) { + out_mean[c_offset] = static_cast(mean_th); + out_var_biased[c_offset] = static_cast(m2_th / count_th); + } + } +} + +// parallel welford kernel to further reduce mean / biased_var +// into mean / unbiased_var / inv_std across multiple processes. +template +__global__ void welford_kernel_parallel( + const scalar_t* __restrict__ mean, + const scalar_t* __restrict__ var_biased, + const int* __restrict__ numel, + scalar_t* __restrict__ out_mean, + scalar_t* __restrict__ out_var, + scalar_t* __restrict__ inv_std, + const int world_size, + const int feature_size, + const float eps) { + + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < feature_size; i += gridDim.x * blockDim.x) { + // load data; + int address = i; + scalar_t x_mean = 0; + scalar_t m_2_n = 0; + int count = 0; + for (int j = 0; j < world_size; j++) { + welford_merge_element(count, x_mean, m_2_n, numel[j], mean[address], var_biased[address]*numel[j]); + address += feature_size; + } + out_mean[i] = x_mean; + out_var[i] = m_2_n/ (count - 1); + inv_std[i] = scalar_t(1) / sqrt(m_2_n/count + eps); + } +} + +// elementwise BN kernel +template < + typename scalar_t, + typename accscalar_t, + typename layerscalar_t, + int PARALLEL_LOADS> +__global__ void batchnorm_forward_c_last_kernel( + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ z, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + const layerscalar_t* __restrict__ weight, + const layerscalar_t* __restrict__ shift, + scalar_t* __restrict__ out, + const int reduction_size, + const int stride, + const bool fuse_relu) { + // tensor dimension (m,c) + // loop along m dimension + int inner_loop_stride = blockDim.y * gridDim.y; + + // offset along m dimension + int m_offset = blockIdx.y * blockDim.y + threadIdx.y; + int c_offset = blockIdx.x * blockDim.x + threadIdx.x; + + auto m_c = mean[c_offset]; + auto inv_std_c = static_cast(inv_std[c_offset]); + auto w_c = weight == NULL ? accscalar_t(1.0) : static_cast(weight[c_offset]); + auto s_c = shift == NULL ? accscalar_t(0.0) : static_cast(shift[c_offset]); + + int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS); + int address_base = m_offset * stride + c_offset; + int address_increment = inner_loop_stride * stride; + + for (int i = 0; i < loop_count; i++) { +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + if (c_offset < stride && m_offset < reduction_size) { + auto tmp = w_c * (static_cast(input[address_base]) - m_c ) * inv_std_c + s_c; + if (z != NULL) { + tmp += z[address_base]; + } + out[address_base] = (fuse_relu && tmp <= accscalar_t(0.0) ? scalar_t(0.0) : static_cast(tmp)); + } + m_offset += inner_loop_stride; + address_base += address_increment; + } + } +} + +// elementwise BN kernel +template < + typename scalar_t, + typename accscalar_t, + typename layerscalar_t, + int PARALLEL_LOADS> +__global__ void relu_backward_c_last_kernel( + const scalar_t* __restrict__ grad_output, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ z, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + const layerscalar_t* __restrict__ weight, + const layerscalar_t* __restrict__ shift, + scalar_t* __restrict__ out, + const int reduction_size, + const int stride) { + // tensor dimension (m,c) + // loop along m dimension + int inner_loop_stride = blockDim.y * gridDim.y; + + // offset along m dimension + int m_offset = blockIdx.y * blockDim.y + threadIdx.y; + int c_offset = blockIdx.x * blockDim.x + threadIdx.x; + + auto m_c = mean[c_offset]; + auto inv_std_c = static_cast(inv_std[c_offset]); + auto w_c = weight == NULL ? accscalar_t(1.0) : static_cast(weight[c_offset]); + auto s_c = shift == NULL ? accscalar_t(0.0) : static_cast(shift[c_offset]); + + int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS); + int address_base = m_offset * stride + c_offset; + int address_increment = inner_loop_stride * stride; + + for (int i = 0; i < loop_count; i++) { +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + if (c_offset < stride && m_offset < reduction_size) { + auto tmp = w_c * (static_cast(input[address_base]) - m_c ) * inv_std_c + s_c; + if (z != NULL) { + tmp += z[address_base]; + } + out[address_base] = (tmp <= accscalar_t(0.0) ? scalar_t(0.0) : grad_output[address_base]); + } + m_offset += inner_loop_stride; + address_base += address_increment; + } + } +} + +// batchnorm backward kernel for c last tensor +template + +__global__ void reduce_bn_c_last_kernel( + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ grad_output, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + accscalar_t* __restrict__ sum_dy_o, + accscalar_t* __restrict__ sum_dy_xmu_o, + layerscalar_t* __restrict__ grad_weight, + layerscalar_t* __restrict__ grad_bias, + volatile accscalar_t* staging_data, + int* semaphores, + const int reduction_size, + const int stride) { + + // hide latency with concurrency + accscalar_t sum_dy[PARALLEL_LOADS]; + accscalar_t sum_dy_xmu[PARALLEL_LOADS]; + +#pragma unroll + for (int i = 0; i < PARALLEL_LOADS; i++) { + sum_dy[i] = accscalar_t(0); + sum_dy_xmu[i] = accscalar_t(0); + } + // tensor dimension (m,c) + + // loop along m dimension + int inner_loop_stride = blockDim.y * gridDim.y; + + // offset along m dimension + int m_offset = blockIdx.y * blockDim.y + threadIdx.y; + int c_offset = blockIdx.x * blockDim.x + threadIdx.x; + + int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS); + int address_base = m_offset * stride + c_offset; + int address_increment = inner_loop_stride * stride; + + auto r_mean = mean[c_offset]; + auto factor = inv_std[c_offset]; + + for (int i = 0; i < loop_count; i++) { + accscalar_t x_input[PARALLEL_LOADS]; + accscalar_t x_grad_output[PARALLEL_LOADS]; + + // load multiple data in +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + if (c_offset < stride && m_offset < reduction_size) { + x_input[j] = input[address_base]; + x_grad_output[j] = grad_output[address_base]; + } else { + x_input[j] = accscalar_t(0); + x_grad_output[j] = accscalar_t(0); + } + m_offset += inner_loop_stride; + address_base += address_increment; + } + + // calculate sum_dy / sum_dy_xmu +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + sum_dy[j] += x_grad_output[j]; + sum_dy_xmu[j] += x_grad_output[j] * (x_input[j] - r_mean); + } + } + + // thread reduction to accumulate sum_dy / sum_dy_xmu between PARALLEL_LOADS +#pragma unroll + for (int j = 1; j < PARALLEL_LOADS; j++) { + sum_dy[0] += sum_dy[j]; + sum_dy_xmu[0] += sum_dy_xmu[j]; + } + + // release array of registers + auto sum_dy_th = sum_dy[0]; + auto sum_dy_xmu_th = sum_dy_xmu[0]; + + // block-wise reduction with shared memory (since reduction cannot be done within a warp) + static __shared__ accscalar_t shmem_sum_dy[MAX_BLOCK_SIZE]; + static __shared__ accscalar_t shmem_sum_dy_xmu[MAX_BLOCK_SIZE]; + + merge_block_vertical(sum_dy_th, sum_dy_xmu_th, shmem_sum_dy, shmem_sum_dy_xmu); + + // grid reduction if needed (coop launch used at the first place) + if (gridDim.y > 1) { + volatile accscalar_t* staging_sum_dy = staging_data; + volatile accscalar_t* staging_sum_dy_xmu = &staging_data[stride*gridDim.y]; + + address_base = c_offset + blockIdx.y * stride; + // write data to staging_data; + if (threadIdx.y == 0 && c_offset < stride) { + staging_sum_dy[address_base] = sum_dy_th; + staging_sum_dy_xmu[address_base] = sum_dy_xmu_th; + } + + __threadfence(); + __syncthreads(); // ensuring writes to staging_ is visible to all blocks + + __shared__ bool is_last_block_done; + // mark block done + if (threadIdx.x == 0 && threadIdx.y == 0) { + int old = atomicAdd(&semaphores[blockIdx.x], 1); + is_last_block_done = (old == (gridDim.y-1)); + } + + __syncthreads(); + + // check that all data is now available in global memory + if (is_last_block_done) { + sum_dy_th = accscalar_t(0.0); + sum_dy_xmu_th = accscalar_t(0.0); + + for (int y = threadIdx.y; y < gridDim.y; y += blockDim.y) { + address_base = c_offset + y * stride; + sum_dy_th += (c_offset < stride ? staging_sum_dy[address_base] : accscalar_t(0.0)); + sum_dy_xmu_th += (c_offset < stride ? staging_sum_dy_xmu[address_base] : accscalar_t(0.0)); + } + + merge_block_vertical(sum_dy_th, sum_dy_xmu_th, shmem_sum_dy, shmem_sum_dy_xmu); + if (threadIdx.y == 0 && c_offset < stride) { + if (grad_bias != NULL) { + grad_bias[c_offset] = static_cast(sum_dy_th); + } + if (grad_weight != NULL) { + grad_weight[c_offset] = static_cast(sum_dy_xmu_th * factor); + } + //mean_dy[c_offset] = sum_dy_th / reduction_size; + //mean_dy_xmu[c_offset] = sum_dy_xmu_th / reduction_size; + sum_dy_o[c_offset] = sum_dy_th; + sum_dy_xmu_o[c_offset] = sum_dy_xmu_th; + } + } + } else { + if (blockIdx.y == 0 && threadIdx.y == 0 && c_offset < stride) { + if (grad_bias != NULL) { + grad_bias[c_offset] = static_cast(sum_dy_th); + } + if (grad_weight != NULL) { + grad_weight[c_offset] = static_cast(sum_dy_xmu_th * factor); + } + //mean_dy[c_offset] = sum_dy_th / reduction_size; + //mean_dy_xmu[c_offset] = sum_dy_xmu_th / reduction_size; + sum_dy_o[c_offset] = sum_dy_th; + sum_dy_xmu_o[c_offset] = sum_dy_xmu_th; + } + } +} + +// elementwise BN kernel +template < + typename scalar_t, + typename accscalar_t, + typename layerscalar_t, + int PARALLEL_LOADS> +__global__ void batchnorm_backward_c_last_kernel( + const scalar_t* __restrict__ grad_output, + const scalar_t* __restrict__ input, + const accscalar_t* __restrict__ mean, + const accscalar_t* __restrict__ inv_std, + const layerscalar_t* __restrict__ weight, + const accscalar_t* __restrict__ sum_dy, + const accscalar_t* __restrict__ sum_dy_xmu, + const int* __restrict__ numel, + scalar_t* __restrict__ grad_input, + const int64_t world_size, + const int reduction_size, + const int stride) { + int64_t div = 0; + for (int i = 0; i < world_size; i++) { + div += numel[i]; + } + // tensor dimension (m,c) + // loop along m dimension + int inner_loop_stride = blockDim.y * gridDim.y; + + // offset along m dimension + int m_offset = blockIdx.y * blockDim.y + threadIdx.y; + int c_offset = blockIdx.x * blockDim.x + threadIdx.x; + + auto m_c = mean[c_offset]; + auto m_dy_c = sum_dy[c_offset] / div; + auto factor_1_c = inv_std[c_offset]; + auto factor_2_c = (weight == NULL? accscalar_t(1.0) : static_cast(weight[c_offset])) * factor_1_c; + factor_1_c = factor_1_c * factor_1_c * sum_dy_xmu[c_offset] / div; + + int loop_count = 1 + (reduction_size - 1) / (inner_loop_stride * PARALLEL_LOADS); + int address_base = m_offset * stride + c_offset; + int address_increment = inner_loop_stride * stride; + + for (int i = 0; i < loop_count; i++) { +#pragma unroll + for (int j = 0; j < PARALLEL_LOADS; j++) { + if (c_offset < stride && m_offset < reduction_size) { + grad_input[address_base] = static_cast( + (static_cast(grad_output[address_base]) - m_dy_c - + (static_cast(input[address_base]) - m_c) * factor_1_c) + * factor_2_c); + } + m_offset += inner_loop_stride; + address_base += address_increment; + } + } +} + +std::vector welford_mean_var_CUDA(const at::Tensor input) { + const auto batch_size = input.size(0); + const auto feature_size = input.size(1); + + auto space_size = get_tensor_spatial_size(input); + auto scalar_type = promote_scalartype(input); + + at::Tensor out_var_biased = at::empty({feature_size}, input.options().dtype(scalar_type)); + at::Tensor out_mean = at::empty({feature_size}, input.options().dtype(scalar_type)); + + int block_y = min(h_last_pow2(batch_size), int(MAX_BLOCK_SIZE / 32)); + int block_x = max(1, min(MAX_BLOCK_SIZE / block_y, h_last_pow2(space_size))); + const dim3 block(block_x, block_y); + const dim3 grid(feature_size); + + auto stream = at::cuda::getCurrentCUDAStream(); + + { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "welford_mean_var_kernel", + using accscalar_t = at::acc_type; + welford_kernel<<>>( + input.DATA_PTR(), + out_mean.DATA_PTR(), + out_var_biased.DATA_PTR(), + batch_size, + feature_size, + space_size); + ); + } + + return {out_mean, out_var_biased}; +} + +at::Tensor batchnorm_forward_CUDA( + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::optional shift) { + const auto batch_size = input.size(0); + const auto feature_size = input.size(1); + at::Tensor out = at::empty_like(input); + + auto space_size = get_tensor_spatial_size(input); + + int block_x = max(32, min(MAX_BLOCK_SIZE, h_last_pow2(space_size)/4)); + int block_y = max(1, min(MAX_BLOCK_SIZE/block_x, h_last_pow2(batch_size)/4)); + const dim3 block(block_x, block_y); + int grid_z = max(1, min(65535, h_last_pow2(space_size)/4/block_x)); + int batch_group_size = max(1, min(65535, h_last_pow2(batch_size)/block_y)); + const dim3 grid(feature_size, batch_group_size, grid_z); + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() && + weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + batchnorm_forward_kernel<<>>( + input.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + shift.has_value() ? shift.value().DATA_PTR() : NULL, + out.DATA_PTR(), + space_size, + batch_size); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + batchnorm_forward_kernel<<>>( + input.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + shift.has_value() ? shift.value().DATA_PTR() : NULL, + out.DATA_PTR(), + space_size, + batch_size); + ); + } + return out; +} + +std::vector reduce_bn_CUDA( + const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight) +{ + const auto batch_size = input.size(0); + const auto feature_size = input.size(1); + + auto scalar_type = promote_scalartype(input); + + at::Tensor sum_dy = at::empty({feature_size}, mean.options()); + at::Tensor sum_dy_xmu = at::empty({feature_size}, mean.options()); + + at::Tensor grad_weight; + at::Tensor grad_bias; + if (weight.has_value()) { + grad_weight = at::empty({feature_size}, weight.value().options()); + grad_bias = at::empty({feature_size}, weight.value().options()); + } else { + grad_weight = at::empty({0}, mean.options()); + grad_bias = at::empty({0}, mean.options()); + } + + auto space_size = get_tensor_spatial_size(input); + + int block_y = min(h_last_pow2(batch_size), int(MAX_BLOCK_SIZE/ 32)); + int block_x = max(1, min(MAX_BLOCK_SIZE/ block_y, h_last_pow2(space_size))); + const dim3 block(block_x, block_y); + const dim3 grid(feature_size); + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() && + weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_backward_reduce", + using accscalar_t = at::acc_type; + reduce_bn_kernel<<>>( + input.DATA_PTR(), + grad_output.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + sum_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + weight.has_value() ? grad_weight.DATA_PTR() : NULL, + weight.has_value() ? grad_bias.DATA_PTR() : NULL, + batch_size, + feature_size, + space_size); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_backward_reduce", + using accscalar_t = at::acc_type; + reduce_bn_kernel<<>>( + input.DATA_PTR(), + grad_output.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + sum_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + weight.has_value() ? grad_weight.DATA_PTR() : NULL, + weight.has_value() ? grad_bias.DATA_PTR() : NULL, + batch_size, + feature_size, + space_size); + ); + } + + return {sum_dy, sum_dy_xmu, grad_weight, grad_bias}; +} + +at::Tensor batchnorm_backward_CUDA( + const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::Tensor sum_dy, + const at::Tensor sum_dy_xmu, + const at::Tensor count) { + const auto batch_size = input.size(0); + const auto feature_size = input.size(1); + + at::Tensor grad_input = at::empty_like(input); + + auto space_size = get_tensor_spatial_size(input); + + int block_x = max(32, min(MAX_BLOCK_SIZE, h_last_pow2(space_size)/4)); + int block_y = max(1, min(MAX_BLOCK_SIZE/block_x, h_last_pow2(batch_size)/4)); + const dim3 block(block_x, block_y); + int grid_z = max(1, min(65535, h_last_pow2(space_size)/4/block_x)); + int batch_group_size = max(1, min(65535, h_last_pow2(batch_size)/block_y)); + const dim3 grid(feature_size, batch_group_size, grid_z); + + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() && + weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_backward", + using accscalar_t = at::acc_type; + batchnorm_backward_kernel<<>>( + grad_output.DATA_PTR(), + input.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + sum_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + count.DATA_PTR(), + grad_input.DATA_PTR(), + count.numel(), + space_size, + batch_size); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_backward", + using accscalar_t = at::acc_type; + batchnorm_backward_kernel<<>>( + grad_output.DATA_PTR(), + input.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + sum_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + count.DATA_PTR(), + grad_input.DATA_PTR(), + count.numel(), + space_size, + batch_size); + ); + } + + return grad_input; +} + +std::vector welford_parallel_CUDA(const at::Tensor mean_feature_nodes, + const at::Tensor var_biased, + const at::Tensor numel, + const float eps) { + const auto world_size = mean_feature_nodes.size(0); + const auto feature_size = mean_feature_nodes.size(1); + + at::Tensor out_var = at::empty({feature_size}, var_biased.options()); + at::Tensor inv_std = at::empty_like(out_var); + at::Tensor out_mean = at::empty_like(out_var); + + at::Tensor mean_feature_nodes_ = mean_feature_nodes.contiguous(); + at::Tensor var_biased_ = var_biased.contiguous(); + at::Tensor numel_ = numel.contiguous(); + + // TODO(jie): tile this for memory coalescing! + const int block = std::min(h_last_pow2(feature_size), MAX_BLOCK_SIZE); + const int grid = std::max(1, feature_size / block); + + auto stream = at::cuda::getCurrentCUDAStream(); + + { + using namespace at; + DISPATCH_FLOAT_AND_HALF(mean_feature_nodes.scalar_type(), 0, "welford_parallel_kernel", + welford_kernel_parallel<<>>( + mean_feature_nodes_.DATA_PTR(), + var_biased_.DATA_PTR(), + numel_.DATA_PTR(), + out_mean.DATA_PTR(), + out_var.DATA_PTR(), + inv_std.DATA_PTR(), + world_size, + feature_size, + eps); + ); + } + + return {out_mean, out_var, inv_std}; +} + +std::vector welford_mean_var_c_last_CUDA(const at::Tensor input) { + const auto stride = input.size(input.ndimension()-1); + const auto reduction_size = input.numel() / stride; + + auto scalar_type = promote_scalartype(input); + auto option = input.options().dtype(scalar_type); + + at::Tensor out_var_biased = at::empty({stride}, option); + at::Tensor out_mean = at::empty({stride}, option); + + dim3 block; + dim3 grid; + flexible_launch_configs(reduction_size, stride, block, grid, true); + + at::Tensor staging_data; + at::Tensor semaphores; + if (grid.y > 1) { + staging_data = at::empty({4*stride*grid.y}, option); + semaphores = at::zeros({grid.x}, input.options().dtype(at::kInt)); + } + + auto stream = at::cuda::getCurrentCUDAStream(); + + { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "welford_mean_var_c_last", + using accscalar_t = at::acc_type; + accscalar_t* staging_data_ptr = grid.y > 1 ? staging_data.DATA_PTR() : nullptr; + int* semaphores_ptr = grid.y > 1 ? semaphores.DATA_PTR() : nullptr; + welford_kernel_c_last + <<>>( + input.DATA_PTR(), + out_mean.DATA_PTR(), + out_var_biased.DATA_PTR(), + staging_data_ptr, + semaphores_ptr, + reduction_size, + stride); + ); + } + + return {out_mean, out_var_biased}; +} + +at::Tensor batchnorm_forward_c_last_CUDA( + const at::Tensor input, + const at::optional z, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::optional shift, + const bool fuse_relu) { + const auto stride = input.size(input.ndimension()-1); + const auto reduction_size = input.numel() / stride; + + at::Tensor out = at::empty_like(input); + + dim3 block; + dim3 grid; + flexible_launch_configs(reduction_size, stride, block, grid); + + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() && weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + batchnorm_forward_c_last_kernel + <<>>( + input.DATA_PTR(), + z.has_value() ? z.value().DATA_PTR() : NULL, + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + shift.has_value() ? shift.value().DATA_PTR(): NULL, + out.DATA_PTR(), + reduction_size, + stride, + fuse_relu); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + batchnorm_forward_c_last_kernel + <<>>( + input.DATA_PTR(), + z.has_value() ? z.value().DATA_PTR() : NULL, + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + shift.has_value() ? shift.value().DATA_PTR(): NULL, + out.DATA_PTR(), + reduction_size, + stride, + fuse_relu); + ); + } + return out; +} + +std::vector reduce_bn_c_last_CUDA( + const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight) { + const auto stride = input.size(input.ndimension()-1); + const auto reduction_size = input.numel() / stride; + + at::Tensor sumn_dy = at::empty({stride}, mean.options()); + at::Tensor sum_dy_xmu = at::empty({stride}, mean.options()); + + at::Tensor grad_weight; + at::Tensor grad_bias; + if (weight.has_value()) { + grad_weight = at::empty({stride}, weight.value().options()); + grad_bias = at::empty({stride}, weight.value().options()); + } else { + // because I cannot return an uninitialized at::Tensor + grad_weight = at::empty({0}, mean.options()); + grad_bias = at::empty({0}, mean.options()); + } + + dim3 block; + dim3 grid; + flexible_launch_configs(reduction_size, stride, block, grid, true); + + at::Tensor staging_data; + at::Tensor semaphores; + if (grid.y > 1) { + staging_data = at::empty({2*stride*grid.y}, mean.options()); + semaphores = at::zeros({grid.x}, input.options().dtype(at::kInt)); + } + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() + && weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_backward_reduce", + using accscalar_t = at::acc_type; + accscalar_t* staging_data_ptr = grid.y > 1 ? staging_data.DATA_PTR() : nullptr; + int* semaphores_ptr = grid.y > 1 ? semaphores.DATA_PTR() : nullptr; + reduce_bn_c_last_kernel + <<>>( + input.DATA_PTR(), + grad_output.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + sumn_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + weight.has_value() ? grad_weight.DATA_PTR() : NULL, + weight.has_value() ?grad_bias.DATA_PTR() : NULL, + staging_data_ptr, + semaphores_ptr, + reduction_size, + stride); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_backward_reduce", + using accscalar_t = at::acc_type; + accscalar_t* staging_data_ptr = grid.y > 1 ? staging_data.DATA_PTR() : nullptr; + int* semaphores_ptr = grid.y > 1 ? semaphores.DATA_PTR() : nullptr; + reduce_bn_c_last_kernel + <<>>( + input.DATA_PTR(), + grad_output.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + sumn_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + weight.has_value() ? grad_weight.DATA_PTR() : NULL, + weight.has_value() ?grad_bias.DATA_PTR() : NULL, + staging_data_ptr, + semaphores_ptr, + reduction_size, + stride); + ); + } + + return {sumn_dy, sum_dy_xmu, grad_weight, grad_bias}; +} + +at::Tensor batchnorm_backward_c_last_CUDA( + const at::Tensor grad_output, + const at::Tensor input, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::Tensor sum_dy, + const at::Tensor sum_dy_xmu, + const at::Tensor count) { + const auto stride = input.size(input.ndimension()-1); + const auto reduction_size = input.numel() / stride; + + at::Tensor grad_input = at::empty_like(input); + + dim3 block; + dim3 grid; + flexible_launch_configs(reduction_size, stride, block, grid); + + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() && weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + batchnorm_backward_c_last_kernel + <<>>( + grad_output.DATA_PTR(), + input.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + sum_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + count.DATA_PTR(), + grad_input.DATA_PTR(), + count.numel(), + reduction_size, + stride); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + batchnorm_backward_c_last_kernel + <<>>( + grad_output.DATA_PTR(), + input.DATA_PTR(), + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + sum_dy.DATA_PTR(), + sum_dy_xmu.DATA_PTR(), + count.DATA_PTR(), + grad_input.DATA_PTR(), + count.numel(), + reduction_size, + stride); + ); + } + + return grad_input; +} + +at::Tensor relu_backward_c_last_CUDA( + const at::Tensor grad_output, + const at::Tensor input, + const at::optional z, + const at::Tensor mean, + const at::Tensor inv_std, + const at::optional weight, + const at::optional shift) { + + const auto stride = input.size(input.ndimension()-1); + const auto reduction_size = input.numel() / stride; + + at::Tensor out = at::empty_like(input); + + dim3 block; + dim3 grid; + flexible_launch_configs(reduction_size, stride, block, grid); + + auto stream = at::cuda::getCurrentCUDAStream(); + + if (input.scalar_type() == at::ScalarType::Half + && weight.has_value() && weight.value().scalar_type() == at::ScalarType::Float) { + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + relu_backward_c_last_kernel + <<>>( + grad_output.DATA_PTR(), + input.DATA_PTR(), + z.has_value() ? z.value().DATA_PTR() : NULL, + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + shift.has_value() ? shift.value().DATA_PTR(): NULL, + out.DATA_PTR(), + reduction_size, + stride); + ); + } else { + if (weight.has_value()) { + TORCH_CHECK(input.scalar_type() == weight.value().scalar_type(), + "input.scalar_type() is not supported with weight.scalar_type()"); + } + using namespace at; + DISPATCH_FLOAT_AND_HALF(input.scalar_type(), 0, "batchnorm_forward", + using accscalar_t = at::acc_type; + relu_backward_c_last_kernel + <<>>( + grad_output.DATA_PTR(), + input.DATA_PTR(), + z.has_value() ? z.value().DATA_PTR() : NULL, + mean.DATA_PTR(), + inv_std.DATA_PTR(), + weight.has_value() ? weight.value().DATA_PTR() : NULL, + shift.has_value() ? shift.value().DATA_PTR(): NULL, + out.DATA_PTR(), + reduction_size, + stride); + ); + } + return out; +} diff --git a/apex/docs/Makefile b/apex/docs/Makefile new file mode 100644 index 00000000..86cc2494 --- /dev/null +++ b/apex/docs/Makefile @@ -0,0 +1,32 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = NVIDIAAPEX +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +gh-pages: + git checkout gh-pages + rm -rf build + rm -rf source + git checkout master -- . + make html + rm -rf ../_modules ../_sources ../_static + mv -fv build/html/* ../ + rm -rf build + git add -A + git commit -m "Generated gh-pages for `git log master -1 --pretty=short --abbrev-commit`" && git push origin gh-pages ; git checkout master + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/apex/docs/source/_static/css/pytorch_theme.css b/apex/docs/source/_static/css/pytorch_theme.css new file mode 100644 index 00000000..45e984c9 --- /dev/null +++ b/apex/docs/source/_static/css/pytorch_theme.css @@ -0,0 +1,118 @@ +body { + font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif; +} + +/* Default header fonts are ugly */ +h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend, p.caption { + font-family: "Lato","proxima-nova","Helvetica Neue",Arial,sans-serif; +} + +/* Use white for docs background */ +.wy-side-nav-search { + background-color: #fff; +} + +.wy-nav-content-wrap, .wy-menu li.current > a { + background-color: #fff; +} + +@media screen and (min-width: 1400px) { + .wy-nav-content-wrap { + background-color: rgba(0, 0, 0, 0.0470588); + } + + .wy-nav-content { + background-color: #fff; + } +} + +/* Fixes for mobile */ +.wy-nav-top { + background-color: #fff; + background-image: url('../img/apex.jpg'); + background-repeat: no-repeat; + background-position: center; + padding: 0; + margin: 0.4045em 0.809em; + color: #333; +} + +.wy-nav-top > a { + display: none; +} + +@media screen and (max-width: 768px) { + .wy-side-nav-search>a img.logo { + height: 60px; + } +} + +/* This is needed to ensure that logo above search scales properly */ +.wy-side-nav-search a { + display: block; +} + +/* This ensures that multiple constructors will remain in separate lines. */ +.rst-content dl:not(.docutils) dt { + display: table; +} + +/* Use our red for literals (it's very similar to the original color) */ +.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal { + color: #F05732; +} + +.rst-content tt.xref, a .rst-content tt, .rst-content tt.xref, +.rst-content code.xref, a .rst-content tt, a .rst-content code { + color: #404040; +} + +/* Change link colors (except for the menu) */ + +a { + color: #F05732; +} + +a:hover { + color: #F05732; +} + + +a:visited { + color: #D44D2C; +} + +.wy-menu a { + color: #b3b3b3; +} + +.wy-menu a:hover { + color: #b3b3b3; +} + +/* Default footer text is quite big */ +footer { + font-size: 80%; +} + +footer .rst-footer-buttons { + font-size: 125%; /* revert footer settings - 1/80% = 125% */ +} + +footer p { + font-size: 100%; +} + +/* For hidden headers that appear in TOC tree */ +/* see http://stackoverflow.com/a/32363545/3343043 */ +.rst-content .hidden-section { + display: none; +} + +nav .hidden-section { + display: inherit; +} + +.wy-side-nav-search>div.version { + color: #000; +} diff --git a/apex/docs/source/_static/img/nv-pytorch2.png b/apex/docs/source/_static/img/nv-pytorch2.png new file mode 100644 index 0000000000000000000000000000000000000000..981268c60032b463b387dd3c9f56ebda9929a266 GIT binary patch literal 6502 zcmX|Gby!s2)89qFMOaw6Vd(}*rMtVNy95-LhFxOuOC#N)Agv%J-LW)CcSuQtNQ2aG zegAl$`<#28GxyASX3pIC%$zyVI$Fx$XH?Gs003B3ML`eU5BzH&Z1lU_xFQGLVcV)J zD*zt9 zvjlfUR|7_~B+euN>?F^^24BjKs^ci=a ziBDP<>6!TFwU5c!%j$O_GNjBgN|jJ3I{-!s!{QLETW$ICGAHP9OXN+nV`UkVOYYb@ zX6u|P`QWx*gJ48dY`ndNYBzGy~qE7Ol;#+Nh-Ihy(g4yWRC3%q@!3Ib! zOD@@7{&(aP02J=M7K6hV>0YZ2-WUZ}p9B`@bbIyZD9C(KDKqy>@1qNxU&Z^{?DWJ| zGS0x#YMTp^^8C~b(>8|+K@4QDn6PWj-S7-}r+go@++9`aUoow@*Ju&a00zKSUzg(k zn-g@d5+ZV15E@T7T=6h$W*PhrN>SBidnqc}PIjk?>z)M>1 z?R&l~8B(<_$`yOZ!~?c?QLRh}5s3BD(rNW=F4|$ym~N9x(`1RH_MNtLzFlUlT12RI zJQfNNBS?S@h79ud&a)rc(p6Ey>l-qqhhpamLtl3;=cIiqTN-*t@~&L?R&Ul0gtz#Q zcvjxd7xaYZS`&@;^b(pa7cok^jvnnbPr)wiy?}=GGmp!b2A8Yb1n_2rYuImz_H6Mv zTmU9u{O7`~SKhYgQu?O2Yc7ZA^7UBG@1Jgmn!{mJIX_hL*M_&<=@woe-AGEV_&vUS z5C_7E^toSgI@Vr!n~m;x2Drgfl$qf~KpO{-90{JovHE&J?^3!BEM=Mb5~T5-j(YH| zf2z0A%*!0E;ND1Nl?JN#wh~HMZ+y;<(M^TJA`!|tfM7N`+Ow1FD=;F2%(64iaAi;1 zN;B;`oO7b5^d?-HX*eRxd9`8HHK;Tcmu#~P4v1f`eB>p;ki(S}k$=Rl z1Hc_WXAnY^Vs?hsU!x2b3@lME z#e%U_8R67Go76BOI&JD=Du)DDv=UYLvND-0O|PtL=j!BA45ubGL7${6~}A2o;{ z1(#IZsejBTZb6_U($uvp>d5!z)pQAYSA-B#E@svAwJKMFr1e8I$4|AP86LmsI{B3Q z9UBlMwAOHSxvVm)gbU2e z4I|3YA^j!UJ>eVWFln7DlSC#@2+T{{J;&x?))gWAzrjYI3;9GZnSS~u*3fYGv8M z>~+oc{Po@mvnT$UYO17`AGNY1b!&QQJsUU9V~g<47I&N^x9*=6LP+{>YSAZkKWZPP z8RopOyRnchs!-0(!uow+*VgH>r&Zi?II>E-?UZ!5a9yLy2Ub6Z_kHPO>vYwR^fN=* zpL-JpX-p8vWwB98maXe)Z54rk1bg(w19TDLL^%MUP2Rc3vpzsK zdfC2Kic=}o@Fv$wMtvz0UrkHL`FCh<46?#678u+{Zvy$?7e!C27jJev2NC-R?@I?i zz&rpL-Heoar0>mXZ2o5VxxW#8JedR2yP^yVwTs3qDe7^7Hc9_}2*C+A!^EqBkpPBQ zZ+BBf`Ro;STGNwz_kOwX6|kDWk&!Wm=7seeVFi>tp9$74ehB$R6$;#nq3*BpOUiPTUaveF$4J?FG&(lfM)R!3H)))DF-^&>qGYARxcfI-k zdlXRzSj+oEtb|8*oKb>vLAe7$_?Am$nVOFqBazQ_&>LIOzug;*%Rc1YQgO0j72+j; zMmxu1BlYKD9F>ZAgdC@NAP|ZfZAP>qnE1oK8vU*53XzwgV3jYmrHQCmv1ANWE*#J& zgkVR<;|f-ppqw!8aDX;CaK-*`%6aalMXvZbu9+Glh(;w%0bvcs90L?D(Ibp%itDM0 zlR+G*nm;3ESRav^zg$rV2AaqLe}C$MrgR5A(v*coHyc49Vt(V8E4Fz0{i@t45xbgn zNgxoIMfcBErlqJQs+a(d_T)UguQ4=u+Wo_0pGTEsUVq8t|IkE{bDsF~7hxE3oU1K) zXvVTpok){JKtY5Sm}dkR2BMOc3IdeI1u(;u1mgb$V55?y&{6jvx?^I<5p&!JMc5cm z^8Wbn9ywkB#p{I?I()czh9UQaMf1cgeqCi)NPXa(9=yQ+uIF`bC#wKiOs3>6n}3ieHqm-(VyN%VX=lY_nufrlX%1h2!BkIOn0bYW$HP8S=Q zzs=;NmXB)+Z&v6cheD@|j5sJ9@haut_2_~?K6|9?ZMKcfYIDyq! zA5pdJS2I?6nAP@&=_`t3$R#V#Nz@7M99^NlAD1Mot1NU-21zq&w_^h;tdq1}h0^T| z4m23)wsXMRi;>eJg55cxh9 zKKtBQ52^nYaBI}h45lJjz&>aY+@m+l%nUwRF;4-mLCN;)A%{cj+&@MC(pelKxS`|% z-E?iNUEkdWKu-gGd(6v}!>7`ot_|l48-EdBLhG$l6wjsAct{TAf=-=-{W$Ad8UE$fq*QUVi$+&i~xrA4w6-sPNJ zLP#>RZe#KbzmkmWJQN1&KZe9F1^xZ8OJnU%4sg#vCEAQe+L_^b-;3N2C>mqd?I z$VBF^4N+Wn5RLn`gzsPAl zyOsuJgk0Zq%x&ur@9A|4I=;d@vZ~`*5iHKwcQw9^NmGHD36Kx#882H%+H9|-ku3yC zcg80VnfdP)U0X0^0zes~eHQLgT#`PLs~ZW0a0DV0#n2c0o)zra+X1A*2v-bky*Q_a z{W&FTT$y;jR_)x?GpBwAVTb!9UQRh`RKLkEj26^=S=@IA%%A=mwsh~l6+Ewd?lk1SI5}4rz9pVl zSK!fU@H=B%3QP0o#k%(vP=>!ng%aU>Gy$NkE4%rLs=F5v z&rcB8BQ~#pn$&iCfsbL!W}@fb*aGi9j&NrtktpH3>>fQk0M${K(1KI)$sYC}l`*A( zOJl8I#UMfW+bnm|A}y?@fw?zBh6@JIvbA|0ANNkSQ@@RcW|z;c+xJ=`e9gr5H)2l^ zBt+p)0M@LbYzLo49gVFrl|k_jrt3Da<=np#nWaI&zIB=$8BMD~oY`tNSG@!{4dN4- zPIFc(?jp6UQ0T8OY2PBSZ{zr>G5y4{*V|BIS(;y$Zzt(oI|g4G1wm%Xx4US$w$-A| zC2gGl`quQ*-rjJ| ztF28BsJegH6IvkTOSVNErHPk7AjyW8tIf3b4Wc^8*uRc{lGO}H>pD**clTI09aXE z$6Dx@D`Op_^h(j#3hS0bdKFkjpy3JojkL2){fx7b^Lm{&(#94O3QnWlGN`@X`?~c+ zi-LwO(NzR8QYY)!X2a`$to@Z6IF}4BrZ;fiPl<^wzX5?rkp{IHIumYYPBjZS|84Il ziQoqB)jLc>iK1?U|MqO84y6xfT>U3!Z<41jEkb8c1vO`-2&jB2L(islsuQKUy87E! z`H4!nnhrvu)mh32;ITLXYd!^dUn~dh^54g@nU{*=v?n92xB(t$oGR<`7w{D=#2N!g z8Cm;I&%^(M8?Dz3D9VLMEGbT%vUf59KqQVq|FXGmf?UIIiZbkaw@0I{gd87M^(K4= z(7|m@?ug*kf3o&PX?(G>oxfx0GcAByUAmiD zJu_izOq^)Z60m+rorc0}=VsF8#u`4Nbs1UU8cjoGwQ%HVaHsqg-#;4v)Smi2#(V!H zlf`l8o#aawF}=${o+oA%-v0O~?NYgXXQTf!s~;I^O9DkFzqQt8rqna;@1xq)SOh*8 zw+!=SS36JBbGqCFauqFm`7NluD<;w;#*saQzx`5SR?1;A4 z_&@m38{LsoaN-lk<)GKL{G^*^Qayeev)X zyZAh-b*nY`b8-cpocWQQ1l!mRfe{vC=`*Vd>BrZ-i^kqHNkbFg%A4B!SBN9Qoz@1@y)^UCw|Z8`A}|1=G&dl`KTS$ z4g!I$-MRgb_q-ZGUcFQ@0Dxmf^v^P|zQfDOh|rwfdJNk=I@b8Gd@JdDos+@%hS*#n zS=0$qg(UE*DLdi?tVv?=5HUwEQhsU5Uctj`Ug)Ge63+a1lDepLLH6ctgS*wO^=x(L z@6n$K5J;0GnVDK(YbH#td10m_nk`FTb5}@DLSxBN(ve$yB@k%LNJs6qF~Jux}IdUZIx+e{q6Ox-V4+W)f65Goeb&BD_eSfbfw=ioc`jK&oY24 z2Gy56^pX5~cj{LwEpo(kPFEtC&=c%IaHrS0yca_9W7!RUF!v)ttzQHG;#`SuMq7Od zYAdZ%&to2LmvtsLn<;RC3slS+-LgWsk1bAOy=hnGHFG~4ijKHPr)9i`pM^q){X9@G zRwJ~skM(Ee8v$=Rw0X1LfA8cMGX*wS#`EeWMKEaQ{Bw@W+}JVE-r@-J`qi~DMNQ=% z&x(N;#v&y0OTIG}Wvd+l6Imk&$x)9EbEn2ZE;g!D$sm8dT!1rCM5CtjOh)d*!=b-Z zD9Rb_GL7Ef*3I%Fag}K~)q8as%!`z{L;|ZfHn^Qy+~aFjs#whrr313oKO3{o*v%79 zeO+;=JYIzbgbV%@@27M(WnahnP9E;|EM`ut{||i?#iQHUZIPh6d7J*xH>tIkUI5G< zctnC3Mg3Fd6st1!$Ho21xpZC+UuoFK6(G8s6cTr&wb^?N68HyRFh#N<)@% zhg~om=Old*LK1e|-IZ+xyJy_)TV^a%(PtmdJOi=tpb0!`89|SAk1WUiq2j*VP9V8qFba?xsO^$OZhG7e4-~V=# zmpfk@!z9^t?SU41%qXokr{7XZW&ofl>ZtW65}pEohU$6=_r9$QCnpg8*lCF_on|O> zT?sk(kG)qp`NU;}22}KXNXIWLy9=aP7}!vl&m4q(v(TfHV{K8UEnD tJlJ8e8gOw?=K$W-{QtX#aNCjx@)CT9tZGP)Ao@lGpsJ{)P$6gi?tiFYTyg*a literal 0 HcmV?d00001 diff --git a/apex/docs/source/_templates/layout.html b/apex/docs/source/_templates/layout.html new file mode 100644 index 00000000..63dfed9c --- /dev/null +++ b/apex/docs/source/_templates/layout.html @@ -0,0 +1,51 @@ +{% extends "!layout.html" %} + {% block sidebartitle %} {{ super() }} + + + {% endblock %} + + {% block footer %} {{ super() }} + + + {% endblock %} diff --git a/apex/docs/source/advanced.rst b/apex/docs/source/advanced.rst new file mode 100644 index 00000000..d6623e62 --- /dev/null +++ b/apex/docs/source/advanced.rst @@ -0,0 +1,219 @@ +.. role:: hidden + :class: hidden-section + +Advanced Amp Usage +=================================== + +GANs +---- + +GANs are an interesting synthesis of several topics below. A `comprehensive example`_ +is under construction. + +.. _`comprehensive example`: + https://github.com/NVIDIA/apex/tree/master/examples/dcgan + +Gradient clipping +----------------- +Amp calls the params owned directly by the optimizer's ``param_groups`` the "master params." + +These master params may be fully or partially distinct from ``model.parameters()``. +For example, with `opt_level="O2"`_, ``amp.initialize`` casts most model params to FP16, +creates an FP32 master param outside the model for each newly-FP16 model param, +and updates the optimizer's ``param_groups`` to point to these FP32 params. + +The master params owned by the optimizer's ``param_groups`` may also fully coincide with the +model params, which is typically true for ``opt_level``\s ``O0``, ``O1``, and ``O3``. + +In all cases, correct practice is to clip the gradients of the params that are guaranteed to be +owned **by the optimizer's** ``param_groups``, instead of those retrieved via ``model.parameters()``. + +Also, if Amp uses loss scaling, gradients must be clipped after they have been unscaled +(which occurs during exit from the ``amp.scale_loss`` context manager). + +The following pattern should be correct for any ``opt_level``:: + + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + # Gradients are unscaled during context manager exit. + # Now it's safe to clip. Replace + # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) + # with + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_norm) + # or + torch.nn.utils.clip_grad_value_(amp.master_params(optimizer), max_) + +Note the use of the utility function ``amp.master_params(optimizer)``, +which returns a generator-expression that iterates over the +params in the optimizer's ``param_groups``. + +Also note that ``clip_grad_norm_(amp.master_params(optimizer), max_norm)`` is invoked +*instead of*, not *in addition to*, ``clip_grad_norm_(model.parameters(), max_norm)``. + +.. _`opt_level="O2"`: + https://nvidia.github.io/apex/amp.html#o2-fast-mixed-precision + +Custom/user-defined autograd functions +-------------------------------------- + +The old Amp API for `registering user functions`_ is still considered correct. Functions must +be registered before calling ``amp.initialize``. + +.. _`registering user functions`: + https://github.com/NVIDIA/apex/tree/master/apex/amp#annotating-user-functions + +Forcing particular layers/functions to a desired type +----------------------------------------------------- + +I'm still working on a generalizable exposure for this that won't require user-side code divergence +across different ``opt-level``\ s. + +Multiple models/optimizers/losses +--------------------------------- + +Initialization with multiple models/optimizers +********************************************** + +``amp.initialize``'s optimizer argument may be a single optimizer or a list of optimizers, +as long as the output you accept has the same type. +Similarly, the ``model`` argument may be a single model or a list of models, as long as the accepted +output matches. The following calls are all legal:: + + model, optim = amp.initialize(model, optim,...) + model, [optim0, optim1] = amp.initialize(model, [optim0, optim1],...) + [model0, model1], optim = amp.initialize([model0, model1], optim,...) + [model0, model1], [optim0, optim1] = amp.initialize([model0, model1], [optim0, optim1],...) + +Backward passes with multiple optimizers +**************************************** + +Whenever you invoke a backward pass, the ``amp.scale_loss`` context manager must receive +**all the optimizers that own any params for which the current backward pass is creating gradients.** +This is true even if each optimizer owns only some, but not all, of the params that are about to +receive gradients. + +If, for a given backward pass, there's only one optimizer whose params are about to receive gradients, +you may pass that optimizer directly to ``amp.scale_loss``. Otherwise, you must pass the +list of optimizers whose params are about to receive gradients. Example with 3 losses and 2 optimizers:: + + # loss0 accumulates gradients only into params owned by optim0: + with amp.scale_loss(loss0, optim0) as scaled_loss: + scaled_loss.backward() + + # loss1 accumulates gradients only into params owned by optim1: + with amp.scale_loss(loss1, optim1) as scaled_loss: + scaled_loss.backward() + + # loss2 accumulates gradients into some params owned by optim0 + # and some params owned by optim1 + with amp.scale_loss(loss2, [optim0, optim1]) as scaled_loss: + scaled_loss.backward() + +Optionally have Amp use a different loss scaler per-loss +******************************************************** + +By default, Amp maintains a single global loss scaler that will be used for all backward passes +(all invocations of ``with amp.scale_loss(...)``). No additional arguments to ``amp.initialize`` +or ``amp.scale_loss`` are required to use the global loss scaler. The code snippets above with +multiple optimizers/backward passes use the single global loss scaler under the hood, +and they should "just work." + +However, you can optionally tell Amp to maintain a loss scaler per-loss, which gives Amp increased +numerical flexibility. This is accomplished by supplying the ``num_losses`` argument to +``amp.initialize`` (which tells Amp how many backward passes you plan to invoke, and therefore +how many loss scalers Amp should create), then supplying the ``loss_id`` argument to each of your +backward passes (which tells Amp the loss scaler to use for this particular backward pass):: + + model, [optim0, optim1] = amp.initialize(model, [optim0, optim1], ..., num_losses=3) + + with amp.scale_loss(loss0, optim0, loss_id=0) as scaled_loss: + scaled_loss.backward() + + with amp.scale_loss(loss1, optim1, loss_id=1) as scaled_loss: + scaled_loss.backward() + + with amp.scale_loss(loss2, [optim0, optim1], loss_id=2) as scaled_loss: + scaled_loss.backward() + +``num_losses`` and ``loss_id``\ s should be specified purely based on the set of +losses/backward passes. The use of multiple optimizers, or association of single or +multiple optimizers with each backward pass, is unrelated. + +Gradient accumulation across iterations +--------------------------------------- + +The following should "just work," and properly accommodate multiple models/optimizers/losses, as well as +gradient clipping via the `instructions above`_:: + + # If your intent is to simulate a larger batch size using gradient accumulation, + # you can divide the loss by the number of accumulation iterations (so that gradients + # will be averaged over that many iterations): + loss = loss/iters_to_accumulate + + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + # Every iters_to_accumulate iterations, call step() and reset gradients: + if iter%iters_to_accumulate == 0: + # Gradient clipping if desired: + # torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), max_norm) + optimizer.step() + optimizer.zero_grad() + +As a minor performance optimization, you can pass ``delay_unscale=True`` +to ``amp.scale_loss`` until you're ready to ``step()``. You should only attempt ``delay_unscale=True`` +if you're sure you know what you're doing, because the interaction with gradient clipping and +multiple models/optimizers/losses can become tricky.:: + + if iter%iters_to_accumulate == 0: + # Every iters_to_accumulate iterations, unscale and step + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + optimizer.zero_grad() + else: + # Otherwise, accumulate gradients, don't unscale or step. + with amp.scale_loss(loss, optimizer, delay_unscale=True) as scaled_loss: + scaled_loss.backward() + +.. _`instructions above`: + https://nvidia.github.io/apex/advanced.html#gradient-clipping + +Custom data batch types +----------------------- + +The intention of Amp is that you never need to cast your input data manually, regardless of +``opt_level``. Amp accomplishes this by patching any models' ``forward`` methods to cast +incoming data appropriately for the ``opt_level``. But to cast incoming data, +Amp needs to know how. The patched ``forward`` will recognize and cast floating-point Tensors +(non-floating-point Tensors like IntTensors are not touched) and +Python containers of floating-point Tensors. However, if you wrap your Tensors in a custom class, +the casting logic doesn't know how to drill +through the tough custom shell to access and cast the juicy Tensor meat within. You need to tell +Amp how to cast your custom batch class, by assigning it a ``to`` method that accepts a ``torch.dtype`` +(e.g., ``torch.float16`` or ``torch.float32``) and returns an instance of the custom batch cast to +``dtype``. The patched ``forward`` checks for the presence of your ``to`` method, and will +invoke it with the correct type for the ``opt_level``. + +Example:: + + class CustomData(object): + def __init__(self): + self.tensor = torch.cuda.FloatTensor([1,2,3]) + + def to(self, dtype): + self.tensor = self.tensor.to(dtype) + return self + +.. warning:: + + Amp also forwards numpy ndarrays without casting them. If you send input data as a raw, unwrapped + ndarray, then later use it to create a Tensor within your ``model.forward``, this Tensor's type will + not depend on the ``opt_level``, and may or may not be correct. Users are encouraged to pass + castable data inputs (Tensors, collections of Tensors, or custom classes with a ``to`` method) + wherever possible. + +.. note:: + + Amp does not call ``.cuda()`` on any Tensors for you. Amp assumes that your original script + is already set up to move Tensors from the host to the device as needed. diff --git a/apex/docs/source/amp.rst b/apex/docs/source/amp.rst new file mode 100644 index 00000000..4bc14051 --- /dev/null +++ b/apex/docs/source/amp.rst @@ -0,0 +1,288 @@ +.. role:: hidden + :class: hidden-section + +apex.amp +=================================== + +This page documents the updated API for Amp (Automatic Mixed Precision), +a tool to enable Tensor Core-accelerated training in only 3 lines of Python. + +A `runnable, comprehensive Imagenet example`_ demonstrating good practices can be found +on the Github page. + +GANs are a tricky case that many people have requested. A `comprehensive DCGAN example`_ +is under construction. + +If you already implemented Amp based on the instructions below, but it isn't behaving as expected, +please review `Advanced Amp Usage`_ to see if any topics match your use case. If that doesn't help, +`file an issue`_. + +.. _`file an issue`: + https://github.com/NVIDIA/apex/issues + +``opt_level``\ s and Properties +------------------------------- + +Amp allows users to easily experiment with different pure and mixed precision modes. +Commonly-used default modes are chosen by +selecting an "optimization level" or ``opt_level``; each ``opt_level`` establishes a set of +properties that govern Amp's implementation of pure or mixed precision training. +Finer-grained control of how a given ``opt_level`` behaves can be achieved by passing values for +particular properties directly to ``amp.initialize``. These manually specified values +override the defaults established by the ``opt_level``. + +Example:: + + # Declare model and optimizer as usual, with default (FP32) precision + model = torch.nn.Linear(D_in, D_out).cuda() + optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + + # Allow Amp to perform casts as required by the opt_level + model, optimizer = amp.initialize(model, optimizer, opt_level="O1") + ... + # loss.backward() becomes: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + ... + +Users **should not** manually cast their model or data to ``.half()``, regardless of what ``opt_level`` +or properties are chosen. Amp intends that users start with an existing default (FP32) script, +add the three lines corresponding to the Amp API, and begin training with mixed precision. +Amp can also be disabled, in which case the original script will behave exactly as it used to. +In this way, there's no risk adhering to the Amp API, and a lot of potential performance benefit. + +.. note:: + Because it's never necessary to manually cast your model (aside from the call ``amp.initialize``) + or input data, a script that adheres to the new API + can switch between different ``opt-level``\ s without having to make any other changes. + +.. _`runnable, comprehensive Imagenet example`: + https://github.com/NVIDIA/apex/tree/master/examples/imagenet + +.. _`comprehensive DCGAN example`: + https://github.com/NVIDIA/apex/tree/master/examples/dcgan + +.. _`Advanced Amp Usage`: + https://nvidia.github.io/apex/advanced.html + +Properties +********** + +Currently, the under-the-hood properties that govern pure or mixed precision training are the following: + +- ``cast_model_type``: Casts your model's parameters and buffers to the desired type. +- ``patch_torch_functions``: Patch all Torch functions and Tensor methods to perform Tensor Core-friendly ops like GEMMs and convolutions in FP16, and any ops that benefit from FP32 precision in FP32. +- ``keep_batchnorm_fp32``: To enhance precision and enable cudnn batchnorm (which improves performance), it's often beneficial to keep batchnorm weights in FP32 even if the rest of the model is FP16. +- ``master_weights``: Maintain FP32 master weights to accompany any FP16 model weights. FP32 master weights are stepped by the optimizer to enhance precision and capture small gradients. +- ``loss_scale``: If ``loss_scale`` is a float value, use this value as the static (fixed) loss scale. If ``loss_scale`` is the string ``"dynamic"``, adaptively adjust the loss scale over time. Dynamic loss scale adjustments are performed by Amp automatically. + +Again, you often don't need to specify these properties by hand. Instead, select an ``opt_level``, +which will set them up for you. After selecting an ``opt_level``, you can optionally pass property +kwargs as manual overrides. + +If you attempt to override a property that does not make sense for the selected ``opt_level``, +Amp will raise an error with an explanation. For example, selecting ``opt_level="O1"`` combined with +the override ``master_weights=True`` does not make sense. ``O1`` inserts casts +around Torch functions rather than model weights. Data, activations, and weights are recast +out-of-place on the fly as they flow through patched functions. Therefore, the model weights themselves +can (and should) remain FP32, and there is no need to maintain separate FP32 master weights. + +``opt_level``\ s +**************** + +Recognized ``opt_level``\ s are ``"O0"``, ``"O1"``, ``"O2"``, and ``"O3"``. + +``O0`` and ``O3`` are not true mixed precision, but they are useful for establishing accuracy and +speed baselines, respectively. + +``O1`` and ``O2`` are different implementations of mixed precision. Try both, and see +what gives the best speedup and accuracy for your model. + +``O0``: FP32 training +^^^^^^^^^^^^^^^^^^^^^^ +Your incoming model should be FP32 already, so this is likely a no-op. +``O0`` can be useful to establish an accuracy baseline. + +| Default properties set by ``O0``: +| ``cast_model_type=torch.float32`` +| ``patch_torch_functions=False`` +| ``keep_batchnorm_fp32=None`` (effectively, "not applicable," everything is FP32) +| ``master_weights=False`` +| ``loss_scale=1.0`` +| +| + +``O1``: Mixed Precision (recommended for typical use) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Patch all Torch functions and Tensor methods to cast their inputs according to a whitelist-blacklist +model. Whitelist ops (for example, Tensor Core-friendly ops like GEMMs and convolutions) are performed +in FP16. Blacklist ops that benefit from FP32 precision (for example, softmax) +are performed in FP32. ``O1`` also uses dynamic loss scaling, unless overridden. + +| Default properties set by ``O1``: +| ``cast_model_type=None`` (not applicable) +| ``patch_torch_functions=True`` +| ``keep_batchnorm_fp32=None`` (again, not applicable, all model weights remain FP32) +| ``master_weights=None`` (not applicable, model weights remain FP32) +| ``loss_scale="dynamic"`` +| +| + +``O2``: "Almost FP16" Mixed Precision +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +``O2`` casts the model weights to FP16, +patches the model's ``forward`` method to cast input +data to FP16, keeps batchnorms in FP32, maintains FP32 master weights, +updates the optimizer's ``param_groups`` so that the ``optimizer.step()`` +acts directly on the FP32 weights (followed by FP32 master weight->FP16 model weight +copies if necessary), +and implements dynamic loss scaling (unless overridden). +Unlike ``O1``, ``O2`` does not patch Torch functions or Tensor methods. + +| Default properties set by ``O2``: +| ``cast_model_type=torch.float16`` +| ``patch_torch_functions=False`` +| ``keep_batchnorm_fp32=True`` +| ``master_weights=True`` +| ``loss_scale="dynamic"`` +| +| + +``O3``: FP16 training +^^^^^^^^^^^^^^^^^^^^^^ +``O3`` may not achieve the stability of the true mixed precision options ``O1`` and ``O2``. +However, it can be useful to establish a speed baseline for your model, against which +the performance of ``O1`` and ``O2`` can be compared. If your model uses batch normalization, +to establish "speed of light" you can try ``O3`` with the additional property override +``keep_batchnorm_fp32=True`` (which enables cudnn batchnorm, as stated earlier). + +| Default properties set by ``O3``: +| ``cast_model_type=torch.float16`` +| ``patch_torch_functions=False`` +| ``keep_batchnorm_fp32=False`` +| ``master_weights=False`` +| ``loss_scale=1.0`` +| +| + +Unified API +----------- + +.. automodule:: apex.amp +.. currentmodule:: apex.amp + +.. autofunction:: initialize + +.. autofunction:: scale_loss + +.. autofunction:: master_params + +Checkpointing +------------- + +To properly save and load your amp training, we introduce the ``amp.state_dict()``, which contains all ``loss_scaler``\ s and their corresponding unskipped steps, as well as ``amp.load_state_dict()`` to restore these attributes. + +In order to get bitwise accuracy, we recommend the following workflow:: + + # Initialization + opt_level = 'O1' + model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level) + + # Train your model + ... + + # Save checkpoint + checkpoint = { + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict() + } + torch.save(checkpoint, 'amp_checkpoint.pt') + ... + + # Restore + model = ... + optimizer = ... + checkpoint = torch.load('amp_checkpoint.pt') + + model, optimizer = amp.initialize(model, optimizer, opt_level=opt_level) + model.load_state_dict(checkpoint['model']) + optimizer.load_state_dict(checkpoint['optimizer']) + amp.load_state_dict(checkpoint['amp']) + + # Continue training + ... + +Note that we recommend restoring the model using the same ``opt_level``. Also note that we recommend calling the ``load_state_dict`` methods after ``amp.initialize``. + +Advanced use cases +------------------ + +The unified Amp API supports gradient accumulation across iterations, +multiple backward passes per iteration, multiple models/optimizers, +custom/user-defined autograd functions, and custom data batch classes. Gradient clipping and GANs also +require special treatment, but this treatment does not need to change +for different ``opt_level``\ s. Further details can be found here: + +.. toctree:: + :maxdepth: 1 + + advanced + +Transition guide for old API users +---------------------------------- + +We strongly encourage moving to the new Amp API, because it's more versatile, easier to use, and future proof. The original :class:`FP16_Optimizer` and the old "Amp" API are deprecated, and subject to removal at at any time. + +For users of the old "Amp" API +****************************** + +In the new API, ``opt-level O1`` performs the same patching of the Torch namespace as the old thing +called "Amp." +However, the new API allows static or dynamic loss scaling, while the old API only allowed dynamic loss scaling. + +In the new API, the old call to ``amp_handle = amp.init()``, and the returned ``amp_handle``, are no +longer exposed or necessary. The new ``amp.initialize()`` does the duty of ``amp.init()`` (and more). +Therefore, any existing calls to ``amp_handle = amp.init()`` should be deleted. + +The functions formerly exposed through ``amp_handle`` are now free +functions accessible through the ``amp`` module. + +The backward context manager must be changed accordingly:: + + # old API + with amp_handle.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + -> + # new API + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + +For now, the deprecated "Amp" API documentation can still be found on the Github README: https://github.com/NVIDIA/apex/tree/master/apex/amp. The old API calls that `annotate user functions`_ to run +with a particular precision are still honored by the new API. + +.. _`annotate user functions`: + https://github.com/NVIDIA/apex/tree/master/apex/amp#annotating-user-functions + + +For users of the old FP16_Optimizer +*********************************** + +``opt-level O2`` is equivalent to :class:`FP16_Optimizer` with ``dynamic_loss_scale=True``. +Once again, the backward pass must be changed to the unified version:: + + optimizer.backward(loss) + -> + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + +One annoying aspect of FP16_Optimizer was that the user had to manually convert their model to half +(either by calling ``.half()`` on it, or using a function or module wrapper from +``apex.fp16_utils``), and also manually call ``.half()`` on input data. **Neither of these are +necessary in the new API. No matter what --opt-level +you choose, you can and should simply build your model and pass input data in the default FP32 format.** +The new Amp API will perform the right conversions during +``model, optimizer = amp.initialize(model, optimizer, opt_level=....)`` based on the ``--opt-level`` +and any overridden flags. Floating point input data may be FP32 or FP16, but you may as well just +let it be FP16, because the ``model`` returned by ``amp.initialize`` will have its ``forward`` +method patched to cast the input data appropriately. diff --git a/apex/docs/source/conf.py b/apex/docs/source/conf.py new file mode 100644 index 00000000..4477a28e --- /dev/null +++ b/apex/docs/source/conf.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# PyTorch documentation build configuration file, created by +# sphinx-quickstart on Fri Dec 23 13:31:47 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('../../apex/parallel/')) +import apex +# import multiproc +import sphinx_rtd_theme + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'sphinx.ext.extlinks', +] + +napoleon_use_ivar = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'Apex' +copyright = '2018' +author = 'Christian Sarofeen, Natalia Gimelshein, Michael Carilli, Raul Puri' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# TODO: change to [:2] at v1.0 +# version = 'master (' + torch.__version__ + ' )' +version = '0.1' +# The full version, including alpha/beta/rc tags. +# TODO: verify this works as expected +release = '0.1.0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = { + 'collapse_navigation': False, + 'display_version': True, + 'logo_only': True, +} + +# html_logo = '_static/img/nv-pytorch2.png' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# html_style_path = 'css/pytorch_theme.css' +html_context = { + 'css_files': [ + 'https://fonts.googleapis.com/css?family=Lato', + '_static/css/pytorch_theme.css' + ], +} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'PyTorchdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'apex.tex', 'Apex Documentation', + 'Torch Contributors', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'Apex', 'Apex Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'Apex', 'Apex Documentation', + author, 'Apex', 'One line description of project.', + 'Miscellaneous'), +] + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'numpy': ('http://docs.scipy.org/doc/numpy/', None), +} + +# -- A patch that prevents Sphinx from cross-referencing ivar tags ------- +# See http://stackoverflow.com/a/41184353/3343043 + +from docutils import nodes +from sphinx.util.docfields import TypedField +from sphinx import addnodes + + +def patched_make_field(self, types, domain, items, **kw): + # `kw` catches `env=None` needed for newer sphinx while maintaining + # backwards compatibility when passed along further down! + + # type: (List, unicode, Tuple) -> nodes.field + def handle_item(fieldarg, content): + par = nodes.paragraph() + par += addnodes.literal_strong('', fieldarg) # Patch: this line added + # par.extend(self.make_xrefs(self.rolename, domain, fieldarg, + # addnodes.literal_strong)) + if fieldarg in types: + par += nodes.Text(' (') + # NOTE: using .pop() here to prevent a single type node to be + # inserted twice into the doctree, which leads to + # inconsistencies later when references are resolved + fieldtype = types.pop(fieldarg) + if len(fieldtype) == 1 and isinstance(fieldtype[0], nodes.Text): + typename = u''.join(n.astext() for n in fieldtype) + typename = typename.replace('int', 'python:int') + typename = typename.replace('long', 'python:long') + typename = typename.replace('float', 'python:float') + typename = typename.replace('type', 'python:type') + par.extend(self.make_xrefs(self.typerolename, domain, typename, + addnodes.literal_emphasis, **kw)) + else: + par += fieldtype + par += nodes.Text(')') + par += nodes.Text(' -- ') + par += content + return par + + fieldname = nodes.field_name('', self.label) + if len(items) == 1 and self.can_collapse: + fieldarg, content = items[0] + bodynode = handle_item(fieldarg, content) + else: + bodynode = self.list_type() + for fieldarg, content in items: + bodynode += nodes.list_item('', handle_item(fieldarg, content)) + fieldbody = nodes.field_body('', bodynode) + return nodes.field('', fieldname, fieldbody) + +TypedField.make_field = patched_make_field diff --git a/apex/docs/source/fp16_utils.rst b/apex/docs/source/fp16_utils.rst new file mode 100644 index 00000000..b6b3da5f --- /dev/null +++ b/apex/docs/source/fp16_utils.rst @@ -0,0 +1,59 @@ +.. role:: hidden + :class: hidden-section + +apex.fp16_utils +=================================== + +This submodule contains utilities designed to streamline the mixed precision training recipe +presented by NVIDIA `on Parallel Forall`_ and in GTC 2018 Sessions +`Training Neural Networks with Mixed Precision: Theory and Practice`_ and +`Training Neural Networks with Mixed Precision: Real Examples`_. +For Pytorch users, Real Examples in particular is recommended. + +Full runnable Python scripts demonstrating ``apex.fp16_utils`` +can be found on the Github page: + +| `Simple FP16_Optimizer demos`_ +| +| `Distributed Mixed Precision Training with imagenet`_ +| +| `Mixed Precision Training with word_language_model`_ +| +| + +.. _`on Parallel Forall`: + https://devblogs.nvidia.com/mixed-precision-training-deep-neural-networks/ +.. _`Training Neural Networks with Mixed Precision: Theory and Practice`: + http://on-demand.gputechconf.com/gtc/2018/video/S8923/ +.. _`Training Neural Networks with Mixed Precision: Real Examples`: + http://on-demand.gputechconf.com/gtc/2018/video/S81012/ +.. _`Simple FP16_Optimizer demos`: + https://github.com/NVIDIA/apex/tree/master/examples/FP16_Optimizer_simple +.. _`Distributed Mixed Precision Training with imagenet`: + https://github.com/NVIDIA/apex/tree/master/examples/imagenet +.. _`Mixed Precision Training with word_language_model`: + https://github.com/NVIDIA/apex/tree/master/examples/word_language_model + +.. automodule:: apex.fp16_utils +.. currentmodule:: apex.fp16_utils + +Automatic management of master params + loss scaling +---------------------------------------------------- + +.. autoclass:: FP16_Optimizer + :members: + +.. autoclass:: LossScaler + :members: + +.. autoclass:: DynamicLossScaler + :members: + +Manual master parameter management +---------------------------------- + +.. autofunction:: prep_param_lists + +.. autofunction:: master_params_to_model_params + +.. autofunction:: model_grads_to_master_grads diff --git a/apex/docs/source/index.rst b/apex/docs/source/index.rst new file mode 100644 index 00000000..c7efc168 --- /dev/null +++ b/apex/docs/source/index.rst @@ -0,0 +1,53 @@ +.. PyTorch documentation master file, created by + sphinx-quickstart on Fri Dec 23 13:31:47 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +:github_url: https://github.com/nvidia/apex + +Apex (A PyTorch Extension) +=================================== + +This site contains the API documentation for Apex (https://github.com/nvidia/apex), +a Pytorch extension with NVIDIA-maintained utilities to streamline mixed precision and distributed training. Some of the code here will be included in upstream Pytorch eventually. The intention of Apex is to make up-to-date utilities available to users as quickly as possible. + +Installation instructions can be found here: https://github.com/NVIDIA/apex#quick-start. + +Some other useful material, including GTC 2019 and Pytorch DevCon 2019 Slides, can be found here: https://github.com/mcarilli/mixed_precision_references. + +.. toctree:: + :maxdepth: 1 + :caption: AMP: Automatic Mixed Precision + + amp + +.. toctree:: + :maxdepth: 1 + :caption: Distributed Training + + parallel + +.. toctree:: + :maxdepth: 1 + :caption: Fused Optimizers + + optimizers + +.. toctree:: + :maxdepth: 1 + :caption: Fused Layer Norm + + layernorm + +.. .. toctree:: + :maxdepth: 1 + :caption: Deprecated mixed precision API + fp16_util + +.. RNN + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` diff --git a/apex/docs/source/layernorm.rst b/apex/docs/source/layernorm.rst new file mode 100644 index 00000000..6eedb4ed --- /dev/null +++ b/apex/docs/source/layernorm.rst @@ -0,0 +1,17 @@ +.. role:: hidden + :class: hidden-section + +apex.normalization.fused_layer_norm +=================================== + +.. automodule:: apex.normalization +.. currentmodule:: apex.normalization + +.. FusedAdam + ---------- + +.. autoclass:: FusedLayerNorm + :members: + +.. autoclass:: FusedRMSNorm + :members: diff --git a/apex/docs/source/optimizers.rst b/apex/docs/source/optimizers.rst new file mode 100644 index 00000000..407f0770 --- /dev/null +++ b/apex/docs/source/optimizers.rst @@ -0,0 +1,23 @@ +.. role:: hidden + :class: hidden-section + +apex.optimizers +=================================== + +.. automodule:: apex.optimizers +.. currentmodule:: apex.optimizers + +.. FusedAdam + ---------- + +.. autoclass:: FusedAdam + :members: + +.. autoclass:: FusedLAMB + :members: + +.. autoclass:: FusedNovoGrad + :members: + +.. autoclass:: FusedSGD + :members: diff --git a/apex/docs/source/parallel.rst b/apex/docs/source/parallel.rst new file mode 100644 index 00000000..73759eeb --- /dev/null +++ b/apex/docs/source/parallel.rst @@ -0,0 +1,25 @@ +.. role:: hidden + :class: hidden-section + +apex.parallel +=================================== + +.. automodule:: apex.parallel +.. currentmodule:: apex.parallel + +.. DistributedDataParallel + ---------- + +.. autoclass:: DistributedDataParallel + :members: + +.. autoclass:: Reducer + :members: + +.. autoclass:: SyncBatchNorm + :members: + +Utility functions +---------------------------------- + +.. autofunction:: convert_syncbn_model diff --git a/apex/examples/README.md b/apex/examples/README.md new file mode 100644 index 00000000..6cb92318 --- /dev/null +++ b/apex/examples/README.md @@ -0,0 +1,4 @@ +This directory contains examples illustrating Apex mixed precision and distributed tools. + +**Note for users of the pre-unification API**: +`deprecated_api` contains examples illustrating the old (pre-unified) APIs. These APIs will be removed soon, and users are strongly encouraged to switch. The separate mixed precision tools called `Amp` and `FP16_Optimizer` in the old API are exposed via different flags/optimization levels in the new API. diff --git a/apex/examples/dcgan/README.md b/apex/examples/dcgan/README.md new file mode 100644 index 00000000..9fc896cb --- /dev/null +++ b/apex/examples/dcgan/README.md @@ -0,0 +1,41 @@ +# Mixed Precision DCGAN Training in PyTorch + +`main_amp.py` is based on [https://github.com/pytorch/examples/tree/master/dcgan](https://github.com/pytorch/examples/tree/master/dcgan). +It implements Automatic Mixed Precision (Amp) training of the DCGAN example for different datasets. Command-line flags forwarded to `amp.initialize` are used to easily manipulate and switch between various pure and mixed precision "optimization levels" or `opt_level`s. For a detailed explanation of `opt_level`s, see the [updated API guide](https://nvidia.github.io/apex/amp.html). + +We introduce these changes to the PyTorch DCGAN example as described in the [Multiple models/optimizers/losses](https://nvidia.github.io/apex/advanced.html#multiple-models-optimizers-losses) section of the documentation:: +``` +# Added after models and optimizers construction +[netD, netG], [optimizerD, optimizerG] = amp.initialize( + [netD, netG], [optimizerD, optimizerG], opt_level=opt.opt_level, num_losses=3) +... +# loss.backward() changed to: +with amp.scale_loss(errD_real, optimizerD, loss_id=0) as errD_real_scaled: + errD_real_scaled.backward() +... +with amp.scale_loss(errD_fake, optimizerD, loss_id=1) as errD_fake_scaled: + errD_fake_scaled.backward() +... +with amp.scale_loss(errG, optimizerG, loss_id=2) as errG_scaled: + errG_scaled.backward() +``` + +Note that we use different `loss_scalers` for each computed loss. +Using a separate loss scaler per loss is [optional, not required](https://nvidia.github.io/apex/advanced.html#optionally-have-amp-use-a-different-loss-scaler-per-loss). + +To improve the numerical stability, we swapped `nn.Sigmoid() + nn.BCELoss()` to `nn.BCEWithLogitsLoss()`. + +With the new Amp API **you never need to explicitly convert your model, or the input data, to half().** + +"Pure FP32" training: +``` +$ python main_amp.py --opt_level O0 +``` +Recommended mixed precision training: +``` +$ python main_amp.py --opt_level O1 +``` + +Have a look at the original [DCGAN example](https://github.com/pytorch/examples/tree/master/dcgan) for more information about the used arguments. + +To enable mixed precision training, we introduce the `--opt_level` argument. diff --git a/apex/examples/dcgan/main_amp.py b/apex/examples/dcgan/main_amp.py new file mode 100644 index 00000000..be1a2894 --- /dev/null +++ b/apex/examples/dcgan/main_amp.py @@ -0,0 +1,274 @@ +from __future__ import print_function +import argparse +import os +import random +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.optim as optim +import torch.utils.data +import torchvision.datasets as dset +import torchvision.transforms as transforms +import torchvision.utils as vutils + +try: + from apex import amp +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") + + +parser = argparse.ArgumentParser() +parser.add_argument('--dataset', default='cifar10', help='cifar10 | lsun | mnist |imagenet | folder | lfw | fake') +parser.add_argument('--dataroot', default='./', help='path to dataset') +parser.add_argument('--workers', type=int, help='number of data loading workers', default=2) +parser.add_argument('--batchSize', type=int, default=64, help='input batch size') +parser.add_argument('--imageSize', type=int, default=64, help='the height / width of the input image to network') +parser.add_argument('--nz', type=int, default=100, help='size of the latent z vector') +parser.add_argument('--ngf', type=int, default=64) +parser.add_argument('--ndf', type=int, default=64) +parser.add_argument('--niter', type=int, default=25, help='number of epochs to train for') +parser.add_argument('--lr', type=float, default=0.0002, help='learning rate, default=0.0002') +parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5') +parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use') +parser.add_argument('--netG', default='', help="path to netG (to continue training)") +parser.add_argument('--netD', default='', help="path to netD (to continue training)") +parser.add_argument('--outf', default='.', help='folder to output images and model checkpoints') +parser.add_argument('--manualSeed', type=int, help='manual seed') +parser.add_argument('--classes', default='bedroom', help='comma separated list of classes for the lsun data set') +parser.add_argument('--opt_level', default='O1', help='amp opt_level, default="O1"') + +opt = parser.parse_args() +print(opt) + + +try: + os.makedirs(opt.outf) +except OSError: + pass + +if opt.manualSeed is None: + opt.manualSeed = 2809 +print("Random Seed: ", opt.manualSeed) +random.seed(opt.manualSeed) +torch.manual_seed(opt.manualSeed) + +cudnn.benchmark = True + + +if opt.dataset in ['imagenet', 'folder', 'lfw']: + # folder dataset + dataset = dset.ImageFolder(root=opt.dataroot, + transform=transforms.Compose([ + transforms.Resize(opt.imageSize), + transforms.CenterCrop(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ])) + nc=3 +elif opt.dataset == 'lsun': + classes = [ c + '_train' for c in opt.classes.split(',')] + dataset = dset.LSUN(root=opt.dataroot, classes=classes, + transform=transforms.Compose([ + transforms.Resize(opt.imageSize), + transforms.CenterCrop(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ])) + nc=3 +elif opt.dataset == 'cifar10': + dataset = dset.CIFAR10(root=opt.dataroot, download=True, + transform=transforms.Compose([ + transforms.Resize(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), + ])) + nc=3 + +elif opt.dataset == 'mnist': + dataset = dset.MNIST(root=opt.dataroot, download=True, + transform=transforms.Compose([ + transforms.Resize(opt.imageSize), + transforms.ToTensor(), + transforms.Normalize((0.5,), (0.5,)), + ])) + nc=1 + +elif opt.dataset == 'fake': + dataset = dset.FakeData(image_size=(3, opt.imageSize, opt.imageSize), + transform=transforms.ToTensor()) + nc=3 + +assert dataset +dataloader = torch.utils.data.DataLoader(dataset, batch_size=opt.batchSize, + shuffle=True, num_workers=int(opt.workers)) + +device = torch.device("cuda:0") +ngpu = int(opt.ngpu) +nz = int(opt.nz) +ngf = int(opt.ngf) +ndf = int(opt.ndf) + + +# custom weights initialization called on netG and netD +def weights_init(m): + classname = m.__class__.__name__ + if classname.find('Conv') != -1: + m.weight.data.normal_(0.0, 0.02) + elif classname.find('BatchNorm') != -1: + m.weight.data.normal_(1.0, 0.02) + m.bias.data.fill_(0) + + +class Generator(nn.Module): + def __init__(self, ngpu): + super(Generator, self).__init__() + self.ngpu = ngpu + self.main = nn.Sequential( + # input is Z, going into a convolution + nn.ConvTranspose2d( nz, ngf * 8, 4, 1, 0, bias=False), + nn.BatchNorm2d(ngf * 8), + nn.ReLU(True), + # state size. (ngf*8) x 4 x 4 + nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf * 4), + nn.ReLU(True), + # state size. (ngf*4) x 8 x 8 + nn.ConvTranspose2d(ngf * 4, ngf * 2, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf * 2), + nn.ReLU(True), + # state size. (ngf*2) x 16 x 16 + nn.ConvTranspose2d(ngf * 2, ngf, 4, 2, 1, bias=False), + nn.BatchNorm2d(ngf), + nn.ReLU(True), + # state size. (ngf) x 32 x 32 + nn.ConvTranspose2d( ngf, nc, 4, 2, 1, bias=False), + nn.Tanh() + # state size. (nc) x 64 x 64 + ) + + def forward(self, input): + if input.is_cuda and self.ngpu > 1: + output = nn.parallel.data_parallel(self.main, input, range(self.ngpu)) + else: + output = self.main(input) + return output + + +netG = Generator(ngpu).to(device) +netG.apply(weights_init) +if opt.netG != '': + netG.load_state_dict(torch.load(opt.netG)) +print(netG) + + +class Discriminator(nn.Module): + def __init__(self, ngpu): + super(Discriminator, self).__init__() + self.ngpu = ngpu + self.main = nn.Sequential( + # input is (nc) x 64 x 64 + nn.Conv2d(nc, ndf, 4, 2, 1, bias=False), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf) x 32 x 32 + nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 2), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*2) x 16 x 16 + nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 4), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*4) x 8 x 8 + nn.Conv2d(ndf * 4, ndf * 8, 4, 2, 1, bias=False), + nn.BatchNorm2d(ndf * 8), + nn.LeakyReLU(0.2, inplace=True), + # state size. (ndf*8) x 4 x 4 + nn.Conv2d(ndf * 8, 1, 4, 1, 0, bias=False), + ) + + def forward(self, input): + if input.is_cuda and self.ngpu > 1: + output = nn.parallel.data_parallel(self.main, input, range(self.ngpu)) + else: + output = self.main(input) + + return output.view(-1, 1).squeeze(1) + + +netD = Discriminator(ngpu).to(device) +netD.apply(weights_init) +if opt.netD != '': + netD.load_state_dict(torch.load(opt.netD)) +print(netD) + +criterion = nn.BCEWithLogitsLoss() + +fixed_noise = torch.randn(opt.batchSize, nz, 1, 1, device=device) +real_label = 1 +fake_label = 0 + +# setup optimizer +optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) +optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) + +[netD, netG], [optimizerD, optimizerG] = amp.initialize( + [netD, netG], [optimizerD, optimizerG], opt_level=opt.opt_level, num_losses=3) + +for epoch in range(opt.niter): + for i, data in enumerate(dataloader, 0): + ############################ + # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z))) + ########################### + # train with real + netD.zero_grad() + real_cpu = data[0].to(device) + batch_size = real_cpu.size(0) + label = torch.full((batch_size,), real_label, device=device) + + output = netD(real_cpu) + errD_real = criterion(output, label) + with amp.scale_loss(errD_real, optimizerD, loss_id=0) as errD_real_scaled: + errD_real_scaled.backward() + D_x = output.mean().item() + + # train with fake + noise = torch.randn(batch_size, nz, 1, 1, device=device) + fake = netG(noise) + label.fill_(fake_label) + output = netD(fake.detach()) + errD_fake = criterion(output, label) + with amp.scale_loss(errD_fake, optimizerD, loss_id=1) as errD_fake_scaled: + errD_fake_scaled.backward() + D_G_z1 = output.mean().item() + errD = errD_real + errD_fake + optimizerD.step() + + ############################ + # (2) Update G network: maximize log(D(G(z))) + ########################### + netG.zero_grad() + label.fill_(real_label) # fake labels are real for generator cost + output = netD(fake) + errG = criterion(output, label) + with amp.scale_loss(errG, optimizerG, loss_id=2) as errG_scaled: + errG_scaled.backward() + D_G_z2 = output.mean().item() + optimizerG.step() + + print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)): %.4f / %.4f' + % (epoch, opt.niter, i, len(dataloader), + errD.item(), errG.item(), D_x, D_G_z1, D_G_z2)) + if i % 100 == 0: + vutils.save_image(real_cpu, + '%s/real_samples.png' % opt.outf, + normalize=True) + fake = netG(fixed_noise) + vutils.save_image(fake.detach(), + '%s/amp_fake_samples_epoch_%03d.png' % (opt.outf, epoch), + normalize=True) + + # do checkpointing + torch.save(netG.state_dict(), '%s/netG_epoch_%d.pth' % (opt.outf, epoch)) + torch.save(netD.state_dict(), '%s/netD_epoch_%d.pth' % (opt.outf, epoch)) + + diff --git a/apex/examples/docker/Dockerfile b/apex/examples/docker/Dockerfile new file mode 100644 index 00000000..5f5469ac --- /dev/null +++ b/apex/examples/docker/Dockerfile @@ -0,0 +1,16 @@ +# Base image must at least have pytorch and CUDA installed. +ARG BASE_IMAGE=nvcr.io/nvidia/pytorch:23.03-py3 +FROM $BASE_IMAGE +ARG BASE_IMAGE +RUN echo "Installing Apex on top of ${BASE_IMAGE}" +# make sure we don't overwrite some existing directory called "apex" +WORKDIR /tmp/unique_for_apex +# uninstall Apex if present, twice to make absolutely sure :) +RUN pip uninstall -y apex || : +RUN pip uninstall -y apex || : +# SHA is something the user can touch to force recreation of this Docker layer, +# and therefore force cloning of the latest version of Apex +RUN SHA=ToUcHMe git clone https://github.com/NVIDIA/apex.git +WORKDIR /tmp/unique_for_apex/apex +RUN pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . +WORKDIR /workspace diff --git a/apex/examples/docker/README.md b/apex/examples/docker/README.md new file mode 100644 index 00000000..3969af61 --- /dev/null +++ b/apex/examples/docker/README.md @@ -0,0 +1,40 @@ +## Option 1: Create a new container with Apex + +**Dockerfile** installs the latest Apex on top of an existing image. Run +``` +docker build -t new_image_with_apex . +``` +By default, **Dockerfile** uses NVIDIA's Pytorch container as the base image, +which requires an NVIDIA GPU Cloud (NGC) account. If you don't have an NGC account, you can sign up for free by following the instructions [here](https://docs.nvidia.com/ngc/ngc-getting-started-guide/index.html#generating-api-key). + +Alternatively, you can supply your own base image via the `BASE_IMAGE` build-arg. +`BASE_IMAGE` must have Pytorch and Cuda installed. For example, any +`-devel` image for Pytorch 1.0 and later from the +[official Pytorch Dockerhub](https://hub.docker.com/r/pytorch/pytorch) may be used: +``` +docker build --build-arg BASE_IMAGE=1.3-cuda10.1-cudnn7-devel -t new_image_with_apex . +``` + +If you want to rebuild your image, and force the latest Apex to be cloned and installed, make any small change to the `SHA` variable in **Dockerfile**. + +**Warning:** +Currently, the non-`-devel` images on Pytorch Dockerhub do not contain the Cuda compiler `nvcc`. Therefore, +images whose name does not contain `-devel` are not eligible candidates for `BASE_IMAGE`. + +### Running your Apex container + +Like any Cuda-enabled Pytorch container, a container with Apex should be run via [nvidia-docker](https://github.com/NVIDIA/nvidia-docker), for example: +``` +docker run --runtime=nvidia -it --rm --ipc=host new_image_with_apex +``` + +## Option 2: Install Apex in a running container + +Instead of building a new container, it is also a viable option to `git clone https://github.com/NVIDIA/apex.git` on bare metal, mount the Apex repo into your container at launch by running, for example, +``` +docker run --runtime=nvidia -it --rm --ipc=host -v /bare/metal/apex:/apex/in/container +``` +then go to /apex/in/container within the running container and +``` +pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . +``` diff --git a/apex/examples/imagenet/README.md b/apex/examples/imagenet/README.md new file mode 100644 index 00000000..257d4a78 --- /dev/null +++ b/apex/examples/imagenet/README.md @@ -0,0 +1,183 @@ +# Mixed Precision ImageNet Training in PyTorch + +`main_amp.py` is based on [https://github.com/pytorch/examples/tree/master/imagenet](https://github.com/pytorch/examples/tree/master/imagenet). +It implements Automatic Mixed Precision (Amp) training of popular model architectures, such as ResNet, AlexNet, and VGG, on the ImageNet dataset. Command-line flags forwarded to `amp.initialize` are used to easily manipulate and switch between various pure and mixed precision "optimization levels" or `opt_level`s. For a detailed explanation of `opt_level`s, see the [updated API guide](https://nvidia.github.io/apex/amp.html). + +Three lines enable Amp: +``` +# Added after model and optimizer construction +model, optimizer = amp.initialize(model, optimizer, flags...) +... +# loss.backward() changed to: +with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() +``` + +With the new Amp API **you never need to explicitly convert your model, or the input data, to half().** + +## Requirements + +- Download the ImageNet dataset and move validation images to labeled subfolders + - The following script may be helpful: https://raw.githubusercontent.com/soumith/imagenetloader.torch/master/valprep.sh + +## Training + +To train a model, create softlinks to the Imagenet dataset, then run `main.py` with the desired model architecture, as shown in `Example commands` below. + +The default learning rate schedule is set for ResNet50. `main_amp.py` script rescales the learning rate according to the global batch size (number of distributed processes \* per-process minibatch size). + +## Example commands + +**Note:** batch size `--b 224` assumes your GPUs have >=16GB of onboard memory. You may be able to increase this to 256, but that's cutting it close, so it may out-of-memory for different Pytorch versions. + +**Note:** All of the following use 4 dataloader subprocesses (`--workers 4`) to reduce potential +CPU data loading bottlenecks. + +**Note:** `--opt-level` `O1` and `O2` both use dynamic loss scaling by default unless manually overridden. +`--opt-level` `O0` and `O3` (the "pure" training modes) do not use loss scaling by default. +`O0` and `O3` can be told to use loss scaling via manual overrides, but using loss scaling with `O0` +(pure FP32 training) does not really make sense, and will trigger a warning. + +Softlink training and validation datasets into the current directory: +``` +$ ln -sf /data/imagenet/train-jpeg/ train +$ ln -sf /data/imagenet/val-jpeg/ val +``` + +### Summary + +Amp allows easy experimentation with various pure and mixed precision options. +``` +$ python main_amp.py -a resnet50 --b 128 --workers 4 --opt-level O0 ./ +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 ./ +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 --keep-batchnorm-fp32 True ./ +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./ +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 --loss-scale 128.0 ./ +$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./ +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./ +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 --loss-scale 128.0 ./ +$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./ +``` +Options are explained below. Again, the [updated API guide](https://nvidia.github.io/apex/amp.html) provides more detail. + +#### `--opt-level O0` (FP32 training) and `O3` (FP16 training) + +"Pure FP32" training: +``` +$ python main_amp.py -a resnet50 --b 128 --workers 4 --opt-level O0 ./ +``` +"Pure FP16" training: +``` +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 ./ +``` +FP16 training with FP32 batchnorm: +``` +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O3 --keep-batchnorm-fp32 True ./ +``` +Keeping the batchnorms in FP32 improves stability and allows Pytorch +to use cudnn batchnorms, which significantly increases speed in Resnet50. + +The `O3` options might not converge, because they are not true mixed precision. +However, they can be useful to establish "speed of light" performance for +your model, which provides a baseline for comparison with `O1` and `O2`. +For Resnet50 in particular, `--opt-level O3 --keep-batchnorm-fp32 True` establishes +the "speed of light." (Without `--keep-batchnorm-fp32`, it's slower, because it does +not use cudnn batchnorm.) + +#### `--opt-level O1` (Official Mixed Precision recipe, recommended for typical use) + +`O1` patches Torch functions to cast inputs according to a whitelist-blacklist model. +FP16-friendly (Tensor Core) ops like gemms and convolutions run in FP16, while ops +that benefit from FP32, like batchnorm and softmax, run in FP32. +Also, dynamic loss scaling is used by default. +``` +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./ +``` +`O1` overridden to use static loss scaling: +``` +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 --loss-scale 128.0 +``` +Distributed training with 2 processes (1 GPU per process, see **Distributed training** below +for more detail) +``` +$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O1 ./ +``` +For best performance, set `--nproc_per_node` equal to the total number of GPUs on the node +to use all available resources. + +#### `--opt-level O2` ("Almost FP16" mixed precision. More dangerous than O1.) + +`O2` exists mainly to support some internal use cases. Please prefer `O1`. + +`O2` casts the model to FP16, keeps batchnorms in FP32, +maintains master weights in FP32, and implements +dynamic loss scaling by default. (Unlike --opt-level O1, --opt-level O2 +does not patch Torch functions.) +``` +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./ +``` +"Fast mixed precision" overridden to use static loss scaling: +``` +$ python main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 --loss-scale 128.0 ./ +``` +Distributed training with 2 processes (1 GPU per process) +``` +$ python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 224 --workers 4 --opt-level O2 ./ +``` + +## Distributed training + +`main_amp.py` optionally uses `apex.parallel.DistributedDataParallel` (DDP) for multiprocess training with one GPU per process. +``` +model = apex.parallel.DistributedDataParallel(model) +``` +is a drop-in replacement for +``` +model = torch.nn.parallel.DistributedDataParallel(model, + device_ids=[arg.local_rank], + output_device=arg.local_rank) +``` +(because Torch DDP permits multiple GPUs per process, with Torch DDP you are required to +manually specify the device to run on and the output device. +With Apex DDP, it uses only the current device by default). + +The choice of DDP wrapper (Torch or Apex) is orthogonal to the use of Amp and other Apex tools. It is safe to use `apex.amp` with either `torch.nn.parallel.DistributedDataParallel` or `apex.parallel.DistributedDataParallel`. In the future, I may add some features that permit optional tighter integration between `Amp` and `apex.parallel.DistributedDataParallel` for marginal performance benefits, but currently, there's no compelling reason to use Apex DDP versus Torch DDP for most models. + +To use DDP with `apex.amp`, the only gotcha is that +``` +model, optimizer = amp.initialize(model, optimizer, flags...) +``` +must precede +``` +model = DDP(model) +``` +If DDP wrapping occurs before `amp.initialize`, `amp.initialize` will raise an error. + +With both Apex DDP and Torch DDP, you must also call `torch.cuda.set_device(args.local_rank)` within +each process prior to initializing your model or any other tensors. +More information can be found in the docs for the +Pytorch multiprocess launcher module [torch.distributed.launch](https://pytorch.org/docs/stable/distributed.html#launch-utility). + +`main_amp.py` is written to interact with +[torch.distributed.launch](https://pytorch.org/docs/master/distributed.html#launch-utility), +which spawns multiprocess jobs using the following syntax: +``` +python -m torch.distributed.launch --nproc_per_node=NUM_GPUS main_amp.py args... +``` +`NUM_GPUS` should be less than or equal to the number of visible GPU devices on the node. The use of `torch.distributed.launch` is unrelated to the choice of DDP wrapper. It is safe to use either apex DDP or torch DDP with `torch.distributed.launch`. + +Optionally, one can run imagenet with synchronized batch normalization across processes by adding +`--sync_bn` to the `args...` + +## Deterministic training (for debugging purposes) + +Running with the `--deterministic` flag should produce bitwise identical outputs run-to-run, +regardless of what other options are used (see [Pytorch docs on reproducibility](https://pytorch.org/docs/stable/notes/randomness.html)). +Since `--deterministic` disables `torch.backends.cudnn.benchmark`, `--deterministic` may +cause a modest performance decrease. + +## Profiling + +If you're curious how the network actually looks on the CPU and GPU timelines (for example, how good is the overall utilization? +Is the prefetcher really overlapping data transfers?) try profiling `main_amp.py`. +[Detailed instructions can be found here](https://gist.github.com/mcarilli/213a4e698e4a0ae2234ddee56f4f3f95). diff --git a/apex/examples/imagenet/main_amp.py b/apex/examples/imagenet/main_amp.py new file mode 100644 index 00000000..c4b0fdfd --- /dev/null +++ b/apex/examples/imagenet/main_amp.py @@ -0,0 +1,543 @@ +import argparse +import os +import shutil +import time + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +import numpy as np + +try: + from apex.parallel import DistributedDataParallel as DDP + from apex.fp16_utils import * + from apex import amp, optimizers + from apex.multi_tensor_apply import multi_tensor_applier +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") + +def fast_collate(batch, memory_format): + + imgs = [img[0] for img in batch] + targets = torch.tensor([target[1] for target in batch], dtype=torch.int64) + w = imgs[0].size[0] + h = imgs[0].size[1] + tensor = torch.zeros( (len(imgs), 3, h, w), dtype=torch.uint8).contiguous(memory_format=memory_format) + for i, img in enumerate(imgs): + nump_array = np.asarray(img, dtype=np.uint8) + if(nump_array.ndim < 3): + nump_array = np.expand_dims(nump_array, axis=-1) + nump_array = np.rollaxis(nump_array, 2) + tensor[i] += torch.from_numpy(nump_array) + return tensor, targets + + +def parse(): + model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + + parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') + parser.add_argument('data', metavar='DIR', + help='path to dataset') + parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') + parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') + parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') + parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') + parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', help='mini-batch size per process (default: 256)') + parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='Initial learning rate. Will be scaled by /256: args.lr = args.lr*float(args.batch_size*args.world_size)/256. A warmup schedule will also be applied over the first 5 epochs.') + parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') + parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)') + parser.add_argument('--print-freq', '-p', default=10, type=int, + metavar='N', help='print frequency (default: 10)') + parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') + parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') + parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') + + parser.add_argument('--prof', default=-1, type=int, + help='Only run 10 iterations for profiling.') + parser.add_argument('--deterministic', action='store_true') + + parser.add_argument("--local_rank", default=os.getenv('LOCAL_RANK', 0), type=int) + parser.add_argument('--sync_bn', action='store_true', + help='enabling apex sync BN.') + + parser.add_argument('--opt-level', type=str) + parser.add_argument('--keep-batchnorm-fp32', type=str, default=None) + parser.add_argument('--loss-scale', type=str, default=None) + parser.add_argument('--channels-last', type=bool, default=False) + args = parser.parse_args() + return args + +def main(): + global best_prec1, args + + args = parse() + print("opt_level = {}".format(args.opt_level)) + print("keep_batchnorm_fp32 = {}".format(args.keep_batchnorm_fp32), type(args.keep_batchnorm_fp32)) + print("loss_scale = {}".format(args.loss_scale), type(args.loss_scale)) + + print("\nCUDNN VERSION: {}\n".format(torch.backends.cudnn.version())) + + cudnn.benchmark = True + best_prec1 = 0 + if args.deterministic: + cudnn.benchmark = False + cudnn.deterministic = True + torch.manual_seed(args.local_rank) + torch.set_printoptions(precision=10) + + args.distributed = False + if 'WORLD_SIZE' in os.environ: + args.distributed = int(os.environ['WORLD_SIZE']) > 1 + + args.gpu = 0 + args.world_size = 1 + + if args.distributed: + args.gpu = args.local_rank + torch.cuda.set_device(args.gpu) + torch.distributed.init_process_group(backend='nccl', + init_method='env://') + args.world_size = torch.distributed.get_world_size() + + assert torch.backends.cudnn.enabled, "Amp requires cudnn backend to be enabled." + + if args.channels_last: + memory_format = torch.channels_last + else: + memory_format = torch.contiguous_format + + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + + if args.sync_bn: + import apex + print("using apex synced BN") + model = apex.parallel.convert_syncbn_model(model) + + model = model.cuda().to(memory_format=memory_format) + + # Scale learning rate based on global batch size + args.lr = args.lr*float(args.batch_size*args.world_size)/256. + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + # Initialize Amp. Amp accepts either values or strings for the optional override arguments, + # for convenient interoperation with argparse. + model, optimizer = amp.initialize(model, optimizer, + opt_level=args.opt_level, + keep_batchnorm_fp32=args.keep_batchnorm_fp32, + loss_scale=args.loss_scale + ) + + # For distributed training, wrap the model with apex.parallel.DistributedDataParallel. + # This must be done AFTER the call to amp.initialize. If model = DDP(model) is called + # before model, ... = amp.initialize(model, ...), the call to amp.initialize may alter + # the types of model's parameters in a way that disrupts or destroys DDP's allreduce hooks. + if args.distributed: + # By default, apex.parallel.DistributedDataParallel overlaps communication with + # computation in the backward pass. + # model = DDP(model) + # delay_allreduce delays all communication to the end of the backward pass. + model = DDP(model, delay_allreduce=True) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().cuda() + + # Optionally resume from a checkpoint + if args.resume: + # Use a local scope to avoid dangling references + def resume(): + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location = lambda storage, loc: storage.cuda(args.gpu)) + args.start_epoch = checkpoint['epoch'] + global best_prec1 + best_prec1 = checkpoint['best_prec1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + resume() + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + + if(args.arch == "inception_v3"): + raise RuntimeError("Currently, inception_v3 is not supported by this example.") + # crop_size = 299 + # val_size = 320 # I chose this value arbitrarily, we can adjust. + else: + crop_size = 224 + val_size = 256 + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(crop_size), + transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), Too slow + # normalize, + ])) + val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(val_size), + transforms.CenterCrop(crop_size), + ])) + + train_sampler = None + val_sampler = None + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset) + + collate_fn = lambda b: fast_collate(b, memory_format) + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, collate_fn=collate_fn) + + val_loader = torch.utils.data.DataLoader( + val_dataset, + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True, + sampler=val_sampler, + collate_fn=collate_fn) + + if args.evaluate: + validate(val_loader, model, criterion) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch) + + # evaluate on validation set + prec1 = validate(val_loader, model, criterion) + + # remember best prec@1 and save checkpoint + if args.local_rank == 0: + is_best = prec1 > best_prec1 + best_prec1 = max(prec1, best_prec1) + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_prec1': best_prec1, + 'optimizer' : optimizer.state_dict(), + }, is_best) + +class data_prefetcher(): + def __init__(self, loader): + self.loader = iter(loader) + self.stream = torch.cuda.Stream() + self.mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).cuda().view(1,3,1,1) + self.std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).cuda().view(1,3,1,1) + # With Amp, it isn't necessary to manually convert data to half. + # if args.fp16: + # self.mean = self.mean.half() + # self.std = self.std.half() + self.preload() + + def preload(self): + try: + self.next_input, self.next_target = next(self.loader) + except StopIteration: + self.next_input = None + self.next_target = None + return + # if record_stream() doesn't work, another option is to make sure device inputs are created + # on the main stream. + # self.next_input_gpu = torch.empty_like(self.next_input, device='cuda') + # self.next_target_gpu = torch.empty_like(self.next_target, device='cuda') + # Need to make sure the memory allocated for next_* is not still in use by the main stream + # at the time we start copying to next_*: + # self.stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(self.stream): + self.next_input = self.next_input.cuda(non_blocking=True) + self.next_target = self.next_target.cuda(non_blocking=True) + # more code for the alternative if record_stream() doesn't work: + # copy_ will record the use of the pinned source tensor in this side stream. + # self.next_input_gpu.copy_(self.next_input, non_blocking=True) + # self.next_target_gpu.copy_(self.next_target, non_blocking=True) + # self.next_input = self.next_input_gpu + # self.next_target = self.next_target_gpu + + # With Amp, it isn't necessary to manually convert data to half. + # if args.fp16: + # self.next_input = self.next_input.half() + # else: + self.next_input = self.next_input.float() + self.next_input = self.next_input.sub_(self.mean).div_(self.std) + + def next(self): + torch.cuda.current_stream().wait_stream(self.stream) + input = self.next_input + target = self.next_target + if input is not None: + input.record_stream(torch.cuda.current_stream()) + if target is not None: + target.record_stream(torch.cuda.current_stream()) + self.preload() + return input, target + + +def train(train_loader, model, criterion, optimizer, epoch): + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + # switch to train mode + model.train() + end = time.time() + + prefetcher = data_prefetcher(train_loader) + input, target = prefetcher.next() + i = 0 + while input is not None: + i += 1 + if args.prof >= 0 and i == args.prof: + print("Profiling begun at iteration {}".format(i)) + torch.cuda.cudart().cudaProfilerStart() + + if args.prof >= 0: torch.cuda.nvtx.range_push("Body of iteration {}".format(i)) + + adjust_learning_rate(optimizer, epoch, i, len(train_loader)) + + # compute output + if args.prof >= 0: torch.cuda.nvtx.range_push("forward") + output = model(input) + if args.prof >= 0: torch.cuda.nvtx.range_pop() + loss = criterion(output, target) + + # compute gradient and do SGD step + optimizer.zero_grad() + + if args.prof >= 0: torch.cuda.nvtx.range_push("backward") + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + if args.prof >= 0: torch.cuda.nvtx.range_pop() + + # for param in model.parameters(): + # print(param.data.double().sum().item(), param.grad.data.double().sum().item()) + + if args.prof >= 0: torch.cuda.nvtx.range_push("optimizer.step()") + optimizer.step() + if args.prof >= 0: torch.cuda.nvtx.range_pop() + + if i%args.print_freq == 0: + # Every print_freq iterations, check the loss, accuracy, and speed. + # For best performance, it doesn't make sense to print these metrics every + # iteration, since they incur an allreduce and some host<->device syncs. + + # Measure accuracy + prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) + + # Average loss and accuracy across processes for logging + if args.distributed: + reduced_loss = reduce_tensor(loss.data) + prec1 = reduce_tensor(prec1) + prec5 = reduce_tensor(prec5) + else: + reduced_loss = loss.data + + # to_python_float incurs a host<->device sync + losses.update(to_python_float(reduced_loss), input.size(0)) + top1.update(to_python_float(prec1), input.size(0)) + top5.update(to_python_float(prec5), input.size(0)) + + torch.cuda.synchronize() + batch_time.update((time.time() - end)/args.print_freq) + end = time.time() + + if args.local_rank == 0: + print('Epoch: [{0}][{1}/{2}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + 'Speed {3:.3f} ({4:.3f})\t' + 'Loss {loss.val:.10f} ({loss.avg:.4f})\t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + epoch, i, len(train_loader), + args.world_size*args.batch_size/batch_time.val, + args.world_size*args.batch_size/batch_time.avg, + batch_time=batch_time, + loss=losses, top1=top1, top5=top5)) + if args.prof >= 0: torch.cuda.nvtx.range_push("prefetcher.next()") + input, target = prefetcher.next() + if args.prof >= 0: torch.cuda.nvtx.range_pop() + + # Pop range "Body of iteration {}".format(i) + if args.prof >= 0: torch.cuda.nvtx.range_pop() + + if args.prof >= 0 and i == args.prof + 10: + print("Profiling ended at iteration {}".format(i)) + torch.cuda.cudart().cudaProfilerStop() + quit() + + +def validate(val_loader, model, criterion): + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + # switch to evaluate mode + model.eval() + + end = time.time() + + prefetcher = data_prefetcher(val_loader) + input, target = prefetcher.next() + i = 0 + while input is not None: + i += 1 + + # compute output + with torch.no_grad(): + output = model(input) + loss = criterion(output, target) + + # measure accuracy and record loss + prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) + + if args.distributed: + reduced_loss = reduce_tensor(loss.data) + prec1 = reduce_tensor(prec1) + prec5 = reduce_tensor(prec5) + else: + reduced_loss = loss.data + + losses.update(to_python_float(reduced_loss), input.size(0)) + top1.update(to_python_float(prec1), input.size(0)) + top5.update(to_python_float(prec5), input.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + # TODO: Change timings to mirror train(). + if args.local_rank == 0 and i % args.print_freq == 0: + print('Test: [{0}/{1}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + 'Speed {2:.3f} ({3:.3f})\t' + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + i, len(val_loader), + args.world_size * args.batch_size / batch_time.val, + args.world_size * args.batch_size / batch_time.avg, + batch_time=batch_time, loss=losses, + top1=top1, top5=top5)) + + input, target = prefetcher.next() + + print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def adjust_learning_rate(optimizer, epoch, step, len_epoch): + """LR schedule that should yield 76% converged accuracy with batch size 256""" + factor = epoch // 30 + + if epoch >= 80: + factor = factor + 1 + + lr = args.lr*(0.1**factor) + + """Warmup""" + if epoch < 5: + lr = lr*float(1 + step + epoch*len_epoch)/(5.*len_epoch) + + # if(args.local_rank == 0): + # print("epoch = {}, step = {}, lr = {}".format(epoch, step, lr)) + + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.reduce_op.SUM) + rt /= args.world_size + return rt + +if __name__ == '__main__': + main() diff --git a/apex/examples/simple/distributed/README.md b/apex/examples/simple/distributed/README.md new file mode 100644 index 00000000..0d939cbb --- /dev/null +++ b/apex/examples/simple/distributed/README.md @@ -0,0 +1,13 @@ +**distributed_data_parallel.py** and **run.sh** show an example using Amp with +[apex.parallel.DistributedDataParallel](https://nvidia.github.io/apex/parallel.html) or +[torch.nn.parallel.DistributedDataParallel](https://pytorch.org/docs/stable/nn.html#distributeddataparallel) +and the Pytorch multiprocess launcher script, +[torch.distributed.launch](https://pytorch.org/docs/master/distributed.html#launch-utility). +The use of `Amp` with DistributedDataParallel does not need to change from ordinary +single-process use. The only gotcha is that wrapping your model with `DistributedDataParallel` must +come after the call to `amp.initialize`. Test via +```bash +bash run.sh +``` + +**This is intended purely as an instructional example, not a performance showcase.** diff --git a/apex/examples/simple/distributed/distributed_data_parallel.py b/apex/examples/simple/distributed/distributed_data_parallel.py new file mode 100644 index 00000000..b364405d --- /dev/null +++ b/apex/examples/simple/distributed/distributed_data_parallel.py @@ -0,0 +1,65 @@ +import torch +import argparse +import os +from apex import amp +# FOR DISTRIBUTED: (can also use torch.nn.parallel.DistributedDataParallel instead) +from apex.parallel import DistributedDataParallel + +parser = argparse.ArgumentParser() +# FOR DISTRIBUTED: Parse for the local_rank argument, which will be supplied +# automatically by torch.distributed.launch. +parser.add_argument("--local_rank", default=0, type=int) +args = parser.parse_args() + +# FOR DISTRIBUTED: If we are running under torch.distributed.launch, +# the 'WORLD_SIZE' environment variable will also be set automatically. +args.distributed = False +if 'WORLD_SIZE' in os.environ: + args.distributed = int(os.environ['WORLD_SIZE']) > 1 + +if args.distributed: + # FOR DISTRIBUTED: Set the device according to local_rank. + torch.cuda.set_device(args.local_rank) + + # FOR DISTRIBUTED: Initialize the backend. torch.distributed.launch will provide + # environment variables, and requires that you use init_method=`env://`. + torch.distributed.init_process_group(backend='nccl', + init_method='env://') + +torch.backends.cudnn.benchmark = True + +N, D_in, D_out = 64, 1024, 16 + +# Each process receives its own batch of "fake input data" and "fake target data." +# The "training loop" in each process just uses this fake batch over and over. +# https://github.com/NVIDIA/apex/tree/master/examples/imagenet provides a more realistic +# example of distributed data sampling for both training and validation. +x = torch.randn(N, D_in, device='cuda') +y = torch.randn(N, D_out, device='cuda') + +model = torch.nn.Linear(D_in, D_out).cuda() +optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + +model, optimizer = amp.initialize(model, optimizer, opt_level="O1") + +if args.distributed: + # FOR DISTRIBUTED: After amp.initialize, wrap the model with + # apex.parallel.DistributedDataParallel. + model = DistributedDataParallel(model) + # torch.nn.parallel.DistributedDataParallel is also fine, with some added args: + # model = torch.nn.parallel.DistributedDataParallel(model, + # device_ids=[args.local_rank], + # output_device=args.local_rank) + +loss_fn = torch.nn.MSELoss() + +for t in range(500): + optimizer.zero_grad() + y_pred = model(x) + loss = loss_fn(y_pred, y) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + +if args.local_rank == 0: + print("final loss = ", loss) diff --git a/apex/examples/simple/distributed/run.sh b/apex/examples/simple/distributed/run.sh new file mode 100644 index 00000000..7a2d85f0 --- /dev/null +++ b/apex/examples/simple/distributed/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash +python -m torch.distributed.launch --nproc_per_node=2 distributed_data_parallel.py diff --git a/apex/requirements.txt b/apex/requirements.txt new file mode 100644 index 00000000..fd202d9b --- /dev/null +++ b/apex/requirements.txt @@ -0,0 +1,6 @@ +cxxfilt>=0.2.0 +tqdm>=4.28.1 +numpy>=1.15.3 +PyYAML>=5.1 +pytest>=3.5.1 +packaging>=14.0 diff --git a/apex/requirements_dev.txt b/apex/requirements_dev.txt new file mode 100644 index 00000000..e1086e0e --- /dev/null +++ b/apex/requirements_dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +flake8>=3.7.9 +Sphinx>=3.0.3 \ No newline at end of file diff --git a/apex/setup.py b/apex/setup.py new file mode 100644 index 00000000..7fc5d0cc --- /dev/null +++ b/apex/setup.py @@ -0,0 +1,806 @@ +import sys +import warnings +import os +from packaging.version import parse, Version + +from setuptools import setup, find_packages +import subprocess + +import torch +from torch.utils.cpp_extension import BuildExtension, CppExtension, CUDAExtension, CUDA_HOME, load + +PYTORCH_HOME = os.path.abspath(os.environ['PYTORCH_HOME']) if 'PYTORCH_HOME' in os.environ else None + +# ninja build does not work unless include_dirs are abs path +this_dir = os.path.dirname(os.path.abspath(__file__)) + + +def get_cuda_bare_metal_version(cuda_dir): + raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True) + output = raw_output.split() + release_idx = output.index("release") + 1 + bare_metal_version = parse(output[release_idx].split(",")[0]) + + return raw_output, bare_metal_version + + +def check_cuda_torch_binary_vs_bare_metal(cuda_dir): + raw_output, bare_metal_version = get_cuda_bare_metal_version(cuda_dir) + torch_binary_version = parse(torch.version.cuda) + + print("\nCompiling cuda extensions with") + print(raw_output + "from " + cuda_dir + "/bin\n") + + # if (bare_metal_version != torch_binary_version): + # raise RuntimeError( + # "Cuda extensions are being compiled with a version of Cuda that does " + # "not match the version used to compile Pytorch binaries. " + # "Pytorch binaries were compiled with Cuda {}.\n".format(torch.version.cuda) + # + "In some cases, a minor-version mismatch will not cause later errors: " + # "https://github.com/NVIDIA/apex/pull/323#discussion_r287021798. " + # "You can try commenting out this check (at your own risk)." + # ) + + +def raise_if_cuda_home_none(global_option: str) -> None: + if CUDA_HOME is not None: + return + raise RuntimeError( + f"{global_option} was requested, but nvcc was not found. Are you sure your environment has nvcc available? " + "If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, " + "only images whose names contain 'devel' will provide nvcc." + ) + + +def check_cudnn_version_and_warn(global_option: str, required_cudnn_version: int) -> bool: + cudnn_available = torch.backends.cudnn.is_available() + cudnn_version = torch.backends.cudnn.version() if cudnn_available else None + if not (cudnn_available and (cudnn_version >= required_cudnn_version)): + warnings.warn( + f"Skip `{global_option}` as it requires cuDNN {required_cudnn_version} or later, " + f"but {'cuDNN is not available' if not cudnn_available else cudnn_version}" + ) + return False + return True + + +if not torch.cuda.is_available(): + # https://github.com/NVIDIA/apex/issues/486 + # Extension builds after https://github.com/pytorch/pytorch/pull/23408 attempt to query torch.cuda.get_device_capability(), + # which will fail if you are compiling in an environment without visible GPUs (e.g. during an nvidia-docker build command). + print( + "\nWarning: Torch did not find available GPUs on this system.\n", + "If your intention is to cross-compile, this is not an error.\n" + "By default, Apex will cross-compile for Pascal (compute capabilities 6.0, 6.1, 6.2),\n" + "Volta (compute capability 7.0), Turing (compute capability 7.5),\n" + "and, if the CUDA version is >= 11.0, Ampere (compute capability 8.0).\n" + "If you wish to cross-compile for a single specific architecture,\n" + 'export TORCH_CUDA_ARCH_LIST="compute capability" before running setup.py.\n', + ) + if os.environ.get("TORCH_CUDA_ARCH_LIST", None) is None and CUDA_HOME is not None: + _, bare_metal_version = get_cuda_bare_metal_version(CUDA_HOME) + if bare_metal_version >= Version("11.8"): + os.environ["TORCH_CUDA_ARCH_LIST"] = "6.0;6.1;6.2;7.0;7.5;8.0;8.6;9.0" + elif bare_metal_version >= Version("11.1"): + os.environ["TORCH_CUDA_ARCH_LIST"] = "6.0;6.1;6.2;7.0;7.5;8.0;8.6" + elif bare_metal_version == Version("11.0"): + os.environ["TORCH_CUDA_ARCH_LIST"] = "6.0;6.1;6.2;7.0;7.5;8.0" + else: + os.environ["TORCH_CUDA_ARCH_LIST"] = "6.0;6.1;6.2;7.0;7.5" + +print("\n\ntorch.__version__ = {}\n\n".format(torch.__version__)) +TORCH_MAJOR = int(torch.__version__.split(".")[0]) +TORCH_MINOR = int(torch.__version__.split(".")[1]) + +if TORCH_MAJOR == 0 and TORCH_MINOR < 4: + raise RuntimeError( + "Apex requires Pytorch 0.4 or newer.\nThe latest stable release can be obtained from https://pytorch.org/" + ) + +cmdclass = {} +ext_modules = [] + +extras = {} + +if "--cpp_ext" in sys.argv or "--cuda_ext" in sys.argv: + if TORCH_MAJOR == 0: + raise RuntimeError( + "--cpp_ext requires Pytorch 1.0 or later, " "found torch.__version__ = {}".format(torch.__version__) + ) + +if "--cpp_ext" in sys.argv: + sys.argv.remove("--cpp_ext") + ext_modules.append(CppExtension("apex_C", ["csrc/flatten_unflatten.cpp"])) + + +# Set up macros for forward/backward compatibility hack around +# https://github.com/pytorch/pytorch/commit/4404762d7dd955383acee92e6f06b48144a0742e +# and +# https://github.com/NVIDIA/apex/issues/456 +# https://github.com/pytorch/pytorch/commit/eb7b39e02f7d75c26d8a795ea8c7fd911334da7e#diff-4632522f237f1e4e728cb824300403ac +version_ge_1_1 = [] +if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 0): + version_ge_1_1 = ["-DVERSION_GE_1_1"] +version_ge_1_3 = [] +if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 2): + version_ge_1_3 = ["-DVERSION_GE_1_3"] +version_ge_1_5 = [] +if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 4): + version_ge_1_5 = ["-DVERSION_GE_1_5"] +version_dependent_macros = version_ge_1_1 + version_ge_1_3 + version_ge_1_5 + +_, bare_metal_version = get_cuda_bare_metal_version(CUDA_HOME) + +if "--distributed_adam" in sys.argv: + sys.argv.remove("--distributed_adam") + raise_if_cuda_home_none("--distributed_adam") + ext_modules.append( + CUDAExtension( + name="distributed_adam_cuda", + sources=[ + "apex/contrib/csrc/optimizers/multi_tensor_distopt_adam.cpp", + "apex/contrib/csrc/optimizers/multi_tensor_distopt_adam_kernel.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3", "--use_fast_math"] + version_dependent_macros, + }, + ) + ) + +if "--distributed_lamb" in sys.argv: + sys.argv.remove("--distributed_lamb") + raise_if_cuda_home_none("--distributed_lamb") + ext_modules.append( + CUDAExtension( + name="distributed_lamb_cuda", + sources=[ + "apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb.cpp", + "apex/contrib/csrc/optimizers/multi_tensor_distopt_lamb_kernel.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3", "--use_fast_math"] + version_dependent_macros, + }, + ) + ) + +if "--cuda_ext" in sys.argv: + sys.argv.remove("--cuda_ext") + raise_if_cuda_home_none("--cuda_ext") + check_cuda_torch_binary_vs_bare_metal(CUDA_HOME) + + ext_modules.append( + CUDAExtension( + name="amp_C", + sources=[ + "csrc/amp_C_frontend.cpp", + "csrc/multi_tensor_sgd_kernel.cu", + "csrc/multi_tensor_scale_kernel.cu", + "csrc/multi_tensor_axpby_kernel.cu", + "csrc/multi_tensor_l2norm_kernel.cu", + "csrc/multi_tensor_l2norm_kernel_mp.cu", + "csrc/multi_tensor_l2norm_scale_kernel.cu", + "csrc/multi_tensor_lamb_stage_1.cu", + "csrc/multi_tensor_lamb_stage_2.cu", + "csrc/multi_tensor_adam.cu", + "csrc/multi_tensor_adagrad.cu", + "csrc/multi_tensor_novograd.cu", + "csrc/multi_tensor_lamb.cu", + "csrc/multi_tensor_lamb_mp.cu", + ], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": [ + "-lineinfo", + "-O3", + # '--resource-usage', + "--use_fast_math", + ] + version_dependent_macros, + }, + ) + ) + ext_modules.append( + CUDAExtension( + name="syncbn", + sources=["csrc/syncbn.cpp", "csrc/welford.cu"], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3"] + version_dependent_macros, + }, + ) + ) + + ext_modules.append( + CUDAExtension( + name="fused_layer_norm_cuda", + sources=["csrc/layer_norm_cuda.cpp", "csrc/layer_norm_cuda_kernel.cu"], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-maxrregcount=50", "-O3", "--use_fast_math"] + version_dependent_macros, + }, + ) + ) + + ext_modules.append( + CUDAExtension( + name="mlp_cuda", + sources=["csrc/mlp.cpp", "csrc/mlp_cuda.cu"], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3"] + version_dependent_macros, + }, + ) + ) + ext_modules.append( + CUDAExtension( + name="fused_dense_cuda", + sources=["csrc/fused_dense.cpp", "csrc/fused_dense_cuda.cu"], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3"] + version_dependent_macros, + }, + ) + ) + + ext_modules.append( + CUDAExtension( + name="scaled_upper_triang_masked_softmax_cuda", + sources=[ + "csrc/megatron/scaled_upper_triang_masked_softmax.cpp", + "csrc/megatron/scaled_upper_triang_masked_softmax_cuda.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + ] + version_dependent_macros, + }, + ) + ) + + ext_modules.append( + CUDAExtension( + name="generic_scaled_masked_softmax_cuda", + sources=[ + "csrc/megatron/generic_scaled_masked_softmax.cpp", + "csrc/megatron/generic_scaled_masked_softmax_cuda.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + ] + version_dependent_macros, + }, + ) + ) + + ext_modules.append( + CUDAExtension( + name="scaled_masked_softmax_cuda", + sources=["csrc/megatron/scaled_masked_softmax.cpp", "csrc/megatron/scaled_masked_softmax_cuda.cu"], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + ] + version_dependent_macros, + }, + ) + ) + + ext_modules.append( + CUDAExtension( + name="scaled_softmax_cuda", + sources=["csrc/megatron/scaled_softmax.cpp", "csrc/megatron/scaled_softmax_cuda.cu"], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + ] + version_dependent_macros, + }, + ) + ) + + if bare_metal_version >= Version("11.0"): + + cc_flag = [] + cc_flag.append("-gencode") + cc_flag.append("arch=compute_70,code=sm_70") + cc_flag.append("-gencode") + cc_flag.append("arch=compute_80,code=sm_80") + if bare_metal_version >= Version("11.1"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_86,code=sm_86") + if bare_metal_version >= Version("11.8"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_90,code=sm_90") + + ext_modules.append( + CUDAExtension( + name="fused_weight_gradient_mlp_cuda", + include_dirs=[os.path.join(this_dir, "csrc")], + sources=[ + "csrc/megatron/fused_weight_gradient_dense.cpp", + "csrc/megatron/fused_weight_gradient_dense_cuda.cu", + "csrc/megatron/fused_weight_gradient_dense_16bit_prec_cuda.cu", + ], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + "--use_fast_math", + ] + version_dependent_macros + cc_flag, + }, + ) + ) + +if PYTORCH_HOME is not None and os.path.exists(PYTORCH_HOME): + nvfuser_is_refactored = "nvfuser" in ( + os.path.join(d) for d in os.listdir(os.path.join(PYTORCH_HOME, "third_party")) + if os.path.isdir(os.path.join(os.path.join(PYTORCH_HOME, "third_party"), d)) + ) + import nvfuser # NOQA + print(PYTORCH_HOME) + include_dirs = [PYTORCH_HOME] + library_dirs = [] + extra_link_args = [] + if nvfuser_is_refactored: + include_dirs.append(os.path.join(PYTORCH_HOME, "third_party/nvfuser/csrc")) + library_dirs = nvfuser.__path__ + extra_link_args.append("-lnvfuser_codegen") + ext_modules.append( + CUDAExtension( + name='instance_norm_nvfuser_cuda', + sources=[ + 'csrc/instance_norm_nvfuser.cpp', + 'csrc/instance_norm_nvfuser_kernel.cu', + ], + include_dirs=include_dirs, + library_dirs=library_dirs, + extra_link_args=extra_link_args, + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3"] + version_dependent_macros + [f"-DNVFUSER_THIRDPARTY={int(nvfuser_is_refactored)}"], + }, + ) + ) + +if "--permutation_search" in sys.argv: + sys.argv.remove("--permutation_search") + + if CUDA_HOME is None: + raise RuntimeError("--permutation_search was requested, but nvcc was not found. Are you sure your environment has nvcc available? If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.") + else: + cc_flag = ['-Xcompiler', '-fPIC', '-shared'] + ext_modules.append( + CUDAExtension(name='permutation_search_cuda', + sources=['apex/contrib/sparsity/permutation_search_kernels/CUDA_kernels/permutation_search_kernels.cu'], + include_dirs=[os.path.join(this_dir, 'apex', 'contrib', 'sparsity', 'permutation_search_kernels', 'CUDA_kernels')], + extra_compile_args={'cxx': ['-O3'] + version_dependent_macros, + 'nvcc':['-O3'] + version_dependent_macros + cc_flag})) + +if "--bnp" in sys.argv: + sys.argv.remove("--bnp") + raise_if_cuda_home_none("--bnp") + ext_modules.append( + CUDAExtension( + name="bnp", + sources=[ + "apex/contrib/csrc/groupbn/batch_norm.cu", + "apex/contrib/csrc/groupbn/ipc.cu", + "apex/contrib/csrc/groupbn/interface.cpp", + "apex/contrib/csrc/groupbn/batch_norm_add_relu.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": [] + version_dependent_macros, + "nvcc": [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + version_dependent_macros, + }, + ) + ) + +if "--xentropy" in sys.argv: + sys.argv.remove("--xentropy") + raise_if_cuda_home_none("--xentropy") + ext_modules.append( + CUDAExtension( + name="xentropy_cuda", + sources=["apex/contrib/csrc/xentropy/interface.cpp", "apex/contrib/csrc/xentropy/xentropy_kernel.cu"], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3"] + version_dependent_macros, + }, + ) + ) + +if "--focal_loss" in sys.argv: + sys.argv.remove("--focal_loss") + raise_if_cuda_home_none("--focal_loss") + ext_modules.append( + CUDAExtension( + name='focal_loss_cuda', + sources=[ + 'apex/contrib/csrc/focal_loss/focal_loss_cuda.cpp', + 'apex/contrib/csrc/focal_loss/focal_loss_cuda_kernel.cu', + ], + include_dirs=[os.path.join(this_dir, 'csrc')], + extra_compile_args={ + 'cxx': ['-O3'] + version_dependent_macros, + 'nvcc':['-O3', '--use_fast_math', '--ftz=false'] + version_dependent_macros, + }, + ) + ) + +if "--index_mul_2d" in sys.argv: + sys.argv.remove("--index_mul_2d") + raise_if_cuda_home_none("--index_mul_2d") + ext_modules.append( + CUDAExtension( + name='fused_index_mul_2d', + sources=[ + 'apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda.cpp', + 'apex/contrib/csrc/index_mul_2d/index_mul_2d_cuda_kernel.cu', + ], + include_dirs=[os.path.join(this_dir, 'csrc')], + extra_compile_args={ + 'cxx': ['-O3'] + version_dependent_macros, + 'nvcc':['-O3', '--use_fast_math', '--ftz=false'] + version_dependent_macros, + }, + ) + ) + +if "--deprecated_fused_adam" in sys.argv: + sys.argv.remove("--deprecated_fused_adam") + raise_if_cuda_home_none("--deprecated_fused_adam") + ext_modules.append( + CUDAExtension( + name="fused_adam_cuda", + sources=[ + "apex/contrib/csrc/optimizers/fused_adam_cuda.cpp", + "apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3", "--use_fast_math"] + version_dependent_macros, + }, + ) + ) + +if "--deprecated_fused_lamb" in sys.argv: + sys.argv.remove("--deprecated_fused_lamb") + raise_if_cuda_home_none("--deprecated_fused_lamb") + ext_modules.append( + CUDAExtension( + name="fused_lamb_cuda", + sources=[ + "apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp", + "apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu", + "csrc/multi_tensor_l2norm_kernel.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3", "--use_fast_math"] + version_dependent_macros, + }, + ) + ) + +# Check, if ATen/CUDAGeneratorImpl.h is found, otherwise use ATen/cuda/CUDAGeneratorImpl.h +# See https://github.com/pytorch/pytorch/pull/70650 +generator_flag = [] +torch_dir = torch.__path__[0] +if os.path.exists(os.path.join(torch_dir, "include", "ATen", "CUDAGeneratorImpl.h")): + generator_flag = ["-DOLD_GENERATOR_PATH"] + +if "--fast_layer_norm" in sys.argv: + sys.argv.remove("--fast_layer_norm") + raise_if_cuda_home_none("--fast_layer_norm") + + cc_flag = [] + cc_flag.append("-gencode") + cc_flag.append("arch=compute_70,code=sm_70") + + if bare_metal_version >= Version("11.0"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_80,code=sm_80") + if bare_metal_version >= Version("11.8"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_90,code=sm_90") + + ext_modules.append( + CUDAExtension( + name="fast_layer_norm", + sources=[ + "apex/contrib/csrc/layer_norm/ln_api.cpp", + "apex/contrib/csrc/layer_norm/ln_fwd_cuda_kernel.cu", + "apex/contrib/csrc/layer_norm/ln_bwd_semi_cuda_kernel.cu", + ], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros + generator_flag, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "-U__CUDA_NO_BFLOAT16_OPERATORS__", + "-U__CUDA_NO_BFLOAT16_CONVERSIONS__", + "-U__CUDA_NO_BFLOAT162_OPERATORS__", + "-U__CUDA_NO_BFLOAT162_CONVERSIONS__", + "-I./apex/contrib/csrc/layer_norm/", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + "--use_fast_math", + ] + version_dependent_macros + generator_flag + cc_flag, + }, + include_dirs=[os.path.join(this_dir, "apex/contrib/csrc/layer_norm")], + ) + ) + +if "--fmha" in sys.argv: + sys.argv.remove("--fmha") + raise_if_cuda_home_none("--fmha") + + if bare_metal_version < Version("11.0"): + raise RuntimeError("--fmha only supported on sm_80 and sm_90 GPUs") + + cc_flag = [] + cc_flag.append("-gencode") + cc_flag.append("arch=compute_80,code=sm_80") + if bare_metal_version >= Version("11.8"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_90,code=sm_90") + + ext_modules.append( + CUDAExtension( + name="fmhalib", + sources=[ + "apex/contrib/csrc/fmha/fmha_api.cpp", + "apex/contrib/csrc/fmha/src/fmha_fill.cu", + "apex/contrib/csrc/fmha/src/fmha_noloop_reduce.cu", + "apex/contrib/csrc/fmha/src/fmha_fprop_fp16_128_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_fprop_fp16_256_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_fprop_fp16_384_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_fprop_fp16_512_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_128_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_256_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_384_64_kernel.sm80.cu", + "apex/contrib/csrc/fmha/src/fmha_dgrad_fp16_512_64_kernel.sm80.cu", + ], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros + generator_flag, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + "--use_fast_math", + ] + version_dependent_macros + generator_flag + cc_flag, + }, + include_dirs=[ + os.path.join(this_dir, "apex/contrib/csrc"), + os.path.join(this_dir, "apex/contrib/csrc/fmha/src"), + ], + ) + ) + + +if "--fast_multihead_attn" in sys.argv: + sys.argv.remove("--fast_multihead_attn") + raise_if_cuda_home_none("--fast_multihead_attn") + + cc_flag = [] + cc_flag.append("-gencode") + cc_flag.append("arch=compute_70,code=sm_70") + + if bare_metal_version >= Version("11.0"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_80,code=sm_80") + if bare_metal_version >= Version("11.1"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_86,code=sm_86") + if bare_metal_version >= Version("11.8"): + cc_flag.append("-gencode") + cc_flag.append("arch=compute_90,code=sm_90") + + subprocess.run(["git", "submodule", "update", "--init", "apex/contrib/csrc/multihead_attn/cutlass"]) + ext_modules.append( + CUDAExtension( + name="fast_multihead_attn", + sources=[ + "apex/contrib/csrc/multihead_attn/multihead_attn_frontend.cpp", + "apex/contrib/csrc/multihead_attn/additive_masked_softmax_dropout_cuda.cu", + "apex/contrib/csrc/multihead_attn/masked_softmax_dropout_cuda.cu", + "apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu", + "apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu", + "apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu", + "apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_additive_mask_cuda.cu", + "apex/contrib/csrc/multihead_attn/self_multihead_attn_bias_cuda.cu", + "apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu", + ], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros + generator_flag, + "nvcc": [ + "-O3", + "-U__CUDA_NO_HALF_OPERATORS__", + "-U__CUDA_NO_HALF_CONVERSIONS__", + "--expt-relaxed-constexpr", + "--expt-extended-lambda", + "--use_fast_math", + ] + + version_dependent_macros + + generator_flag + + cc_flag, + }, + include_dirs=[ + os.path.join(this_dir, "apex/contrib/csrc/multihead_attn/cutlass/include/"), + os.path.join(this_dir, "apex/contrib/csrc/multihead_attn/cutlass/tools/util/include") + ], + ) + ) + +if "--transducer" in sys.argv: + sys.argv.remove("--transducer") + raise_if_cuda_home_none("--transducer") + ext_modules.append( + CUDAExtension( + name="transducer_joint_cuda", + sources=[ + "apex/contrib/csrc/transducer/transducer_joint.cpp", + "apex/contrib/csrc/transducer/transducer_joint_kernel.cu", + ], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros + generator_flag, + "nvcc": ["-O3"] + version_dependent_macros + generator_flag, + }, + include_dirs=[os.path.join(this_dir, "csrc"), os.path.join(this_dir, "apex/contrib/csrc/multihead_attn")], + ) + ) + ext_modules.append( + CUDAExtension( + name="transducer_loss_cuda", + sources=[ + "apex/contrib/csrc/transducer/transducer_loss.cpp", + "apex/contrib/csrc/transducer/transducer_loss_kernel.cu", + ], + include_dirs=[os.path.join(this_dir, "csrc")], + extra_compile_args={ + "cxx": ["-O3"] + version_dependent_macros, + "nvcc": ["-O3"] + version_dependent_macros, + }, + ) + ) + +if "--cudnn_gbn" in sys.argv: + sys.argv.remove("--cudnn_gbn") + raise_if_cuda_home_none("--cudnn_gbn") + if check_cudnn_version_and_warn("--cudnn_gbn", 8500): + subprocess.run(["git", "submodule", "update", "--init", "apex/contrib/csrc/cudnn-frontend/"]) + ext_modules.append( + CUDAExtension( + name="cudnn_gbn_lib", + sources=[ + "apex/contrib/csrc/cudnn_gbn/norm_sample.cpp", + "apex/contrib/csrc/cudnn_gbn/cudnn_gbn.cpp", + ], + include_dirs=[os.path.join(this_dir, "apex/contrib/csrc/cudnn-frontend/include")], + extra_compile_args={"cxx": ["-O3", "-g"] + version_dependent_macros + generator_flag}, + ) + ) + +if "--peer_memory" in sys.argv: + sys.argv.remove("--peer_memory") + raise_if_cuda_home_none("--peer_memory") + ext_modules.append( + CUDAExtension( + name="peer_memory_cuda", + sources=[ + "apex/contrib/csrc/peer_memory/peer_memory_cuda.cu", + "apex/contrib/csrc/peer_memory/peer_memory.cpp", + ], + extra_compile_args={"cxx": ["-O3"] + version_dependent_macros + generator_flag}, + ) + ) + +# NOTE: Requires NCCL >= 2.10.3 +if "--nccl_p2p" in sys.argv: + sys.argv.remove("--nccl_p2p") + raise_if_cuda_home_none("--nccl_p2p") + # Check NCCL version. + _nccl_version_getter = load( + name="_nccl_version_getter", + sources=["apex/contrib/csrc/nccl_p2p/nccl_version.cpp", "apex/contrib/csrc/nccl_p2p/nccl_version_check.cu"], + + ) + _available_nccl_version = _nccl_version_getter.get_nccl_version() + if _available_nccl_version >= (2, 10): + ext_modules.append( + CUDAExtension( + name="nccl_p2p_cuda", + sources=[ + "apex/contrib/csrc/nccl_p2p/nccl_p2p_cuda.cu", + "apex/contrib/csrc/nccl_p2p/nccl_p2p.cpp", + ], + extra_compile_args={"cxx": ["-O3"] + version_dependent_macros + generator_flag}, + ) + ) + else: + warnings.warn( + f"Skip `--nccl_p2p` as it requires NCCL 2.10.3 or later, but {_available_nccl_version[0]}.{_available_nccl_version[1]}" + ) + +# note (mkozuki): Now `--fast_bottleneck` option (i.e. apex/contrib/bottleneck) depends on `--peer_memory` and `--nccl_p2p`. +if "--fast_bottleneck" in sys.argv: + sys.argv.remove("--fast_bottleneck") + raise_if_cuda_home_none("--fast_bottleneck") + if check_cudnn_version_and_warn("--fast_bottleneck", 8400): + subprocess.run(["git", "submodule", "update", "--init", "apex/contrib/csrc/cudnn-frontend/"]) + ext_modules.append( + CUDAExtension( + name="fast_bottleneck", + sources=["apex/contrib/csrc/bottleneck/bottleneck.cpp"], + include_dirs=[os.path.join(this_dir, "apex/contrib/csrc/cudnn-frontend/include")], + extra_compile_args={"cxx": ["-O3"] + version_dependent_macros + generator_flag}, + ) + ) + + +if "--fused_conv_bias_relu" in sys.argv: + sys.argv.remove("--fused_conv_bias_relu") + raise_if_cuda_home_none("--fused_conv_bias_relu") + if check_cudnn_version_and_warn("--fused_conv_bias_relu", 8400): + subprocess.run(["git", "submodule", "update", "--init", "apex/contrib/csrc/cudnn-frontend/"]) + ext_modules.append( + CUDAExtension( + name="fused_conv_bias_relu", + sources=["apex/contrib/csrc/conv_bias_relu/conv_bias_relu.cpp"], + include_dirs=[os.path.join(this_dir, "apex/contrib/csrc/cudnn-frontend/include")], + extra_compile_args={"cxx": ["-O3"] + version_dependent_macros + generator_flag}, + ) + ) + + +setup( + name="apex", + version="0.1", + packages=find_packages( + exclude=("build", "csrc", "include", "tests", "dist", "docs", "tests", "examples", "apex.egg-info",) + ), + install_requires=["packaging>20.6",], + description="PyTorch Extensions written by NVIDIA", + ext_modules=ext_modules, + cmdclass={"build_ext": BuildExtension} if ext_modules else {}, + extras_require=extras, +) diff --git a/apex/tests/L0/run_amp/__init__.py b/apex/tests/L0/run_amp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/tests/L0/run_amp/test_add_param_group.py b/apex/tests/L0/run_amp/test_add_param_group.py new file mode 100644 index 00000000..d3e90c43 --- /dev/null +++ b/apex/tests/L0/run_amp/test_add_param_group.py @@ -0,0 +1,148 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +from apex.amp import _amp_state +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn import Parameter + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +class MyModel(torch.nn.Module): + def __init__(self, unique): + super(MyModel, self).__init__() + self.weight0 = Parameter(unique + + torch.arange(2, device='cuda', dtype=torch.float32)) + self.weight1 = Parameter(1. + unique + torch.arange(2, device='cuda', dtype=torch.float16)) + + @staticmethod + def ops(input, weight0, weight1): + return ((input*(weight0.float()))*(weight1.float())).sum() + + def forward(self, input): + return self.ops(input, self.weight0, self.weight1) + + +# Abandon all hope, ye who enter here. + + +class TestAddParamGroup(unittest.TestCase): + def setUp(self): + self.x = torch.ones((2), device='cuda', dtype=torch.float32) + common_init(self) + + def tearDown(self): + pass + + def zero_grad(self, models, optimizer, how_to_zero): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + elif how_to_zero == "optimizer": + optimizer.zero_grad() + + def test_add_param_group(self): + for opt_level in ("O0", "O1", "O2", "O3"): + for zero_before_add in (True, False): + for try_accumulation in (True, False): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + + optimizer.zero_grad() + loss = model0(self.x) + loss.backward() + optimizer.step() + + if zero_before_add: + optimizer.zero_grad() + optimizer.add_param_group({'params' : model1.parameters(), 'lr' : 0.5}) + if not zero_before_add: + optimizer.zero_grad() + + loss = model0(self.x) + model1(self.x) + loss.backward(retain_graph=try_accumulation) + if try_accumulation: + loss.backward() + optimizer.step() + + # Once more to make sure the new params pick up momemtums properly + optimizer.zero_grad() + loss = model0(self.x) + model1(self.x) + loss.backward(retain_graph=try_accumulation) + if try_accumulation: + loss.backward() + optimizer.step() + + reference_params = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + for how_to_zero in "none", "model", "optimizer": + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1], optimizer = amp.initialize([model0, model1], + optimizer, + opt_level=opt_level, + verbosity=0, + cast_model_type=False) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + + self.zero_grad([model0, model1], optimizer, how_to_zero) + loss = model0(self.x) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + + if zero_before_add: + self.zero_grad([model0, model1], optimizer, how_to_zero) + optimizer.add_param_group({'params' : model1.parameters(), 'lr' : 0.5}) + if not zero_before_add: + self.zero_grad([model0, model1], optimizer, how_to_zero) + + loss = model0(self.x) + model1(self.x) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward(retain_graph=try_accumulation) + if try_accumulation: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + + # Once more to make sure the new params pick up momentums properly + self.zero_grad([model0, model1], optimizer, how_to_zero) + loss = model0(self.x) + model1(self.x) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward(retain_graph=try_accumulation) + if try_accumulation: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + + final_params = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + for reference, final in zip(reference_params, final_params): + self.assertTrue(torch.allclose(reference.to(final.dtype), final), + "opt_level = {}, how_to_zero = {}, zero_before_add = {}".format( + opt_level, how_to_zero, zero_before_add)) + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_basic_casts.py b/apex/tests/L0/run_amp/test_basic_casts.py new file mode 100644 index 00000000..5d4d81d1 --- /dev/null +++ b/apex/tests/L0/run_amp/test_basic_casts.py @@ -0,0 +1,143 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +import torch +from torch import nn +import torch.nn.functional as F + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +def run_layer_test(test_case, fns, expected, input_shape, test_backward=True): + for fn, typ in it.product(fns, expected.keys()): + x = torch.randn(input_shape, dtype=typ).requires_grad_() + y = fn(x) + test_case.assertEqual(y.type(), expected[typ]) + if test_backward: + y.float().sum().backward() + test_case.assertEqual(x.grad.type(), MATCH_INPUT[typ]) + +class TestBasicCasts(unittest.TestCase): + def setUp(self): + self.handle = amp.init(enabled=True) + common_init(self) + + def tearDown(self): + self.handle._deactivate() + + def test_linear_is_half(self): + m = nn.Linear(self.h, self.h) + f = ft.partial(F.linear, weight=m.weight, bias=m.bias) + run_layer_test(self, [m, f], ALWAYS_HALF, (self.b, self.h)) + + def test_conv2d_is_half(self): + m = nn.Conv2d(self.c, self.c, self.k) + f = ft.partial(F.conv2d, weight=m.weight, bias=m.bias) + run_layer_test(self, [m, f], ALWAYS_HALF, (self.b, self.c, self.h, self.h)) + + def test_softmax_is_float(self): + m = nn.Softmax(dim=1) + f = ft.partial(F.softmax, dim=1) + run_layer_test(self, [m, f], ALWAYS_FLOAT, (self.b, self.h)) + + def test_group_norm_is_float(self): + m = nn.GroupNorm(num_groups=4, num_channels=self.c) + run_layer_test(self, [m], ALWAYS_FLOAT, (self.b, self.c, self.h, self.h)) + + def test_mse_loss_is_float(self): + shape = (self.b, self.h) + target = torch.randn(shape) + mod = nn.MSELoss() + m = lambda x: mod(x, target) + f = ft.partial(F.mse_loss, target=target) + run_layer_test(self, [m], ALWAYS_FLOAT, shape) + + def test_relu_is_match(self): + run_layer_test(self, [nn.ReLU(), F.relu], MATCH_INPUT, (self.b, self.h)) + + def test_batch_norm_is_match(self): + m = nn.BatchNorm2d(num_features=self.c) + f = ft.partial(F.batch_norm, running_mean=m.running_mean, running_var=m.running_var, + weight=m.weight, bias=m.bias, training=True) + run_layer_test(self, [m], MATCH_INPUT, (self.b, self.c, self.h, self.h)) + + # Test forward-only for BN inference + m.eval() + f = ft.partial(F.batch_norm, running_mean=m.running_mean, running_var=m.running_var, + weight=m.weight, bias=m.bias, training=False) + run_layer_test(self, [m, f], MATCH_INPUT, (self.b, self.c, self.h, self.h), + test_backward=False) + +class TestBannedMethods(unittest.TestCase): + def setUp(self): + self.handle = amp.init(enabled=True) + common_init(self) + + def tearDown(self): + self.handle._deactivate() + + def bce_common(self, assertion): + shape = (self.b, self.h) + target = torch.rand(shape) + mod = nn.BCELoss() + m = lambda x: mod(x, target) + f = ft.partial(F.binary_cross_entropy, target=target) + for fn in [m, f]: + x = torch.rand(shape, dtype=torch.half) + assertion(fn, x) + + def test_bce_raises_by_default(self): + assertion = lambda fn, x: self.assertRaises(NotImplementedError, fn, x) + self.bce_common(assertion) + + def test_bce_is_float_with_allow_banned(self): + self.handle._deactivate() + self.handle = amp.init(enabled=True, allow_banned=True) + assertion = lambda fn, x: self.assertEqual(fn(x).type(), FLOAT) + self.bce_common(assertion) + +class TestTensorCasts(unittest.TestCase): + def setUp(self): + self.handle = amp.init(enabled=True) + common_init(self) + + def tearDown(self): + self.handle._deactivate() + + def test_matmul_method_is_half(self): + other = torch.randn(self.h, self.h) + lhs = lambda x: x.matmul(other) + rhs = lambda x: other.matmul(x) + run_layer_test(self, [lhs, rhs], ALWAYS_HALF, (self.h, self.h)) + + def test_matmul_op_is_half(self): + other = torch.randn(self.h, self.h) + lhs = lambda x: x @ other + rhs = lambda x: other @ x + run_layer_test(self, [lhs, rhs], ALWAYS_HALF, (self.h, self.h)) + + def test_pow_method_is_float(self): + fn = lambda x: x.pow(2.) + run_layer_test(self, [fn], ALWAYS_FLOAT, (self.b, self.h)) + + def test_pow_op_is_float(self): + fn = lambda x: x ** 2. + run_layer_test(self, [fn], ALWAYS_FLOAT, (self.b, self.h)) + + def test_cpu_is_float(self): + fn = lambda x: x.cpu() + always_cpu_float = {torch.float: 'torch.FloatTensor', + torch.half: 'torch.FloatTensor'} + run_layer_test(self, [fn], always_cpu_float, (self.b, self.h)) + + def test_sum_is_float(self): + fn = lambda x: x.sum() + run_layer_test(self, [fn], ALWAYS_FLOAT, (self.b, self.h)) + + # TODO: maybe more tests on disabled casting? + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_cache.py b/apex/tests/L0/run_amp/test_cache.py new file mode 100644 index 00000000..b58d2665 --- /dev/null +++ b/apex/tests/L0/run_amp/test_cache.py @@ -0,0 +1,137 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +from apex.amp import _amp_state +import torch +from torch import nn +import torch.nn.functional as F + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +def get_reference_grad(i, w, ops): + # Creating new tensors ensures, among other things, that the new tensors are not in the cache. + # In fact, they are guaranteed not to use the cache because they are not torch.nn.Parameters. + fp32_i = i.detach().clone().float() + fp32_w = w.detach().clone().float().requires_grad_() + loss = ops(fp32_i, fp32_w) + loss.backward() + return fp32_w.grad + +class WhitelistModule(torch.nn.Module): + def __init__(self, dtype): + super(WhitelistModule, self).__init__() + self.weight = torch.nn.Parameter(torch.arange(8*8, device='cuda', dtype=dtype).view(8,8)) + + @staticmethod + def ops(input, weight): + return (input.mm(weight)).mm(weight).sum() + + def forward(self, input): + return self.ops(input, self.weight) + + +class BlacklistModule(torch.nn.Module): + def __init__(self, dtype): + super(BlacklistModule, self).__init__() + self.weight = torch.nn.Parameter(torch.arange(2*8, device='cuda', dtype=dtype).view(2,8)) + + @staticmethod + def ops(input, weight): + return (input + torch.pow(weight, 2) + torch.pow(weight, 2)).sum() + + def forward(self, input): + return self.ops(input, self.weight) + + +class PromoteModule(torch.nn.Module): + def __init__(self, dtype): + super(PromoteModule, self).__init__() + self.weight = torch.nn.Parameter(torch.arange(2*8, device='cuda', dtype=dtype).view(2,8)) + + @staticmethod + def ops(input, weight): + return ((input*weight)*weight).sum() + + def forward(self, input): + return self.ops(input, self.weight) + +class TestCache(unittest.TestCase): + def setUp(self): + self.x = torch.ones((2, 8), device='cuda', dtype=torch.float32) + common_init(self) + + def tearDown(self): + pass + + def train_eval_train_test(self, module, t): + model = module(t).cuda() + optimizer = torch.optim.SGD(model.parameters(), lr=1.0) + + _amp_state.allow_incoming_model_not_fp32 = True + model, optimizer = amp.initialize(model, optimizer, opt_level="O1", verbosity=0) + _amp_state.allow_incoming_model_not_fp32 = False + + def training_step(): + for param in model.parameters(): + param.grad = None + + loss = model(self.x).sum() + _amp_state.loss_scalers[0]._loss_scale = 4.0 + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + self.assertEqual(len([p.grad for p in model.parameters() if p.grad is not None]), 1) + self.assertEqual(model.weight.grad.type(), model.weight.type()) + + reference_grad = get_reference_grad(self.x, model.weight, model.ops) + + # Currently there's no difference in the allclose calls, so no need for branching, + # but I'm keeping this in case we want different tolerances for fp16 and fp32 checks. + if model.weight.grad.type() == "torch.cuda.HalfTensor": + self.assertTrue(torch.allclose(model.weight.grad.float(), reference_grad)) + elif model.weight.grad.type() == "torch.cuda.FloatTensor": + self.assertTrue(torch.allclose(model.weight.grad.float(), reference_grad)) + else: + raise RuntimeError("model.weight.grad.type = {}".format(model.weight.grad.type())) + + model.weight.data -= 1. + + # Simulates first epoch + training_step() + + # Simulates eval + with torch.no_grad(): + loss = model(self.x).sum() + + # Simulates resuming training after eval + training_step() + + _amp_state.handle._deactivate() + + # I could easily have these as a set of for loops in a single test, + # instead of going for granularity. + def test_whitelist_module_fp16_weight(self): + self.train_eval_train_test(WhitelistModule, torch.float16) + + def test_whitelist_module_fp32_weight(self): + self.train_eval_train_test(WhitelistModule, torch.float32) + + def test_blacklist_module_fp16_weight(self): + self.train_eval_train_test(BlacklistModule, torch.float16) + + def test_blacklist_module_fp32_weight(self): + self.train_eval_train_test(BlacklistModule, torch.float32) + + def test_promote_module_fp16_weight(self): + self.train_eval_train_test(PromoteModule, torch.float16) + + def test_promote_module_fp32_weight(self): + self.train_eval_train_test(PromoteModule, torch.float32) + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_checkpointing.py b/apex/tests/L0/run_amp/test_checkpointing.py new file mode 100644 index 00000000..921985cd --- /dev/null +++ b/apex/tests/L0/run_amp/test_checkpointing.py @@ -0,0 +1,267 @@ +import unittest + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +from apex import amp + + +from utils import common_init, FLOAT + + +class MyModel(torch.nn.Module): + def __init__(self): + super(MyModel, self).__init__() + self.conv1 = nn.Conv2d(3, 6, 3, 1, 1) + self.bn1 = nn.BatchNorm2d(6) + self.param = nn.Parameter(torch.randn(1)) + + def forward(self, x): + x = x * self.param + x = F.relu(self.conv1(x)) + x = self.bn1(x) + return x + + +class TestCheckpointing(unittest.TestCase): + def setUp(self): + self.initial_lr = 1e-3 + self.test_opt_levels = ("O0", "O1", "O2", "O3") + + def seed(self): + torch.manual_seed(2809) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + + def check_state_dict_fp32(self, state_dict): + for key in state_dict: + if 'num_batches_tracked' in key: + continue + param = state_dict[key] + self.assertEqual(param.type(), FLOAT, + 'Parameter in state_dict not FLOAT') + + def train_step(self, model, optimizer, data, loss_ids): + optimizer.zero_grad() + + output = model(data) + + # Call backward for num_losses-1 + for idx in loss_ids: + loss = output.mean() + with amp.scale_loss(loss, optimizer, loss_id=idx) as scaled_loss: + scaled_loss.backward(retain_graph=True) + + optimizer.step() + return output + + def compare_models(self, modelA, modelB, test_setup=''): + state_dictA = modelA.state_dict() + state_dictB = modelB.state_dict() + self.assertEqual(len(state_dictA), len(state_dictB), + 'state_dicts have different lengths' + test_setup) + for key in state_dictA: + paramA = state_dictA[key] + paramB = state_dictB[key] + self.assertTrue((paramA==paramB).all(), + msg='Parameters in state_dices not equal.' + + 'key: {}\nparam: {}\nrestored: {}\ndiff: {} for {}'.format( + key, paramA, paramB, paramA - paramB, test_setup)) + + def test_restoring(self): + nb_epochs = 10 + nb_epochs_restore = nb_epochs // 2 + for opt_level in self.test_opt_levels: + for res_opt_level in self.test_opt_levels: + for amp_before_load in [True, False]: + for num_losses in range(1, 3): + test_setup = ('#' * 75 + '\n' + \ + f'opt_level {opt_level}\n' + \ + f'restore_opt_level {res_opt_level}\n' + \ + f'amp_before_load {amp_before_load}\n' + \ + f'num_losses {num_losses}\n') + + self.seed() + + # Create reference model + model = MyModel().to('cuda') + + optimizer = optim.SGD(model.parameters(), + lr=self.initial_lr) + + # Initialize with num_losses*2 for the original model and the restored one + model, optimizer = amp.initialize( + model, optimizer, opt_level=opt_level, + num_losses=num_losses*2, verbosity=0) + + # Compare training behavior for same restore option + # We cannot really generalize it, since a saved model in O0 + # would introduce a skipped step in O1, which will raise an error + if opt_level == res_opt_level: + # train for nb_epochs and restore after nb_epochs_restore + for epoch in range(nb_epochs): + + x = torch.randn(16, 3, 24, 24, device='cuda') + output = self.train_step( + model, optimizer, x, range(num_losses)) + # Initialize model one step before comparing. + # Otherwise the batchnorm layers will be updated + # additionally in restore_model + if epoch == (nb_epochs_restore - 1): + # Load model and optimizer + checkpoint = { + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'amp': amp.state_dict() + } + # Check state_dict for FP32 tensors + self.check_state_dict_fp32(checkpoint['model']) + + # Restore model + restore_model = MyModel().to('cuda') + restore_optimizer = optim.SGD( + restore_model.parameters(), + lr=self.initial_lr) + + if amp_before_load: + restore_model, restore_optimizer = amp.initialize( + restore_model, + restore_optimizer, + opt_level=res_opt_level, + num_losses=num_losses*2, + verbosity=0) + + restore_model.load_state_dict(checkpoint['model']) + restore_optimizer.load_state_dict(checkpoint['optimizer']) + # FIXME: We cannot test the amp.state_dict in the same script + # amp.load_state_dict(checkpoint['amp']) + + if not amp_before_load: + restore_model, restore_optimizer = amp.initialize( + restore_model, + restore_optimizer, + opt_level=res_opt_level, + num_losses=num_losses*2, + verbosity=0) + + elif epoch >= nb_epochs_restore: + restore_output = self.train_step( + restore_model, + restore_optimizer, + x, + range(num_losses, num_losses*2)) + self.assertTrue( + torch.allclose(output.float(), restore_output.float()), + 'Output of reference and restored models differ for ' + test_setup) + self.compare_models(model, restore_model, test_setup) + # if opt_level != res_opt_level + else: + # skip tests for different opt_levels + continue + + def test_loss_scale_decrease(self): + num_losses = 3 + nb_decrease_loss_scales = [0, 1, 2] + for opt_level in self.test_opt_levels: + #print('#' * 75 + f'\n opt_level {opt_level}\n') + # Create new tmp copy for this run + nb_decrease_loss_scales_tmp = list(nb_decrease_loss_scales) + + model = MyModel().to('cuda') + + optimizer = optim.SGD(model.parameters(), + lr=self.initial_lr) + + model, optimizer = amp.initialize( + model, optimizer, opt_level=opt_level, num_losses=num_losses, + verbosity=0) + + if amp._amp_state.opt_properties.loss_scale != 'dynamic': + #print('Static loss scale set. Skipping opt_level.') + continue + + # force to skip some updates to decrease the loss_scale + initial_loss_scales = [] + for idx in range(num_losses): + initial_loss_scales.append( + amp._amp_state.loss_scalers[idx].loss_scale()) + + for _ in range(len(nb_decrease_loss_scales)): + x = torch.randn(16, 3, 24, 24, device='cuda') + for idx in range(num_losses): + while nb_decrease_loss_scales_tmp[idx] > 0: + optimizer.zero_grad() + output = model(x * 2**17) + loss = output.mean() + + with amp.scale_loss(loss, optimizer, loss_id=idx) as scaled_loss: + scaled_loss.backward(retain_graph=True) + optimizer.step() + nb_decrease_loss_scales_tmp[idx] -= 1 + + # Check loss scales afterwards + updated_loss_scales = [] + for idx in range(num_losses): + updated_loss_scales.append( + amp._amp_state.loss_scalers[idx].loss_scale()) + for factor, update_ls, init_ls in zip(nb_decrease_loss_scales, + updated_loss_scales, + initial_loss_scales): + self.assertEqual(update_ls, init_ls / 2**factor) + + # Check state dict + amp_state_dict = amp.state_dict() + for scaler_idx, factor, init_ls in zip(amp_state_dict, + nb_decrease_loss_scales, + initial_loss_scales): + scaler = amp_state_dict[scaler_idx] + self.assertEqual(scaler['loss_scale'], init_ls / 2**factor) + unskipped_target = 0 + self.assertEqual(scaler['unskipped'], unskipped_target) + + def test_state_dict(self): + for opt_level in self.test_opt_levels: + # Skip O3 + if opt_level == 'O3': + continue + + model = MyModel().to('cuda') + optimizer = optim.Adam(model.parameters(), lr=1e-3) + model, optimizer = amp.initialize( + model, optimizer, opt_level=opt_level, verbosity=0) + + # Export state_dict and check for Half + state_dict = model.state_dict() + for key in state_dict: + self.assertFalse('Half' in state_dict[key].type()) + + # Check, if model is still trainable + # Create dummy data + data = torch.randn(10, 3, 4, 4, device='cuda') + target = torch.randn(10, 6, 4, 4, device='cuda') + + # Get initnial loss + optimizer.zero_grad() + output = model(data) + loss = F.mse_loss(output, target) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + last_loss = loss.item() + + # train for some epochs + for epoch in range(10): + optimizer.zero_grad() + output = model(data) + loss = F.mse_loss(output, target) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + self.assertTrue(loss.item() < last_loss) + last_loss = loss.item() + +if __name__=='__main__': + unittest.main() + diff --git a/apex/tests/L0/run_amp/test_fused_sgd.py b/apex/tests/L0/run_amp/test_fused_sgd.py new file mode 100644 index 00000000..7f592128 --- /dev/null +++ b/apex/tests/L0/run_amp/test_fused_sgd.py @@ -0,0 +1,794 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +from apex.amp import _amp_state +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn import Parameter + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + + +try: + import amp_C + disabled = False + from apex.optimizers import FusedSGD as FusedSGD +except ImportError as err: + print("amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was ", err) + disabled = True + + +class MyModel(torch.nn.Module): + def __init__(self, unique): + super(MyModel, self).__init__() + self.weight0 = Parameter(unique + + torch.arange(2, device='cuda', dtype=torch.float32)) + self.weight1 = Parameter(1. + unique + torch.arange(2, device='cuda', dtype=torch.float16)) + + @staticmethod + def ops(input, weight0, weight1): + return ((input*(weight0.float()))*(weight1.float())).sum() + + def forward(self, input): + return self.ops(input, self.weight0, self.weight1) + +# Abandon all hope, ye who enter here. + +# This is hands down the ugliest code I have ever written, but it succeeds in testing +# multiple models/optimizers/losses fairly thoroughly. Many of the different test cases +# require slightly divergent code in a way that seems near-impossible to genericize into a simple +# cross product or nested loops. + +class TestMultipleModelsOptimizersLosses(unittest.TestCase): + def setUp(self): + self.x = torch.ones((2), device='cuda', dtype=torch.float32) + common_init(self) + + def tearDown(self): + pass + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_2models2losses1optimizer(self): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.125) + + reference_grads = [] + for i in range(2): + optimizer.zero_grad() + loss0 = model0(self.x) + loss1 = model1(self.x) + loss0.backward() + loss1.backward() + + reference_grads.append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + optimizer.step() + + final_params = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + for materialize_master_grads in (False, True): + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (False, True): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + else: + iters = 2 + + model0 = MyModel(1) + model1 = MyModel(2) + + models = [model0, model1] + + optimizer = FusedSGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.125, + materialize_master_grads=materialize_master_grads) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1], optimizer = amp.initialize( + [model0, model1], + optimizer, + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer.zero_grad() + + loss0 = model0(self.x) + loss1 = model1(self.x) + + with amp.scale_loss(loss0, optimizer, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if inject_inf_loc == "fp32": + model0.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model0.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, optimizer, loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if inject_inf_loc == "fp32": + model1.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model1.weight1.grad[0] = float('inf') + + if i != inject_inf: + master_params = amp.master_params(optimizer) + for param, reference_grad in zip(master_params, reference_grads[unskipped]): + if opt_level == "O2" and not materialize_master_grads: + continue + else: + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float()), + "opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers)) + unskipped += 1 + optimizer.step() + + model_params = [p for p in model0.parameters()] + [p for p in model1.parameters()] + for model, master, reference in zip( + model_params, + amp.master_params(optimizer), + final_params): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_3models2losses1optimizer(self): + + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}, + {'params' : model2.parameters(), 'lr' : 0.125}], + momentum=0.125) + + reference_grads = [] + for i in range(2): + optimizer.zero_grad() + loss0 = model0(self.x) + model2(self.x) + loss1 = model1(self.x) + model2(self.x) + loss0.backward() + loss1.backward() + + reference_grads.append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()] + + [param.grad.data.clone() for param in model2.parameters()]) + + optimizer.step() + + + final_params = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + \ + [param.data.clone() for param in model2.parameters()] + + for materialize_master_grads in (False, True): + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (False, True): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + if which_backward == 0: + which_models = (0, 2) + elif which_backward == 1: + which_models = (1, 2) + else: + iters = 2 + which_models = (None,) + + for which_model in which_models: + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + models = [model0, model1, model2] + + optimizer = FusedSGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}, + {'params' : model2.parameters(), 'lr' : 0.125}], + momentum=0.125, + materialize_master_grads=materialize_master_grads) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1, model2], optimizer = amp.initialize( + [model0, model1, model2], + optimizer, + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer.zero_grad() + + loss0 = model0(self.x) + model2(self.x) + loss1 = model1(self.x) + model2(self.x) + + with amp.scale_loss(loss0, optimizer, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if which_model == 0: + inj_model = model0 + elif which_model == 2: + inj_model = model2 + else: + raise RuntimeError(which_model + " invalid for loss 0") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, optimizer, loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if which_model == 1: + inj_model = model1 + elif which_model == 2: + inj_model = model2 + else: + raise RuntimeError(which_model + " invalid for loss 1 ") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + + if i != inject_inf: + master_params = amp.master_params(optimizer) + for param, reference_grad in zip(master_params, reference_grads[unskipped]): + if opt_level == "O2" and not materialize_master_grads: + continue + else: + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float()), + "opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} which_model {} use_multiple_loss_scalers {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, which_model, use_multiple_loss_scalers)) + unskipped += 1 + + optimizer.step() + + model_params = [p for p in model0.parameters()] + \ + [p for p in model1.parameters()] + \ + [p for p in model2.parameters()] + for model, master, reference in zip( + model_params, + amp.master_params(optimizer), + final_params): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_2models2losses2optimizers(self): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.25) + + # Don't do it like this: reference_grads = [[]]*5 + # because then it creates a list of 5 references to the same "[]" and appending + # to any of them effectively makes you append to all of them, which multiplies + # the resulting size of reference_grads by 5x and needless to say makes the test fail. + reference_grads = [[], [], [], [], []] + final_params = [None, None, None, None, None] + for i in range(2): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + loss1 = model1(self.x) + loss0.backward() + loss1.backward() + + reference_grads[0].append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + optimizer0.step() + optimizer1.step() + + final_params[0] = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + def what_got_skipped(which_iter, which_backward): + if which_iter == 0 and which_backward == 0: + return 1 + if which_iter == 0 and which_backward == 1: + return 2 + if which_iter == 1 and which_backward == 0: + return 3 + if which_iter == 1 and which_backward == 1: + return 4 + return 0 + + for which_iter in (0,1): + for which_backward in (0,1): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.25) + + for i in range(3): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + loss1 = model1(self.x) + loss0.backward() + loss1.backward() + + if i != which_iter: + reference_grads[what_got_skipped(which_iter, which_backward)].append( + [param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + if i == which_iter: + if which_backward == 0: + optimizer1.step() + else: + optimizer0.step() + else: + optimizer0.step() + optimizer1.step() + + final_params[what_got_skipped(which_iter, which_backward)] = \ + [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + for materialize_master_grads in (False, True): + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (False, True): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + else: + iters = 2 + + model0 = MyModel(1) + model1 = MyModel(2) + + models = [model0, model1] + + optimizer0 = FusedSGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125, materialize_master_grads=materialize_master_grads) + optimizer1 = FusedSGD([{'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.25, materialize_master_grads=materialize_master_grads) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1], [optimizer0, optimizer1] = amp.initialize( + [model0, model1], + [optimizer0, optimizer1], + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer0.zero_grad() + optimizer1.zero_grad() + + loss0 = model0(self.x) + loss1 = model1(self.x) + + with amp.scale_loss(loss0, optimizer0, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if inject_inf_loc == "fp32": + model0.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model0.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, optimizer1, loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if inject_inf_loc == "fp32": + model1.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model1.weight1.grad[0] = float('inf') + + # print("opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers)) + + if i != inject_inf: + master_params = list(amp.master_params(optimizer0)) + \ + list(amp.master_params(optimizer1)) + for param, reference_grad in zip(master_params, + reference_grads[what_got_skipped(inject_inf, which_backward)][unskipped]): + if opt_level == "O2" and not materialize_master_grads: + continue + else: + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float())) + unskipped += 1 + + optimizer0.step() + optimizer1.step() + + model_params = [p for p in model0.parameters()] + [p for p in model1.parameters()] + master_params = [p for p in amp.master_params(optimizer0)] + \ + [p for p in amp.master_params(optimizer1)] + for model, master, reference in zip( + model_params, + master_params, + final_params[what_got_skipped(inject_inf, which_backward)]): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_3models2losses2optimizers(self): + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 1.0}], + momentum=0.5) + optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}], + momentum=0.25) + + # Again, can't do this: reference_grads = [[]]*9 + reference_grads = [[], [], [], [], [], [], [], [], []] + final_params = [None, None, None, None, None, None, None, None, None] + for i in range(2): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + model1(self.x) + loss1 = model2(self.x) + model1(self.x) + loss0.backward() + loss1.backward() + + reference_grads[0].append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + optimizer0.step() + optimizer1.step() + + final_params[0] = \ + [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + \ + [param.data.clone() for param in model2.parameters()] + + def what_got_skipped(which_iter, which_backward, which_model): + if which_iter == 0: + if which_backward == 0: + if which_model == 0: + return 1 + if which_model == 1: + return 2 + if which_backward == 1: + if which_model == 2: + return 3 + if which_model == 1: + return 4 + if which_iter == 1: + if which_backward == 0: + if which_model == 0: + return 5 + if which_model == 1: + return 6 + if which_backward == 1: + if which_model == 2: + return 7 + if which_model == 1: + return 8 + return 0 + + for which_iter in (0,1): + for which_backward in (0,1): + if which_backward == 0: + which_models = (0,1) + if which_backward == 1: + which_models = (2,1) + for which_model in which_models: + + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 1.0}], + momentum=0.5) + optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}], + momentum=0.25) + + for i in range(3): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + model1(self.x) + loss1 = model2(self.x) + model1(self.x) + loss0.backward() + loss1.backward() + + if i != which_iter: + reference_grads[what_got_skipped(which_iter, + which_backward, which_model)].append( + [param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + if i == which_iter: + if which_backward == 0: + # if which_model == 0: + optimizer1.step() + # if which_model == 1: + # optimizer1.step() + if which_backward == 1: + # if which_model == 2: + # optimizer0.step() + # if which_model == 1: + continue + else: + optimizer0.step() + optimizer1.step() + + final_params[what_got_skipped(which_iter, which_backward, which_model)] = \ + [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + \ + [param.data.clone() for param in model2.parameters()] + + for materialize_master_grads in (False, True): + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (False, True): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + if which_backward == 0: + which_models = (0, 1) + elif which_backward == 1: + which_models = (2, 1) + else: + iters = 2 + which_models = (None,) + + for which_model in which_models: + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + models = [model0, model1, model2] + + optimizer0 = FusedSGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 1.0}], + momentum=0.5, materialize_master_grads=materialize_master_grads) + optimizer1 = FusedSGD([{'params' : model2.parameters(), 'lr' : 0.5}], + momentum=0.25, materialize_master_grads=materialize_master_grads) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1, model2], [optimizer0, optimizer1] = amp.initialize( + [model0, model1, model2], + [optimizer0, optimizer1], + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer0.zero_grad() + optimizer1.zero_grad() + + loss0 = model0(self.x) + model1(self.x) + loss1 = model2(self.x) + model1(self.x) + + with amp.scale_loss(loss0, optimizer0, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if which_model == 0: + inj_model = model0 + elif which_model == 1: + inj_model = model1 + else: + raise RuntimeError(which_model + " invalid for loss 0") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, [optimizer0, optimizer1], loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if which_model == 2: + inj_model = model2 + elif which_model == 1: + inj_model = model1 + else: + raise RuntimeError(which_model + " invalid for loss 1 ") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + + if i != inject_inf: + master_params = list(amp.master_params(optimizer0)) + \ + list(amp.master_params(optimizer1)) + for param, reference_grad in zip(master_params, + reference_grads[what_got_skipped(inject_inf, + which_backward, which_model)][unskipped]): + if opt_level == "O2" and not materialize_master_grads: + continue + else: + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float())) + unskipped += 1 + + optimizer0.step() + optimizer1.step() + + model_params = [p for p in model0.parameters()] + \ + [p for p in model1.parameters()] + \ + [p for p in model2.parameters()] + master_params = [p for p in amp.master_params(optimizer0)] + \ + [p for p in amp.master_params(optimizer1)] + + # print("opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {} which_model {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers, which_model)) + + for model, master, reference in zip( + model_params, + master_params, + final_params[what_got_skipped(inject_inf, which_backward, which_model)]): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_larc.py b/apex/tests/L0/run_amp/test_larc.py new file mode 100644 index 00000000..f4f3e838 --- /dev/null +++ b/apex/tests/L0/run_amp/test_larc.py @@ -0,0 +1,53 @@ +import unittest + +import torch +from torch import nn +from torch.nn import Parameter + +from apex import amp +from apex.parallel.LARC import LARC +from utils import common_init + + +class MyModel(torch.nn.Module): + def __init__(self, unique): + super(MyModel, self).__init__() + self.weight0 = Parameter( + unique + torch.arange(2, device="cuda", dtype=torch.float32) + ) + + def forward(self, input): + return (input * self.weight0).sum() + + +class TestLARC(unittest.TestCase): + def setUp(self): + self.x = torch.ones((2), device="cuda", dtype=torch.float32) + common_init(self) + + def tearDown(self): + pass + + def test_larc_mixed_precision(self): + for opt_level in ["O0", "O1", "O2", "O3"]: + model = MyModel(1) + + optimizer = LARC( + torch.optim.SGD( + [{"params": model.parameters(), "lr": 0.25}], momentum=0.125 + ) + ) + + model, optimizer = amp.initialize( + model, optimizer, opt_level=opt_level, verbosity=0 + ) + + optimizer.zero_grad() + loss = model(self.x) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/tests/L0/run_amp/test_multi_tensor_axpby.py b/apex/tests/L0/run_amp/test_multi_tensor_axpby.py new file mode 100644 index 00000000..0b439bb8 --- /dev/null +++ b/apex/tests/L0/run_amp/test_multi_tensor_axpby.py @@ -0,0 +1,180 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +import torch +from torch import nn +import torch.nn.functional as F +from math import floor + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +try: + import amp_C + from amp_C import multi_tensor_axpby + from apex.multi_tensor_apply import MultiTensorApply + disabled = False +except ImportError as err: + print("amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was ", err) + disabled = True + +TORCH_MAJOR = int(torch.__version__.split('.')[0]) +TORCH_MINOR = int(torch.__version__.split('.')[1]) +try_nhwc = (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 4) + + +class TestMultiTensorAxpby(unittest.TestCase): + + def setUp(self): + common_init(self) + + self.a = 2.0 + self.b = 8.0 + self.xval = 4.0 + self.yval = 16.0 + self.overflow_buf = torch.cuda.IntTensor(1).zero_() + self.ref = torch.full((1,), 136.0, device="cuda", dtype=torch.float32) + + def tearDown(self): + pass + + # The tensor creation here is written for convenience, not speed. + def axpby(self, sizea, sizeb, applier, repeat_tensors, + x_type, y_type, out_type, inplace=False, nhwc=False): + self.overflow_buf.zero_() + sizea = sizea if isinstance(sizea, tuple) else (sizea,) + sizeb = sizeb if isinstance(sizeb, tuple) else (sizeb,) + t1 = torch.full(sizea, 1.0, device="cuda", dtype=torch.float32) + t2 = torch.full(sizeb, 1.0, device="cuda", dtype=torch.float32) + + def to_fmt(t, tp): + if nhwc: + return t.clone().to(tp, memory_format=torch.channels_last) + else: + return t.clone().to(tp) + + y_list = [] + for i in range(repeat_tensors): + y_list += [to_fmt(t1, y_type)*self.yval, to_fmt(t2, y_type)*self.yval] + + x_list = [to_fmt(x, x_type)*(self.xval/self.yval) for x in y_list] + + if inplace: + out_list = y_list + else: + out_list = [to_fmt(out, out_type)*3.0 for out in y_list] + + applier(multi_tensor_axpby, self.overflow_buf, [x_list, y_list, out_list], self.a, self.b, -1) + + self.assertTrue(all([torch.allclose(out, self.ref.to(out_type)) for out in out_list]), + msg="{} {} {} {} {} {} {}".format(sizea, sizeb, repeat_tensors, + x_type, y_type, out_type, inplace)) + self.assertTrue(self.overflow_buf.item() == 0, + msg="{} {} {} {} {} {} {}".format(sizea, sizeb, repeat_tensors, + x_type, y_type, out_type, inplace)) + + # def find_inf(self, sizea, sizeb, applier, repeat_tensors, in_type, out_type, t, ind, val, inplace=False): + # self.overflow_buf.zero_() + # a = torch.cuda.FloatTensor(sizea).fill_(self.scale) + # b = torch.cuda.FloatTensor(sizeb).fill_(self.scale) + + # out_list = [] + # for i in range(repeat_tensors): + # out_list += [a.clone().to(out_type), b.clone().to(out_type)] + + # if inplace: + # in_list = out_list + # else: + # in_list = [out.clone().to(in_type) for out in out_list] + + # applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale) + + # self.overflow_buf.zero_() + # in_list[t][ind] = val + # applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale) + # self.assertTrue(self.overflow_buf.item()) + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_fuzz(self): + input_size_pairs = ( + (7777*77, 555*555), + (777, 555), + (555, 2048*32+1), + (2048*32+1, 555), + (555, 2048*32), + (2048*32, 555), + (33333, 555), + (555, 33333)) + appliers = ( + MultiTensorApply(2048*32), + MultiTensorApply(333), + MultiTensorApply(33333)) + repeat_tensors = ( + 1, + 55) + + for sizea, sizeb in input_size_pairs: + for applier in appliers: + for repeat in repeat_tensors: + for x_type in (torch.float32, torch.float16): + for y_type in (torch.float32, torch.float16): + for out_type in (torch.float32, torch.float16): + for inplace in (True, False): + if inplace is True and (y_type is not out_type): + continue + else: + self.axpby(sizea, sizeb, applier, repeat, + x_type, y_type, out_type, inplace=inplace) + # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + # 0, 0, float('nan'), inplace=inplace) + # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + # 2*repeat-1, sizeb-1, float('inf'), inplace=inplace) + # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + # 2*(repeat//2), sizea//2, float('inf'), inplace=inplace) + + @unittest.skipIf(disabled, "amp_C is unavailable") + @unittest.skipIf(not try_nhwc, "torch version is 1.4 or earlier, may not support nhwc") + def test_fuzz_nhwc(self): + input_size_pairs = ( + ((7, 77, 7, 77), (5, 55, 5, 55)), + ((1, 1, 777, 1), (1, 1, 555, 1)), + ((5, 47, 5, 55), (1, 1, 1, 2048*32 + 1)), + ((1, 1, 1, 2048*32 + 1), (55, 47, 5, 55)), + ((555, 1, 1, 1), (32, 8, 32, 8)), + ((32, 8, 32, 8), (55, 47, 5, 55)), + ((1, 1, 33333, 1), (55, 47, 55, 5)), + ((55, 47, 55, 5), (1, 1, 33333, 1))) + appliers = ( + MultiTensorApply(2048*32), + MultiTensorApply(333), + MultiTensorApply(33333)) + repeat_tensors = ( + 1, + 55) + + for sizea, sizeb in input_size_pairs: + for applier in appliers: + for repeat in repeat_tensors: + for x_type in (torch.float32, torch.float16): + for y_type in (torch.float32, torch.float16): + for out_type in (torch.float32, torch.float16): + for inplace in (True, False): + if inplace is True and (y_type is not out_type): + continue + else: + self.axpby(sizea, sizeb, applier, repeat, + x_type, y_type, out_type, inplace=inplace, nhwc=True) + # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + # 0, 0, float('nan'), inplace=inplace) + # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + # 2*repeat-1, sizeb-1, float('inf'), inplace=inplace) + # self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + # 2*(repeat//2), sizea//2, float('inf'), inplace=inplace) + + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_multi_tensor_l2norm.py b/apex/tests/L0/run_amp/test_multi_tensor_l2norm.py new file mode 100644 index 00000000..ed3cbd19 --- /dev/null +++ b/apex/tests/L0/run_amp/test_multi_tensor_l2norm.py @@ -0,0 +1,87 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +import torch +from torch import nn +import torch.nn.functional as F + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +try: + import amp_C + from amp_C import multi_tensor_l2norm + from apex.multi_tensor_apply import MultiTensorApply + disabled = False +except ImportError as err: + print("amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was ", err) + disabled = True + + +class TestMultiTensorL2Norm(unittest.TestCase): + + def setUp(self): + common_init(self) + self.val = 4.0 + self.overflow_buf = torch.cuda.IntTensor(1).zero_() + + def tearDown(self): + pass + + # The tensor creation here is written for convenience, not speed. + def l2norm(self, sizea, sizeb, applier, repeat_tensors, in_type, per_tensor): + self.overflow_buf.zero_() + a = torch.cuda.FloatTensor(sizea).fill_(self.val) + b = torch.cuda.FloatTensor(sizeb).fill_(self.val) + + in_list = [] + for i in range(repeat_tensors): + in_list += [a.clone().to(in_type), b.clone().to(in_type)] + + if per_tensor: + norm, norm_per_tensor = applier(multi_tensor_l2norm, self.overflow_buf, [in_list], True) + normab = torch.cat((a.norm().view(1), b.norm().view(1))) + norm_per_tensor = norm_per_tensor.view(-1, 2) + else: + norm, _ = applier(multi_tensor_l2norm, self.overflow_buf, [in_list], True) + + reference = torch.cuda.FloatTensor((sizea + sizeb)*repeat_tensors).fill_(self.val).norm() + + self.assertTrue(torch.allclose(norm, reference)) + if per_tensor: + self.assertTrue(torch.allclose(norm_per_tensor, normab)) + self.assertTrue(self.overflow_buf.item() == 0) + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_fuzz(self): + input_size_pairs = ( + (7777*77, 555*555), + (777, 555), + (555, 2048*32+1), + (2048*32+1, 555), + (555, 2048*32), + (2048*32, 555), + (33333, 555), + (555, 33333)) + appliers = ( + MultiTensorApply(2048*32), + MultiTensorApply(333), + MultiTensorApply(33333)) + repeat_tensors = ( + 1, + 55) + + for sizea, sizeb in input_size_pairs: + for applier in appliers: + for repeat in repeat_tensors: + for in_type in (torch.float32, torch.float16): + for per_tensor in (False, True): + self.l2norm(sizea, sizeb, applier, repeat, in_type, per_tensor) + + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_multi_tensor_scale.py b/apex/tests/L0/run_amp/test_multi_tensor_scale.py new file mode 100644 index 00000000..22da2490 --- /dev/null +++ b/apex/tests/L0/run_amp/test_multi_tensor_scale.py @@ -0,0 +1,126 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +import torch +from torch import nn +import torch.nn.functional as F + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +try: + import amp_C + from amp_C import multi_tensor_scale + from apex.multi_tensor_apply import MultiTensorApply + disabled = False +except ImportError as err: + print("amp_C fused kernels unavailable, disabling TestMultiTensorApply. ImportError was ", err) + disabled = True + + +class TestMultiTensorScale(unittest.TestCase): + + def setUp(self): + common_init(self) + self.scale = 4.0 + self.overflow_buf = torch.cuda.IntTensor(1).zero_() + self.ref = torch.cuda.FloatTensor([1.0]) + + def tearDown(self): + pass + + # The tensor creation here is written for convenience, not speed. + def downscale(self, sizea, sizeb, applier, repeat_tensors, in_type, out_type, inplace=False): + self.overflow_buf.zero_() + a = torch.cuda.FloatTensor(sizea).fill_(self.scale) + b = torch.cuda.FloatTensor(sizeb).fill_(self.scale) + + out_list = [] + for i in range(repeat_tensors): + out_list += [a.clone().to(out_type), b.clone().to(out_type)] + + if inplace: + in_list = out_list + else: + in_list = [out.clone().to(in_type) for out in out_list] + + applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale) + + self.assertTrue(all([torch.allclose(out, self.ref.to(out_type)) for out in out_list])) + self.assertTrue(self.overflow_buf.item() == 0) + + def find_inf(self, sizea, sizeb, applier, repeat_tensors, in_type, out_type, t, ind, val, inplace=False): + self.overflow_buf.zero_() + a = torch.cuda.FloatTensor(sizea).fill_(self.scale) + b = torch.cuda.FloatTensor(sizeb).fill_(self.scale) + + out_list = [] + for i in range(repeat_tensors): + out_list += [a.clone().to(out_type), b.clone().to(out_type)] + + if inplace: + in_list = out_list + else: + in_list = [out.clone().to(in_type) for out in out_list] + + applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale) + + self.overflow_buf.zero_() + in_list[t][ind] = val + applier(multi_tensor_scale, self.overflow_buf, [in_list, out_list], 1./self.scale) + self.assertTrue(self.overflow_buf.item()) + + # Currently, the fused kernel gives a hard error if you attempt to downscale + # into fp16 output, which imo is the desired behavior. Maybe someday we + # will learn otherwise. + # @unittest.skipIf(disabled, "amp_C is unavailable") + # def test_fp16_to_fp16(self): + # self.downscale(self.fp16, self.fp16, self.fp16_ref) + # + # @unittest.skipIf(disabled, "amp_C is unavailable") + # def test_fp32_to_fp16(self): + # self.downscale(self.fp32, self.fp16, self.fp16_ref) + + @unittest.skipIf(disabled, "amp_C is unavailable") + def test_fuzz(self): + input_size_pairs = ( + (7777*77, 555*555), + (777, 555), + (555, 2048*32+1), + (2048*32+1, 555), + (555, 2048*32), + (2048*32, 555), + (33333, 555), + (555, 33333)) + appliers = ( + MultiTensorApply(2048*32), + MultiTensorApply(333), + MultiTensorApply(33333)) + repeat_tensors = ( + 1, + 55) + + for sizea, sizeb in input_size_pairs: + for applier in appliers: + for repeat in repeat_tensors: + for in_type in (torch.float32, torch.float16): + for out_type in (torch.float32, torch.float16): + for inplace in (True, False): + if inplace is True and (out_type is not in_type): + continue + else: + self.downscale(sizea, sizeb, applier, repeat, in_type, out_type, inplace=inplace) + self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + 0, 0, float('nan'), inplace=inplace) + self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + 2*repeat-1, sizeb-1, float('inf'), inplace=inplace) + self.find_inf(sizea, sizeb, applier, repeat, in_type, out_type, + 2*(repeat//2), sizea//2, float('inf'), inplace=inplace) + + + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_multiple_models_optimizers_losses.py b/apex/tests/L0/run_amp/test_multiple_models_optimizers_losses.py new file mode 100644 index 00000000..068c8453 --- /dev/null +++ b/apex/tests/L0/run_amp/test_multiple_models_optimizers_losses.py @@ -0,0 +1,762 @@ +import unittest + +import functools as ft +import itertools as it + +from apex import amp +from apex.amp import _amp_state +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn import Parameter + +from utils import common_init, HALF, FLOAT,\ + ALWAYS_HALF, ALWAYS_FLOAT, MATCH_INPUT + +class MyModel(torch.nn.Module): + def __init__(self, unique): + super(MyModel, self).__init__() + self.weight0 = Parameter(unique + + torch.arange(2, device='cuda', dtype=torch.float32)) + self.weight1 = Parameter(1. + unique + torch.arange(2, device='cuda', dtype=torch.float16)) + + @staticmethod + def ops(input, weight0, weight1): + return ((input*(weight0.float()))*(weight1.float())).sum() + + def forward(self, input): + return self.ops(input, self.weight0, self.weight1) + +# Abandon all hope, ye who enter here. + +# This is hands down the ugliest code I have ever written, but it succeeds in testing +# multiple models/optimizers/losses fairly thoroughly. Many of the different test cases +# require slightly divergent code in a way that seems near-impossible to genericize into a simple +# cross product or nested loops. + +class TestMultipleModelsOptimizersLosses(unittest.TestCase): + def setUp(self): + self.x = torch.ones((2), device='cuda', dtype=torch.float32) + common_init(self) + + def tearDown(self): + pass + + def test_2models2losses1optimizer(self): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.125) + + reference_grads = [] + for i in range(2): + optimizer.zero_grad() + loss0 = model0(self.x) + loss1 = model1(self.x) + loss0.backward() + loss1.backward() + + reference_grads.append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + optimizer.step() + + final_params = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (True, False): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + else: + iters = 2 + + model0 = MyModel(1) + model1 = MyModel(2) + + models = [model0, model1] + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.125) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1], optimizer = amp.initialize( + [model0, model1], + optimizer, + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer.zero_grad() + + loss0 = model0(self.x) + loss1 = model1(self.x) + + with amp.scale_loss(loss0, optimizer, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if inject_inf_loc == "fp32": + model0.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model0.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, optimizer, loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if inject_inf_loc == "fp32": + model1.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model1.weight1.grad[0] = float('inf') + + if i != inject_inf: + for param, reference_grad in zip(amp.master_params(optimizer), + reference_grads[unskipped]): + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float())) + unskipped += 1 + optimizer.step() + + model_params = [p for p in model0.parameters()] + [p for p in model1.parameters()] + for model, master, reference in zip( + model_params, + amp.master_params(optimizer), + final_params): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + + def test_3models2losses1optimizer(self): + + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}, + {'params' : model2.parameters(), 'lr' : 0.125}], + momentum=0.125) + + reference_grads = [] + for i in range(2): + optimizer.zero_grad() + loss0 = model0(self.x) + model2(self.x) + loss1 = model1(self.x) + model2(self.x) + loss0.backward() + loss1.backward() + + reference_grads.append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()] + + [param.grad.data.clone() for param in model2.parameters()]) + + optimizer.step() + + + final_params = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + \ + [param.data.clone() for param in model2.parameters()] + + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (True, False): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + if which_backward == 0: + which_models = (0, 2) + elif which_backward == 1: + which_models = (1, 2) + else: + iters = 2 + which_models = (None,) + + for which_model in which_models: + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + models = [model0, model1, model2] + + optimizer = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 0.5}, + {'params' : model2.parameters(), 'lr' : 0.125}], + momentum=0.125) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1, model2], optimizer = amp.initialize( + [model0, model1, model2], + optimizer, + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer.zero_grad() + + # print("opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} which_model {} use_multiple_loss_scalers {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, which_model, use_multiple_loss_scalers)) + + loss0 = model0(self.x) + model2(self.x) + loss1 = model1(self.x) + model2(self.x) + + with amp.scale_loss(loss0, optimizer, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if which_model == 0: + inj_model = model0 + elif which_model == 2: + inj_model = model2 + else: + raise RuntimeError(which_model + " invalid for loss 0") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, optimizer, loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if which_model == 1: + inj_model = model1 + elif which_model == 2: + inj_model = model2 + else: + raise RuntimeError(which_model + " invalid for loss 1 ") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + + if i != inject_inf: + for param, reference_grad in zip(amp.master_params(optimizer), + reference_grads[unskipped]): + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float())) + unskipped += 1 + + optimizer.step() + + model_params = [p for p in model0.parameters()] + \ + [p for p in model1.parameters()] + \ + [p for p in model2.parameters()] + for model, master, reference in zip( + model_params, + amp.master_params(optimizer), + final_params): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + + def test_2models2losses2optimizers(self): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.25) + + # Don't do it like this: reference_grads = [[]]*5 + # because then it creates a list of 5 references to the same "[]" and appending + # to any of them effectively makes you append to all of them, which multiplies + # the resulting size of reference_grads by 5x and needless to say makes the test fail. + reference_grads = [[], [], [], [], []] + final_params = [None, None, None, None, None] + for i in range(2): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + loss1 = model1(self.x) + loss0.backward() + loss1.backward() + + reference_grads[0].append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + optimizer0.step() + optimizer1.step() + + final_params[0] = [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + def what_got_skipped(which_iter, which_backward): + if which_iter == 0 and which_backward == 0: + return 1 + if which_iter == 0 and which_backward == 1: + return 2 + if which_iter == 1 and which_backward == 0: + return 3 + if which_iter == 1 and which_backward == 1: + return 4 + return 0 + + for which_iter in (0,1): + for which_backward in (0,1): + model0 = MyModel(1) + model1 = MyModel(2) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.25) + + for i in range(3): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + loss1 = model1(self.x) + loss0.backward() + loss1.backward() + + if i != which_iter: + reference_grads[what_got_skipped(which_iter, which_backward)].append( + [param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + if i == which_iter: + if which_backward == 0: + optimizer1.step() + else: + optimizer0.step() + else: + optimizer0.step() + optimizer1.step() + + final_params[what_got_skipped(which_iter, which_backward)] = \ + [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (True, False): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + else: + iters = 2 + + model0 = MyModel(1) + model1 = MyModel(2) + + models = [model0, model1] + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}], + momentum=0.125) + optimizer1 = torch.optim.SGD([{'params' : model1.parameters(), 'lr' : 0.5}], + momentum=0.25) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1], [optimizer0, optimizer1] = amp.initialize( + [model0, model1], + [optimizer0, optimizer1], + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer0.zero_grad() + optimizer1.zero_grad() + + loss0 = model0(self.x) + loss1 = model1(self.x) + + with amp.scale_loss(loss0, optimizer0, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if inject_inf_loc == "fp32": + model0.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model0.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, optimizer1, loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if inject_inf_loc == "fp32": + model1.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + model1.weight1.grad[0] = float('inf') + + # print("opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers)) + + if i != inject_inf: + master_params = list(amp.master_params(optimizer0)) + \ + list(amp.master_params(optimizer1)) + for param, reference_grad in zip(master_params, + reference_grads[what_got_skipped(inject_inf, which_backward)][unskipped]): + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float())) + unskipped += 1 + + optimizer0.step() + optimizer1.step() + + model_params = [p for p in model0.parameters()] + [p for p in model1.parameters()] + master_params = [p for p in amp.master_params(optimizer0)] + \ + [p for p in amp.master_params(optimizer1)] + for model, master, reference in zip( + model_params, + master_params, + final_params[what_got_skipped(inject_inf, which_backward)]): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + + def test_3models2losses2optimizers(self): + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 1.0}], + momentum=0.5) + optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}], + momentum=0.25) + + # Again, can't do this: reference_grads = [[]]*9 + reference_grads = [[], [], [], [], [], [], [], [], []] + final_params = [None, None, None, None, None, None, None, None, None] + for i in range(2): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + model1(self.x) + loss1 = model2(self.x) + model1(self.x) + loss0.backward() + loss1.backward() + + reference_grads[0].append([param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + optimizer0.step() + optimizer1.step() + + final_params[0] = \ + [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + \ + [param.data.clone() for param in model2.parameters()] + + def what_got_skipped(which_iter, which_backward, which_model): + if which_iter == 0: + if which_backward == 0: + if which_model == 0: + return 1 + if which_model == 1: + return 2 + if which_backward == 1: + if which_model == 2: + return 3 + if which_model == 1: + return 4 + if which_iter == 1: + if which_backward == 0: + if which_model == 0: + return 5 + if which_model == 1: + return 6 + if which_backward == 1: + if which_model == 2: + return 7 + if which_model == 1: + return 8 + return 0 + + for which_iter in (0,1): + for which_backward in (0,1): + if which_backward == 0: + which_models = (0,1) + if which_backward == 1: + which_models = (2,1) + for which_model in which_models: + + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 1.0}], + momentum=0.5) + optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}], + momentum=0.25) + + for i in range(3): + optimizer0.zero_grad() + optimizer1.zero_grad() + loss0 = model0(self.x) + model1(self.x) + loss1 = model2(self.x) + model1(self.x) + loss0.backward() + loss1.backward() + + if i != which_iter: + reference_grads[what_got_skipped(which_iter, + which_backward, which_model)].append( + [param.grad.data.clone() for param in model0.parameters()] + + [param.grad.data.clone() for param in model1.parameters()]) + + if i == which_iter: + if which_backward == 0: + # if which_model == 0: + optimizer1.step() + # if which_model == 1: + # optimizer1.step() + if which_backward == 1: + # if which_model == 2: + # optimizer0.step() + # if which_model == 1: + continue + else: + optimizer0.step() + optimizer1.step() + + final_params[what_got_skipped(which_iter, which_backward, which_model)] = \ + [param.data.clone() for param in model0.parameters()] + \ + [param.data.clone() for param in model1.parameters()] + \ + [param.data.clone() for param in model2.parameters()] + + for opt_level in ("O0", "O1", "O2", "O3"): + for how_to_zero in ("none", "model", "optimizer"): + for use_multiple_loss_scalers in (True, False): + if opt_level == "O1" or opt_level == "O2": + inject_inf_iters = (-1, 0, 1) + else: + inject_inf_iters = (-1,) + + for inject_inf in inject_inf_iters: + if inject_inf >= 0: + inject_inf_locs = ("fp16", "fp32") + which_backwards = (0, 1) + else: + inject_inf_locs = ("fdsa",) + which_backwards = (None,) + + for inject_inf_loc in inject_inf_locs: + for which_backward in which_backwards: + if use_multiple_loss_scalers: + num_losses = 2 + loss_ids = [0, 1] + else: + num_losses = 1 + loss_ids = [0, 0] + + if inject_inf >= 0: + iters = 3 + if which_backward == 0: + which_models = (0, 1) + elif which_backward == 1: + which_models = (2, 1) + else: + iters = 2 + which_models = (None,) + + for which_model in which_models: + model0 = MyModel(1) + model1 = MyModel(2) + model2 = MyModel(3) + + models = [model0, model1, model2] + + optimizer0 = torch.optim.SGD([{'params' : model0.parameters(), 'lr' : 0.25}, + {'params' : model1.parameters(), 'lr' : 1.0}], + momentum=0.5) + optimizer1 = torch.optim.SGD([{'params' : model2.parameters(), 'lr' : 0.5}], + momentum=0.25) + + _amp_state.allow_incoming_model_not_fp32 = True + [model0, model1, model2], [optimizer0, optimizer1] = amp.initialize( + [model0, model1, model2], + [optimizer0, optimizer1], + opt_level=opt_level, + verbosity=0, + cast_model_type=False, + num_losses=num_losses) + _amp_state.allow_incoming_model_not_fp32 = False + + _amp_state.loss_scalers[0]._loss_scale = 4.0 + if use_multiple_loss_scalers: + _amp_state.loss_scalers[1]._loss_scale = 16.0 + + unskipped = 0 + for i in range(iters): + if how_to_zero == "none": + for model in models: + for param in model.parameters(): + param.grad = None + elif how_to_zero == "model": + for model in models: + model.zero_grad() + else: + optimizer0.zero_grad() + optimizer1.zero_grad() + + loss0 = model0(self.x) + model1(self.x) + loss1 = model2(self.x) + model1(self.x) + + with amp.scale_loss(loss0, optimizer0, loss_id=loss_ids[0]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 0: + if which_model == 0: + inj_model = model0 + elif which_model == 1: + inj_model = model1 + else: + raise RuntimeError(which_model + " invalid for loss 0") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + with amp.scale_loss(loss1, [optimizer0, optimizer1], loss_id=loss_ids[1]) as scaled_loss: + scaled_loss.backward() + if i == inject_inf and which_backward == 1: + if which_model == 2: + inj_model = model2 + elif which_model == 1: + inj_model = model1 + else: + raise RuntimeError(which_model + " invalid for loss 1 ") + if inject_inf_loc == "fp32": + inj_model.weight0.grad[0] = float('inf') + elif inject_inf_loc == "fp16": + inj_model.weight1.grad[0] = float('inf') + + if i != inject_inf: + master_params = list(amp.master_params(optimizer0)) + \ + list(amp.master_params(optimizer1)) + for param, reference_grad in zip(master_params, + reference_grads[what_got_skipped(inject_inf, + which_backward, which_model)][unskipped]): + self.assertTrue(torch.allclose(param.grad.float(), reference_grad.float())) + unskipped += 1 + + optimizer0.step() + optimizer1.step() + + model_params = [p for p in model0.parameters()] + \ + [p for p in model1.parameters()] + \ + [p for p in model2.parameters()] + master_params = [p for p in amp.master_params(optimizer0)] + \ + [p for p in amp.master_params(optimizer1)] + + # print("opt_level {} i {} inject_inf {} which_backward {} inject_inf_loc {} use_multiple_loss_scalers {} which_model {}".format(opt_level, i, inject_inf, which_backward, inject_inf_loc, use_multiple_loss_scalers, which_model)) + + for model, master, reference in zip( + model_params, + master_params, + final_params[what_got_skipped(inject_inf, which_backward, which_model)]): + self.assertTrue(torch.allclose(model, reference)) + self.assertTrue(torch.allclose(model, master.to(model.dtype))) + + if opt_level == "O1": + _amp_state.handle._deactivate() + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_promotion.py b/apex/tests/L0/run_amp/test_promotion.py new file mode 100644 index 00000000..f5ef30c1 --- /dev/null +++ b/apex/tests/L0/run_amp/test_promotion.py @@ -0,0 +1,75 @@ +import unittest + +import itertools as it + +from apex import amp +import torch +from torch import nn +import torch.nn.functional as F + +from utils import common_init, HALF, FLOAT, DTYPES + +class TestPromotion(unittest.TestCase): + def setUp(self): + self.handle = amp.init(enabled=True) + common_init(self) + + def tearDown(self): + self.handle._deactivate() + + def run_binary_promote_test(self, fns, input_shape, x_inplace=False): + type_pairs = it.product(DTYPES, DTYPES) + for fn, (xtype, ytype) in it.product(fns, type_pairs): + x = torch.randn(input_shape, dtype=xtype).requires_grad_() + x_leaf = x + if x_inplace: + # We need a non-leaf to call in place on + x = x.clone() + y = torch.randn(input_shape, dtype=ytype) + out = fn(x, y) + if x_inplace: + # In place: always match xtype + self.assertEqual(out.type(), x.type()) + else: + # Out of place: match widest type + if xtype == torch.float or ytype == torch.float: + self.assertEqual(out.type(), FLOAT) + else: + self.assertEqual(out.type(), HALF) + out.float().sum().backward() + self.assertEqual(x_leaf.grad.dtype, xtype) + + def test_atan2_matches_widest(self): + fns = [lambda x, y : torch.atan2(x, y), + lambda x, y : x.atan2(y)] + self.run_binary_promote_test(fns, (self.b,)) + + def test_mul_matches_widest(self): + fns = [lambda x, y : torch.mul(x, y), + lambda x, y: x.mul(y)] + self.run_binary_promote_test(fns, (self.b,)) + + def test_cat_matches_widest(self): + shape = self.b + ys = [torch.randn(shape, dtype=torch.half) for _ in range(5)] + x_float = torch.randn(shape) + out = torch.cat(ys + [x_float]) + self.assertEqual(out.type(), FLOAT) + x_half = torch.randn(shape, dtype=torch.half) + out = torch.cat(ys + [x_half]) + self.assertEqual(out.type(), HALF) + + def test_inplace_exp_is_error_for_half(self): + xs = torch.randn(self.b) + xs.exp_() + self.assertEqual(xs.type(), FLOAT) + xs = torch.randn(self.b, dtype=torch.half) + with self.assertRaises(NotImplementedError): + xs.exp_() + + def test_inplace_add_matches_self(self): + fn = lambda x, y: x.add_(y) + self.run_binary_promote_test([fn], (self.b,), x_inplace=True) + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/test_rnn.py b/apex/tests/L0/run_amp/test_rnn.py new file mode 100644 index 00000000..c49a5f00 --- /dev/null +++ b/apex/tests/L0/run_amp/test_rnn.py @@ -0,0 +1,116 @@ +import unittest + +from apex import amp +import random +import torch +from torch import nn + +from utils import common_init, HALF + +class TestRnnCells(unittest.TestCase): + def setUp(self): + self.handle = amp.init(enabled=True) + common_init(self) + + def tearDown(self): + self.handle._deactivate() + + def run_cell_test(self, cell, state_tuple=False): + shape = (self.b, self.h) + for typ in [torch.float, torch.half]: + xs = [torch.randn(shape, dtype=typ).requires_grad_() + for _ in range(self.t)] + hidden_fn = lambda: torch.zeros(shape, dtype=typ) + if state_tuple: + hidden = (hidden_fn(), hidden_fn()) + else: + hidden = hidden_fn() + outputs = [] + for i in range(self.t): + hidden = cell(xs[i], hidden) + if state_tuple: + output = hidden[0] + else: + output = hidden + outputs.append(output) + for y in outputs: + self.assertEqual(y.type(), HALF) + outputs[-1].float().sum().backward() + for i, x in enumerate(xs): + self.assertEqual(x.grad.dtype, x.dtype) + + def test_rnn_cell_is_half(self): + cell = nn.RNNCell(self.h, self.h) + self.run_cell_test(cell) + + def test_gru_cell_is_half(self): + cell = nn.GRUCell(self.h, self.h) + self.run_cell_test(cell) + + def test_lstm_cell_is_half(self): + cell = nn.LSTMCell(self.h, self.h) + self.run_cell_test(cell, state_tuple=True) + +class TestRnns(unittest.TestCase): + def setUp(self): + self.handle = amp.init(enabled=True) + common_init(self) + + def tearDown(self): + self.handle._deactivate() + + def run_rnn_test(self, rnn, layers, bidir, state_tuple=False): + for typ in [torch.float, torch.half]: + x = torch.randn((self.t, self.b, self.h), dtype=typ).requires_grad_() + hidden_fn = lambda: torch.zeros((layers + (layers * bidir), + self.b, self.h), dtype=typ) + if state_tuple: + hidden = (hidden_fn(), hidden_fn()) + else: + hidden = hidden_fn() + output, _ = rnn(x, hidden) + self.assertEqual(output.type(), HALF) + output[-1, :, :].float().sum().backward() + self.assertEqual(x.grad.dtype, x.dtype) + + def test_rnn_is_half(self): + configs = [(1, False), (2, False), (2, True)] + for layers, bidir in configs: + rnn = nn.RNN(input_size=self.h, hidden_size=self.h, num_layers=layers, + nonlinearity='relu', bidirectional=bidir) + self.run_rnn_test(rnn, layers, bidir) + + def test_gru_is_half(self): + configs = [(1, False), (2, False), (2, True)] + for layers, bidir in configs: + rnn = nn.GRU(input_size=self.h, hidden_size=self.h, num_layers=layers, + bidirectional=bidir) + self.run_rnn_test(rnn, layers, bidir) + + def test_lstm_is_half(self): + configs = [(1, False), (2, False), (2, True)] + for layers, bidir in configs: + rnn = nn.LSTM(input_size=self.h, hidden_size=self.h, num_layers=layers, + bidirectional=bidir) + self.run_rnn_test(rnn, layers, bidir, state_tuple=True) + + def test_rnn_packed_sequence(self): + num_layers = 2 + rnn = nn.RNN(input_size=self.h, hidden_size=self.h, num_layers=num_layers) + for typ in [torch.float, torch.half]: + x = torch.randn((self.t, self.b, self.h), dtype=typ).requires_grad_() + lens = sorted([random.randint(self.t // 2, self.t) for _ in range(self.b)], + reverse=True) + # `pack_padded_sequence` breaks if default tensor type is non-CPU + torch.set_default_tensor_type(torch.FloatTensor) + lens = torch.tensor(lens, dtype=torch.int64, device=torch.device('cpu')) + packed_seq = nn.utils.rnn.pack_padded_sequence(x, lens) + torch.set_default_tensor_type(torch.cuda.FloatTensor) + hidden = torch.zeros((num_layers, self.b, self.h), dtype=typ) + output, _ = rnn(packed_seq, hidden) + self.assertEqual(output.data.type(), HALF) + output.data.float().sum().backward() + self.assertEqual(x.grad.dtype, x.dtype) + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_amp/utils.py b/apex/tests/L0/run_amp/utils.py new file mode 100644 index 00000000..7aa20c36 --- /dev/null +++ b/apex/tests/L0/run_amp/utils.py @@ -0,0 +1,21 @@ +import torch + +HALF = 'torch.cuda.HalfTensor' +FLOAT = 'torch.cuda.FloatTensor' + +DTYPES = [torch.half, torch.float] + +ALWAYS_HALF = {torch.float: HALF, + torch.half: HALF} +ALWAYS_FLOAT = {torch.float: FLOAT, + torch.half: FLOAT} +MATCH_INPUT = {torch.float: FLOAT, + torch.half: HALF} + +def common_init(test_case): + test_case.h = 64 + test_case.b = 16 + test_case.c = 16 + test_case.k = 3 + test_case.t = 10 + torch.set_default_tensor_type(torch.cuda.FloatTensor) diff --git a/apex/tests/L0/run_deprecated/test_deprecated_warning.py b/apex/tests/L0/run_deprecated/test_deprecated_warning.py new file mode 100644 index 00000000..f1f33f76 --- /dev/null +++ b/apex/tests/L0/run_deprecated/test_deprecated_warning.py @@ -0,0 +1,56 @@ +import unittest + +import torch + +import apex +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase + + +def init_model_and_optimizer(): + model = torch.nn.Linear(1, 1, bias=False).cuda() + optimizer = torch.optim.SGD(model.parameters(), 1.0) + return model, optimizer + + +@unittest.skipUnless(torch.cuda.is_available(), "") +class TestDeprecatedWarning(unittest.TestCase): + + def test_amp(self): + model, optimizer = init_model_and_optimizer() + with self.assertWarns(apex.DeprecatedFeatureWarning): + _ = apex.amp.initialize(model, optimizer) + + def test_fp16_model(self): + model, _ = init_model_and_optimizer() + with self.assertWarns(apex.DeprecatedFeatureWarning): + _ = apex.fp16_utils.FP16Model(model) + + def test_fp16_optimizer(self): + _, optimizer = init_model_and_optimizer() + with self.assertWarns(apex.DeprecatedFeatureWarning): + _ = apex.fp16_utils.FP16_Optimizer(optimizer) + + def test_fp16_loss_scaler(self): + with self.assertWarns(apex.DeprecatedFeatureWarning): + apex.fp16_utils.LossScaler() + + +class TestParallel(NcclDistributedTestBase): + + @property + def world_size(self): + return min(torch.cuda.device_count(), 2) + + def test_distributed_data_parallel(self): + model, _ = init_model_and_optimizer() + with self.assertWarns(apex.DeprecatedFeatureWarning): + _ = apex.parallel.DistributedDataParallel(model) + + def test_convert_syncbn_model(self): + model, _ = init_model_and_optimizer() + with self.assertWarns(apex.DeprecatedFeatureWarning): + _ = apex.parallel.convert_syncbn_model(model) + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/tests/L0/run_fp16util/__init__.py b/apex/tests/L0/run_fp16util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/tests/L0/run_fp16util/test_fp16util.py b/apex/tests/L0/run_fp16util/test_fp16util.py new file mode 100644 index 00000000..eecddbc0 --- /dev/null +++ b/apex/tests/L0/run_fp16util/test_fp16util.py @@ -0,0 +1,75 @@ +import unittest + +import torch +import torch.nn as nn + +from apex.fp16_utils import FP16Model + + +class DummyBlock(nn.Module): + def __init__(self): + super(DummyBlock, self).__init__() + + self.conv = nn.Conv2d(10, 10, 2) + self.bn = nn.BatchNorm2d(10, affine=True) + + def forward(self, x): + return self.conv(self.bn(x)) + + +class DummyNet(nn.Module): + def __init__(self): + super(DummyNet, self).__init__() + + self.conv1 = nn.Conv2d(3, 10, 2) + self.bn1 = nn.BatchNorm2d(10, affine=False) + self.db1 = DummyBlock() + self.db2 = DummyBlock() + + def forward(self, x): + out = x + out = self.conv1(out) + out = self.bn1(out) + out = self.db1(out) + out = self.db2(out) + return out + + +class DummyNetWrapper(nn.Module): + def __init__(self): + super(DummyNetWrapper, self).__init__() + + self.bn = nn.BatchNorm2d(3, affine=True) + self.dn = DummyNet() + + def forward(self, x): + return self.dn(self.bn(x)) + + +class TestFP16Model(unittest.TestCase): + def setUp(self): + self.N = 64 + self.C_in = 3 + self.H_in = 16 + self.W_in = 32 + self.in_tensor = torch.randn((self.N, self.C_in, self.H_in, self.W_in)).cuda() + self.orig_model = DummyNetWrapper().cuda() + self.fp16_model = FP16Model(self.orig_model) + + def test_params_and_buffers(self): + exempted_modules = [ + self.fp16_model.network.bn, + self.fp16_model.network.dn.db1.bn, + self.fp16_model.network.dn.db2.bn, + ] + for m in self.fp16_model.modules(): + expected_dtype = torch.float if (m in exempted_modules) else torch.half + for p in m.parameters(recurse=False): + assert p.dtype == expected_dtype + for b in m.buffers(recurse=False): + assert b.dtype in (expected_dtype, torch.int64) + + def test_output_is_half(self): + out_tensor = self.fp16_model(self.in_tensor) + assert out_tensor.dtype == torch.half + diff --git a/apex/tests/L0/run_fused_layer_norm/test_fused_layer_norm.py b/apex/tests/L0/run_fused_layer_norm/test_fused_layer_norm.py new file mode 100644 index 00000000..3a8b5249 --- /dev/null +++ b/apex/tests/L0/run_fused_layer_norm/test_fused_layer_norm.py @@ -0,0 +1,273 @@ +import torch + +from apex.normalization import FusedLayerNorm +from apex.normalization import FusedRMSNorm +from apex.normalization import MixedFusedLayerNorm +from apex.normalization import MixedFusedRMSNorm + +from torch.testing._internal import common_utils +from torch.testing._internal.common_device_type import instantiate_device_type_tests + +from itertools import product + +def _prep_inputs(batch_size, normalized_shape, dtype): + shape = (batch_size, *normalized_shape) + fused = torch.randn(shape).cuda().requires_grad_(True) + with torch.no_grad(): + native = fused.clone().to(dtype).requires_grad_(True) + return native, fused + +autocast_dtypes = (torch.half, torch.bfloat16) if torch.cuda.is_bf16_supported() else (torch.half,) + +class TestFusedLayerNorm(common_utils.TestCase): + + def _test_fused_layer_norm( + self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + fwd_thresholds=dict(rtol=None, atol=None), bwd_thresholds=dict(rtol=None, atol=None) + ): + + normalized_shape = [32, 16] + + if not mixed_fused: + module_cpu_ = FusedLayerNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine).cpu() + module_cuda_ = FusedLayerNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine).to(device="cuda", dtype=dtype) + else: + assert elementwise_affine + module_cpu_ = MixedFusedLayerNorm( + normalized_shape=normalized_shape).cpu() + module_cuda_ = MixedFusedLayerNorm( + normalized_shape=normalized_shape).to(device="cuda", dtype=dtype) + + torch.cuda.manual_seed(42) + if contiguous: + input_shape = [batch_size] + normalized_shape + input_ = torch.randn(input_shape, device="cpu").requires_grad_(True) + input_cuda_ = input_.to(device="cuda", dtype=dtype).detach().requires_grad_(True) + self.assertTrue(input_.is_contiguous()) + self.assertTrue(input_cuda_.is_contiguous()) + else: + input_shape = [batch_size] + normalized_shape + input_shape = [batch_size * 3] + [normalized_shape[0] * 5, normalized_shape[1] * 3] + input_src_ = torch.randn(input_shape, device="cpu") + input_ = input_src_[::3, ::5, ::3].detach().requires_grad_(True) + input_cuda_ = input_src_.to(device="cuda", dtype=dtype)[::3, ::5, ::3].detach().requires_grad_(True) + # make sure that tensors are NOT contiguous. + self.assertFalse(input_.is_contiguous()) + self.assertFalse(input_cuda_.is_contiguous()) + out_cpu_ = module_cpu_(input_) + gO = torch.rand_like(out_cpu_) + out_cpu_.backward(gO) + out_cuda_ = module_cuda_(input_cuda_) + + gO = gO.to(device="cuda", dtype=dtype) + out_cuda_.backward(gO) + self.assertFalse(out_cpu_.is_cuda) + self.assertTrue(out_cuda_.is_cuda) + torch.testing.assert_close( + out_cpu_.to(device="cuda", dtype=dtype), out_cuda_, **fwd_thresholds) + torch.testing.assert_close( + input_.grad.to(device="cuda", dtype=dtype), input_cuda_.grad, **bwd_thresholds) + + def _test_fused_rms_norm( + self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + fwd_thresholds=dict(rtol=None, atol=None), bwd_thresholds=dict(rtol=None, atol=None) + ): + + normalized_shape = [32, 16] + + if not mixed_fused: + module_cpu_ = FusedRMSNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine).cpu() + module_cuda_ = FusedRMSNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine).to(device="cuda", dtype=dtype) + else: + assert elementwise_affine + module_cpu_ = MixedFusedRMSNorm( + normalized_shape=normalized_shape).cpu() + module_cuda_ = MixedFusedRMSNorm( + normalized_shape=normalized_shape).to(device="cuda", dtype=dtype) + + torch.cuda.manual_seed(42) + if contiguous: + input_shape = [batch_size] + normalized_shape + input_ = torch.randn(input_shape, device="cpu").requires_grad_(True) + input_cuda_ = input_.to(device="cuda", dtype=dtype).detach().requires_grad_(True) + self.assertTrue(input_.is_contiguous()) + self.assertTrue(input_cuda_.is_contiguous()) + else: + input_shape = [batch_size] + normalized_shape + input_shape = [batch_size * 3] + [normalized_shape[0] * 5, normalized_shape[1] * 3] + input_src_ = torch.randn(input_shape, device="cpu") + input_ = input_src_[::3, ::5, ::3].detach().requires_grad_(True) + input_cuda_ = input_src_.to(device="cuda", dtype=dtype)[::3, ::5, ::3].detach().requires_grad_(True) + # make sure that tensors are NOT contiguous. + self.assertFalse(input_.is_contiguous()) + self.assertFalse(input_cuda_.is_contiguous()) + out_cpu_ = module_cpu_(input_) + gO = torch.rand_like(out_cpu_) + out_cpu_.backward(gO) + out_cuda_ = module_cuda_(input_cuda_) + + torch.testing.assert_close( + out_cpu_.to(device="cuda", dtype=dtype), out_cuda_.clone().detach(), **fwd_thresholds) + gO = gO.to(device="cuda", dtype=dtype) + out_cuda_.backward(gO) + self.assertFalse(out_cpu_.is_cuda) + self.assertTrue(out_cuda_.is_cuda) + torch.testing.assert_close( + input_.grad.to(device="cuda", dtype=dtype), input_cuda_.grad, **bwd_thresholds) + if elementwise_affine: + torch.testing.assert_close(module_cpu_.weight.grad.to(device="cuda", dtype=dtype), + module_cuda_.weight.grad, **bwd_thresholds) + + # layer norm tests + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16, 65536), (True, False), (False,), (False,), (torch.float,))) + ) + def test_layer_norm_regular(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_layer_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16, 65536), (True, False), (True,), (False,), (torch.float,))) + ) + def test_layer_norm_elemwise(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_layer_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16, 65536), (True, False), (True,), (True,), (torch.float,))) + ) + def test_layer_norm_mixed(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_layer_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16,), (True, False), (True,), (False,), (torch.half,))) + ) + def test_layer_norm_half(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_layer_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + fwd_thresholds=dict(rtol=1e-3, atol=1e-3), bwd_thresholds=dict(rtol=1e-3, atol=1e-3)) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16,), (True, False), (True,), (False,), (torch.bfloat16,))) + ) + def test_layer_norm_bfloat16(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_layer_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + fwd_thresholds=dict(rtol=1.6e-2, atol=3e-4), bwd_thresholds=dict(rtol=1.6e-2, atol=3e-3)) + + # rms norm tests + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16, 65536), (True, False), (False,), (False,), (torch.float,))) + ) + def test_rms_norm_regular(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_rms_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16, 65536), (True, False), (True,), (False,), (torch.float,))) + ) + def test_rms_norm_elemwise(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_rms_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + bwd_thresholds=dict(rtol=2e-3, atol=2e-4)) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16, 65536), (True, False), (True,), (True,), (torch.float,))) + ) + def test_rms_norm_mixed(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_rms_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + bwd_thresholds=dict(rtol=2e-3, atol=2e-4)) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16,), (True, False), (True,), (False,), (torch.half,))) + ) + def test_rms_norm_half(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_rms_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + bwd_thresholds = dict(rtol=1.6e-2, atol=3e-3)) + + @common_utils.parametrize( + "batch_size, contiguous, elementwise_affine, mixed_fused, dtype", + list(product((16,), (True, False), (True,), (False,), (torch.bfloat16,))) + ) + def test_rms_norm_bfloat16(self, batch_size, contiguous, elementwise_affine, mixed_fused, dtype): + self._test_fused_rms_norm(batch_size, contiguous, elementwise_affine, mixed_fused, dtype, + fwd_thresholds=dict(rtol=1.6e-2, atol=3e-4), bwd_thresholds=dict(rtol=1.6e-2, atol=3e-2)) + + @common_utils.parametrize( + "dtype, elementwise_affine", + list(product(autocast_dtypes, (True, False))) + ) + def test_autocast_fused_layer_norm(self, dtype, elementwise_affine): + bf16_fwd_thresholds = dict(rtol=1.6e-2, atol=3e-4) + bf16_bwd_thresholds = dict(rtol=1.6e-2, atol=3e-3) + batch_size = 16 + normalized_shape = [32, 16] + native = torch.nn.LayerNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine + ).to(device="cuda", dtype=dtype) + fused = FusedLayerNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine + ).cuda() + native_x, fused_x = _prep_inputs(batch_size, normalized_shape, dtype) + + expected = native(native_x) + with torch.cuda.amp.autocast(dtype=dtype): + actual = fused(fused_x) + tols = {'rtol': None, 'atol': None} if dtype == torch.half else bf16_fwd_thresholds + # original tests used torch.testing.assert_allclose, which disables dtype checking by default. + # link to issue here: https://github.com/pytorch/pytorch/issues/61844 + torch.testing.assert_close(actual, expected, **tols, check_dtype=False) + + g_native = torch.rand_like(expected) + with torch.no_grad(): + g_fused = g_native.clone() + expected.backward(g_native) + actual.backward(g_fused) + + tols = {'rtol': None, 'atol': None} if dtype == torch.half else bf16_bwd_thresholds + torch.testing.assert_close(native_x.grad, fused_x.grad, **tols, check_dtype=False) + + @common_utils.parametrize( + "dtype, elementwise_affine", + list(product(autocast_dtypes, (True, False))) + ) + def test_autocast_fused_rms_norm(self, dtype, elementwise_affine): + bf16_fwd_thresholds = dict(rtol=1.6e-2, atol=3e-4) + bf16_bwd_thresholds = dict(rtol=1.6e-2, atol=3e-3) + batch_size = 16 + normalized_shape = [32, 16] + native = FusedRMSNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine + ).to(dtype=dtype) + fused = FusedRMSNorm( + normalized_shape=normalized_shape, elementwise_affine=elementwise_affine + ).cuda() + native_x, fused_x = _prep_inputs(batch_size, normalized_shape, dtype) + + expected = native(native_x.cpu()) + with torch.cuda.amp.autocast(dtype=dtype): + actual = fused(fused_x) + tols = {'rtol': None, 'atol': None} if dtype == torch.half else bf16_fwd_thresholds + torch.testing.assert_close(actual, expected.detach().clone().cuda(), **tols, check_dtype=False) + + g_native = torch.rand_like(expected) + with torch.no_grad(): + g_fused = g_native.detach().clone().cuda() + expected.backward(g_native) + actual.backward(g_fused) + + tols = {'rtol': 1e-3, 'atol': 1e-3} if dtype == torch.half else bf16_bwd_thresholds + torch.testing.assert_close(native_x.grad.cuda(), fused_x.grad, **tols, check_dtype=False) + + +instantiate_device_type_tests(TestFusedLayerNorm, globals(), only_for=("cuda",)) + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_instance_norm_nvfuser/__init__.py b/apex/tests/L0/run_instance_norm_nvfuser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/tests/L0/run_instance_norm_nvfuser/test_instance_norm_nvfuser.py b/apex/tests/L0/run_instance_norm_nvfuser/test_instance_norm_nvfuser.py new file mode 100644 index 00000000..dca3c280 --- /dev/null +++ b/apex/tests/L0/run_instance_norm_nvfuser/test_instance_norm_nvfuser.py @@ -0,0 +1,107 @@ +import itertools +import unittest + +import torch +import torch.nn as nn + +import apex +from apex.normalization import InstanceNorm3dNVFuser + + +class TestInstanceNormNVFuser(unittest.TestCase): + dtype = torch.float + track_running_stats = False + channels_last = False + affine = False + batch_size = 5 + channel_size = 7 + spatial_size = 3 + + def init_modules(self): + self.m = InstanceNorm3dNVFuser(self.channel_size, affine=self.affine, track_running_stats=self.track_running_stats, device='cuda', dtype=self.dtype) + self.reference_m = torch.nn.InstanceNorm3d(self.channel_size, affine=self.affine, track_running_stats=self.track_running_stats, device='cuda', dtype=self.dtype) + + def check_same_output(self): + torch.manual_seed(42) + for i in range(2): # exercise JIT + caching + inp = torch.randint(0, 2, (self.batch_size, self.channel_size, self.spatial_size, self.spatial_size, self.spatial_size), device='cuda', requires_grad=True, dtype=self.dtype) + inp2 = inp.detach().clone() + inp2.requires_grad = True + if self.channels_last: + _inp = inp.to(memory_format=torch.channels_last_3d) + else: + _inp = inp + out = self.m(_inp) + out2 = self.reference_m(inp2) + if self.m.running_mean is None: + assert self.reference_m.running_mean is None + assert self.m.running_var is None + assert self.reference_m.running_var is None + else: + torch.testing.assert_close(self.m.running_mean, self.reference_m.running_mean) + if self.dtype == torch.float16: + torch.testing.assert_close(self.m.running_var, self.reference_m.running_var, atol=5e-3, rtol=5e-3) + else: + torch.testing.assert_close(self.m.running_var, self.reference_m.running_var) + torch.testing.assert_close(out, out2) + grad_out = torch.randn_like(inp) + out.backward(grad_out) + out2.backward(grad_out) + if self.dtype == torch.float16: + torch.testing.assert_close(inp.grad, inp2.grad, atol=5e-3, rtol=5e-3) + elif self.dtype == torch.bfloat16: + torch.testing.assert_close(inp.grad, inp2.grad, atol=2e-2, rtol=2e-2) + else: + torch.testing.assert_close(inp.grad, inp2.grad) + if self.m.weight is not None: + if self.dtype == torch.float16: + torch.testing.assert_close(self.m.weight.grad, self.reference_m.weight.grad, atol=5e-2, rtol=5e-2) + elif self.dtype == torch.bfloat16: + torch.testing.assert_close(self.m.weight.grad, self.reference_m.weight.grad, atol=7e-2, rtol=8e-2) + else: + torch.testing.assert_close(self.m.weight.grad, self.reference_m.weight.grad) + if self.m.bias is not None: + if self.dtype == torch.float16: + torch.testing.assert_close(self.m.bias.grad, self.reference_m.bias.grad, atol=5e-3, rtol=7e-2) + elif self.dtype == torch.bfloat16: + torch.testing.assert_close(self.m.bias.grad, self.reference_m.bias.grad, atol=5e-2, rtol=1e-2) + else: + torch.testing.assert_close(self.m.bias.grad, self.reference_m.bias.grad) + + def test_sweep(self): + dtypes = [torch.float, torch.half] + if torch.cuda.get_device_capability() >= (8, 0): + dtypes.append(torch.bfloat16) + for dtype, track_running_stats, channels_last, affine in itertools.product(dtypes, (False, True), (False, True), (False, True)): + with self.subTest(dtype=dtype, track_running_stats=track_running_stats, channels_last=channels_last, affine=affine): + self.dtype = dtype + self.track_running_stats = track_running_stats + self.channels_last = channels_last + self.affine = affine + self.init_modules() + self.check_same_output() + + @unittest.skipIf(torch.cuda.device_count() < 2, "more than 1 GPU required") + def test_multigpu(self): + class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.norm = InstanceNorm3dNVFuser(4) + + def forward(self, x): + x = self.norm(x) + x = torch.sum(x, dim=(1, 2, 3, 4)) + return x + + device = torch.device(f"cuda:1") + model = Model().to(device) + + x = torch.randn(2, 4, 128, 128, 128, device=device, requires_grad=True) + y = torch.randn(2, device=device) + pred = model(x) + loss = nn.functional.mse_loss(pred, y.float()) + loss.backward() + + +if __name__ == "__main__": + unittest.main() diff --git a/apex/tests/L0/run_mlp/test_mlp.py b/apex/tests/L0/run_mlp/test_mlp.py new file mode 100644 index 00000000..6d019dd9 --- /dev/null +++ b/apex/tests/L0/run_mlp/test_mlp.py @@ -0,0 +1,206 @@ +"""Tests for c++ MLP""" +from itertools import product +from time import time + +import torch +from torch import nn +from torch.testing._internal import common_utils +from torch.testing._internal.common_device_type import instantiate_device_type_tests +from torch.testing._internal.common_device_type import onlyCUDA + +from apex.mlp import MLP + + +batch_size = 1024 +mlp_sizes = [480, 1024, 1024, 512, 256, 1] +num_iters = 10 + + +# note(crcrpar): On Ampere, this test should be run without TF32 enabled. +class TestMLP(common_utils.TestCase): + def test_creation(self): + MLP(mlp_sizes) + + def test_numeric(self): + mlp = MLP(mlp_sizes).cuda() + + mlp_layers = [] + for i in range(mlp.num_layers): + linear = nn.Linear(mlp_sizes[i], mlp_sizes[i + 1]) + with torch.no_grad(): + mlp.weights[i].copy_(linear.weight) + mlp.biases[i].copy_(linear.bias) + mlp_layers.append(linear) + mlp_layers.append(nn.ReLU()) + + ref_mlp = nn.Sequential(*mlp_layers).cuda() + + test_input = ( + torch.empty(batch_size, mlp_sizes[0], device="cuda") + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + ref_input = test_input.clone().detach().requires_grad_() + mlp_out = mlp(test_input) + ref_out = ref_mlp(ref_input) + self.assertEqual(mlp_out, ref_out) + + # Use mean value as scalar loss. Multiply 10 to make it big enough not zero out + mlp_out.mean().mul(10.0).backward() + ref_out.mean().mul(10.0).backward() + self.assertEqual(test_input.grad, ref_input.grad) + self.assertEqual(mlp.biases[0].grad, ref_mlp[0].bias.grad) + + def _test_mlp_impl(self, use_activation: str, bias: bool, enable_autocast: bool): + mlp = MLP(mlp_sizes, bias=bias, activation=use_activation).cuda() + + mlp_layers = [] + for i in range(mlp.num_layers): + linear = nn.Linear(mlp_sizes[i], mlp_sizes[i + 1], bias=bias) + with torch.no_grad(): + mlp.weights[i].copy_(linear.weight) + if bias: + mlp.biases[i].copy_(linear.bias) + mlp_layers.append(linear) + if use_activation == "relu": + mlp_layers.append(nn.ReLU()) + if use_activation == "sigmoid": + mlp_layers.append(nn.Sigmoid()) + + ref_mlp = nn.Sequential(*mlp_layers).cuda() + + test_input = ( + torch.empty(batch_size, mlp_sizes[0], device="cuda") + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + ref_input = test_input.clone().detach().requires_grad_() + + with torch.cuda.amp.autocast_mode.autocast(enabled=enable_autocast): + mlp_out = mlp(test_input) + mlp_loss = mlp_out.mean().mul(10.0) + # Use mean value as scalar loss. Multiply 10 to make it big enough not zero out + ref_out = ref_mlp(ref_input) + ref_loss = ref_out.mean().mul(10.0) + + mlp_loss.backward() + ref_loss.backward() + if enable_autocast: + self.assertEqual(mlp_out.dtype, torch.float16) + self.assertEqual(ref_out.dtype, torch.float16) + else: + self.assertEqual(mlp_out, ref_out) + self.assertEqual(test_input.grad, ref_input.grad) + self.assertEqual(mlp.weights[0].grad, ref_mlp[0].weight.grad) + + @common_utils.parametrize( + "use_activation,bias", + list(product(("none", "relu", "sigmoid"), (True, False))), + ) + def test_mlp(self, use_activation: str, bias: bool): + self._test_mlp_impl(use_activation, bias, enable_autocast=False) + + @common_utils.parametrize( + "use_activation,bias", + list(product(("none", "relu", "sigmoid"), (True, False))), + ) + def test_mlp_autocast_fp16(self, use_activation: str, bias: bool): + self._test_mlp_impl(use_activation, bias, enable_autocast=True) + + def test_no_grad(self): + mlp = MLP(mlp_sizes).cuda() + + mlp_layers = [] + for i in range(mlp.num_layers): + linear = nn.Linear(mlp_sizes[i], mlp_sizes[i + 1]) + with torch.no_grad(): + mlp.weights[i].copy_(linear.weight) + mlp.biases[i].copy_(linear.bias) + mlp_layers.append(linear) + mlp_layers.append(nn.ReLU(inplace=True)) + + ref_mlp = nn.Sequential(*mlp_layers).cuda() + + test_input = torch.empty(batch_size, mlp_sizes[0], device="cuda").uniform_(-1.0, 1.0) + ref_input = test_input.clone().detach() + mlp_out = mlp(test_input) + ref_out = ref_mlp(ref_input) + self.assertEqual(mlp_out, ref_out) + + # Use mean value as scalar loss. Multiply 10 to make it big enough not zero out + mlp_out.mean().mul(10.0).backward() + ref_out.mean().mul(10.0).backward() + self.assertEqual(mlp.weights[0].grad, ref_mlp[0].weight.grad) + + def test_performance_half(self): + mlp = MLP(mlp_sizes).cuda().half() + + mlp_layers = [] + for i in range(mlp.num_layers): + linear = nn.Linear(mlp_sizes[i], mlp_sizes[i + 1]) + mlp.weights[i].data.copy_(linear.weight) + mlp.biases[i].data.copy_(linear.bias) + mlp_layers.append(linear) + mlp_layers.append(nn.ReLU(inplace=True)) + + ref_mlp = nn.Sequential(*mlp_layers).cuda().half() + + test_input = ( + torch.empty(batch_size, mlp_sizes[0], device="cuda", dtype=torch.half) + .fill_(10.0) + .requires_grad_() + ) + ref_input = ( + torch.empty(batch_size, mlp_sizes[0], device="cuda", dtype=torch.half) + .fill_(10.0) + .requires_grad_() + ) + + # Warm up GPU + for _ in range(100): + ref_out = ref_mlp(ref_input) + ref_loss = ref_out.mean() + ref_mlp.zero_grad() + ref_loss.backward() + mlp_out = mlp(test_input) + test_loss = mlp_out.mean() + mlp.zero_grad() + test_loss.backward() + + torch.cuda.profiler.start() + torch.cuda.synchronize() + start_time = time() + for _ in range(num_iters): + ref_out = ref_mlp(ref_input) + ref_loss = ref_out.mean() + ref_mlp.zero_grad() + ref_loss.backward() + torch.cuda.synchronize() + stop_time = time() + ref_time = (stop_time - start_time) * 1000.0 / num_iters + print(f"\nPytorch MLP time {ref_time:.4f} ms") + + torch.cuda.synchronize() + start_time = time() + for _ in range(num_iters): + mlp_out = mlp(test_input) + test_loss = mlp_out.mean() + mlp.zero_grad() + test_loss.backward() + torch.cuda.synchronize() + stop_time = time() + actual_time = (stop_time - start_time) * 1000.0 / num_iters + print(f"C++ MLP time {actual_time:.4f} ms") + torch.cuda.profiler.stop() + self.assertLessEqual( + actual_time, + ref_time, + msg=f"Custom extension took {actual_time:.4f} while PyTorch took {ref_time:.4f}", + ) + + +instantiate_device_type_tests(TestMLP, globals(), only_for=("cuda",)) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_optimizers/__init__.py b/apex/tests/L0/run_optimizers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/tests/L0/run_optimizers/test_adam.py b/apex/tests/L0/run_optimizers/test_adam.py new file mode 100644 index 00000000..ef51d25b --- /dev/null +++ b/apex/tests/L0/run_optimizers/test_adam.py @@ -0,0 +1,238 @@ +import copy +import math +import random +import unittest + +import torch +import torch.nn.functional as F +from torch import nn + +try: + import apex +except ImportError as e: + HAS_APEX = False +else: + HAS_APEX = True + + +class Model(torch.nn.Module): + def __init__(self): + super(Model, self).__init__() + self.conv1 = nn.Conv2d(1, 6, 5) + self.relu1 = nn.ReLU() + self.pool1 = nn.MaxPool2d(2) + self.conv2 = nn.Conv2d(6, 16, 5) + self.relu2 = nn.ReLU() + self.pool2 = nn.MaxPool2d(2) + self.fc1 = nn.Linear(256, 120) + self.relu3 = nn.ReLU() + self.fc2 = nn.Linear(120, 84) + self.relu4 = nn.ReLU() + self.fc3 = nn.Linear(84, 10) + self.relu5 = nn.ReLU() + + def forward(self, x): + y = self.conv1(x) + y = self.relu1(y) + y = self.pool1(y) + y = self.conv2(y) + y = self.relu2(y) + y = self.pool2(y) + y = y.reshape(y.shape[0], -1) + y = self.fc1(y) + y = self.relu3(y) + y = self.fc2(y) + y = self.relu4(y) + y = self.fc3(y) + y = self.relu5(y) + return y + + +@unittest.skipIf(not HAS_APEX, "`apex` is not found.") +class AdamTest(unittest.TestCase): + def setUp(self, seed=0): + super().setUp() + torch.manual_seed(seed) + + self.model = Model().cuda() + self.model_ = Model().cuda() + self.model_.load_state_dict(copy.deepcopy(self.model.state_dict())) + + self.lr = 0.00001 + params = [p for p in self.model.parameters() if p.requires_grad] + self.optimizer = torch.optim.Adam(params, lr=self.lr) + + def testGradScaler(self): + params_ = [p for p in self.model_.parameters() if p.requires_grad] + optimizer_ = apex.optimizers.FusedAdam(params_, lr=self.lr, capturable=False) + scaler = torch.cuda.amp.GradScaler(enabled=True) + scaler_ = torch.cuda.amp.GradScaler(enabled=True) + + for i in range(100): + x = torch.rand([32, 1, 28, 28]).cuda().to(memory_format=torch.channels_last) + x_ = x.clone() + gt = torch.rand([32, 10]).cuda() + gt_ = gt.clone() + + # Reference + with torch.cuda.amp.autocast(enabled=True): + y = self.model(x) + loss = ((gt - y) ** 2).mean() + + scaler.scale(loss).backward() + scaler.step(self.optimizer) + scaler.update() + + # DUT + with torch.cuda.amp.autocast(enabled=True): + y = self.model_(x) + loss_ = ((gt_ - y) ** 2).mean() + + scaler_.scale(loss_).backward() + scaler_.step(optimizer_) + scaler_.update() + + for module in zip(self.model.modules(), self.model_.modules()): + m = module[0] + m_ = module[1] + if isinstance(m, nn.Conv2d) or isinstance(m_, nn.Linear): + torch.testing.assert_close(m.weight, m_.weight, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(m.weight.grad, m_.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + # Init for next iteration + self.optimizer.zero_grad() + optimizer_.zero_grad() + + self.model_.load_state_dict(copy.deepcopy(self.model.state_dict())) + + def testGradScalerCapturable(self): + params_ = [p for p in self.model_.parameters() if p.requires_grad] + optimizer_ = apex.optimizers.FusedAdam(params_, lr=self.lr, capturable=True) + scaler = torch.cuda.amp.GradScaler(enabled=True) + scaler_ = torch.cuda.amp.GradScaler(enabled=True) + + for i in range(100): + x = torch.rand([32, 1, 28, 28]).cuda().to(memory_format=torch.channels_last) + x_ = x.clone() + gt = torch.rand([32, 10]).cuda() + gt_ = gt.clone() + + # Reference + with torch.cuda.amp.autocast(enabled=True): + y = self.model(x) + loss = ((gt - y) ** 2).mean() + + scaler.scale(loss).backward() + scaler.step(self.optimizer) + scaler.update() + + # DUT + with torch.cuda.amp.autocast(enabled=True): + y = self.model_(x) + loss_ = ((gt_ - y) ** 2).mean() + + scaler_.scale(loss_).backward() + scaler_.step(optimizer_) + scaler_.update() + + for module in zip(self.model.modules(), self.model_.modules()): + m = module[0] + m_ = module[1] + if isinstance(m, nn.Conv2d) or isinstance(m_, nn.Linear): + torch.testing.assert_close(m.weight, m_.weight, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(m.weight.grad, m_.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + # Init for next iteration + self.optimizer.zero_grad() + optimizer_.zero_grad() + + self.model_.load_state_dict(copy.deepcopy(self.model.state_dict())) + + def testGradScalerCapturableMaster(self): + # Cast conv layers to FP16 + for m in self.model_.modules(): + if m.__class__ in [torch.nn.Conv2d]: + m.half() + params_ = [p for p in self.model_.parameters() if p.requires_grad] + optimizer_ = apex.optimizers.FusedAdam(params_, lr=self.lr, capturable=True, master_weights=True) + scaler = torch.cuda.amp.GradScaler(enabled=True) + scaler_ = torch.cuda.amp.GradScaler(enabled=True) + + for i in range(100): + x = torch.rand([32, 1, 28, 28]).cuda().to(memory_format=torch.channels_last) + x_ = x.clone() + gt = torch.rand([32, 10]).cuda() + gt_ = gt.clone() + + # Reference + with torch.cuda.amp.autocast(enabled=True): + y = self.model(x) + loss = ((gt - y) ** 2).mean() + + scaler.scale(loss).backward() + scaler.step(self.optimizer) + scaler.update() + + # DUT + with torch.cuda.amp.autocast(enabled=True): + y = self.model_(x) + loss_ = ((gt_ - y) ** 2).mean() + + scaler_.scale(loss_).backward() + scaler_.step(optimizer_) + scaler_.update() + + for module in zip(self.model.modules(), self.model_.modules()): + m = module[0] + m_ = module[1] + if isinstance(m, nn.Conv2d) or isinstance(m_, nn.Linear): + torch.testing.assert_close(m.weight, m_.weight.float(), atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(m.weight.grad, m_.weight.grad.float(), atol=1e-3, rtol=1e-3, equal_nan=True) + + # Init for next iteration + self.optimizer.zero_grad() + optimizer_.zero_grad() + + self.model_.load_state_dict(copy.deepcopy(self.model.state_dict())) + + def testNative(self): + params_ = [p for p in self.model_.parameters() if p.requires_grad] + optimizer_ = apex.optimizers.FusedAdam(params_, lr=self.lr, capturable=False) + + for i in range(100): + x = torch.rand([32, 1, 28, 28]).cuda().to(memory_format=torch.channels_last) + x_ = x.clone() + gt = torch.rand([32, 10]).cuda() + gt_ = gt.clone() + + # Reference + y = self.model(x) + loss = ((gt - y) ** 2).mean() + + loss.backward() + self.optimizer.step() + + # DUT + y = self.model_(x) + loss_ = ((gt_ - y) ** 2).mean() + + loss_.backward() + optimizer_.step() + + for module in zip(self.model.modules(), self.model_.modules()): + m = module[0] + m_ = module[1] + if isinstance(m, nn.Conv2d) or isinstance(m_, nn.Linear): + torch.testing.assert_close(m.weight, m_.weight, atol=1e-3, rtol=1e-3, equal_nan=True) + torch.testing.assert_close(m.weight.grad, m_.weight.grad, atol=1e-3, rtol=1e-3, equal_nan=True) + + # Init for next iteration + self.optimizer.zero_grad() + optimizer_.zero_grad() + + self.model_.load_state_dict(copy.deepcopy(self.model.state_dict())) + + +if __name__ == '__main__': + unittest.main() + diff --git a/apex/tests/L0/run_optimizers/test_fused_novograd.py b/apex/tests/L0/run_optimizers/test_fused_novograd.py new file mode 100755 index 00000000..fa94e710 --- /dev/null +++ b/apex/tests/L0/run_optimizers/test_fused_novograd.py @@ -0,0 +1,170 @@ +import torch +from torch.optim import Optimizer +import math +import apex +import unittest + +from test_fused_optimizer import TestFusedOptimizer +from itertools import product + +class Novograd(Optimizer): + """ + Implements Novograd algorithm. + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.95, 0)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + grad_averaging: gradient averaging + amsgrad (boolean, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + """ + + def __init__(self, params, lr=1e-3, betas=(0.95, 0), eps=1e-8, + weight_decay=0, grad_averaging=False, amsgrad=False): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, + weight_decay=weight_decay, + grad_averaging=grad_averaging, + amsgrad=amsgrad) + + super(Novograd, self).__init__(params, defaults) + + def __setstate__(self, state): + super(Novograd, self).__setstate__(state) + for group in self.param_groups: + group.setdefault('amsgrad', False) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('Sparse gradients are not supported.') + amsgrad = group['amsgrad'] + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros([]).to(state['exp_avg'].device) + if amsgrad: + # Maintains max of all exp. moving avg. of sq. grad. values + state['max_exp_avg_sq'] = torch.zeros([]).to(state['exp_avg'].device) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + if amsgrad: + max_exp_avg_sq = state['max_exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + norm = torch.sum(torch.pow(grad, 2)) + + if exp_avg_sq == 0: + exp_avg_sq.copy_(norm) + else: + exp_avg_sq.mul_(beta2).add_(norm, alpha=1 - beta2) + + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq) + # Use the max. for normalizing running avg. of gradient + denom = max_exp_avg_sq.sqrt().add_(group['eps']) + else: + denom = exp_avg_sq.sqrt().add_(group['eps']) + + grad.div_(denom) + if group['weight_decay'] != 0: + grad.add_(p.data, alpha=group['weight_decay']) + if group['grad_averaging']: + grad.mul_(1 - beta1) + exp_avg.mul_(beta1).add_(grad) + + p.data.add_(exp_avg, alpha=-group['lr']) + + return loss + + +class TestFusedNovoGrad(TestFusedOptimizer): + + def __init__(self, *args, **kwargs): + super(TestFusedNovoGrad, self).__init__(*args, **kwargs) + + # The options for NovoGrad and FusedNovoGrad are very specific if they + # are expected to behave the same. + self.options = {'lr':1e-3, 'betas':(0.95, 0), 'eps':1e-8, + 'weight_decay':0, 'grad_averaging':False, 'amsgrad':False} + + self.tst_options = {'lr':1e-3, 'betas':(0.95, 0), 'eps':1e-8, + 'weight_decay':0, 'grad_averaging':False, 'amsgrad':False, + 'bias_correction':False, 'reg_inside_moment':True, + 'norm_type':2, 'init_zero':False, 'set_grad_none':True} + + self.ref_optim = Novograd + self.fused_optim = apex.optimizers.FusedNovoGrad + + def test_float(self): + self.gen_single_type_test(param_type=torch.float) + + def test_half(self): + self.gen_single_type_test(param_type=torch.float16) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_device(self): + devices = ("cuda:1", "cuda:0") + for current_dev, tensor_dev in product(devices, devices): + with torch.cuda.device(current_dev): + torch.cuda.synchronize() + self.gen_single_type_test(param_type=torch.float, device=tensor_dev) + + + def test_multi_params(self): + sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]] + + tensors = [] + for size in sizes: + tensors.append(torch.rand(size, dtype=torch.float, device="cuda")) + ref_param, tst_param, ref_optim, tst_optim = self.gen_param_optim( + tensors, self.options, self.tst_options + ) + + for _ in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_optimizers/test_fused_optimizer.py b/apex/tests/L0/run_optimizers/test_fused_optimizer.py new file mode 100644 index 00000000..e5f5e8c4 --- /dev/null +++ b/apex/tests/L0/run_optimizers/test_fused_optimizer.py @@ -0,0 +1,311 @@ +from itertools import product +import random +import unittest + +import torch + +import apex + + +class TestFusedOptimizer(unittest.TestCase): + def setUp(self, max_abs_diff=1e-3, max_rel_diff=1, iters=7): + self.max_abs_diff = max_abs_diff + self.max_rel_diff = max_rel_diff + self.iters = iters + torch.manual_seed(9876) + + def tearDown(self): + pass + + def gen_param_optim(self, tensors, options, tst_options=None): + + # Adding this to make backward compatible with existing tests. Just in + # case "tst_options" are not provided, it gets a copy of options + # which contains the parameters for the reference optimizer + if tst_options == None: + tst_options = options + + ref_param = [] + tst_param = [] + for tensor in tensors: + ref_param.append(torch.nn.Parameter(tensor.clone())) + tst_param.append(torch.nn.Parameter(tensor.clone())) + + ref_optim = self.ref_optim(ref_param, **options) + tst_optim = self.fused_optim(tst_param, **tst_options) + + return (ref_param, tst_param, ref_optim, tst_optim) + + def gen_grad(self, ref_param, tst_param): + for p_ref, p_tst in zip(ref_param, tst_param): + p_ref.grad = torch.rand_like(p_ref) + p_tst.grad = p_ref.grad + + def gen_mixed_grad(self, ref_param, tst_param, scale=1.0): + half_grads = [] + for p_ref, p_tst in zip(ref_param, tst_param): + half_grads.append(torch.rand_like(p_ref).half()) + p_ref.grad = half_grads[-1].float() / scale + return half_grads + + def get_max_diff(self, ref_param, tst_param): + max_abs_diff = max_rel_diff = 0 + for p_ref, p_tst in zip(ref_param, tst_param): + max_abs_diff_p = (p_ref - p_tst).abs().max().item() + max_rel_diff_p = ((p_ref - p_tst) / p_ref).abs().max().item() + + if max_abs_diff_p > max_abs_diff: max_abs_diff = max_abs_diff_p + if max_rel_diff_p > max_rel_diff: max_rel_diff = max_rel_diff_p + + return max_abs_diff, max_rel_diff + + def gen_single_type_test(self, param_type=torch.float, device='cuda', *, skip_assert: bool = False): + nelem = 278011 + + # Some ref and test optimizers may require different set of options. + # This is a quick workaround to add that functionality while making + # minimum changes in existing code. + # If there is no "tst_options" field provided, safe to initialize + # the test optimizer with the parameters of reference optimizer. + if not hasattr(self, 'tst_options'): + self.tst_options = self.options + + tensor = torch.rand(nelem, dtype=param_type, device=device) + + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], self.options, self.tst_options) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + if skip_assert: + return + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + +class TestFusedAdam(TestFusedOptimizer): + + def setUp(self): + super().setUp() + self.options = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08, + 'weight_decay': 0, 'amsgrad': False} + self.ref_optim = torch.optim.Adam + self.fused_optim = apex.optimizers.FusedAdam + + def test_float(self): + self.gen_single_type_test(param_type=torch.float) + + # NOTE(mkozuki): Current threshold values look too small for BFloat16. + # TODO(mkozuki): Refactor `TestFusedOptimizer` + def test_half(self): + self.gen_single_type_test(param_type=torch.float16, skip_assert=True) + + def test_bfloat16(self): + self.gen_single_type_test(param_type=torch.bfloat16, skip_assert=True) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_device(self): + devices = ("cuda:0", "cuda:1") + for current_dev, tensor_dev in product(devices, devices): + with torch.cuda.device(current_dev): + self.gen_single_type_test(param_type=torch.float, device=tensor_dev) + + @unittest.skip('Disable until 8/1/2019 adam/adamw upstream picked') + def test_multi_params(self): + sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]] + + tensors = [] + for size in sizes: + tensors.append(torch.rand(size, dtype=torch.float, device='cuda')) + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim(tensors, self.options) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + @unittest.skip('No longer support fuse scaling') + def test_scale(self): + nelem = 278011 + tensor = torch.rand(nelem, dtype=torch.float, device='cuda') + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], self.options) + + for i in range(self.iters): + scale = random.random() * 1000 + half_grads = self.gen_mixed_grad(ref_param, tst_param, scale) + ref_optim.step() + tst_optim.step(grads=half_grads, scale=scale) + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + @unittest.skip('No longer support output fp16 param') + def test_fp16_output(self): + nelem = 278011 + + tensor = torch.rand(nelem, dtype=torch.float, device='cuda') + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], self.options) + + fp16_param = torch.nn.Parameter(tensor.clone().half()) + + for i in range(self.iters): + half_grads = self.gen_mixed_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step(grads=half_grads, output_params=[fp16_param]) + + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + max_abs_diff, max_rel_diff = self.get_max_diff(tst_param, \ + [fp16_param.float()]) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + def test_adam_option(self): + nelem = 1 + adam_option = {'lr':0.01, 'betas':(0.6, 0.9), 'eps':3e-06, + 'weight_decay':0, 'amsgrad':False} + + tensor = torch.rand(nelem, dtype=torch.float, device='cuda') + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], adam_option) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + def test_frozen_model(self): + nelem = 1 + adam_option = {'lr':0.01, 'betas':(0.6, 0.9), 'eps':3e-06, + 'weight_decay':0, 'amsgrad':False} + + tensor = torch.rand(nelem, dtype=torch.float, device='cuda') + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], adam_option) + + #Add an empty param group which may occur for pipeline parallel p-tuning + tst_optim.add_param_group({"params": []}) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + +class TestFusedAdagrad(TestFusedOptimizer): + def __init__(self, *args, **kwargs): + super(TestFusedAdagrad, self).__init__(*args, **kwargs) + self.options = {"lr": 5e-4, "eps": 1e-08, "weight_decay": 1.0e-5} + self.ref_optim = torch.optim.Adagrad + self.fused_optim = apex.optimizers.FusedAdagrad + + def test_float(self): + self.gen_single_type_test(param_type=torch.float) + + @unittest.skip("PyTorch optimizer is not numerically correct for fp16") + def test_half(self): + self.gen_single_type_test(param_type=torch.float16) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_device(self): + devices = ("cuda:0", "cuda:1") + for current_dev, tensor_dev in product(devices, devices): + with torch.cuda.device(current_dev): + self.gen_single_type_test(param_type=torch.float, device=tensor_dev) + + + def test_multi_params(self): + sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]] + adagrad_option = {"lr": 5e-4, "eps": 1e-08, "weight_decay": 0} + + tensors = [] + for size in sizes: + tensors.append(torch.rand(size, dtype=torch.float, device="cuda")) + ref_param, tst_param, ref_optim, tst_optim = self.gen_param_optim( + tensors, adagrad_option + ) + + for _ in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_params_different_devices_throws(self): + sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]] + adagrad_option = {"lr": 5e-4, "eps": 1e-08, "weight_decay": 0} + + tensors = [] + for i, size in enumerate(sizes): + tensors.append(torch.rand(size, dtype=torch.float, device="cuda:"+str(i % 2))) + ref_param, tst_param, ref_optim, tst_optim = self.gen_param_optim( + tensors, adagrad_option + ) + self.gen_grad(ref_param, tst_param) + with self.assertRaisesRegex(RuntimeError, "not on the same device"): + tst_optim.step() + + def test_adagrad_option(self): + nelem = 1 + adagrad_option = {"lr": 0.01, "eps": 3e-06, "weight_decay": 0} + + tensor = torch.rand(nelem, dtype=torch.float, device="cuda") + ref_param, tst_param, ref_optim, tst_optim = self.gen_param_optim( + [tensor], adagrad_option + ) + + for _ in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + +class TestFusedSGD(TestFusedOptimizer): + def __init__(self, *args, **kwargs): + super(TestFusedSGD, self).__init__(*args, **kwargs) + self.options = {"lr": .25, "momentum": .125} + self.ref_optim = torch.optim.SGD + self.fused_optim = apex.optimizers.FusedSGD + + def test_float(self): + self.gen_single_type_test(param_type=torch.float) + + def test_half(self): + self.gen_single_type_test(param_type=torch.float16) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_device(self): + devices = ("cuda:0", "cuda:1") + for current_dev, tensor_dev in product(devices, devices): + with torch.cuda.device(current_dev): + self.gen_single_type_test(param_type=torch.float, device=tensor_dev) + +if __name__ == '__main__': + unittest.main() diff --git a/apex/tests/L0/run_optimizers/test_lamb.py b/apex/tests/L0/run_optimizers/test_lamb.py new file mode 100644 index 00000000..4900fe5a --- /dev/null +++ b/apex/tests/L0/run_optimizers/test_lamb.py @@ -0,0 +1,336 @@ +import unittest +import os + +import torch +from torch.optim import Optimizer +import apex +from apex.multi_tensor_apply import multi_tensor_applier +from itertools import product + +class RefLAMB(Optimizer): + r"""Implements Lamb algorithm. + + It has been proposed in `Large Batch Optimization for Deep Learning: Training BERT in 76 minutes`_. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-6) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0.01) + + .. _Large Batch Optimization for Deep Learning: Training BERT in 76 minutes: + https://arxiv.org/abs/1904.00962 + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.01): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay) + super(RefLAMB, self).__init__(params, defaults) + if multi_tensor_applier.available: + import amp_C + self.multi_tensor_l2norm=amp_C.multi_tensor_l2norm + # Skip buffer + self._dummy_overflow_buf = torch.tensor([0], dtype=torch.int, device=self.param_groups[0]["params"][0].device) + self.multi_tensor_lamb = amp_C.multi_tensor_lamb + else: + raise RuntimeError('apex.optimizers.FusedLAMB requires cuda extensions') + + def step(self, closure=None): + """Performs a single optimization step. + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + # create separate grad lists for fp32 and fp16 params + g_all_32, g_all_16 = [], [] + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + if p.dtype == torch.float32: + g_all_32.append(p.grad.data) + elif p.dtype == torch.float16: + g_all_16.append(p.grad.data) + else: + raise RuntimeError('FusedLAMB only support fp16 and fp32.') + + device = self.param_groups[0]["params"][0].device + g_norm_32, g_norm_16 = torch.zeros(1, device=device), torch.zeros(1, device=device) + # compute grad norm for two lists + if len(g_all_32) > 0: + g_norm_32 = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [g_all_32], False)[0] + if len(g_all_16) > 0: + g_norm_16 = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [g_all_16], False)[0] + + # blend two grad norms to get global grad norm + global_grad_norm = multi_tensor_applier(self.multi_tensor_l2norm, + self._dummy_overflow_buf, + [[g_norm_32, g_norm_16]], + False)[0] + + max_grad_norm = 1.0 + clipped_ratio = max_grad_norm / max(global_grad_norm, max_grad_norm) + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + p.grad.data *= clipped_ratio + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('Lamb does not support sparse gradients, consider SparseAdam instad.') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['m'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['v'] = torch.zeros_like(p.data) + + m_t, v_t = state['m'], state['v'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + # m_t = beta1 * m + (1 - beta1) * g_t + m_t.mul_(beta1).add_(grad, alpha=1-beta1) + # v_t = beta2 * v + (1 - beta2) * (g_t * g_t) + v_t.mul_(beta2).addcmul_(grad, grad, value=1-beta2) + + # Debiasing + m_t_hat = m_t / (1.0 - beta1 ** state['step']) + v_t_hat = v_t / (1.0 - beta2 ** state['step']) + + update = m_t_hat / v_t_hat.sqrt().add(group['eps']) + + if group['weight_decay'] != 0: + update.add_(p.data, alpha=group['weight_decay']) + + trust_ratio = 1.0 + w_norm = p.data.pow(2).sum().sqrt() + g_norm = update.pow(2).sum().sqrt() + if w_norm > 0 and g_norm > 0: + trust_ratio = w_norm / g_norm + + state['w_norm'] = w_norm + state['g_norm'] = g_norm + state['trust_ratio'] = trust_ratio + + step_size = group['lr'] + + p.data.add_(update, alpha=-step_size*trust_ratio) + + return loss + +class TestLamb(unittest.TestCase): + def setUp(self, max_abs_diff=1e-3, max_rel_diff=1, iters=7): + self.max_abs_diff = max_abs_diff + self.max_rel_diff = max_rel_diff + self.iters = iters + torch.cuda.manual_seed(9876) + + + def tearDown(self): + pass + + def gen_param_optim(self, tensors, lamb_option): + ref_param = [] + tst_param = [] + for tensor in tensors: + ref_param.append(torch.nn.Parameter(tensor.clone())) + tst_param.append(torch.nn.Parameter(tensor.clone())) + + ref_optim = self.ref_optim(ref_param, **lamb_option) + tst_optim = self.tst_optim(tst_param, use_nvlamb=True, **lamb_option) + + return (ref_param, tst_param, ref_optim, tst_optim) + + def gen_grad(self, ref_param, tst_param): + for p_ref, p_tst in zip(ref_param, tst_param): + p_ref.grad = torch.rand_like(p_ref) + p_tst.grad = p_ref.grad + + def gen_mixed_grad(self, ref_param, tst_param, scale=1.0): + half_grads = [] + for p_ref, _ in zip(ref_param, tst_param): + half_grads.append(torch.rand_like(p_ref).half()) + p_ref.grad = half_grads[-1].float() / scale + return half_grads + + def get_max_diff(self, ref_param, tst_param): + max_abs_diff = max_rel_diff = 0 + for p_ref, p_tst in zip(ref_param, tst_param): + max_abs_diff_p = (p_ref - p_tst).abs().max().item() + max_rel_diff_p = ((p_ref - p_tst) / p_ref).abs().max().item() + + if max_abs_diff_p > max_abs_diff: max_abs_diff = max_abs_diff_p + if max_rel_diff_p > max_rel_diff: max_rel_diff = max_rel_diff_p + + return max_abs_diff, max_rel_diff + + def gen_single_type_test(self, param_type=torch.float, device="cuda"): + nelem = 278011 + tensor = torch.rand(nelem, dtype=param_type, device=device) + weight_decay = [0, 0.01] + + for wd in weight_decay: + lamb_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08, 'weight_decay':wd} + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], lamb_option) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + torch.cuda.synchronize() + tst_optim.step() + torch.cuda.synchronize() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + +class TestFusedLAMB(TestLamb): + def __init__(self, *args, **kwargs): + super(TestLamb, self).__init__(*args, **kwargs) + self.ref_optim = RefLAMB + self.tst_optim = apex.optimizers.FusedLAMB + + + def test_float(self): + self.gen_single_type_test(param_type=torch.float) + + @unittest.skip("PyTorch optimizer is not numerically correct for fp16") + def test_half(self): + self.gen_single_type_test(param_type=torch.float16) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_device(self): + devices = ("cuda:0", "cuda:1") + for current_dev, tensor_dev in product(devices, devices): + with torch.cuda.device(current_dev): + self.gen_single_type_test(param_type=torch.float, device=tensor_dev) + + def test_multi_params(self): + sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]] + weight_decay = [0, 0.01] + + for wd in weight_decay: + lamb_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08, 'weight_decay':wd} + tensors = [] + for size in sizes: + tensors.append(torch.rand(size, dtype=torch.float, device='cuda')) + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim(tensors, lamb_option) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + def test_lamb_option(self): + nelem = 1 + tensor = torch.rand(nelem, dtype=torch.float, device='cuda') + weight_decay = [0, 0.01] + + for wd in weight_decay: + lamb_option = {'lr':0.01, 'betas':(0.6, 0.9), 'eps':3e-06, 'weight_decay':wd} + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], lamb_option) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + +class TestFusedMixedPrecisionLamb(TestLamb): + def __init__(self, *args, **kwargs): + super(TestLamb, self).__init__(*args, **kwargs) + self.ref_optim = RefLAMB + self.tst_optim = apex.optimizers.FusedMixedPrecisionLamb + + + def test_float(self): + self.gen_single_type_test(param_type=torch.float) + + @unittest.skip("PyTorch optimizer is not numerically correct for fp16") + def test_half(self): + self.gen_single_type_test(param_type=torch.float16) + + @unittest.skipIf(torch.cuda.device_count()<2, "more than 1 GPU required") + def test_multi_device(self): + devices = ("cuda:0", "cuda:1") + for current_dev, tensor_dev in product(devices, devices): + with torch.cuda.device(current_dev): + self.gen_single_type_test(param_type=torch.float, device=tensor_dev) + + def test_multi_params(self): + sizes = [[4096, 1024], [4096], [4096, 2048], [32320, 1024], [1]] + weight_decay = [0, 0.01] + + for wd in weight_decay: + lamb_option = {'lr':5e-4, 'betas':(0.9, 0.999), 'eps':1e-08, 'weight_decay':wd} + tensors = [] + for size in sizes: + tensors.append(torch.rand(size, dtype=torch.float, device='cuda')) + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim(tensors, lamb_option) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + + def test_lamb_option(self): + nelem = 1 + tensor = torch.rand(nelem, dtype=torch.float, device='cuda') + weight_decay = [0, 0.01] + + for wd in weight_decay: + lamb_option = {'lr':0.01, 'betas':(0.6, 0.9), 'eps':3e-06, 'weight_decay':wd} + ref_param, tst_param, ref_optim, tst_optim = \ + self.gen_param_optim([tensor], lamb_option) + + for i in range(self.iters): + self.gen_grad(ref_param, tst_param) + ref_optim.step() + tst_optim.step() + max_abs_diff, max_rel_diff = self.get_max_diff(ref_param, tst_param) + + self.assertLessEqual(max_abs_diff, self.max_abs_diff) + self.assertLessEqual(max_rel_diff, self.max_rel_diff) + +if __name__ == '__main__': + script_path = os.path.dirname(os.path.realpath(__file__)) + unittest.main() diff --git a/apex/tests/L0/run_test.py b/apex/tests/L0/run_test.py new file mode 100644 index 00000000..f460d0c2 --- /dev/null +++ b/apex/tests/L0/run_test.py @@ -0,0 +1,94 @@ +"""L0 Tests Runner. + +How to run this script? + +1. Run all the tests: `python /path/to/apex/tests/L0/run_test.py` If you want an xml report, + pass `--xml-report`, i.e. `python /path/to/apex/tests/L0/run_test.py --xml-report` and + the file is created in `/path/to/apex/tests/L0`. +2. Run one of the tests (e.g. fused layer norm): + `python /path/to/apex/tests/L0/run_test.py --include run_fused_layer_norm` +3. Run two or more of the tests (e.g. optimizers and fused layer norm): + `python /path/to/apex/tests/L0/run_test.py --include run_optimizers run_fused_layer_norm` +""" +import argparse +import os +import unittest +import sys + + +TEST_ROOT = os.path.dirname(os.path.abspath(__file__)) +TEST_DIRS = [ + "run_amp", + "run_deprecated", + "run_fp16util", + "run_optimizers", + "run_fused_layer_norm", + "run_mlp", + "run_transformer", + "run_instance_norm_nvfuser", +] +DEFAULT_TEST_DIRS = [ + "run_optimizers", + "run_fused_layer_norm", + "run_mlp", + "run_transformer", + "run_instance_norm_nvfuser", +] + + +def parse_args(): + parser = argparse.ArgumentParser( + description="L0 test runner", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--include", + nargs="+", + choices=TEST_DIRS, + default=DEFAULT_TEST_DIRS, + help="select a set of tests to run (defaults to ALL tests).", + ) + parser.add_argument( + "--xml-report", + action="store_true", + help="pass this argument to get a junit xml report. (requires `xmlrunner`)", + ) + args, _ = parser.parse_known_args() + return args + + +def main(args: argparse.Namespace) -> None: + test_runner_kwargs = {"verbosity": 2} + Runner = unittest.TextTestRunner + if args.xml_report: + import xmlrunner + from datetime import date # NOQA + Runner = xmlrunner.XMLTestRunner + + errcode = 0 + for test_dir in args.include: + if args.xml_report: + this_dir = os.path.abspath(os.path.dirname(__file__)) + xml_output = os.path.join( + this_dir, + f"""TEST_{test_dir}_{date.today().strftime("%y%m%d")}""", + ) + test_runner_kwargs["output"] = xml_output + + runner = Runner(**test_runner_kwargs) + test_dir = os.path.join(TEST_ROOT, test_dir) + suite = unittest.TestLoader().discover(test_dir) + + print("\nExecuting tests from " + test_dir) + + result = runner.run(suite) + + if not result.wasSuccessful(): + errcode = 1 + + sys.exit(errcode) + + +if __name__ == '__main__': + args = parse_args() + main(args) diff --git a/apex/tests/L0/run_transformer/__init__.py b/apex/tests/L0/run_transformer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/tests/L0/run_transformer/gpt_scaling_test.py b/apex/tests/L0/run_transformer/gpt_scaling_test.py new file mode 100644 index 00000000..eb70e257 --- /dev/null +++ b/apex/tests/L0/run_transformer/gpt_scaling_test.py @@ -0,0 +1,116 @@ +import subprocess +import os + +from apex.transformer.testing.commons import TEST_SUCCESS_MESSAGE + + +def run_gpt(cmd): + args = list(cmd.split(" ")) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + outs, errs = p.communicate() + outs = list(str((outs).decode("utf-8")).splitlines()) + success = False + runtime = 0 + num_params = 0 + for out in outs: + out = str(out) + if "Average Iteration Time:" in str(out): + slicey = out[out.find(":") + 2 :] + try: + runtime = float(slicey) + except: + print(slicey) + quit() + if "Number of Parameters:" in str(out): + slicey = out[out.find(":") + 2 :] + try: + num_params = int(slicey) + except: + print(slicey) + quit() + if str(out) == str(TEST_SUCCESS_MESSAGE): + success = True + return runtime, round(float(int(num_params)) / 10.0 ** 9, 3), success, errs + + +def plot(runtimes): + import matplotlib.pyplot as plt + + for distributed_setting in runtimes.keys(): + plt.scatter( + runtimes[distributed_setting].keys(), + runtimes[distributed_setting].values(), + label=distributed_setting, + ) + plt.legend() + plt.xlabel("Parameters (Billions)") + plt.ylabel("Training Iteration time (s)") + plt.title(str("GPT Scaling w/ Offloading")) + plt.savefig("offload_gpt_scaling.png") + plt.close() + if not os.path.exists("/my_workspace/"): + os.system("mkdir /my_workspace/") + os.system("cp *.png /my_workspace/") + + +def main(): + runtimes = {} + nlist = ( + list(range(2000, 10000, 2000)) + + list(range(10000, 50000, 5000)) + + list(range(50000, 100000, 10000)) + ) + print("N-List:", nlist) + for data_parr, tens_parr, pipe_parr in [(8, 1, 1), (4, 2, 1), (2, 1, 4), (1, 2, 4)]: + for offload in [True, False]: + dist_setting = ( + "ddp=" + + str(data_parr) + + ", tensor_parr=" + + str(tens_parr) + + ", pipe_parr=" + + str(pipe_parr) + + ", offload=" + + str(offload) + ) + runtimes[dist_setting] = {} + print("Beginning Testing for", dist_setting) + for n in nlist: + cmd = "python3 -m torch.distributed.launch --nproc_per_node=8 run_gpt_minimal_test.py" + cmd += ( + " --micro-batch-size 1 --num-layers " + + str(n) + + " --hidden-size 128 --num-attention-heads 16" + ) + cmd += ( + " --max-position-embeddings 128 --seq-length 128 --tensor-model-parallel-size " + + str(tens_parr) + ) + cmd += ( + " --pipeline-model-parallel-size " + + str(pipe_parr) + + (" --cpu-offload" if offload else "") + ) + print(cmd) + runtime, bill_params, success, errs = run_gpt(cmd) + if success: + runtimes[dist_setting][bill_params] = runtime + print( + str(runtime) + "s per training iter for", + str(bill_params) + "B parameter GPT-2", + ) + if n >= 10000: + plot(runtimes) + else: + print("GPT-2 w/", n, "layers failed using", dist_setting) + print("Moving on to the next distributed setting...") + print("#" * (25)) + print() + plot(runtimes) + break + print(runtimes) + plot(runtimes) + + +if __name__ == "__main__": + main() diff --git a/apex/tests/L0/run_transformer/test_batch_sampler.py b/apex/tests/L0/run_transformer/test_batch_sampler.py new file mode 100644 index 00000000..99b0f1ae --- /dev/null +++ b/apex/tests/L0/run_transformer/test_batch_sampler.py @@ -0,0 +1,141 @@ +import torch +from torch.testing._internal import common_utils +from torch.utils.data import Dataset +from torch.utils.data import DataLoader + +from apex.transformer.pipeline_parallel.utils import _split_batch_into_microbatch as split_batch_into_microbatch + + +class MyIterableDataset(Dataset): + def __init__(self, start, end): + super().__init__() + assert end > start, "this example code only works with end >= start" + self.start = start + self.end = end + self.samples = list(range(self.start, self.end)) + + def __iter__(self): + return iter(range(self.start, self.end)) + + def __getitem__(self, index): + return self.samples[index] + + +class MegatronPretrainingRandomSampler: + + def __init__(self, total_samples, consumed_samples, micro_batch_size, + data_parallel_rank, data_parallel_size): + # Keep a copy of input params for later use. + self.total_samples = total_samples + self.consumed_samples = consumed_samples + self.micro_batch_size = micro_batch_size + self.data_parallel_rank = data_parallel_rank + self.data_parallel_size = data_parallel_size + self.micro_batch_times_data_parallel_size = \ + self.micro_batch_size * data_parallel_size + self.last_batch_size = \ + self.total_samples % self.micro_batch_times_data_parallel_size + + # Sanity checks. + assert self.total_samples > 0, \ + 'no sample to consume: {}'.format(self.total_samples) + assert self.micro_batch_size > 0 + assert data_parallel_size > 0 + assert self.data_parallel_rank < data_parallel_size, \ + 'data_parallel_rank should be smaller than data size: {}, ' \ + '{}'.format(self.data_parallel_rank, data_parallel_size) + + def __len__(self): + return self.total_samples + + def __iter__(self): + active_total_samples = self.total_samples - self.last_batch_size + self.epoch = self.consumed_samples // active_total_samples + current_epoch_samples = self.consumed_samples % active_total_samples + assert current_epoch_samples % self.micro_batch_times_data_parallel_size == 0 + + # data sharding and random sampling + bucket_size = (self.total_samples // self.micro_batch_times_data_parallel_size) * self.micro_batch_size + bucket_offset = current_epoch_samples // self.data_parallel_size + start_idx = self.data_parallel_rank * bucket_size + + g = torch.Generator() + g.manual_seed(self.epoch) + random_idx = torch.randperm(bucket_size, generator=g).tolist() + idx_range = [start_idx + x for x in random_idx[bucket_offset:]] + + batch = [] + # Last batch if not complete will be dropped. + for idx in idx_range: + batch.append(idx) + if len(batch) == self.micro_batch_size: + self.consumed_samples += self.micro_batch_times_data_parallel_size + yield batch + batch = [] + + +# Samples 8 tensors in total. +# First sample 4 tensors twice, then sample 2 tensors fourth. +class TestBatchSamplerBehavior(common_utils.TestCase): + def tearDown(self) -> None: + torch.cuda.empty_cache() + super().tearDown() + + def test_batch_sampler_behavior(self): + dataset = MyIterableDataset(0, 100) + + for num_workers in (1, 2, 4): + torch.manual_seed(42) + loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, 4, 0, 1), num_workers=num_workers) + samples = [] + for i, batch in enumerate(loader): + samples.append(batch) + if i == 2 - 1: + break + + torch.manual_seed(42) + loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, 2, 0, 1), num_workers=num_workers) + samples2 = [] + for i, batch in enumerate(loader): + samples2.append(batch) + if i == 4 - 1: + break + self.assertEqual(torch.cat(samples), torch.cat(samples2), msg=f"num_workers={num_workers}") + + def test_split_batch(self): + + class MyIterableDataset(Dataset): + def __init__(self, start, end): + super().__init__() + assert end > start, "this example code only works with end >= start" + self.start = start + self.end = end + self.samples = list(range(self.start, self.end)) + + def __len__(self): + return self.end - self.start + + def __iter__(self): + return iter(range(self.start, self.end)) + + def __getitem__(self, index): + return (torch.tensor([index, index]), torch.tensor([index // 2, index // 2])) + + dataset = MyIterableDataset(0, 100) + torch.manual_seed(42) + global_batch_size = 16 + loader = DataLoader(dataset, batch_sampler=MegatronPretrainingRandomSampler(100, 0, global_batch_size, 0, 1), num_workers=2) + batch = next(iter(loader)) + + for _micro_batch_size in (1, 2, 4, 8): + microbatches = list(split_batch_into_microbatch( + batch, + _micro_batch_size=_micro_batch_size, + _global_batch_size=global_batch_size, + )) + self.assertEqual(len(microbatches), global_batch_size // _micro_batch_size) + self.assertEqual(len(microbatches[0][0]), _micro_batch_size) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_bert_minimal.py b/apex/tests/L0/run_transformer/test_bert_minimal.py new file mode 100644 index 00000000..e7f6d56d --- /dev/null +++ b/apex/tests/L0/run_transformer/test_bert_minimal.py @@ -0,0 +1,250 @@ +import torch +import unittest +from apex.transformer.testing import global_vars +from apex.transformer.testing.standalone_bert import bert_model_provider +from apex.transformer.pipeline_parallel.schedules.common import ( + _get_params_for_weight_decay_optimization, build_model +) +from apex.transformer.pipeline_parallel.schedules import get_forward_backward_func +from apex.transformer.pipeline_parallel.utils import ( + average_losses_across_data_parallel_group, unwrap_model, setup_microbatch_calculator +) +from apex.transformer.log_util import set_logging_level +from apex.transformer import tensor_parallel, parallel_state +from apex.transformer.enums import ModelType +from apex.transformer._ucc_util import HAS_UCC +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase, NcclDistributedTestBase +import logging + +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + + +logging.getLogger("apex").setLevel(logging.WARNING) + + +set_logging_level("WARNING") + + +class BertTestBase: + + def _download_fancy_data(self): + text = """ + An original sentence not subject to any license restrictions, copyright, or royalty payments. Nothing to see here. Commercial or non-commercial use. Research or non-research purposes. The quick brown fox jumps over the lazy dog. Lorem ipsum. + """ + text = text * 1024 + encoded = text.encode("ascii", "replace") + ints = [int(encoded[i]) for i in range(len(encoded))] + return torch.tensor(ints) + + # build a batch given sequence_len and batch size + def _generate_fancy_data_labels(self, sequence_len, batch_size): + temps = [] + for i in range(batch_size): + if self.inds is None or self.data_idx >= len(self.inds): + # hack as use of RNG will fall out of sync due to pipelines being different + torch.manual_seed(self.MANUAL_SEED) + self.inds = torch.randperm( + self.effective_length, device="cuda") + self.masks = ( + torch.rand( + len(self.inds) // batch_size + 1, batch_size, sequence_len, device="cuda" + ) + >= self.MASK_PROB + ).long() + self.MANUAL_SEED += 1 + self.data_idx = 0 + if self.rank == 0: + print("new epoch", len(self.inds)) + print("my start", self.inds[0:5]) + print("masks_checksum:", torch.sum(self.masks)) + if self.EASY_MODE: + data_idx_ = self.data_idx % self.EASY_MODE_SIZ + else: + data_idx_ = self.data_idx + offset = self.inds[data_idx_] # * SEQUENCE_LEN + self.data_idx += 1 + + curr = self.fancy_data[offset: offset + + sequence_len].clone().detach() + temps.append(curr) + temp = torch.stack(temps, dim=0).cuda() + mask = self.masks[self.data_idx // batch_size] + mask_not = torch.logical_not(mask).long() + data = mask * temp + mask_not * 124 + label = temp + if parallel_state.get_tensor_model_parallel_rank() == 0: + data_dict = {"text": data, "label": label, "mask_not": mask_not} + else: + data_dict = None + keys = ["text", "label", "mask_not"] + broadcasted_data = tensor_parallel.broadcast_data( + keys, data_dict, torch.long) + return ( + broadcasted_data["text"].long(), + broadcasted_data["label"].long(), + broadcasted_data["mask_not"], + ) + + def _fwd_step_func(self, batch, model): + data, label, loss_mask = batch + y = model(data, torch.ones_like(data), lm_labels=label) + + def loss_func(output_tensor): + output_tensor, _ = output_tensor + lm_loss_ = output_tensor.float() + lm_loss = torch.sum(lm_loss_.view(-1) * + loss_mask.reshape(-1)) / loss_mask.sum() + averaged_loss = average_losses_across_data_parallel_group([ + lm_loss]) + if self.data_idx >= 1536: + # NOTE (patwang): Loss cutoff might be excessively high but roughly one in five + # unlucky random seeds do cause loss to spike to just under 8.0 + self.assertLess(averaged_loss, 8.0) + return lm_loss, {"avg": averaged_loss} + + return y, loss_func + + def _train( + self, model, optim, virtual_pipeline_model_parallel_size, pipeline_model_parallel_size, async_comm + ): + args = global_vars.get_args() + sequence_len = args.seq_length + micro_batch_size = args.micro_batch_size + hidden_size = args.hidden_size + global_batch_size = args.global_batch_size + forward_backward_func = get_forward_backward_func( + virtual_pipeline_model_parallel_size, pipeline_model_parallel_size + ) + tensor_shape = (sequence_len, micro_batch_size, hidden_size) + for _ in range(16): + batch = self._generate_fancy_data_labels( + sequence_len, global_batch_size) + optim.zero_grad() + forward_backward_func( + self._fwd_step_func, + batch, + model, + forward_only=False, + tensor_shape=tensor_shape, + async_comm=async_comm, + sequence_parallel_enabled=args.sequence_parallel, + ) + # All-reduce layernorm parameters across model parallel nodes + # when sequence parallelism is used + if parallel_state.get_tensor_model_parallel_world_size() > 1 and args.sequence_parallel: + for model_module in model: + unwrapped_model = unwrap_model(model_module) + for param in unwrapped_model.parameters(): + if getattr(param, 'sequence_parallel_enabled', False): + grad = param.grad + torch.distributed.all_reduce( + grad, group=parallel_state.get_tensor_model_parallel_group()) + + optim.step() + + @unittest.skipUnless(torch.cuda.device_count() > 2, "requires at least 3 gpus") + def test_bert_without_interleaving(self): + self._test_bert(virtual_pipeline_model_parallel_size=None) + + @unittest.skipUnless(torch.cuda.device_count() > 2, "requires at least 3 gpus") + def test_bert_with_interleaving(self): + if self.DISTRIBUTED_BACKEND == 'ucc': + self.skipTest('skip interleaving with ucc') + self._test_bert(virtual_pipeline_model_parallel_size=2) + + def _test_bert(self, virtual_pipeline_model_parallel_size): + + self.MANUAL_SEED = 42 + self.inds = None + self.masks = None + self.data_idx = 0 + self.MASK_PROB = 0.1 + self.EASY_MODE = False + self.EASY_MODE_SIZ = 32 + + num_devices = torch.cuda.device_count() + tensor_model_parallel_size = 2 if num_devices % 2 == 0 and num_devices > 4 else 1 + pipeline_model_parallel_size = num_devices // tensor_model_parallel_size + + override_args = { + "micro_batch_size": 2, + "num_layers": 16, + "hidden_size": 256, + "num_attention_heads": 8, + "max_position_embeddings": 512, + "seq_length": 512, + "global_batch_size": 128, + "pipeline_model_parallel_size": pipeline_model_parallel_size, + "tensor_model_parallel_size": tensor_model_parallel_size, + "bert_binary_head": False, + "world_size": self.world_size, + "rank": self.rank, + } + + global_vars.set_global_variables(override_args=override_args, ignore_unknown_args=True) + args = global_vars.get_args() + + self.fancy_data = self._download_fancy_data() + self.effective_length = self.fancy_data.size(0) // args.seq_length + self.effective_length = self.fancy_data.size(0) - args.seq_length + + if self.rank == 0: + print( + f'testing backend: {self.DISTRIBUTED_BACKEND} with virtual_pipeline_model_parallel_size: {virtual_pipeline_model_parallel_size}') + async_comm = not args.sequence_parallel and virtual_pipeline_model_parallel_size is None + self.data_idx = 0 + args.padded_vocab_size = 128 # needed in standalone gpt + args.model_type = ModelType.encoder_or_decoder + setup_microbatch_calculator( + args.rank, + args.rampup_batch_size, + args.global_batch_size, + args.micro_batch_size, + args.data_parallel_size, + ) + parallel_state.initialize_model_parallel( + args.tensor_model_parallel_size, + args.pipeline_model_parallel_size, + virtual_pipeline_model_parallel_size, + default_backend="nccl", + p2p_backend=self.DISTRIBUTED_BACKEND, + ) + + tensor_parallel.random.model_parallel_cuda_manual_seed(0) + model = build_model( + bert_model_provider, + wrap_with_ddp=parallel_state.get_data_parallel_world_size() > 1, + virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, + cpu_offload=args.cpu_offload, + ) + assert isinstance(model, list) + assert len(model) == ( + 1 + if virtual_pipeline_model_parallel_size is None + else virtual_pipeline_model_parallel_size + ) + _param_groups = _get_params_for_weight_decay_optimization(model) + optim = torch.optim.Adam(_param_groups) + self._train( + model, + optim, + virtual_pipeline_model_parallel_size, + args.pipeline_model_parallel_size, + async_comm, + ) + torch.cuda.synchronize() + + +class NcclBertTest(BertTestBase, NcclDistributedTestBase): + pass + + +@unittest.skipUnless(HAS_UCC, "requires pytorch to be built with native ucc") +class UccBertTest(BertTestBase, UccDistributedTestBase): + pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_cross_entropy.py b/apex/tests/L0/run_transformer/test_cross_entropy.py new file mode 100644 index 00000000..d1e1f9b1 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_cross_entropy.py @@ -0,0 +1,97 @@ +import logging +from typing import Tuple + +import torch +import torch.nn.functional as F +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer import tensor_parallel +from apex.transformer.tensor_parallel import cross_entropy +from apex.transformer.testing.commons import set_random_seed, IdentityLayer +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + +logging.getLogger("apex").setLevel(logging.WARNING) + + +def torch_cross_entropy( + batch_size: int, seq_length: int, vocab_size: int, logits_scale: float, seed: int, label_smoothing: float = 0.0 +) -> Tuple[torch.Tensor, torch.Tensor]: + set_random_seed(seed) + identity = IdentityLayer( + (batch_size, seq_length, vocab_size), scale=logits_scale + ).cuda() + logits = identity() + target = torch.cuda.LongTensor(size=(batch_size, seq_length)).random_(0, vocab_size) + loss = ( + F.cross_entropy( + logits.view(-1, logits.size()[-1]), target.view(-1), reduction="none", label_smoothing=label_smoothing + ) + .view_as(target) + .mean() + ) + loss.backward() + return loss, identity.weight.grad + + +def tensor_sharded_cross_entropy( + batch_size, seq_length, vocab_size, logits_scale, seed, label_smoothing=0.0 +): + set_random_seed(seed) + identity = IdentityLayer( + (batch_size, seq_length, vocab_size), scale=logits_scale + ).cuda() + logits = identity() + logits_parallel = tensor_parallel.scatter_to_tensor_model_parallel_region(logits) + target = torch.cuda.LongTensor(size=(batch_size, seq_length)).random_(0, vocab_size) + logits_parallel_ = logits_parallel.clone().detach() + loss = cross_entropy.vocab_parallel_cross_entropy(logits_parallel, target, label_smoothing=label_smoothing).mean() + loss.backward() + # check for mutation + assert torch.equal(logits_parallel_, logits_parallel) + return loss, identity.weight.grad + + +class VocabParallelCrossEntropyTestBase: + def test_cross_entropy(self): + batch_size, sequence_length, vocab_size_per_partition = 13, 17, 11 + logits_scale = 1000.0 + seed = 1234 + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + ) + vocab_size = vocab_size_per_partition * tensor_model_parallel_world_size + loss_torch, grad_torch = torch_cross_entropy( + batch_size, sequence_length, vocab_size, logits_scale, seed + ) + ( + loss_tensor_parallel, + grad_tensor_parallel, + ) = tensor_sharded_cross_entropy( + batch_size, sequence_length, vocab_size, logits_scale, seed + ) + + self.assertEqual( + loss_torch, loss_tensor_parallel, + msg=f"tensor_model_parallel_size: {tensor_model_parallel_world_size}", + ) + self.assertEqual( + grad_torch, grad_tensor_parallel, + msg=f"tensor_model_parallel_size: {tensor_model_parallel_world_size}", + ) + + parallel_state.destroy_model_parallel() + + +class NcclVocabParallelCrossEntropyTest(VocabParallelCrossEntropyTestBase, NcclDistributedTestBase): pass +class UccVocabParallelCrossEntropyTest(VocabParallelCrossEntropyTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_data.py b/apex/tests/L0/run_transformer/test_data.py new file mode 100644 index 00000000..38dc752e --- /dev/null +++ b/apex/tests/L0/run_transformer/test_data.py @@ -0,0 +1,64 @@ +import logging + +import torch.testing +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer.tensor_parallel import data as data_utils +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + +logging.getLogger("torch").setLevel(logging.WARNING) + + +class BroadcastDataTestBase: + def test_broadcast_data(self): + tensor_model_parallel_world_size: int = self.world_size // ( + 1 + self.world_size > 1 + ) + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size + ) + + target_key_size = { + "key1": [7, 11], + "key2": [8, 2, 1], + "key3": [13], + "key4": [5, 1, 2], + "key5": [5, 12], + } + keys = [k for k in target_key_size] + + data = {} + data_t = {} + with torch.no_grad(): + for key in target_key_size: + data[key] = torch.randint(0, 1000, size=target_key_size[key]) + data_t[key] = data[key].clone() + # "key_x" is supposed to be ignored. + data["key_x"] = torch.rand(5) + data_t["key_x"] = data["key_x"].clone() + if parallel_state.get_tensor_model_parallel_rank() != 0: + data = None + + data_utils._check_data_types(keys, data_t, torch.int64) + key_size, _, _ = data_utils._build_key_size_numel_dictionaries(keys, data) + + for key in keys: + self.assertEqual(target_key_size[key], key_size[key]) + + broadcasted_data = data_utils.broadcast_data(keys, data, torch.int64) + for key in keys: + self.assertEqual(broadcasted_data[key], data_t[key].cuda()) + + parallel_state.destroy_model_parallel() + + +class NcclBroadcastDataTest(BroadcastDataTestBase, NcclDistributedTestBase): pass +class UccBroadcastDataTest(BroadcastDataTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_dynamic_batchsize.py b/apex/tests/L0/run_transformer/test_dynamic_batchsize.py new file mode 100644 index 00000000..6a9d2006 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_dynamic_batchsize.py @@ -0,0 +1,220 @@ +from typing import Tuple, List + +import torch +import unittest + +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel.utils import get_num_microbatches +from apex.transformer.pipeline_parallel.schedules.common import ( + _get_params_for_weight_decay_optimization, build_model +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_with_interleaving import ( + _forward_backward_pipelining_with_interleaving, +) +from apex.transformer.pipeline_parallel.utils import ( + setup_microbatch_calculator, _reconfigure_microbatch_calculator, update_num_microbatches +) +from apex.transformer.testing import global_vars +from apex.transformer.testing.commons import ( + print_separator, fwd_step_func, model_provider_func +) +from apex.transformer.log_util import get_transformer_logger +from apex.transformer._data import MegatronPretrainingRandomSampler, MegatronPretrainingSampler +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase + +from torch.testing._internal import common_utils + +# note(mkozuki): To see warmup, steady, cooldown iterations, uncomment the line below +# set_logging_level("INFO") +_logger = get_transformer_logger("pipeline_parallel_test") +# note(mkozuki): To see if local batch size increases, uncomment the line below +# _logger.setLevel("INFO") + + +NUM_ITERATIONS = 20 +NUM_SAMPLES = 16384 // 2 +HIDDEN_SIZE = 16 + + +def Dataset(num_samples: int) -> List[Tuple[torch.Tensor, torch.Tensor]]: + return [ + ( + torch.randn(HIDDEN_SIZE, HIDDEN_SIZE), + torch.randn(HIDDEN_SIZE // 2, HIDDEN_SIZE // 2), + ) + for _ in range(num_samples) + ] + + +# Run forward & backward with dynamic batch size. +def run_interleaved_with_dynamic_batch_size( + pipeline_model_parallel_size: int, forward_only: bool, BatchSamplerCls, +) -> None: + args = global_vars.get_args() + _reconfigure_microbatch_calculator( + args.rank, + args.rampup_batch_size, + args.global_batch_size, + args.micro_batch_size, + 1, # args.data_parallel_size, + ) + virtual_pipeline_model_parallel_size = 2 + # NOTE (mkozuki): `virtual_pipeline_model_parallel_size` is a requisite for the interleaving scheduling + # In megatron, `args.virtual_pipeline_model_parallel_size` is computed in megatron/arguments.py and + # used ubiquitously but this test uses custom model so it's safe to abuse. + parallel_state.initialize_model_parallel( + 1, pipeline_model_parallel_size, virtual_pipeline_model_parallel_size + ) + pipeline_model_parallel_size = ( + parallel_state.get_pipeline_model_parallel_world_size() + ) + + print_separator( + f"BatchSamplerCls: {BatchSamplerCls.__name__}, forward_only: {forward_only}" + ) + + model = build_model( + model_provider_func, + wrap_with_ddp=True, + virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, + hidden_size=HIDDEN_SIZE, + ) + assert isinstance(model, list) + assert len(model) == virtual_pipeline_model_parallel_size + optimizer = torch.optim.Adam( + _get_params_for_weight_decay_optimization(model)) + + initial_local_minibatch_size = get_num_microbatches() * args.micro_batch_size + dataset = Dataset(NUM_SAMPLES) + data_loader = torch.utils.data.DataLoader( + dataset, + batch_sampler=BatchSamplerCls( + NUM_SAMPLES, + 0, + initial_local_minibatch_size, + parallel_state.get_data_parallel_rank(), + parallel_state.get_data_parallel_world_size(), + ), + ) + data_iter = iter(data_loader) + + def get_num_samples(batch): + if isinstance(batch, torch.Tensor): + return len(batch) + assert isinstance(batch, (list, tuple)) + return [get_num_samples(b) for b in batch] + + tensor_shape = [args.micro_batch_size, HIDDEN_SIZE, HIDDEN_SIZE] + consumed_samples = 0 + for i in range(NUM_ITERATIONS): + update_num_microbatches(consumed_samples, consistency_check=False) + local_batch_size = get_num_microbatches() * args.micro_batch_size + data_iter._index_sampler.local_minibatch_size = local_batch_size + local_mini_batch = next(data_iter) + + _logger.info( + f"iter: {i} / {NUM_ITERATIONS} " + f"local batchsize: {get_num_samples(local_mini_batch)} " + f"consumed_samples: {consumed_samples} / {NUM_SAMPLES}" + ) + _forward_backward_pipelining_with_interleaving( + fwd_step_func, + local_mini_batch, + model, + forward_only=forward_only, + tensor_shape=tensor_shape, + ) + + consumed_samples += ( + parallel_state.get_data_parallel_world_size() + * get_num_microbatches() + * args.micro_batch_size + ) + + if not forward_only: + for m in model: + for p in m.parameters(): + if p.grad is None: + raise RuntimeError("grad not found") + else: + optimizer.zero_grad(set_to_none=True) + + torch.cuda.synchronize() + + +class DynamicBatchsizeTestBase: + @unittest.skipUnless(torch.cuda.device_count() > 2, "requires at least 3 gpus") + def test_dynamic_batchsize(self): + + n_tests = 0 + failures = [] + + override_args = { + "micro_batch_size": 2, + "num_layers": 16, + "hidden_size": 256, + "num_attention_heads": 8, + "max_position_embeddings": 512, + "seq_length": 512, + "global_batch_size": 128, + "use_cpu_initialization": True, + "world_size": self.world_size, + "rank": self.rank, + } + + global_vars.set_global_variables( + args_defaults={"global_batch_size": 512, + "rampup_batch_size": [64, 64, 1000], }, + ignore_unknown_args=True, + override_args=override_args, + ) + + args = global_vars.get_args() + + setup_microbatch_calculator( + args.rank, + args.rampup_batch_size, + args.global_batch_size, + args.micro_batch_size, + 1, # args.data_parallel_size, + ) + for BatchSamplerCls in ( + MegatronPretrainingSampler, + MegatronPretrainingRandomSampler, + ): + for forward_only in (False, True): + n_tests += 1 + pipeline_model_parallel_size = self.world_size + try: + run_interleaved_with_dynamic_batch_size( + pipeline_model_parallel_size, forward_only, BatchSamplerCls, + ) + except Exception as e: + msg = ( + f"\tforward_only: {forward_only}\n" + f"pipeline rank: {parallel_state.get_pipeline_model_parallel_rank()}, " + f"virtual pipeline rank: {parallel_state.get_virtual_pipeline_model_parallel_rank()}\n" + f"{str(e)}" + ) + raise RuntimeError(msg) + finally: + parallel_state.destroy_model_parallel() + if failures: + print_separator("TEST FAILED:") + print("\n".join(failures)) + msg = f"{len(failures)} / {n_tests} cases failed" + raise RuntimeError(msg) + else: + if torch.distributed.get_rank() == 0: + print_separator("TEST RESULT: ### PASS!") + + +class NcclDynamicBatchsizeTest(DynamicBatchsizeTestBase, NcclDistributedTestBase): + pass + +# TODO: (Fuzzkatt) UCC still doesn't work with fwd_bwd_pipelining_with_interleaving + + +if __name__ == "__main__": + torch.backends.cuda.matmul.allow_tf32 = False + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_fused_softmax.py b/apex/tests/L0/run_transformer/test_fused_softmax.py new file mode 100644 index 00000000..c4996707 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_fused_softmax.py @@ -0,0 +1,371 @@ +"""Test for fused softmax functions. + +Ref: https://github.com/NVIDIA/Megatron-LM/blob/40becfc96c4144985458ac0e0fae45dbb111fbd2/megatron/fused_kernels/tests/test_fused_kernels.py +""" # NOQA +import itertools + +import torch +from torch.testing._internal import common_utils + +from apex.transformer import AttnMaskType +from apex.transformer.functional import FusedScaleMaskSoftmax + + +def attention_mask_func(attention_scores, attention_mask): + return attention_scores.masked_fill(attention_mask, -10000.0) + +def forward_torch_softmax(input, mask, scale): + input = input * scale + mask_output = attention_mask_func(input, mask) if mask is not None else input + probs = torch.nn.Softmax(dim=-1)(mask_output) + all_k_masked = mask.all(axis=-1) + zero_attention_mask = (1.0 - all_k_masked.float())[:, :, :, None] + probs = probs * zero_attention_mask + return probs + +autocast_dtypes = ( + (torch.half, torch.bfloat16) if torch.cuda.is_bf16_supported() else (torch.half,) +) + + +class TestFusedScaleMaskSoftmax(common_utils.TestCase): + def _setup_fused_softmax( + self, + input_in_fp16, + input_in_bf16, + scale=None, + softmax_in_fp32=False, + attn_mask_type=AttnMaskType.padding, + ): + fused_fn = FusedScaleMaskSoftmax( + input_in_fp16=input_in_fp16, + input_in_bf16=input_in_bf16, + mask_func=attention_mask_func, + scale=scale, + softmax_in_fp32=softmax_in_fp32, + attn_mask_type=attn_mask_type, + scaled_masked_softmax_fusion=True, + ) + torch_fn = FusedScaleMaskSoftmax( + input_in_fp16=input_in_fp16, + input_in_bf16=input_in_bf16, + mask_func=attention_mask_func, + scale=scale, + softmax_in_fp32=softmax_in_fp32, + attn_mask_type=attn_mask_type, + scaled_masked_softmax_fusion=False, + ) + return fused_fn, torch_fn + + def tearDown(self) -> None: + torch.cuda.empty_cache() + super().tearDown() + + def test_fused_scale_mask_softmax(self): + """ + attention_scores.shape = [4, 12, 24, 24] + mask.shape = [4, 1, 24, 24] + """ + for (dtype, scale, softmax_in_fp32, shape) in itertools.product( + (torch.half, torch.bfloat16), (None, 2.0), (False, True), ((4, 12, 24, 24), (32, 12, 4, 214)) + ): + msg = f"{dtype}-{scale}-{softmax_in_fp32}" + input_in_fp16 = dtype == torch.half + input_in_bf16 = dtype == torch.bfloat16 + if not (scale is None or softmax_in_fp32): + with self.assertRaises(RuntimeError, msg=msg): + self._setup_fused_softmax( + input_in_fp16, + input_in_bf16, + scale, + softmax_in_fp32, + AttnMaskType.padding, + ) + return + fused_fn, torch_fn = self._setup_fused_softmax( + input_in_fp16, + input_in_bf16, + scale, + softmax_in_fp32, + AttnMaskType.padding, + ) + + attention_scores_0 = ( + torch.randn(shape) + .to(device="cuda", dtype=dtype) + .requires_grad_(True) + ) + with torch.no_grad(): + attention_scores_1 = attention_scores_0.clone().requires_grad_(True) + mask_shape = (shape[0],) + (1,) + shape[2:] + mask = torch.randint(0, 2, mask_shape, device="cuda").bool() + expected = fused_fn(attention_scores_0, mask) + actual = torch_fn(attention_scores_1, mask) + self.assertEqual(actual, expected, msg=msg) + + g0 = torch.rand_like(actual) + with torch.no_grad(): + g1 = g0.clone() + expected.backward(g0) + actual.backward(g1) + + def test_autocast_fused_scale_mask_softmax(self): + for dtype in autocast_dtypes: + msg = f"dtype: {dtype}" + input_in_fp16 = dtype == torch.half + input_in_bf16 = dtype == torch.bfloat16 + fused_fn, torch_fn = self._setup_fused_softmax( + input_in_fp16, input_in_bf16, attn_mask_type=AttnMaskType.padding + ) + + attention_scores_0 = ( + torch.randn((4, 12, 24, 24)).cuda().requires_grad_(True) + ) + with torch.no_grad(): + attention_scores_1 = ( + attention_scores_0.clone().to(dtype).requires_grad_(True) + ) + mask = torch.randint(0, 2, (4, 1, 24, 24)).bool().cuda() + + expected = torch_fn(attention_scores_1, mask) + with torch.cuda.amp.autocast(dtype=dtype): + actual = fused_fn(attention_scores_0, mask) + self.assertEqual(actual.dtype, dtype, msg=msg) + self.assertEqual(actual, expected, msg=msg) + + g0 = torch.rand_like(actual) + with torch.no_grad(): + g1 = g0.clone() + expected.backward(g0) + actual.backward(g1) + + def test_fused_scale_softmax(self): + """ + attention_scores.shape = [4, 12, 24, 24] + mask = None + """ + for (dtype, scale, softmax_in_fp32, shape) in itertools.product( + (torch.half, torch.bfloat16), (None, 2.0), (False, True), ((4, 12, 24, 24), (32, 12, 4, 214)) + ): + msg = f"{dtype}-{scale}-{softmax_in_fp32}" + input_in_fp16 = dtype == torch.half + input_in_bf16 = dtype == torch.bfloat16 + if not (scale is None or softmax_in_fp32): + with self.assertRaises(RuntimeError, msg=msg): + self._setup_fused_softmax( + input_in_fp16, + input_in_bf16, + scale, + softmax_in_fp32, + AttnMaskType.padding, + ) + return + fused_fn, torch_fn = self._setup_fused_softmax( + input_in_fp16, + input_in_bf16, + scale, + softmax_in_fp32, + AttnMaskType.padding, + ) + + attention_scores_0 = ( + torch.randn(shape) + .to(device="cuda", dtype=dtype) + .requires_grad_(True) + ) + with torch.no_grad(): + attention_scores_1 = attention_scores_0.clone().requires_grad_(True) + mask = None + + expected = fused_fn(attention_scores_0, mask) + actual = torch_fn(attention_scores_1, mask) + self.assertEqual(actual, expected, msg=msg) + + g0 = torch.rand_like(actual) + with torch.no_grad(): + g1 = g0.clone() + expected.backward(g0) + actual.backward(g1) + + def test_autocast_fused_scale_softmax(self): + for dtype in autocast_dtypes: + msg = f"dtype: {dtype}" + input_in_fp16 = dtype == torch.half + input_in_bf16 = dtype == torch.bfloat16 + fused_fn, torch_fn = self._setup_fused_softmax( + input_in_fp16, input_in_bf16, attn_mask_type=AttnMaskType.padding + ) + + attention_scores_0 = ( + torch.randn((4, 12, 24, 24)).cuda().requires_grad_(True) + ) + with torch.no_grad(): + attention_scores_1 = ( + attention_scores_0.clone().to(dtype).requires_grad_(True) + ) + mask = None + + expected = torch_fn(attention_scores_1, mask) + with torch.cuda.amp.autocast(dtype=dtype): + actual = fused_fn(attention_scores_0, mask) + self.assertEqual(actual.dtype, dtype, msg=msg) + self.assertEqual(actual, expected, msg=msg) + + g0 = torch.rand_like(actual) + with torch.no_grad(): + g1 = g0.clone() + expected.backward(g0) + actual.backward(g1) + + def test_fused_upper_triangle_mask_softmax(self): + """ + attn_weights.shape: [4, 12, 24, 24] + total_mask.shape: [4, 1, 24, 24] + + total_mask[0, 0], a 24x24 matrix is like a lower triangular matrix, but + upper elements are True and lower elements and diagonal are False. + """ + for (dtype, scale, softmax_in_fp32) in itertools.product( + (torch.half, torch.bfloat16), (None, 2.0), (False, True), + ): + msg = f"{dtype}-{scale}-{softmax_in_fp32}" + input_in_fp16 = dtype == torch.half + input_in_bf16 = dtype == torch.bfloat16 + if not (scale is None or softmax_in_fp32): + with self.assertRaises(RuntimeError, msg=msg): + self._setup_fused_softmax( + input_in_fp16, + input_in_bf16, + scale, + softmax_in_fp32, + AttnMaskType.causal, + ) + return + fused_fn, torch_fn = self._setup_fused_softmax( + input_in_fp16, + input_in_bf16, + scale, + softmax_in_fp32, + AttnMaskType.causal, + ) + + attn_weights_0 = ( + torch.randn((4, 12, 24, 24)) + .to(device="cuda", dtype=dtype) + .requires_grad_(True) + ) + with torch.no_grad(): + attn_weights_1 = attn_weights_0.clone().requires_grad_(True) + total_mask = ( + ~(torch.tril(torch.randn((24, 24), device="cuda")).bool()) + .unsqueeze(0) + .unsqueeze(0) + ) + total_mask = total_mask.repeat((4, 1, 1, 1)) + expected = fused_fn(attn_weights_0, total_mask) + actual = torch_fn(attn_weights_1, total_mask) + self.assertEqual(actual, expected, msg=msg) + + g0 = torch.randn_like(actual) + with torch.no_grad(): + g1 = g0.clone() + actual.backward(g0) + expected.backward(g1) + + def test_autocast_fused_upper_triangle_mask_softmax(self): + for dtype in autocast_dtypes: + msg = f"dtype: {dtype}" + input_in_fp16 = dtype == torch.half + input_in_bf16 = dtype == torch.bfloat16 + fused_fn, torch_fn = self._setup_fused_softmax( + input_in_fp16, input_in_bf16, attn_mask_type=AttnMaskType.causal + ) + + attn_weights_0 = ( + torch.randn((4, 12, 24, 24)).cuda().requires_grad_(True) + ) + with torch.no_grad(): + attn_weights_1 = ( + attn_weights_0.clone().to(dtype).requires_grad_(True) + ) + total_mask = ( + ~(torch.tril(torch.randn((24, 24), device="cuda")).bool()) + .unsqueeze(0) + .unsqueeze(0) + ) + + with torch.cuda.amp.autocast(dtype=dtype): + actual = fused_fn(attn_weights_0, total_mask) + self.assertEqual(actual.dtype, dtype, msg=msg) + expected = torch_fn(attn_weights_1, total_mask) + self.assertEqual(actual, expected, msg=msg) + + g0 = torch.randn_like(actual) + with torch.no_grad(): + g1 = g0.clone() + actual.backward(g0) + expected.backward(g1) + + +class TestGenericFusedSoftmaxKernel(common_utils.TestCase): + + def setUp(self): + super().setUp() + self.batch = 2 + self.attn = 16 + self.scale_t = 1.0 + self.dtype = torch.float16 + self.device = torch.cuda.current_device() + self.thresh = {"atol": 1e-3, "rtol": 1e-3} + + qlen = [1, 2] + klen = [1, 2, 3, 4, 5, 8, 10, 11, 13, 128, 256, 1200, 1234] + available_cuda_mem = torch.cuda.memory.mem_get_info(self.device)[0] / (1024 ** 3) + if available_cuda_mem > 40: + qlen.extend([1234, 2322, 2348]) + klen.extend([2048, 3123, 4096, 4128, 7234, 8192]) + + self.q_k_lens = itertools.product(qlen, klen) + + def tearDown(self) -> None: + torch.cuda.empty_cache() + super().tearDown() + + def test_forward(self, allmasked: bool=False): + import generic_scaled_masked_softmax_cuda + for qlen, klen in self.q_k_lens: + inputs = torch.normal(0, 2, (self.batch, self.attn, qlen, klen), dtype=self.dtype, device=self.device) + masks = ( + torch.randint(0, 2, (self.batch, 1, qlen, klen), dtype=torch.bool, device=self.device) + if not allmasked else torch.ones((self.batch, 1, qlen, klen), dtype=torch.bool, device=self.device) + ) + softmax_results = generic_scaled_masked_softmax_cuda.forward(inputs, masks, self.scale_t) + softmax_results_torch = forward_torch_softmax(inputs, masks, self.scale_t) + self.assertEqual( + softmax_results_torch.to(self.dtype), softmax_results, **self.thresh, msg=f"(q, k) = ({qlen, klen})") + + def test_backward(self, allmasked: bool=False): + import generic_scaled_masked_softmax_cuda + for qlen, klen in self.q_k_lens: + inputs = torch.normal(0, 2, (self.batch, self.attn, qlen, klen), dtype=self.dtype, device=self.device) + backward = torch.rand_like(inputs, dtype=torch.float16, device=self.device) + masks = ( + torch.randint(0, 2, (self.batch, 1, qlen, klen), dtype=torch.bool, device=self.device) + if not allmasked else torch.ones((self.batch, 1, qlen, klen), dtype=torch.bool, device=self.device) + ) + softmax_results = generic_scaled_masked_softmax_cuda.forward(inputs, masks, self.scale_t) + back_grad = generic_scaled_masked_softmax_cuda.backward(backward, softmax_results, self.scale_t) + inputs.requires_grad = True + softmax_results_torch = forward_torch_softmax(inputs, masks, self.scale_t) + softmax_results_torch.backward(backward) + self.assertEqual(back_grad, inputs.grad, **self.thresh, msg=f"(q, k) = ({qlen, klen})") + + def test_allmasked(self): + self.test_forward(True) + + def test_allmask_backward(self): + self.test_backward(True) + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_gpt_minimal.py b/apex/tests/L0/run_transformer/test_gpt_minimal.py new file mode 100644 index 00000000..5fba86a7 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_gpt_minimal.py @@ -0,0 +1,232 @@ +from functools import partial +from typing import List +import time + +import torch + +import unittest + +from apex.transformer._ucc_util import HAS_UCC +from apex.transformer import parallel_state +from apex.transformer.enums import ModelType +from apex.transformer.tensor_parallel import model_parallel_cuda_manual_seed +from apex.transformer.pipeline_parallel.utils import ( + average_losses_across_data_parallel_group, unwrap_model, setup_microbatch_calculator, + get_ltor_masks_and_position_ids +) +from apex.transformer.pipeline_parallel.schedules.common import ( + _get_params_for_weight_decay_optimization, build_model +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_without_interleaving import ( + forward_backward_pipelining_without_interleaving, +) +from apex.transformer.testing.standalone_gpt import gpt_model_provider +from apex.transformer.testing import global_vars + +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase, NcclDistributedTestBase + +from torch.testing._internal import common_utils +from torch.testing._internal.common_device_type import instantiate_device_type_tests + + +class GptTestBase: + + def _download_fancy_data(self): + text = """ + An original sentence not subject to any license restrictions, copyright, or royalty payments. Nothing to see here. Commercial or non-commercial use. Research or non-research purposes. The quick brown fox jumps over the lazy dog. Lorem ipsum. + """ + text = text * 1024 + encoded = text.encode("ascii", "replace") + ints = [int(encoded[i]) for i in range(len(encoded))] + return torch.tensor(ints) + + # build a batch given sequence_len and batch size + def _generate_fancy_data_labels(self, sequence_len, batch_size): + temps = list() + for i in range(batch_size): + if self.inds is None or self.data_idx >= len(self.inds): + # hack as use of RNG will fall out of sync due to pipelines being different + model_parallel_cuda_manual_seed(self.MANUAL_SEED) + self.inds = torch.randperm(effective_length, device="cuda") + self.MANUAL_SEED += 1 + self.data_idx = 0 + data_idx_ = self.data_idx + offset = self.inds[data_idx_] + self.data_idx += 1 + curr = fancy_data[offset: offset + + sequence_len + 1].clone().detach() + temps.append(curr) + temp = torch.stack(temps, dim=0).cuda() + return temp + + def _get_batch(self, int_tensors: List[torch.Tensor]): + data = int_tensors[0] + # Unpack. + tokens_ = data.long() + labels = tokens_[:, 1:].contiguous() + tokens = tokens_[:, :-1].contiguous() + # Get the masks and position ids. + attention_mask, loss_mask, position_ids = get_ltor_masks_and_position_ids( + tokens, + self.N_VOCAB, # tokenizer.eod, + False, # args.reset_position_ids, + False, # args.reset_attention_mask, + False, # args.eod_mask_loss, + ) + return tokens, labels, loss_mask, attention_mask, position_ids + + # Ref: https://github.com/NVIDIA/Megatron-LM/blob/b31e1296354e979722627a6c4dedafe19b51fa97/pretrain_gpt.py#L75 + def _loss_func(self, loss_mask, output_tensor): + losses = output_tensor.float() + loss_mask = loss_mask.view(-1).float() + loss = torch.sum(losses.view(-1) * loss_mask) / loss_mask.sum() + + # Reduce loss for logging. + averaged_loss = average_losses_across_data_parallel_group([loss]) + + return loss, {"lm loss": averaged_loss[0]} + + # Ref: https://github.com/NVIDIA/Megatron-LM/blob/b31e1296354e979722627a6c4dedafe19b51fa97/pretrain_gpt.py#L86 + def _fwd_step_func(self, batch, model): + """Forward step.""" + tokens, labels, loss_mask, attention_mask, position_ids = self._get_batch( + batch) + output_tensor = model(tokens, position_ids, + attention_mask, labels=labels) + return output_tensor, partial(self._loss_func, loss_mask) + + def _train(self, model, optim, pipeline_model_parallel_size, async_comm): + args = global_vars.get_args() + fwd_bwd_func = forward_backward_pipelining_without_interleaving + + tensor_shape = (args.seq_length, args.micro_batch_size, + args.hidden_size) + runtime = 0 + # training loop + for i in range(3): + since = time.time() + if torch.distributed.get_rank() == 0: + print("begin iter", i) + batch = [ + self._generate_fancy_data_labels( + args.seq_length, args.global_batch_size) + for _ in range(pipeline_model_parallel_size) + ] + if torch.distributed.get_rank() == 0: + print("finished making batch...") + optim.zero_grad() + fwd_bwd_func( + self._fwd_step_func, + batch, + model, + forward_only=False, + tensor_shape=tensor_shape, + async_comm=async_comm, + sequence_parallel_enabled=args.sequence_parallel, + ) + if torch.distributed.get_rank() == 0: + print("finished forward step") + # All-reduce layernorm parameters across model parallel nodes + # when sequence parallelism is used + if parallel_state.get_tensor_model_parallel_world_size() > 1 and global_vars.get_args().sequence_parallel: + for model_module in model: + unwrapped_model = unwrap_model(model_module) + for param in unwrapped_model.parameters(): + if getattr(param, 'sequence_parallel_enabled', False): + grad = param.grad + torch.distributed.all_reduce( + grad, group=parallel_state.get_tensor_model_parallel_group()) + optim.step() + if torch.distributed.get_rank() == 0: + print("finished iter", i) + runtime += time.time() - since + return runtime / 3.0 + + @unittest.skipUnless(torch.cuda.device_count() > 2, "requires at least 3 gpus") + def test_gpt(self): + self.MANUAL_SEED = 42 + self.inds = None + self.data_idx = 0 + self.N_VOCAB = 128 + init = True + + num_devices = torch.cuda.device_count() + tensor_model_parallel_size = 2 if num_devices % 2 == 0 and num_devices >= 4 else 1 + pipeline_model_parallel_size = num_devices // tensor_model_parallel_size + + override_args = { + "micro_batch_size": 2, + "num_layers": 16, + "hidden_size": 256, + "num_attention_heads": 8, + "max_position_embeddings": 512, + "seq_length": 512, + "global_batch_size": 128, + "pipeline_model_parallel_size": pipeline_model_parallel_size, + "tensor_model_parallel_size": tensor_model_parallel_size, + "world_size": self.world_size, + "rank": self.rank, + } + + global_vars.set_global_variables(override_args=override_args, ignore_unknown_args=True) + args = global_vars.get_args() + + for async_comm in (False,) if args.sequence_parallel else (False, True): + global fancy_data + global effective_length + + if init: + init = False + + fancy_data = self._download_fancy_data() + args = global_vars.get_args() + args.model_type = ModelType.encoder_or_decoder + effective_length = fancy_data.size(0) // args.seq_length + effective_length = fancy_data.size(0) - args.seq_length + + args.padded_vocab_size = 128 + setup_microbatch_calculator( + args.rank, + args.rampup_batch_size, + args.global_batch_size, + args.micro_batch_size, + args.data_parallel_size, + ) + + print(args.tensor_model_parallel_size, "MODEL PARALLEL SIZE") + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=args.tensor_model_parallel_size, + pipeline_model_parallel_size_=args.pipeline_model_parallel_size, + default_backend="nccl", + p2p_backend=self.DISTRIBUTED_BACKEND, + ) + + model_parallel_cuda_manual_seed(0) + model = build_model( + gpt_model_provider, + wrap_with_ddp=parallel_state.get_data_parallel_world_size() > 1, + virtual_pipeline_model_parallel_size=None, + cpu_offload=args.cpu_offload, + ) + assert isinstance(model, list), model + _param_groups = _get_params_for_weight_decay_optimization(model) + optim = torch.optim.Adam(_param_groups) + runtime = self._train( + model, optim, args.pipeline_model_parallel_size, async_comm) + + parallel_state.destroy_model_parallel() + torch.cuda.synchronize() + + +class NcclGptTest(GptTestBase, NcclDistributedTestBase): + pass + + +@unittest.skipUnless(HAS_UCC, "requires pytorch to be built with native ucc") +class UccGptTest(GptTestBase, UccDistributedTestBase): + pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_layers.py b/apex/tests/L0/run_transformer/test_layers.py new file mode 100644 index 00000000..e2df2acf --- /dev/null +++ b/apex/tests/L0/run_transformer/test_layers.py @@ -0,0 +1,561 @@ +import logging +import unittest +import typing + +import torch +import torch.nn as nn +from torch.testing._internal import common_utils + +from apex.transformer import parallel_state +from apex.transformer.tensor_parallel import layers +from apex.transformer.testing.commons import set_random_seed +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + + +logging.getLogger("torch").setLevel(logging.WARNING) +logging.getLogger("apex").setLevel(logging.WARNING) + + +# N.B.(mkozuki): Disable TF32 matrix multiply. +# Matrices used in this test are so small that TF32 matmul +# can be less precise so that `self.assertEqual` raises. +torch.backends.cuda.matmul.allow_tf32 = False + + +class TensorParallelLayerTestBase: + + BATCH_SIZE: int = 8 + SEQUENCE_LENGTH: int = 128 + VOCAB_SIZE: int = 1024 + HIDDEN_SIZE: int = 256 + INPUT_SIZE_COEFF: int = 256 + OUTPUT_SIZE_COEFF: int = 256 + SEED: int = 123456 + + @property + def tensor_shape(self) -> typing.Sequence[int]: + return [self.SEQUENCE_LENGTH, self.BATCH_SIZE, self.HIDDEN_SIZE] + + @torch.no_grad() + @unittest.skipIf(torch.cuda.device_count() < 2, "Requires >=2 GPUs") + def test_all_gather_parity(self) -> None: + if self.DISTRIBUTED_BACKEND == "ucc": + self.skipTest("torch_ucc does NOT support `torch.distributed._all_gather_base` as of 2022/06/15") + from torch.distributed.distributed_c10d import all_gather, _all_gather_base # NOQA + + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + ) + tensor_model_parallel_rank = parallel_state.get_tensor_model_parallel_rank() + cur_tensor_model_device = torch.device(f"cuda:{tensor_model_parallel_rank}") + with torch.no_grad(): + tensor = tensor_model_parallel_rank * torch.ones( + self.tensor_shape, dtype=torch.float32, device=cur_tensor_model_device) + numel = tensor.numel() + numel_gathered = tensor_model_parallel_world_size * numel + gathered = torch.empty( + torch.Size((numel_gathered,)), + device=cur_tensor_model_device, + dtype=torch.float32, + requires_grad=False, + ) + chunks = [ + gathered[i * numel : (i + 1) * numel] + for i in range(tensor_model_parallel_world_size) + ] + all_gather(chunks, tensor, group=parallel_state.get_tensor_model_parallel_group()) + + gathered_for_base = torch.empty( + torch.Size((numel_gathered,)), + device=cur_tensor_model_device, + dtype=torch.float32, + requires_grad=False, + ) + _all_gather_base( + gathered_for_base, + tensor, + group=parallel_state.get_tensor_model_parallel_group(), + ) + + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + self.assertEqual(gathered, gathered_for_base, msg=msg) + parallel_state.destroy_model_parallel() + + @torch.no_grad() + @unittest.skipIf(torch.cuda.device_count() < 2, "Requires >=2 GPUs") + def test_reduce_scatter_parity(self) -> None: + if self.DISTRIBUTED_BACKEND == "ucc": + self.skipTest("torch_ucc does NOT support `torch.distributed._reduce_scatter_base` as of 2022/06/15") + from torch.distributed.distributed_c10d import reduce_scatter, _reduce_scatter_base # NOQA + + for tensor_model_parallel_world_size in range(2, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + ) + tensor_model_parallel_rank = parallel_state.get_tensor_model_parallel_rank() + cur_tensor_model_device = torch.device(f"cuda:{tensor_model_parallel_rank}") + with torch.no_grad(): + input = torch.cat([ + i * torch.ones(self.tensor_shape, dtype=torch.float32, device=cur_tensor_model_device) + for i in range(tensor_model_parallel_world_size) + ]) + input_list = [t.clone() for t in input.chunk(tensor_model_parallel_world_size)] + output = torch.empty( + self.tensor_shape, + device=cur_tensor_model_device, + dtype=torch.float32, + requires_grad=False, + ) + reduce_scatter( + output, input_list, + group=parallel_state.get_tensor_model_parallel_group(), + ) + + output_for_base = torch.empty( + self.tensor_shape, + device=cur_tensor_model_device, + dtype=torch.float32, + requires_grad=False, + ) + _reduce_scatter_base( + output_for_base, + input, + group=parallel_state.get_tensor_model_parallel_group(), + ) + + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + self.assertEqual(output, output_for_base, msg=msg) + self.assertEqual(input, torch.cat(input_list), msg=msg) + parallel_state.destroy_model_parallel() + + def test_parallel_embedding(self) -> None: + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + ) + set_random_seed(self.SEED + 1) + input_tensor = torch.randint( + 0, + self.VOCAB_SIZE, + ( + self.BATCH_SIZE, + self.SEQUENCE_LENGTH, + ), + device="cuda", + ) + loss_weight = torch.randn( + ( + self.BATCH_SIZE, + self.SEQUENCE_LENGTH, + self.HIDDEN_SIZE, + ), + device="cuda", + ) + + set_random_seed(self.SEED) + embedding_torch = nn.Embedding( + self.VOCAB_SIZE, + self.HIDDEN_SIZE, + ).cuda() + output_torch = embedding_torch(input_tensor) + loss_torch = torch.mul(output_torch, loss_weight).sum() + loss_torch.backward() + + # N.B.(mkozuki): With affine weight initialization on GPU, + # it's super difficult to keep the consistency with nn.Embedding. + # Thus, turning on `use_cpu_initialization`. + set_random_seed(self.SEED) + embedding_vocab_parallel = layers.VocabParallelEmbedding( + self.VOCAB_SIZE, + self.HIDDEN_SIZE, + init_method=nn.init.normal_, + use_cpu_initialization=True, + ).cuda() + output_vocab_parallel = embedding_vocab_parallel(input_tensor) + loss_vocab_parallel = torch.mul( + output_vocab_parallel, loss_weight + ).sum() + loss_vocab_parallel.backward() + + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + self.assertEqual(output_torch, output_vocab_parallel, msg=msg) + self.assertEqual(loss_torch, loss_vocab_parallel, msg=msg) + + splitted_weight_torch = torch.split( + embedding_torch.weight.grad, + self.VOCAB_SIZE + // tensor_model_parallel_world_size, + 0, + )[parallel_state.get_tensor_model_parallel_rank()] + self.assertEqual( + splitted_weight_torch, embedding_vocab_parallel.weight.grad, msg=msg, + ) + + parallel_state.destroy_model_parallel() + + def _affine_weight_init_test_impl( + self, init_device: str, is_column_parallel: bool + ) -> None: + dim = int(not is_column_parallel) + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size + ) + input_size: int = self.INPUT_SIZE_COEFF * tensor_model_parallel_world_size + output_size: int = self.OUTPUT_SIZE_COEFF * tensor_model_parallel_world_size + + weight_shape = ( + (self.OUTPUT_SIZE_COEFF, input_size) + if is_column_parallel + else (output_size, self.INPUT_SIZE_COEFF) + ) + weight = torch.empty(weight_shape) + set_random_seed(self.SEED) + + sharding_dim_size = ( + self.OUTPUT_SIZE_COEFF + if is_column_parallel + else self.INPUT_SIZE_COEFF + ) + + if init_device == "cpu": + layers._initialize_affine_weight_cpu( + weight, + output_size, + input_size, + sharding_dim_size, + dim, + nn.init.normal_, + params_dtype=torch.float32, + ) + else: + layers._initialize_affine_weight_gpu( + weight, torch.nn.init.normal_, dim + ) + # Target + set_random_seed(self.SEED) + if init_device == "cpu": + main_weight = torch.empty(output_size, input_size) + nn.init.normal_(main_weight) + curr_weight = torch.split(main_weight, sharding_dim_size, dim=dim)[ + parallel_state.get_tensor_model_parallel_rank() + ] + else: + curr_weight = torch.empty(*weight_shape) + nn.init.normal_(curr_weight) + + self.assertEqual( + curr_weight, weight, msg=f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}") + parallel_state.destroy_model_parallel() + + def test_affine_weight_init_column_parallel_cpu(self) -> None: + self._affine_weight_init_test_impl(init_device="cpu", is_column_parallel=True) + + def test_affine_weight_init_column_parallel_gpu(self) -> None: + self._affine_weight_init_test_impl(init_device="gpu", is_column_parallel=True) + + def test_affine_weight_init_row_parallel_cpu(self) -> None: + self._affine_weight_init_test_impl(init_device="cpu", is_column_parallel=False) + + def test_affine_weight_init_row_parallel_gpu(self) -> None: + self._affine_weight_init_test_impl(init_device="gpu", is_column_parallel=False) + + def test_row_parallel_linear(self) -> None: + self._row_parallel_linear_test_impl(False, False, False) + + def test_row_parallel_linear_gradient_accumulation_fusion(self) -> None: + self._row_parallel_linear_test_impl(True, False, False) + + def test_row_parallel_linear_gradient_accumulation_fusion_in_fp16(self) -> None: + self._row_parallel_linear_test_impl(True, True, False) + + # fails on native ucc and torch ucc: ucc does not support reduce scatter + @unittest.skipIf(torch.cuda.device_count() < 2, "Sequence Parallel requires >=2 GPUs") + def test_row_parallel_linear_sequence_parallel(self) -> None: + self._row_parallel_linear_test_impl(False, False, True) + + # TODO(mkozuki): Merge this with `_column_parallel_linear_test_impl` + # Note that `input_is_parallel` is unique to `RowParallelLinear` which could make the merge complicated. + def _row_parallel_linear_test_impl( + self, + gradient_accumulation_fusion: bool, + accumulation_in_fp16: bool, + sequence_parallel_enabled: bool, + ) -> None: + tensor_shape = ( + self.SEQUENCE_LENGTH, + self.BATCH_SIZE, + self.HIDDEN_SIZE, + ) + for tensor_model_parallel_world_size in range( + 1 + int(sequence_parallel_enabled), self.world_size + 1 + ): + if self.world_size % tensor_model_parallel_world_size: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + ) + set_random_seed(self.SEED) + + linear = layers.RowParallelLinear( + self.HIDDEN_SIZE, + self.HIDDEN_SIZE, + keep_master_weight_for_test=True, + params_dtype=torch.float32, + use_cpu_initialization=True, + gradient_accumulation_fusion=gradient_accumulation_fusion, + accumulation_in_fp16=accumulation_in_fp16, + sequence_parallel_enabled=sequence_parallel_enabled, + # n.b.(mkozuki): RowParallelLinear is constructed with `input_is_parallel=True` + # by default, e.g. https://github.com/NVIDIA/NeMo/blob/782b4e1652aaa43c8be390d9\ + # db0dc89544afa080/nemo/collections/nlp/modules/common/megatron/transformer.py#L204 + input_is_parallel=True, + ).cuda() + if accumulation_in_fp16: + linear = linear.half() + # Simulate the situation where fusion of weight grad calculation and gradient accumulation is enabled. + if gradient_accumulation_fusion: + with torch.no_grad(): + linear.weight.main_grad = torch.zeros_like(linear.weight) + + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + + with torch.no_grad(): + orig_input_tensor = torch.randn(tensor_shape, requires_grad=True, device="cuda") + orig_loss_weight = torch.randn(tensor_shape, device="cuda") + input_tensor = orig_input_tensor.chunk( + chunks=tensor_model_parallel_world_size, + dim=2, + )[parallel_state.get_tensor_model_parallel_rank()].contiguous() + if sequence_parallel_enabled: + loss_weight = orig_loss_weight.chunk( + chunks=tensor_model_parallel_world_size, + dim=0, + )[parallel_state.get_tensor_model_parallel_rank()] + else: + loss_weight = orig_loss_weight + if accumulation_in_fp16: + orig_input_tensor = orig_input_tensor.half() + input_tensor = input_tensor.half() + loss_weight = loss_weight.half() + input_tensor.requires_grad_() + output, _ = linear(input_tensor) + loss = torch.mul(output, loss_weight).sum() + loss.backward() + self.assertIsNotNone(input_tensor.grad, msg=msg) + + ref_linear = nn.Linear( + in_features=self.HIDDEN_SIZE, + out_features=self.HIDDEN_SIZE, + bias=False, + device="cuda", + ) + with torch.no_grad(): + dldy = orig_loss_weight.clone() + x = orig_input_tensor.clone() + ref_linear.weight.copy_(linear.master_weight) + if accumulation_in_fp16: + ref_linear = ref_linear.half() + x.requires_grad_() + expected_output = ref_linear(x) + expected_loss = torch.mul(expected_output, dldy).sum() + expected_loss.backward() + + if not accumulation_in_fp16: + if sequence_parallel_enabled: + self.assertEqual( + x=output, + y=expected_output.chunk( + chunks=tensor_model_parallel_world_size, + dim=0, + )[parallel_state.get_tensor_model_parallel_rank()], + msg=msg, + ) + else: + self.assertEqual( + x=output, + y=expected_output, + msg=msg, + ) + + grad_attr_name = "main_grad" if gradient_accumulation_fusion else "grad" + # NOTE(mkozuki): Numerical errors seems to be enlarged by tensor model parallel. + if tensor_model_parallel_world_size == 1: + self.assertEqual( + x=getattr(linear.weight, grad_attr_name), + y=ref_linear.weight.grad.chunk( + chunks=tensor_model_parallel_world_size, + dim=0, + )[parallel_state.get_tensor_model_parallel_rank()], + msg=msg, + ) + + parallel_state.destroy_model_parallel() + + def test_column_parallel_linear(self): + self._column_parallel_linear_test_impl(False, False, False, False) + + def test_column_parallel_linear_async(self): + self._column_parallel_linear_test_impl(True, False, False, False) + + def test_column_parallel_linear_gradient_accumulation_fusion(self): + self._column_parallel_linear_test_impl(False, True, False, False) + + def test_column_parallel_linear_gradient_accumulation_fusion_in_fp16(self): + self._column_parallel_linear_test_impl(False, True, True, False) + + def test_column_parallel_linear_sequence_parallel(self): + if self.DISTRIBUTED_BACKEND == "ucc": + self.skipTest("Backward's reduce_scatter fails. as of 2022/06/15") + self._column_parallel_linear_test_impl(False, False, False, True) + + @unittest.skipIf(torch.cuda.device_count() < 2, "Sequence Parallel requires >= 2 GPUs") + def test_column_parallel_linear_exception(self): + with self.assertRaisesRegex( + RuntimeError, + "`async_tensor_model_parallel_allreduce` and `sequence_parallel_enabled` cannot be enabled at the same time.", + ): + self._column_parallel_linear_test_impl(True, False, False, True) + + def _column_parallel_linear_test_impl( + self, + async_tensor_model_parallel_allreduce: bool, + gradient_accumulation_fusion: bool, + accumulation_in_fp16: bool, + sequence_parallel_enabled: bool, + ): + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if async_tensor_model_parallel_allreduce and sequence_parallel_enabled: + if tensor_model_parallel_world_size == 1: + continue + if self.world_size % tensor_model_parallel_world_size: + continue + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + ) + + input_tensor_shape = self.tensor_shape + expected_output_shape = self.tensor_shape + # When sequence parallel, `gather_output` is disabled, i.e., + # output of matmul isn't gathered in dimension of feature/hidden (last dim). + if sequence_parallel_enabled: + expected_output_shape[-1] //= tensor_model_parallel_world_size + + # tensor's shape is [sequence length, batch size, hidden size] + set_random_seed(self.SEED) + linear = layers.ColumnParallelLinear( + self.HIDDEN_SIZE, + self.HIDDEN_SIZE, + bias=False, + keep_master_weight_for_test=True, + params_dtype=torch.float32, + use_cpu_initialization=True, + gather_output=not sequence_parallel_enabled, + no_async_tensor_model_parallel_allreduce=not async_tensor_model_parallel_allreduce, + gradient_accumulation_fusion=gradient_accumulation_fusion, + accumulation_in_fp16=accumulation_in_fp16, + sequence_parallel_enabled=sequence_parallel_enabled, + ).cuda() + if accumulation_in_fp16: + linear = linear.half() + + # Simulate the situation where fusion of weight grad calculation and gradient accumulation happens. + if gradient_accumulation_fusion: + with torch.no_grad(): + linear.weight.main_grad = torch.zeros_like(linear.weight) + + orig_input_tensor = torch.randn(input_tensor_shape, device="cuda", requires_grad=True) + if accumulation_in_fp16: + orig_input_tensor = orig_input_tensor.half() + if sequence_parallel_enabled: + input_tensor = list( + orig_input_tensor.chunk(tensor_model_parallel_world_size, dim=0) + )[parallel_state.get_tensor_model_parallel_rank()] + else: + input_tensor = orig_input_tensor + output, _ = linear(input_tensor) + # The order of dimension is expected to be (sequence, batch, hidden) + self.assertEqual(output.shape, expected_output_shape, msg=msg) + + orig_loss_weight = torch.randn(input_tensor_shape, device="cuda") + if accumulation_in_fp16: + orig_loss_weight = orig_loss_weight.half() + if sequence_parallel_enabled: + loss_weight = orig_loss_weight.chunk( + tensor_model_parallel_world_size, dim=2, + )[parallel_state.get_tensor_model_parallel_rank()] + else: + loss_weight = orig_loss_weight + loss = torch.mul(output, loss_weight).sum() + loss.backward() + + with torch.no_grad(): + dldy = orig_loss_weight.clone() + x = orig_input_tensor.clone() + ref_linear = nn.Linear( + in_features=self.HIDDEN_SIZE, + out_features=self.HIDDEN_SIZE, + bias=False, + device="cuda", + ) + if accumulation_in_fp16: + ref_linear = ref_linear.half() + # NOTE(mkozuki): `master_weight` is available because `keep_master_weight_for_test` is set. + ref_linear.weight.copy_(linear.master_weight) + x.requires_grad_() + expected_output = ref_linear(x) + if sequence_parallel_enabled: + chunk = expected_output.chunk( + tensor_model_parallel_world_size, + dim=2, + )[parallel_state.get_tensor_model_parallel_rank()] + self.assertEqual( + x=output, + y=chunk, + msg=msg, + ) + else: + self.assertEqual( + x=output, + y=expected_output, + msg=msg, + ) + + expected_loss = torch.mul(expected_output, dldy).sum() + expected_loss.backward() + grad_attr_name = "main_grad" if gradient_accumulation_fusion else "grad" + # NOTE(mkozuki): Numerical errors seems to be enlarged by tensor model parallel. + if tensor_model_parallel_world_size == 1: + self.assertEqual( + x=getattr(linear.weight, grad_attr_name), + y=ref_linear.weight.grad.chunk( + chunks=tensor_model_parallel_world_size, + dim=0, + )[parallel_state.get_tensor_model_parallel_rank()], + msg=msg, + ) + + parallel_state.destroy_model_parallel() + + +class NcclTensorParallelLayerTest(TensorParallelLayerTestBase, NcclDistributedTestBase): + pass + + +class UccTensorParallelLayerTest(TensorParallelLayerTestBase, UccDistributedTestBase): + pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_mapping.py b/apex/tests/L0/run_transformer/test_mapping.py new file mode 100644 index 00000000..18fa0763 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_mapping.py @@ -0,0 +1,87 @@ +import logging + +import torch +from torch.testing._internal import common_utils + +from apex.transformer import parallel_state +from apex.transformer.tensor_parallel import mappings +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + + +logging.getLogger("torch").setLevel(logging.WARNING) +logging.getLogger("apex").setLevel(logging.WARNING) + + +class MappingTestBase: + def test_reduce(self): + for tensor_model_paralell_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_paralell_world_size > 0: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_paralell_world_size + ) + t = torch.full((10, 10, 10, 10), 50, device=f"cuda:{self.rank}") + expected = torch.full( + (10, 10, 10, 10), + 50 * tensor_model_paralell_world_size, + device=f"cuda:{self.rank}", + ) + self.assertTrue( + torch.equal(mappings._reduce(t), expected), + msg=f"tensor_model_paralell_world_size: {tensor_model_paralell_world_size}", + ) + parallel_state.destroy_model_parallel() + + def test_split(self): + for tensor_model_paralell_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_paralell_world_size > 0: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_paralell_world_size + ) + + tensors = [ + torch.randn(10, 1) + for _ in range(tensor_model_paralell_world_size) + ] + x = torch.cat(tensors, 1) + out = mappings._split_along_last_dim(x) + self.assertTrue( + torch.equal( + out, tensors[parallel_state.get_tensor_model_parallel_rank()] + ), + msg=f"tensor_model_paralell_world_size: {tensor_model_paralell_world_size}" + ) + parallel_state.destroy_model_parallel() + + def test_gather(self): + for tensor_model_paralell_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_paralell_world_size > 0: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_paralell_world_size + ) + device = f"cuda:{self.rank}" + gathered = mappings._gather_along_last_dim( + torch.tensor( + [parallel_state.get_tensor_model_parallel_rank()], device=device + ) + ) + expected = torch.tensor( + [rank for rank in range(tensor_model_paralell_world_size)], + device=device, + ) + self.assertTrue( + torch.equal(gathered, expected), + msg=f"tensor_model_paralell_world_size: {tensor_model_paralell_world_size}", + ) + parallel_state.destroy_model_parallel() + + +class NcclMappingTest(MappingTestBase, NcclDistributedTestBase): pass +class UccMappingTest(MappingTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_microbatches.py b/apex/tests/L0/run_transformer/test_microbatches.py new file mode 100644 index 00000000..13e64881 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_microbatches.py @@ -0,0 +1,85 @@ +import logging +from typing import List, Optional + +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel.utils import ( + _reconfigure_microbatch_calculator, + get_micro_batch_size, + get_num_microbatches, + get_current_global_batch_size, + update_num_microbatches, +) +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + +logging.getLogger("apex").setLevel(logging.WARNING) + + +class MicrobatchCalculatorTestBase: + + GLOBAL_BATCH_SIZE: int = 1024 + MICRO_BATCH_SIZE: int = 1 + + def _test(self, rampup_batch_size: Optional[List[int]]) -> None: + for data_parallel_size in range(1, self.world_size + 1): + + expected_global_batch_size = self.GLOBAL_BATCH_SIZE + expected_micro_batch_size = self.MICRO_BATCH_SIZE + if rampup_batch_size: + expected_global_batch_size = rampup_batch_size[0] + num_consumed_samples = 0 + step_of_global_batch_size = rampup_batch_size[1] + threshold = rampup_batch_size[2] + + if data_parallel_size > 1 and data_parallel_size % 2 != 0: + continue + if self.world_size % data_parallel_size != 0: + continue + msg = f"data_parallel_size: {data_parallel_size}" + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=self.world_size // data_parallel_size, + pipeline_model_parallel_size_=1, + ) + self.assertEqual(data_parallel_size, parallel_state.get_data_parallel_world_size(), msg=msg) + + _reconfigure_microbatch_calculator( + self.rank, + rampup_batch_size, + self.GLOBAL_BATCH_SIZE, + self.MICRO_BATCH_SIZE, + data_parallel_size, + ) + + self.assertEqual(get_micro_batch_size(), expected_micro_batch_size, msg=msg) + self.assertEqual(get_num_microbatches(), expected_global_batch_size / expected_micro_batch_size / data_parallel_size, msg=msg) + current_global_batch_size = get_current_global_batch_size() + self.assertEqual(current_global_batch_size, expected_global_batch_size, msg=msg) + + # Make sure `global_batch_size` equals to the final global batch size after + # certain number of updates. + if rampup_batch_size: + update_num_microbatches(current_global_batch_size) + for i in range(100): + current_global_batch_size = get_current_global_batch_size() + update_num_microbatches(current_global_batch_size) + current_global_batch_size = get_current_global_batch_size() + self.assertEqual(get_current_global_batch_size(), self.GLOBAL_BATCH_SIZE, msg=msg) + parallel_state.destroy_model_parallel() + + def test_constant_microbatch_calculator(self): + self._test(rampup_batch_size=None) + + def test_dynamic_microbatch_calculator(self): + self._test(rampup_batch_size=[256, 128, 500]) + + +class NcclMicrobatchCalculatorTest(MicrobatchCalculatorTestBase, NcclDistributedTestBase): pass +class UccMicrobatchCalculatorTest(MicrobatchCalculatorTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_p2p_comm.py b/apex/tests/L0/run_transformer/test_p2p_comm.py new file mode 100644 index 00000000..c93b19af --- /dev/null +++ b/apex/tests/L0/run_transformer/test_p2p_comm.py @@ -0,0 +1,122 @@ +import logging +import unittest + +import torch +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel import p2p_communication +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + +logging.getLogger("apex").setLevel(logging.DEBUG) + + +# [P2P Ops Involved in Pipeline Model Parallel forward/backward] +# **forward_backward_pipelining_without_interleaving** +# - send_forward / recv_forward +# - send_backward / recv_backward +# - send_forward_recv_backward +# - send_backward_recv_forward +# **forward_backward_pipelining_with_interleaving** +# - send_backward_recv_backward +# - recv_backward +# - recv_forward +# - send_forward_backward_recv_forward_backward +# - send_forward_recv_forward +class P2PCommTestBase: + + numel = 4 + shape = (2, 2) + dtype = torch.float32 + + @property + def world_size(self): + return min(2, torch.cuda.device_count()) + + def _init_model_parallel(self): + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=1, + pipeline_model_parallel_size_=self.world_size, + virtual_pipeline_model_parallel_size_=None, + ) + + def create_tensor(self, value: int = None): + return torch.tensor( + [value] * self.numel).view(self.shape).to(device="cuda", dtype=self.dtype) + + # Brief: Simulate warm-up. + # Brief: test `recv_forward` & `send_forward`. + def test_no_interleaving_warmup(self): + self.assertEqual(self.world_size, 2) + self._init_model_parallel() + input_tensor = None + if parallel_state.is_pipeline_first_stage(): + tensor = self.create_tensor(self.rank) + print(tensor) + p2p_communication.send_forward(output_tensor=tensor, tensor_shape=self.shape, dtype=self.dtype) + else: + input_tensor = p2p_communication.recv_forward(tensor_shape=self.shape, dtype=self.dtype) + + if parallel_state.is_pipeline_first_stage(): + self.assertIsNone(input_tensor) + else: + expected_input_tensor = self.create_tensor(self.rank - 1) + self.assertEqual(input_tensor, expected_input_tensor) + + # Brief: test `send_forward`, `send_forward_recv_forward`, and `recv_forward`. + def test_send_forward_recv_forward(self): + self._init_model_parallel() + prev_tensor = None + tensor = self.create_tensor(self.rank) + if parallel_state.is_pipeline_first_stage(): + p2p_communication.send_forward(output_tensor=tensor, tensor_shape=self.shape, dtype=self.dtype) + elif parallel_state.is_pipeline_last_stage(): + prev_tensor = p2p_communication.recv_forward(tensor_shape=self.shape, dtype=self.dtype) + else: + prev_tensor = p2p_communication.send_forward_recv_forward( + output_tensor=tensor, + recv_prev=True, + tensor_shape=self.shape, + dtype=self.dtype, + ) + + if parallel_state.is_pipeline_first_stage(): + self.assertIsNone(prev_tensor) + else: + expected_prev_tensor = self.create_tensor(self.rank - 1) + self.assertEqual(prev_tensor, expected_prev_tensor) + + # Brief: test `send_backward`, `send_backward_recv_backward`, and `recv_backward`. + def test_send_backward_recv_backward(self): + self._init_model_parallel() + tensor = self.create_tensor(self.rank) + + next_tensor = None + if parallel_state.is_pipeline_first_stage(): + next_tensor = p2p_communication.recv_backward(tensor_shape=self.shape, dtype=self.dtype) + elif parallel_state.is_pipeline_last_stage(): + p2p_communication.send_backward(input_tensor_grad=tensor, tensor_shape=self.shape, dtype=self.dtype) + else: + next_tensor = p2p_communication.send_backward_recv_backward( + input_tensor_grad=tensor, + recv_next=True, + tensor_shape=self.shape, + dtype=self.dtype, + ) + + if parallel_state.is_pipeline_last_stage(): + self.assertIsNone(next_tensor) + else: + expected_next_tensor = self.create_tensor(self.rank + 1) + self.assertEqual(next_tensor, expected_next_tensor) + + +# n.b.(mkozuki): Intentionally skip NCCL backend tests as I trust pytorch/pytorch repo. +class UccP2PCommTest(P2PCommTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_parallel_state.py b/apex/tests/L0/run_transformer/test_parallel_state.py new file mode 100644 index 00000000..a86c2fb3 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_parallel_state.py @@ -0,0 +1,187 @@ +import logging +import os + +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + +logging.getLogger("apex").setLevel(logging.WARNING) + + +os.environ["BACKEND"] = "NCCL" +DATA_PARALLEL_WORLD_SIZE: int = 1 + + +def calc_expected_tensor_model_paralell_rank( + rank: int, tensor_model_parallel_world_size: int, +) -> int: + return rank % tensor_model_parallel_world_size + + +class ParallelStateTestBase: + def test_initialize_model_parallel(self) -> None: + + self.assertFalse(parallel_state.model_parallel_is_initialized()) + + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + msg = f"tensor_model_parallel_world_siz: {tensor_model_parallel_world_size}" + if self.world_size % tensor_model_parallel_world_size: + continue + + pipeline_model_parallel_world_size = ( + self.world_size // tensor_model_parallel_world_size + ) + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + ) + self.assertEqual( + tensor_model_parallel_world_size, + parallel_state.get_tensor_model_parallel_world_size(), + msg=msg, + ) + expected_tensor_model_parallel_rank = calc_expected_tensor_model_paralell_rank( + self.rank, tensor_model_parallel_world_size + ) + self.assertEqual( + expected_tensor_model_parallel_rank, + parallel_state.get_tensor_model_parallel_rank(), + msg=msg, + ) + + expected_tensor_model_parallel_src_rank = ( + self.rank // tensor_model_parallel_world_size + ) * tensor_model_parallel_world_size + self.assertEqual( + expected_tensor_model_parallel_src_rank, + parallel_state.get_tensor_model_parallel_src_rank(), + msg=msg, + ) + + parallel_state.destroy_model_parallel() + self.assertFalse(parallel_state.model_parallel_is_initialized(), msg=msg) + + def test_initialize_model_parallel_with_virtual_and_split(self) -> None: + if self.world_size < 4: + self.skipTest("requires >= 4 GPUs") + self.assertFalse(parallel_state.model_parallel_is_initialized()) + + tensor_model_parallel_world_size = 1 + int(self.world_size > 4) + pipeline_model_parallel_world_size = ( + self.world_size // tensor_model_parallel_world_size + ) + virtual_pipeline_model_parallel_world_size = 2 + pipeline_model_parallel_split_rank = pipeline_model_parallel_world_size // 2 + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + virtual_pipeline_model_parallel_size_=virtual_pipeline_model_parallel_world_size, + pipeline_model_parallel_split_rank_=pipeline_model_parallel_split_rank, + ) + self.assertEqual( + calc_expected_tensor_model_paralell_rank( + self.rank, tensor_model_parallel_world_size + ), + parallel_state.get_tensor_model_parallel_rank(), + ) + self.assertEqual( + pipeline_model_parallel_world_size, + parallel_state.get_pipeline_model_parallel_world_size(), + ) + self.assertEqual( + virtual_pipeline_model_parallel_world_size, + parallel_state.get_virtual_pipeline_model_parallel_world_size(), + ) + + expected_pipeline_rank = ( + self.rank - (self.rank % tensor_model_parallel_world_size) + ) % pipeline_model_parallel_world_size + self.assertEqual( + expected_pipeline_rank, parallel_state.get_pipeline_model_parallel_rank(), + ) + # virtual pipeline model parallel rank is lazily set, i.e., right after the call of + # `initialize_model_parallel`, it's set to 0. + self.assertEqual( + 0, parallel_state.get_virtual_pipeline_model_parallel_rank(), + ) + self.assertEqual( + pipeline_model_parallel_split_rank, + parallel_state.get_pipeline_model_parallel_split_rank(), + ) + + fake_split_rank = 77 + parallel_state.set_pipeline_model_parallel_split_rank(fake_split_rank) + self.assertEqual( + fake_split_rank, parallel_state.get_pipeline_model_parallel_split_rank() + ) + + # relative position embedding groups check + self.assertEqual( + expected_pipeline_rank < pipeline_model_parallel_split_rank, + parallel_state.is_rank_in_encoder_relative_position_embedding_group(), + ) + self.assertEqual( + expected_pipeline_rank >= pipeline_model_parallel_split_rank, + parallel_state.is_rank_in_decoder_relative_position_embedding_group(), + ) + + parallel_state.destroy_model_parallel() + + def test_initialize_model_parallel_decoder_only(self) -> None: + """Initialize model parallelism for decoder-only Transformers like GPT-3""" + + self.assertFalse(parallel_state.model_parallel_is_initialized()) + + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + if self.world_size % tensor_model_parallel_world_size: + continue + + pipeline_model_parallel_world_size = ( + self.world_size // tensor_model_parallel_world_size + ) + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + pipeline_model_parallel_split_rank_=0, + ) + self.assertEqual( + tensor_model_parallel_world_size, + parallel_state.get_tensor_model_parallel_world_size(), + msg=msg, + ) + expected_tensor_model_parallel_rank = calc_expected_tensor_model_paralell_rank( + self.rank, tensor_model_parallel_world_size + ) + self.assertEqual( + expected_tensor_model_parallel_rank, + parallel_state.get_tensor_model_parallel_rank(), + msg=msg, + ) + + expected_tensor_model_parallel_src_rank = ( + self.rank // tensor_model_parallel_world_size + ) * tensor_model_parallel_world_size + self.assertEqual( + expected_tensor_model_parallel_src_rank, + parallel_state.get_tensor_model_parallel_src_rank(), + msg=msg, + ) + + parallel_state.destroy_model_parallel() + self.assertFalse(parallel_state.model_parallel_is_initialized(), msg=msg) + + +class NcclParallelStateTest(ParallelStateTestBase, NcclDistributedTestBase): pass +class UccParallelStateTest(ParallelStateTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_pipeline_parallel_fwd_bwd.py b/apex/tests/L0/run_transformer/test_pipeline_parallel_fwd_bwd.py new file mode 100644 index 00000000..a1193f1c --- /dev/null +++ b/apex/tests/L0/run_transformer/test_pipeline_parallel_fwd_bwd.py @@ -0,0 +1,716 @@ +import contextlib +import logging +import itertools +import os +from packaging.version import parse, Version +import re +from typing import Optional, Tuple, List +import unittest + +import torch +from torch.testing._internal import common_utils + +from apex._autocast_utils import _get_autocast_dtypes +from apex.transformer import parallel_state +from apex.transformer.enums import ModelType +from apex.transformer.pipeline_parallel import utils as pp_utils +from apex.transformer.pipeline_parallel.schedules.common import ( + FwdStepFunc, + build_model, + _get_params_for_weight_decay_optimization, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_no_pipelining import ( + forward_backward_no_pipelining, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_with_interleaving import ( + _forward_backward_pipelining_with_interleaving, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_without_interleaving import ( + forward_backward_pipelining_without_interleaving, +) +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase +from apex.transformer.testing.distributed_test_base import HAS_TORCH_UCC_COMPAT_NVIDIA_DRIVER +from apex.transformer.testing import commons as testing_utils +from apex.transformer._ucc_util import HAS_UCC + +logging.getLogger("torch").setLevel(logging.WARNING) +logging.getLogger("apex").setLevel(logging.WARNING) + +weight_coeff = 1024 + +# Guard for https://github.com/pytorch/pytorch/pull/82450 +CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV = False +ngc_container_2209, pytorch_113 = Version("22.09"), Version("1.13") +if parse(os.getenv("NVIDIA_PYTORCH_VERSION", "22.08")) >= ngc_container_2209: + CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV = True +elif parse(torch.__version__) >= pytorch_113: + CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV = True +else: + CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV = False + + +def get_init_weights_func(offset: int = 0): + @torch.no_grad() + def init_weights(m): + rank = parallel_state.get_pipeline_model_parallel_rank() + if isinstance(m, torch.nn.Linear): + m.weight.fill_((rank + offset + 1.0) / weight_coeff) + m.bias.fill_(1.0) + return init_weights + + +def get_dtype_for_comparison(): + if(torch.cuda.get_device_capability() >= (8, 0)): + return torch.float64 + return torch.float32 + + +def get_target_loss_and_model(global_batch_shape: tuple, hidden_size: int, total_layers: int) -> Tuple[torch.Tensor, List[torch.Tensor]]: + model = [] + dtype = get_dtype_for_comparison() + data = torch.ones(global_batch_shape, dtype=dtype) + for i in range(total_layers): + w = torch.ones((hidden_size, hidden_size), dtype=dtype) * (i + 1.0) / weight_coeff + b = torch.ones(hidden_size, dtype=dtype) + + w.requires_grad_() + b.requires_grad_() + + # don't need to care about transpose semantics as all values are the same + data = torch.matmul(w, data) + b + model.append([w, b]) + + loss = data.sum() / global_batch_shape[0] + loss.backward() + + return loss, model + + +def _get_default_world_sizes_model_parallel_world_size(pipeline_model_parallel_world_size: Optional[int] = None + ) -> Tuple[int, int, int]: + # TODO: revisit if we can fold this into the class for skip logic / avoid duplication + # of world size computation + world_size = torch.cuda.device_count() + tensor_model_parallel_world_size = 1 + data_parallel_size = 1 + (world_size >= 8 and world_size % 2 == 0) + + if pipeline_model_parallel_world_size is None: + pipeline_model_parallel_world_size = world_size // (tensor_model_parallel_world_size * data_parallel_size) + else: + data_parallel_size = world_size // (tensor_model_parallel_world_size * pipeline_model_parallel_world_size) + + return tensor_model_parallel_world_size, data_parallel_size, pipeline_model_parallel_world_size + + +class PipelineParallelForwardBackwardTestBase: + + GLOBAL_BATCH_SIZE = 16 + MICRO_BATCH_SIZE = 2 + HIDDEN_SIZE = 32 + + deallocate_options = (True, False) + # If :obj:`None`, (torch.float32, torch.float16, torch.bfloat16) are dtype options on Ampere. + # You can limit the options by overriding the following `dtypes`. + dtypes = None + + def _forward_backward_test_impl( + self, + forward_only: bool, + fwd_bwd_func: FwdStepFunc, + pipeline_model_parallel_world_size: Optional[int], + virtual_pipeline_model_parallel_size: Optional[int], + async_comm: bool = False, + *, + default_backend: Optional[str] = None, + p2p_backend: Optional[str] = None, + sync_batch_comm: bool = True, + ) -> None: + if fwd_bwd_func == _forward_backward_pipelining_with_interleaving: + self.assertIsNotNone(virtual_pipeline_model_parallel_size) + self.assertGreater(virtual_pipeline_model_parallel_size, 1) + dtype_options = self.dtypes or [torch.float32, torch.double] + _get_autocast_dtypes() + + for dtype, deallocate_pipeline_outputs in itertools.product( + dtype_options, self.deallocate_options, + ): + grad_scaler = ( + torch.cuda.amp.GradScaler(init_scale=4.0) + if dtype == torch.half + else None + ) + + (tensor_model_parallel_world_size, + data_parallel_size, + pipeline_model_parallel_world_size) = _get_default_world_sizes_model_parallel_world_size(pipeline_model_parallel_world_size) + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + virtual_pipeline_model_parallel_size_=virtual_pipeline_model_parallel_size, + default_backend=default_backend, + p2p_backend=p2p_backend, + ) + pp_utils._reconfigure_microbatch_calculator( + rank=parallel_state.get_tensor_model_parallel_rank(), + rampup_batch_size=None, + global_batch_size=self.GLOBAL_BATCH_SIZE, + micro_batch_size=self.MICRO_BATCH_SIZE, + data_parallel_size=parallel_state.get_data_parallel_world_size(), + ) + + global_batch_shape = ( + self.GLOBAL_BATCH_SIZE + // parallel_state.get_data_parallel_world_size(), + self.HIDDEN_SIZE, + self.HIDDEN_SIZE, + ) + + batch = None + if parallel_state.is_pipeline_first_stage(): + batch = (torch.ones(global_batch_shape, dtype=dtype).cuda(), ) + + model = build_model( + testing_utils.model_provider_func, + # Use DDP only when it's better to have + wrap_with_ddp=data_parallel_size > 1, + virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, + hidden_size=self.HIDDEN_SIZE, + ) + + offset = pipeline_model_parallel_world_size if virtual_pipeline_model_parallel_size is not None else 0 + for idx, model_module in enumerate(model): + model_module = model_module.to(dtype) + model_module.apply(get_init_weights_func(idx*offset)) + + _param_groups = _get_params_for_weight_decay_optimization(model) + optimizer = torch.optim.Adam(_param_groups, lr=1e-3) + + pp_utils.update_num_microbatches(0) + + loss = fwd_bwd_func( + testing_utils.fwd_step_func, + batch, + model, + forward_only=forward_only, + # `tensor_shape` is the shape of micro batch. + tensor_shape=( + self.MICRO_BATCH_SIZE, + self.HIDDEN_SIZE, + self.HIDDEN_SIZE, + ), + dtype=dtype, + async_comm=async_comm, + grad_scaler=grad_scaler, + deallocate_pipeline_output=deallocate_pipeline_outputs, + sync_batch_comm=sync_batch_comm, + ) + + if dtype == get_dtype_for_comparison(): + torch.cuda.synchronize() + hidden_size = self.HIDDEN_SIZE + microbatch_size = self.MICRO_BATCH_SIZE + total_layers = pipeline_model_parallel_world_size + if virtual_pipeline_model_parallel_size is not None: + total_layers *= virtual_pipeline_model_parallel_size + target_loss, target_model = get_target_loss_and_model(global_batch_shape, hidden_size, total_layers) + + for loss_item in loss: + x = loss_item['avg'] + self.assertEqual(x.item() / microbatch_size, target_loss.item()) + + if not forward_only: + for vm_id, model_module in enumerate(model): + params = list(model_module.parameters()) + rank = params[0].get_device() + offset = pipeline_model_parallel_world_size + param_id = rank // data_parallel_size + vm_id * offset + target_params = target_model[param_id] + + self.assertEqual(params[0].cpu(), target_params[0]) + self.assertEqual(params[1].cpu(), target_params[1]) + self.assertEqual(params[0].grad.cpu() / microbatch_size, target_params[0].grad) + self.assertEqual(params[1].grad.cpu() / microbatch_size, target_params[1].grad) + + if not forward_only: + for m in model: + for p in m.parameters(): + self.assertIsNotNone(p.grad) + optimizer.step() + optimizer.zero_grad(set_to_none=True) + + parallel_state.destroy_model_parallel() + + def test_learning_no_pipelining(self): + self._forward_backward_test_impl(False, forward_backward_no_pipelining, 1, None) + + def test_inference_no_pipelining(self): + self._forward_backward_test_impl(True, forward_backward_no_pipelining, 1, None) + + def test_learning_pipelining_without_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + False, forward_backward_pipelining_without_interleaving, None, None, sync_batch_comm=sync_batch_comm, + ) + + def test_inference_pipelining_without_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + True, forward_backward_pipelining_without_interleaving, None, None, sync_batch_comm=sync_batch_comm, + ) + + def test_learning_async_pipelining_without_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + False, forward_backward_pipelining_without_interleaving, None, None, async_comm=True, + sync_batch_comm=sync_batch_comm, + ) + + def test_inference_async_pipelining_without_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + True, forward_backward_pipelining_without_interleaving, None, None, async_comm=True, + sync_batch_comm=sync_batch_comm, + ) + + # fails on native ucc: times out + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_learning_pipelining_with_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + False, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2, + sync_batch_comm=sync_batch_comm, + ) + + # fails on native ucc: times out + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_inference_pipelining_with_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + True, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2, + sync_batch_comm=sync_batch_comm, + ) + + # fails on native ucc: times out + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_learning_async_pipelining_with_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + False, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2, async_comm=True, + sync_batch_comm=sync_batch_comm, + ) + + # fails on native ucc: times out + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_inference_async_pipelining_with_interleaving(self, sync_batch_comm: bool = True): + self._forward_backward_test_impl( + True, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2, async_comm=True, + sync_batch_comm=sync_batch_comm, + ) + + +class NcclPipelineParallelForwardBackwardTest(NcclDistributedTestBase, PipelineParallelForwardBackwardTestBase): + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 8) + + def _run_hybrid_distributed_backend(self, forward_only: bool) -> None: + self._forward_backward_test_impl( + forward_only, forward_backward_pipelining_without_interleaving, None, None, + default_backend="nccl", p2p_backend="ucc", + ) + + @unittest.skipUnless(HAS_TORCH_UCC_COMPAT_NVIDIA_DRIVER, "Needs driver >= 470.42.01") + def _test_hybrid_backends(self, forward_only: bool) -> None: + if HAS_UCC: + self._run_hybrid_distributed_backend(forward_only) + else: + with self.assertRaisesRegex( + ImportError, + re.escape("UCC backend requires pytorch source build with UCC installed and enabled"), + ): + self._run_hybrid_distributed_backend(forward_only) + + def test_learning_pipelining_without_interleaving_ucc_for_p2p(self): + self._test_hybrid_backends(False) + + def test_inference_pipelining_without_interleaving_ucc_for_p2p(self): + self._test_hybrid_backends(True) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_learning_pipelining_without_interleaving_skyp_sync_after_batch_isend_irecv(self): + self.test_learning_pipelining_without_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_inference_pipelining_without_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_inference_pipelining_without_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_learning_async_pipelining_without_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_learning_async_pipelining_without_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_inference_async_pipelining_without_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_inference_async_pipelining_without_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_learning_pipelining_with_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_learning_pipelining_with_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_inference_pipelining_with_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_inference_pipelining_with_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_learning_async_pipelining_with_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_learning_async_pipelining_with_interleaving(sync_batch_comm=False) + + @unittest.skipUnless(CAN_SKIP_SYNC_AFTER_BATCH_ISEND_IRECV, "Requires https://github.com/pytorch/pytorch/pull/82450") + def test_inference_async_pipelining_with_interleaving_skip_sync_after_batch_isend_irecv(self): + self.test_inference_async_pipelining_with_interleaving(sync_batch_comm=False) + + +# n.b.(mkozuki): pipeline parallel w/o interleaving with UCX_TLS=tcp,sm fails. +class UccPipelineParallelForwardBackwardTest(UccDistributedTestBase, PipelineParallelForwardBackwardTestBase): + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 8) + + deallocate_options = (False,) + dtypes = (torch.float32,) + + +# Sanity checking the functionality of `forward_backward_pipelining_without_interleaving` with +# `model_type=ModelType.encoder_and_decoder` which is used for pipeline training of transformer +# models such as T5. +@unittest.skipIf(torch.cuda.device_count() < 4, "Requires >= 4 GPUs") +class NcclPipelineParallelWithToyParallelMLP(NcclDistributedTestBase): + + GLOBAL_BATCH_SIZE: int = 16 + MICRO_BATCH_SIZE: int = 2 + HIDDEN_SIZE: int = 64 + # TODO(mkozuki): Change `DECODER_SEQUENCE_LENGTH` to a value different from `ENCODER_SEQUENCE_LENGTH`. + # To test forward_backward_pipelining_without_interleaving with `model_type=ModelType.encoder_and_decoder`, + # `decoder_seq_length` is necessary and ideally should be different from `encoder_sequence_length` + # but my laziness let me use the same value. + # Note that you may have to either update `MyModel` def or define another `MyModel`. + # to support different `DECODER_SEQUENCE_LENGTH`. + ENCODER_SEQUENCE_LENGTH: int = 32 + DECODER_SEQUENCE_LENGTH: int = 32 + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 8) + + # TODO(mkozuki): Set `tensor_model_parallel>1` for encoder_and_decoder as well if there's enough GPUs + # in order to let `sequence_parallel_enabled` have an effect on tensor shape logic. + def _forward_backward_test_impl( + self, + *, + forward_only: bool, + sequence_parallel_enabled: bool, + model_type: ModelType, + dtype: torch.dtype = torch.float32, + ) -> None: + # N.B.(mkozuki): It might be better to set `tensor_model_parallel_size` to >1 + # if `self.world_size > 5`. Otherwise, `pipeline_model_parallel_split_rank` + # can be 1, which can be too far real usecase. + tensor_model_parallel_size = 1 + int(self.world_size >= 4) + pipeline_model_parallel_world_size = self.world_size // tensor_model_parallel_size + if model_type == ModelType.encoder_and_decoder: + pipeline_model_parallel_split_rank = pipeline_model_parallel_world_size // 2 + else: + pipeline_model_parallel_split_rank = None + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + virtual_pipeline_model_parallel_size_=None, + pipeline_model_parallel_split_rank_=pipeline_model_parallel_split_rank, + ) + testing_utils.set_random_seed(567) + pp_utils._reconfigure_microbatch_calculator( + rank=parallel_state.get_tensor_model_parallel_rank(), + rampup_batch_size=None, + global_batch_size=self.GLOBAL_BATCH_SIZE, + micro_batch_size=self.MICRO_BATCH_SIZE, + data_parallel_size=parallel_state.get_data_parallel_world_size(), + ) + # TODO(mkozuki): Call `build_model` with `model_type`. + model = build_model( + testing_utils.mlp_provider_func, + wrap_with_ddp=False, + virtual_pipeline_model_parallel_size=None, + hidden_size=self.HIDDEN_SIZE, + sequence_parallel_enabled=sequence_parallel_enabled, + ) + model = [m.to(dtype=dtype) for m in model] + + if parallel_state.is_pipeline_first_stage(): + batch: Tuple[torch.Tensor] = ( + torch.ones( + (self.GLOBAL_BATCH_SIZE, self.ENCODER_SEQUENCE_LENGTH, self.HIDDEN_SIZE), + dtype=dtype, + device="cuda", + ), + ) + else: + batch = None + + forward_backward_pipelining_without_interleaving( + forward_step_func=testing_utils.ToyParallelMLPFwdBwdStepFunc( + sequence_parallel_enabled=sequence_parallel_enabled, + ), + batch=batch, + model=model, + forward_only=forward_only, + tensor_shape=( + self.ENCODER_SEQUENCE_LENGTH, + self.MICRO_BATCH_SIZE, + self.HIDDEN_SIZE, + ), + model_type=model_type, + decoder_sequence_length=self.DECODER_SEQUENCE_LENGTH, + async_comm=False, + grad_scaler=None, + deallocate_pipeline_outputs=False, + dtype=dtype, + sequence_parallel_enabled=sequence_parallel_enabled, + ) + + def test_pipelining_without_interleaving_encoder_and_decoder(self) -> None: + self._forward_backward_test_impl(forward_only=False, sequence_parallel_enabled=False, model_type=ModelType.encoder_and_decoder) + + def test_pipelining_without_interleaving_inferenc_encoder_and_decoder(self) -> None: + self._forward_backward_test_impl(forward_only=True, sequence_parallel_enabled=False, model_type=ModelType.encoder_and_decoder) + + def test_pipelining_without_interleaving_sequence_paralle_encoder_and_decoder(self) -> None: + self._forward_backward_test_impl(forward_only=False, sequence_parallel_enabled=True, model_type=ModelType.encoder_and_decoder) + + def test_pipelining_without_interleaving_inference_sequence_paralle_encoder_and_decoder(self) -> None: + self._forward_backward_test_impl(forward_only=True, sequence_parallel_enabled=True, model_type=ModelType.encoder_and_decoder) + + def test_pipelining_without_interleaving_encoder_or_decoder(self) -> None: + self._forward_backward_test_impl(forward_only=False, sequence_parallel_enabled=False, model_type=ModelType.encoder_or_decoder) + + def test_pipelining_without_interleaving_sequence_parallel_encoder_or_decoder(self) -> None: + self._forward_backward_test_impl(forward_only=False, sequence_parallel_enabled=True, model_type=ModelType.encoder_or_decoder) + + def test_pipelining_without_interleaving_sequence_parallel_encoder_or_decoder_half(self) -> None: + self._forward_backward_test_impl(forward_only=False, sequence_parallel_enabled=True, model_type=ModelType.encoder_or_decoder, dtype=torch.half) + + +class NcclPipelineParallelWithCustomSyncContextHandler(NcclDistributedTestBase): + + GLOBAL_BATCH_SIZE = 32 + MICRO_BATCH_SIZE = 1 + HIDDEN_SIZE = 1 + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 8) + + @unittest.skipIf(torch.cuda.device_count() < 2 or torch.cuda.device_count() % 2 != 0, "Requires >= 2 GPUs") + def test_pipelining_without_interleaving_with_custom_sync_context_handler(self) -> None: + + # Parallel configuration + world_size = torch.cuda.device_count() + tensor_model_parallel_world_size = 1 + data_parallel_size = 2 if world_size > 2 else 1 + pipeline_model_parallel_world_size = world_size // data_parallel_size + + # Initialize pipeline parallelism + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + ) + pp_utils._reconfigure_microbatch_calculator( + rank=parallel_state.get_tensor_model_parallel_rank(), + rampup_batch_size=None, + global_batch_size=self.GLOBAL_BATCH_SIZE, + micro_batch_size=self.MICRO_BATCH_SIZE, + data_parallel_size=parallel_state.get_data_parallel_world_size(), + ) + pp_utils.update_num_microbatches(0) + + # Construct synthetic data + dtype = get_dtype_for_comparison() + hidden_size = self.HIDDEN_SIZE + microbatch_size = self.MICRO_BATCH_SIZE + global_batch_shape = ( + self.GLOBAL_BATCH_SIZE + // parallel_state.get_data_parallel_world_size(), + hidden_size, + hidden_size, + ) + batch = None + if parallel_state.is_pipeline_first_stage(): + batch = (torch.ones(global_batch_shape, dtype=dtype).cuda(), ) + + # Construct model + model = build_model( + testing_utils.model_provider_func, + wrap_with_ddp=True, + hidden_size=hidden_size, + )[0] + model = model.to(dtype) + model.module.apply(get_init_weights_func(0)) + + # Construct context that destroys all grads on exit + has_entered_grad_sync_context = False + has_exited_grad_sync_context = False + has_called_grad_sync_func = False + @contextlib.contextmanager + def custom_grad_sync_context(): + try: + nonlocal has_entered_grad_sync_context + has_entered_grad_sync_context = True + yield + finally: + nonlocal has_exited_grad_sync_context + has_exited_grad_sync_context = True + for param in model.parameters(): + param.grad = None + def custom_grad_sync_func(): + nonlocal has_called_grad_sync_func + has_called_grad_sync_func = True + + # Training step with pipeline parallelism + loss = forward_backward_pipelining_without_interleaving( + testing_utils.fwd_step_func, + batch, + model, + forward_only=False, + tensor_shape=(microbatch_size, hidden_size, hidden_size), + dtype=dtype, + async_comm=False, + grad_scaler=None, + deallocate_pipeline_outputs=False, + sequence_parallel_enabled=False, + custom_sync_context_handler=custom_grad_sync_context, + custom_grad_sync_func=custom_grad_sync_func, + ) + torch.cuda.synchronize() + + # Check if model has initialized gradients + has_any_grads = any(param.grad is not None for param in model.parameters()) + has_all_grads = all(param.grad is not None for param in model.parameters()) + + # Check context behavior + self.assertTrue(has_entered_grad_sync_context, 'Has not entered custom sync context') + self.assertTrue(has_exited_grad_sync_context, 'Has not exited custom sync context') + self.assertEqual( + has_any_grads, + has_all_grads, + 'Expected gradients to all be uninitialized or all be initialized', + ) + self.assertEqual( + has_all_grads, + parallel_state.is_pipeline_first_stage(), + 'Expected gradients to be initialized only in first pipeline stage', + ) + + # Clean up + parallel_state.destroy_model_parallel() + + @unittest.skipIf(torch.cuda.device_count() < 4 or torch.cuda.device_count() % 2 != 0, "Requires >= 4 GPUs") + def test_pipelining_with_interleaving_with_custom_sync_context_handler(self) -> None: + + # Parallel configuration + world_size = torch.cuda.device_count() + tensor_model_parallel_world_size = 1 + data_parallel_size = 2 if world_size > 4 else 1 + pipeline_model_parallel_world_size = world_size // data_parallel_size + virtual_pipeline_model_parallel_size = 2 + + # Initialize pipeline parallelism + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + virtual_pipeline_model_parallel_size_=virtual_pipeline_model_parallel_size, + ) + pp_utils._reconfigure_microbatch_calculator( + rank=parallel_state.get_tensor_model_parallel_rank(), + rampup_batch_size=None, + global_batch_size=self.GLOBAL_BATCH_SIZE, + micro_batch_size=self.MICRO_BATCH_SIZE, + data_parallel_size=parallel_state.get_data_parallel_world_size(), + ) + pp_utils.update_num_microbatches(0) + + # Construct synthetic data + dtype = get_dtype_for_comparison() + hidden_size = self.HIDDEN_SIZE + microbatch_size = self.MICRO_BATCH_SIZE + global_batch_shape = ( + self.GLOBAL_BATCH_SIZE + // parallel_state.get_data_parallel_world_size(), + hidden_size, + hidden_size, + ) + batch = None + if parallel_state.is_pipeline_first_stage(): + batch = (torch.ones(global_batch_shape, dtype=dtype).cuda(), ) + + # Construct model + model = build_model( + testing_utils.model_provider_func, + wrap_with_ddp=True, + virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, + hidden_size=hidden_size, + ) + for module in model: + module.to(dtype) + module.module.apply(get_init_weights_func(0)) + + # Construct context that keeps track whenever entered/exited + grad_sync_context_enter_count = 0 + grad_sync_context_exit_count = 0 + @contextlib.contextmanager + def custom_grad_sync_context(): + try: + nonlocal grad_sync_context_enter_count + grad_sync_context_enter_count += 1 + yield + finally: + nonlocal grad_sync_context_exit_count + grad_sync_context_exit_count += 1 + for module in model: + for param in module.parameters(): + param.grad = None + + # Training step with pipeline parallelism + loss = _forward_backward_pipelining_with_interleaving( + testing_utils.fwd_step_func, + batch, + model, + forward_only=False, + tensor_shape=(microbatch_size, hidden_size, hidden_size), + dtype=dtype, + async_comm=False, + grad_scaler=None, + deallocate_pipeline_outputs=False, + sequence_parallel_enabled=False, + custom_sync_context_handler=custom_grad_sync_context, + ) + torch.cuda.synchronize() + + # Check context behavior + self.assertTrue( + grad_sync_context_enter_count > 0, + 'Has not entered custom sync context', + ) + self.assertEqual( + grad_sync_context_enter_count, + grad_sync_context_exit_count, + 'Has not entered and exited custom sync context ' + 'the same number of times', + ) + self.assertEqual( + grad_sync_context_exit_count, + virtual_pipeline_model_parallel_size + 1, + 'Expected to exit custom sync context once per model chunk ' + 'and once at the function end', + ) + + # Clean up + parallel_state.destroy_model_parallel() + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_random.py b/apex/tests/L0/run_transformer/test_random.py new file mode 100644 index 00000000..bd737ee5 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_random.py @@ -0,0 +1,117 @@ +import logging + +import torch +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer import tensor_parallel +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase + +logging.getLogger("apex").setLevel(logging.WARNING) + + +class TransformerRandomTestBase: + def test_set_cuda_rng_state(self): + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size + ) + + size, seed = 123, 1234 + torch.cuda.manual_seed(seed) + tensor = torch.cuda.FloatTensor(size) + + rng_state = torch.cuda.get_rng_state() + rng_state_clone = rng_state.clone() + + for _ in range(5): + torch.randn(size, out=tensor) + result_1 = tensor.clone() + + self.assertEqual(rng_state.sub(rng_state_clone).max(), 0, msg=msg) + self.assertGreater( + torch.cuda.get_rng_state().sub(rng_state_clone).max(), 0, + msg=msg, + ) + + new_rng_state = torch.cuda.get_rng_state() + self.assertGreater(new_rng_state.sub(rng_state).max(), 0, msg=msg) + + tensor_parallel.random._set_cuda_rng_state(rng_state) + for _ in range(5): + torch.randn(size, out=tensor) + tensor_parallel.random._set_cuda_rng_state(rng_state) + for _ in range(5): + torch.randn(size, out=tensor) + result_2 = tensor.clone() + + self.assertEqual(result_2, result_1, msg=msg) + + self.assertEqual(rng_state.sub(rng_state_clone).max(), 0, msg=msg) + + parallel_state.destroy_model_parallel() + + def test_cuda_rng_tracker(self): + for tensor_model_parallel_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_parallel_world_size: + continue + msg = f"tensor_model_parallel_world_size: {tensor_model_parallel_world_size}" + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size + ) + + seed_1, seed_2, size = 1234, 4321, [12, 21] + tensor = torch.cuda.FloatTensor(size) + + torch.cuda.manual_seed(seed_1) + torch.randn(size, out=tensor) + target_11 = tensor.clone() + torch.randn(size, out=tensor) + target_12 = tensor.clone() + + torch.cuda.manual_seed(seed_2) + torch.randn(size, out=tensor) + targt_21 = tensor.clone() + torch.randn(size, out=tensor) + target_22 = tensor.clone() + + torch.cuda.manual_seed(seed_1) + tensor_parallel.random.get_cuda_rng_tracker().add("test", seed_2) + + torch.randn(size, out=tensor) + result_11 = tensor.clone() + + with tensor_parallel.random.get_cuda_rng_tracker().fork("test"): + torch.randn(size, out=tensor) + result_21 = tensor.clone() + + torch.randn(size, out=tensor) + result_12 = tensor.clone() + + with tensor_parallel.random.get_cuda_rng_tracker().fork("test"): + torch.randn(size, out=tensor) + result_22 = tensor.clone() + + self.assertEqual(target_11, result_11, msg=msg) + self.assertEqual(target_12, result_12, msg=msg) + self.assertEqual(targt_21, result_21, msg=msg) + self.assertEqual(target_22, result_22, msg=msg) + self.assertNotEqual(result_11, result_21, msg=msg) + self.assertNotEqual(result_21, result_22, msg=msg) + + tensor_parallel.random.get_cuda_rng_tracker().reset() + parallel_state.destroy_model_parallel() + + +class NcclTransformerRandomTest(TransformerRandomTestBase, NcclDistributedTestBase): pass +class UccTransformerRandomTest(TransformerRandomTestBase, UccDistributedTestBase): pass + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L0/run_transformer/test_transformer_utils.py b/apex/tests/L0/run_transformer/test_transformer_utils.py new file mode 100644 index 00000000..4164eca8 --- /dev/null +++ b/apex/tests/L0/run_transformer/test_transformer_utils.py @@ -0,0 +1,40 @@ +import logging + +import torch +from torch.testing._internal import common_utils + +logging.getLogger("torch").setLevel(logging.WARNING) + +from apex.transformer import parallel_state +from apex.transformer.tensor_parallel import utils +from apex.transformer.testing.distributed_test_base import NcclDistributedTestBase + +logging.getLogger("apex").setLevel(logging.WARNING) + + +class TransformerUtilsTest(NcclDistributedTestBase): + def test_split_tensor_along_last_dim(self): + for tensor_model_paralell_world_size in range(1, self.world_size + 1): + if self.world_size % tensor_model_paralell_world_size > 0: + continue + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_paralell_world_size + ) + + device = "cpu" + input_tensor = torch.randn((100, 100, 100), device=device) + splits = utils.split_tensor_along_last_dim(input_tensor, 10) + last_dim_shapes = torch.tensor( + [int(split.size()[-1]) for split in splits] + ) + + self.assertTrue( + torch.equal(last_dim_shapes, torch.full((10,), 10),), + msg=f"tensor_model_paralell_world_size: {tensor_model_paralell_world_size}", + ) + + parallel_state.destroy_model_parallel() + + +if __name__ == "__main__": + common_utils.run_tests() diff --git a/apex/tests/L1/common/compare.py b/apex/tests/L1/common/compare.py new file mode 100644 index 00000000..74374d41 --- /dev/null +++ b/apex/tests/L1/common/compare.py @@ -0,0 +1,64 @@ +import argparse +import torch + +parser = argparse.ArgumentParser(description='Compare') +parser.add_argument('--opt-level', type=str) +parser.add_argument('--keep-batchnorm-fp32', type=str, default=None) +parser.add_argument('--loss-scale', type=str, default=None) +parser.add_argument('--fused-adam', action='store_true') +parser.add_argument('--use_baseline', action='store_true') +args = parser.parse_args() + +base_file = str(args.opt_level) + "_" +\ + str(args.loss_scale) + "_" +\ + str(args.keep_batchnorm_fp32) + "_" +\ + str(args.fused_adam) + +file_e = "True_" + base_file +file_p = "False_" + base_file +if args.use_baseline: + file_b = "baselines/True_" + base_file + +dict_e = torch.load(file_e) +dict_p = torch.load(file_p) +if args.use_baseline: + dict_b = torch.load(file_b) + +torch.set_printoptions(precision=10) + +print(file_e) +print(file_p) +if args.use_baseline: + print(file_b) + +# ugly duplication here... +if not args.use_baseline: + for n, (i_e, i_p) in enumerate(zip(dict_e["Iteration"], dict_p["Iteration"])): + assert i_e == i_p, "i_e = {}, i_p = {}".format(i_e, i_p) + + loss_e = dict_e["Loss"][n] + loss_p = dict_p["Loss"][n] + assert loss_e == loss_p, "Iteration {}, loss_e = {}, loss_p = {}".format(i_e, loss_e, loss_p) + print("{:4} {:15.10f} {:15.10f} {:15.10f} {:15.10f}".format( + i_e, + loss_e, + loss_p, + dict_e["Speed"][n], + dict_p["Speed"][n])) +else: + for n, (i_e, i_p) in enumerate(zip(dict_e["Iteration"], dict_p["Iteration"])): + assert i_e == i_p, "i_e = {}, i_p = {}".format(i_e, i_p) + + loss_e = dict_e["Loss"][n] + loss_p = dict_p["Loss"][n] + loss_b = dict_b["Loss"][n] + assert loss_e == loss_p, "Iteration {}, loss_e = {}, loss_p = {}".format(i_e, loss_e, loss_p) + assert loss_e == loss_b, "Iteration {}, loss_e = {}, loss_b = {}".format(i_e, loss_e, loss_b) + print("{:4} {:15.10f} {:15.10f} {:15.10f} {:15.10f} {:15.10f} {:15.10f}".format( + i_e, + loss_b, + loss_e, + loss_p, + dict_b["Speed"][n], + dict_e["Speed"][n], + dict_p["Speed"][n])) diff --git a/apex/tests/L1/common/main_amp.py b/apex/tests/L1/common/main_amp.py new file mode 100644 index 00000000..106a0f63 --- /dev/null +++ b/apex/tests/L1/common/main_amp.py @@ -0,0 +1,526 @@ +import argparse +import os +import shutil +import time + +import torch +import torch.nn as nn +import torch.nn.parallel +import torch.backends.cudnn as cudnn +import torch.distributed as dist +import torch.optim +import torch.utils.data +import torch.utils.data.distributed +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as models + +import numpy as np + +try: + from apex.parallel import DistributedDataParallel as DDP + from apex.fp16_utils import * + from apex import amp, optimizers + from apex.multi_tensor_apply import multi_tensor_applier +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") + +model_names = sorted(name for name in models.__dict__ + if name.islower() and not name.startswith("__") + and callable(models.__dict__[name])) + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Training') +parser.add_argument('data', metavar='DIR', + help='path to dataset') +parser.add_argument('--arch', '-a', metavar='ARCH', default='resnet18', + choices=model_names, + help='model architecture: ' + + ' | '.join(model_names) + + ' (default: resnet18)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 4)') +parser.add_argument('--epochs', default=90, type=int, metavar='N', + help='number of total epochs to run') +parser.add_argument('--start-epoch', default=0, type=int, metavar='N', + help='manual epoch number (useful on restarts)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', help='mini-batch size per process (default: 256)') +parser.add_argument('--lr', '--learning-rate', default=0.1, type=float, + metavar='LR', help='Initial learning rate. Will be scaled by /256: args.lr = args.lr*float(args.batch_size*args.world_size)/256. A warmup schedule will also be applied over the first 5 epochs.') +parser.add_argument('--momentum', default=0.9, type=float, metavar='M', + help='momentum') +parser.add_argument('--weight-decay', '--wd', default=1e-4, type=float, + metavar='W', help='weight decay (default: 1e-4)') +parser.add_argument('--print-freq', '-p', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--resume', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true', + help='evaluate model on validation set') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') + +parser.add_argument('--prof', dest='prof', action='store_true', + help='Only run 10 iterations for profiling.') +parser.add_argument('--deterministic', action='store_true') + +parser.add_argument("--local_rank", default=0, type=int) +parser.add_argument('--sync_bn', action='store_true', + help='enabling apex sync BN.') + +parser.add_argument('--has-ext', action='store_true') +parser.add_argument('--opt-level', type=str) +parser.add_argument('--keep-batchnorm-fp32', type=str, default=None) +parser.add_argument('--loss-scale', type=str, default=None) +parser.add_argument('--fused-adam', action='store_true') + +parser.add_argument('--prints-to-process', type=int, default=10) + +cudnn.benchmark = True + +def fast_collate(batch): + imgs = [img[0] for img in batch] + targets = torch.tensor([target[1] for target in batch], dtype=torch.int64) + w = imgs[0].size[0] + h = imgs[0].size[1] + tensor = torch.zeros( (len(imgs), 3, h, w), dtype=torch.uint8 ) + for i, img in enumerate(imgs): + nump_array = np.asarray(img, dtype=np.uint8) + if(nump_array.ndim < 3): + nump_array = np.expand_dims(nump_array, axis=-1) + nump_array = np.rollaxis(nump_array, 2) + + tensor[i] += torch.from_numpy(nump_array) + + return tensor, targets + +best_prec1 = 0 +args = parser.parse_args() + +# Let multi_tensor_applier be the canary in the coalmine +# that verifies if the backend is what we think it is +assert multi_tensor_applier.available == args.has_ext + +print("opt_level = {}".format(args.opt_level)) +print("keep_batchnorm_fp32 = {}".format(args.keep_batchnorm_fp32), type(args.keep_batchnorm_fp32)) +print("loss_scale = {}".format(args.loss_scale), type(args.loss_scale)) + + +print("\nCUDNN VERSION: {}\n".format(torch.backends.cudnn.version())) + +if args.deterministic: + cudnn.benchmark = False + cudnn.deterministic = True + torch.manual_seed(args.local_rank) + torch.set_printoptions(precision=10) + +def main(): + global best_prec1, args + + args.distributed = False + if 'WORLD_SIZE' in os.environ: + args.distributed = int(os.environ['WORLD_SIZE']) > 1 + + args.gpu = 0 + args.world_size = 1 + + if args.distributed: + args.gpu = args.local_rank % torch.cuda.device_count() + torch.cuda.set_device(args.gpu) + torch.distributed.init_process_group(backend='nccl', + init_method='env://') + args.world_size = torch.distributed.get_world_size() + + assert torch.backends.cudnn.enabled, "Amp requires cudnn backend to be enabled." + + # create model + if args.pretrained: + print("=> using pre-trained model '{}'".format(args.arch)) + model = models.__dict__[args.arch](pretrained=True) + else: + print("=> creating model '{}'".format(args.arch)) + model = models.__dict__[args.arch]() + + if args.sync_bn: + import apex + print("using apex synced BN") + model = apex.parallel.convert_syncbn_model(model) + + model = model.cuda() + + # Scale learning rate based on global batch size + args.lr = args.lr*float(args.batch_size*args.world_size)/256. + if args.fused_adam: + optimizer = optimizers.FusedAdam(model.parameters()) + else: + optimizer = torch.optim.SGD(model.parameters(), args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay) + + model, optimizer = amp.initialize( + model, optimizer, + # enabled=False, + opt_level=args.opt_level, + keep_batchnorm_fp32=args.keep_batchnorm_fp32, + loss_scale=args.loss_scale + ) + + if args.distributed: + # By default, apex.parallel.DistributedDataParallel overlaps communication with + # computation in the backward pass. + # model = DDP(model) + # delay_allreduce delays all communication to the end of the backward pass. + model = DDP(model, delay_allreduce=True) + + # define loss function (criterion) and optimizer + criterion = nn.CrossEntropyLoss().cuda() + + # Optionally resume from a checkpoint + if args.resume: + # Use a local scope to avoid dangling references + def resume(): + if os.path.isfile(args.resume): + print("=> loading checkpoint '{}'".format(args.resume)) + checkpoint = torch.load(args.resume, map_location = lambda storage, loc: storage.cuda(args.gpu)) + args.start_epoch = checkpoint['epoch'] + best_prec1 = checkpoint['best_prec1'] + model.load_state_dict(checkpoint['state_dict']) + optimizer.load_state_dict(checkpoint['optimizer']) + print("=> loaded checkpoint '{}' (epoch {})" + .format(args.resume, checkpoint['epoch'])) + else: + print("=> no checkpoint found at '{}'".format(args.resume)) + resume() + + # Data loading code + traindir = os.path.join(args.data, 'train') + valdir = os.path.join(args.data, 'val') + + if(args.arch == "inception_v3"): + crop_size = 299 + val_size = 320 # I chose this value arbitrarily, we can adjust. + else: + crop_size = 224 + val_size = 256 + + train_dataset = datasets.ImageFolder( + traindir, + transforms.Compose([ + transforms.RandomResizedCrop(crop_size), + transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), Too slow + # normalize, + ])) + val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ + transforms.Resize(val_size), + transforms.CenterCrop(crop_size), + ])) + + train_sampler = None + val_sampler = None + if args.distributed: + train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset) + val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset) + + train_loader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), + num_workers=args.workers, pin_memory=True, sampler=train_sampler, collate_fn=fast_collate) + + val_loader = torch.utils.data.DataLoader( + val_dataset, + batch_size=args.batch_size, shuffle=False, + num_workers=args.workers, pin_memory=True, + sampler=val_sampler, + collate_fn=fast_collate) + + if args.evaluate: + validate(val_loader, model, criterion) + return + + for epoch in range(args.start_epoch, args.epochs): + if args.distributed: + train_sampler.set_epoch(epoch) + + # train for one epoch + train(train_loader, model, criterion, optimizer, epoch) + if args.prof: + break + # evaluate on validation set + prec1 = validate(val_loader, model, criterion) + + # remember best prec@1 and save checkpoint + if args.local_rank == 0: + is_best = prec1 > best_prec1 + best_prec1 = max(prec1, best_prec1) + save_checkpoint({ + 'epoch': epoch + 1, + 'arch': args.arch, + 'state_dict': model.state_dict(), + 'best_prec1': best_prec1, + 'optimizer' : optimizer.state_dict(), + }, is_best) + +class data_prefetcher(): + def __init__(self, loader): + self.loader = iter(loader) + self.stream = torch.cuda.Stream() + self.mean = torch.tensor([0.485 * 255, 0.456 * 255, 0.406 * 255]).cuda().view(1,3,1,1) + self.std = torch.tensor([0.229 * 255, 0.224 * 255, 0.225 * 255]).cuda().view(1,3,1,1) + # With Amp, it isn't necessary to manually convert data to half. + # if args.fp16: + # self.mean = self.mean.half() + # self.std = self.std.half() + self.preload() + + def preload(self): + try: + self.next_input, self.next_target = next(self.loader) + except StopIteration: + self.next_input = None + self.next_target = None + return + with torch.cuda.stream(self.stream): + self.next_input = self.next_input.cuda(non_blocking=True) + self.next_target = self.next_target.cuda(non_blocking=True) + # With Amp, it isn't necessary to manually convert data to half. + # if args.fp16: + # self.next_input = self.next_input.half() + # else: + self.next_input = self.next_input.float() + self.next_input = self.next_input.sub_(self.mean).div_(self.std) + + def next(self): + torch.cuda.current_stream().wait_stream(self.stream) + input = self.next_input + target = self.next_target + self.preload() + return input, target + + +def train(train_loader, model, criterion, optimizer, epoch): + batch_time = AverageMeter() + data_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + # switch to train mode + model.train() + end = time.time() + + run_info_dict = {"Iteration" : [], + "Loss" : [], + "Speed" : []} + + prefetcher = data_prefetcher(train_loader) + input, target = prefetcher.next() + i = -1 + while input is not None: + i += 1 + + # No learning rate warmup for this test, to expose bitwise inaccuracies more quickly + # adjust_learning_rate(optimizer, epoch, i, len(train_loader)) + + if args.prof: + if i > 10: + break + # measure data loading time + data_time.update(time.time() - end) + + # compute output + output = model(input) + loss = criterion(output, target) + + # measure accuracy and record loss + prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) + + if args.distributed: + reduced_loss = reduce_tensor(loss.data) + prec1 = reduce_tensor(prec1) + prec5 = reduce_tensor(prec5) + else: + reduced_loss = loss.data + + losses.update(to_python_float(reduced_loss), input.size(0)) + top1.update(to_python_float(prec1), input.size(0)) + top5.update(to_python_float(prec5), input.size(0)) + + # compute gradient and do SGD step + optimizer.zero_grad() + + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + + # for param in model.parameters(): + # print(param.data.double().sum().item(), param.grad.data.double().sum().item()) + + # torch.cuda.synchronize() + torch.cuda.nvtx.range_push("step") + optimizer.step() + torch.cuda.nvtx.range_pop() + + torch.cuda.synchronize() + # measure elapsed time + batch_time.update(time.time() - end) + + end = time.time() + + # If you decide to refactor this test, like examples/imagenet, to sample the loss every + # print_freq iterations, make sure to move this prefetching below the accuracy calculation. + input, target = prefetcher.next() + + if i % args.print_freq == 0 and i > 1: + if args.local_rank == 0: + print('Epoch: [{0}][{1}/{2}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + 'Speed {3:.3f} ({4:.3f})\t' + 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' + 'Loss {loss.val:.10f} ({loss.avg:.4f})\t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + epoch, i, len(train_loader), + args.world_size * args.batch_size / batch_time.val, + args.world_size * args.batch_size / batch_time.avg, + batch_time=batch_time, + data_time=data_time, loss=losses, top1=top1, top5=top5)) + run_info_dict["Iteration"].append(i) + run_info_dict["Loss"].append(losses.val) + run_info_dict["Speed"].append(args.world_size * args.batch_size / batch_time.val) + if len(run_info_dict["Loss"]) == args.prints_to_process: + if args.local_rank == 0: + torch.save(run_info_dict, + str(args.has_ext) + "_" + str(args.opt_level) + "_" + + str(args.loss_scale) + "_" + str(args.keep_batchnorm_fp32) + "_" + + str(args.fused_adam)) + quit() + + +def validate(val_loader, model, criterion): + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + # switch to evaluate mode + model.eval() + + end = time.time() + + prefetcher = data_prefetcher(val_loader) + input, target = prefetcher.next() + i = -1 + while input is not None: + i += 1 + + # compute output + with torch.no_grad(): + output = model(input) + loss = criterion(output, target) + + # measure accuracy and record loss + prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) + + if args.distributed: + reduced_loss = reduce_tensor(loss.data) + prec1 = reduce_tensor(prec1) + prec5 = reduce_tensor(prec5) + else: + reduced_loss = loss.data + + losses.update(to_python_float(reduced_loss), input.size(0)) + top1.update(to_python_float(prec1), input.size(0)) + top5.update(to_python_float(prec5), input.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if args.local_rank == 0 and i % args.print_freq == 0: + print('Test: [{0}/{1}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' + 'Speed {2:.3f} ({3:.3f})\t' + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + i, len(val_loader), + args.world_size * args.batch_size / batch_time.val, + args.world_size * args.batch_size / batch_time.avg, + batch_time=batch_time, loss=losses, + top1=top1, top5=top5)) + + input, target = prefetcher.next() + + print(' * Prec@1 {top1.avg:.3f} Prec@5 {top5.avg:.3f}' + .format(top1=top1, top5=top5)) + + return top1.avg + + +def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'): + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, 'model_best.pth.tar') + + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def adjust_learning_rate(optimizer, epoch, step, len_epoch): + """LR schedule that should yield 76% converged accuracy with batch size 256""" + factor = epoch // 30 + + if epoch >= 80: + factor = factor + 1 + + lr = args.lr*(0.1**factor) + + """Warmup""" + if epoch < 5: + lr = lr*float(1 + step + epoch*len_epoch)/(5.*len_epoch) + + # if(args.local_rank == 0): + # print("epoch = {}, step = {}, lr = {}".format(epoch, step, lr)) + + for param_group in optimizer.param_groups: + param_group['lr'] = lr + + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def reduce_tensor(tensor): + rt = tensor.clone() + dist.all_reduce(rt, op=dist.reduce_op.SUM) + rt /= args.world_size + return rt + +if __name__ == '__main__': + main() diff --git a/apex/tests/L1/common/run_test.sh b/apex/tests/L1/common/run_test.sh new file mode 100644 index 00000000..f4ae06c8 --- /dev/null +++ b/apex/tests/L1/common/run_test.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +print_banner() { + printf "\n\n\n\e[30m\e[42m$1\e[0m\n\n\n\n" +} + +print_banner "Distributed status: $1" + +echo $2 +DATADIR=$2 + +if [ -n "$3" ] +then + USE_BASELINE="" +else + USE_BASELINE="--use_baseline" +fi + +if [ "$1" == "single_gpu" ] +then + BASE_CMD="python main_amp.py -a resnet50 --b 128 --workers 4 --deterministic --prints-to-process 5" +fi + +if [ "$1" == "distributed" ] +then + BASE_CMD="python -m torch.distributed.launch --nproc_per_node=2 main_amp.py -a resnet50 --b 128 --workers 4 --deterministic --prints-to-process 5" +fi + +ADAM_ARGS="--opt-level O2 --keep-batchnorm-fp32 False --fused-adam" + +keep_batchnorms=( +"" +"--keep-batchnorm-fp32 True" +"--keep-batchnorm-fp32 False" +) + +loss_scales=( +"" +"--loss-scale 1.0" +"--loss-scale 128.0" +"--loss-scale dynamic" +) + +opt_levels=( +"O0" +"O1" +"O2" +"O3" +) + +rm True* +rm False* + +set -e + +print_banner "Installing Apex with --cuda_ext and --cpp_ext" + +pushd ../../.. +pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . +popd + +for opt_level in "${opt_levels[@]}" +do + for loss_scale in "${loss_scales[@]}" + do + for keep_batchnorm in "${keep_batchnorms[@]}" + do + if [ "$opt_level" == "O1" ] && [ -n "${keep_batchnorm}" ] + then + print_banner "Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}" + continue + fi + print_banner "${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --has-ext $DATADIR" + set -x + ${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --has-ext $DATADIR + set +x + done + done +done + +# Handle FusedAdam separately due to limited support. +# FusedAdam will not be tested for bitwise accuracy against the Python implementation. +# The L0 tests already do so. These tests are here to ensure that it actually runs, +# and get an idea of performance. +for loss_scale in "${loss_scales[@]}" +do + print_banner "${BASE_CMD} ${ADAM_ARGS} ${loss_scale} --has-ext $DATADIR" + set -x + ${BASE_CMD} ${ADAM_ARGS} ${loss_scale} --has-ext $DATADIR + set +x +done + +print_banner "Reinstalling apex without extensions" + +pushd ../../.. +pip install -v --no-cache-dir . +popd + +for opt_level in "${opt_levels[@]}" +do + for loss_scale in "${loss_scales[@]}" + do + for keep_batchnorm in "${keep_batchnorms[@]}" + do + if [ "$opt_level" == "O1" ] && [ -n "${keep_batchnorm}" ] + then + print_banner "Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}" + continue + fi + print_banner "${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} $DATADIR" + set -x + ${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} $DATADIR + set +x + done + done +done + +print_banner "Checking for bitwise accuracy between Python-only and cpp/cuda extension installs" + +for opt_level in "${opt_levels[@]}" +do + for loss_scale in "${loss_scales[@]}" + do + for keep_batchnorm in "${keep_batchnorms[@]}" + do + echo "" + if [ "$opt_level" == "O1" ] && [ -n "${keep_batchnorm}" ] + then + echo "Skipping ${opt_level} ${loss_scale} ${keep_batchnorm}" + continue + fi + echo "${BASE_CMD} --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} [--has-ext] $DATADIR" + set -x + python compare.py --opt-level ${opt_level} ${loss_scale} ${keep_batchnorm} --use_baseline + set +x + done + done +done + +print_banner "Reinstalling Apex with --cuda_ext and --cpp_ext" + +pushd ../../.. +pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" . +popd diff --git a/apex/tests/L1/cross_product/run.sh b/apex/tests/L1/cross_product/run.sh new file mode 100644 index 00000000..7ccf9ec4 --- /dev/null +++ b/apex/tests/L1/cross_product/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# DATADIR="/home/mcarilli/Desktop/pt18data/apex_stale/examples/imagenet/bare_metal_train_val/" +# DATADIR="/opt/home/apex/examples/imagenet/" +cp ../common/* . +bash run_test.sh single_gpu $1 diff --git a/apex/tests/L1/cross_product_distributed/run.sh b/apex/tests/L1/cross_product_distributed/run.sh new file mode 100644 index 00000000..917ec11e --- /dev/null +++ b/apex/tests/L1/cross_product_distributed/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cp ../common/* . +bash run_test.sh distributed $1 diff --git a/apex/tests/L1/transformer/pipeline_parallel_fwd_bwd_ucc_async.py b/apex/tests/L1/transformer/pipeline_parallel_fwd_bwd_ucc_async.py new file mode 100644 index 00000000..786c0ed5 --- /dev/null +++ b/apex/tests/L1/transformer/pipeline_parallel_fwd_bwd_ucc_async.py @@ -0,0 +1,219 @@ +import os +import logging +import itertools +from typing import Optional, Tuple, List +import unittest + +import torch +from torch.testing._internal import common_utils +from torch.testing._internal import common_cuda +from torch.testing._internal import common_distributed + +from apex._autocast_utils import _get_autocast_dtypes +from apex.transformer import parallel_state +from apex.transformer.pipeline_parallel import utils as pp_utils +from apex.transformer.pipeline_parallel.schedules.common import ( + FwdStepFunc, + build_model, + _get_params_for_weight_decay_optimization, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_no_pipelining import ( + forward_backward_no_pipelining, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_with_interleaving import ( + _forward_backward_pipelining_with_interleaving, +) +from apex.transformer.pipeline_parallel.schedules.fwd_bwd_pipelining_without_interleaving import ( + forward_backward_pipelining_without_interleaving, +) +from apex.transformer.testing.distributed_test_base import UccDistributedTestBase +from apex.transformer.testing import commons as testing_utils + + +logging.getLogger("torch").setLevel(logging.WARNING) +logging.getLogger("apex").setLevel(logging.WARNING) + + +def _get_default_world_sizes_model_parallel_world_size(pipeline_model_parallel_world_size: Optional[int] = None + ) -> Tuple[int, int, int]: + # TODO: revisit if we can fold this into the class for skip logic / avoid duplication + # of world size computation + world_size = torch.cuda.device_count() + tensor_model_parallel_world_size = 1 + data_parallel_size = 1 + (world_size >= 8 and world_size % 2 == 0) + + if pipeline_model_parallel_world_size is None: + pipeline_model_parallel_world_size = world_size // (tensor_model_parallel_world_size * data_parallel_size) + else: + data_parallel_size = world_size // (tensor_model_parallel_world_size * pipeline_model_parallel_world_size) + + return tensor_model_parallel_world_size, data_parallel_size, pipeline_model_parallel_world_size + + +class UccPipelineParallelForwardBackwardProf(UccDistributedTestBase): + + # The purpose of this class is to test and confirm asynchronous communication via profiling. + # Having that in mind, it is safe to skip all the numerical checks. + # For unit testing with numerical checks please refer to `tests/L0/run_transformer/test_pipeline_parallel_fwd_bwd.py`. + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.GLOBAL_BATCH_SIZE = 1024 + self.MICRO_BATCH_SIZE = 64 + self.HIDDEN_SIZE = 256 + self.NUM_FWD_BWD_ITERATIONS = 4 + self.deallocate_options = (False,) + self.dtypes = (torch.float32,) + + @property + def world_size(self) -> int: + return min(torch.cuda.device_count(), 8) + + def _forward_backward_test_impl( + self, + forward_only: bool, + fwd_bwd_func: FwdStepFunc, + pipeline_model_parallel_world_size: Optional[int], + virtual_pipeline_model_parallel_size: Optional[int], + async_comm: bool = False, + *, + default_backend: Optional[str] = None, + p2p_backend: Optional[str] = None, + ) -> None: + if fwd_bwd_func == _forward_backward_pipelining_with_interleaving: + self.assertIsNotNone(virtual_pipeline_model_parallel_size) + self.assertGreater(virtual_pipeline_model_parallel_size, 1) + dtype_options = self.dtypes or [torch.float32, torch.double] + _get_autocast_dtypes() + + for dtype, deallocate_pipeline_outputs in itertools.product( + dtype_options, self.deallocate_options, + ): + grad_scaler = ( + torch.cuda.amp.GradScaler(init_scale=4.0) + if dtype == torch.half + else None + ) + + (tensor_model_parallel_world_size, + data_parallel_size, + pipeline_model_parallel_world_size) = _get_default_world_sizes_model_parallel_world_size(pipeline_model_parallel_world_size) + + parallel_state.initialize_model_parallel( + tensor_model_parallel_size_=tensor_model_parallel_world_size, + pipeline_model_parallel_size_=pipeline_model_parallel_world_size, + virtual_pipeline_model_parallel_size_=virtual_pipeline_model_parallel_size, + default_backend=default_backend, + p2p_backend=p2p_backend, + ) + pp_utils._reconfigure_microbatch_calculator( + rank=parallel_state.get_tensor_model_parallel_rank(), + rampup_batch_size=None, + global_batch_size=self.GLOBAL_BATCH_SIZE, + micro_batch_size=self.MICRO_BATCH_SIZE, + data_parallel_size=parallel_state.get_data_parallel_world_size(), + ) + + global_batch_shape = ( + self.GLOBAL_BATCH_SIZE + // parallel_state.get_data_parallel_world_size(), + self.HIDDEN_SIZE, + self.HIDDEN_SIZE, + ) + + batch = None + if parallel_state.is_pipeline_first_stage(): + batch = (torch.ones(global_batch_shape, dtype=dtype).cuda(), ) + + model = build_model( + testing_utils.model_provider_func, + # Use DDP only when it's better to have + wrap_with_ddp=data_parallel_size > 1, + virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, + hidden_size=self.HIDDEN_SIZE, + ) + + + offset = pipeline_model_parallel_world_size if virtual_pipeline_model_parallel_size is not None else 0 + for idx, model_module in enumerate(model): + model_module = model_module.to(dtype) + + _param_groups = _get_params_for_weight_decay_optimization(model) + optimizer = torch.optim.Adam(_param_groups, lr=1e-3) + + pp_utils.update_num_microbatches(0) + + for _ in range(self.NUM_FWD_BWD_ITERATIONS): + loss = fwd_bwd_func( + testing_utils.fwd_step_func, + batch, + model, + forward_only=forward_only, + # `tensor_shape` is the shape of micro batch. + tensor_shape=( + self.MICRO_BATCH_SIZE, + self.HIDDEN_SIZE, + self.HIDDEN_SIZE, + ), + dtype=dtype, + async_comm=async_comm, + grad_scaler=grad_scaler, + deallocate_pipeline_output=deallocate_pipeline_outputs, + ) + + parallel_state.destroy_model_parallel() + + def test_learning_no_pipelining(self): + self._forward_backward_test_impl(False, forward_backward_no_pipelining, 1, None) + + def test_inference_no_pipelining(self): + self._forward_backward_test_impl(True, forward_backward_no_pipelining, 1, None) + + def test_learning_pipelining_without_interleaving(self): + self._forward_backward_test_impl( + False, forward_backward_pipelining_without_interleaving, None, None + ) + + def test_inference_pipelining_without_interleaving(self): + self._forward_backward_test_impl( + True, forward_backward_pipelining_without_interleaving, None, None + ) + + def test_learning_async_pipelining_without_interleaving(self): + self._forward_backward_test_impl( + False, forward_backward_pipelining_without_interleaving, None, None, async_comm=True + ) + + def test_inference_async_pipelining_without_interleaving(self): + self._forward_backward_test_impl( + True, forward_backward_pipelining_without_interleaving, None, None, async_comm=True + ) + + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_learning_pipelining_with_interleaving(self): + self._forward_backward_test_impl( + False, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2 + ) + + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_inference_pipelining_with_interleaving(self): + self._forward_backward_test_impl( + True, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2 + ) + + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_learning_async_pipelining_with_interleaving(self): + self._forward_backward_test_impl( + False, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2, async_comm=True + ) + + @unittest.skipUnless(_get_default_world_sizes_model_parallel_world_size()[-1] > 2, "Interleaved schedule requires pipeline_model_parallel_world_size > 2") + def test_inference_async_pipelining_with_interleaving(self): + self._forward_backward_test_impl( + True, _forward_backward_pipelining_with_interleaving, None, virtual_pipeline_model_parallel_size=2, async_comm=True + ) + + +if __name__ == "__main__": + os.environ["UCC_TLS"] = "ucp,cuda" + common_distributed.TIMEOUT_DEFAULT = 500 + common_utils.run_tests() diff --git a/apex/tests/distributed/DDP/ddp_race_condition_test.py b/apex/tests/distributed/DDP/ddp_race_condition_test.py new file mode 100644 index 00000000..761a3359 --- /dev/null +++ b/apex/tests/distributed/DDP/ddp_race_condition_test.py @@ -0,0 +1,69 @@ +import torch +import torch.distributed as dist +from torch.nn import Parameter +from torch.nn import Module +from apex.parallel import DistributedDataParallel as DDP +import argparse +import os + + +parser = argparse.ArgumentParser(description='allreduce hook example') +parser.add_argument("--local_rank", default=0, type=int) +args = parser.parse_args() + +args.distributed = False +if 'WORLD_SIZE' in os.environ: + args.distributed = int(os.environ['WORLD_SIZE']) > 1 + +if args.distributed: + args.gpu = args.local_rank % torch.cuda.device_count() + torch.cuda.set_device(args.gpu) + torch.distributed.init_process_group(backend='nccl', + init_method='env://') + args.world_size = torch.distributed.get_world_size() + +torch.set_printoptions(precision=10) +torch.manual_seed(args.local_rank) + +class Model(Module): + def __init__(self): + super(Model, self).__init__() + self.a = Parameter(torch.cuda.FloatTensor(4096*4096).fill_(1.0)) + self.b = Parameter(torch.cuda.FloatTensor(4096*4096).fill_(2.0)) + def forward(self, input): + return (input*self.a)*self.b + +model = Model() +# model = DDP(model, message_size=1, gradient_predivide_factor=8.0) +# model = DDP(model, delay_allreduce=True) +# model = DDP(model, message_size=1, allreduce_trigger_params=[model.b]) +model = DDP(model, message_size=1, allreduce_trigger_params=[model.b], num_allreduce_streams=3) + +x = torch.cuda.FloatTensor(4096*4096) + +passed = True +torch.cuda.cudart().cudaProfilerStart() +for i in range(10): + x.fill_(i + args.local_rank) # fill x with new values every iteration for sanity + model.zero_grad() + out = model(x) + loss = out.sum() + # torch.cuda.nvtx.range_push("backward") + loss.backward() + # torch.cuda.nvtx.range_pop() + + # torch.cuda.nvtx.range_push("synchronize() + info") + # torch.cuda.synchronize() + print("i = {}".format(i)) + def info(name, param, val): + expected = val*4096*4096*(2.*i+1)/2. + actual = param.grad.data.sum().item() + print(name+": grad.data_ptr() = {}, expected sum {}, got {}".format( + param.grad.data_ptr(), expected, actual)) + return (expected == actual) + if not info("model.a", model.module.a, 2.): passed = False + if not info("model.b", model.module.b, 1.): passed = False + # torch.cuda.nvtx.range_pop() +torch.cuda.cudart().cudaProfilerStop() + +print("passed = ", passed) diff --git a/apex/tests/distributed/DDP/run_race_test.sh b/apex/tests/distributed/DDP/run_race_test.sh new file mode 100644 index 00000000..2c2bd266 --- /dev/null +++ b/apex/tests/distributed/DDP/run_race_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch --nproc_per_node=2 ddp_race_condition_test.py diff --git a/apex/tests/distributed/amp_master_params/amp_master_params.py b/apex/tests/distributed/amp_master_params/amp_master_params.py new file mode 100644 index 00000000..4af5092f --- /dev/null +++ b/apex/tests/distributed/amp_master_params/amp_master_params.py @@ -0,0 +1,70 @@ +import torch +import argparse +import os +from apex import amp +# FOR DISTRIBUTED: (can also use torch.nn.parallel.DistributedDataParallel instead) +from apex.parallel import DistributedDataParallel + +parser = argparse.ArgumentParser() +# FOR DISTRIBUTED: Parse for the local_rank argument, which will be supplied +# automatically by torch.distributed.launch. +parser.add_argument("--local_rank", default=0, type=int) +args = parser.parse_args() + +# FOR DISTRIBUTED: If we are running under torch.distributed.launch, +# the 'WORLD_SIZE' environment variable will also be set automatically. +args.distributed = False +if 'WORLD_SIZE' in os.environ: + args.distributed = int(os.environ['WORLD_SIZE']) > 1 + +if args.distributed: + # FOR DISTRIBUTED: Set the device according to local_rank. + torch.cuda.set_device(args.local_rank) + + # FOR DISTRIBUTED: Initialize the backend. torch.distributed.launch will provide + # environment variables, and requires that you use init_method=`env://`. + torch.distributed.init_process_group(backend='nccl', + init_method='env://') + + torch.manual_seed(torch.distributed.get_rank()) + +torch.backends.cudnn.benchmark = True + +N, D_in, D_out = 64, 1024, 16 + +# Each process receives its own batch of "fake input data" and "fake target data." +# The "training loop" in each process just uses this fake batch over and over. +# https://github.com/NVIDIA/apex/tree/master/examples/imagenet provides a more realistic +# example of distributed data sampling for both training and validation. +x = torch.randn(N, D_in, device='cuda') +y = torch.randn(N, D_out, device='cuda') + +model = torch.nn.Linear(D_in, D_out).cuda() +optimizer = torch.optim.SGD(model.parameters(), lr=1e-3) + +model, optimizer = amp.initialize(model, optimizer, opt_level="O2") + +if args.distributed: + # FOR DISTRIBUTED: After amp.initialize, wrap the model with + # apex.parallel.DistributedDataParallel. + model = DistributedDataParallel(model) + # torch.nn.parallel.DistributedDataParallel is also fine, with some added args: + # model = torch.nn.parallel.DistributedDataParallel(model, + # device_ids=[args.local_rank], + # output_device=args.local_rank) + +loss_fn = torch.nn.MSELoss() + +for t in range(500): + optimizer.zero_grad() + y_pred = model(x) + loss = loss_fn(y_pred, y) + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + optimizer.step() + +if args.local_rank == 0: + print("final loss = ", loss) + +torch.save(list(model.parameters()), "rank{}model.pth".format(torch.distributed.get_rank())) +torch.save(list(amp.master_params(optimizer)), "rank{}master.pth".format(torch.distributed.get_rank())) diff --git a/apex/tests/distributed/amp_master_params/compare.py b/apex/tests/distributed/amp_master_params/compare.py new file mode 100644 index 00000000..e5cbf20c --- /dev/null +++ b/apex/tests/distributed/amp_master_params/compare.py @@ -0,0 +1,28 @@ +import torch + +model_params_rank0 = torch.load("rank0model.pth", + map_location = lambda storage, loc: storage.cuda(0)) +model_params_rank1 = torch.load("rank1model.pth", + map_location = lambda storage, loc: storage.cuda(0)) +master_params_rank0 = torch.load("rank0master.pth", + map_location = lambda storage, loc: storage.cuda(0)) +master_params_rank1 = torch.load("rank1master.pth", + map_location = lambda storage, loc: storage.cuda(0)) + +for model_rank0, model_rank1, master_rank0, master_rank1 in zip( + model_params_rank0, + model_params_rank1, + master_params_rank0, + master_params_rank1): + assert torch.allclose(model_rank0, model_rank1), "Model param mismatch" + assert torch.allclose(master_rank0, master_rank1), "Master param mismatch" + # Some debugging/investigation assistance code: + # maxval, maxind = torch.max(((torch.abs(model_rank0).float())/torch.abs(master_rank0)).view(-1), 0) + # offending_val_half = model_rank0.view(-1)[maxind.item()] + # offending_val_float = master_rank0.view(-1)[maxind.item()] + # print(maxval.item(), maxind.item(), offending_val_half.item(), offending_val_float.item(), + # offending_val_float.half().item()) + # rtol needs to be > 2^-11 because of denormals... + assert torch.allclose(model_rank0, master_rank0.half(), rtol=.005), "Model-master mismatch" + +print("OK: Model and master params match across ranks.") diff --git a/apex/tests/distributed/amp_master_params/run.sh b/apex/tests/distributed/amp_master_params/run.sh new file mode 100644 index 00000000..8599dbbb --- /dev/null +++ b/apex/tests/distributed/amp_master_params/run.sh @@ -0,0 +1,4 @@ +#!/bin/bash +python -m torch.distributed.launch --nproc_per_node=2 amp_master_params.py + +python compare.py diff --git a/apex/tests/distributed/synced_batchnorm/python_single_gpu_unit_test.py b/apex/tests/distributed/synced_batchnorm/python_single_gpu_unit_test.py new file mode 100644 index 00000000..80c26825 --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/python_single_gpu_unit_test.py @@ -0,0 +1,111 @@ +import torch +import numpy as np +import apex + +def compare(desc, inp1, inp2, error): + a = inp1.clone().detach().cpu().numpy() + b = inp2.clone().detach().cpu().numpy() + close = np.allclose(a,b, error, error) + if not close: + print(desc, close) + z = a - b + index = (np.abs(z) >= error + error * np.abs(b)).nonzero() + print("dif : ", z[index]) + print("inp1 : ", a[index]) + print("inp2 : ", b[index]) + return close + +feature_size = 10 +space_size = 16 +batch_size = 5 + + +error = 1e-5 + +np.random.seed(1) +dtype = np.float32 +inp = (np.random.randn(batch_size, feature_size, space_size, space_size)).astype(dtype) +grad = (np.random.randn(batch_size, feature_size, space_size, space_size)).astype(dtype) +weight = (np.random.randn(feature_size)).astype(dtype) +bias = (np.random.randn(feature_size)).astype(dtype) + +type_tensor = torch.cuda.FloatTensor +ref_tensor = torch.cuda.DoubleTensor + +inp_t = type_tensor(inp) +weight_t = type_tensor(weight) +bias_t = type_tensor(bias) + +inp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +inp2_r = ref_tensor(inp) +weight_r = ref_tensor(weight).view(-1, 1, 1) +bias_r = ref_tensor(bias).view(-1, 1, 1) + +grad_output_t = type_tensor(grad) + +m = inp_r.mean(1) +b_v = inp_r.var(1, unbiased=False) +unb_v = inp_r.var(1, unbiased=True) + +eps = 1e-5 + +bn = torch.nn.BatchNorm2d(feature_size).cuda() +bn.momentum = 1.0 +bn.weight.data = weight_t.clone() +bn.bias.data = bias_t.clone() +inp_bn = inp_t.clone().requires_grad_() +grad_bn = grad_output_t.clone().detach() +out_bn = bn(inp_bn) +out_bn.backward(grad_bn) + +from apex.parallel.sync_batchnorm import SyncBatchNorm + +sbn = SyncBatchNorm(feature_size).cuda() +sbn.momentum = 1.0 +sbn.weight.data = weight_t.clone() +sbn.bias.data = bias_t.clone() +inp_sbn = inp_t.clone().requires_grad_() +grad_sbn = grad_output_t.clone().detach() +out_sbn = sbn(inp_sbn) +out_sbn.backward(grad_sbn) + +sbn_result = True +sbn_result_c_last = True +bn_result = True + +out_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r + +compare("comparing bn output: ", out_bn, out_r, error) + +grad_output_t = type_tensor(grad) + +grad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +grad_output2_r = ref_tensor(grad) + +grad_bias_r = grad_output_r.sum(1) +grad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1) + +mean_dy_r = grad_output_r.mean(1) +mean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1) + +grad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1) + +compare("comparing bn input grad: ", inp_bn.grad, grad_input_r, error) +sbn_result = compare("comparing sbn input grad: ", inp_sbn.grad, grad_input_r, error) and sbn_result + +compare("comparing bn/sbn output: ", out_bn, out_sbn, error) +sbn_result = compare("comparing running_mean: ", bn.running_mean.data, sbn.running_mean.data, error) and sbn_result +sbn_result = compare("comparing running_variance: ", bn.running_var.data, sbn.running_var.data, error) and sbn_result +compare("comparing grad_input: ", inp_bn.grad, inp_sbn.grad, error) +compare("comparing grad_bias: ", bn.bias.grad, sbn.bias.grad, error) +compare("comparing grad_bias bn to ref: ", bn.bias.grad, grad_bias_r, error) +sbn_result = compare("comparing grad_bias sbn to ref: ", sbn.bias.grad, grad_bias_r, error) and sbn_result +compare("comparing grad_weight: ", bn.weight.grad, sbn.weight.grad, error) +compare("comparing grad_weight bn to ref: ", bn.weight.grad, grad_weight_r, error) +sbn_result = compare("comparing grad_weight sbn to ref: ", sbn.weight.grad, grad_weight_r, error) and sbn_result + +if sbn_result: + print("====SBN single gpu passed tests") +else: + print("*SBN single gpu failed*") + diff --git a/apex/tests/distributed/synced_batchnorm/single_gpu_unit_test.py b/apex/tests/distributed/synced_batchnorm/single_gpu_unit_test.py new file mode 100644 index 00000000..6fcbd0d1 --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/single_gpu_unit_test.py @@ -0,0 +1,159 @@ +import torch +import numpy as np +import apex +if True: + print("using setup tools") + import syncbn +else: + print("using jit") + from torch.utils.cpp_extension import load + syncbn = load(name='syncbn', sources=['../../csrc/syncbn.cpp', '../../csrc/welford.cu']) + +def compare(desc, inp1, inp2, error): + a = inp1.clone().detach().cpu().numpy() + b = inp2.clone().detach().cpu().numpy() + close = np.allclose(a,b, error, error) + if not close: + print(desc, close) + z = a - b + index = (np.abs(z) >= error + error * np.abs(b)).nonzero() + print("dif : ", z[index]) + print("inp1 : ", a[index]) + print("inp2 : ", b[index]) + return close + +feature_size = 10 +space_size = 16 +batch_size = 5 + + +error = 1e-5 + +np.random.seed(1) +dtype = np.float32 +inp = (np.random.randn(batch_size, feature_size, space_size, space_size)).astype(dtype) +grad = (np.random.randn(batch_size, feature_size, space_size, space_size)).astype(dtype) +weight = (np.random.randn(feature_size)).astype(dtype) +bias = (np.random.randn(feature_size)).astype(dtype) +count = torch.cuda.IntTensor([batch_size*space_size**2]) + +type_tensor = torch.cuda.FloatTensor +ref_tensor = torch.cuda.DoubleTensor + +inp_t = type_tensor(inp) +weight_t = type_tensor(weight) +bias_t = type_tensor(bias) + +inp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +inp2_r = ref_tensor(inp) +weight_r = ref_tensor(weight).view(-1, 1, 1) +bias_r = ref_tensor(bias).view(-1, 1, 1) + +grad_output_t = type_tensor(grad) + +m = inp_r.mean(1) +b_v = inp_r.var(1, unbiased=False) +unb_v = inp_r.var(1, unbiased=True) + +eps = 1e-5 + +#mean, var, var_biased = syncbn.welford_mean_var(inp_t) +mean, var_biased = syncbn.welford_mean_var(inp_t) +inv_std = 1.0 / torch.sqrt(var_biased + eps) + +bn = torch.nn.BatchNorm2d(feature_size).cuda() +bn.momentum = 1.0 +bn.weight.data = weight_t.clone() +bn.bias.data = bias_t.clone() +inp_bn = inp_t.clone().requires_grad_() +grad_bn = grad_output_t.clone().detach() +out_bn = bn(inp_bn) +out_bn.backward(grad_bn) + +sbn = apex.parallel.SyncBatchNorm(feature_size).cuda() +sbn.momentum = 1.0 +sbn.weight.data = weight_t.clone() +sbn.bias.data = bias_t.clone() +inp_sbn = inp_t.clone().requires_grad_() +grad_sbn = grad_output_t.clone().detach() +out_sbn = sbn(inp_sbn) +out_sbn.backward(grad_sbn) + +sbn_c_last = apex.parallel.SyncBatchNorm(feature_size, channel_last=True).cuda() +sbn_c_last.momentum = 1.0 +sbn_c_last.weight.data = weight_t.clone() +sbn_c_last.bias.data = bias_t.clone() +inp_sbn_c_last = inp_t.clone().transpose(-1, 1).contiguous().requires_grad_() +grad_sbn_c_last = grad_output_t.clone().transpose(-1, 1).contiguous().detach() +out_sbn_c_last = sbn_c_last(inp_sbn_c_last) +out_sbn_c_last.backward(grad_sbn_c_last) + +sbn_result = True +sbn_result_c_last = True +bn_result = True + +sbn_result = compare("comparing mean: ", mean, m, error) and sbn_result +#sbn_result = compare("comparing variance: ", var, unb_v, error) and sbn_result +sbn_result = compare("comparing biased variance: ", var_biased, b_v, error) and sbn_result + + +out = syncbn.batchnorm_forward(inp_t, mean, inv_std, weight_t, bias_t) +out_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r + +sbn_result = compare("comparing output: ", out, out_r, error) and sbn_result +compare("comparing bn output: ", out_bn, out_r, error) + +grad_output_t = type_tensor(grad) + +grad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +grad_output2_r = ref_tensor(grad) + +grad_bias_r = grad_output_r.sum(1) +grad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1) + +sum_dy_r = grad_output_r.sum(1) +mean_dy_r = grad_output_r.mean(1) +sum_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1) +mean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1) + +grad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1) + +sum_dy, sum_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output_t, inp_t, mean, inv_std, weight_t) +grad_input = syncbn.batchnorm_backward(grad_output_t, inp_t, mean, inv_std, weight_t, sum_dy, sum_dy_xmu, count) +sbn_result = compare("comparing bias grad: ", grad_bias, grad_bias_r, error) and sbn_result +sbn_result = compare("comparing weight grad: ", grad_weight, grad_weight_r, error) and sbn_result +sbn_result = compare("comparing sum_dy grad: ", sum_dy, sum_dy_r, error) and sbn_result +sbn_result = compare("comparing sum_dy_xmu grad: ", sum_dy_xmu, sum_dy_xmu_r, error) and sbn_result +sbn_result = compare("comparing input grad: ", grad_input, grad_input_r, error) and sbn_result +compare("comparing bn input grad: ", inp_bn.grad, grad_input_r, error) +sbn_result = compare("comparing sbn input grad: ", inp_sbn.grad, grad_input_r, error) and sbn_result + +compare("comparing bn/sbn output: ", out_bn, out_sbn, error) +sbn_result = compare("comparing running_mean: ", bn.running_mean.data, sbn.running_mean.data, error) and sbn_result +sbn_result = compare("comparing running_variance: ", bn.running_var.data, sbn.running_var.data, error) and sbn_result +compare("comparing grad_input: ", inp_bn.grad, inp_sbn.grad, error) +compare("comparing grad_bias: ", bn.bias.grad, sbn.bias.grad, error) +compare("comparing grad_bias bn to ref: ", bn.bias.grad, grad_bias_r, error) +sbn_result = compare("comparing grad_bias sbn to ref: ", sbn.bias.grad, grad_bias_r, error) and sbn_result +compare("comparing grad_weight: ", bn.weight.grad, sbn.weight.grad, error) +compare("comparing grad_weight bn to ref: ", bn.weight.grad, grad_weight_r, error) +sbn_result = compare("comparing grad_weight sbn to ref: ", sbn.weight.grad, grad_weight_r, error) and sbn_result + +compare("comparing channel last bn/sbn output: ", out_bn, out_sbn_c_last.transpose(-1, 1).contiguous(), error) +sbn_result_c_last = compare("comparing channel last running_mean: ", bn.running_mean.data, sbn_c_last.running_mean.data, error) and sbn_result_c_last +sbn_result_c_last = compare("comparing channel last running_variance: ", bn.running_var.data, sbn_c_last.running_var.data, error) and sbn_result_c_last +compare("comparing channel last grad_input: ", inp_bn.grad, inp_sbn_c_last.grad.transpose(-1, 1).contiguous(), error) +compare("comparing channel last grad_bias: ", bn.bias.grad, sbn_c_last.bias.grad, error) +sbn_result_c_last = compare("comparing channel last grad_bias sbn to ref: ", sbn_c_last.bias.grad, grad_bias_r, error) and sbn_result_c_last +compare("comparing channel last grad_weight: ", bn.weight.grad, sbn_c_last.weight.grad, error) +sbn_result_c_last = compare("comparing channel last grad_weight sbn to ref: ", sbn_c_last.weight.grad, grad_weight_r, error) and sbn_result_c_last + +if sbn_result: + print("====SBN single gpu passed tests") +else: + print("*SBN single gpu failed*") + +if sbn_result_c_last: + print("====SBN channel last single gpu passed tests") +else: + print("*SBN channel last single gpu failed*") diff --git a/apex/tests/distributed/synced_batchnorm/test_batchnorm1d.py b/apex/tests/distributed/synced_batchnorm/test_batchnorm1d.py new file mode 100644 index 00000000..f35ac473 --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/test_batchnorm1d.py @@ -0,0 +1,18 @@ +import torch +import apex + +model = apex.parallel.SyncBatchNorm(4).cuda() +model.weight.data.uniform_() +model.bias.data.uniform_() +data = torch.rand((8,4)).cuda() + +model_ref = torch.nn.BatchNorm1d(4).cuda() +model_ref.load_state_dict(model.state_dict()) +data_ref = data.clone() + +output = model(data) +output_ref = model_ref(data_ref) + +assert(output.allclose(output_ref)) +assert(model.running_mean.allclose(model_ref.running_mean)) +assert(model.running_var.allclose(model_ref.running_var)) diff --git a/apex/tests/distributed/synced_batchnorm/test_groups.py b/apex/tests/distributed/synced_batchnorm/test_groups.py new file mode 100644 index 00000000..d028cc39 --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/test_groups.py @@ -0,0 +1,185 @@ +import torch +import numpy as np +import apex +import syncbn +import os +import argparse +import torch.optim as optim + +def compare(desc, inp1, inp2, error): + a = inp1.clone().detach().cpu().numpy() + b = inp2.clone().detach().cpu().numpy() + close = np.allclose(a,b, error, error) + if not close: + print(desc, close) + z = a - b + index = (np.abs(z) >= error + error * np.abs(b)).nonzero() + print("dif : ", z[index]) + print("inp1 : ", a[index]) + print("inp2 : ", b[index]) + return close + +feature_size = 10 +space_size = 40 +batch_size = 32 + + +from apex.parallel import DistributedDataParallel as DDP +parser = argparse.ArgumentParser() +parser.add_argument("--local_rank", default=0, type=int) +parser.add_argument("--fp16", action='store_true', default=False) +parser.add_argument("--fp64", action='store_true', default=False) +parser.add_argument("--group_size", default=0, type=int) +args = parser.parse_args() + +try: + args.world_size = int(os.environ['WORLD_SIZE']) +except: + print("This is a multi-gpu test. To run it please use 'python -m torch.distributed.launch --nproc_per_node= test_groups.py '") + exit(1) + +torch.cuda.set_device(args.local_rank) +torch.distributed.init_process_group(backend='nccl', init_method='env://') + +start = (args.local_rank%args.group_size) * batch_size//args.group_size +finish = (args.local_rank%args.group_size + 1) * batch_size//args.group_size + +error = 1e-5 +dtype = np.float32 +if args.fp16: + error = 1e-3 + dtype = np.float16 +elif args.fp64: + error = 1e-8 + dtype = np.float64 + + +np.random.seed(18 + args.local_rank//args.group_size) + +inp = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype) +grad = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype) +weight = np.random.randn(feature_size).astype(dtype) +bias = np.random.randn(feature_size).astype(dtype) + + +type_tensor = torch.cuda.FloatTensor +if args.fp16: + type_tensor = torch.cuda.HalfTensor +if args.fp64: + type_tensor = torch.cuda.DoubleTensor + +ref_tensor = torch.cuda.DoubleTensor + +inp_t = type_tensor(inp) +weight_t = type_tensor(weight) +bias_t = type_tensor(bias) + +inp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +inp2_r = ref_tensor(inp) +weight_r = ref_tensor(weight).view(-1, 1, 1) +bias_r = ref_tensor(bias).view(-1, 1, 1) + +grad_output_t = type_tensor(grad) + +m = inp_r.mean(1) +b_v = inp_r.var(1, unbiased=False) +unb_v = inp_r.var(1, unbiased=True) + +eps = 1e-5 + +mean, var_biased = syncbn.welford_mean_var(inp_t) +inv_std = 1.0 / torch.sqrt(var_biased + eps) + +bn = torch.nn.BatchNorm2d(feature_size).cuda() +bn.momentum = 1.0 +bn.weight.data = weight_t.clone() +bn.bias.data = bias_t.clone() +if args.fp16: + bn.half() +if args.fp64: + bn.double() +bn = DDP(bn) +inp_bn = inp_t.clone().requires_grad_() +grad_bn = grad_output_t.clone().detach() +out_bn = bn(inp_bn) +out_bn.backward(grad_bn) +# compensating the averaging over processes done by DDP +# in order to produce mathematically equivalent result +# https://github.com/NVIDIA/apex/issues/134#issuecomment-458307368 +for param in bn.parameters(): + param.grad = param.grad / args.group_size +bn_opt = optim.SGD(bn.parameters(), lr=1.0) + +sbn = apex.parallel.SyncBatchNorm(feature_size, process_group=apex.parallel.create_syncbn_process_group(args.group_size)).cuda() +sbn.momentum = 1.0 +sbn.weight.data = weight_t.clone() +sbn.bias.data = bias_t.clone() +if args.fp16: + sbn.half() +if args.fp64: + sbn.double() +sbn = DDP(sbn) +sbn_opt = optim.SGD(sbn.parameters(), lr=1.0) +inp_sbn = inp_t.clone().requires_grad_() +grad_sbn = grad_output_t.clone().detach() +out_sbn = sbn(inp_sbn[start:finish]) +out_sbn.backward(grad_sbn[start:finish]) + +sbn_result = True +bn_result = True + +if args.local_rank == 0: + sbn_result = compare("comparing mean: ", mean, m, error) and sbn_result + sbn_result = compare("comparing biased variance: ", var_biased, b_v, error) and sbn_result + +out = syncbn.batchnorm_forward(inp_t, mean, inv_std, weight_t, bias_t) +out_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r + +if args.local_rank == 0: + sbn_result = compare("comparing output: ", out, out_r, error) and sbn_result + compare("comparing bn output: ", out_bn, out_r, error) + +grad_output_t = type_tensor(grad) + +grad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +grad_output2_r = ref_tensor(grad) + +grad_bias_r = grad_output_r.sum(1) +grad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1) + +mean_dy_r = grad_output_r.mean(1) +mean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1) + +grad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1) + +mean_dy, mean_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output_t, inp_t, mean, inv_std, weight_t) +grad_input = syncbn.batchnorm_backward(grad_output_t, inp_t, mean, inv_std, weight_t, mean_dy, mean_dy_xmu) + +if args.local_rank == 0: + sbn_result = compare("comparing bias grad: ", grad_bias, grad_bias_r, error) and sbn_result + sbn_result = compare("comparing weight grad: ", grad_weight, grad_weight_r, error) and sbn_result + sbn_result = compare("comparing mean_dy grad: ", mean_dy, mean_dy_r, error) and sbn_result + sbn_result = compare("comparing mean_dy_xmu grad: ", mean_dy_xmu, mean_dy_xmu_r, error) and sbn_result + sbn_result = compare("comparing input grad: ", grad_input, grad_input_r, error) and sbn_result + compare("comparing bn input grad: ", inp_bn.grad, grad_input_r, error) + +if args.local_rank == 0: + sbn_result = compare("comparing running_mean: ", bn.module.running_mean.data, sbn.module.running_mean.data, error) and sbn_result + sbn_result = compare("comparing running_variance: ", bn.module.running_var.data, sbn.module.running_var.data, error) and sbn_result + +# execute by both +compare("comparing layers output: ", out_bn[start:finish], out_sbn, error) and sbn_result +compare("comparing layers grad_input: ", inp_bn.grad[start:finish], inp_sbn.grad[start:finish], error) and sbn_result + +bn_opt.step() +sbn_opt.step() + +if args.local_rank == 0: + compare("comparing bn vs sbn bias: ", bn.module.bias, sbn.module.bias, error) + compare("comparing bn vs sbn weight: ", bn.module.weight, sbn.module.weight, error) + + +if sbn_result: + print("====SBN group test passed") +else: + print("*SBN group test failed*") diff --git a/apex/tests/distributed/synced_batchnorm/two_gpu_test_different_batch_size.py b/apex/tests/distributed/synced_batchnorm/two_gpu_test_different_batch_size.py new file mode 100755 index 00000000..a9e8cb64 --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/two_gpu_test_different_batch_size.py @@ -0,0 +1,158 @@ +import torch +import torch.nn as nn +from torch.nn.parallel import DistributedDataParallel as DDP +from apex.parallel import SyncBatchNorm as ApexSyncBatchNorm + +import argparse +import os +import numpy as np + +var_batch = 16 + +def compare(desc, inp1, inp2, error= 1e-5): + a = inp1.clone().detach().cpu().numpy() + b = inp2.clone().detach().cpu().numpy() + close = np.allclose(a,b, error, error) + if not close: + print(desc, close) + z = a - b + index = (np.abs(z) >= error + error * np.abs(b)).nonzero() + print("dif : ", z[index]) + print("inp1 : ", a[index]) + print("inp2 : ", b[index]) + return close + +parser = argparse.ArgumentParser() +parser.add_argument('--local_rank', type=int, default=0) +parser.add_argument('--apex', action='store_true') +args = parser.parse_args() + + +torch.manual_seed(2809) +# Setup DDP +torch.cuda.set_device(args.local_rank) +device = torch.device('cuda:{}'.format(args.local_rank)) + +torch.distributed.init_process_group( + 'nccl', + init_method='env://', + rank=args.local_rank, +) + +# Setup model +if args.apex: + model = nn.Sequential( + nn.Conv2d(3, 6, 3, 1, 1), + ApexSyncBatchNorm(6) + ) +else: + model = nn.Sequential( + nn.Conv2d(3, 6, 3, 1, 1), + nn.SyncBatchNorm(6) + ) + +# Setup reference model +model_reference = nn.Sequential( + nn.Conv2d(3, 6, 3, 1, 1), + nn.BatchNorm2d(6) +) + +with torch.no_grad(): + model_reference[0].weight.copy_(model[0].weight) + model_reference[0].bias.copy_(model[0].bias) +model_reference.to(device) + +model = model.to(device) +model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank) + +global_batch_size = var_batch + 8 +# Create random data +if args.local_rank == 0: + data = torch.randn(var_batch, 3, 8, 8, device=device, dtype=torch.float) * 50.0 + grad = torch.randint(0, 10, (var_batch, 6, 8, 8), device=device, dtype=torch.float) / 10.0 +else: + data = torch.randn(8, 3, 8, 8, device=device) + grad = torch.randint(0, 10, (8, 6, 8, 8), device=device, dtype=torch.float) / 10.0 + +data.requires_grad_() +data.retain_grad = True + +weighted_gradient = True + +# DDP forward/backward +output = model(data) + +if weighted_gradient: + output.backward(grad * 2 / global_batch_size) +else: + output.backward(grad / output.size(0)) + +d_list = [torch.randn(8, 3, 8, 8, device=device) for i in range(int(os.environ['WORLD_SIZE']))] +y_list = [torch.randn(8, 6, 8, 8, device=device) for i in range(int(os.environ['WORLD_SIZE']))] +dgrad_list = [torch.randn(8, 3, 8, 8, device=device) for i in range(int(os.environ['WORLD_SIZE']))] +grad_list = [torch.randn(8, 6, 8, 8, device=device) for i in range(int(os.environ['WORLD_SIZE']))] +if args.local_rank == 0: + # placeholder, these random data will later be discarded. + torch.distributed.all_gather(d_list, torch.randn(8, 3, 8, 8, device=device)) + torch.distributed.all_gather(y_list, torch.randn(8, 6, 8, 8, device=device)) + torch.distributed.all_gather(dgrad_list, torch.randn(8, 3, 8, 8, device=device)) + torch.distributed.all_gather(grad_list, torch.randn(8, 6, 8, 8, device=device)) +else: + torch.distributed.all_gather(d_list, data) + torch.distributed.all_gather(y_list, output) + torch.distributed.all_gather(dgrad_list, data.grad) + torch.distributed.all_gather(grad_list, grad) + +torch.distributed.barrier() + +if args.local_rank == 0: + ref_tensor = d_list[1:] + ref_tensor.insert(0, data) + assert(ref_tensor[0].equal(data)) + ref_tensor = torch.cat(ref_tensor, 0) + ref_tensor = ref_tensor.detach() + ref_tensor.requires_grad_() + ref_tensor.retain_grad() + + # Reference forward/backward + output_reference = model_reference(ref_tensor) + grad_tensor = grad_list[1:] + grad_tensor.insert(0, grad) + assert(grad_tensor[0].equal(grad)) + grad_tensor = torch.cat(grad_tensor, 0) + if weighted_gradient: + output_reference.backward(grad_tensor / output_reference.size(0)) + else: + output_reference.backward(grad_tensor / output_reference.size(0)) + + dgrad_tensor = dgrad_list[1:] + dgrad_tensor.insert(0, data.grad) + dgrad_tensor = torch.cat(dgrad_tensor, 0) + # check output + output_tensor = y_list[1:] + output_tensor.insert(0, output) + output_tensor = torch.cat(output_tensor, 0) + passed = True + passed = passed and compare("check output", + output_tensor, + output_reference) + # check stats + passed = passed and compare("check running mean failed", + model_reference[1].running_mean, + model.module[1].running_mean) + passed = passed and compare("check running var failed", + model_reference[1].running_var, + model.module[1].running_var) + passed = passed and compare("bn wgrad check failed!", + model_reference[1].weight.grad, + model.module[1].weight.grad, 1e-6) + passed = passed and compare("conv wgrad check failed!", + model_reference[0].weight.grad, + model.module[0].weight.grad) + # can't really compare dgrad directly, as we need to scale it to account for + # DDP + # passed = passed and compare("dgrad check failed!", ref_tensor.grad, dgrad_tensor) + if passed: + print("====SBN two gpu with different batches test passed") + else: + assert("*failed two gpu with different batches tests*") diff --git a/apex/tests/distributed/synced_batchnorm/two_gpu_unit_test.py b/apex/tests/distributed/synced_batchnorm/two_gpu_unit_test.py new file mode 100644 index 00000000..ea89e7ae --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/two_gpu_unit_test.py @@ -0,0 +1,180 @@ +import torch +import numpy as np +import apex +import syncbn +import os +import argparse +import torch.optim as optim + +def compare(desc, inp1, inp2, error): + a = inp1.clone().detach().cpu().numpy() + b = inp2.clone().detach().cpu().numpy() + close = np.allclose(a,b, error, error) + if not close: + print(desc, close) + z = a - b + index = (np.abs(z) >= error + error * np.abs(b)).nonzero() + print("dif : ", z[index]) + print("inp1 : ", a[index]) + print("inp2 : ", b[index]) + return close + +feature_size = 10 +space_size = 40 +batch_size = 32 + + +from apex.parallel import DistributedDataParallel as DDP +parser = argparse.ArgumentParser() +parser.add_argument("--local_rank", default=0, type=int) +parser.add_argument("--fp16", action='store_true', default=False) +parser.add_argument("--fp64", action='store_true', default=False) +args = parser.parse_args() +args.world_size = int(os.environ['WORLD_SIZE']) +torch.cuda.set_device(args.local_rank) +torch.distributed.init_process_group(backend='nccl', init_method='env://') +start = args.local_rank * batch_size//args.world_size +finish = (args.local_rank + 1) * batch_size//args.world_size + +error = 1e-5 +dtype = np.float32 +if args.fp16: + error = 1e-3 + dtype = np.float16 +elif args.fp64: + error = 1e-8 + dtype = np.float64 + +np.random.seed(18) +inp = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype) +grad = np.random.randn(batch_size, feature_size, space_size, space_size).astype(dtype) +weight = np.random.randn(feature_size).astype(dtype) +bias = np.random.randn(feature_size).astype(dtype) + + +type_tensor = torch.cuda.FloatTensor +if args.fp16: + type_tensor = torch.cuda.HalfTensor +if args.fp64: + type_tensor = torch.cuda.DoubleTensor + +ref_tensor = torch.cuda.DoubleTensor + +inp_t = type_tensor(inp) +weight_t = type_tensor(weight) +bias_t = type_tensor(bias) + +inp_r = ref_tensor(inp.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +inp2_r = ref_tensor(inp) +weight_r = ref_tensor(weight).view(-1, 1, 1) +bias_r = ref_tensor(bias).view(-1, 1, 1) + +grad_output_t = type_tensor(grad) + +m = inp_r.mean(1) +b_v = inp_r.var(1, unbiased=False) +unb_v = inp_r.var(1, unbiased=True) + +eps = 1e-5 + +mean, var_biased = syncbn.welford_mean_var(inp_t) +inv_std = 1.0 / torch.sqrt(var_biased + eps) + +bn = torch.nn.BatchNorm2d(feature_size).cuda() +bn.momentum = 1.0 +bn.weight.data = weight_t.clone() +bn.bias.data = bias_t.clone() +if args.fp16: + bn.half() +if args.fp64: + bn.double() +inp_bn = inp_t.clone().requires_grad_() +grad_bn = grad_output_t.clone().detach() +out_bn = bn(inp_bn) +out_bn.backward(grad_bn) +# compensating the averaging over processes done by DDP +# in order to produce mathematically equivalent result +# https://github.com/NVIDIA/apex/issues/134#issuecomment-458307368 +for param in bn.parameters(): + param.grad = param.grad / args.world_size +bn_opt = optim.SGD(bn.parameters(), lr=1.0) + +sbn = apex.parallel.SyncBatchNorm(feature_size).cuda() +sbn.momentum = 1.0 +sbn.weight.data = weight_t.clone() +sbn.bias.data = bias_t.clone() +if args.fp16: + sbn.half() +if args.fp64: + sbn.double() +sbn = DDP(sbn) +sbn_opt = optim.SGD(sbn.parameters(), lr=1.0) +inp_sbn = inp_t.clone().requires_grad_() +grad_sbn = grad_output_t.clone().detach() +out_sbn = sbn(inp_sbn[start:finish]) +out_sbn.backward(grad_sbn[start:finish]) + +count = [ space_size**2 * ( (i+1) * batch_size // args.world_size - i * batch_size // args.world_size ) for i in range(0, args.world_size)] +count = torch.cuda.IntTensor(count) + +print("--- count : " , count) + +sbn_result = True +bn_result = True + +if args.local_rank == 0: + sbn_result = compare("comparing mean: ", mean, m, error) and sbn_result + sbn_result = compare("comparing biased variance: ", var_biased, b_v, error) and sbn_result + +out = syncbn.batchnorm_forward(inp_t, mean, inv_std, weight_t, bias_t) +out_r = weight_r * (inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) + bias_r + +if args.local_rank == 0: + sbn_result = compare("comparing output: ", out, out_r, error) and sbn_result + compare("comparing bn output: ", out_bn, out_r, error) + +grad_output_t = type_tensor(grad) + +grad_output_r = ref_tensor(grad.transpose(1, 0, 2, 3).reshape(feature_size, -1)) +grad_output2_r = ref_tensor(grad) + +grad_bias_r = grad_output_r.sum(1) +grad_weight_r = ((inp2_r - m.view(-1, 1, 1)) * torch.rsqrt(b_v.view(-1,1,1) + eps) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1) + +sum_dy_r = grad_output_r.sum(1) +mean_dy_r = grad_output_r.mean(1) +mean_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).mean(1) +sum_dy_xmu_r = ((inp2_r - m.view(-1, 1, 1)) * grad_output2_r).transpose(1,0).contiguous().view(feature_size, -1).sum(1) + +grad_input_r = (grad_output2_r - mean_dy_r.view(-1, 1, 1) - (inp2_r - m.view(-1, 1, 1)) / (b_v.view(-1,1,1) + eps) * mean_dy_xmu_r.view(-1, 1, 1) ) * torch.rsqrt(b_v.view(-1,1,1) + eps) * weight_r.view(-1,1,1) + +sum_dy, sum_dy_xmu, grad_weight, grad_bias = syncbn.reduce_bn(grad_output_t, inp_t, mean, inv_std, weight_t) +grad_input = syncbn.batchnorm_backward(grad_output_t, inp_t, mean, inv_std, weight_t, sum_dy, sum_dy_xmu, count) +if args.local_rank == 0: + sbn_result = compare("comparing bias grad: ", grad_bias, grad_bias_r, error) and sbn_result + sbn_result = compare("comparing weight grad: ", grad_weight, grad_weight_r, error) and sbn_result + sbn_result = compare("comparing sum_dy grad: ", sum_dy, sum_dy_r, error) and sbn_result + sbn_result = compare("comparing sum_dy_xmu grad: ", sum_dy_xmu, sum_dy_xmu_r, error) and sbn_result + sbn_result = compare("comparing input grad: ", grad_input, grad_input_r, error) and sbn_result + compare("comparing bn input grad: ", inp_bn.grad, grad_input_r, error) + +if args.local_rank == 0: + sbn_result = compare("comparing running_mean: ", bn.running_mean.data, sbn.module.running_mean.data, error) and sbn_result + sbn_result = compare("comparing running_variance: ", bn.running_var.data, sbn.module.running_var.data, error) and sbn_result + +# execute by both +compare("comparing layers output: ", out_bn[start:finish], out_sbn, error) and sbn_result +compare("comparing layers grad_input: ", inp_bn.grad[start:finish], inp_sbn.grad[start:finish], error) and sbn_result + +bn_opt.step() +sbn_opt.step() + +if args.local_rank == 0: + compare("comparing bn vs sbn bias: ", bn.bias, sbn.module.bias, error) + compare("comparing bn vs sbn weight: ", bn.weight, sbn.module.weight, error) + + +if sbn_result: + print("====SBN two gpu passed tests") +else: + print("*SBN two gpu failed*") diff --git a/apex/tests/distributed/synced_batchnorm/unit_test.sh b/apex/tests/distributed/synced_batchnorm/unit_test.sh new file mode 100755 index 00000000..2165f5ec --- /dev/null +++ b/apex/tests/distributed/synced_batchnorm/unit_test.sh @@ -0,0 +1,8 @@ +python python_single_gpu_unit_test.py +python single_gpu_unit_test.py +python test_batchnorm1d.py +python -m torch.distributed.launch --nproc_per_node=2 two_gpu_unit_test.py +python -m torch.distributed.launch --nproc_per_node=2 two_gpu_unit_test.py --fp16 +python -m torch.distributed.launch --nproc_per_node=2 two_gpu_test_different_batch_size.py --apex +#beware, you need a system with at least 4 gpus to test group_size Date: Thu, 25 Dec 2025 19:45:04 +0400 Subject: [PATCH 05/39] Prune old stage_1_alignment logs and tensorboard events; keep latest artifacts untracked --- ...20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log | 1044 - ...10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 53109 ---------------- ...54_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 7690 --- ...21_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 18531 ------ ...45_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 18530 ------ ...03_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 1038 - ...22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 1038 - ...56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log | 1038 - ...51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log | 1038 - ...14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log | 1038 - ...51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log | 1038 - ...30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log | 1058 - ...s.out.tfevents.1766400270.gpu-52.3342268.0 | Bin 48602 -> 0 bytes ...ts.out.tfevents.1766426196.gpu-04.111446.0 | Bin 48602 -> 0 bytes ...ts.out.tfevents.1766426626.gpu-04.112116.0 | Bin 48602 -> 0 bytes ...ts.out.tfevents.1766427132.gpu-04.112924.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766567162.gpu-51.2669674.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766568374.gpu-51.2671666.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766569231.gpu-51.2673246.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766569685.gpu-51.2674567.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766571661.gpu-51.2676736.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766571804.gpu-51.2677199.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766571900.gpu-51.2677735.0 | Bin 48602 -> 0 bytes ...s.out.tfevents.1766572119.gpu-51.2678290.0 | Bin 48602 -> 0 bytes 24 files changed, 106190 deletions(-) delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_22:02:54_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-22_22:11:21_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:05:45_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:26:03_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766400270.gpu-52.3342268.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766426196.gpu-04.111446.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766426626.gpu-04.112116.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766427132.gpu-04.112924.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766567162.gpu-51.2669674.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766568374.gpu-51.2671666.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569231.gpu-51.2673246.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569685.gpu-51.2674567.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766571661.gpu-51.2676736.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766571804.gpu-51.2677199.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766571900.gpu-51.2677735.0 delete mode 100644 stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766572119.gpu-51.2678290.0 diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log deleted file mode 100644 index 40c7dfc2..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-22_14:44:20_tp2_pp1_seqlen8192_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1044 +0,0 @@ -W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] -W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] ***************************************** -W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1222 14:44:22.943000 3342258 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 8192 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 8192 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.030 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1222 14:44:31.761652648 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.346 seconds -time to initialize megatron (seconds): 3.999 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-22 14:44:34 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 3False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 FalseFalse - -3 3False - False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc2.linear.weight - module.adapter.layernorm.bias - module.adapter.layernorm.weight - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (7289.26, 7289.29) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 14:45:19 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-22 14:45:19 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (44573.39, 44573.43) - train/valid/test-data-iterators-setup ..........: (840.26, 842.88) -training ... -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-22 14:45:19 -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 356.27 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward\n core_attn_out = self.core_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward\n matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor\n self.buffer[(name, dtype)] = torch.empty(\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward -[rank1]: core_attn_out = self.core_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward -[rank1]: matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor -[rank1]: self.buffer[(name, dtype)] = torch.empty( -[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 356.27 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward\n core_attn_out = self.core_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward\n matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor\n self.buffer[(name, dtype)] = torch.empty(\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 471, in forward -[rank0]: core_attn_out = self.core_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/dot_product_attention.py", line 146, in forward -[rank0]: matmul_input_buffer = parallel_state.get_global_memory_buffer().get_tensor( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/utils.py", line 415, in get_tensor -[rank0]: self.buffer[(name, dtype)] = torch.empty( -[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 356.27 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.47 GiB is free. Including non-PyTorch memory, this process has 10.08 GiB memory in use. Of the allocated memory 8.31 GiB is allocated by PyTorch, and 890.48 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -[rank1]:[W1222 14:45:23.420611800 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1222 14:45:28.727847326 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1222 14:45:29.902000 3342258 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 3342267 closing signal SIGTERM -E1222 14:45:30.116000 3342258 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 3342268) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-22_14:45:29 - host : gpu-52 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 3342268) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log deleted file mode 100644 index 77af4bde..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-22_21:55:10_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,53109 +0,0 @@ -W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] -W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] ***************************************** -W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1222 21:55:29.684000 111435 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 4096 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.066 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1222 21:56:35.174676658 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -> setting tensorboard ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 1.931 seconds -time to initialize megatron (seconds): 19.957 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-22 21:56:53 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.weight - module.adapter.linear_fc1.linear.bias - module.adapter.layernorm.bias - module.adapter.linear_fc2.linear.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (7206.33, 7206.41) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 21:57:38 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not existdataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist - -[after dataloaders are built] datetime: 2025-12-22 21:57:41 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (44392.15, 44392.16) - train/valid/test-data-iterators-setup ..........: (3101.55, 3101.55)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-22 21:57:41 -packing_seq_len:4096----<<<<<----4176 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ - for idx, (sample_idx, inner_sample) in enumerate( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx - x = next(it) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator - sample = next(it) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample - l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples - super().pack_selected_samples(samples), - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 351, in pack_selected_samples - raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") -ValueError: Packed sample exceeds the maximum sequence length of 4096: [Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img000_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, - 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, 198, - 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), total_len=315, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7777, ..., 0.8803, 0.8661, 0.8803], - [-1.7923, -1.7923, -1.7923, ..., -1.4660, -1.4660, -1.4518], - ..., - [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], - [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 46]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img001_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 60424, - 264, 9814, 4008, 315, 279, 2661, 2168, 13, 151645, - 198, 151644, 77091, 198, 1782, 31535, 369, 16301, 55155, - 304, 6303, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 1782, 31535, 369, 16301, 55155, - 304, 6303, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img002_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 1782, 1656, 29428, - 14126, 22361, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 1782, 1656, 29428, - 14126, 22361, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img003_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 276, 2168, 315, 264, 41530, 15616, 28642, 448, - 52484, 12624, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 276, 2168, 315, 264, 41530, 15616, 28642, 448, - 52484, 12624, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img004_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 6750, 264, 2797, 323, 63594, 12126, 315, - 279, 6548, 13, 151645, 198, 151644, 77091, 198, 1782, - 1901, 36780, 15273, 19818, 4621, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, 1782, - 1901, 36780, 15273, 19818, 4621, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.4048, 1.4048, 1.4048, ..., 1.7620, 1.7762, 1.7762], - [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], - [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], - ..., - [ 1.3026, 1.3026, 1.3026, ..., -1.0252, -1.0252, -1.0252], - [-0.1572, 0.6019, 0.4997, ..., -0.9399, -0.9399, -0.9683], - [-0.2740, -0.4784, 0.6457, ..., -0.9683, -0.9683, -0.9683]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img005_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, - 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 78243, 75105, 220, 23, - 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, - 6430, 12136, 56432, 3421, 1947, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 78243, 75105, 220, 23, - 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, - 6430, 12136, 56432, 3421, 1947, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.3835, -1.3835, -1.3835, ..., -0.5275, -0.5275, -0.5275], - [-1.3689, -1.3543, -1.3251, ..., -0.3853, -0.3995, -0.3995], - [-1.3835, -1.3689, -1.3689, ..., -0.4848, -0.4706, -0.4422], - ..., - [-1.3981, -1.4127, -1.4273, ..., -0.4137, -0.3995, -0.3995], - [-1.2375, -0.2594, -1.6609, ..., -0.5275, -0.5133, -0.5133], - [-1.1937, -0.3032, 0.3391, ..., -0.5417, -0.5417, -0.5417]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img006_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, - 264, 9814, 4008, 315, 279, 2168, 13, 151645, 198, - 151644, 77091, 198, 1782, 10550, 29674, 645, 12426, 448, - 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, - 259, 481, 36724, 481, 2953, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 1782, 10550, 29674, 645, 12426, 448, - 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, - 259, 481, 36724, 481, 2953, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.9893, -1.0039, -1.0039, ..., -0.2715, -0.2715, -0.2715], - [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2573], - [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2715], - ..., - [-1.0915, -1.0915, -1.0769, ..., -0.2857, -0.3000, -0.3142], - [-1.1207, -1.1207, -1.1207, ..., -0.3853, -0.3568, -0.3568], - [-1.1061, -1.1061, -1.1061, ..., -0.3142, -0.3000, -0.3000]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img007_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 1782, 42646, 331, 595, 3821, 18, 21, 19, - 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, - 323, 264, 24461, 9509, 27860, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 1782, 42646, 331, 595, 3821, 18, 21, 19, - 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, - 323, 264, 24461, 9509, 27860, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img008_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 67, 20367, 19684, 17, 220, 17, - 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, - 323, 18579, 71738, 11, 4024, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 67, 20367, 19684, 17, 220, 17, - 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, - 323, 18579, 71738, 11, 4024, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - ..., - [-1.2959, -1.3397, -1.7923, ..., 2.0748, 2.0321, 2.1459], - [-1.4273, -1.7193, -0.9602, ..., 1.7620, 1.9184, 1.9610], - [-1.5003, -1.4419, -1.7923, ..., 1.9895, 2.0748, 2.0321]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img009_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, - 198, 151644, 77091, 198, 759, 75242, 12426, 11, 66274, - 32300, 4603, 32300, 4603, 1841, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 759, 75242, 12426, 11, 66274, - 32300, 4603, 32300, 4603, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.2375, -1.5733, -1.7923, ..., -0.8261, -0.6412, -0.6270], - [-1.7923, -1.7923, -1.7777, ..., -0.9683, -0.9683, -0.9683], - [-1.7923, -1.7923, -1.7631, ..., -0.9683, -0.9683, -0.9683], - ..., - [-1.7339, -1.5879, -1.4565, ..., 2.1459, 2.1459, 2.1459], - [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459], - [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img010_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 64, 2766, 1809, 448, - 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, - 279, 18967, 13962, 32372, 4426, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 64, 2766, 1809, 448, - 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, - 279, 18967, 13962, 32372, 4426, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img011_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, - 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, - 151645, 198, 151644, 77091, 198, 1782, 11623, 369, 1435, - 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, - 15786, 369, 279, 5593, 364, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 1782, 11623, 369, 1435, - 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, - 15786, 369, 279, 5593, 364, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3807, -1.3807, -1.3807], - ..., - [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img012_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, - 198, 151644, 77091, 198, 5082, 352, 263, 6837, 39698, - 1899, 69105, 938, 81625, 819, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 5082, 352, 263, 6837, 39698, - 1899, 69105, 938, 81625, 819, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.6536, -1.4419, -1.7923, ..., -1.4518, -1.4518, -1.4518], - [-1.7631, -1.7631, -1.7485, ..., -1.3096, -1.4233, -1.4802], - [-1.7631, -1.7631, -1.7631, ..., -1.4518, -1.4518, -1.4518], - ..., - [-1.7631, -1.7631, -1.7047, ..., -1.4518, -1.4518, -1.4518], - [-0.0113, 0.2953, -1.5733, ..., -1.3238, -1.3807, -1.3949], - [ 0.8063, -0.5368, -1.7777, ..., -1.4518, -1.4518, -1.4518]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img013_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 74785, - 279, 2168, 3529, 285, 974, 13, 151645, 198, 151644, - 77091, 198, 19789, 6078, 304, 279, 6083, 315, 264, - 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, - 5382, 3054, 2114, 10576, 902, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, 19789, 6078, 304, 279, 6083, 315, 264, - 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, - 5382, 3054, 2114, 10576, 902, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5771, 1.5771], - [ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5629, 1.5487], - [ 1.3464, 1.3464, 1.3464, ..., 1.5771, 1.5771, 1.5629], - ..., - [-0.9310, -0.9310, -0.9018, ..., 0.9514, 1.9753, 1.9895], - [-0.7412, -0.7266, -0.7120, ..., -0.7834, -0.7408, -0.7408], - [-0.5806, -0.5660, -0.5368, ..., 0.9230, 2.0606, 2.0748]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img014_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, - 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, - 151645, 198, 151644, 77091, 198, 2277, 481, 61794, 18342, - 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, - 311, 259, 481, 6470, 34957, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 2277, 481, 61794, 18342, - 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, - 311, 259, 481, 6470, 34957, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], - [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7762, 1.7762], - [1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], - ..., - [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7620, 1.7335], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.8281, 1.7114, 1.5800, ..., 2.1317, 2.1317, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img015_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 14593, 15478, 11, 825, 3338, - 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, - 481, 10658, 1419, 19507, 2062, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 14593, 15478, 11, 825, 3338, - 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, - 481, 10658, 1419, 19507, 2062, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - ..., - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img016_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, - 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 500, 5064, 5750, 11, - 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, - 28451, 22601, 11, 12188, 22601, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 500, 5064, 5750, 11, - 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, - 28451, 22601, 11, 12188, 22601, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img017_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 1782, 6623, 60037, - 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, - 738, 448, 19752, 3931, 48051, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 1782, 6623, 60037, - 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, - 738, 448, 19752, 3931, 48051, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img018_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 60424, 264, 9814, 4008, 315, 279, 2661, - 2168, 13, 151645, 198, 151644, 77091, 198, 2024, 3045, - 7821, 2103, 488, 11030, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, 2024, 3045, - 7821, 2103, 488, 11030, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [-0.2886, -0.2302, 0.0179, ..., 2.0890, 2.1175, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img019_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, - 264, 2805, 323, 2797, 16148, 315, 279, 17420, 2168, - 13, 151645, 198, 151644, 77091, 198, 64, 2781, 504, - 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, - 389, 264, 7002, 4034, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 64, 2781, 504, - 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, - 389, 264, 7002, 4034, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., -0.5986, -0.5986, -0.6270], - ..., - [-1.2083, -1.2083, -1.2083, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img020_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, - 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, - 198, 1782, 1081, 72725, 550, 315, 6470, 296, 377, - 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, - 29724, 315, 279, 93767, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 1782, 1081, 72725, 550, 315, 6470, 296, 377, - 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, - 29724, 315, 279, 93767, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img021_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 7985, 264, 50537, 714, - 38219, 12126, 315, 279, 6802, 624, 151652, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, - 198, 151644, 77091, 198, 1782, 3691, 1555, 4013, 281, - 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, - 1555, 2550, 2355, 8149, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 1782, 3691, 1555, 4013, 281, - 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, - 1555, 2550, 2355, 8149, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img022_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 19789, 87361, 14201, - 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, - 2472, 315, 1059, 59511, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 19789, 87361, 14201, - 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, - 2472, 315, 1059, 59511, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[-1.2083, -1.2083, -1.2083, ..., -0.9256, -0.9256, -0.9399], - [-1.1207, -1.1207, -1.0915, ..., -0.8545, -0.8688, -0.8688], - [-1.2959, -1.2959, -1.2813, ..., -1.0821, -1.0678, -1.0678], - ..., - [-0.7704, -0.7850, -0.7704, ..., -0.4990, -0.4990, -0.4990], - [-0.7558, -0.7704, -0.7850, ..., -0.3142, -0.3142, -0.2857], - [-0.8142, -0.8142, -0.8142, ..., -0.2715, -0.2431, -0.2289]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img023_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 3838, 374, 304, 279, - 6548, 5267, 151652, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 18824, 315, 13779, 9317, 1173, - 553, 9080, 567, 84946, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 18824, 315, 13779, 9317, 1173, - 553, 9080, 567, 84946, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., -0.7123, -0.7266, -0.7266], - [ 1.9303, 1.9303, 1.9303, ..., -0.7834, -0.7977, -0.7977], - [-1.0039, -1.3251, -1.2083, ..., -0.7977, -0.7692, -0.7408], - ..., - [ 1.0836, 1.1128, 1.2442, ..., 2.0890, 2.1032, 2.1175], - [ 0.9814, 0.9814, 0.9814, ..., 2.1175, 2.1175, 2.1317], - [ 1.3464, 1.4048, 1.4924, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img024_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 64, 22360, 3745, 448, - 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, - 4158, 67590, 3745, 26334, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 64, 22360, 3745, 448, - 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, - 4158, 67590, 3745, 26334, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.7260, 1.7260, 1.7260, ..., 1.9042, 1.9042, 1.9042], - [1.7260, 1.7260, 1.7260, ..., 1.8899, 1.8899, 1.8899], - [1.6822, 1.6822, 1.6822, ..., 1.8473, 1.8473, 1.8473], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img025_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, - 198, 151644, 77091, 198, 64, 3691, 16049, 8511, 25279, - 11, 825, 7100, 962, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 64, 3691, 16049, 8511, 25279, - 11, 825, 7100, 962, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.7844, 1.7990, 1.8135, ..., 2.0179, 2.0464, 2.0748], - [1.8719, 1.8719, 1.8573, ..., 2.0890, 2.1459, 2.1459], - [1.8427, 1.8427, 1.8573, ..., 2.0606, 2.0606, 2.0748], - ..., - [1.7990, 1.7698, 1.7844, ..., 2.0606, 2.0606, 2.0606], - [1.8427, 1.8427, 1.8427, ..., 2.0321, 2.0321, 2.0464], - [1.7990, 1.7844, 1.7990, ..., 2.0890, 2.1032, 2.1032]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img026_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 1151, 6918, 5636, 304, - 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, - 11, 9710, 5099, 8166, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 1151, 6918, 5636, 304, - 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, - 11, 9710, 5099, 8166, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.7406, 1.7406, 1.7406, ..., 2.0037, 2.0037, 2.0037], - [ 1.7406, 1.7406, 1.6968, ..., 2.0606, 2.0464, 2.0890], - [ 1.7406, 1.7406, 1.7406, ..., 1.7620, 1.5344, 1.5060], - ..., - [ 1.7406, 1.7406, 1.7406, ..., 2.1459, 2.0321, 2.0748], - [ 0.2223, -0.3616, 0.6165, ..., 2.1459, 2.0037, 2.1317], - [ 0.3975, -0.0842, -0.0696, ..., 2.1459, 2.0748, 2.0890]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img027_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, - 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, - 2953, 3198, 52679, 19125, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, - 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, - 2953, 3198, 52679, 19125, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img028_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, - 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 5220, 374, 12233, 458, 220, - 23, 15, 82, 259, 481, 15478, 448, 458, 2168, - 315, 264, 46652, 1841, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 5220, 374, 12233, 458, 220, - 23, 15, 82, 259, 481, 15478, 448, 458, 2168, - 315, 264, 46652, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 1.6055, 1.6055, 1.6055], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317], - [1.3756, 1.4486, 1.6676, ..., 2.1175, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img029_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 1782, 9382, 642, 56727, 11, 448, - 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, - 1632, 438, 1008, 12396, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 1782, 9382, 642, 56727, 11, 448, - 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, - 1632, 438, 1008, 12396, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - ..., - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img030_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, - 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, - 198, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, - 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, - 49496, 8968, 52001, 8778, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, - 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, - 49496, 8968, 52001, 8778, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.7114, 1.4632, ..., 1.6482, 1.6482, 1.6624], - [1.3318, 1.3318, 1.3318, ..., 1.6909, 1.6909, 1.6766], - [1.9303, 1.8135, 1.4340, ..., 1.6055, 1.6055, 1.5913], - ..., - [0.9522, 1.0252, 0.7479, ..., 1.3069, 1.3069, 1.3354], - [1.3026, 1.2734, 1.2588, ..., 1.7193, 1.7477, 1.7477], - [1.5508, 1.5362, 1.5508, ..., 1.7335, 1.7193, 1.7193]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img031_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 9190, 5612, 551, 279, - 9124, 2213, 315, 279, 2168, 624, 151652, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151653, 151645, 198, 151644, 77091, 198, 267, 287, - 10464, 93957, 45993, 1173, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, 267, 287, - 10464, 93957, 45993, 1173, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - ..., - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None)] ----------- -PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.056 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1222 22:03:46.728785705 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -> setting tensorboard ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 0.921 seconds -time to initialize megatron (seconds): 12.880 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-22 22:03:57 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc1.linear.bias - module.adapter.layernorm.bias - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.weight - module.adapter.linear_fc2.linear.bias -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (7090.80, 7090.82) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 22:04:44 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-22 22:04:45 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (46747.78, 46747.84) - train/valid/test-data-iterators-setup ..........: (1066.80, 1066.81)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-22 22:04:45 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ - for idx, (sample_idx, inner_sample) in enumerate( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx - x = next(it) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator - sample = next(it) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample - l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples - super().pack_selected_samples(samples), - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 346, in pack_selected_samples - sample = sample[:max_length] -TypeError: 'Qwen2VLImageTaskSample' object is not subscriptable ----------- -PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.054 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1222 22:12:12.619771483 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -> setting tensorboard ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 1.358 seconds -time to initialize megatron (seconds): 15.380 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-22 22:12:26 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False3 - False -3 False -3 False -33 False -False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc1.linear.bias - module.adapter.layernorm.bias - module.adapter.linear_fc2.linear.weight - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (6811.19, 6811.28) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-22 22:13:11 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-22 22:13:12 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (45536.07, 45541.33) - train/valid/test-data-iterators-setup ..........: (1021.23, 1021.28)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-22 22:13:12 -packing_seq_len:4096----<<<<<----4176 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ - for idx, (sample_idx, inner_sample) in enumerate( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx - x = next(it) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator - sample = next(it) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample - l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples - super().pack_selected_samples(samples), - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 380, in pack_selected_samples - raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") -ValueError: Packed sample exceeds the maximum sequence length of 4096: [Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img000_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, - 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, 198, - 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), total_len=315, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7777, ..., 0.8803, 0.8661, 0.8803], - [-1.7923, -1.7923, -1.7923, ..., -1.4660, -1.4660, -1.4518], - ..., - [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], - [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 46]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img001_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 60424, - 264, 9814, 4008, 315, 279, 2661, 2168, 13, 151645, - 198, 151644, 77091, 198, 1782, 31535, 369, 16301, 55155, - 304, 6303, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 1782, 31535, 369, 16301, 55155, - 304, 6303, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img002_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 1782, 1656, 29428, - 14126, 22361, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 1782, 1656, 29428, - 14126, 22361, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img003_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 276, 2168, 315, 264, 41530, 15616, 28642, 448, - 52484, 12624, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 276, 2168, 315, 264, 41530, 15616, 28642, 448, - 52484, 12624, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img004_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 6750, 264, 2797, 323, 63594, 12126, 315, - 279, 6548, 13, 151645, 198, 151644, 77091, 198, 1782, - 1901, 36780, 15273, 19818, 4621, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, 1782, - 1901, 36780, 15273, 19818, 4621, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.4048, 1.4048, 1.4048, ..., 1.7620, 1.7762, 1.7762], - [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], - [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], - ..., - [ 1.3026, 1.3026, 1.3026, ..., -1.0252, -1.0252, -1.0252], - [-0.1572, 0.6019, 0.4997, ..., -0.9399, -0.9399, -0.9683], - [-0.2740, -0.4784, 0.6457, ..., -0.9683, -0.9683, -0.9683]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img005_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, - 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 78243, 75105, 220, 23, - 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, - 6430, 12136, 56432, 3421, 1947, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 78243, 75105, 220, 23, - 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, - 6430, 12136, 56432, 3421, 1947, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.3835, -1.3835, -1.3835, ..., -0.5275, -0.5275, -0.5275], - [-1.3689, -1.3543, -1.3251, ..., -0.3853, -0.3995, -0.3995], - [-1.3835, -1.3689, -1.3689, ..., -0.4848, -0.4706, -0.4422], - ..., - [-1.3981, -1.4127, -1.4273, ..., -0.4137, -0.3995, -0.3995], - [-1.2375, -0.2594, -1.6609, ..., -0.5275, -0.5133, -0.5133], - [-1.1937, -0.3032, 0.3391, ..., -0.5417, -0.5417, -0.5417]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img006_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, - 264, 9814, 4008, 315, 279, 2168, 13, 151645, 198, - 151644, 77091, 198, 1782, 10550, 29674, 645, 12426, 448, - 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, - 259, 481, 36724, 481, 2953, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 1782, 10550, 29674, 645, 12426, 448, - 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, - 259, 481, 36724, 481, 2953, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.9893, -1.0039, -1.0039, ..., -0.2715, -0.2715, -0.2715], - [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2573], - [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2715], - ..., - [-1.0915, -1.0915, -1.0769, ..., -0.2857, -0.3000, -0.3142], - [-1.1207, -1.1207, -1.1207, ..., -0.3853, -0.3568, -0.3568], - [-1.1061, -1.1061, -1.1061, ..., -0.3142, -0.3000, -0.3000]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img007_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 1782, 42646, 331, 595, 3821, 18, 21, 19, - 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, - 323, 264, 24461, 9509, 27860, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 1782, 42646, 331, 595, 3821, 18, 21, 19, - 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, - 323, 264, 24461, 9509, 27860, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img008_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 67, 20367, 19684, 17, 220, 17, - 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, - 323, 18579, 71738, 11, 4024, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 67, 20367, 19684, 17, 220, 17, - 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, - 323, 18579, 71738, 11, 4024, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - ..., - [-1.2959, -1.3397, -1.7923, ..., 2.0748, 2.0321, 2.1459], - [-1.4273, -1.7193, -0.9602, ..., 1.7620, 1.9184, 1.9610], - [-1.5003, -1.4419, -1.7923, ..., 1.9895, 2.0748, 2.0321]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img009_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, - 198, 151644, 77091, 198, 759, 75242, 12426, 11, 66274, - 32300, 4603, 32300, 4603, 1841, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 759, 75242, 12426, 11, 66274, - 32300, 4603, 32300, 4603, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.2375, -1.5733, -1.7923, ..., -0.8261, -0.6412, -0.6270], - [-1.7923, -1.7923, -1.7777, ..., -0.9683, -0.9683, -0.9683], - [-1.7923, -1.7923, -1.7631, ..., -0.9683, -0.9683, -0.9683], - ..., - [-1.7339, -1.5879, -1.4565, ..., 2.1459, 2.1459, 2.1459], - [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459], - [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img010_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 64, 2766, 1809, 448, - 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, - 279, 18967, 13962, 32372, 4426, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 64, 2766, 1809, 448, - 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, - 279, 18967, 13962, 32372, 4426, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img011_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, - 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, - 151645, 198, 151644, 77091, 198, 1782, 11623, 369, 1435, - 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, - 15786, 369, 279, 5593, 364, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 1782, 11623, 369, 1435, - 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, - 15786, 369, 279, 5593, 364, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3807, -1.3807, -1.3807], - ..., - [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img012_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, - 198, 151644, 77091, 198, 5082, 352, 263, 6837, 39698, - 1899, 69105, 938, 81625, 819, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 5082, 352, 263, 6837, 39698, - 1899, 69105, 938, 81625, 819, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.6536, -1.4419, -1.7923, ..., -1.4518, -1.4518, -1.4518], - [-1.7631, -1.7631, -1.7485, ..., -1.3096, -1.4233, -1.4802], - [-1.7631, -1.7631, -1.7631, ..., -1.4518, -1.4518, -1.4518], - ..., - [-1.7631, -1.7631, -1.7047, ..., -1.4518, -1.4518, -1.4518], - [-0.0113, 0.2953, -1.5733, ..., -1.3238, -1.3807, -1.3949], - [ 0.8063, -0.5368, -1.7777, ..., -1.4518, -1.4518, -1.4518]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img013_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 74785, - 279, 2168, 3529, 285, 974, 13, 151645, 198, 151644, - 77091, 198, 19789, 6078, 304, 279, 6083, 315, 264, - 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, - 5382, 3054, 2114, 10576, 902, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, 19789, 6078, 304, 279, 6083, 315, 264, - 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, - 5382, 3054, 2114, 10576, 902, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5771, 1.5771], - [ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5629, 1.5487], - [ 1.3464, 1.3464, 1.3464, ..., 1.5771, 1.5771, 1.5629], - ..., - [-0.9310, -0.9310, -0.9018, ..., 0.9514, 1.9753, 1.9895], - [-0.7412, -0.7266, -0.7120, ..., -0.7834, -0.7408, -0.7408], - [-0.5806, -0.5660, -0.5368, ..., 0.9230, 2.0606, 2.0748]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img014_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, - 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, - 151645, 198, 151644, 77091, 198, 2277, 481, 61794, 18342, - 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, - 311, 259, 481, 6470, 34957, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 2277, 481, 61794, 18342, - 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, - 311, 259, 481, 6470, 34957, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], - [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7762, 1.7762], - [1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], - ..., - [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7620, 1.7335], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.8281, 1.7114, 1.5800, ..., 2.1317, 2.1317, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img015_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 14593, 15478, 11, 825, 3338, - 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, - 481, 10658, 1419, 19507, 2062, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 14593, 15478, 11, 825, 3338, - 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, - 481, 10658, 1419, 19507, 2062, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - ..., - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img016_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, - 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 500, 5064, 5750, 11, - 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, - 28451, 22601, 11, 12188, 22601, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 500, 5064, 5750, 11, - 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, - 28451, 22601, 11, 12188, 22601, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img017_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 1782, 6623, 60037, - 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, - 738, 448, 19752, 3931, 48051, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 1782, 6623, 60037, - 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, - 738, 448, 19752, 3931, 48051, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img018_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 60424, 264, 9814, 4008, 315, 279, 2661, - 2168, 13, 151645, 198, 151644, 77091, 198, 2024, 3045, - 7821, 2103, 488, 11030, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, 2024, 3045, - 7821, 2103, 488, 11030, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [-0.2886, -0.2302, 0.0179, ..., 2.0890, 2.1175, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img019_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, - 264, 2805, 323, 2797, 16148, 315, 279, 17420, 2168, - 13, 151645, 198, 151644, 77091, 198, 64, 2781, 504, - 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, - 389, 264, 7002, 4034, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 64, 2781, 504, - 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, - 389, 264, 7002, 4034, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., -0.5986, -0.5986, -0.6270], - ..., - [-1.2083, -1.2083, -1.2083, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img020_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, - 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, - 198, 1782, 1081, 72725, 550, 315, 6470, 296, 377, - 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, - 29724, 315, 279, 93767, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 1782, 1081, 72725, 550, 315, 6470, 296, 377, - 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, - 29724, 315, 279, 93767, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img021_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 7985, 264, 50537, 714, - 38219, 12126, 315, 279, 6802, 624, 151652, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, - 198, 151644, 77091, 198, 1782, 3691, 1555, 4013, 281, - 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, - 1555, 2550, 2355, 8149, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 1782, 3691, 1555, 4013, 281, - 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, - 1555, 2550, 2355, 8149, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img022_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 19789, 87361, 14201, - 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, - 2472, 315, 1059, 59511, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 19789, 87361, 14201, - 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, - 2472, 315, 1059, 59511, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[-1.2083, -1.2083, -1.2083, ..., -0.9256, -0.9256, -0.9399], - [-1.1207, -1.1207, -1.0915, ..., -0.8545, -0.8688, -0.8688], - [-1.2959, -1.2959, -1.2813, ..., -1.0821, -1.0678, -1.0678], - ..., - [-0.7704, -0.7850, -0.7704, ..., -0.4990, -0.4990, -0.4990], - [-0.7558, -0.7704, -0.7850, ..., -0.3142, -0.3142, -0.2857], - [-0.8142, -0.8142, -0.8142, ..., -0.2715, -0.2431, -0.2289]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img023_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 3838, 374, 304, 279, - 6548, 5267, 151652, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 18824, 315, 13779, 9317, 1173, - 553, 9080, 567, 84946, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 18824, 315, 13779, 9317, 1173, - 553, 9080, 567, 84946, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., -0.7123, -0.7266, -0.7266], - [ 1.9303, 1.9303, 1.9303, ..., -0.7834, -0.7977, -0.7977], - [-1.0039, -1.3251, -1.2083, ..., -0.7977, -0.7692, -0.7408], - ..., - [ 1.0836, 1.1128, 1.2442, ..., 2.0890, 2.1032, 2.1175], - [ 0.9814, 0.9814, 0.9814, ..., 2.1175, 2.1175, 2.1317], - [ 1.3464, 1.4048, 1.4924, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img024_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 64, 22360, 3745, 448, - 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, - 4158, 67590, 3745, 26334, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 64, 22360, 3745, 448, - 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, - 4158, 67590, 3745, 26334, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.7260, 1.7260, 1.7260, ..., 1.9042, 1.9042, 1.9042], - [1.7260, 1.7260, 1.7260, ..., 1.8899, 1.8899, 1.8899], - [1.6822, 1.6822, 1.6822, ..., 1.8473, 1.8473, 1.8473], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img025_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, - 198, 151644, 77091, 198, 64, 3691, 16049, 8511, 25279, - 11, 825, 7100, 962, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 64, 3691, 16049, 8511, 25279, - 11, 825, 7100, 962, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.7844, 1.7990, 1.8135, ..., 2.0179, 2.0464, 2.0748], - [1.8719, 1.8719, 1.8573, ..., 2.0890, 2.1459, 2.1459], - [1.8427, 1.8427, 1.8573, ..., 2.0606, 2.0606, 2.0748], - ..., - [1.7990, 1.7698, 1.7844, ..., 2.0606, 2.0606, 2.0606], - [1.8427, 1.8427, 1.8427, ..., 2.0321, 2.0321, 2.0464], - [1.7990, 1.7844, 1.7990, ..., 2.0890, 2.1032, 2.1032]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img026_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 1151, 6918, 5636, 304, - 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, - 11, 9710, 5099, 8166, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 1151, 6918, 5636, 304, - 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, - 11, 9710, 5099, 8166, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.7406, 1.7406, 1.7406, ..., 2.0037, 2.0037, 2.0037], - [ 1.7406, 1.7406, 1.6968, ..., 2.0606, 2.0464, 2.0890], - [ 1.7406, 1.7406, 1.7406, ..., 1.7620, 1.5344, 1.5060], - ..., - [ 1.7406, 1.7406, 1.7406, ..., 2.1459, 2.0321, 2.0748], - [ 0.2223, -0.3616, 0.6165, ..., 2.1459, 2.0037, 2.1317], - [ 0.3975, -0.0842, -0.0696, ..., 2.1459, 2.0748, 2.0890]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img027_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, - 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, - 2953, 3198, 52679, 19125, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, - 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, - 2953, 3198, 52679, 19125, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img028_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, - 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 5220, 374, 12233, 458, 220, - 23, 15, 82, 259, 481, 15478, 448, 458, 2168, - 315, 264, 46652, 1841, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 5220, 374, 12233, 458, 220, - 23, 15, 82, 259, 481, 15478, 448, 458, 2168, - 315, 264, 46652, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 1.6055, 1.6055, 1.6055], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317], - [1.3756, 1.4486, 1.6676, ..., 2.1175, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img029_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 1782, 9382, 642, 56727, 11, 448, - 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, - 1632, 438, 1008, 12396, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 1782, 9382, 642, 56727, 11, 448, - 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, - 1632, 438, 1008, 12396, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - ..., - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img030_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, - 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, - 198, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, - 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, - 49496, 8968, 52001, 8778, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, - 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, - 49496, 8968, 52001, 8778, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.7114, 1.4632, ..., 1.6482, 1.6482, 1.6624], - [1.3318, 1.3318, 1.3318, ..., 1.6909, 1.6909, 1.6766], - [1.9303, 1.8135, 1.4340, ..., 1.6055, 1.6055, 1.5913], - ..., - [0.9522, 1.0252, 0.7479, ..., 1.3069, 1.3069, 1.3354], - [1.3026, 1.2734, 1.2588, ..., 1.7193, 1.7477, 1.7477], - [1.5508, 1.5362, 1.5508, ..., 1.7335, 1.7193, 1.7193]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img031_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 9190, 5612, 551, 279, - 9124, 2213, 315, 279, 2168, 624, 151652, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151653, 151645, 198, 151644, 77091, 198, 267, 287, - 10464, 93957, 45993, 1173, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, 267, 287, - 10464, 93957, 45993, 1173, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - ..., - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None)] ----------- -PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 13:06:02.340272124 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 0.413 seconds -time to initialize megatron (seconds): 7.457 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 13:06:09 -building llava-ov-1.5-4b model ... -3 False -3 False -33 FalseFalse - -33 False False - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -3 False -3 False -33 False False - -3 3False -False -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 False -False -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -3 False -3 False -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -3 False3 - False -33 FalseFalse - -3 False -3 False -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 False -False -33 FalseFalse - -3 False -3 False -33 False - False -33 FalseFalse - -33 False False - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - -3 False -3 False -3 False -3 False -33 FalseFalse - -3 False -3 False -33 FalseFalse - -33 FalseFalse - -33 FalseFalse - - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc2.linear.bias - module.adapter.layernorm.bias - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (8605.30, 8605.43) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:06:57 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 13:06:58 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (47788.52, 47788.53) - train/valid/test-data-iterators-setup ..........: (1018.18, 1018.19)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 13:06:58 -packing_seq_len:4096----<<<<<----4176 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/map_dataset.py", line 150, in __iter__ - for idx, (sample_idx, inner_sample) in enumerate( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/wrappers/base.py", line 182, in iter_ctx - x = next(it) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/megatron/energon/task_encoder/base.py", line 168, in seed_wrapper_generator - sample = next(it) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 201, in encode_sample - l_sample_packed = self.pack_selected_samples(l_Qwen2VLImageTaskSample) - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py", line 554, in pack_selected_samples - super().pack_selected_samples(samples), - File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/data/multimodal/task_encoder.py", line 380, in pack_selected_samples - raise ValueError(f"Packed sample exceeds the maximum sequence length of {packing_seq_len}: {samples}") -ValueError: Packed sample exceeds the maximum sequence length of 4096: [Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img000_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, - 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, 198, - 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), total_len=315, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - 64, 21206, 429, 2727, 7324, 2948, 389, 432, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7777, ..., 0.8803, 0.8661, 0.8803], - [-1.7923, -1.7923, -1.7923, ..., -1.4660, -1.4660, -1.4518], - ..., - [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], - [-1.7777, -1.7777, -1.7777, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 46]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img001_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 60424, - 264, 9814, 4008, 315, 279, 2661, 2168, 13, 151645, - 198, 151644, 77091, 198, 1782, 31535, 369, 16301, 55155, - 304, 6303, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 1782, 31535, 369, 16301, 55155, - 304, 6303, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img002_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 1782, 1656, 29428, - 14126, 22361, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 1782, 1656, 29428, - 14126, 22361, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img003_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 276, 2168, 315, 264, 41530, 15616, 28642, 448, - 52484, 12624, 151645]), total_len=183, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 276, 2168, 315, 264, 41530, 15616, 28642, 448, - 52484, 12624, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img004_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 6750, 264, 2797, 323, 63594, 12126, 315, - 279, 6548, 13, 151645, 198, 151644, 77091, 198, 1782, - 1901, 36780, 15273, 19818, 4621, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, 1782, - 1901, 36780, 15273, 19818, 4621, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.4048, 1.4048, 1.4048, ..., 1.7620, 1.7762, 1.7762], - [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], - [ 1.4632, 1.4632, 1.4632, ..., 1.8188, 1.8188, 1.8188], - ..., - [ 1.3026, 1.3026, 1.3026, ..., -1.0252, -1.0252, -1.0252], - [-0.1572, 0.6019, 0.4997, ..., -0.9399, -0.9399, -0.9683], - [-0.2740, -0.4784, 0.6457, ..., -0.9683, -0.9683, -0.9683]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img005_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, - 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 78243, 75105, 220, 23, - 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, - 6430, 12136, 56432, 3421, 1947, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 78243, 75105, 220, 23, - 6460, 6084, 10775, 448, 22279, 6430, 4367, 11, 389, - 6430, 12136, 56432, 3421, 1947, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.3835, -1.3835, -1.3835, ..., -0.5275, -0.5275, -0.5275], - [-1.3689, -1.3543, -1.3251, ..., -0.3853, -0.3995, -0.3995], - [-1.3835, -1.3689, -1.3689, ..., -0.4848, -0.4706, -0.4422], - ..., - [-1.3981, -1.4127, -1.4273, ..., -0.4137, -0.3995, -0.3995], - [-1.2375, -0.2594, -1.6609, ..., -0.5275, -0.5133, -0.5133], - [-1.1937, -0.3032, 0.3391, ..., -0.5417, -0.5417, -0.5417]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img006_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, - 264, 9814, 4008, 315, 279, 2168, 13, 151645, 198, - 151644, 77091, 198, 1782, 10550, 29674, 645, 12426, 448, - 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, - 259, 481, 36724, 481, 2953, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 1782, 10550, 29674, 645, 12426, 448, - 43001, 11, 81258, 11, 8887, 11, 323, 264, 6774, - 259, 481, 36724, 481, 2953, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.9893, -1.0039, -1.0039, ..., -0.2715, -0.2715, -0.2715], - [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2573], - [-1.0185, -1.0185, -1.0185, ..., -0.2715, -0.2715, -0.2715], - ..., - [-1.0915, -1.0915, -1.0769, ..., -0.2857, -0.3000, -0.3142], - [-1.1207, -1.1207, -1.1207, ..., -0.3853, -0.3568, -0.3568], - [-1.1061, -1.1061, -1.1061, ..., -0.3142, -0.3000, -0.3000]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img007_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 1782, 42646, 331, 595, 3821, 18, 21, 19, - 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, - 323, 264, 24461, 9509, 27860, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 1782, 42646, 331, 595, 3821, 18, 21, 19, - 15, 87, 5798, 304, 23387, 448, 24461, 9509, 6248, - 323, 264, 24461, 9509, 27860, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img008_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 67, 20367, 19684, 17, 220, 17, - 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, - 323, 18579, 71738, 11, 4024, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 67, 20367, 19684, 17, 220, 17, - 448, 45567, 33417, 11, 5796, 4171, 11, 18579, 1643, - 323, 18579, 71738, 11, 4024, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - [-0.7850, -0.7850, -0.7850, ..., 2.0464, 2.0464, 2.0464], - ..., - [-1.2959, -1.3397, -1.7923, ..., 2.0748, 2.0321, 2.1459], - [-1.4273, -1.7193, -0.9602, ..., 1.7620, 1.9184, 1.9610], - [-1.5003, -1.4419, -1.7923, ..., 1.9895, 2.0748, 2.0321]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img009_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, - 198, 151644, 77091, 198, 759, 75242, 12426, 11, 66274, - 32300, 4603, 32300, 4603, 1841, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 759, 75242, 12426, 11, 66274, - 32300, 4603, 32300, 4603, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-1.2375, -1.5733, -1.7923, ..., -0.8261, -0.6412, -0.6270], - [-1.7923, -1.7923, -1.7777, ..., -0.9683, -0.9683, -0.9683], - [-1.7923, -1.7923, -1.7631, ..., -0.9683, -0.9683, -0.9683], - ..., - [-1.7339, -1.5879, -1.4565, ..., 2.1459, 2.1459, 2.1459], - [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459], - [ 1.8865, 1.8865, 1.8865, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img010_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 64, 2766, 1809, 448, - 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, - 279, 18967, 13962, 32372, 4426, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 64, 2766, 1809, 448, - 279, 2168, 315, 279, 94635, 304, 279, 1917, 5700, - 279, 18967, 13962, 32372, 4426, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img011_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, - 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, - 151645, 198, 151644, 77091, 198, 1782, 11623, 369, 1435, - 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, - 15786, 369, 279, 5593, 364, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 1782, 11623, 369, 1435, - 17711, 448, 279, 2265, 4172, 4382, 369, 1435, 11, - 15786, 369, 279, 5593, 364, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3807, -1.3807, -1.3807], - ..., - [ 1.2442, 1.2442, 1.2442, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665], - [ 1.2588, 1.2588, 1.2588, ..., -1.3665, -1.3665, -1.3665]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img012_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 3838, 374, 304, 279, 6548, 30, 151645, - 198, 151644, 77091, 198, 5082, 352, 263, 6837, 39698, - 1899, 69105, 938, 81625, 819, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 5082, 352, 263, 6837, 39698, - 1899, 69105, 938, 81625, 819, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[-0.6536, -1.4419, -1.7923, ..., -1.4518, -1.4518, -1.4518], - [-1.7631, -1.7631, -1.7485, ..., -1.3096, -1.4233, -1.4802], - [-1.7631, -1.7631, -1.7631, ..., -1.4518, -1.4518, -1.4518], - ..., - [-1.7631, -1.7631, -1.7047, ..., -1.4518, -1.4518, -1.4518], - [-0.0113, 0.2953, -1.5733, ..., -1.3238, -1.3807, -1.3949], - [ 0.8063, -0.5368, -1.7777, ..., -1.4518, -1.4518, -1.4518]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img013_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 74785, - 279, 2168, 3529, 285, 974, 13, 151645, 198, 151644, - 77091, 198, 19789, 6078, 304, 279, 6083, 315, 264, - 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, - 5382, 3054, 2114, 10576, 902, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, 19789, 6078, 304, 279, 6083, 315, 264, - 4746, 10041, 18824, 3691, 4158, 7002, 1947, 38468, 369, - 5382, 3054, 2114, 10576, 902, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5771, 1.5771], - [ 1.9303, 1.9303, 1.9303, ..., 1.5771, 1.5629, 1.5487], - [ 1.3464, 1.3464, 1.3464, ..., 1.5771, 1.5771, 1.5629], - ..., - [-0.9310, -0.9310, -0.9018, ..., 0.9514, 1.9753, 1.9895], - [-0.7412, -0.7266, -0.7120, ..., -0.7834, -0.7408, -0.7408], - [-0.5806, -0.5660, -0.5368, ..., 0.9230, 2.0606, 2.0748]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img014_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 7985, - 264, 50537, 714, 38219, 12126, 315, 279, 6802, 13, - 151645, 198, 151644, 77091, 198, 2277, 481, 61794, 18342, - 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, - 311, 259, 481, 6470, 34957, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 2277, 481, 61794, 18342, - 6470, 10058, 369, 43816, 11, 75102, 11, 43816, 342, - 311, 259, 481, 6470, 34957, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], - [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7762, 1.7762], - [1.6676, 1.5946, 1.5362, ..., 1.7762, 1.7762, 1.7762], - ..., - [1.5508, 1.5508, 1.5508, ..., 1.7762, 1.7620, 1.7335], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.8281, 1.7114, 1.5800, ..., 2.1317, 2.1317, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img015_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 14593, 15478, 11, 825, 3338, - 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, - 481, 10658, 1419, 19507, 2062, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 14593, 15478, 11, 825, 3338, - 389, 279, 2115, 323, 1378, 3338, 389, 279, 1290, - 481, 10658, 1419, 19507, 2062, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - ..., - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175], - [1.9011, 1.9011, 1.9011, ..., 2.1175, 2.1175, 2.1175]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img016_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 35127, 264, 2805, 323, - 2797, 16148, 315, 279, 17420, 2168, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 500, 5064, 5750, 11, - 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, - 28451, 22601, 11, 12188, 22601, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 500, 5064, 5750, 11, - 12188, 4935, 2752, 11, 9317, 4935, 2752, 11, 12188, - 28451, 22601, 11, 12188, 22601, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img017_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 1782, 6623, 60037, - 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, - 738, 448, 19752, 3931, 48051, 151645]), total_len=195, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 1782, 6623, 60037, - 19805, 10058, 448, 4778, 19752, 3931, 26210, 323, 26906, - 738, 448, 19752, 3931, 48051, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img018_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151653, 198, 60424, 264, 9814, 4008, 315, 279, 2661, - 2168, 13, 151645, 198, 151644, 77091, 198, 2024, 3045, - 7821, 2103, 488, 11030, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, 2024, 3045, - 7821, 2103, 488, 11030, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [-0.2886, -0.2302, 0.0179, ..., 2.0890, 2.1175, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img019_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 35127, - 264, 2805, 323, 2797, 16148, 315, 279, 17420, 2168, - 13, 151645, 198, 151644, 77091, 198, 64, 2781, 504, - 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, - 389, 264, 7002, 4034, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 64, 2781, 504, - 279, 70454, 8260, 25484, 6254, 304, 2155, 8964, 7249, - 389, 264, 7002, 4034, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., -0.5986, -0.5986, -0.6270], - ..., - [-1.2083, -1.2083, -1.2083, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [ 1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img020_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, - 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, - 198, 1782, 1081, 72725, 550, 315, 6470, 296, 377, - 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, - 29724, 315, 279, 93767, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 1782, 1081, 72725, 550, 315, 6470, 296, 377, - 258, 3179, 448, 279, 3421, 315, 279, 2311, 279, - 29724, 315, 279, 93767, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img021_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 7985, 264, 50537, 714, - 38219, 12126, 315, 279, 6802, 624, 151652, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, - 198, 151644, 77091, 198, 1782, 3691, 1555, 4013, 281, - 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, - 1555, 2550, 2355, 8149, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 1782, 3691, 1555, 4013, 281, - 18, 29659, 81516, 2355, 60066, 13352, 448, 264, 3175, - 1555, 2550, 2355, 8149, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img022_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 21195, - 264, 16830, 4008, 315, 279, 6548, 594, 1376, 4419, - 13, 151645, 198, 151644, 77091, 198, 19789, 87361, 14201, - 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, - 2472, 315, 1059, 59511, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, 19789, 87361, 14201, - 448, 825, 315, 1105, 12233, 16971, 82514, 2163, 279, - 2472, 315, 1059, 59511, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[-1.2083, -1.2083, -1.2083, ..., -0.9256, -0.9256, -0.9399], - [-1.1207, -1.1207, -1.0915, ..., -0.8545, -0.8688, -0.8688], - [-1.2959, -1.2959, -1.2813, ..., -1.0821, -1.0678, -1.0678], - ..., - [-0.7704, -0.7850, -0.7704, ..., -0.4990, -0.4990, -0.4990], - [-0.7558, -0.7704, -0.7850, ..., -0.3142, -0.3142, -0.2857], - [-0.8142, -0.8142, -0.8142, ..., -0.2715, -0.2431, -0.2289]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img023_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 3838, 374, 304, 279, - 6548, 5267, 151652, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 18824, 315, 13779, 9317, 1173, - 553, 9080, 567, 84946, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 18824, 315, 13779, 9317, 1173, - 553, 9080, 567, 84946, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.9303, 1.9303, 1.9303, ..., -0.7123, -0.7266, -0.7266], - [ 1.9303, 1.9303, 1.9303, ..., -0.7834, -0.7977, -0.7977], - [-1.0039, -1.3251, -1.2083, ..., -0.7977, -0.7692, -0.7408], - ..., - [ 1.0836, 1.1128, 1.2442, ..., 2.0890, 2.1032, 2.1175], - [ 0.9814, 0.9814, 0.9814, ..., 2.1175, 2.1175, 2.1317], - [ 1.3464, 1.4048, 1.4924, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 26]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img024_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 64, 22360, 3745, 448, - 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, - 4158, 67590, 3745, 26334, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 64, 22360, 3745, 448, - 264, 67590, 5383, 304, 6651, 323, 264, 3691, 323, - 4158, 67590, 3745, 26334, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.7260, 1.7260, 1.7260, ..., 1.9042, 1.9042, 1.9042], - [1.7260, 1.7260, 1.7260, ..., 1.8899, 1.8899, 1.8899], - [1.6822, 1.6822, 1.6822, ..., 1.8473, 1.8473, 1.8473], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img025_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, - 198, 151644, 77091, 198, 64, 3691, 16049, 8511, 25279, - 11, 825, 7100, 962, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, 64, 3691, 16049, 8511, 25279, - 11, 825, 7100, 962, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.7844, 1.7990, 1.8135, ..., 2.0179, 2.0464, 2.0748], - [1.8719, 1.8719, 1.8573, ..., 2.0890, 2.1459, 2.1459], - [1.8427, 1.8427, 1.8573, ..., 2.0606, 2.0606, 2.0748], - ..., - [1.7990, 1.7698, 1.7844, ..., 2.0606, 2.0606, 2.0606], - [1.8427, 1.8427, 1.8427, ..., 2.0321, 2.0321, 2.0464], - [1.7990, 1.7844, 1.7990, ..., 2.0890, 2.1032, 2.1032]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img026_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 21195, 264, 16830, 4008, - 315, 279, 6548, 594, 1376, 4419, 624, 151652, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151653, - 151645, 198, 151644, 77091, 198, 1151, 6918, 5636, 304, - 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, - 11, 9710, 5099, 8166, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, 1151, 6918, 5636, 304, - 9710, 2472, 96440, 11, 5570, 539, 11, 293, 32600, - 11, 9710, 5099, 8166, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[ 1.7406, 1.7406, 1.7406, ..., 2.0037, 2.0037, 2.0037], - [ 1.7406, 1.7406, 1.6968, ..., 2.0606, 2.0464, 2.0890], - [ 1.7406, 1.7406, 1.7406, ..., 1.7620, 1.5344, 1.5060], - ..., - [ 1.7406, 1.7406, 1.7406, ..., 2.1459, 2.0321, 2.0748], - [ 0.2223, -0.3616, 0.6165, ..., 2.1459, 2.0037, 2.1317], - [ 0.3975, -0.0842, -0.0696, ..., 2.1459, 2.0748, 2.0890]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img027_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 74785, 279, 2168, 3529, - 285, 974, 624, 151652, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151653, 151645, 198, 151644, 77091, - 198, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, - 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, - 2953, 3198, 52679, 19125, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 70, 63212, 17553, 16049, 8511, 52679, 19352, 22011, - 17553, 2562, 52679, 369, 2953, 22011, 17553, 19125, 369, - 2953, 3198, 52679, 19125, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img028_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 12115, 264, 63594, 22845, - 315, 279, 2168, 3897, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 64, 5220, 374, 12233, 458, 220, - 23, 15, 82, 259, 481, 15478, 448, 458, 2168, - 315, 264, 46652, 1841, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 64, 5220, 374, 12233, 458, 220, - 23, 15, 82, 259, 481, 15478, 448, 458, 2168, - 315, 264, 46652, 1841, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 1.6055, 1.6055, 1.6055], - [1.9303, 1.9303, 1.9303, ..., 2.1459, 2.1459, 2.1459], - ..., - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317], - [1.3756, 1.4486, 1.6676, ..., 2.1175, 2.1459, 2.1459], - [1.9303, 1.9303, 1.9303, ..., 2.1317, 2.1317, 2.1317]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img029_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 60424, 264, 9814, 4008, - 315, 279, 2661, 2168, 624, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 151645, 198, - 151644, 77091, 198, 1782, 9382, 642, 56727, 11, 448, - 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, - 1632, 438, 1008, 12396, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, 1782, 9382, 642, 56727, 11, 448, - 279, 9382, 642, 389, 264, 5312, 19059, 11, 438, - 1632, 438, 1008, 12396, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - ..., - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802], - [-1.7923, -1.7923, -1.7923, ..., -1.4802, -1.4802, -1.4802]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img030_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 151652, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151653, 198, 3838, - 374, 304, 279, 6548, 30, 151645, 198, 151644, 77091, - 198, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, - 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, - 49496, 8968, 52001, 8778, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, 22028, 23790, 17553, 10900, 1778, 1424, 52394, 22786, - 311, 2338, 17899, 3460, 8654, 22786, 16882, 17899, 11153, - 49496, 8968, 52001, 8778, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.9303, 1.7114, 1.4632, ..., 1.6482, 1.6482, 1.6624], - [1.3318, 1.3318, 1.3318, ..., 1.6909, 1.6909, 1.6766], - [1.9303, 1.8135, 1.4340, ..., 1.6055, 1.6055, 1.5913], - ..., - [0.9522, 1.0252, 0.7479, ..., 1.3069, 1.3069, 1.3354], - [1.3026, 1.2734, 1.2588, ..., 1.7193, 1.7477, 1.7477], - [1.5508, 1.5362, 1.5508, ..., 1.7335, 1.7193, 1.7193]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 24, 24]]), video_grid_thw=None), Qwen2VLImageTaskSample(__key__='pretrain-000002.tar/ps_00013874.img031_jpg', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, num_tiles=[1], tokens=tensor([151644, 8948, 198, 2610, 525, 264, 10950, 17847, 13, - 151645, 198, 151644, 872, 198, 9190, 5612, 551, 279, - 9124, 2213, 315, 279, 2168, 624, 151652, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, 151655, - 151655, 151653, 151645, 198, 151644, 77091, 198, 267, 287, - 10464, 93957, 45993, 1173, 151645]), total_len=194, labels=tensor([ -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, -100, -100, - -100, -100, -100, -100, -100, -100, -100, 267, 287, - 10464, 93957, 45993, 1173, 151645]), attn_mask=tensor([False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False, False, False, False, False, False, False, - False, False, False, False]), imgs=[tensor([[1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - ..., - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464], - [1.8281, 1.8281, 1.8281, ..., 2.0464, 2.0464, 2.0464]])], pixel_values_videos=None, image_grid_thw=tensor([[ 1, 26, 24]]), video_grid_thw=None)] ----------- -PackedCaptioningSample(__key__='pretrain-000002.tar/ps_00013874', __restore_key__=('MapDataset', 0, 'Webdataset', 'pretrain-000002.tar', 0), __subflavor__=None, __subflavors__={}, images=[ AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.026 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 13:26:14.872313642 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 0.410 seconds -time to initialize megatron (seconds): 4.290 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 13:26:17 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias - module.adapter.layernorm.bias - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (6330.73, 6330.82) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:27:03 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 13:27:04 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (45737.89, 45737.93) - train/valid/test-data-iterators-setup ..........: (849.20, 849.21) -training ... -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 13:27:04 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img020_jpg from 194 to fit remaining capacity 114. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img016_jpg from 265 to fit remaining capacity 22. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img013_jpg from 299 to fit remaining capacity 221. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision\n t = (t * cos_) + (_rotate_half(t) * sin_)\n', 'RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0\n'] -WARNING:megatron.core.utils:The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision\n t = (t * cos_) + (_rotate_half(t) * sin_)\n', 'RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision -[rank1]: t = (t * cos_) + (_rotate_half(t) * sin_) -[rank1]: RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 241, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 32, in apply_rotary_pos_emb_vision -[rank0]: t = (t * cos_) + (_rotate_half(t) * sin_) -[rank0]: RuntimeError: The size of tensor a (12848) must match the size of tensor b (19328) at non-singleton dimension 0 -[rank1]:[W1224 13:27:07.107477592 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 13:27:12.100223646 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 13:27:14.317000 2671643 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2671665 closing signal SIGTERM -E1224 13:27:14.532000 2671643 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2671666) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_13:27:14 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2671666) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log deleted file mode 100644 index 4e68402f..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:40:22_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1038 +0,0 @@ -W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] -W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] ***************************************** -W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1224 13:40:24.129000 2673230 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 4096 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 0 -0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.026 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 13:40:32.550238037 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.347 seconds -time to initialize megatron (seconds): 3.307 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 13:40:34 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 3False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False3 - False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.bias - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (7411.22, 7411.28) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:41:20 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 13:41:21 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (45636.49, 45637.28) - train/valid/test-data-iterators-setup ..........: (967.35, 967.44)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 13:41:21 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img020_jpg from 194 to fit remaining capacity 114. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img016_jpg from 265 to fit remaining capacity 22. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0)\n', 'RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision -[rank1]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0) -[rank1]: RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 -WARNING:megatron.core.utils:repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0)\n', 'RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1\n'] -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img013_jpg from 299 to fit remaining capacity 221. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 282, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 60, in apply_rotary_pos_emb_vision -[rank0]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor, dim=0) -[rank0]: RuntimeError: repeats must have the same size as input along dim, but got repeats.size(0) = 32 and input.size(0) = 1 -[rank1]:[W1224 13:41:23.232043330 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 13:41:28.264405723 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 13:41:30.309000 2673230 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2673245 closing signal SIGTERM -E1224 13:41:30.524000 2673230 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2673246) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_13:41:30 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2673246) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log deleted file mode 100644 index c9f1c9e7..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-24_13:47:56_tp2_pp1_seqlen4096_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1038 +0,0 @@ -W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] -W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] ***************************************** -W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1224 13:47:58.055000 2674551 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 4096 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 4096 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.028 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 13:48:06.611399525 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.221 seconds -time to initialize megatron (seconds): 3.304 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 13:48:08 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False3 False - -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 3 False -False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.layernorm.bias - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc2.linear.bias - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (6690.53, 6690.67) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 13:48:52 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 13:48:53 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (43774.48, 43778.90) - train/valid/test-data-iterators-setup ..........: (842.12, 859.14) -training ... -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 13:48:53 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img020_jpg from 194 to fit remaining capacity 114. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img016_jpg from 265 to fit remaining capacity 22. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img013_jpg from 299 to fit remaining capacity 221. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision -[rank1]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) -[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision -[rank0]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) -[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.56 GiB is free. Including non-PyTorch memory, this process has 9.99 GiB memory in use. Of the allocated memory 8.24 GiB is allocated by PyTorch, and 863.13 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -[rank1]:[W1224 13:48:56.545373505 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 13:49:01.858122644 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 13:49:02.846000 2674551 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2674566 closing signal SIGTERM -E1224 13:49:03.011000 2674551 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2674567) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_13:49:02 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2674567) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log deleted file mode 100644 index 1bbc0001..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:20:51_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1038 +0,0 @@ -W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] -W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] ***************************************** -W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1224 14:20:52.982000 2676720 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 1024 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 1024 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.024 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 14:21:01.966538345 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 0.355 seconds -time to initialize megatron (seconds): 4.533 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 14:21:05 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False3 False - -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc2.linear.weight - module.adapter.layernorm.weight - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias - module.adapter.layernorm.bias -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (6866.03, 6866.22) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:21:50 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 14:21:51 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (45766.41, 45766.42) - train/valid/test-data-iterators-setup ..........: (860.32, 860.33)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 14:21:51 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img004_jpg from 195 to fit remaining capacity 160. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img004_jpg from 265 to fit remaining capacity 130. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img003_jpg from 299 to fit remaining capacity 139. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision -[rank1]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) -[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 1 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision\n cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0)\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 319, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 93, in apply_rotary_pos_emb_vision -[rank0]: cos_ = torch.repeat_interleave(cos_seg, counts_tensor.to(cos_seg.device), dim=0) -[rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 89.07 GiB. GPU 0 has a total capacity of 39.56 GiB of which 29.68 GiB is free. Including non-PyTorch memory, this process has 9.87 GiB memory in use. Of the allocated memory 8.13 GiB is allocated by PyTorch, and 857.50 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -[rank1]:[W1224 14:21:54.192514656 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 14:22:00.551070334 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 14:22:01.564000 2676720 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2676735 closing signal SIGTERM -E1224 14:22:01.728000 2676720 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2676736) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_14:22:01 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2676736) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log deleted file mode 100644 index d10d8c2f..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:23:14_tp2_pp1_seqlen1024_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1038 +0,0 @@ -W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] -W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] ***************************************** -W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1224 14:23:15.893000 2677183 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 1024 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 1024 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.026 seconds -> compiling and loading fused kernels ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 14:23:24.177105851 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.296 seconds -time to initialize megatron (seconds): 3.627 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 14:23:27 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 3 False -False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.bias - module.adapter.linear_fc2.linear.weight - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (6575.26, 6575.33) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:24:11 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 14:24:11 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (43559.41, 43564.47) - train/valid/test-data-iterators-setup ..........: (849.64, 849.72)training ... - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 14:24:11 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img004_jpg from 195 to fit remaining capacity 160. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img004_jpg from 265 to fit remaining capacity 130. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img003_jpg from 299 to fit remaining capacity 139. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0\n'] -WARNING:megatron.core.utils:The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision -[rank1]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) -[rank1]: RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision -[rank0]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) -[rank0]: RuntimeError: The size of tensor a (1440) must match the size of tensor b (2048) at non-singleton dimension 0 -[rank1]:[W1224 14:24:14.066149140 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 14:24:19.100043220 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 14:24:21.085000 2677183 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2677198 closing signal SIGTERM -E1224 14:24:21.250000 2677183 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2677199) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_14:24:21 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2677199) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log deleted file mode 100644 index d4d051ea..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:24:51_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1038 +0,0 @@ -W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] -W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] ***************************************** -W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1224 14:24:53.245000 2677719 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 2048 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 2048 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.025 seconds -> compiling and loading fused kernels ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 14:25:00.304975724 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.346 seconds -time to initialize megatron (seconds): 4.384 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 14:25:04 -building llava-ov-1.5-4b model ... -33 False - False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -33 False - False -3 False -3 False -3 False -3 False -3 False -3 False -3 3 False -False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.layernorm.bias - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc2.linear.weight - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (4443.62, 4443.67) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:25:46 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 14:25:47 -done with setup ... -training ...(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (41972.18, 41972.22) - train/valid/test-data-iterators-setup ..........: (840.93, 840.94) - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 14:25:47 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img010_jpg from 195 to fit remaining capacity 14. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img008_jpg from 265 to fit remaining capacity 94. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img006_jpg from 299 to fit remaining capacity 266. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision -[rank1]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) -[rank1]: RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 -WARNING:megatron.core.utils:The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision\n out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk)\n', 'RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0\n'] -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 335, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 76, in apply_rotary_pos_emb_vision -[rank0]: out[start:end] = (slice_t * cos_chunk) + (_rotate_half(slice_t) * sin_chunk) -[rank0]: RuntimeError: The size of tensor a (848) must match the size of tensor b (2048) at non-singleton dimension 0 -[rank1]:[W1224 14:25:49.461266190 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 14:25:55.544067802 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 14:25:56.532000 2677719 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2677734 closing signal SIGTERM -E1224 14:25:56.747000 2677719 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2677735) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_14:25:56 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2677735) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log b/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log deleted file mode 100644 index a40f7bf7..00000000 --- a/stage_1_alignment_llava_ov_4b/run_2025-12-24_14:28:30_tp2_pp1_seqlen2048_mbs1_gbs2_5steps.log +++ /dev/null @@ -1,1058 +0,0 @@ -W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] -W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] ***************************************** -W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W1224 14:28:32.170000 2678274 site-packages/torch/distributed/run.py:803] ***************************************** -False -False -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/utils.py:20: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_l2norm - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/optimizer.py:28: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/optimizer/clip_grads.py:29: UserWarning: Transformer Engine and Apex are not installed. Falling back to local implementations of multi_tensor_applier, multi_tensor_l2norm, and multi_tensor_scale - warnings.warn( -INFO:datasets:PyTorch version 2.9.1 available. -INFO:datasets:PyTorch version 2.9.1 available. -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:TE_DType import failed, FP8 communication will be disabled: No module named 'transformer_engine.pytorch' -WARNING:megatron.core.transformer.moe.fused_a2a:Float8BlockQuantizer, Float8BlockwiseQTensor import failed, FP8 not available: No module named 'transformer_engine.pytorch' --------------- Configure model to llava-ov-1.5-4b -------------- - num_layers = 36 - hidden_size = 2560 - ffn_hidden_size = 9728 - num_attention_heads = 32 - group_query_attention = True - num_query_groups = 8 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = True - vocab_size_in_config_file = 151936 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-06 - rotary_base = 5000000 - kv_channels = 128 - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 2, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 2, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:741: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... True - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.flash - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... qwen2-vl - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_llava_ov_4b/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 36 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 2048 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 2 - ffn_hidden_size ................................. 9728 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 2 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 - hidden_dropout .................................. 0 - hidden_size ..................................... 2560 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 128 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 5 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-1.5-4b - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-06 - normalization ................................... RMSNorm - num_attention_heads ............................. 32 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 36 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 8 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 1 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... None - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 4 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 5000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_llava_ov_4b - save_ema ........................................ None - save_interval ................................... 2000 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 2048 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 2 - tensorboard_dir ................................. stage_1_alignment_llava_ov_4b/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 5 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 4096 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. True - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 151936 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 2 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 2 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -WARNING: tokenizer already has an EOS token, replace <|im_end|> with <|im_end|>, and will add 0 new tokens to tokenizer. - > padded vocab (size: 151936) with 128 dummy tokens (new size: 152064) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 2 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -make: Nothing to be done for 'default'. -make: Leaving directory '/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.026 seconds -> compiling and loading fused kernels ... -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W1224 14:28:39.475120241 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 ->>> done with compiling and loading fused kernels. Compilation time: 0.344 seconds -time to initialize megatron (seconds): 3.439 -/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2025-12-24 14:28:42 -building llava-ov-1.5-4b model ... -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 3 False -False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False3 False - -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False -3 False - > number of parameters on (tensor, pipeline) model parallel rank (1, 0): 4201988608 - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4201988608 -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (27271680 elements, 27271680 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.bias - module.adapter.layernorm.weight -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine - loading release checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 - successfully loaded checkpoint from /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1 [ t 1/2, p 1/1 ] at iteration 0 -(min, max) time across ranks (ms): - load-checkpoint ................................: (6846.22, 6846.29) -[after model, optimizer, and learning rate scheduler are built] datetime: 2025-12-24 14:29:27 -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 10 - validation: 200 - test: 200 -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -You have video processor config saved in `preprocessor.json` file which is deprecated. Video processor configs should be saved in their own `video_preprocessor.json` file. You can rename the file or load and save the processor back which renames it automatically. Loading from `preprocessor.json` will be removed in v5.0. -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_00/train_dataloader_dprank000.pt does not exist -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 5058), pretrain-000001.tar[0, 5036), pretrain-000002.tar[0, 5039), pretrain-000004.tar[0, 3821), pretrain-000000.tar[0, 5034)] sum(count)=23988 -dataset state stage_1_alignment_llava_ov_4b/dataloader/iter_0000000/mp_rank_01/train_dataloader_dprank000.pt does not exist -[after dataloaders are built] datetime: 2025-12-24 14:29:28 -done with setup ... -training ...(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (44510.70, 44515.78) - train/valid/test-data-iterators-setup ..........: (847.39, 868.47) - -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2025-12-24 14:29:28 -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000002.tar/ps_00013874.img010_jpg from 195 to fit remaining capacity 14. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00000708.img008_jpg from 265 to fit remaining capacity 94. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -WARNING:megatron.core.utils:Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. -cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 - 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 - 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 - 577 577 577 577 625 577 577 577 577 625 577 625 577 577 - 577 577 577 625]. -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision\n raise RuntimeError(\n', 'RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328.\ncu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971\n 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991\n 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577\n 577 577 577 577 625 577 577 577 577 625 577 625 577 577\n 577 577 577 625].\n'] -[rank0]: Traceback (most recent call last): -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank0]: main() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank0]: trainer.train() -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank0]: pretrain( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank0]: iteration, num_floating_point_operations_so_far = train( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank0]: train_step(forward_step_func, -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank0]: losses_reduced = forward_backward_func( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank0]: output_tensor, num_tokens = forward_step( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank0]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank0]: output_tensor = model( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank0]: return self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank0]: outputs = self.module(*inputs, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank0]: image_embeddings, window_index = self.vision_model(images, \ -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward -[rank0]: x = self.decoder( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank0]: hidden_states = self._checkpointed_forward( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank0]: hidden_states, context = checkpoint_handler( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank0]: return tensor_parallel.checkpoint( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank0]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank0]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank0]: outputs = run_function(*args) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank0]: hidden_states, context = layer( -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank0]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank0]: attention_output_with_bias = self.self_attention( -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank0]: return self._call_impl(*args, **kwargs) -[rank0]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank0]: return forward_call(*args, **kwargs) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank0]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank0]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision -[rank0]: raise RuntimeError( -[rank0]: RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. -[rank0]: cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 -[rank0]: 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 -[rank0]: 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 -[rank0]: 577 577 577 577 625 577 577 577 577 625 577 625 577 577 -[rank0]: 577 577 577 625]. -WARNING:megatron.core.utils:Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. -cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 - 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 - 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 - 577 577 577 577 625 577 577 577 577 625 577 625 577 577 - 577 577 577 625]. -['Traceback (most recent call last):\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step\n output_tensor = model(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward\n return self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward\n outputs = self.module(*inputs, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward\n image_embeddings, window_index = self.vision_model(images, \\\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward\n x = self.decoder(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward\n hidden_states = self._checkpointed_forward(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward\n hidden_states, context = checkpoint_handler(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler\n return tensor_parallel.checkpoint(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint\n return CheckpointFunction.apply(function, distribute_saved_activations, *args)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward\n outputs = run_function(*args)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward\n hidden_states, context = layer(\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward\n attention_output_with_bias = self.self_attention(\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n', ' File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl\n return forward_call(*args, **kwargs)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward\n query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q)\n', ' File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision\n raise RuntimeError(\n', 'RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328.\ncu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971\n 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991\n 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577\n 577 577 577 577 625 577 577 577 577 625 577 625 577 577\n 577 577 577 625].\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 21, in -[rank1]: main() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py", line 17, in main -[rank1]: trainer.train() -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/megatron_trainer.py", line 48, in train -[rank1]: pretrain( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 282, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 1059, in train -[rank1]: train_step(forward_step_func, -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/training_utils.py", line 550, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 464, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/pipeline_parallel/schedules.py", line 286, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py", line 261, in forward_step -[rank1]: output_tensor = model( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/distributed/data_parallel_base.py", line 23, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/legacy/model/module.py", line 189, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py", line 325, in forward -[rank1]: image_embeddings, window_index = self.vision_model(images, \ -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 366, in forward -[rank1]: x = self.decoder( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 105, in forward -[rank1]: hidden_states = self._checkpointed_forward( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 248, in _checkpointed_forward -[rank1]: hidden_states, context = checkpoint_handler( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 230, in checkpoint_handler -[rank1]: return tensor_parallel.checkpoint( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 437, in checkpoint -[rank1]: return CheckpointFunction.apply(function, distribute_saved_activations, *args) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/autograd/function.py", line 581, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/tensor_parallel/random.py", line 371, in forward -[rank1]: outputs = run_function(*args) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/qwen_vl/vision_transformer_block.py", line 196, in custom_forward -[rank1]: hidden_states, context = layer( -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 840, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/transformer_layer.py", line 487, in forward -[rank1]: attention_output_with_bias = self.self_attention( -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/transformer/attention.py", line 449, in forward -[rank1]: query = self.apply_rotary_fn(query, q_pos_emb, config=self.config, cu_seqlens=cu_seqlens_q) -[rank1]: File "/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/rice_vision_model.py", line 71, in apply_rotary_pos_emb_vision -[rank1]: raise RuntimeError( -[rank1]: RuntimeError: Token length mismatch: t.shape[0]=6992 != total_tokens(from cu_seqlens)=19328. -[rank1]: cu_seqlens=[ 0 1105 1682 2259 2836 3461 4038 4615 5192 5769 6394 6971 -[rank1]: 7548 8173 8750 9327 9904 10481 11058 11683 12260 12837 13414 13991 -[rank1]: 14616 15193 15818 16395 16972 17549 18126 18703 19328], counts=[1105 577 577 577 625 577 577 577 577 625 577 577 625 577 -[rank1]: 577 577 577 577 625 577 577 577 577 625 577 625 577 577 -[rank1]: 577 577 577 625]. -WARNING:aiak_training_llm.data.multimodal.task_encoder:Truncating sample pretrain-000004.tar/ps_00019328.img006_jpg from 299 to fit remaining capacity 266. -WARNING:aiak_training_llm.data.multimodal.task_encoder:No remaining packing capacity; stopping packing of further samples. -[rank1]:[W1224 14:29:30.229872060 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank0]:[W1224 14:29:35.162733945 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W1224 14:29:37.154000 2678274 site-packages/torch/distributed/elastic/multiprocessing/api.py:908] Sending process 2678289 closing signal SIGTERM -E1224 14:29:37.318000 2678274 site-packages/torch/distributed/elastic/multiprocessing/api.py:882] failed (exitcode: 1) local_rank: 1 (pid: 2678290) of binary: /home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/python3.10 -Traceback (most recent call last): - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/bin/torchrun", line 7, in - sys.exit(main()) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 357, in wrapper - return f(*args, **kwargs) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 936, in main - run(args) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/run.py", line 927, in run - elastic_launch( - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 156, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/home/rana.zayed/.conda/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 293, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -============================================================ -/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/aiak_training_llm/train.py FAILED ------------------------------------------------------------- -Failures: - ------------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2025-12-24_14:29:37 - host : gpu-51 - rank : 1 (local_rank: 1) - exitcode : 1 (pid: 2678290) - error_file: - traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html -============================================================ diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766400270.gpu-52.3342268.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766400270.gpu-52.3342268.0 deleted file mode 100644 index 83ec48654cb8825f5c98de3a19a2a970208bf7ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48602 zcmb__2Y6If^EVy#(xfJ|gkF=71Ofz*DhPr!QAEUbv%5)_Y|EBVM8$$g6BWgR9Z|4> zND&1)uYjT=3Zf{AC|E$SD~R9BxpVfOxw{v4`G3*J*9Y_d&fGb3rq4MeD(OFG?i}%; z?O^TALq|;bba3;!MIndBpEIn&?T_@$pOB+vMcn>yAXF4^gj_k{sL$sJ&CHn|@z~v6v zQ$(G!r})Yd>bEa=fl$xa`e~Ui_t{bF#b~WICg67))P8d0kG_Kmx6H~)AAFu$Nzt^)Y)lf}ab@P}_p*V=xSfPJ+-TFc^Mzdaey!ru9P z4A7On6ImA$=u2;&T5fx215w#u>#Ma!UOFPM{);@mN zJlnL51Upi@P|GkNiyY1}x8KEC&?3gu27@CJ{}BCma*DlXZc1XNI+5oL>NhlBv z+RLNv&`g6RoNi_7CWfMlENn^3=rNR_pP$>|5!gJzVo9eZ0}0+A=)6VeLS)~GBao`$e}<`;l{fyUmZPxz&rdhC94rg;j4h=*hY@1$n}JaFBt>BXvp=v+dJ%Favgb- zP*JX%A*$US412r*KL=H|FVCHwt9Tr~te-|Oxz2ctAcb5zf;9CC*}uBgLncLu>N}2PyPBJoXZ|H<~QsmaKML8M9mbPi1zvJn9I!U3Sp4NkhIh?o5mR zMCA5OD<;@pmymf{cdb!)x~IeprW|K(aC7haUz34^HQrs}u$8<`mIR*o!=VRQBj2aV9FLoX7)b%BC^pwo{@CcY8`o&BoAjq1BxioM76bdJS zjF?1u&g?25GZWQ(CxN4hQd~V3XgeUE<0{ULVox3|>U25to!^j{=sUL{H#Y^q0Ue$n z%K&}v5RnMjOlPMZe0_(t3d>a$@H?xv-puSp<97)V?L{Uy5+D+(!;a6Iu!s5h`y?#- z_^rTGJ_peEo^{{H^!XFJh*s3+?NutlggQg^aAc;}bTk^q9Ew_?jnGeu7@GHkaX*~T z!n|wVqXjU`YwGlRg7%V-`CO>hL1pb@*Z%| z1i`0r9PIc7R@zn0SCPLJnUL!YwOI zWp}uyu08z2&dl?7enh0A=ie1hc*qUjvI1=P6x;o7w<~$4bnv*6&sm_TeN1bJKrsXv z10X))Mw$0Vy$(o@n6!#QesR?6H8`F3_r$Ll`RgP!lHbi)>UNgdivy74aG<+u_XfgY zVHe_$m$C*oJGVIJ7c%mv?N!KcW2_Mlddk4PfhD-)DZ*^B1=XsI89JKI+HW5{_K2bpmg59ZSA-XkzzIpu0fZ*zQt)lH z6qpB#{8gEHzx-389@Tr}U?_koKMrQSkMHih;ymU{Rr-w3L>7 zOf@#`ne4lAE~8BwlYMBNt_JDQltVX*c~9BY&9J|!ELE>JZ4beK7=?#VLu6TQE&F5859dE?Qo_shF z*+o%rSt4LdvhOLsCctq2wZ@$uPp~EKcTaWg&e*Z?8=@E4(NVIaC=iH%&2^xcmLQ3* z!26v}#m9N~i-y0k(>VHDg2&U?4poOU7zOVJlGhe#AO)LC-e33IhB54)RqT##UobM$ z9`(Z-vGeYQs+E@?NyXYKCg$Q#zN*2F%Y8?PZjlUfxxpUzfa-9>;{>1GWm47F^0BNu zvx<|%@A4^lOM3a&qxcrevo?xt6_?^P4)*v+ChJZ(dCm0C*4l15M$`_}&evKC=&+~655Xc-=3HKP(mp}M zQv>@>lgKw5E3suACuF-e6%3$JlZJG(3AIPez;*$D62gu=JE)ZsKL*uyuh&uG0K;kT zSp+&DR@pY(wK*$Geq`)ADpwZl@;Sm~pi9Dm;s};yMEi+8`aghd5Q zf1-~PHHRV(oNLUbd2`u{@sn5%1nc1%>PF{Nq}32V^r>MkD7JiRX^sPOc#up<}^MJTt!BX^AM`euNK}Li6J^FR zyS`ET{mgLhl5oOB4@fxS8U%`qy#aGN(VEo1559Jc8N2e|iBvRpoeXTkwHHIoh1C|$ z3njY9Q}dPcPnFzt1?i?nf%UK2(*Gd%vD(F2F3N!(2pJ1u1_ynd6hGvpVHppy?9fYc zC6U}B_g_bjG0c@;YA?+rr^aP0 zHa{aFF*a`@A%hM``X!5b;OYJQm@<0!FCr6_QA?L2;y@8W?i5wxNF$@HlOZmhTW=i*6}|ljntyF68if-CnRNUZ{CR%?$*WeEBNx z+r>=gzm`i0nfF6z4vq}?lX}6=jx|OulGfYU`h)bD%m~aqMW2C2AOkAqQRtTf|Bi@H z7Po(=a1EA)>vWoMMHUXl!XwH^PG=PAN>qtYU=^)N?|Xae)vP__Be?{&hjezigAlNT zZ|bt+(egp;4*vvq+uY}4y~3NP4`0BTGwUCfISA`@mpGg=Q^dXYhf7ZnBH0n8hL&dt z7fTIIeUJ?X*QdmpEVHkB%~6wCn5_~IElg$%&%y;%TuLR3Inw=^XV_`H>R%#KoJIo_ z%BI%g1V7?|5n~vF6{s*K8}7H875#ysK3wS^y?ZbLs@dbWJat=# z93IHj_0YRg6~`t(@;smvptSA{-6c%;$_oCV&JH~xN~EMpw+QdQx2YlIYyQlMR-1H>CO zry~WZiY0>9V8FCRDJ%0Usm|A$yG;J7-SfMZTW9zAjUy;c7&> z$SOFakktc7xx0zlP8GKBjePls*)v$mwoOgK7AadZwvoA=~S6!R}hN)*uW;T1(RN@_Q(SvCDi>2Mhh&^@wy4 z`Uy=VJs&$15?r7fFyF}Gt#|aUR~q+W{BB-fS-n0oRu5jLA~{`?dpz7=Qie5!J}w`d(A4rpOx*nwzVnrP7)$evH|P zCJhJ)EprQRNN=TqkB@64QBa-GpR9Jw?_RunAB)pAOH_>0G68jZ7|aAV=_EpK5<4HY z-7|tv`h@-|)w9nSKIS*pX)&cC5iOdEVnipMKJ4l(uF1G?Mgj`|BnB8xDT8>lZ@=YC zIX~Kn$VBDbhI1h$|2(e1w;wD5eRsvfZzzL{L>?J(9^nq`f-Nt=4-iHBS^{G&8Jo2 zqN#~>-&329v5>8EI)RCh4gJNC$K^&}RD^>*=twSwI5vg!>>DdXA>PR-m%vWO?if&s zDn>AXrtbf+(9;(#YQ!YM-%OE+W;m=9g>1bmjy;ybyQRE!fF0t7%?VK)VgoS|(=AaS zZ*g8P#`D)CD)PJmcBpWz-6^KLTAejXZG-`Z@4tFA6FYYcB3zUPa0v@XLZLbz^tu6= z^ZOr}3{q>ULPA19po*Y{ldnK>*LnCuxA$X8_ZbO^N;h3XhRiMXmVjwfhD}&ON8^RJ zU(KBWwye7sg%M&!F#3+1SulI%>h5<>hu-^eq{8d#K65|G5RW!?oZbGx7a?AOI)Ij*Sc9R;qaCO zxa_3Iq`n^Z7{VA5rTUEXzH0rz$5;+O+Lk^@w6Ye1cNU5IO`1wEG^XyqHot7|!V(i{ z?FddJCLo*_6;;sA#*yMQTeAN5lQ-64l)fUNMYTb289T<3gwwip+rp9sj`l>UNEU#k zWB;BA36f|*;JL7&1qUBx{#%s}ge`(=lP!ckNK?w&n!gu(JGT>S_j_C-W4m8ABKsga z0El}2#Ne{{L(UuPWtw!^`wNr7aUJOcPzDQd84=@Qb%(NKB3*OV<&>S}>{{_+C?G1v%wi17FLqkcTT^uzhfZuGIUJ&bsLIVaNy}m3y7+*UYKM0JFKXjmKV_RawZk@t8nB%75s5CQcLgsTVYrY0 zA{Ux>ft*7q>WoA~NgKo`2TZZCtzDK&pMYE2#yDKqy6%klp$nHYJ0oVp8xt{sGRXQs zdP6h^8>g)HprVfbi22RiJ#(e4FppNs)pE23#lgH@9NAh*9$r>Dgk^~iNGOqoD=Ica zS@QVp%l`R+S;A}ci6~(Su~rCKS}d4?rM1TcThr*;_}#lq)>`!-WFc!}Dgq~UO*^Gf z%9Yg9k2JfAc`lzyJoH>LF>nVzK%cfqfUC6r_bMpp(M#Hy043(0s<%+oyXVwK<*dyut)b!5T93-JjdPW=V(aUIdFt zhqTg};9%iMs!2wbq==HbnetrWOU&x-mSDo_76*Ie!`{cuw{lXg1>gq9%J-PNKfO26 zDmo$bmLdI0g)%pd=*mIsWY<`9RsDS|iQK#ots#=g#vK;2`3r;?zbje%Pi8gEV6~E$ zB|g?lx;tbA9WFwUjRkt!1)i4pg_M2es_yaOH4ZUn_sT;04$)?Sy20-C*;zkSvfuDZ ziy7Cl$Mu-3Khl>77j;8<-^)2HqDcMNN00Ysf;zMx zErNpDB~~yL9R{%&D(TU=CF3tTzxhXX%$8x#O|d5g9(8P0tEnc-Wp3KI?G!7xej}F= z1y{KY+z0c2S>=_gFGg9b&-?-OF{0JSpjvc6S9n4ZY~P4|Rf5vAB7!Li4Be4#7>390Y8Y8GnN($R zOr3F?TXXh3%%9pJ;e|gXS+4k1$#VU+0+*k0z3c*&>n?XO9O~HEpj`};;K&h9xhA{D z9d%3oW&%@h7_ET<(?-V>D?vM*-3#5LrXW@^5{l+qZF{c<^SoSe;O__xu4d@^2ScrWHNv23%QbBw1<_TrV+vpesF-C0qD8J5#OW*FubMiaOULS zFp|Cmo%}ZP_+XmUXD@*{z2tK=W2y)Kv{U#cu5)R75kZUVToOTpZyqT%`oAE!G-ofZ zIrYiM56xgXl{e*bqTs3v<+*%E)xd~F%y@)G(RT>r0WpdgCDxf^*BvjMJ&GB{B*p*L zspXxROWJ%its`7gOqxM2Nr~M(1Fvv{4F`SCp{<#-WZ$9XER1jO{4{wzV2|X;+tkV988sb^! z1GpZ3q@wU?R;Hahj=lrSv@H>y+VG0QuB2t<{+5L+Sk~#a{JDeJ02oxU@k?KNra0oMKdQ2p2rtzm6X_&PQm`bhZpyJ-EQfO^+z{LR|LgMPPc*;$TR%DhtL<6_}`Y9{eUUWH8uNQ35MKQz`XL17m zVowP;H0B|F%O%zxdv6hQDvrq|gj0cf3A*2dK^)fgL0T%#^R!kS4_uwbyD@H`OwsbU3{6@@!)iJOg|EsyDl^w6wFtBQzDQGkRxx$fNRTul|d=X=ke&saOb zsj^m3tosQmkX-!ykfDqXze`YkS_*VJ-ux)8=dm_R+?6`D?1{eYn6Gvbq57Cu0Pgrg zYPri~eJUT=k+qEBO>`@GX>>boiTO|}Ti++IP*IdeYIm5$oQc0Ap6C;nIqQ^hCw#Sy z&m=A{COQSK%<3kA%gy|%=eD0<;qZtO!o_eHZr6fz9$fMP!lmC69LEVo#lw~Bh6V^+ z_}s6IvA;`T+^BTDM;PV@%tj}`8QH_uLI7U+>uB2;uS;aviFTN2(xpxlP9>{D(| zo^pMK|L0EkGYfQqhu}nM5ymf|rvkq@9#f$ZFpV#spL6R*ra_NLNO8Xg`oMbkib>WL zi>sjhd`0^0Y-1KqQ#J<7IOuQVo??%Apx9dCx0b)~81uSv$_Q9^T{^H)phL!To?1gc zc>Jw5*n!_Kp~Ym88%8?BKBaOkUil}7_ArO{7B7)29A3PV5ZXvMy_i)&C4fB*=0ubt z5nQvZzwSxi;nB@U3!#GTEJ7IFY(i?z;}zr4FsKtZ%AI&SS8m+)7CR0fNOVzkH?D-{ zyjG~?=XU4e8Gkaoke{ek@ZztQv&7r+*ql`iuW>-d^SSZP!4&X5zq|LN%v;_h@x=In z{=INgLk@eK>?#~oT#9>xXG)lfDi0Ecx=Vo?@N0va3*N7sK!pn)@s#N! zSzz^W*gZ+q@9tm9r;%!e2o?L%5^rozyUg8dO<9g}8Ic8fc^WmBhhVWs2ulxV^Ti@Us2fa{TVdsTM&uAD){(dmo8yr)vh@}%_FnNL z!yP$Gp*Is!rZBz?eO|%|Pr?0uecnF@SYh&aiHqf6xGYn;c90T4ZzmE6mce|q6Yj5a zddvwqrL3x6$T{$trIpyJzW*Ac9Zz+8V~w!5Y62UzQpnj+kiGfG(Ni_sM2#tq1@Q(YIZ?8ZkDS)YLc!{mexk+4||ECHwWJv^3mKF++J zNX5>_h8|yu-PB{Cd~|-kGJo7^L*Et*srei;lBq0eg*5m?_s^KN+AWb#TQx)LoA{L@ zMQisk{QD2rIuEeo*zy~QFf5KW1#y83TjMN%a&YpL@Tk+uUteLO9lVjiP_$bS^DU-6 zl@DeyQt+nJepVXne-nXXX|TP1Qa}y`dWzV)JNs{77Pk7$ge@$rz?L-_Zk!aE;U4$W ze;<393E{gE5{0lCz6Jrah<1Epfg@Udf7>ft@jIyfbBRV#+r_6=a2AVZX;M%`1IPV( z2kZ1tyM=(!p=%!xoM8M)sXz(-dx~covUTXMhF7s@?`64w2*YUs?5HV18xvnMP*_so zQJxpr{pAjBwCB|RyNJiH5WJwQURSnv(OU@@_YSV>fp9#YN@ai@ z%1>}}fWd*nO-H>f0BJj)07bq)04B;;D<96zLZ`p3sLs^<4v8n!JSM9}>lWBPr%5v@ zADx$9poAcAyJlBmYTmzqXhb#N8p@A2j)Y0-7e-BW|p>!rf|cp)RLd zkvo`ZS4`sAXxe5bcgG|;%3X#)PZqYfapaDT-qnqj_?9doqOru6WkobR1O-RNC9$#5_-(((pNC?R^bH?$Br-Sx19mLs{*#b z8-v8*08{TS=dzaggmld4U7a1hz*3?R&va`9O!lcvPU-A>dws;Rcn$6%V3EZ`V2X8# z$0Gjsxm_O2?!{|TOO||m$I%Sd%CSx^BpTu37oz?$&K>K6es7KKP1#YLcsG3r9>rFO zFM5L^YALSMJ9)p^$0F`!%ZPjtaVzV%;*l+xKL5kL_p(-?KKBqRwgPojQ1vhnZ{CV8lq>Z_ znhG)KZn3G|H%yOwB3BUUL}dkxV)_MmIGCeZuUZPol*UR9zxJpgGqnHC* zZ)g9*SG$>|X>lKcqorx9kc&ace>@(+Ptvt@7CghCw^%{zJ?ioNu-wK;>t6D)guzw! z6V>7#0EKFiDxhFK;oFOrYrHm~@lLiz&E?7(U6nPc`6RX$?qaUEJh^>I>Ai4*57m!K{8i?G*#T;015!oF-M?()h%GB5%2^&*mvxMeL1ZpXpV;+9w3zlw)K1{@l zbPE#e@#71E(>$V;H}0h}OXN)zjfb?L-QW}*@=2l zV&kb5%2%&pKm(r0QK0V`cdxI~R0tDudXuuDet)gnZUjqFPJWc=$0#xb>8IoBg8stf zo3Y=0b;)sNtxri@(Re9vb%)OB2trm89-v4R16uO^(%0wt*vQobj}hJC)(63NFsCt} z12-401dxkyVN}$C>GFoa?HEo4IZ zjl>p%EBFbz*ra^lWXX~}53@rw?{T6U4^c~<`QYoq>u%78mn<(HX}Piv6P3;@&7y*; zF&cEvgtraNR~IRKF!6Zru%V-vYq~??irkLaN_ayA-!Myb)4etD$teSyGK1=Uf(RCM z9yHJ}G#(LoBbTU`z|BYI8*>#OLy3M)e!XBc%R!v?BteUMIJ8ef`wMV4uQ<=tw_r_f zv*p$AWwN$bVxlT(!!Tj4#tE0U@~u;HFwhF@$afgBmc8Pq^(vR<48m*w}>?q?mw1uA~B1M0W4u!+r@fmk3$nEJhTQGgs^f`hQ`VOloV(teLY$;q8 z|4oOwyV!YrQKAadj;N?Y;JzQ&dPuh7FabZ1)@Oau@t+=IMS`o>5XD#|Xo1*pS8O0! zQVNDESb@xrVNGYH77dOw$ki%jD}2oYTt_4~=?9~_ z3>n35@*Mj-Q7L?}A_v^N2#$;x1RI@a$B|IZ07Z%L1X7o+UsPp=yiJmno3c%!vU`65%2gQ&m@y4H>w=`oy*82s5 zMj>mbLPL2w`X3n+UJ9JD^0h2Be6&wui{T?7WWZf`xD7lbWu+w9y=l#gw^)*K_Ie^+ z%n=Boh~43FZc8lt{l|}3@}%VkS^|?NIk<%A4c4b>;NAJNE@Lg`bq=04jCrfC$aO?+ z3D<#(o$(#SvvJBAzf#tX)0ii3-$>sf=C=_a8VJ2`FEY-ZpW7PiPrshQY)REkv<8~^ z=F_1nQ-nQ_*wMmwSc~ygs zw-7MSPxtWnVYFVf#^{X|XawLgAa^;GoX{{PxKeaNzm1~(ra)0Aj2pp|K zrh=L<j*FX^tjM3z@zG`VvuoIy zS@9ZCEG)VqfB3M6RdbI5tyE9Hes5;}HP?AG@SE9pNAYxX{ATl5CeoTFuF=f$N374oj>$Cp&0YHma;ht`+C;D;$vnc zBX1FXqHzepQt{rFZ_K@(g@tL`2oA%-HVEfH38L4N!5WzwjTI^tofXMkvzH~IUy`tx zgl-#yg)IgpgEwKBuenuj{r3f6J`R`;&5H(8UtZX#gz;g@{}etn#T8O{8+U&9+Xpf} z{3GE+?F@{jKaBT3g0}L&yP=k1DOxNHDzA8=a211n=xv3)t&qxdxzC}??q%gnbr0N5 zutIg~*EAWO0^yV{;miKwp*(i{4@q!5{$kibWc9)@*`Ql`j2+{z`AaMrO?qAZCJPsC z+CikFVd#kQsDTa%zErTQY~9wE#iX5g61Ip*LC)afZ29uKy1y zhBiD0?9z8K-(uo1;~hmjGO!s;yyK1@FC-I8eKnRkY|Wn2qnOpnc$Z*BURH<2$01`q zu*otb-R;d{vhlh^7N$W*7THs$K7pEO^45~>;%R64Fn0KMDeMpfUjazw>#gm&lyJfo zkyBD>SNN|Fuu4+;djyM>q%Jzq5|)~c=)qGCuVio7bXA`h`_*8<(N?(t21i+Z0b?8} zr8Rb)&b^bFP5b-w5ok7B>ucx;lcUSJf6iJm22vu~f7^wJfIBbV^ zd`JCd)-H+yUj6m2eqr1BxC9rrMF%(70dAftPoiw6Vp|&CG;|@`&SATi?d+_>!dM_Q zF+z4+XGS8wUJ*l{g<1aiz&lx&?@_sc{$3GG#c;V$khn9k=h6e;GAC`pheRqmY3)j9 z7QqW0%7B=7ota}ZnN$3Le=1kJ&vaeR9)cE4x&)2SilqR$Yg*4Am^kd0pePRQ2^8B> z#LFU3vP)s|==ze0P1v}{^&b(%V%%eQZxaupBzn8n+Iv5&RrF!$r)_c#Oh2{q;cMrZGle;j_^M@EEGJP{T>O!PZ`($C zDCgtE-l&J=Y)bahCt%JdL%>tk_nh!oe|P)O%yQZG5k6Y3=9v0$RFs?>?RP)Z>r2L( z)e>8m6Nm;wazI%KHfy$<8zY6X%&! zrQ9h&vQ#9KUYIa}snW@x65YZ*Re;4qwp@i+ zv!m`syRw*3>GB!DqETrD_nO$@okn|##FigjJ({^6UrSWseyFIRN#Y+?SLlV#3blwc zvfKHeGqJe)KmxHS#`lh#WpNKoD!3*eFYPYRgN(g>K3CZbr6H#sP%()EX%Un0vxdEX z{3IGOClUCY?|I{8=v4+$n|<16^`sc1-=QC-LuFb*Zp4WUszvX zyRQgZ^an$`gt)LUg&UjuUNL?s%Ps7Xh$3-?HVeypSlCVW1phpsR~jQSct|0#xeqQW z68FDh0~5z&j)mf0^%-_UIqOSF`>H5RBe)1#R zxXp4Ik@J;lAcrWUft`h?KRNx@$8N;;OAa#VsfQJrXd#lNVj4SMj1>hxV)#Tkwr$A7 z`}5NKK1OM#YYKl$w^jOvpqEwFmR8m>N{jVmtiX?-(kqD_ld0!&U5@b-%YQssWl&tB z?Y4u=B(0DXi{2EJIea9`pgLJDKXJqK0ORrn-zr>gqc>!nZOGet18Y6;XDxP4j!0;6 zPGr9y+%j(p7!=45IQ~ky=(paP*D!zO)+0o%s3c+sAr@3&Jmzd9TiWLv#x*ZtF-gy( z1dkzRN8<_wOlOeZkH4`%Dd6T67A|Jn_?cWlRECiy5Stn6@LgoU`MGKX zN5;%a)tNz=^Bqx+o<(yQLIl}>aS(X&-NbA~FPBeu`}4CayoT7NoxJm*`{EA?N}qw zPD!X}1XBM0oIXoYICIluix@*gKPU{%KtQmNAO`gHNy1iisdFiVZTO=Z*0^K}n3^Q) zuT?+!i+OPGN=9?ZB>%WP!V^yu2}slO8`#)QSvW3=i&BDaH%4<3~OeqVsz7=%e&U zjq5sd4U4Ep|3ugrQMZi|?eokKrP=t~L6x%$=LD|bG!9vp)APS&0pxKBE?QI(95X@? z2Se2y?y(FfzU0SJl=qE#bpjKPWhaPuQL53!SWK-#8s8*w3QJ7;c=Dg49hnn9@Fby% z*v^B^9Qu7J`i*JHZlkkGC9cUbekNQIBVkB@^>oSdWMN0y=geW!@vDRt(jj|`JaVlW5t?Z6jo$PlC8>jmK)dQ+x7Dq?!;eJT-4=o5d@kdxQ5y9_L?FZ zOWbb`wAss!V8(BRE8<(i#ryA51fIP5?J7*two724hM?(L++mU;>b>vwo6feb{C6T3 zw=RRo)raHb54F;E-Z?L;HU8bjID0>CUb&a)zC=$ z|3u!CCGbDbMxSI}`wJ3Sv<;vn5UjXJEV02z%!nt)J2x?}ebPTfGkWbUgtb5g94rN= zk4u;mi-LMK4%xi$+|VHMBE{Qte@RGexk|%+CU_#z;W5=?m9OWS{WUe#@E5ioIYT7k zp>3^Wouva(>CnknvM28>+IxZ}yaxYEu$b^_Q{f1C^d70$J%!dMj1yciYRDPpogS9R z=$)pK@p1c-j*Wl&>qr&9-i3$&nkY5N(*jN!K4nOz<|P z;=72{z45hg1aEA?lt{a(gs3YKamIDuHW&y%FbR18F6zqJQ6%W@2irW%6!Zy+D|{DS zcj&k-ci2>ZQph#AS}*n4ei5_Ro2wDIWUu`tQG9npPm{%`LoD~{%cK43vpVpt)d?8u zz!~DK3y}w8cyQh%?n;(rN+*X4jJrFe-uJt$;fqBcI`+M4_xIZ-)*$q&v@5li7>l4J zper8t;Cg;6ZtN^xT(ajubBU+K??aDAB4Aw ze7Mihb{OzR<2%PK_qk29?vqRbH?|R~P{0sn!4n}f&gMY&$Wzl8p0{tilBIxOl6WEo zjK(q%o|0;Sp_ZNnx$14M&u>^5VQ12=5si2zTj>_vic&?hLF;*s0;N26=bYgWG2BYE z6X04m}jsF>Y+wahXh>#1eTNPI7G&`7iZ9DLsjFKLSN6&3ZSM8Wq~ z?{_nE4HncT!h~xe0O5lZ0V*{1?xi#-7Obq5r1DG>Ya&KhJkg1 z@eymCtT=Yz9nsyab$_>90bBR8V*U&~@#?l$c*U0K#|(?_!)vJ73^&7IcH%Z5lqdOCU@7;}F-O4mgjfR9QTH!J8jjw))=fOFp)+)Sk&kyz3-Iz~GOuQSj0Cpw)0n~^)|7ySbBE6=1kVNc=}XoOVFcGsOGvPu zzfFPQ4`vTo!ZgPz35=S9il6abUB?)YTV9NhAWejc`yAQ-2IXVAu-$Nr7TrmqxQ6xvk(bDqz6W}fqz`Dn)Pf9ut^m+trdoX>gAbN2H*{YvRSe?Q*u z8OwQF*ZH<#Uv{rkAunKadDD9rJH4S!*~8Mc)*+`i=nLfeYyn4lFzoTz0ww7a1Fn!W zknY6avgW(oPOJVWZ(N&m;%37A;b8|Q1Hp`tb5h7k za6N7)cYW1ss+K8Nx<{FucDIz%uG6l;|32u2C)V~Mv_S6q7c4tpCbXNhj#>k!*Y0yT z1J)!_T^&9?+fV)W$5s()7wsmkfx~%a)QS;WuNmy~I-}Iq`R;H=cfu{Ta_cHfk5>q{ zoz_;XRp@d!oL+0tb(fQWyaJe=+b%N-xM#`1UcCwM>^Ft4TfUcoowN>Gz5INyH4)H` z?Xx>XLEnApmJxjkbi=kk7h9hCFHzZ9>!hV3FKr>%@{r31@J^fK3R{&Og?(yj^+Ur6 z);hHJ49kR93AVr1S4)XP=Gp8;POpQrpl*bxRl0{l-X8kzr02VB!NTYoWp3Ntwi~U{ zyyk`ZmXz0Mja)59t5sBNwHMmFUZ>kYa)k(b*({Y6`$rrdJ%I3LSQaj@?38%jw41d? z1p!~!Z=Dc!21=qV;dCooH#`tlWTE`3_j~sw=+)DkK4+QoI+5BzYp2ymF6+Qz5d*#D zBlMP5HtwLk!tTKYxqbb$|FyJOO^`jc?phO%ZIadR3%bOJ^=ULHi~UF`c`wCt1UJ+y9GlUQVHsKjqB$N`^U;l@`3D@`0m;7!k7zs~aD z8X~=~)>~^8aJp?FSFtn7WpSkQ!XPOj!xyNULJ_|U<6oP}#IM5}v;vA>>o_YQzZ|Y$ z$mX>hKEq609(0YYdpmu_!Q+*-SWe0{ZqfQ_t-?X4HQ)<@c1&s?`q(E@q4pn>Qrk)6FhH&gsX&&`F!HAXOD?C=(0qA=E#L^}g^ zIQCq?RbHeks%h;<8&dldneCcZTyFVFLf))()*29`%L_)r?X(p;4d-2vL&Nvi9`O0& zqX>9bedk)sytPE@U~Qn5Nx(tp1gjgAsaU}8L-7G^m2b0$d;$Ie$_eG#`o++#jhG(n zxsDb>Jy>7gO~2C`wA9>hoJfDqi4o4iROO)vEH*V9064Rev`o$*8(Dek~N9(1v@Ysr+)?%OCmM5$i9GhZS z(3J;9Kd*!@p-|5>ZA0tG(#(+mDHlLPUdQVTc!0}yiEZSFid+vHblYI~#i(2#SocK_ zCfD0H5ZNf#wGq|o^aovTpO=Fw+n42Rlc{(dZ3|kQU~=u>NRUFVZGOMIME76>dY0c5 z*dW)>oNGVBN-Q4IwfTdVx|;}ixHe47)qw-fpvw`qxvh2|m@&H$YS6r)=!%Ihx6N94 zNe^1C_FAr13#8C%b6E?V?r@@r)}_7MGGqMukpwMUxO&6N}M3sF^TYM%L657HOc86RbROseK zjV}KP<%}3a`HSJRComH=c?*37nkdE9bAYx3@;R>J+$i>B*Ps0~<~ujpN|@+7*CjVM z3Bbb}SHH*r_o;xjb#_|8*SDFguuN3}=ifQq!R*Do+lWH67Y)FX0Fg)@cG|8gcQ7Bn z`F4UuAHP0$%2xsU$|wEHGkyM<1Vw$`M5Pi;s6Ai}hDzLqqfs^DP?Q60gnm*es(G94 zZER;@-aYRV)fnd0u)AG;YeB$xE>!EFvi7~s|7*j}ZmSOn4bN^(o5SX@<~c*QEG|h3 zTxNR5C~+?TsixgX6^Jn5kc7i9Aqtnx;grJ!a0=dl#kaW`c@KT4kawdX56lclQ;rXs znA|9No6{!zhe>AcM+AqWUOQOeuzGwRIGJG&hcx@%zn{1~ifQJf5>Uh%9+y4fv*y`C z_QE*9r*a&#qBM4C@Xy`sh|k$UG>Ri$;P&N#l}Z3panW!8X-Ii?fI93XOgum-0h`AU z;g%VuvO8Q;Kem4NGtBcpDPhs`ZwV(n-~?}30k*pGtzM_ok+@SjcKJ`6S)e#;7m+Rk z#Qo=PZ~HMpBl)fDg-&~sHQxtG4ja0= zR<|!06m}u@cqwaev-5bB_X-*LU&}R+{D#prf__&KxHqr_mpnz74YuIui+8>`kj~o0 z{mY)S%-&7pch|aVjiS)NlmfvYV?MzgyXk|wtKCM}6{ZY+%#!m7VfWMeXe}^!3%K%- z?Lmw%tl<5-iiHcAF!|q|y>|k0_YTV?(A}#a-B^z^ROmC9dBrsidtI?n*hAZKKkX(`-`yNMgx18|FQ1$=E}%1GQVUbUS#uBD{bEPC$AN zAT$w|f^Vy-z})cE-5OKx2jw!T-mCiqK1}&>F!Oypab4M$xi7Vsh!$BqLIWG*^us^E zjB;Ql;?Gh%hFLdQ(wU0N_>53dMb&4hG5IDHQ&CZ4cJF$GQF=mR3Q95Msmqr&sv6gU1hI9139eDddL_T5HcrmIms2kgzcg)Q~`f zUYkEy=nGl>wooC5t{w5LD#_nU_q*&xZY2%$SA&}GGf6xz7ePs^7xF`h=^iq$$S&FH4_q(&k-%5wY-S^ffEI&!yURn>Wsgd45(CUb_x8{64bXu#$ z$liK59@%+ecTqfGQ?eiY&jaHa?t&wxxD#UuHpRW=m9%4w9lehdF0!MUWJjLQ7Xq7W zLoY2(5?_J$+U<&ubAHRk1MD>ZAmQ;eHb&K9_lLo|f#kJG8c4zBlJ~>VXGb&aCC7+# z#BS#F_(LVuuou>doOd@=t=#-bD%MspG23gu{s=oReU1~hNCr8aU=KV%bui?zgU{|T zsA_Zhs95hKhn>W4by2l1EI&!$JG2p6C*5nJD6J4mAmG{r(C>5j!A$twF1szLr#_2q zh7U32Q|^#2>ai_ectY9YrjX>cgX0Rmlpjn_$Ok2wr2N3Wb5?9+tm^)i!m3md8gN5` zC0={7bT=Ncegtzx&Prf(MQZ85XqXL_LD^9y_Bys@%}f?5%{xhCVyIMy$b?A5?KI$- zGIqe(qZ?Ut*z0S8#Nf5I+gD)n$qI!^HfYF*Xq@xX}IS~Wf0{lq`JF={xRtmfrR9oF{ zTd@rcr?qV!=zvINJL<0G9DimeW7l4}vS63T7Ayi?67=PVu!NK!4npaHYZGNBxY zb~}#>rkdDl&z^3HZm~{W+>`?EO zxWZi!xRjDh4tSbxQCAinP5w^dLkcCsu~d>2d#^dT>Q?3g=bRx}bb*`cIuv98(vo_V z8yDW3&xcpc-NJHp$0R=H>gu8$6BTPrsTxGW9CXX_6>XTOH~lOTEj&Fe>6!4ELPxi9+F``y`xj4Sf0eZl5uoXijR~A*(hqW7p{i!bX24Es9OJ)_jP$ zu-d|Tp+py1YQA!9gCQS{Cf!uc_vU++?y|Dv%X0J21Uy6=tYzDT*P>er;pDj?qzl;GZl@cpiW_QPVPgY< zDPInKyW;vto^ri=_EB*7D6-!i>P9ztCr(5lDfGc^LYo zz`rBn6UA+Hu{-<|+-+l@kNFDaKL7k6W6rY|Rpub9+gV_bppLW)Ky(5}z{ z;tm_rk?|#NUC*&KS&A(6A40_xS%$7N9ahxYgbEhe=Yhl_WUtX<=gf(}$bH9`-Nlq? zYN@e$tEtE;*u#+314y~M@!C!mw(yO-X-B8kEMl^!-U`{Vh zUm*Ml=oUqF=k_a=4e(eYhK{Vq`DCvD-mr{KV{uwm8CpriX@Y9e(2_Lr{6jsn?;s7h zY<9-OmSYlmkTyU|_xS!leL=f%KMZC{Y*tzN0#sskJZ@X^T={g$^k0}u(e@g`6!{%W znL|PsR0HIujICwn>>m8?*LG&mj!R%PXlecc^sQm`S>l?g^U9Gp_PQ)XOW)M8eFif_ zvE*7JU9<+P*pPwXVqz-w!$T`K*=R3I&3)8i30+5^BedaKTNM=iS@b2uhv?a&P$3jD z3Vm*a->E1z{TnhB%QU!f`m>#A!K(AmoU{zMo)*l|+GtgRP^2{_Ju5p$DU7{#*B5Cl z^lMO#kVNPwG>!CptWZdBfNH>eBZoKN(R-Hu@*Cs#ZxUbd8=+uBu|^0A%$#WGC{fBy zZ*O>V2y4?=ZXu$vO(RVOon$L^fon^3#zd*k>|z_nOJgoKv4t~;Q&(!j^ZG?FN&cIZ!5JLW6hQmrwI(_WXT7^gJ=)QLeb6WF8^ z5BbK(`6%z41JX3`b(Qc$f z5##Pze9T`COn#L`%=cF%OiZreVW7{C)i6p*fpwRCJ#zEkT}TuDr~mnLmaf$ZGF!{k zszFNDYCNq97Y$9U&ED9$f`x3SC71}=&|eI=98UB_ML6hzj^sj!W0OeFUg-5nE7r;Q zXmz3&I~g-Dpb}LKzYk5_|6w85zTUN&M6|z=7D9=rjl()&$ksbz*kdZZtBi(Rky-_tGtGY7^j-rL}pbwJNch!El7PMHy}sMu-)`=-blog4r`ihSxa} zdhdgwVz(z_;6U4m?rm=MI!8cDyU*JuD?K+OD!IXU|GiDy9&N;yWl}C89>P4n&Y(H4Ik{#;Y`m_cf>XwEz+%O>lt^4>Li7!!c?Mqw6k#rw=-Mh(hST1p! zHdJe6zJ$$P;N!AWDWI)vVPN%g`YA?4@ziJZ4g|>jkG@@KIkTc*|f6Ekh%e0}7vaCn#<^+pbkM;~U?1xN3 zq*fq!g}!aP&oGzIp0i4qVQIhj<$@yZCl=%iTsOaFJ7K=k!m2&bFi8rvP$a3T*-EAx z%@4BsEmx0jWV%H0qh3ow*XJxm;Vg-b7~5#e^^Dpr5>NOG@u-b;{giFy)P8q!sm3fP z?QBKVis@Z}ZVSSN1Q5Bok7lPRjAWG45X@G${_yPK~MI2nE^}k(iZcaP3J77cQ z-}0FAe_29{?0|?Wv6(LptdbpI4F}I0X!#J+e9JS5TA}%(UvHi0g4E_jzViwbFa;Zx zp^C1ut60*ZcNW27(xGNy2{>3dl4_7qB`Kn$Zc^?XbPLPnot9w2>gM}%vSIII=36y z7{4n~{O*14^RQaU0f~>bk_?-ypuDyO6T4T-9})Tecc=cBi(b?+|SU zs2i+qkCpX9CHf8iI&HeM$|TGEcb00p&0q+lpI3?ea9!lQF7u77ME^Vw6|6FmCtS@IjMY8mA#wl98u z3X`S1av32@rtqR5kCPO9!xdTLigLy9n&sK&JljF}oOH~VVb4vzD*zsKWL2}NCevju zWTf_C1=pgEw2UaY%4Og_nE%Tv(+@p&h_(8(??kJJR-Y)zF5SK`tel4sa=nFJOaW$;eVk>`<4AgSl2&bQ67)kOVRQsZUkv7uw)q)W1)R zTN#V(reAc&w$<6p2=(kt>xkx6*@fWG)@M_=e3>iux5AF0XSyWoW&%cCB7!Li4Be4V7>390 zY8Y8Gm{etQOr3Gd?%DNO=1(1v@WP*xELZ%hM7chH>Blt~*Gsw*;ezW9XFeS2$k?D& z43pr<5l*=#x<=i8AI)F_liE!Yn1(v0SP9zctZwKYH3YGWkx(>Wm+5!D#dy9=!XnR` zQPm#2b}x2YTJ>9mVs3Cz#GD(?pSC&HFsjFQSEx=AjpV{AlPAl$VG3Jg%FcSMfLyr; z!HNR114cVgC&az;NCD-_c}l^U4h(c{45$tm$9kE9j`Qtk$28ZNo(Id!H*|O z#cXmR+)ZJ{nIHJmV4lJKG(VO@%a>op{H=fFN_x>AR)U&F06X}>0kZp`=MqPJ6BfX2 z+Kvm%$zR@!z6728hVuAen$%-0fH}Ryb2Q)QmFs7v@Jn3h(ziE3i|btCL4$7|Dvb7j zL2zlzUYc{tQzc^}Q(!;Hh?ePvgmC=KO$6=ZrrxJphuL3)i{Bs2P$i4qa!T6+PObLVtTbLxcxr3 z`9&P0sK!-TY!zLUd@*xwbV_}58>`HolBSx6o$^JuK6# z(*B7W;uYruxE_A-%*pYrOq(@;;IT|w582N zs=-a51qDvfVNlzFNn~hD0#$*Ru))7Uxezv*3v9JWV$0GNk`Ur2`YkIywYSTswE{QI z7;o_nrf(UejnX<7IpJ!6e18X=jrRuOMl9HS8&n^x6ANKZ5qizBPN4UDCL;_P<4^Qw zK}YplX;ln5GSMEZu3q$iw9DBM!- zN!HpkW(d)atv!t(g()srl_kJ9I})GT%L`}R%v_4{LkSoCl-hP0JdZ8ZDk-rsY5r_~ znlg?*@vEWluru+d1QqvQ6+&y94K5A<5fWFQ#Zqoyv?8jhhM zQKI6CGdVtQzN-Kn8sm_@=@RE(=>8RRDoWi(O9-a|^%8Wy`~5hq>w&aXjOS^tIxbYL zwU61J7bUc?J;eZO(x_ufZSJZz=UMyApy5QVxCf4=iDO5f%!#!8;r)?`^LdFV+Gi9* z>-ez8pqNeRw7lWVVPZG`c17$OsHpL7c~oRsNl7+&tn?DQ7Vsu=9)|O2tTz zkQGHq=lic6s?SP#Z%9m0(t|p=?%e8JO%{FAro$~5 zYsZXL_e)T$`w2;qoVN2=K4U}aaVi_&`g6Gc0-R>N0m%n7<)0^)WFY z-0_9fa+k^cRCcVA@;$>FWhEL#HH>b@EifKRW$W8!c2sUX{pMniROU?lCGkX`u*_K} zjeBFujYk;n(*F>h0#{~r6TszWe(1Rm%Cm5Iu#Ipr9ERJqAe{%7e1LH2_XNjqLQ(N> z<$6>D)EGFcD`V^*5*Qg@*O&t#~yplqqR>6zCTFw-2rT?Kb3@^o{;(45S=U@_enJdD=-&$`HDI&G$gaXc#ijV|!E4qt6XhIF6zVPoUbGW!t4olLW4qj4nYrK&KW>9vEJG+}I@2}5F z$D`atHkOW?L>5R8oQu|HGs;Y2Jo!h0izwUXfT;}ZoC8e}GsJ4H8`=-4SeYHK=RHKS z@Lvq5aDtQa{twNl{TWl|L%oD90t#pl5|>NIiQN?*b2hBrGwCAJlYdA^)RRr~U{qf( z*n)+?`vx&%<8T=+8Hg{+r+$B|7b~8w@Da6GJk5$iW1b4qLV7cXOhkxkI+sLqw(l>z z(3lyeA%0p4jna+Cm@9UEah~*^|NQmL2>&k8gb@~Zc_#tazVr!xo#m1VL>!&I5IBZO zhY6QgJz0afN<9OFimpDz41He02~WcP9$s(q53DfRAw;-Xn1stRrE3Q%0rYl4 zK7SF+N8924D!a?*kSk?X^+L`)-={yyPW4f_2A=9B(KUkNstIh=N+OH1{N6w8Oyz$S zrY{gGU*EuF*>dc=@_#TVWnnRaihwU#H%Nlj<9b{=#wwlNClV$W^cz4YnR62O&}KI> zD-y@vde}PqUfm74w0Zy3`_xk z&-FgfJ0DAkRP21L>hcs=4Lt_RM`vd%^T&JcIkS!-U3-@i$xs$GL;C26pJp;`^{PZd zZB-ksZ~Rw|6s?^T_3u}{^{Imu$A(WP!mv121H=U`Y>lx1%E8G}!lOlw!C@xa*WXQG zDBAUj`6g4J$_KL;DY$6CWmXzoEkUs~*hD`mAcq1yN$i(~o%onp*xRQN!NS4{Y*~Zh z#z~PG?r}eR{I46C5dJA4Q3z|}YY;GtXvHTMIHKA2fAY$^(%IsO{oYD>#cq zvouL4qRRPxK3+cPUIIpku1PF#obfBA0wwtG$}fq^)=T5w%4L*alnaP3oEE^2nmn{I zu{8sQCB-h~d4Vx?2VBdH%e(gx298F68nm1h@Ek>+Njs9-T_KxOZ?} z4}{~fR4N1PP=11&1EL)Guyp>nEC9)$Mt~w;AOPd#tCbIDXJNwnLz9@AuXI1*2{n(& zYSFp{w$EY0-7e-Lr$!> z!rf|cp)RLdkvo`ZS4`siNv+B-x$8cY&_&a_Ku;95ee}pR57m2~mH19cSS<0iHX|Ay zf`TJsaBmg)%TZ2D{+Vf?ur$Ezhlp}9b3UO+zn(4jb~c3<+-USIxA`{~`ew``MA0>k z;~n73US-Ge{$*?Tvu)fb!9@;;z^Q3Ev4XHtL_2gzYs_?J$!sD$;&kaN7Wi9>*sqom?0qp2)mw18sTCWqW&_@9rJ@eEdQ~U z>?p32_@Z?e@kMViMC}Dmk0Bdi4qNKuFLtozu{-Ay+1NbR3Sr?1O|)4UUW3JUL(>HI zYQ)-D#m}&XCyBY6=#iiJ?jJQ+Zm0C4w2p|lm317k$fiucc7Dbt)(Z5ZM8#I1W(ukv z2I9?Iv4wJ_o=8(6QMx}VckV@|M{aqH=oaZjWd)34`DQAjt{ld>C0REyJ@SJ@L_N}2 zK@?Z1>30-!VDs%<^uX#-%+kz%oXAB>(?|iwAmm>jkKiY1%&HBu7<9)cOrZ51^;mwG zZsSW2U24q|21g_`CJa;|6sZCV=HtG-Xu8IeFIhfhYfPO-ve0l zsPHZm#}@Nx4bhpVtRb!grb*P;Qk1g4c)5<^No;qYEnq9`j#&ZIL39!3)hL9&*EIW; zHB?PlK;M83Rjrk8fQea14so@RsGPpr{)zD{QGMf+w2BzOR#t&NW$*_IL~A9qk0rW5 zX76!frY<+g1w>*57m!K{8i<_v`5ahr5!oF-2j*?B$<&zjDWW@~#_kfDFCM6=Y!2P@ z*9w+yxppC;igXJS>+<3Yg5zDHl{e<4GE?M3+sA#u0`rv;S@i58G8XW4V-4w?f+S+o zJ&l(2VJGU=MMO28D4~4y8U{4ri5vy`o-z0ODouqjF{d{vN7e7Fo&ULyMUfZe8W=^U zApLY)9nfExcr$i-E#H&OTEDQEXcvu_0#|qF>^47SCE)>zcrl8ey zh>6PoBx*!dP&J1A_7Zs8(0Fx`!UqG7>;1F%Gv=Dke3qz1*R+AyN_ayA-!O}J)4e(H z?&`m7WCk_oIRX}S9yHJ}G#(OpBZsJ$z|BX-8*>#OLy3M4dq1|Y9K<0BE$ZRWJ_+qF zz}>9;EJNRdIk~47UALRb+LWb4F3MU%h6!^ucDS^aZ=I5ZfmUE!wk;}a<%+}GS;Dm8 zGQ!4$>5cjo#EB23tZ7xv%EFT!5>CWB@b0t^%D=JW1g6J1FVj=Zazwu6L?7meYQru3 zCWvZus`!A-I@W5*@KT;v@M=MHDsm$+IbFppv3&hsFYL51Yj;NCiTn`YLBlo9f|=kd zWVTbn{~ZSw9b?Q9yrq{Yu`QimcA-KmW+y zvFs*=J6)9SJG) z9ad4q+z%w!lDI6sW7Arh>^$055}`u(BPyy8xbFwH9+Is%Ou!GM`B{HCG;b{{5?uQ- zVPlb?E@H!7vA%FYAsDViK}+r2u$pmpg#<;;idS~P1QkGa-;jndr*mP|_uDbZ0jreF zs*kT(fa{3lCj4MjolgEnOh|u}kir+sv%$TK;K+zUuz^`t90}zNP?QKyAmv|ECX*SG zd9M($Xh>2NVsT2r_>hI7J>jF^Lhq>dyrI-4Co?1+|4YEakmy;XJReM+2v)@Ja8L}1 z5^to{xp;yJ*-;6NLe^M?hVpj!Uos}V6jqiL@a*<$!e2pMn}9&Q8ANLeXC zc9-dt?PW>EX0H*vm?IEE5xK+T>Xul(a`J4JJlQ3ez~o6fE+Kk@^{EPHb#7CAaujM$T)X?b!+Up zTyrV2B|GIBXyWTkgsMy)_B>)o3*TWa(uZCuD)U+I?6fco>R&@7qeZT(!-0DZZ|)io zXk5GzpB@b|yc^$$#S=Fu#@q`S7q9i#<(*73ZIpPTY6&dGP%EBVQuGH;z~sHVr6zOg z^4=s8g;R&Sgd=DL1xefiJ3P4P8g@voeTy*hkl_2g;>pnnE;O3JBJjRzIAOp1+b+Y~ zGVQcdE+L8^QD}Q807X~q2;uv$$Sq#fqx8~tKG83QY19F0iF#46i9+id6Jl~0(&XUr z)!utk`VM9D)8{2H&QIsKyf9iXT4VIa3N!+68IW@Vl$_8o#=Asaak1Jc3ifPv=kY9R zd1f7bf*!TN^JIF19v(ZN&pfZ2#czduxze?>S(dTy+k`8!jF8gBkW=(cMH){MHoK>{ zhVq$JI4{A`Dl||~gr@sfdXo zf?HI!`vk`0&Kp!7Qv$&rcG%EK@*9e>=D4@L{?uCLDx8$K=qfac0?G?7a9rHLWJUgz zj*o>cf8_L>I0uu^PufafC0?4QzF|`Hvk-=|8itZ-3r0=^dguU2CmDkAxvDsF251 z=$<(PFEin)_bvgUaMeV*;zh1XfdIye@&4hcyjE}V;WE~ovqeIR$#sPEz)*~FTua#; zg?&Z)-zi~6()k|ICmM$kEE(_ZvqRm_v9PfGW`e`8upz?PP=e_7WUxktMq`CaMQ1fU z{_6ymgnmuJVw$^A1Qxa!lnma4WxVEAx%J=UgZVgMIy5gvnVM>Sav|fx@GS}-YTydV zyp21--EK1D!|xJK)Xu)+pf3W$=Ff+k-`o!@a2PK zzTVod%OEFQ5jiE5w))RA_p(ZoZwJ9*C8>o@w1lN*BYN;Ai-!Bc5Nch#}9y3>>`peb(iB-)^E?f3FCpVz^u=NZgsob_F)Gz$WJt zLP964abZawywITxh>6#kIX06y#SaKvsP_ZYbw?z$XwoHUd{!(8&`A$1-NnRV=BJ7{ zG$BxIOA#-NK*=tN$)kOl-EC~#BV!MNi*b(E69WF{YpDd+@b$%$dR*NNm-zW+W$3L0tTi zj&IwBa+LG&^!DoISLO|FZb3`2eVw-KEg-KRR>caw&H@Tqy5qA zDwbiaIU=zIYb17X!qvfN{P6bEEFN31pGX%mF9iwMZj5OXW5cQWj{a5p{;G`Zn-3^E z+8_!TKhLB(r4*L7aVw3&O>(w2X4!_XCA{cSQ{V$(Z=Af7sUqorzU~&LN~eELbPM-X z0TvJ0aus6Ejy)4T?ZAvm^MeG7Mx{R7Yhs0W8m&nZJ8e>j<;?xqB~gX@p`wB&iG5gI zp%*$U)FRHY?cEwPvG5;?BNqAi-jTg1=7C8C*Wlx|s`unqjJ*xNP}vKmA-fe&F^L0d z5rgqFhaEEH{go`1d|$$1ESW;L(&!Q#>)ypp8iqU@dDDvbONG*gpqSyh8<$rO{lzv8 zDK=HHCZ`{L{C;*NZ}r0ji&yf-z{YC3+7WG6NyJLxv0y<>G#L7&xVR3>PQEP{!R%zs zfD`2`_8t@^7evWMCAN01aU5e7)^UUgM+;k1@3oCh@G9iG-L_&|8(%TL3Dl--UX`v+V=~7=aj&%9%`39Ll=8<3QS_w*ooJFEloI~8Xi0~>Jos1c}b{(ZJ5a)1RDv^JDkbH(gdR=n*FrnW!t0rD7U8R*V$|->=Vb zIkp{@iJr?B>|91^rpX;I*0TI6p%;|ao+z!Q6z1#6Sb-lqrB@I+CS$MWx}4}!EdTOk zRg~hEPi+2+@3d$Tlk`<*oQ7*3;ed$BS<<2J+E;rO0GOje_ZN7m6rlbyG z=j4Qh7Ux9v`@t>qhJZnV41nXWq>EfF${k>a|KYERT2V>F4ni!b!g$P;NH(?41@F)L zFN;ZX{zvc_lQfH7p@8WO()+PD7AOVWtlX@QIm%4l%ok5~VcU36E+8ty$P$RnqU-Qo zWWd>(Y6Hi*Uu%_Srf>2oq8vSoIxvI?vH{~D@Z`IJ*@|ACFwyDFZd2@zie0rEIzPRT z4$90?dk0vWeM7KwN^7%9YYl>6PjN60uL{u_!bu(*IVrbY)kzUIz6CR4mq{NVREr74 zSwHhZzg?10(Fmmc|2ci8qR{um)@6*Lq0o6rNgkt?Kw z92M|P?btn|-yM}G_WryEzcN;|{!w8?14*)3`Ob9Xnhz-4!*CBUT-4=o5d@kdxQ5y5 zbQ>ZXQ`{CeH*Ld?;IyBJUJ>6CF5Z8iByi=a$CfckYj&Q1g&KmUXK{x~lBgBRtoO04 z+b>aZ>r#kZeKTzXV0BKpI>`_&(B0LN?QZMhOTxPfJmaZu*@xu zhcHGo`bA-cBpCzIWKe7T_qKF)G(M21NNkE;bP+NEPdP)Q0+(M?aoFc5^EHEYT~K%c z2`m`#(nYNh^RNaDVG8?1-ox7&4=P+#cmQd)2<$|c;|jOm@my@_cedTe&ctgH71d57 zB2;`ooQ!xk{3xNb5-;_+{HLGUy4x-hwP@?=A+_S#H>jM(Tu^7qf|DEXYsn7UB?&4H z8azG^*Duqh|M9#hQ{V|N)Evyb_Qk&vwW4hRC4peYMPl&{Mtnv*mT}-~=CybIjiAwM zuPdwtDqw#hIDK5glvothbM%lkyM65WZsbLZx97f}P@}5^XL+Jy!X8p4l&0 zp87Ep*xA1miFgiDb*w9NKq?(N*-G|g^6nnnSi&p)4}uj9p~bduaXbc=5A zzcP+x^_yQNK#|oK{cth#DI3HO>32WfvyGwcWN2cNEOE4F4-{Tt$r1NoL>p#sYUxi6 z#(A4k@m)mfKKiw9FBxrTN~H4Ngs3YKamIDu*6;H{FbR18F6zqJQ6wn8N2Nk0m}?}i z@LhD>q2oH7K|}dTA=lt)^&fC?BU2+I{~>b8UV96|`0j?DCW}pnnC{c)TJ@}~4qUp_ zxEE~0zU%b}EtP7C`WO#7iAnr<*WlWmQ0mj`KQ1AO4Hb46>7ajX{JJ7jVUNU8*C3qcC+(v7KO>l;DsF>Yswans~d*AEKNYuWbaL`E91sr_e3NLAj z@f8*IrbNMMm&cuAuEDz!P`CyH5I#5_ph9E&c1n}t;&UH%VjX6VazrC`n6(C{h`%oo zc9-ClVPM^0e8gNQD~_G_&2N8at^4IIv;wy7w~qKT@WiXrTI?2ErXMpXHX97G(jJ)A zA+ueMdM19Ued&9)&Kh-{jJWGSH`+u{fB)iB^6Q@`-F$LedymnDqoW{ge&aLuKFDQ97J_a_7FEG#WgG~rpOt=px1$mX~W z9!ZUen!h2U)U&Kg<9GTKhYvDM)9VI87On7@_lB5Ve_NYVYOca}yxxBVyBqU6iHUb( z=E6=zngkR%iJh0A|@-swDxD&|%o@-eDtfW}L74kiU!_)xQJSkHP+ zMFJJC70Q>cJt1RIe2OqCPFRD9j`K|6H>ga&!nwm|75rC&Yp-sp$Ozsh zaXG(w=j@VMR}d<1DYOx&l-{x>Ke(}nF(wMIQv> z@Rl0E!~BE~idBX#$oR{&d#m#hHtugvkX-LHcF^UwYL zmfC7QvbsXyurGSnzoIbY@c6T@D|7oJUGj!yYi%QLe>e~-3^+or>~PfQbA-yXCx<){ zcPQJ9zhy7-c-?mWPj3CW|M%Ar?*31E9JMXT^HoLM*7;hlR>vRp*}aZ(cPQL0;+_(* z6I`#$t6W;+;u;rKso3kH{0`Sw(JE^f<9}b(cJt?b2rX3b+S9g??Fp@`)>+GN`<(%o zJ7iB0_3)Ir?tbdG*D4^?tF^9LhRc0^)GK4OUOYJ9cN^3`zd65QPr|LRYSVMJ%@VhR z)?TYy;&Hj$etXz6)y+R%0n97tkYfPe+HKeK*Ad{c?@QL${2hqUE?Ot8VNsFao(yQ` zj(MF7&`rz#DC$d~8{hl;5nJ7k1lmpOqGcj49TC{_h$jH>ZinkUTa{$H=yth0{$dky)fn<6`FR~X zDCoC;e9_`TgnqSi_X=CDu7uu8>!CG`N47`GgVur^3Ir8yw7%!JZ-)_hvtyU8x1EsS zeYNYfHX*my5%H9{4K9l#T^I#PiI~1X)f9^OUH;YRT1@bti7SJN8TYyGq~(XiVd3PeCVrnC=z9Nw_Pst2Ag-qxEI zxpqwU65H=`k3u={zPVnW>+q^4eCyi*J$0e41)Cd!AN-Bjxx9Dyeo2O`u+(WT5KFc z!1L1FuiDN?;KABJEr)=^?n!npC{wXOFo5C%+N#Lmj08gb1C$fWwe>UKj?ZFxaCHy* z0@Qbvq1X7nQ(VejlTA%oFw$ zg3&K5=SwKmGfjJ<<+~R%Lq7Uiq8<%-eSaY211?V$+sF|WxgIuX_+a=&L$1Ge1zu%x z-MBZQqFmQQRJ%JE_ILw+4ytTlt~)D7@i=~6y6Y<@*E=MnkZVUU=q=Yh7=fM}v;;QE z^~_g3z1mJJzM*@|*|sUy5v?P%VOoI>9CC*}uBgLncLuO;$Q)C#n^AccO1$6oCAMw3O{H05yzWAs96)8)1sw%hLTyX=8zBp8j@C%ZkxC1zu2xlpD4g_kmIk#{{UB(#Mu>Wz3nsL;(b zj4uBO<&2m_`MLL74q+zhD+!J!N^$jEpzVNsj;lB~ialvOZr6C`JI@|K1f%cVfZW^^ z09&+fdm97H8>j%*)7fbUU*BP^!g5pv+$;C}rOaL&kbr0}GQg1lkw_i(@+zmgF&}@< zAfgj}{50^CF95W{e;=>F^m*Q30!4k^RHYJ3s54{_N6NjXqfsN~P*eeJgnm-Q(7aBM z=Mfg>9h3`Tm{;5B^#tw3A@jLVt%J(iCGL?c*x8*sgviCSTgTyY`0R!5h$ELvk^+~L z-Ps`S@LMC#_8?Ut!i4rW5FCaH23#J8Qw|frDR`TEKmHIS?})@h@@fh4z|3$o<@lh9 zDKN-u`t9uvm}K5Jl*mI-uNN+M*?j>YoXn_?Lt1_BmLEG_%rtYCVFV~*4WGvu3fK!B z5obw);8Qt{xdx3JU2_hwBYsRmiX&d^4HSZvN&-`H(HAVh&6&E-fFogOR`FMuq$~&Jwq?)Ls;TB!>gtUAs3B z4hy>wf4r16xY_yj=P%bY^81gZHIV#9#v0+Erxe^9Sb|HQBFrXR@OE*lWdrG~{l0tI z6Sm(ZbWg3j*4TgsrW6bQnDYtN*xSx-{9rg?UvbOe`)rSnBC`8weY92>yoEf4$o4Qs z7z)n8r4+`Yb|X$f@q(u|Guxg#Y3lbKgs)2QDAd!)Fgo$Gw~F%$M* zB`^wmBLmnfccq-m_J*|By8bv|6tU#{jSt>!dvpxZJ5ak`%XWgNE5Zv%;Dn^-074UU zDfqTp3e3}Mw?4+yd!Mni46665f}sGW{5Y8PJ_e4AS;Ks(UnH8y;t?9yAa@Y{0cMl~ zD-nOL;xXLN=fMYk_req{-+Tg5jU$UK(r~|bAS{69{;EsXgJP|y85aUP)vtJ>RRrNc3 zCN?X3#`R!YxxL{Hw9123AGyP}Y68)Iop!C(0{ndp*g$g!?7$nR8`Mam+>?UOmB{J` zYFx&Y$k3YzUnmibmxJXHeuI@E-kTzEYXLdEa$OHC(RF^S`H z6qYZ`n!b~%rB@}QP)m6HLSSayz5qJ#ph)0}?x9J*f2v#Kh8D?8|MmU@*m(65(%v71Sv%|VNxBsPo$Aw=}~?13`KuQ*I% zXia)}QSUdHx_e2YqPoj8q1v5ArnHA599;N%e71WT%X*A0ByvSc-3;uo#^pj+Qe|{M zhFi&bK3sEIL7|u9uKc;Xa|j(4Z{Hyg*j7v2-dZoMnVH^D*zStAx7K`a_|5~f8QC|w z5|LdP^_C_Awj}%V@8<7jxaX{KC&v?Pi929jwGS9OR=9~?WJhz!j>1470yfuyURr`A zz5?%eIu#$M(06kcb{dBi5j>v8Ca5}`!6C@;)2b-_NlBP_di4eZfe% zJ?e)wV&~lpRVyz)l8UueOw9HDUTnjT%d%pkTO@;AZm*3We5oLqo=5;nG%5Lk{d#rHV60l|QCO7;LIZ9{xZLkdmF}J| zopv!-q~AmWMpvY+4vdD`WEqqlRbsD8bH;wdLZx3N8iq>siA;z@ylxYoC1ZQ`KCqBQ zhbu~nK#UISc>~23pR7=*L|)p*n<_90hjO!eNswQh{N)yLPpfDWJoiVm}0nP?>Xi-AVfd|CB6S zI#D9eI8kgH=_h2nHUSKvQIm#rv%jXD}f-VUMiXvD-DvE}o^uV==vJ>3w0x0X^AM`euNK}Li6J^FR8|^%NH8b3M zB%E;30}@WS27#g?Z@`>Rv?jG^e#g(5v71pwq@uBFVPF%ky$E71thR7oDA7f(ny-AX zdHAcbq?>96UVht_HJRXtXoIvIlmk5wG8V!N4*EDLe#nYTD$Zrup)04*N+P*M?!S&4 zW0)(i#9oqTN>*4itJ3h7hcS!uhFn8foFG^Pq#WHzldLFv%k|vjPi?<~#pcE3L^j6e z4J2gH0ZG4PG24|tdlgegHK!6LDx-!jN5p|5g5FD9V}g>)hOQ!a2<(v|h_B>rsWFar zvq8m+=GnfJz&B`vwLFLLT69YxoIEdtbRmb=>-K_G@j}fjYHlE~s4Gz=K7m!VCOv=p=f_!lNVi*Q32YB(>2e1lUS)L+pm96WZ??JL7N(~EdBg(PVP}>LD zU~qkkoyjtLLSAuQW?_2YPIzcxGGcfZE}-I4Dq+m05AJ`RoyOA=RGdZw6w0R7;RHY7 zn{NN49Vk%{vw+`*J-@7c77$SOFakktc7 zxx0zlP8GKBjr{sx?o^htt@j_o7AadZwvo{?@-Dd61t!oAU9=hEwg5K zzdHsUWd<$t9s)*#)*={!zBSA~OI!HCV-l3o{fY;=ADvC}2h8)E* z?QfU?3jfFpbYpZcDE!a-W(yE7{NNY)YZeG4p7@IkFm5+sf8zhnl{e-5Go{t?0 z2`*3#m~Z6p);s#l__=)N^0%tj6ssxn288D1C~2ubPjooN?8LbHiBz-` z4ZI<}l?FaOu8~ARbwYo#+A*Kgz2-6&r&V}>P%%!+0MyB0Fca9MlL)z1?0j72o)U!8 zC-hILp8ZX8UOCA+Emlf+(Nq*8I_dOb-#gmg$ha`@L4^xxF~D$28N}Rs&h%i)`IJOO z<=lvKAtmIu{yw^f#RQ8M61^fZ#E=s(!BSCl`K{MAOmSZG5a9~NiMWoC54t%L$JNR) zLpf7U#Ij@6zhjlaBN7-Zfej7d1k@(>624kFXcn^@>lYEJXgAUg#Dsg681sr{4c=oB z^WujI6C-9k4D|W&8b(PeuEi$kw~!*kdWY1;Oci*dZSED3LBg7XvX7(=Ac& z>~{Kn#`8K$6`ohe4i&DoTf~%C+If@IMi{VS*W(A7*zJ}}h(rx8Vc|$9ROf+SHz42n z%$ds|r$0ti3rI*v2viZYaPk#M?mGY7?(c7y(rx-UL5e9T2^liC)LQ}`TU~s3IUS94 zAH1Dy+a>`QRn#7=s8x^83`S$TEp6vTVT4!_jJ_j#D$Jg_+WFm+q4z!b`AmPTw~ zY2q>>83&e{nO1O?f;Ngz-C2so?1c}Ovef8R%LyG*qisy+)UhZQiK`b`u5s#6y$0-1 zzb@B6McUf5h8HFzpmiU=Bl#t&-CjQTCX()bYy5B6+)vSG4A+LneTK4x!&@BSvQsxE z^;cq#A&enWs?Rv@4QF)whUM_9K258MR@Nf$&LUC2NmD6?#?<{cA=CQ-OH8~h;Y4Br z!g*0q1?_AcDNeH`>&L#Jjb)TNS16R$7F@=Tu_WO(e!8rTB@3#oBsh^Q07=LGJrNQl z(SpEpVXl4?|6=~zixOJ|*CtyCeUPS5t+dwF*eYxf)V3{j2kep!g@gX{nx>iH9c z%i<3?Z>*R3&kN46Oa}iamq8gUz-2^?ht(a*l8JQ9d6!dmmb2@~)|_ux)br#j`X~|g zh?+4dAD<@-!9ua6n6ZHGKUSO&R^aNO(a% z!YhhX20VUUwbR@g6*NHzF`N7o+ z`&$~E(P3dFnSJNtVzHOq#O zr0)=EKd~TJ;JW#>+6n8G=4X9#lu1&HHMEkjz*Z|+ZZtp04OjoLqz9QUQT+H=qU&=O zqHvbNMzlY&?;N8xvi9Gojd%T&Z8lL`UTXl$N!Ndg(8ctw;DsX$7ZO0^Lh~+=a|lJ9 zk!UDsgII0RYu(w_ejpdYt!-o+E^J+QM*Pr)%bA@Xv*A|}F@aLZ`apU^GzS}&yhWGF z^}7=D>o-|F-sWFNpOd3yYt@T_xg9vNwUpfEUR9N4iE6$~P$COgSY(E>2eziM$34-#OxDJ|LNp3l6H^g5scYIPg;K7hCOrPcU(9o< z^eW+@=aPYeJNNZAEwTe5s>Ei#1h7hWfHfQp zxV>l<(|m*06SYF~8DDRo?19wgWWMtX6EFp9$WZR)O@FbZ!>2HO8N9v4AC!56d+~64|)JLNW1>ZmkU@#k@~SUuHDE4wb`4r2nuSeSiw+q z7{p?zq(|o-v=!XVeD>`U9)0#)`3)DejPeyve*IDdCQISBR9Uiw7X^8oq~IH_$P!nS z<+=ac#XhIu+q90DIKpfh_S_VCLf}!yR<)XHvRo#+?p@tk!Sy}4j3~IuW#B%T|H~?m zO+VR~wfanahdxHM`WRG;F6c5(D1z-9v9C%{nzsC)Q}3^+&OCvt@6uAD=>V6K_yRU8 zmW(Zx!VcwMmi7*lgqP(?LaT`JPkpLVzt9E`qyBwj+{#>RxBQ|l->(WVBQ$mseGwX= zG}(m^%+qI6xO`bF_FvLkY-Wo6oP=d9ZW30e){DH+uxYNwdX0*UX6#}i$%Eidk^!3u7dLsjcnc9OxbAWn!J&?g4cf&p3631$lxwnUxLU7aaF03+nvYZ>1uzx2gn(Ko=Y6@OW1* zQlGsT=Jb-!(bR18R7X37U*bBKvX6*#ah*#dXz=b<%9kl$kMC+okSm|SKEWzAAp-*#6dDNuEJuw=%VC{S#!hw z>o@7FGMl}NRuh$3bu|$;UhqPXzt(yy)5Dj1LJJ8!EYqyg{)rmmdFKPT9{$Cbv6Gc) zH%oXd(>6qSYQrlEyONfbn)h};#;RVWpAylc)mU%2!Pa8^v_ead12Ujx9&?MgHUF>A zt~Q17zs6?@|J(9Tq;Lsbs)=%sW0etiv1tfdX`aHmDgLq%z3)#i;L|Rz%7svSwum)m zQ^ExB;8=}EYpvA>*&|y&9fk|X&OafJb zm$1RVLAek%nhR{rSYpf89+D8^$M}}d!>dY)XsuA!ITLM_KBu+DX=Ai*rEa(ypeWc0 zXXE{0xDgBX-T~DI`{WXsQ-oe~tP|+{o~Z~!WLLK?Ea+GzSH++s2ko)y8pb3T^dI=_ z&`fXg;u33=+)rm0FhRay4}Duqko9R1Bt7X|!qUv)O;~Hs9}*Q?dm2LuQ(UkrOMnS> zBr&zOuWh}NxfD}h>*DYES_=`qZOIu4$&Rgww}rgw!g@Q^nxwAD2BM=OisXGKtQE#lrpcEyAfly#(Fw!5|Ln`XDV8=XqMIj;$-3RARQL^A`jyY)=`0S~TidQaf~2 z>6@&5=AeWW_rTFKas23$HIW5Z%!x8_p7|vaDcWZgMEk_3&!m_w>3pc`e-<+hi;UeCHjHc32ezNv#(mY@#xmltv1;+^YIxUQ}@HhwYZQT9K@yOn}UK)1F}ZB z3-OP_?ue>b_S7pcVbkIDzb5i`YLT)T}#_|Zh(Q(Gz4D%I+ zDd0q_K?-!vePU@%R?-`HnCKJ*NT`$R&aKYXRMDrp;hu65PH{iB?TE@+L9y;9q(Jgh zD_cCy*&spnX(`a@c=MyUp2ylOaaU^JEgMSMF(30C5vq@g1>lY^q?Wr()~C|;W%dMy z*W`NzuMXXgTWmg*%GS5f>8#v(dgEmk^O!TSL*j`(VVSc|88`an>1Fx;*M={&gP1B6SzCpeB1ii(FT*9{Hue#!mCjIrA#Fm6;` zy+;`42h2$@1&+r=zU z>z@ctlonz90(vU&o8vJR3IWsj;u)VlR+VYcof1;quYo?W-o0Xyb;aT;XkWc}b~xLZ zvSZ4|fEfqwT)p$~T}^RNTISwf4+ zA~%e5h~XTI za8Pk6il!Ys!A#WnQ$(TeQs6~9Mq6EyY^?R%^H(t!{Gwk8RJh;~PpLkV1y&D--IGM^ zz5ccRtTF6aiHdz~En}!CQ(O1|e~|bb{Dj;W1~!9e3SxljL_GTbj!+;mJ=h&w~E$=?-ln$EKI78Uoae3aqN_)DQT z15&0iz6^a{!U<2o{XRD8g}tmW+3Ih?#lj?9mML93NC}{~6A1)MVLsXk_g6VR=7gM5 zR#h+LtnklU5q7F~$~Ev*H#OD>i>oHEQ7eTk&Tj!vF;n@I&(aqNm9KALs%&YJzpE>A zQX2k`Kt)ty)D2Q#_4uB>>at4bCW(m!{S4?Nb58*u+UiDTMdH|751ZC*f1MSrJ^v7? zVw_RJT3?K&C^v3EYM+_vXW5OG=LlIJEds;jgbtCgSqv-z=k$IdjdebLCIPYYv4+Q2 zY&Z27C?B1dr_3K0EZES1Ax*2`*WbO0A(_geR!DDsGq*9*R;?=%5^Ae@XnhmEa-?YO ze8ayl8hYo~tT^_Ggu~)kZ4ei@ur(#Y%%2l?W6|gH8350&*zOQ^elz+VX|W!fun;!omt{S%cxmNs$@uai`xjwU`NE z=wc!lg|HsJ1_85(c6?%iBU*j`*=y{xSm)dGf-Gk=7|`F_QUVrv1Gvi$452lwvUd@WUm>{B@IgMdcV{Io?j2m$1L1f)mC67+l%L?{0E5ua z_qFcI0+2mbh)|I)5P*sD)yjvnv!FeAZUj^FciIR~sCi6Qi`Ff$eNK~RQa(C2uRsYw zTDfkT$kcq6MwqDPGokz#0B;IQFR1V0D|d=6P;*9Yk1n~3d767AEP9$v6vwrM?UiTF=I=lFKPIwSmlL$e90};c zKm=Skcz>rf0XLMLW0|8VLu|Df@NN~ji2yDNx!urvCz=?=l`EVZiULJzi0WhK5zRf%XU@wK%g8XkgzBV%%J75U3oPE7r^ zv!7sTfV$Oa2{Cg%snqH{CWRN=XgqtYOJf%LzAX_&*EEiIfG>N29moHj7kZd&qr1AY zjV%eBnx>O02rEUjkv9kXGo5)^4MLAOUHS?p-YUG{;n?6xsYgti(iQP%Q$ze4|>yeORi%_aY!xt5Yf7e_@Xx$qV{69 z&y)?YhMin_`Fhqo_M3#o=CL*i3r}dG&BE{+EVdV#Ca_l{-o`3^h9f#f%-tAAe%PdM zr?QB9acv@BMBK_cu6Sfirk|d;_#A5my1EXbVk=N{1yv6N@#d}gLb*~;q^S^t?m698 zYD|wDkt>LFqOt-;v3xTXQCAM*tPRVTGCeY{F43&N>t%2p{6%?3F$cEZ&i@=;`W~}1 z?dlP@FuMx57=--G;}QHMt@&=t6b8M=3R>?`kLQQwHs1Q!kh3gdFuguejR^x)2t}%Z zg877RFIui~L;j?xY>mbZlr>r_oT2Vx(g&$rulwnOzjrWkyrChjAv)8PHNIBt zic&sU7Gj0V;2hl~C7orfZbrp|d4OL6i=o_%1s;%-3Ffj|sA+8pZ zmD3fUPUy(unxb@CMGRmot3aPJ_yfhFwG!ILl3k$Zod@_+Zyhpd0g)KN1*DRK1|oM+ z5eHUWM0Ur|m|MgDVQQ>$Cc(zk*i@nU5`kLE=G5<=?!wY7>m;g3w;-_|KfWM1(IZ-U z<6bJWM84|tKBrhW+$q8W5UK$E0nKZ!+-`n zk)uH0GwxnrrKu1m=JY0IL;a2!``%M5ihNbBfl*{S(oe_L1^tD|H)HoZ;K*dw`lcpC zyJ)-=xVl5}8IgE@`)9Jsk~C4gLv3!|bA zOqX}8sPWK6YH{DwU|5UfrCT37iNMKRtK?4m#;}MZJ za*28g+|#*Y(N9_RH7ySHaKqIPWv5OYNJ;1+%hL^V29e87R%oW70W9Z>P= zLUbx}BXK!h#VoOW{dYeu-p7K2nQe(aksks)Xt>5%FbiCT%nnNUe`L?GF>G71vj`Tq zrB0azniZ8FAKt#5X_UPZ4mC>SkpA%c`Mf(7S)JF?Vcj*+eMbq)^j`m?mko zf@=KGFB@38xm7!Y68$+k6b^UCXWXqI>&)qM{0c`+i{SA=!$<1pGi+pY=D&(%xl7g1lV9#v(xj#D=?K1JU9VFkH!k zo@u}JUdGwO5)?TrUfBT?R1DR9QyRjW&cSPCTR_k#WKC3P zC~rsqC1b)%fq&PYRhd1CcJ~2!v3? z?(n#Ym^{hGB}8wqK2-zn&YyP~YcW5w;ns@GTiqqs5xFH?2QGHT zcM#9VDQo-@nGX+Vp8S+f^c`Z781bQj&CdA7RHax>L)Ad~mPgE^|r5I`@QcH^d;0c%t4Hk4_PTeU9DV#doB^*I3E>7VN z*cw}A-og&aqHaVc9uj#fy5BUfSAQ*KA=L^?M188nuZ5t!qq($ze#7gC|yd2VVGL9h;xN?;4^R z=cn^Mei*G6tucCI1sVak49GnRN=|4P6J4UNxL8?=g1zzdqsLj)(y2SGq(?3AJel60 zhsQ1uu+Hn|@>^lQ4fME&Wf@ONT#;pjlrDyxqHikJc#^c)Eqce6&RmZrJ&0(u3K7dS3%VzMHCO2>!$Uf(iyX4+j# z$iku<@`n$5ST*-3&`S06WaBm8vr5w*2`vhS9NMb)UeQrdK~)0JYHMc(Y*=Y)!5s)NHM-RxK_qsF251=7P zs)KYTid>Zf0gM&n{lkX5=D*#~!Y5EYbl$fuy5s#>^E4zRjV(7 ziN+xWOT~NpVYiQmg&QRthJ}p~&Vdp{uP1{wGBp}2R4O{F-jxRzuq5i0$qW%gWYU2v2yp7v(?-u@6;WH9W z)XuutIg~ z*EAWO0^yV{;rnCHkG0tGzkdMHjK^OL8;Go47$zHZOOLT*{55}xC8O@b9ZQ&D=roYf z(J(Z}c+^0L1YauH&vw{$oW-OECANr3LC)afZ29uKZ`HTVb0#Q`8z-#Nm!AW)nV~*$XE|-vdm~+ zy~{gHHiAPG*=VLCi|naWpFmACd231c5089wjdvHCQEStAxc$QY)Qk2}{jJ^x!FnSF$&3y6UyB0+m>B6dFo&V{p`#FJO!VrL;!g zD|2@-vsrf-t$}7UQ(r?*2*Wi!g@N;Affd%wAKLW{QzUOoRMD|wpyI#bupQp<9rc%5 zyC@2HPW!<}*>*aI$8V>>4si2Kc@kwi72C4wx|?@eoR) zw`;Av_a-Czw`S?5$43*zn0`w0;cMrZGle;j_^M@{SWe;!aq&kszHJ-HSI$SL&9Coc zIh#IX2p)4b=>ndzz88eQ#79$GA5$NWvf>M)z2A)KqZw=N8>_NLVuvSP z82qD`q-U{stY93$tMM4N8)KTp+;D2Wqlfx8-owZ~AaQX=GYr7Qc_!5TCfhEFr2 zvO~h6QAvY)P3-Vaqdi4p|Ju8K1Pgk@HxaSI{ZLUslf*x)uFwme6>1SD$J=fX6N?7+ z1Y%Ky?;SZy;~tn)a7{kmwB`%$VeH*1aRqy!G~~1cDkgCtEn+f$*03jUpLIElB?C7T zxfn~P)2%eR1joB~36qA+*5qwsl%_cpL1+Yu8LqqW_~p=FeB+Q}Qx$8n=%=P_*_FIo zBrIOZ8wVS&?drv}T~!e)iN}HkHAXOWC~~wr%TAUS6786stP^shoWDT@K5woy$oP>!MwvOIw8=v4+$n|<1Wsa;s8NLaW)xOXmckQ9RH?Y3Gk0i9{4~BLL zabaN!H#Xmy`Ne!D4^fvQ5B1PyVR;V=yUCv5g-@S7&WLQ}CP*Z*z7H-b68FDh0~5z& zj)mf0jXZJeQP!98p+ppYDPcF7BnYL1|1Db5AukX9l{dAg>u-Ojnug?fMwr$A7^h168%P7q> zt@H17ZSP9xhbn3hRMgT-iu7cxz>lBOD~=tL2^Vr*j`0-BzdTuGP~737=bg+Xm6Q-O zL~jbp96pj|P@OE7XJ^h2F)mm0C|qu&H)Nb|$lH1Ymppc&KRYL{N@$VbmHmEj%e*OI zP#{C#_$%q6f-RT(nc*Kdk*F1wMC>5Mf+~#1oR4Hn`&>S#&tVpmR465Qj7gdsS14dQ zgY7wxX9OO?Lb9vdX-M*k#;U?Zd@%Q09*LWPt6AguSPtHm{TvnKr&&bZB}CX5 zQ8$he?ek0#rP=t~L6x%$=LD|bG!EH&1K$3F1&{|NxM)#DaLfon91K-+xW_V__>vz> zQQlK`@%Jn|m=#upqoFRwVrmuA_$G-{SYq18vd3$OnG@eSLZ~9P^I$WFeqV}yV_LG? zc*wQxS$0B>N?hT@V@QDYbjk8$VY|M3xs6H3mZ&NnvbWd+4P?+gqu(8sEOyCvw@hQK zm{z8+B14jFRlc*_xT6mrsKanuO;&MHm%~L6Xo}z(X208OifAlx$A3Mdh#kS55?92x zgp2pzrwBZHaOnmnX=PK0V4;Sf=~>)ik|JvL!t9D{>zb4+TbEAc>cjEzhgxYnZ<*UR z6aQ{yoV~*pj-O=GwoPKAv}F)BbhX0(L<+ryKmS>qU;5*psxU&5jDct>sEum<^bI4n z)-*yzV$=1ai;xL;${89Jxcpj*!~CMYX$*D)gT=Zb@u39suqF**2|MJA(zlo}Oqj0l z0Mc$T*vTH(d2YYsx!C_bdh-@`CN8>#P*LqPCPKyc!^uc=!;ca=EAi5aoh#^UQwspn4xNfjuqp@Em07Sm)`0R62C>lW39m&nh)mHCTIL9O^d70$J%!dMj1&B6^@4wxcY5_5gpA&49Wp*{U(&Jh!+$QA z%K*Pq0c!&iaMbBAjesaVNKSTV_%XI<^*?8avaJ5RnM9$;>WhB3IQo zTDv<{G%-n*JX*6OkG;r}BcDk$OpesmA4W{@Hl^abh}6CDwbxXAa0F8#Q)Ur;x)Kp* zT=#8*fdB-PkO$zRuACi3f=*pB?OrCBnX?I3_%6Ec&~aVvu&Ml{kZX3es-!=})W~)T zOZM7d9L0Ax^fXy~I>d6H?!9I0r>qVfokR3uk3hOO>q6uK86KQBiMx_z8P~$$0^{xu zsrUW%x_Qmh9y<0NF6-vEHJD52H)-Rwh8T;WB%muE_uzV7EN*NmUR<*0LUW0y*zZG+ zN96kKfiiph!gdCVvAyT^40(k)3~$SYMI<2oF?maih>H03TJMEzN`)~j1L6J;eH3OO znqfH}`lG|b&jHU&u8}PKyVnm&V@C6`c?2&!D31^K8QKm5-e`R1xaB_YS@F>>rhwN< zRH1+&%7Q0Crku}#%+FQR7`>YQu!5z4$J|9!iWD#!%S3oessV;t+85-gx4CAuDnHB4 zo=-ezXzh#uI?tDVd(%N7Xoar1YW;ahQ zv$P!W{BdR^-jq0Kbs7K;zHfz>G{yOfih5I`;30E^eVJ?Eyq5?Qu7LoA4^9NA(Acqq z(xez&(quF1FtaTnNbE3c3r-P#Um)r&$1B6Yy21E}wN6$XyGOcinZ{c8*UA;Jb-!)Q zpMfV{-S#rC*fRZ?L9y9nkd^kp+)gw**c@{qc0JU^CVmcy3rTyrU4 z%2{~5$=+Oc7WzLxV0ael>2|3IvN>LpM^Zbc<}Z&a^<2Bs_+72~qdS?V`B@^1R(Q;N zy>)fM4@P#P_L(J*KXl>=G3%ZMA+!mG=Y!Y}x+?K7yurwQ4C+%el-RgrK7LW&@g}UZtn5*u3OmcLLaM+6Oa^J|!sWLA E2i;~pPXGV_ diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766427132.gpu-04.112924.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766427132.gpu-04.112924.0 deleted file mode 100644 index 077fef1cf8dd2fde6413d41b1aeafda5e528c435..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48602 zcmb__3A~L}_dgmqPa&DdYo6Wtnq|n4c_xYD+;h&o_qb5zcmZBTpPZEwBTlW@zd-?!2BkHl@SyZOf}fZ2KNGY!DAeXAVnLx87_7j3p}7(;}1);enS3JU!8WI#K0$nI!> z{ypQ|{rw1Z=SzRyZyPg~K)Y(4wT8$`M+CM!;t2q}+u^#vR%IKo&wX|7gHZ%)AKqua zt?~5)+h6OaH83Fa9nNC6-^E!_H^$Q{JtGl+Fa3AY3%rhSk+H_K7ISayL2ITUl@*8j4fPBpy!p2KmfL2JBbs|?*JzCkLxE_} zJ|*f7l^QJJbSqmoDil>@;pe3{kG__mx7^a~VOy&k2)d)zUaOB>)`7($CVH=k(OXv8 zxILeJHDU-szP#hIO}6(X$P$e}<`;l}76N?jueyxFPp+ik6GB=CM( zAFWNu?R7*vC2oVu;z;L5K~f^7FHkjwB7Xh+d@oSr41>~2@6OK6iPSa;t ziOZ^~Wt)$tulVTm%6n}yC(u_6(5}h`}R=z%Lqdol!aWFDBQ0U4)8q zT^mvD?qJyC4fr{zvVB?ZcA1LDaht7wJ0{oXB&3jQM=s96)8)1s zw%hLTyX=8zBp8j@r@B3bMP_4YxzN1IlO{54F=jF?B(#Mu>Wz3nsL;(bj4uBO<&2m_ z+4p&ckC}=3OM;_`Qd~V3XgeUE<0{ULVo%<9`P7HZcV1ge1f%a#)>pQGfSf;9gy~CGOW%lAX35fP04IBv&iPT}wZTmrEKK?^Kq7!}m`rs*F z1ZXAy|H4e4_w^Gf>hq>5m0&`hA$vGd>NOpWt78sD1<*$5Cq)d+`|?%I3R#$URxW^H zUJa+$6SNnG%;!S24k~MBZ$AARJG)y0L@u7)nhux4XU}&>99dkF6u8XvP6lxW>k9tu zL8?H62@8V+hhc&Nm(Agn!vt^&-kFKNY-8lPrYPi97vzDN;b_Y7K@*c_koV5hzOhU) zJBA1jMZI>o&}H`pd~h0of#L^p4G}1YAY%Z;N8Bj${;1ah z$q|!QQOGZddc6jxD}5R0$H<>vqLAOlS>$#W+Y17ahE z>ii}nzvWa~1Ice>tPu`+iov~sCAj1%!fdhyH9OuqbP%1j^Y5>D*!H%B?x}Ux8XM5S zltRHDb3VZu`}~VHUo(=hE6y5nk1aZl$i7bNtF^-5E#%2Zwudpou!HySDG@GY(&S&+ z{GsX0-K$qhOQ5?~-`H56J5m%dnR&%Ejru*XM~Z9u=I~!{VAf)v1V&+RWB^;`u9S0` zYe-AqiP_Gv#FFDXm)>D3oKEx((gtYhPVjU^cmWBVkn|itXksn}-&RY3`RdR;8dL9; zX3#RI-m3*e0ZjREFzbE%s%F#2nJ=|fqKPaXp@9u@2jL%JMmew&@n!-Ceue=rs0 zm`OCEimK00 z7}~TU?T$Td+apm2YW=l#;Pit#29EP&@c2QDBO%Owg+x}>@9>$}tn3-rgIUk7dSV!@ zvb6I3^K8YliS|C)wOR}C_c34t%^k1j6Wj#9GOu!{;9e~bNFMt1rwiP(`J^%f@rwj}$m z2k-34aNo1Wof=QDCGMo$e_Jtj%(zuyM{~)J{6HWAHrIh(T7o3L0`GS^6(6Vip5>3Q z)7WS}!Q*Lcf~vzAjDmLq$!m)=kb=!6Z`6YJyBYTDDt2?XFBmDcNByuy?7VxSYUSle zQn9v*i5W2Ky5HDw3Ef6?i)4_?4fenXREHxTC;03xld86skDm|S_^Ok{Z%c92!?wD& z6Y%xg7_GDJHBppS1SJq~Z9?exxq@IOf?ki)5!O?mB@WYvSn}zs$KPAdws@~xQb;@` zIi29Rf-e;W(-R3mi6$jK@S7`g{$i~1EKpe05QGNYkZ`HrnJV2M6y7?Hxgu2-5-_?V zwRB)K%qGjA?5GlZy`S-J9TqC>kZ2ex)gdw=67jlCc$SQPx&Mzg79Fm;g9yanwYE1< zXz|Gkg-YaAd0qQHjKT?b5}csWJ~=7SE1RkjUxZNk`N zM;N<;cdP93Il{%DOTvMI2$qlvqG2dKaBZUO1UEZB?zkh3MFl>|vqQaXF_A0W1%XQ`xzvDnPk8n;<3sl) z3LhF!G8{`KDY3Wqo3lS+F7OEni!N|;U5A1UKw46daudQ^^Lb)MZW)%VTd|bL##~)p zv}2-TjVV==NLYiG-@LvS^YnTwBT(V#VPQuw8j4VEhe^<_QD@!T(a+S#Nr@`5^@utJ z>g9;cW!D<=?P=BTXNvBwtFmUCWX7(_3c^NX*TTRiTzdh;Tv%=4yilTxEHz)bb6U%HZXn%M zHL!J$ZMTFUstwjMQ4aJ#$XEz7IOyY~_#x@{&+5;zL!SHSLqu|m+y|CIz}D~q z0v@If(Xt)FYtb!*aPqtm(uEveuiFb&#S1mBsJVf_k}uc3vSbO9`B&vqLgxJtnu8+) z{-j>;vtx~si=_27)?E4QFU$z|AEeJfBhUaU=27UE0{@PPPZoF6(5wi{!qr$sxFQRO zV&M^GB&Rb9btS6AC$Ngvq>p>!*Y2!6<<3~ciY_O zW4*$OmXG|zm=kzNWe&o6-GvTkX^OZTzRs)BgJegP8frdFxL9ha;e%{2xITr>WSPC@ zXxj?R!n`2y(88p}@GM+F#idljm@m(7>&i}}do>X$PNM+|WmD^Lf}ilxvLBpaR;%I~ z!bGc8Uq*~!2v(rNm~6OT{-w)jGt_+-Ks9^(mZxq@$*+5ux}LL^Xhn6M74!t%Xk5*Q zTO?TiE70Y^=v&Ly{lfg3HjmIs=-<_GWkWevs%b6~iiU!`Hm2%ahK8Ena9(fbk$xy) zg#$0Gzy;tfdGp}Ly>Bs&-}I=mj|~x0G)jSXg$@vJ)SQk?EOEbmGJh9Kk+pt|P%%Z8 zq3cYS9d$OLf&~uvAaMxUYxLMTa}qC7)A>j%kuyu$1k`5?iEf&DcgdKc!VR{O{xyOfVnbKwlvI2|H#l@0J%A%>2v$N6Nf z|IQEJT9(CWGd9vnB2E)ji-wkzk(ZW@W?oMkaLvMuJ8a{hAmqW?KrP)D`2X|;6$X#) z%#>J{O|%Xwu{u7lBXzEv`FmPEb17;)NthzPLn(7e=z?m1+?2Vs%$nUx>OA)wGiYU> zB49LVErKEFTf^+L#I;c8SHy4z*jwwcHtqm9yXRZ#F}(U%Y(qGyXDMNr5n3V2O^r=r;OZ^%?E)57D;r*@_V zuU_`uFub zZK=+fEY)QXzIOxTZ-cE0e=B=Uv6>=pKxk&Bl9sx1+rev?oj4_t(B9YehV)h%`1rU+ z5(U)>{mE*_{F-BHTCg~6`!=Ev=_@wId`p4q=A-S)q|MB}|> zwWVdXD)E`YXpFbT8D11dh!w%;JJP4a?3pXW@16?1_u)v1*OxJ9kYh~Gc0>K{G0@T; z@VCoK&&x0*H=O9dx6AF-PHb5wq8f{}jr;bIjNL;1~zLIPU%@jH@V;^AWrU!6$O{qbi1E?d)AX^D~A@VL)VmT-6r16+1$#iYIx_87t# z5~cc#^M1PF`hzTo|At%*bNJN@z&ndX{U%MN7#dUe->lh=rBqku77Zp{| z&c>1AG+VO%!@1vXWR!M4K+vMvAh?VjV@bl*x@X*EmMp0J8o`NV0Z2Ob?}?Bgi53K& z3%hBqIrS`?W)4A7lprQO}SZ8?+3{U>+_bVmz$wP?k)jYc9B)va_6B-Um+d=Jfq?H4*iQnlUIJ zpC=5#LZPMPSbt8(GB;ObQHA$S`W(^ffS-fCqjXYZ$>n6ZHTjlD53;E3>bD49(2wwn z;uL~9H4uW1zGN4eP&qh(>CJ6&0Ws+5h(hxr_MSND6Hsy;Nk_8Hw{9+A)_0@5vkgJXtW#_2|H0qd~|d#A*eCSLoX&`V4FNEE^qqfTjIT%LPT+ zPb|n4xNd%}cEWn4f6l(}I+LU&@6x9U3v9KLR zB%?}FL`mID95QJcv$}PT5SXyK1;Lzb*!#HoR!*w50DQWrMib`l?~?GM6GCqp(yvq~ zcGHNi9JEe$jZ@Elwu~i_CwxMmAd<+&9Tu|r3xpWID_Q(Te^y`4Y9(bqC48)vWH@96 z9WFwUjRkt!1)i4pg_M2es;=MO$cxO`T`SiSZ3d_t>|URp^+P554K0ptUBxC#`+i2> zfRm*y68A~O-+7W)hR%Jr=_jV6Pf2i5Hr=NnZ0d*-tb`kdg=XU~$~a8b)BUvd8Tv)h?0os-K5S+ax|1$msL;2W;U5?7R0 zG|v2oea^Bkl+Q`WY#H|46nH}5QO8!bnrgCKrqjAvy;#Au)0ec2D7eaH;69lD%PQ5r zT{oPy`W%(3h*lqiYS9H<;t55teIxc&2};wJAN0mOZ{NZ^fqBR1gGAE-E+z2=Y*;KA zTPlSeS}^7JSD7R<{fbr+k|4%E^{GnzLK{4c`uB-(D|4~k@{8WpvgR@)v|p|xnpb5P zLNHsOP2uuot=KEwwC)R`bG*@H2#;BoX=CY8a;y1L02uaW_ z() zC%T0{C0VZcRmpPw&My}8?AhNES#aIuE`UQF8ymEXVG%ld=C4JT8!uYP7*Bgyg60v!E5(px20XbMJVnD7e&mu z@oe8^XFW#sDG4upEzw9WtTK7BoEw&~pR~AjB`YA`^(|2?3dk-P?LeK7@XjLzlq=`= z|0utcfo6Ya2Gs!*ST9S^+MoYji)pUI5>)R>0ZN15#}lPuHn|Ayrm*A85BzC3-{gK; zAIoZg-rS1$TQg4)`FhbFR)U&F2s`+}0dfYQ=MqPJlNP{sEpFbyocus%5+gKr)wGWx$DxHM-ktvO{c=yjSYu%hqj zOGLp{7s`wIj!$>rQOT;$Fdh)2h*4slIdy6yY%CxJ_ z(RW~(wjRP$8(u-!m9(t9GWW*6S=DRwPlPI3jrEorY%SJLE41`DAOl+BF}HYI^MAVti15zHwTWBxDng&1^XqCHk!!(e4gdeXUs?>g1wqrh)SRBY{O3@J=;!Ky3) zCfJe0)K>cDp>oWnnDZM^i+)ONCk>v*7iyK1*!UK~>|hIJ9KZ5kT|QuEqWSLxD(<~1 zg4Q+%TpR!*B(6S-r`*J7MP|7})NT>HRK2XCQ`A6TocC~>G_Yk zJvtoTczK~Q3Z8+zf2!>OOKX24A;tU%#EB3vrZ8B(>E$`6SeMA{{}PGVB~lZwEI}RW zF%7y|?{cYFzfXP2T| zpUV84!p9lj^2>=vQ4OQpaSP3dQrY_4%udR!r(2tU%KL;nmM1*XCoFT;DdYC;dFck$ zfBY50m08^+aJiX3USsErEF6Bj0?~`%Fx;*M={&gP1B6SzCpeB1ii(FT*9{Hu%t!aF zVT_$)Qy5!I?-7Ri0do>e!DEHH-*&X_%`8x!MtGu&3GWiX+mi4m0_8S*WuG$Bvy|&A z+U)zaGqXUSNjOnjgz*dLslacJ$5bc;Oyi5MZL#!ira|XjK@^JnHP8pvyH`xIu2@_J zt=o}S->{8obEUE|V8%gz8}}4@%mc;N5+7MxbQ|-!-jTq<>(YUZ0v$4z^VAypuEB3y z#}0f+MWR_u7P(=hL+n#3*W%s#@KqVi;cZ-rFona5R}w-S38xpcDyRgor@@?vQY3kl|ImDjqNXYB@{1+0~a+WO&;dp3jYU4yJ(TZSrk5<}Ht}LiCC81O0p9 zq=p>!IN4P=sJIlD4W8j+CaTQUgs8g|c+rm0R+l6jyS{kzN9KY*CP9S@9`O|GBUxbe zaM(Rb)amwGFSEw50ab}q>}zWoLq(YiLpuugM~HPynnBqu?(BB!vSB+b9e*zuz|wKk z*aAs{bI~fO^u-d!lf~7DdJ$ziTrib^opYcm;)YnQbwkmNOXjiTl~6>d9vLFsiQ?Y{5d{eS?^> zad}La3?vri2EDHOf)!7j)Ffyuo@N=)n5TlYklu_T6A_}C&Lz>B?IZu!>l$X14#=g@ zC{;(sT(I+t^W>&qsyLYuo>q$p7e-jz<(&fD>a)x5Vv<(BHo?*9i-2R8a+q-I(ldjZ ztMsNsMHj50&P=cn`H;>| z)-c@abrsw+NSVU;GW2<+8ilO5a{Ve+4&Etou`mgjWlGl$QUd7hL;}HLn2&bC{Z&qn z*&$cTs_KQDUo9TnjGgL<^@w&n)lH2x!s4n4Y}862i!*%o_p_MFzob5)3ze^LV5)5C z>Rxq}IVtNTs0hxCxHd_ z(fX`}72}K&*7{;JMY(YU((Z|?GFWzFc^VO{&%l6Tazckl*enK?fV*1?yKF5&0h(j`!MXivE4)5r|wAI2!L=$SO+Gu?fzjCB# z?Ht3uU(~t51Xdi&XiRWe9IFB10vEQ%SpenWWGUg%Wh3kT%S8J_35KFwpO|kk^{IR? zi;)^e)7G=n;LIjOA(jT4>L&%{P@t!Xy|?A5hnR(J+?23|g%#Mc2E&b$A~W3Me%asn zcP51UB_s-AZF~&^W)bc9!~#dO`hMrcmtJ5olcyQcC~CX-)C$gG(JW00iYVvE=3UI8 ztKFP{(V=S^51e59N~u5z{(A~a4cS`oY^8&Y^4H}8A`GVmu%jj)ZA^U4Kw(LVM|oc0 z&o$qE#EeUz1yL^Y5pb6s7(#1UW$z*$ze4cyPj&p1?OolLgo}Fz*Y!X+9#5q*zz*dn zxH-V!K$WT^2D1QUp9I7Jsg?jtl&@AkoSg+*m7PUQ%@?#H0)?8#WVLAB0^8>_X(r{P zv$FG)5M+1RYGJ14ZLJ9t)qF!JKL)^?!qN-syZFkTq6^fV(ZX$mKVzQe6A~6Z&87-0 z^qB_3*xdr27C#aSwA%LMEPRspZH#S06eDzF0WHp1GOyodXXYx;oW1$ysY6U;|CG=| zWCe77AObENyuVYNfE&urvCPrNJsF!B@anciGXggezy%?<8+z|V6Qj6tg>yqu;7Uqm z^@G2^$oSm5ox%(lDS+)0SGQQfTKjOjZC_Mle$WpRT0}PjnkLyJZmhS$-D+^5E~i?N zJD6x!OybUP=gLg(mZmFm*Fc~r3){gsa-WS}x`mbaI%W_omiXFQ5e*MP!I3e!w~GAb zC@1EoYbqRNX@E;IX$diNKBd^|Jtl=0+-U4C`RPj*`finoqH7w*JHVH{$d2RB!fiie z+c+VM$Q3yt0;i_wtM#N3T> zvNTJ<>!$6j!O~cNBAA>+P&t zZPP$zX)1LhaA9^8a128JPW#ykeS#R^*QQIF?`|SKUS6{3`TV(sxe`p z3ZY09P%xkH?M2Hq9w@zHJ6q!yb&b{vXQ=y_^g$}u>pt+?un(CyuIi$QV@qWXaUC#C zqQ;k^l>NoabrescUA|t@u!fx~(VgOrN1^Sf1A1D;9mC!zx>;j#dl* zfyiA@z=0JPk=^m*DO$`Q!Td6Yo;iCB?tL1I0Ad_i!s zN3`vOgo!!5N!d`p@4cT^fklz7ozL@}Tx-y1y^8Nf!a9+&vy)(63NFsCt}12-401dxkyVN}$C z>GF=<&7bVSws+*U=Ix~|fcOoHw(#tPsXbM}#)5O$V12bYi_bHm{8eI$!4>=jU2Iam zZ?bu<&kC6@u%b6nE$Tcv^TF4J*WI8GFIisfn-p%rM5R+7v#6kIj0T;h@V24(>LP^? zCLUk;*FFbxO+S&i=$fX9t%Ns3@C~y>H{DwUmkvJiH#4ZW_a%Zwod*px42?%b-pD2D zC2;eR`Nmwu$55i5Kc5;hn&lvR^&@Ce4~Oytc6Zy4rd9|1=kV1m@uubUqPJwP|ETrch6<lgCwRAhBt|FfS5y~b`*s5O|7@g{|;KEyOhqZQQF zm6vC*bn`9=CHix8C>-vN&$wGbzI%6W9@BTuAw-?fcUVOcb3c$^OX0HkZgpmk{S!vy?5TA%d|S6ssHFyA1tu}DxCvEi=RK(w$33|F$C-ZS^z z%{V)77!iz|6|d}o2`Yr@z9|i1O=tIKb21s^4=QARe9Zz}M0;jMRbyx!kz2xb;9_Td2k}Cj zvc^yElUJ2_@_Qt{m?TDgXdv{$y~sFsesOEe|MB^MnJw{+rEfr6QfDeuW%9A-5j$G= z4r{SK^fHFbw=Mg}S!O}6x}M=?w%aoI2bk96>8AOyLgL8hcjX z#tzBj5)%&zzRxS39F5^ZqX{em@4H5m_RDkk{&WS?PLswH*{Gcw8PLv12#T)Q5yJOh zkz2f|N9m=#^tI~im_}`RBSDLTjRCD|Oo+*0NRxvnR(tIWE56Oo;DAeOvI%QCi^NVp=)2q|3*IYr-8tnnmivui)ruN<=q?@4fUJ<=4^gdx9@DE7gN zE%26?-#FN}rXOCz4o=iggbOc0f1FG2I_Ep7h>0nJ8#1%`UyR3fCaFB81cEc_a-fqG zG!UQQT?2)+WDl|0!<%JhGE^cD7B7aK9$H-5gUc}B!v4bcU7Tu6PeAvUP zxkrIks;8I#@^>RvX{wS>(4t_-p{;uF6&(c?R3-4dW8k^*%+=i{(S@r^ti~}=9N|lL z1JA9`_@`4D{bynM4rgsQIEm(Tt*v%dTv|{ekE_s!rfhwI3D@5e2!*RA(v>K3RSEJewe||E+Q%gEK+$f_c=T@VaPK~b*Fg6MNXwK0g!Iuj5 zvdG$rtZ~g&OxPkO1v!I@v*pX{mRo2t4J{s-c$g8qQG$vf+R%m5d7m^94z8Xs%wrkCL{?tSpAHj;etPYEhL&kbwlV!$g|CcM7Z2TjUg=x@{ zMfTLGPoO56ytSnJv$Z`BF?KxTSJ)v2z5zpRopDL}AT zNou7NEn%tIh#oxU@JjZEO;=r)*Karrj>-pV0Su1X@&$}>pp@3A)a~2vnAzMQ*Fdw` zP+vn&2*Wi!`GE^$ffd%=>+19}QzS#C5aps{#X!Y>!(ltT<2&jvwsuhz@IJ?%9?rJ& zoCFuPMF%(70dAftPoiw6Vq50!8T1|7&U-`3cDB@EVJr}u7$G~ZGb52-uZSVf!t8l@ z)^^tA+c``N=ck6+D)^YGgYrU=30sU>_A(TXK*IIk;M_!s)mZhJn zO{Fz3{Z!wFubpGg6y`wUtClrmIf;tm;*WHE+cuJ;oR5(ku6Um1Y_>~y%-J*$@Raqv zD11lf8$V!{Yuq#?3NMcKU1x*m8Ebx**n%|@J3Qs$;Mbr2{v?aXR+SRz zBIcza0o#o+O=501wcgQj2L{(}Z++m^ja*I;9krw(}~D!Uxl{kFspTZxUYg zs44KFs6Rp8$x@NLHfh}LOqD(`gXk81qyj7+vgInonjI@U<^0KvO23%|i$jM3?uUvBnk4>Vb%kE&tWb+MU#y?Mnu*22Sqa3V0N*=u7RNm> zso$Be>vfbk+67ARO`vwU`3AY6%K3k?zP_vH5wz$JhIR>YVPOh4Hh;3K{1KL0 z*dY-`;tFjRmiMr*o9qd`yS3{SBhq!VLS!8uTv8#Mg#l5OqIHnxyODX?9 zLKJ-|VK9y8w->IZdP7a zryOM_&wa~@(QF$(lM9H-FtP+ME8+3iZahS3ht*=j-0o$zvPPf}B%z`aNcsPB`Yc7EQefP*jG;H( zp)j-o0)mAEF`%zc686i!{&@_x*_~!s9^ifnD?C6!EnZKS8nGP~ zy*-r$r;`^Eu_6(Ns9_xI7Ig_usd9u(PTF%F4)kV6;L5uQRA@4}Q&{12JFu%GS>VEb z505rN>cA%X_)HmKuL}P-bag1o6XPPL@ z#@`OAoLx94aQ&un$Z9`x>~j`C{wu*niz&21H%Xkr64O5B*gdV86F*`xp^DhfgUuZJeJT2lX~}M5rGayIu@mx> z#1%0Th6Gqomn=^fw%5S{Uo+`gv4qGK(m{?2c&1M5p3(1)N)~(F&^iqmE4nUKSdk`4 zwkqFQZd|!qSFK~XUol+Nq9U;k z^rDN933$pG8Wp(wT8hJn&$n!0uygKHcmN4381d3YtqAk5CJkW;+hpm@{OXmaD-|9< z+ARh<)#JLr?RPvE>)YM#G&>WoOH@=ljfqh4{cti8-SDG?&Pu#A@ReLIn_+Bc}2#$8Zn$%2_(UYx=XTI~l2R2(#Td>*b}rc3`5c~6$WxfP!5z`XWd5?Hhi zpd=8ixJWFq!AQ)AMV(sR$GrAQ4-(DjwbvEa0u^wu2%J7HVM;6t>e)DCn>)@~*n_-C z@%CJ~RRoDGTs5(u37$xFcue(Jl4Na?$)x~-OM{ZBazWNtx3km?Mpf~?ligjr3~=? zhlx}KtN}>CQK!Q+0;2dJndzP2$JnADf13I(%j)-7O@Jb+FZ$u)=uWNtQfXg&V7kV#$%)*AQ)(!KtM`jF{kUO2u~(se9vV7gsxefGLsmwS=fE5pl+K z-!>QsKrjh;050mv*-<2@%dmA7m|%V=afR=q>kb{)OKJ<7* zuFoDQvFGMz7%0Z}t{eJZ9p*61dW=3x_jzHZU2C{;8Yw4UeWDdoAJe?Rj$!wqa8T2)*NiWS`Unb}!c3hs!HtIc56qS{8n zMQc$XuT%>_L)d?EA}=eaVUvRUPn4NYfER1we6J1?X))71ZO&jirLLp%PjTR z{8EP*3HuX7AsUIgfP?Q_;U!IRzM`Vulqk4%`mh4#8kE~afWkEpfbhYI02LZLv{#xG zEAJb97wa&4R6=5hSzB<5`1=A;Zz*0G2G$M6N33t(xl%0jVrwI(tLT%kH6+kw}Yw}2H#MJziF{PemR~o;+&+a*ZX_~(zvS@|Jyf?(` z`rF!^Qfn2yzE4*dyBl-=Geqq$xEnJMb}H5+7<=MR{{*a=z`Iue;SeZXj=ovFGK(r! zZ>6O$sz^iQB{~OF0v-0^gq5sk-MNiG#cPG~rNZQCx3n9l9>{6I3~8ljX(cqIwXM4v zYQ^@olHN6uFWM&5>*>Zc3N&L*&C?Paoth?a^6~Chmb5iOiMd{!ZNF}R}_`_dS}ze`+Bu%5q7f#Bmeyj7QJj=j$j!Kk6A_!;ljHIMPQ z-e7zLX%tM{r_25~C?C^>?S@;l;N9z+b|uYmVEft;HqQ=PCriuFYT$ikVt$FAqr^;Z z_WIFWR$lC~ld#0YsThEY9T$;e00MD%OO4=Rej)(HDpMC^;$=LxjSsVN&$`dkGC1y8 z-38-f@egDuP|8u9vSLpbgF0Y_5*wGy$1e+X*upx?{JV%M>@2H>RDlPW4ARxjx?KDJ E02PDBGynhq diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766567162.gpu-51.2669674.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766567162.gpu-51.2669674.0 deleted file mode 100644 index f8e5e1701a1cedc46563f68cac416665a4d59d62..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48602 zcmb__2bfb;^EU{Jxq?)I1(sd<(w6NM5$Ot21Q8X~m`$>~Vbe(20w|~`DyS$HEEH)1 z0#Yo1NV8D{MUnC<7ElCHv0}mU&73?X>EDlF~WlwO)jK_=DbGTV8y4`VhqJkgH{Djl5xx)orVE27+Be&Wezg z;QH3QxJJF|^)9%$YTpZT3kF}TU8GgVf4?;Msw*3Xm7ZSHX z%hMW{xEv0r*BW$9aPp5=0CVySvZ8>!Dm-fk5a6lfC95s>%_2g3Xx+4CMMYj~BB0&7 z=5&jKUisUO->)Ljjc=TN#*+0If%epTXz9pHTL`v1!tCpVipV-&$! zM+}%|Y4JG04$`jDnnxiEZT3>9*TGrPG{V!meL^8`U;TG7i`=$gNpy|;XI}oTH?5J; z=$FSW@5wdtwOp-nX_?htV)J^PZUf1+BIwn!RaOkrg0Egfc+)ITJz*((f@too_0n1v z2Yg|_b$r+vsEo3N)2(dXs6beeg+{&q_jo^oUUN^I`IeSX5_C7MKubj~>%d|W1HG3- z=&h=3T)N#hWf(!ex#7Ze5=yW(--eLoC?FjOPxj9`6 z6!g7ohPNC_=vUbHFR@IRO$7JVdTXs?k*%Rhzqud>e13%+%geU49ZBGAPStqT(%~rr zze*dRbqqM&wvemL8RfD#(uHA=l#t;IR866X->QPi>zMc*kSn10b&9hB^2^~0hHPHD z;WNy{W&Oz=y|1UQIQUuZZI-*9rmq;RU9EKt2c6b{F9g~#seKq|a|abxwL3CnSbtii z-%XiMTQZ)ZMY?NUwZ^nSp)F`2x^84=YvgB%pXj+M+&$uTCX63SOcchZl4xhZ4#%Dg zxXO!6MKyJ5S5kivky+5@(uJ0Z&k~uvw4PcTLAtzPB-~D0nbUCI6*)9~|Mu1uSvL{z z1F6neEa`Iyc$hXs%Oc>QbG+3J%2X`i_o4WJwkopOL%sn20Of>oZN2Nc{25FS?v)Fn z9!%AD)9AtB?^t1!Hq%JAI5I|s4z*?Jz~ z;`wZ4b2#EiU7|#{9y{lY5ybRopF6zC@~#9QsP)&{du*jnYnjh(D-_lXj!l^>=qd!G zUs%bPP^f2`_V2eVn=(UQIiILUL*B&e3wVIb6T~)hL`AMg4!v#|{9;tDbJ9l4W^&!* zIYLFbPC-zY>?}* z7ayEwB^FV8-l1s6q3FqAMo4T$SMq>-44N zDu4Daw{%`e%XQWAwZ~x0{Mf~}S?~gHN@0W;z*}?H)TfphCf~HLvGX3qj zi+d80J6>Hl(lTKY5t*%Z(dq}wUByN)PoQc4+iWyM9byPe?cv*wjL2AY?6cc~RM;kHnd%pmHJyS!cMIa$CHFh!n~# z3{IER8nxY4o7Z9Wg+u;u$Xf1n6_*%|q3J@KuH1V!(-uRXr-g*J@PyqV7YG%)c~PUw zKSDVp22uX-tK1{ZMExYe(L^b(o&&TUkk4@y=SHz73pccC$$aP8ONn6gotu)In*?Cm zu!%JoVBcj5V2aL8EBN|0a}}1QD&WvFzYb>h;-mybdyxi?1c*fPu(tBGyP1#w&~l;^ zef(7Ll+OdSUaJo4nLf{Zfk088w^peH6KW4wgP}^d;b_#0I20FyHbOrs6xF<~8viko zg?ayz3t*Vn(C&8mt;GT3xlpZx%G$wykMXdxJ7ooti)Xiy&0+Ic3!NcbHkTv?E-SNp zl(;=Zo_@49sR9uuv{^}T7$!vFayXoFm;g?}TYaR)Q;fV1Bp#A?nII3$3`bLr51N?# zD0v%q9LQyoIbju%hoYVmEOuBuJ`bGCu!lpMeQ%@p_V8Gv`HKW7VhxYW9`IQUZ6SL} zoZwSAj@eNf>n&TmmmTq)5>g!TVz;jltW*M+ii`f>N8Ns92Pm|f$ixHGJYe(qA>1;< zRCb4J>e7s=k223cWevfi=idQNc)$tXvI1;%6XcdM0qOjW?<@DBRudHO`+tw=NceIx{?WNWtA0#({K=lz5q7QHhdp8$^AeGLwKh;|kHK5ORfud4VuWD@ z@84A>T*!pUw_Ug^jk$aO$|ca+}@E0(n99W6? zvlWkFdHK#enTqPao={Omr83l*e3Oc)sHjCJCzmivf0USlQcQX3@?{PAtie*3!TVQG z$KDd%w5zP6(?-*#4eNY#t!4EFqVyVVkk%QTesIUYaUKUAKZtQCfZ4Br$f|m69s`?| zJ>z`qW4@p4ZJK35{+E^WUnmibm;IFxeuI@E-Wwuua{=l0^ux1Qu(Df1W3bXhZYAi}Qp3qsOybxqh2;&a zjcLQwQpF}BR;VRBegQDEPLB^Acu*wpcP@42)YkdvhYML6sOjs3jcK431RC_({J|1m z$m+L+N;q^%#Ivd`e=F1PvX{D*G|aJ)rk&fyv-GFMf7a7tXig0k@n_O-2ewOv9xrKm5O5F%-GsopZx9hL24H)j~ zbK)Ay3JSd(_pxz{TMwth;=XG5EX(y-ynEAjjd31*!^MfZXkJWk_J++x#aa5-QZG&{i%xG*6HzwDy?BJ ztPwfyZm3$h`H@tttzu%vFF!n&9hbY`B)Ua1$l(Ng-~p!e26KZZgd0# zY>WRVmlP5YNlrUBuHZ}g!SsZDP@+l74;**6{ZoupliyZYl@3A!Zb-1wYfqN$x{LPi zV6I5UI|PibNMjus4YR>AC_AdeUhm|!N?}Q?_az#JN==ANh(z2@1D+{kd*+5Fu;}pi z9Yi2ThbeAfvB@VZ6e^Kd`H@8{7=;--2~JSxtZ;=)@JuP(^s=*}4FP4A2+A?DoQG2C!g_bU$gRWvP1dC9abGV%e`vecytUhm? zM85O8V#~1IglyGr0RtGVNkcl?fSMaIuyq3Tma;i$S&3?Y5QKz;Igg3PA@% zD%(+aZOiEWn;5(5yh|$!c6n^UQqUzqUr`84NJZfwlpeS?QFelxowI)p-^QYXEfQMP z912}owb*6g}WedDJ7R2 z@LSccY{mG{@I3++sQ^lbW2q!5_Wm&Y)o+;#d|1Mw3*1)Mp&$d0meix%xbWtD?l<@O zuUM|`&i9FI%+)nTJ0>dDm{K)}ggNM-ddo&KPp|bp0u`Pf7Ip-qp$O%)83f%NwRPK1 zpJZy}9f>Nk^@uti>g9;cW!D^X!}S-{V2aN30THTa*!>}w2YMR}Br3v%iL&T1)3$cH zj~VW|9}=8!(R~t5xCXwWBDc?&PBbU=)DwUI%Z%N6iH*jtT@;&etwj)XVYP+xLWwT2 z)qLfODVJxBA>CBp_tF;2&HIV;;o49w3*|r$grW;!1_yndBtN9xN8RhP>`>K@XeE){ zBKKcMjxfxTQ(`U2F(fO@nYE|kRSTG?ERt&oi{l52fRv*#wbQL)RV2`4LcqFxK z*jU=lrd2O^!1C}xBKBHsn3iJ`UW;xigp=onkS<_zyPa;ZDsHHGg^djarhK_`=VdQ2 zna?>yO9`3xLTC<-4EU3J!Ow~{MlO=(+c?l3;xWeoiH$~}IaJKU&@To49TA@>ZtuQR zj#KT0h$iksmxI`JrZVy9Ui7N4NtfD#T$LcgV%GyIR{zpq-dq{@E>4$(Fd{c)N zkCq2wclax~+r~a0^A)B(`{GB;LcS+g5X?bXx3k!0uS^p6iuwipdz0*lQbYJ-q8v*N z4Ly(z2G^(9o+z^`N>6QL7AEBr!b1y_7Qr)d0Tq{031ixCnDjh5jju~kaT=qbP&PFW zC-@0ZS-0v|X0>cbh)}dzsWM^=La+iA#ze#Yqx(Ag80tlznxh&$e$!KTXwtqlOkFRR zsHm>9{Vu-~jjQo+iv-Jm1-d*KUAt#rnE5rsKO_3luW91QfpV@?(_ADJ4F!2^Ow~D} z8mf4sw;S_Ff03}lBb8R*9Pp;R8L{Z8jf~^7KUX-Oj*y~J3bZS9fVjiPbYy&qTfgK( zH%pQA_<~R|Mb<^vnGP%JY(fPK?DIh45VF_kv2*6cU*t^rma0sd9+8WPtb#oZSv`Q1 zyBn|VRACF>$h&Vo{Wwe6-gT5dK%{Jq*wJ);QmZ`k&&_rwm~D^ID#DL|Zc$WsZog95 z0FM=7=*W7UPv-is!5cpLrc=gC*CPdzmYWHg6UmmSbjTB$V0Vj zv`mlh|I-(&oi=4MQ(~*Wrgcz>HSxG@$#Z4rR!=|XQj9u5m?FPJDRW5Zf@*-=l(DtU zoZX?XJHEyY+Mf~_4O%;Y0Q%N2`z&!y)Onpq9D7l=p`~y1sljKNp_q4)NEfZaDmG*w zxR{tq{Wa^mtg+EvR(W`)!*cC61Ugz9rRAxh;LoBjAwERU7KKWnkWu1u8~jd1vFYEC zrC6qvp8cEkpatta{>@R#&vL;oT4$|p5Q?;>q-W>kDuuBz%bPA|q2FVth;$M92~8tC zA1f3R9H1I7-^k(3cl3|XZ=S*UJ?vYR-v|X8iZwz|VCFFXoaoJ1+FdC855;?&#S?8jKB52rv;F|wcUnTO_4VsG%HIAJa&ppxU87S?!q5`*QW?EKZwxn&`tgEe%l1gJ34GNhcoiWs&o7iL=5F zrBCReQa$^b_vCbDofa8C5WHwAiV&S}`mjrlsWllFc1u9v^+W)JNo5dU`L}IrrkuS$ z5}Bx+TW~HUg?xYJ-u2Awz4RwS7KtH-9ES;}iXyA_m2Rdu*GXKVI1$$t@IW_5{J2^< zW+-RMiRfKE>jPE^y#8k*7%PFzqQG&ejqfFN_Z9YHcH?)6h<2lS6fy3e#mDUVd^3$j z%*%cuDluZl!$6-Ot6`Ls0_!fToV}p>6{HDo8}#G1mPdXi$Q&(8s}CtztMRldTr@PX z4%|Mk0SnoBogpw0vZ22ia5KxGPQOLV* znLdX>*85Y1goK1Z6+sIpUxDPV^Q~+B`)zRt zX?f@@(fDLl?Xjv_-Pp`vIKtb~E^ZV?h!w%;+cGD>?3tsB*I5p|_rXw^+tX#p5ZmZJ zorim!qoJkU=k1)GncpQUxxskIg(=UDxI#ah;wm0@9Q&HBs0R-{>v z1rucJy>n6u8CyS)kdmzeGI6$c$ynEd`KZCaiAMBM+k0W2f|5~~-APQixFl4aIptn9 zu+-omS^)=^+89=_mx4BmP2HJ_#h8xi53$tfL5Ys3(T)am>R1$u#MFyS*T{RJ@g8=l zC;dxbfQPz+VGTD-NI>g8en;X<-1N|UpWjN-eQ33JlO?sv41HkoI&DPEXDCb9+{HdF zJB=e!e<}7D!Wa^z`l9E3eX}3NupIstxteHYEduW>6!sc4m11ZN-GB9*X9lvw#BEiH zc#)WZa9&hYK|4o}6sOsg^?&v_n8qlzTtLvG+90@$9b*Z?1&)5Oh9wJ@OE{4%07=LG zJrNQl(1O5oVUPT|WG(aGhEyX`MR0Afh0q6SNO_y{H}jS+wy}1)Sv~#prxkZuZmiF4+k zEW296LcpSG%z$J4EmPP@U!4A%Wj&TkSj>8KWw2pCWD+8^0>LZvZR350xqR;5SEYod z{RV6F9U|=~7UT+CH@{{(VZPGy>u#LLB zft*7iY!8J42^++Ty%&AJwl-Kx+1eJ-!-cKu_K+94a5=M^M{IZlBF0w=SszGmi00s^ zC7+m8aQJd!ev?)!ZniYMlvc{pGPSxz{_FydY%V2-{&?tOmL*y*p+pw0u*e8y%Huz_ z^}m8y!m+i9C}9b)RtQ;IESQ3&wZ;OQ(|E(A2kJ6e`%5AVSrbzcIH_ycDTPw5q*g9i z_6GA@=G7re(Q`?|z#aSmecB=puG0Enke{DhpmqnW*)zG2Isb)q30h;P*x$hp7hd8YX;tVfVS^F_bjTJD0>W;x$^g$bB~jmpsAH7{Juk`B*HSWG%JDyakq z3rA87GO8p+l+;bNg(s&ot2?MZ(I~8Lkv}&F_C995m6K{N0Nw)^Y-jHNDG4t+A@r6Z z{Yr&WCynUJLF+`*QElVOl*nmDkB$1kx;<9b50&UQ^qSJL3!5x0 zZ%E&OlcnwA_esRwc@kfS*0lOy0MpSajR;)S4ds0==dp+)_0u{W8^{E8gIoj!wSA;u zC^`%xF;v2%a|O=XPcWZ-Y-9Qc^x3oJH=Nfp%2(tUuDX}W(%*6!Axoz4q9Bix6nw)K zS>lRv#MeJou+N#FqI^y!X3MbWrpOflk2#CjU0p>hq;sMYQ@vsTN((Wv)O7+czR#6{j?9`ayG-KD>*00@Iq(2Z^QwTuR~# z*q~T4vQ!c~^xWokt(hdWYep*xNf6_o`c$QUp$#5J{rkkYm9f}v`b7gLtoxQ3p}?xi`?O$VXnq}jn`_vw3me>mgcmE z2uaW_(&JA-QVEQ@Ls}kjUw(U7Ps+2~L!^uAT;f54ZyqX%_J2WeY0O@lb830Ddd-;v z>)n=?69rdYD9`6RZoDWfz>LRN5?B}yh*88SvCbT;?s(zsQN$=FDE__khqq%c=|k=4 z1B6S8Ni*mrDYiN*@CrBBaM1T0+MGFEav#~p!kGLF!WLnSo->a%sET$`N;k6tqbgbS zc1VJWBAeSb0eVDfSd9~CdZ2b3Z2lNauU57v`Y^?x2X4O)ZhjF5DXMW57F$IZC11>( z8?8QZcVU&;ln%6-sLZOXiMa8+7n(mPmj`_7NLzH`rRNpH^t;u|Wp3%w=rxHs}A= zjE5#O{?F~C@V^u9L<*L`rJ5-BI93^Q78{0;mF6j|o8m7=qj%>Wfn7{{_UcRvq4sPS zY0RdC3tZY2cicr0TfPBh8@DdMnN-})J%2oG`Ap)E(1vSp6KHX<6Lc8Vc3=`28k0a( z;3aJEPf#v|jphPdKa$w8sjiAs{xAq z-EcPE8-yFNVDD{EeXy37z?>rVnq!?n@Aphb7_K{BX<iy-L<=MtWHq+=h}+GEcm+Of5#C8RLL1*@_I z7-vV~Q(I^G+4q@CQ8Sxx(N9US)8Kh*p;k$Wjcw=8@wZdP@kg}$q#rvIt0btn_o@V1 z+iY-g0Em#d`Ye`m1EUq0)JxF)?)T%ct_RXmF`lQn>KKzh;v;5zX66$0!uFH_s7a%aDYXk5 zRxM%eGg)~ADei%zY2w(?Cvzfy+4QZIiSzpsQMAt}h}Lmok3lh;(mA}K&T=Mpp?pQ` z(p1!Vw>&DctfVB1HdfeK4Y_dvfnp805so`p<1~_~Sy(k)ns@#3oostvlW<~tbU3{6 z@?2vSJOjIS^}_>LTKnd%M4^};fjAKY#v}&I|DAbL3)Us_uf)SHkw$oB3F=UnVbINd zxAT5)aP7mSn7yZters9QjR?HEsy3yn){yL(@%Y>vQTHRqwmY959K@yOoBaH4*L1$Y zS%`lWbcR&L;{K~%KQ3Qtf^quL zLK6C-_b2yHW1#bT5Qzd5WW`035J`akdQ*qmtgEs2&~st)nw6+%o|w7Slj9fm9>Im-A_n@nEq9sBPi02V@c_fC zb)|yWh;GL%HXcf4>+`a@E4Q8&w_o%ab0%Jtc%n~O=B$&(Z8P-98g~NYHTnvZdb}dNf!6hFcT>3r1F`Q6TJY2aR)c^%cUu(}8yQq)C*v5K~ zFw74alVA!SE8M;I?+f2E3pBhh;fXFLyh{LYOTwE7l-ux?eagzrR<5r&8NBxwW`WK~ zI8j=J@eAmwz;BMlR44?@=of$2c*=K7gFe-dC=~Z=pbxBfuNY)qvA7CajXA$|U>nn; zKcR^{1=RnXkiVC=l)H=r#pV)k|7>9z^SVBkz{2a&fujXFWGv^YIrL3We)2Ip@RJ4* z&0@012_qe1pOU#2?}bidYB7hm?Lfj54liCw2yG;sUd*bX62P7YV< z6zp!f5GvRV5yI$Z6H;?7w-}FxL7kXU?)ckzd!5JsVaLI96@7-Nx<{{s=Db#@<>&Uv z8JD^kUdyXv@nWx*GsSB-`I55??=ObuapIkWN#Jc>y!bftmRAfS`o#Ew{=INg12$`n z>?#~oT#CUP?m5j&)QG`^sJj$+(N46jE4m$p^Jb58id5<(s5#Ug~yx?&)w8~AJdb2B_!&}Hia;%uNQ2=Lg0Oan6YuV43`YV z7v)p`EbGpSr)9&5S}dMsN1-uK1!*C@8AB!_L^YjDqB+|qT3Qz|qtxVDS_+NQWyqLw zc7Ac5JU?t>Iy1s=N;F}F#a-S>z>S{Xx;c|H{|F+EPG1Nvk4idB`1{1xyO^uga3rCk ztCX%Y6D&j_q_gAs3}*Iy&crHR8SZ)&w>hRO#GRqoQ(@gcF{G`@M3(igv6psf{9BEKI^>nbNg`lmL1=A)mh#=A-R!f0f;3Ovou^ zRrNy7=BdL6F{Al{Tmw&a>*yLman%GiY9*1yd8yTs1x)3Syq>;5sC<0`lV!`#&SPVl zlk&R+6;VmFZjc14H~sGWkX1UL9!-Q|r85mW$($A7Lz~^mtVkSt>!EAG-=|s8y5|i9 zE5;cmtog-gigM!yWKy8-0hZnPN+RnsFkqPM&><2uih(I$zmK1s#5y1EyOBu6&c}K# zPqEd|W1xI=PL48vJZ(=YuWZG)4L_%$qg4Q?wD@Tgf&W-x_ zJ1eKQWW_Px7$OXdV+}!E;KJ4z3!ogFY$ZJUX2ga76YU0L2@FL$m6&fb^{IR?i;=o5 ztHp1J+#o@*G}u}{DIkXeJxT1%hu*w_S=d`|CW3{971*){!;O<7Gu-3eySUd4KoK+HQ!6-&MYA+XD5A4BE`6Ihbi-~X zO3|Tf9Sa<1{7R`n3I4l^DxL3}M8vJElNj=xJpgGqnHDmZ|67N-zO?Kvw#L+uvPK7mGt_-d`XHI>bqlAS{fUWVr!cJ{I@6Rj#C5sC-c&(!N7UE^q50y0n#$&YyzK{Ay5&!aD$*@Ttjmip2##}!R^FJG z%1n_fAIo3CdUlsq61}2l7m=}muN!Mf=M*Oqn-!@x#~WN@WrhUg708XBR&UiE?fy97vsXHr~||0 z9rd%%c4XUoOfD1IUfKeP-=Js<&rTTHQx$A1I0vJwzxDNYl}sq_xt%^i46fiO=wg%f zeUr!6?OVqVQM)?`S=4!S=7X;bue(7XUZTACeCE?rn5gWMs1Z>?)fo2ME8%TJ*4rpI~KsGoSV&NX)vAm)ft;1+%pL^V29e8B(pZ}k+*IsLBUHHPR^ zUh`Ap~uqAO>{Dd|$rm^#Q z$9+Vo(EW&tDg^HPfvtyRD-ILz18IKN&tA78g%t@}-A~w9Bxs7*a96A^TwDT%D^bwf zZQp#yIQzB)Mb3&>cEAJ`Lv`PfhA^k|?!ougVvwZ|D4UgvuUUZWh~y^xU{uAv;)|G& z)}Bs~!WS#F!M%&%$cRC(A=y?O3FQn>ln75CxtctFoEefe5*ZCibA?!(QZPPbp=eL| zD7er&sy(;OKK(s2B%@{!$-ouA(tZxnvFmD!R#at*X4P0FDvQ;0o} z*wMmwSc~+bSBc8}(&=X}Viwf%2$75yxv35Z?lru*YdoNF@m~9E(+-AL>(N*|af4#a zy?}A?8t-|s8`DfHC7!5S0!uN}il>$o{lOD3^-jEa2XpGKokb)Hrw(@sN6?CklehzR zapyG2SKWii_1*QLryxK7BlkS}LEQPtcX0s&pA_-*$O9uO?R)7D)V$xcR2wTLYAZKuKwtRWrbPElpq2baVUonE$NKg?(N44R3 zU@zLdil4qwixu%`j?G}=9e4D2A(>$4t5JlR>#*&w&dXy~=T`|U^0GQCJ`Ne_flZVd z3wAEPjmgH7ONd}$8gyilJ+tNJ4Y5+VNKx5hubklvPYt#B54;z#ec$KJG|pN z>@78SQ55iQOFzDgZKrQ}>~==k0dAflPoiw6Vp|%!m+-=1-4_T}Ot9!Q!&o3RF+z4+ zXGT1~UJ*lX3)b#>YYyx3eOWG`zgGlPFz47s|g-+Hq8Y*Wqr>J|HE3_|6-PFuf#{o)dW)?wzA^$qy62XeNHmg_|~Yb zk=ViU=LbKx&Y@3PJXU)x!Hbxef&^?g#x#ks;naLbD~=2;W@N9CxVWQfQNZ|lCeEkwqe9NB3tyRDe!@?H%{KkRFRziX!<0kN`H~S!aY@h#Y47Sg_yHr z!kEFwm{FPY648rBB^B;9vBEoz)+C8tKYZyV=6+oHGNB6hLq!Em68o^aLN9bys70Kg zKMyQsVsTU=t71`v?;Y7oV;-1Pa194NYxjl*hu7x{$OaA5Em9EabxqA8;7oB@=)_NMIKVnW?^{`3%iM);QiZoe9DMiBO#H< zCLXw?NZkL14NM%9ITnh0_3yC8(^y~1=uJeV=t~JY(Ii1ACH!yEk`8%!@UPs-Jzalf z;14&mjr&(FBXYhn4P+B#G_bSq^e3m^{Ma2Cocbt(Uhq170YWzw$x<s; zC^@zrm5Gt{4mDapX{L$Ye{F0Tu$iErsH)AXsx>bu(vz_QKXyv5IC4yGIiKrtqEE5> z$CFi2iYtmPE@dX^q@-B%rl8E>BUw?Z6Xo*$7IhjiE>GX0aJhxvka4adZ}ScOYRAD^kTqK*?=T+By z*_VZw!EHn_#w2Z{S14dQgY0vTPL|rh@$A%9jhI1MFM-jZG=U*RkPR3IfhXS$%vSXB_;RN=r*oM*Dt4ED*EHiP zIw<$uwC@^=?G2*$!K&H=RkgGr*i#(L!>dAchH#Q^iJX+{&g-Oz8{dLmVwcGr7gUQ0 z!ww$2gEazO_$Co58iADmp3`S43YOA)DeQUW4FnPDcbIVB#6{0j7dpoFdcHx}B^_$T{cAc$fZ5BW_ z-$~%2MHRs@BLs0URL$WY%V7LVeoRGq^57|-FyS~R@kOad7h^HC3Tb?U#3?K>?BnDy zce$Apf8Q>mS~&48Z06AKOVMvkOLQCW_-NdanJ48*puNl*rVd2E{BUC z&=kQn%wDJ45Yd?8et7HMQ`iyQC~-x6OSpLdeUiW*%&NYeN!rbOiD03Ipy^rMVUi?j z`GIMh+16e39--paH79cQ;rQ4?t+btIWOqu(|F@5xy}23PZfDZAOk$(7r4crCwZi~J z61{~bGj8b17%}91g%Ofu3`CPb4L#Yh8zc66i7JeTUUU&M0Z%zYqXL&-Q*o$!-<3}@ z*a!C!u>uzBhQx;w%)=Tqgeh#*)3<)jc+mL+g$Iy!i@=t<9OttqXie)S|6xhSZ8{-=K0Db3vUc3!FRCpJWHE z!F~c22Mr#dhwGQ=(*JnglPT~8O*Vhby!MwRFvdo0P!b4MTqG9XV8my{oUzv|W?uXC z9}&&ywKo;k0u`{o1e`uDVM;6t>N$GI?i^KeL2vRR#oKegN=R`t8TK>56NxsLp&qMz zJT;L>}fBKH)U zpD<2v-Qt>!n0IVZZlW=VO%gh@ZGr&6z5vd5+5RibQcAH@YMDangGP}c1 zbcUTIyfFi3e`r%^eQ#Obn()94Y2N~L4h9)M-5=XoB-~LBfa-{r!L>neY z8tV@u#(A4k@m)mfKKiv)emF9pDUoI$6QZs}#2MFpTfff-!6f7XxTq^zZYJLCgvtKbaV*7;1C422H4&%EUdYUXY9b&pq$3K1A z&Fa9~5dy|KaC33ig~$UkJUDLj(x!;J-wC{68%=~ zW~~{q&o1-qIqXBEG%md-2I{Max(Q;wOoX8HhGmj)(r}pzw3RGm~p13V(F^eFe;D z&iaf<7ao+$gZm6^hdy^WwsYKcpF7rUKY=OWg3k$6C}4=P;E9lmb2*T?*=ibNQo8LE zmI6L3@k9z3jb%JMCDi~!EqVD_>TRxne|G=l>`YGjf@s7unW|fKGfEZB2Ce70`AT_i zV7+h38E)FqIJhPhE4T$&Ioa6??yk>|l`w0uP2!@pNX0AFe9#c~pPb0g&h3`PuBUoF zZQ#IebZ!<_-hP9{af~SLtaZdDIKw$q%x;caW?6T|n?=k>RR5B2&`2}|9DLsjFKLSL z6&3ZSM8TWq2hy2quvh{L*FXTm2gd_cXzW^`G%22Ze9uhQVK(3^q7ggHI)PKf-xmnG zEAh%Oux>CuVy=@F$FAn@kLR=2{S$HpY~Ak^@n_(PSEseiEw)TQW>9Q47-XeAFuPk; zL9TixYOe174qNBJDY{)Mf^3f4 z;E^se1f zMWEufLiti*;9kXdKzsOZBdaLeVy%UQhS@m|d03sVVxF zz5<<^R&et1?pK%#7Fkw3A!AT{iZCip*toyTUS$fu#&-lPoI8A0!GAuu8OwriGJ=;% zTu!i_zfFSRCJ(&&3ey~=-xI;8IjHy<>(#Z5@VMDfd<1C}Ox$P6{x>Ke(}nF(x9Fky z2T$}Q&9URvg(EEoPt!WtS{JP$-d85(m-sn~&*aN0n|#O0i;w?6SmNPS3_!(>i%8K2 zfjGRSM({8{;e%q8p$jtpGI!kh;oAJlyVe5IMMG%26fm7 lB{nXRkN>3C?n_x`+3cT)D(oz4fK-78m)tdZ;zy4|_U%x&)GE;DG?qD1u#?TY$Fo4!h$ zkW^ZfNcKvoP!uJE)c>64bDn#j^SSqW&+`nar2qW=(2#|e zD&uB)kBr{kuW7B~pv~>e8CdT0g?r|Y&e1xDoxYGiSnRh29XX+h*J}$-Hy!_Hui z6MxI9hMC&Y(3# z)Z&MmwjH8=`;%V~>SbClt+~T_e$?7gS}z{q_c>$KHdsHQupi-8dg1+-EF(7&Zda{P zYf|cVIGjFf$UVc!KVAXMFY1~b16)7n%#?uyc=$l+a?4*5u&35tYvywKtjU1(=$79- z2D)hDHQNRg=sO>reZunmW}>pU)>CVNytIX3%foIzz&mY@^K4aq4EB{Vo4y-Mu+~un zXIaX>B-o+aU@bESS!}bHIeiY!g7hd)>-GzWef{;{$#HpXq0-nI7cCvStS_yR)%cGE zmbP1HjUuf;Yf@HjwU^p_KBvb(a;+%(MR_VK%G>N(IgIdTSsr`XvRUHw)%s|yN`n4K zz&bVJ3{H=+gww5T-PmA6k%gK~(to~!pkKYK?IO#~UlFO@wXRwQa#;r!ix}vw5v8}X zvT^yDe-)1)$PZUmU1rJJN|0A*{j@e-+cayyA99O*b9#%N4u{)UVnD7FMXp+q->s{H zK4E0DZ>}Qr%j{d1SiY3#{k6VYn|NevczVEGkc0k!!i|}KRzE$Oz}p_K_Lk+wZAAKD zZJ^dE==9ja?s8|0%i>5EM?g}-hA&Vxg(7~_Kl=DXCVri^(+VhlofE8p{BpQMVVlox z_zW{~dAV!h$Z_-)JHD&4&azFeaiumy>l6t&twDbnv|~#9Fv#W!DXhBMKh-*b7P(?# z&f}I_cF>3P(7I_&Xn|r|$Ut=6=+0Iv$`wDcbMwOYdrmQ7?6i|GQ5e%D(axY9jy)G} zl@~dRYN|IXW9(2OvuoSh&sw%h$Ua(ctvNxuePAR!PFuOtaNZR;G<^T$pV#)DNWil* zoNrog-9@yH(1vTd1RQcswR%9AiUk6G6d%x5E}K2<5AqLCPAJ#b5B^=<#`IwJZdwTS zV1~Y%0jD)&w|U?oaa42qThzGbFf%SYBrF=27I-csWI}oshIdyPKF>F*20Nd(eof@! z`D|@-I1)%*(nRL*Rq70a^#It;}gH_uFm7!g|56DR+n5#bES{ zr}HHg>Y1i(UA?#(GvvR^1<;T;_4$Kd;PMQyjT}*t>(N(@83Df-lj|)dkKE4W`t5It zY?SLXM725tA-BixyV{AmEvl9dt#Un4zy4#|CN5EsX(OQuX9CU`eW>iwN8(IWP&tvGy!yp9rra8SPpCqN!B$0r?zPac&fQ(&qCX7Unx&`XgbY@0?Cm4Lm^zk#m zQ@#Mu$3|(Vm_C0;f}%ceqf!Yb)E=~k!qYv5qfsyFP*ewPgnm*urg?|`eaCMs%qu=b zRAZRe$nJ3mtR+F?xlpZx%GzhQ9C(S>`8ht=!%!pV$yIi%V5KJfO8GNzduA0a>yYk1xEpx;_-3)@Q*1fR-r z%!|>ucGxB_JK{$rq&VUw9)B@dsU$EJ7d`Oa&YtW5E%}AW!~>KWw0Q#%Zkb^!yTdj0 z2g4ecGtYncuLO&pe@8gsK___23b57fvih7(NAgZ-k9R)k#sbBYat#qE1|eer#D|?I z^S+432FVeFR#C`zMLeDur@iltxs8$k#8HL(PWDo#z0B(JLz2UW?ylA24~2wXh(BJ+ z8r^tex2U%p%LL61tyu zxz;KM4NNH!{4wSe%(1WDyK>DK!mf4mi2qpT|3+jF(FSQ9Fn9~Pi;?Xij4-U={kzMB z3z;Nl8q#Wjui+^|QAYufUO2h*9be=mVi*jvVc z&2m@Dxh#xH%X9ZU-fRM~7)3@E0(n99W6?^AwN4 zy=z5drlRgTK{TR@%3!E*`6d-pQBm(+6AUs+^G_0{pcGS{x_nu~eru@AZSejT)Jaof zn>Ke-%Ms&g(?(?Pf6elZL>;CL)w03q2X_n{=NrJ|2Qdx@G5Zx1Syi9SYhbgoXIu|D zTg}TDNvq7S^TbTc?Wc(Lf!Y;Xd+_%$U<1wVw*qhMPEaF>a!(3AS0cB)-1$wWMB4pH z_(F+byd0Pg;WtiusKXh;#xzh%0uA|Wfl#SGYz^4L zr5rjf>RHv1zm*ek+siyk8mLXyi55%}H_1g%5}Sns5F&cLR)0CF|#tzp%3Om|Kb`<;lVX(P2^wJU}@fCQV-LCjJ zSH9Y5Jv)sxD>c)LLU!Dn|ERJFN$bhy4ze>;ia zt7Y|fSuUwUz}INwwVt}yL{VBPlt93>38LTU2!NRgc-(edNKbv1+YBFK%BSyZr+vz{ z_ATGhRbsEPl};>Vq0+M#6PXw)H6=1367e_szIXup!eS$%o&wg}+ zM85r(63br_*{V$e0~o7GLps`kS`am`9l)Q2up`e3YNf=7LABN6v6b7va9RtCK?g)D z+c9^|x@7T8#;#S>Rd#u8p)$}VA-^k(B_vlQ1f>VAO_ZJBW@qp++xILga9Rjj)EtW4 zaIP^Yn-JApo$SAA`aotDYG?#2tgnF8lc@=TP04*}(9SE_p?+233U@)^Qc5m0;53(3 z%=mCq4TTSxlnlpGNlNTZ|FQqC%mu!xCc&Z$+)me_AOnz=)T7*l@aBB(cg>#7ELWFN zi|{d5myUK!RID+jY7hx?&_xfFJjp!0BN9}2dRW*IjD{kV(`FEKbJPvD{CI?^kxy$A zsiHK3s8gX{j>ueg%^`1nF=I4Sba&SwNIkvb=MEcd*Ra!2}fgT9O7Qze;`Zy_m$nWp9cd_hHi+Z$@NN$n)uOmkp z=EyI#mgXCh73R!JE9`p*6P5LH4PkKtU=fgVbSDk6qU3ri^MeAWT$7%^bF{4MhaKm$=3RC6^6dMb04DqnIF0vOla%qTNid zTxGUpg#;d{jnMLK!fVkjg>dpb5Yh#09*@%lR>cD~uZXdMz?3gp-F6LSGJj1&`UWBM zJ_yahkpX{FFZfxp#>hp|d>c3Bmrr3v;H<<(BajId^9b}yfqzHDCyRTP=Iq6$|czG0S4;kK=mcaIqEQd1y0Xz7n4l5olFU0Qf zS8%tDeLm(ZOzC>>os2ohg)YA_4aGmY@j!ZeTKnYe(8OR0o0-S(^+#7^Tr2`Wxw3>3?~|yguJZ!! zfD?_Y@oJQ!VfY29_quX&{z(U1OJQ%63ObETT*BB5v~$ZKP&&Joj4_YA(` zB=bmb${<+bz)LG|9(YsU?7MRQos8pGW-1(Sfsmq63bZS9fOsOtbYx-}qoXIP5t zltjf8Sr=VrI;^O(2^B1`-wTOD$X=tz&Y6>Vk%A@bzh=sGVRIr~WEJcY$m#*4+}%WN zrwUv6Mn3Y=&L>#Pwtox47AadJb}XHr(kiw4PQ8=~X2aIBitr<#TNKruC!ka|z+;6N zI=UX`lezv|-J$tCEKXY|R}yiWpjtGvq>NnqnR_Q*LmKdz`?}1rl(wO7xJnzQ<#_%7 zpT3~UqWon{iPdUL>!1>A>h;)C=St5#t=?uX#R`cj@;j6=hlDPu2FOhrTg%MZJ!t3s zR%XzyZAZkSL2DlfLf;x@pCzt|I=?g;$6k_WXz9Cd?zGm-P*iG9D~Q%$6&o@TTue-* z{^+U&?QFD{l^%G|VOb(U$7^G?LKPJJS@b2uhv?a&a48frO8p*#->E1z{Tp%>%QX7# zh9C8$1?w&NdB5eVEc%KrTDDd<1Vvg?()02Ql)~8h^xlUr8;A>RF8grzl-sAzgz(MTgPLF)f9OHLUVJKwA8J`HuYzA;?7P)D%$&WPf%~A zfsc=CBvDZ9(4VY!%pW{Cu04y>+I1#WjMJI}YDEal1UBg;LT(s6A2poQ0#N#d{wdY7 zul&@QUaZsNYY8u!ilRg(oj%+#=HB&;3pZvfT*!z5hEmEP&aVE-y-Yc0+aL@}K$)ym-rjVYUyKnKItdp@v0WE=@j9oCG5>Zz=eYywk7r7^WjBHpQ&18zXl$uB1>9D8$JA%(Xsq~nZI0yw3HV55 zZGL5~ZhU4i66I}K7Y_;}#EM|_Z8I~mZaV^J&;S1&SMM^9my~8@`{tUUPscsbGdJ|rB`oSVvIH_?lY7nY@QN7mz^e2sjrPahA@Ui zslM2G|E_pkI?LgICs)H9enS^{XW@v?ps5r?W9a^CaP05{EHN?jG9q3iCLo*_6;;sA zu_MK4Hf8o$wE`(^hfWNi1#Mr1E!2LMsepTxK<{*d#= ze3@&4M<+5F+$onq87#tOM2v^k9mz1^~zhF_%jDGY{BI*$}V^BU` zcL;)o5>v@B|D3B|%{aoMikAIp0nzG!pM$-lbW)?q^rAl*l6j-F`J6GUd^t<|Jv4~EL!|x0 zf?R>?=GSZ|%vWmsal_e6lJW;DlGN60CDV=O2l;<@T@>s~rb`q*wn%h+&O#KX?%8|{bHmC92;=gZUvNmiu(I{k1Ohw?Nu3@JXO1Y96cWv{X%yaow;-Tl# z90Pao1N3Q&1h`7;f7ha-g05XBzVyYp|=d_S1Ob_X+&2J zS|__k*RARx7cMqHn-jNf(=}puDzmT%8T-6QT_S|yj>|Q*Y))8$6s2i*vua)&fCHoCOOkB{IO_nZ^*f?3*K5?H! z{GBI>WvFlUGi{lU9x{dq7j;8<-^&FoqDcKM_Y}@%f_g|Uf`Zy1S}+tH2GJNQ>Cw55 zvhG~TeD-_C(l?;bo+rQIf|gOf;r?hQ zpUZW`#1Ur8u;<3*4uVG=UDa%=$#j{CCnkKy3a;Mq^f{v7Dwl!#VE!+wbeOg5b=K-r zYXYqzT76O@*cG_T4ogh0MNo5JPG zT(NKOo!x~g_U9xlYjKmXGPUmVL_&tS8uK+OJR4qTA<2+yiF6T?pj)O-5<;(KN|Ox- zZeJQ?y5z6~MqMI;DG3bSkxm$f$L?wvSu~hbWpYfNaZBGYc|G%|=1wBIg+C=(uJ~2S za($rl$A1{tyG$lz!F7kz1&2C1HfR;YBsg+}Q?ALbv8vyfk1~PTEZ0DRX{lq1m7tx@ z>VfW2LlCPN2}Sdj{C(M4#`CFDh;-z6JF41)*Y3k^ORIj1P}~hJikNfb>DZV5KBKz9 zbp$VbEzw9WtTK7BoExUF9kxC)k`<8OlCYv+?|{(`)CmdiJW@cpa{g@n#=jZpcq`Ew z1=Rr)ST9r1xn1x0nQ5-m5>)R>0ZIel#}lPuHn|k;rm*775BzDU*x-JeA4}&46W?Y< z`^T>*^7W!UtOPZUAa?MB17!C@&n1rdCM|&WA??D<$?s(&e01_#%HxA+Qm?fH=Jb-! z(G=x7@3&I;C9ZS%T0)EKToOTpZyqj<^?yNdY0O@lbLywjgC{Trc5^X(i72@0LU|$I z@y{sI#^D9b@U$gAx+ctA*h9`{CvnagbsfS7EVLbW!rf%(-#ksZC2+W%gbd zeSoOUs;h~(@q!mRw>|PY)5BRMw2;ulGR-ROpQs_8cRqmY;mzxc9jr{dLBfl<2!y9L zJXgq(w5&XpzHlq6dU;EUXwhn{x7=WBv3^>irN;&t&~mr2#oL_!5qD?~<9}VZ!vD^^ z6Dd>*mujNi<5*?bSz;JMR+^`Ggi{6KH*}8{m&)4Kas6E?98?!0l0+)6r9CuN~ zmT$lx-f@j4lZyLvbHA$PO^)7S5)c~$Qcbtv)h2Tak*n1mP zAFLInFsBH;=2$1t`#n<;hN^p-{g(wDkCo9EV$hL`_E>cdqY@1I4}5lLrZ;$Ti8acZ zZfg!PLB7mGt3(A^pB6#Vlg=d^O5bn`Ywg)1QL(kB6{Il51*@_Im|#Z|Q@iuQexEXz z;z2J_i!Mc)od(b23$;p0Y*PC`exSWFj{jWt)HUo(6!{2L+UQTzwW# zxq;D&%yNh5M5u&x)kSy(k)I`sZ6GuZa@53Adw!{LpW=NqHo8Q8UB zXYTJydQX&44oOHcKLT+g1dJ&RmhYT6{|M_6xi3N_V)IR7ys`v!sM|2;X1?2B&6zs( z0V2?Mc>fVgk8%RMqq25eWvvm}Gvo1@998$DC$+zj9vsA_=9{9T?!&StIE(R*Le8+N zSkel2?Pb&9-%9duI=o&2$4OCBuoa!*e=#dG?DOopObG9&ppO;dlwh1bw2*?nxTNKX zxeT=ZG!+zN#Z8kCDS%$Pw#hfF8-9ZX6(c=DZcKK@^awthv*sa&Iej`2Dli3{Xf;TI z&a=a+Ww4T7%Nc|zT7aNVt~<9nS5rkF^wrlh7;D!{bUpmV@eGuT3aY9k?aOHYT15B&Cem`UETem73)kNcoTtg8@{qnxjA{t^%VpBCt5KJH1~Fb6QxBM zzkr?!{N{K}g+joLees-U`oGOI=#-fRDel)mA6V~RG03`NaTTVHni-^*Ld-Nu1pbBUjS__KSN*Y&al7G9SQ94pWvV>wUFp|`vG^sa%U!>4UEhFo*XKi76ajypj;wNI1QiRY4_yJq^Z0lp+yav;4C0%sWiMK5`ck zj|w(RgfP08!efMS|<^8mjW-^iM7=w$;NR9x?7nG?zor86fSt!U8av@fz`ud_asq|{q#~1 zZw#AFsMyz*6-7mv3PC#x_D6_y44OgNE$-|Ve{^Ytm5!g63y3EMq6;Jm&PB^TbLHiX zCqwU3c+$+~fT;}ZoC8e}H^geL8-|s>*qa@%BNA2kF9uXN!6|wF6(6qja&-n1 zE1vF@&{#api$Pw`ojx?nAIW`cz% zhIDo!pTW$&cj%dR9mD-c#m&TYg}5^moBSPqyWuRGZ_$xS_w-`8i~pn0+Z0D3!Nf1nKJqwR2imECQ0$d$6HdLd^~V0%_mS1oW<$} zDMUKWFaF@QIz&Q7F)#(Z_UwP!v(CpA5)eBd>$$xpRzr`0^3nPE z%KUMk&|hm9(&$Hsv?!9HENX_d(lPoUrmfCMB-B=EXnhmEa-?YOf|!5bI%`UV6~~@@ zlnBG(SR)V@xUeQnh(79+LZl+l2d2KPx& zEDg5NPYTGPKu;07D7^u{25aWyM6j^10$bK#xN%ZshI`ygmbUH6gs|-s1c^eJhOa@u zETR>kSm20e-@oAKys9i_S}y@b%!E&^;4Bu+(xjk>F8TdnCFan1pCn4rp=%QloM8M) zsXz(-yIs>`vej_=nHr4pItyt55r)$O*iloAHYUDips=LetvoL<;;xr}V8-QD2`usv zaF-nzLUUMU@4{}MLh!EAw(Z#7O?XP#JGia~!tr=2l>v4rKf%obF%FFV?&~L60CH9W zVt~{{04B;;D<96z!p%n?FJ)?e;nPH*Q1h6q7Oh)g`|JkIq{`U%69sftoYgyKUDT=4tMcu;^*FQDC9ZG!Vk>7VxzAkx-!3 zwx=DXJ7zQJyA~702;E9Ri*uID>v#FNg~~H$PrqLA857y8X9!w|tbi`|hrxw|_jk$? za6{QSmN{}>e)KE@{zL_CEr4A?rxSYbL=&U9a)on4QQ%5S<*BD!4>M+zJ*zMSMham2 z#MLcku;xCT?c^QvnIF{nzXUC!8v#v|>|rO?Tj6dsxKNiN29mjcd4-R45*ng?AjadXvP1DI0 zgq0%Ns1w)iU^?@lM2|XM`U(c#D!ky~*o()0eUt&texArhz;wJZNE{9@_0FkW))JqP zv8y^|vZI&%0s-QgZh?TwK9$KS{qXU?uPlqVNdk*39s*OWOFS0wzt`z-V|FiIlbW)m zV%wp^td%41B7K2qgo|H@`pY2{gpWrt1Mx*~FhuPoPOl*wU=G{< zK-yx~Joc)D#pbb22n$bWqRqna8Z5R4nkKMUBi_a;eugbFP0ZcIj{JeC8~iNd9{noAP90v(sA*b3B6LDj=Rym>3WP_EPyX(}W}x3=QWPE3zHvW(~! z=|p7(jAHp_Dx$6&#zi{^9AbK;`zwT~zv~s_xVTD9zoVD~n{VgTuiiYuEX~&vT$o*j zTns|~a^`XAA4gw}i1;`GjvT zny&H2{)t`L8f(-wIx3u@?qkvisa&u7_7?j!OdN|}rSA}(Y04VnI$)YajW0zh`-_+B zD4v8Xr!#+r^`h5|D_}Z^F2cMJg)sia=RdH9s{hIru%W86@(nOC3&|m_7Lt|IIZK0= zvAAa73i=K)fUT?oeaheul!(?!Xdg>2h&k#YJRy z{FFZZ{vAw>-Saxp9aUp9gyu^GYATy8v`ag)bW6uK2vww8kXW}5Ul6>(En0cwUMe$1 zUfjfWDGSU$lgOfH7m=}muN!Mf=ai%no1XdZ$C~U!dEX?e@ze_CtJg4~0Z-&8(D#hH z*H>vOgo!!5Njau|7c_feCW|6#zeQ_c6q$+i({Xh`e_`^?*n^L`7BFkQT;htxOM$C9 zbaq<+vXbxsMWPtclwk=i%3g^Y6%|yCk$`XZ3B@j%PWDk0rFIheP`$w7&p%^IUm`z6En~hb;NBg2|eD4Uvno){vLo(E z<<{?+MmZ?q(9E?8>JP7<&%0BR)p`9}=iT-+yGdd8`$Q<-q)^|Bm-0P1loGS#v(yFV#8gr{zyqF7_MYNt9)@Jot4U8 zl%U92@yZUEpc1I=8`2QwbQX4bqB4UV^^vk!8Tgt7xQ<9}(ho*`n(W}4xi{JH(ezJ=Y%zQ!gbcU~54VA5q^y)AyMO$)>uoH_n7)DF#TeV8Q(!XAE(Um zkB|9d81v-+lK5hh81bQj&;$1(u}&+!<)Ot1DX)8Y4G*~3~zq|JaL0!+`WJa@zy?a zY8ul_cYi_jiK->A6ho~!}jv)V1G4kix0MUBVHxl9CkefSvvRp&IOv zd@3>Vkl_2g;>pn{E;O3JBJjRzBx%3=KWiqw!L*ZiGm(wjsbvh>9u7j$6+1%s{ws2e z7xgH;w2LYx6*G<6;!A=?joLZ}t!qq($ze#7gC|yd@7_J5FPoqKPy*xpbb;FkqxGUS zMsKV@BLJ5HIj2I&2@PYSOVkw?D_c>p#lDwcXHiSp7WxD|YJum;^aedVc7DHkUN?{5 z3cKN@3H*+i`d<;Q$TC7o7eh|bHx+F>N!skrdhgn|nN?UN!O#^Vm#R31|T!5(qg&`Amy zin8XoPu4n`!CZxn5*J;CHZeeX;RTM18z~b z732NGF?sFOZEz-Q&RHZO#pF6ddSNKWIIg8^j>5hshb}2*Mza5IqE9pqAy_Kj+efYU z|A&QzKS($X3tJ+b4JC+PPX=pbXf#%+RCLy(Urw;FH22+K6P1|eZWV=vEe0inH(?pC zxm9lc_xfQz4ww$ji!r9|+4uT!#)qtL6h1V<6;gQ{x6Xc7b;gHJC7h_8fzkAZ@cu{8 zR&IDV)Kn}*i$#pe;9D*$gT3KfqEf&jl^1fKL*KEZJFo&#jXeY_RJVRjQ>;@Ul+q=9 zS;IEYWXJy%3695K3>%27UI-=|bW4x1WBfILi6vw9i}}ZyVHoipk&cF;9mbG(#N+aP zig;vVGnjbC9X(!1CK&o^Om*1D&AoPJb!zS>ShPAVbXa^GGTH;1EHkPteEKRT8!IKU zFbz7g$e!Bu3DiWBx2AMYI9_s;v7_{Rg&ktx%MZzXy|rDJ5>B`xa!M-g+A)uwWtF75 zKM*Wdk~-)_OIT_)q6beoypp|P(^aqR?sHn@^~#p=-e^&e(uuE}9S6A1~dRhD=)b)+BYorZ)L1mDzOD?Bz9=(g~7kHGG{i6 z$AGrBf&^?g#x#ks;naLbtL)6lVq_nZxJY*M7+~T&lj@XGSeoro8ik9RI15>} z;ohS}w&+n);DZrgg1nQdBGFufY)qAAA0uGlM=HSLAzQ9O%-NB<{)Qeb7TY9Y(Wqp= zy(U(8r_q`svE#<}n8MtTz;8sXa6eR3&?NB>t1I+EXN6kCu~aD^#l)ii?+L`hh3_5N z%iVoXf9au_Wm18AnF%?$;4x z$xOPHMwj4t_by@5@bvQS)flCxC8V&apqSyh8@Epm{lzy9DK=HHCbOH28_2HYUHAu4 zidXWsG{VMfyR@jbt0Q71@mR3<&tkz)zj0rDz_OErPtYQmoopO*qMXIwgQDbuDA}mQ z)+Y|!_Aaxqze-HBu#NRz+xP^pLaxVSE4O9)%kfR1?84$0xlP`!zL53xJ#>=D75%}` zE+H-~OyS1n$zknJCJ&dLQsf~GZ5EdIu&|r#39kC_#Wfg_-%Cg&vZ)s?DH8X;VFMG# zWR8X6Ufoq%{SDTaa`&G^r07ctIng9RC?))F(UJ~%dGN11sXblaCy?EfZCuW2T1Mo2 zWg5sP%4lF`;ptCKzxlCy=l1^X8T3~wbh=2EifQb4F;*1(kU?YR*mg`N<_(`y;~7db z-Pq$q6HECSqWa;=+PunIW~ob0#tQuSDZP^DF`059*X6{XV)>URt6~(NZgXoTW|Eqm zC5lCF3d$Tlk`<#mSuO|G%wEN~{I&!Z8>lyAoNvh6d;~$dh4a_y?RLNKr||4ni!b!g$R2NH(?4eNIL`XE8~YzX={g%yzLW z6fm7ZdO!Zg0;PbPSCrSIK$*!q-ghG(!g^jVAS%Pi5{S)W>+oGpRVd^a##(aTdSoWA_*a!*X`#(eX~?8oS!+%xgZVU|-8c5Y>D zc4e)32<#~i=HXQ#Izu?gQ=%tj%mtkkapPO4OZ+l9H-yw;!qDT3nzKfr$17#%Hd-_S zDgQmE&r}p19sSz^#?S$k2@n~YiGW}sK@8~YlZ5@Fcu*mOJrxHVyJQNOnj~z-p1&Sq z9^eyIh+5$R3Tp9svebxe?CHFb1*d&4B25?- zUy6QXTC&^tbmpW@?1Ze6xWb9YkO1rHlI6+5?yP#`cNRt8QC*b|a#X-GwPW{;es@%| z*i+9`InCmkp%#S|%_YfZ)2!aB!=5WQ*lw3!$lBiir^Y%pVMQAXiRa7r&J%o zj$rK?ge&4(!o~aVQv}|c9(j&QTK$>?EYuJ*J&QX`Qbg_FWW_1Ab**Xi!MSY;3;QlRN(S!Dh|g#y76KL8@yD7#kwK!p%nA51`S~f z`~D{%)MPvuTvy=%q}`&h6>i6QZolKX*fX<_Z)9g8tsbGG+G$0EitmS$k?4jWC3IHe zr4ub~n$2w8VTp^jt{GA*u6={bY1{>MrY!hzTjQpUHhN{3Fql06xnS5k>3ypBm&(GXg03%d0lspvh0<|m939IE?VCi70$H6eo0J8ewH z$LUQvHr{xs*6$4Pku(Lc5g-9a?KZ;*h~k6f=JbG{*cR35_Fg*6>i2I-fFi3e`r+c} zQ#Obn(nkJu{TSLoh9)M-l1H1|vhE?49GRO=v|)0jiT*HRg10FZ-$kVEV_%z-KYuk- zBKge-QCA}3jO)H_!0(4(67m3C)RnWNNKh}&C0yXd+@$8|VEhVqj_uEEu6 zcVg+yOpV-}LFAIX_LW5N-3>iW7M~6=-KV{c>fXre!0j^$80)~9;;aji2V{6~-XQKu zmSs|Vn*)ryGpOG8yXAqaj=1UAcdgOeXW1apuhS-L%`g^0NkCUT?!oo^Xxx}3UR<)~ zL34?_#OFniN96jf{&H(!ahDiMYd(HRazVh); zu?$4*7KDu%h_+achyLi0@N>X3lWQakKd8;I^~`9lknqBTa(i)~q3zJ`iNtq~o9^@M zjJgMz0-n;6h!zSMqAYkKWZL;0$bvjIjZt{XOJ`ULxN|GQ6DeRcmWl9`R09mP6c**G zx4HiP{xv~%CeKJrJd+u^MK_~V(QMFqUQncz=bpX!HYdYf-JMm^;BOw7k=BF&dsyaZ=PUDZ%fG8S|@CR zGn_-k?B=UwmdqA|&M_mgR^o`e8vqC2x57)B;(SF#y(v*}w_6s3nQL%;J0eWD1_BU1 zI1!*iW4EqKlj5KIcHhZ5%qq1fNbE4{3{DY$Um)U{j#q|(b%XH{bDgX>b`Q^O^cQR0 z?~+9;VC#P8s6PWwygIGr9t@X8NHC(C zF*XIjHJ1XWoP`r#efA1F3-3xWJPT>MU2;J-$7AqF8b#H7&8Sk(vnq|>ZF)JjGfiXd zL}ZFqc+7i4%&xz!%_%ik;TLy1-k#l!S*bH&;@y}65ZF@de=m;Xq!;4r~7Nz?w6TUQ?CnQqf^rwPCnlK3Uk3C%c>`242n+? zM#Twp9qP54Dg5swuyF41S%tub;GXPytPUgi#oUAh>-pOh2!3k!n$Apf%*i8Q)KFCX zjQ8r=MS0xpDtrWKEKJ4Gzvh@-QJIfj%Rp0?8 LgEap6rFs7kQc7In(Eyq1CjX zzwRIUxNTeYqXSQj-Ftajz2cD5hN|)^ zL!O8l%2V;T93>vF>d^jV!Zize^(EXryZRrtZFqb5XvFPUsMwWe{;1F4bylgNaMy@B zCE_5s0T(n|hBQOn^UpU$(~ftqypp(o@MODe*g! z0qxbJpr-+PeEPVNLkM*Jrn8UP9{M*?*+;olX^FgaMqtY$o&dnBPWL&ss=$E#VX0Dm zJi$7~44!8TzDuxIDnpbk1G3oZDp&n(&Vuw9Pa9qyiTDR-zmr$ub%x7~HJY#geocQ` zqkXee5884z(Hcccq0+p(!r>}&`u(ccL~^|tdJVhGia}3L%Nb61^K6SBvVElE^;h~T zIi;aMH0YQdRYO$?e_RfP000Q$TbTK zdUTi2>t6B1xRHc@nd`G>Z6E9C1C;(sn|NeLq$+4F$e}<`;)Zu|j|pQ5yzLLQU$@QO zLZlB-1}mLHs@ECuRHz1*#gQ(Kf}})DUm$A=N&FsvY3eX0e)(Hz1r)!|306RUxjo^C z)9*5UhLyOi{IyTBtLZCt@2~%F+a7(5VaiaYQ#7nPLV*Zq$CUQr3a2+LvFgm~$tMTV zA_K3_d(?K@Hu{iWN)M$uEl}(Xn}}{0+u7Ph`QpboHxtu*$C)s8exEQ=7}Is4)sPF0 zJr{797kQFux_R#xnO72--P>OLlHBLeZ@cPx z0-m3#zGj>A0ns{28KLA8a9Ew}@PaZG3j_lwKA^2ioUTY9#6Lhfps5-(frxy+qN42K^ecQ)Pm~q*o!=iC%iRVIxOh~W9@SZZm-~VFXX?8wm zZYOf_e71JF-3g>FS)y;Qb9VI@V)~LN_q=WE@DYJup$t+w_?+dcqaxsP77ObI$ELy) z_7sEBFRtQCNYpb;+jdFC)69_X))zoSp5_mPe8A_Mj!O zNv?fO7aejCi%0isKiBrN4m@5Ns}yO#AvNrAN1a}WD*$H9C4?F@Z^T$J+2wkCWgjts zmaF=o{{`E!Pv|>(C`C$hkV3!H<0w_V(PR1pGPA>zqt1}(c7Uc$8uHY% z!7ukABDcN%)L7duI;36csx%H)dP>b;(sAYn_t+=Z?EMI9j9THeJ-&;`>!uVaO@W_N z9ARgPNoz`QTssWh)_4jN*ioMn913h^u^NgviamZ*_UI#VCQ7KB$TwYA&t=N3#%F{o zlv@;>F4bY!ZimzFb_Al4U^L>WR6V6-W@Bi%P}Wa_Pcm(>QeQ}D3t!Y5@qkdFn`anZ z{t?m{F^TfV)_3)0CTjfW^c85LBv;Q3+78I)xRP@t*^?7rmN=R3oc;x2qVJqeZf**I zwWsFX!~lPl0b6M7bbzn#v{qsHvI2hckI(Bcd$DylQHb^;8ypD`iPT|>UJOiUKK|oh z5-j@oncyj(2WXpz`wnIL+_#57QJ=SwsRR@13OT}&DzE8iG>SPCbwC@TpA<1P@BCFW z!z|3p-%AT%nAgCX)BPpUwK33K)laTq2Ta0MJrI!pj3;eC@nZ5$)7$5(`hWYSBEH>c3T9V>vUN3{5!%452@fSOTZ3KiNmj|?&O`)CFym(Vu9k#2Z(eL zD25xATpNvlZYmqfi@gVWD8xbHqje!YVPjpTQ7m8q_BM@ayZ98Pq1 z9o|4VEbK!3@sifyW@q)fwAqaOjrtl$ek)^*aL`i@?hP!#B~KD&lP#Du^_jm$5chxi zZ22;q_gfqrjNSM*7Q5V9;#fSbim*(r*3T0h5`RT+^uE1A8R7ruF(X*vy2z z>JZV6!rsaNw#r>e=dzn2EqkZ_y8K#V$@l9YSYYe$AA%jB3{&!4;OUC+0uneO-E#n; ziMbSfTP+2qR^!M?rrvk!%b$>x z%7LYb-!6F!N7lB9G8I*Hm{3tgWir&be3OhRsi?l=sws@pLpr9Q6jPp>d^sWkN4VT$ z^8O{%i8mUXmbGm|&1-1WMs@l2W!u6dMCowlN~H@p{osy)<2(sGeh}kG2(w=ykyZ6O zeI_8{jOx;~~f>2T2wKSnRTqUNoha?5SpA*+yR*>lBxL?0L>&2_+uy}`Dwa9kee~I2f$^fOUnch&?;f}Yr)_nea*2Fg$ z+4X)(#E#;qw>%NBCD~)#&;G`6->}B5j3?LTm_4;N3v-+9C}kVROlA^VgWx47>gRBdPoRsKW^g)hV8c1)e2^ht~|go>}gN zzY~QbdJ&NBsOH%KwgW^u*^i&PtKqC+B*-sS?ptFUdxoIW5$vDsyhH znzT=Fr|s+6lXT=+CrWLXoF!z3aw8Z(qb3dMXcKB-%)oX4e-gqDy93lpsUL%Chu7<@ zaDw4oAs3iyL`@YIp~sbpd^ANq>^YDN)KF{NISvJ&bX5P zr&v^Q%OCVnqUKQSfpd*H*@URp>SW9l8+S6ZkozaW3hOIiwPb2SU`ujq4e$RWJJdUM zTydxcE~VsB1OEM`VqSxu`j^CqEJ}uBsU#)#t~uMx&RpO&e-kXa!0j|03Nip`NiE7v z2ye}2XYWx*Sg!619UpUb>1fA9#TrwpCXuiP-TdeBQs(K+sy0IFm=K;G7Ip-qp$Mfq zO@eNX+WKIRnoNy!txl+-G=iv;pwoGH3}I;56i4@Nvb=xs2OC&Tt^AS zTv%=4ypW;`yPU6lug>DfCXjAw99Z{`?Su}0l`>MvM>)^}A!8xT;GmC_;)m3}yZ2=* zJGA&h`Vf)aBKKcIjxo$#Q06ErFeNLjndP6iZVwZc!8IjONehBSK+4gbG|7szw_MLn z*^}vKvH8zBB*y0HI%LoZNxx(Jj56HLh!aHwy_dMg1SyvdT}5gL z?2#de*DsiO-9*~W^y(MPw@s`~z@wE>N`X^&Et;hePM#M+x{%ZBRlQ(UyioIsni~i# z`Eu~fj)$1c*Q!HH37PjpXbz4H_>)?}&w({YE|S*UxZ&`KrpySe)v?hCWI@F|3jI>x z-x2Z2;&$5k@EVqdE4C5MA`6FN;Spsdmn#Z&C91?Hu!`2CF9^&V%GyI3E3^c*hqQOA zK?vBvH+4JkX!#&^hkt^*ZSM22UZL#QlYcYjysNJun1irhwbbdVN)cCiXY11bBs-$i z;IB)RW2vF353<4F`jonoW%f@w!wxeGlY0^2p@qqg;aRwVl1nLtFiRn!71gyp=n1N5T+N4DBv}3_(B;AC?wQ%0n16R)eWD-zyEJzJlyh}8%|$}e zP>|QgRGr(Ow%ru01`P7_p%hL)6(>)tSF)^(%-m*3TO zfvrt5LLR9MSMq#;|4&~Kd}&4{Q)2a-(>kcc(tKWL>Rh@1VrTA6H6!{%WnL|Ps zR0HIu%&leC?B3OSZ8c`lzHLFoqCv|IhM;c^v(GxNg*tB#i(_lqO)Y($?as!`P^?O$ z6+~;Wj13tGE+&>zUv1elO`Wut)fV3CwoOPU&})?ON;er4{8{uR#D{3vqDUDOGRgv8 zliw*RHtieoCChYX$(`X#X~9Mh9{bi-GlLfFs&r8rhM`DnNxHqDP%4ZSzwrD0EcAO- zM-ri*&@{T|5~BIRS&F{{*kd`BHAu|S;S_bjKU27*Y z<-AEpMdjRzb0MYNDezAmA0$a2tV5|gY7{CdrP3$G~7+1e8vm2XqM6?@O24ccJON^OQ zT(^Wp%>H&nB_>z!Fwp16YZxh|z`DyW&A2H4GSY-MU-|QKTkBkcEKu^5#*mVAm`|(3 zMN<=N(?5FGXCd1T9ZZC5=r4vmZWVn|5f1vGBe@LX*c8&U-7mRoD(hsdY)|xJCu3I( zs6-Vb7(i3^e^{uYGKNb;b_Yo!THvrw6tea1IQCcy?-u(bSF=OBRYw$|i-DMk>6WOU zbeQ`KTN7;nqFdQli5Rs^H(%$o|cXYQ_kwGw*o!;uQFuj_~r z&TB63a+P1b23pz!{x0^sqOOMIh7p-*A|*b#b1FtpYN6wmv)623QGEp*SZZrp!Br00C_Z&(DHfyhx~yfX(M9=0zi6B@p;O1ASR}4qWVyzO zYX@y%hx$@Gt$~NSqiGE{zZ+ZJx}GH_ zx)u_gNK8ODFDj~_osA>KX|`nj##cWb!zkUYLyKyI;4*fMB?*_+tE44M7R=~Il!{~l zNILfKiI5w?HW z$PNIamOnALEdG%5#(J54^&SZ`8JyjnJ^*E~2$vBt9#(fKOD57a=Uh(OS)|M9mnKkIxf^V4>7fa;!fmZTfWziz@c%3y9_({2c5ZrIQ*6XBYMvi-*WmhwM6R>C?XTb6PmL=?vjZ@dMtjGI0EM`4=FxY4iG6}I-f#4PTwuwH& zT0XxY<=o2BepP+wJ4D(~EXWnOW`3=9!g{3#{5|$DNosYOBuQ$)rAhb~;s?5vm#zl4Yhltb1B z(i@^V*f?e0YDJ5B6Z6wrue!lDr9XX6zLKXjED74Xb7X5N`RtPR53(##>&ppBWZ{ZS z%utp*{(0UtmorPaQ%4e(5Nm~yrNx3NSXxIsur-Z4Z*A9+$=WRgh(;l6Vk!bBbxk`Z zQOcFnkr{6`XP!&PfrN*iOEw1X;0I{a771{r*8lEBMTOnv?tq!KdtJz!|Ic-3ksT0G zB{uUVfR(ZXtl^;c(pOe7%{OBZQ7bf`@%4^M52Q9L`OZsBz!a<@L*M*e-kv2Lt{hCT zm~?1XRs{|gj-;AoR7#3Sshba+w|vN~?kOEiSlyCfVFB!Y+txq>Hd54uC6PxBp*2Jj*|@_(Z~g)y#_viN-?^sVK~^g{ zrQ>6*q^nb}pu^?b^zC*Mbpl)z@eGb+SmFzbh zaQ}W6n=Bo6C1K-aX>Q^^iTFED63fsI=d4joNB^vYi@KqH-^+O{B1!%DRx{5sL0vqI zz5oTaL#$vZIt*elRMMk!O4euZGM~NQaDqpl-L8Mbc`YM-#l2bcxGWvfml3jL2`>ur zI7z`bT#_ZOD7US&UB*7=z7f*rF(`Y0D2fUQv5K^91IOq7M>H z2e_1uFJQxB$=FgU>`<+%w*HH0m5x`@NYlpwp)JDMMt-P z$BfV?`Z}U{RqsLw7HG36T)wOod+zc&VW!wCM-%z1#Z8CRQ|l$(XxKDYbB;B-p5V7S zWR9UVL`Z^enKnrXy_P9WHvFsR%UMj9Y}J8Lmxy3W2ZruQ6^7xlyBbCoO(s>E9Fu3< zo*V~K9zPw6a|{HkQReq>0`e>1LMGEU~YTP=Y@9UB{Th+z^OIl?K|WY?%U z>h;fDUv;%cQ z!aI*7P_CRu3~%xa1HJwlqBRDp0Vc3smY_T2~5C8U#O{C>68GWpFoz z1808VPs7C~_tW}VPJ6qJm-$=I=__ePdsqo-8X@f92M5R%fSyYn@l9F)x7=`M8*}nU zT}NMnPJS!>_+XmU=O~3az2tK=cYNw!=AiIPT<3C9hZfhlB!ULtJW^)#e?f3*&R$w` z%H6YC7R#wTay@;CD7b1uc|PB9Uv@ukJO)o7U|~EUMiHaL8gm?)(X z*de+o`C``G7{BlPI;=9g{6_i!QJIxj6LI5tFSO*XO$V4BzVaqoNa$fb&8pi!QA0fE zd;r(Ohc>KQ#mcn*)#1fl1j17rUP;)Uw5-gmbMiM<^?J}jM2l8qt>p$=i?!1VEj>=i zfL3_SE#B7r-(KgZV#fbIH%t8Q%sY|7WpJq`$~}%%M$}T%5VF)fg>_T>Wg~iTbfj~0 zrah193!(PRjWuRd!UZnvN;vK!i7nrNu`~aydjqMs_doi5q3uy8eaIN)Dg|x=EiF|+ zhe2%zCXt~r2~-7M!Uq2aO)J5cOKY7F`rWTyZ8R;4kr% zf-r!q73X;XGHRk*9wk|pQj%{!UdZo+JFSCa4Y?VPJ0uz%mb0*Oy7Y0+dw;O) zSyoOoitW+h@W#t?jZyFn?A)J^Ucu7Z!@LA3=0_k-gn%)H!E!bH>Jio@a#F{`=9^}C zWeMs~k7>}&dbiX5(>HG+DQ5o<-yXLu^$~%$Raa(KSDKPNGasKDV(NbE#N6}g!9iSV zz9}l|IlRlYYBBy%SdGYvCBMP%-Pm;aC_j;h)8UO0I8Kt9g01Kbe;HP2z$-sFm=K=U z(L^{U7^e*_q@XXpI(g@J4D|5;ktjexRy;Hbkpk$7nj0pvuEs$@0u>`YLT)T}#_|X{ zFZyXZ!#u_?1)OL#NP*7BjxX)WN_vYX6P=<32y zF8KiA((Vb4V;(5BmiU(&?VQZKt^+!-@VYc$qdb?VmulKb>c?36L04;FC7Z9p^t}Q7hraUoB^ecVOz88jQXMx5{`v74IBO0k7w<&i$FU{FaU<#t*dbg_9a` zI^twk;-KVGyq~+}LS~}eGl@dYrNE1JjJCQY*?6t(cs=HVH=ISF!Ud0b%C(U!uzEP` zo+RquGsix}8pGbzQAJ}|3>9T64DBe`A0gH;X$EPxxU+k0m(V^|I`+;cvc(evu?3O@ z=b|@M+`v*F-p2P%VBH$RN946F%Vdy&MDowqE zh(s5xrN&IK5XF$rPUJIK+4nh%w(nrL_B$oqEKFBG_a-^O@=7q_p<`cbG4u;Dzb=MA30_g2T0>N^ak9NWRRW6U& zA(zUk@`aokAO1Xro$5Jv(Hf$`)>tDfuA0C`trW62&;99}$5ejt0zwxmU)#V`*>dLU zopYFzQgAncih$3k8>GPMQ=iX2$SR%H?;%XAbY?>*nK}i0Xsa8UC5dBiJzPF>_P#u;^3>x?S|RQJv!2Sd)%^R2Ce&6f(E28Rv|v@kEc=^V2AP( z+#FyKx_ox68(0AHKOGPQn&tv9QNCLGaCR29UvN_?Q}Y`a6M;g_Y@fKg#R}HihnsME$!_KcZP%eibR(c?l0Bkgy%p|OQ_oS-fr)m> zBpzOo&L2#vSW1M8Mk;}xENlM_B=a|*WI&z1>45- zHC zx;>cPi`S%xoFXIByuqb`G?0N_(^(g_LO@V^pz_uptTO@t|e5FZb4!_ zetbc2l1H@i#=TT#i9GT0rK4G3zFS8YJ-djE1$@m|LprB4h1j(I(Q41JQ#<=*q8d+> zP`+9X0~+u|jsktpxO;u2rb3vQ)0&hG^?P{FxEvNmc6f!>z$h{c>8IoBhW^6jo3Y2P zS^GP))}QLQqVZDTY7U*t8HB7PJV22s2DIe+g!g_M!A7pC))C#})(63NFsCt}12-40 z1ke}b!YHW&)8!rS-ktFw+ukm((lW8_r7eK?4T`q#?1ZU3Rl>%CbJ$@08)Gu3GNJrj z#}wcARv4m;qdZHH-rcJafh?5^m zx$kpZ9TuK6dyC*iyaVq}2cY~LKTcqIoI_jw9K>=&n{+_T5w(C@_$?6S=v49nk45G@ z%kcbf6M-V<1ktI;jl|`2C9}lx^%HkoJ%?GlCjTNlksks)XxPA6FbiCX%JV{!){VYdxwzm zCWXd6#575x71Ziw>T;HD-lRi`{u~Vohr8o5?pBZ&Omog-PJ?$NQ77~rR#C*<4?Gsp$V7rcuMS^t1hPz?| z(b6(7T*-oNcHaII<80--L@;tzys`r(s1&OErZj{#ofF*~y$rJ5CJ8bVU$X$$5y?&Z z!Kly^?I$oH{aA+-zF4sn?p*{&Mht?Dusd)hlrun5B0PaK_`ZuyGDA}N9ubR%BugR| zrxeT&SxDLwJ_;`MHniua55ByU8IqjM1S|}RmNhC4z~qTwMdA(z$&g6#M#ZlCgX% zkuK&4giyrp@HoFEu3u9821}ka*+xrX@+1$J5WT_LR1Lg4f6isB#k~0HjBl8?`j)wNw;eVHT@~-#kJ49@Z_|QP;g?o{4?)?1La71tE&TL8D4`>av zC25sVl_|!aN9<_fJFLX|(5o3TKd{k|SC|ETS%*Ulny$ftdkt^ynh$6~ynjD`&p3uR z;X@)%!V@Ne<*!l}bu z!V$F6(iHB1t-3U$gdGy~BO((I3BJ!Oo*a$gLZb;R0`I#crVfo7wY34QX-tU8VMvpMCsun)3R{n5 z^V7v26U{h3UFh+{XuW8S(Hbkz2*71P>SQQ6pB_6+U%}a^U6eK6{hVZ zqR}d3OQ;D$ekoDxgBM%iEib=uu;2EdakGPC-$mHMOVA$Y(z?$1PD)~8ir^kwcSDr% zc)t!QcuWZdSJdr9Cn;zu%39-|T)uxba}^eRN~EHz(8d7NFTB8UaTAjz`I9<6{`bLk z``DT3`57S#i*CptKI~!D+#^9t)zfdf{gBNnO^0-7Q847tR=xL7r9Ph)La~{}Dw28@eg!I8sjCov3+8l{}%ioAJ zVn(w6mjosnhY&0k@2%5_3C&qp_@fSoVPPwTbD{*%>&ak^OpV48m6Fby{NU`dED61M z4^fF}?wlAbY%wSqya~&E&8>9nzb^pualmwFUNo3`oBizLj1NQiN_=RFE2Q!^ZsD!r z(~J+NbU0Bv1Ec8=W@!LR2U z`4zzm)vaCAWONFIQ@Vujy3)kh@`6VN`{(gzz z977wP2llETXI;R=V}p*3;*o{TVB#Hj^mrkeVCt)})M3l}zSfjk9p3>WS>$CkSbQ8Z z)&rX?Gu~ahZXlD5mIn!0mL@1SV`)@VX4`O9z5mnQuc;TSBzv{4Jf<>bl#sZ;< z5whbNGZOjrk{I$VOkm};Zr0_y{ClEXd#?zlVz^vLNZgrta?Kl0Feh!yVM0PDt#w&d zF}%rZ5K*U$tx&%SqG|7k}j8+qRKH z>3leH*Ue`+8}$eJ1kBlF33$r-o)>D>Vvdjaz#=%Qn2N!;2m@2|g6{C&)WlDv~Sq-QSj}(%^rIZsA8tz~UiWu0pKY zv1i@`w=tuV_7lOPQOSgRO&st}qa#IP=hk))VeZEk9aXp=GAd}2_=nXcdZDvIF5=j# zN7}J4ed^B%#G(Y>J93rBJuoTZntZ&dZU5bjy+cmQ?1j>h%K@mE#DTPk$@p2r=6>ET z#A3-)IxNPLS#&FnCc*LUUBaYcvrEQ*&M1BAe-fpwKrzF0Hy*z}^cUYaB-vESnr!Pi zbRoNvcg!h*#VdK^VB@u2iBxi2Da~|CuV0$m0%wTohpH=!sw-J#C0a68;KxttmBxd z;+*LHesIgYDPWKwL*V#J>7v^U2d-p>Kl4w56qQ8mAjELcnz_$?cF+UF&&gUuHQM_ zwo8Y-ySg&Jx{@6Rdy0d3cvXnT5Ki)qv6C|HyiSU^@h#jnewn;UVY!&FqRyX-StHQh z)kcm!F?R3e9YrIM^#AAdS&D-H?>0G%p}ndTATl%y0l`9o7|_=z2^-k?R~dsn90zM$ zG6hUc61L)x+uAY@aM1-st?&Q^wRk;QYQ#=nG2tQ>oc6Cls3H-Es9_xI7Ig_usdR)* zPMYn4@BQou9M?gGCewEcE1aqmyE>8uex}_P-Xgi|LZTMqq?{NY+^tfA7iPzgd`Lvw zyBGG9`lFs3^6VWfq8?R~ura}s6C>K^nIcNF@wbCYXBW;1T)$}?vPU;&UC08+-*s@& zqKe>{5rQ}vs^)NyWjOIAKbE3=`;&{^OgNseB?(7{CdOiF71H=7iIZ4j+Q;1C&0l6t z{J7eLDq=eiHgjnArD!*%CA*E=x<0dkosd6tToEH-NPzWpo#n~Gwr$*(KR~~)jw~H| zZ?Oj&$e?>hyE`gbZ2zPC1~FFLWs_Kut&?n3zO&r8J8#)s$Z&fpGA`YY}8Rv?c@-l z;``xbB)Z{83Z12R>DKE?cQ9L5-jJw8TbF^ z*o-I?S$)wD7e}A8LHv;Zebe$Q8QQSsGMbnqOCD{9yWj_w968&9&@egDTzeQX!P}IQ z?;=w7#@DtlIa`YtI7zjWx33&i6>eAVfB&h250h^d$M$-vb_%536 z&~V*q*i?R!$ThiI>B^+1nHm|LL9k@6{iRWScSB2)#iv6o_i5$@D-N?daK}sn#yW78 zIO{^>0T~{gH;KEHWto`kbc1nML-KvU@4bCwfQOEK_qu)jwgWo)P09^Q2F4;N3FwN) zJ-A*FiyPaE7ndA%XfE-T`hDo}h+Ll|P~qrS+|@uawzp;X`8mvCcrA-QOGE;?Kc?T( zBBCO`z1Dj%!#(^SmVsEBP1u-$Xp7}|=#LHyKL;dD?$|t7@{nAB4oS=>1Nit|FymXw(fV1`7`jutLmumiY?QQ85EmM23cwkwD-*KUMQc5>}RTN zVC&r7p1wpp&XaH*=tipq_4f}xCBOc8$vYQ+(}Ser@R1L$xAp2k>vUE+Dh=aj-K4#g z*iyb{%W>Atcv6QE-HgT*0M}d+m~~73joe2}~#w>!JiZuzw zp7_%~0c$4kuGK#{1QM4gHF$U;iz?RZOJP)zjmArK4yFWZKl0~qtY`iCE=0e0tx$id zFnQW7?Z%!C*SeV@t;(a7(2%yU?rLZr+t-V=u8CsNHX&b6cVEwvTFj|w*Ojo*sc8)- zAMbvJxnPlH)e z8f&X4pmpp@SEVW5S0?6{_&G|<LmR*8Wfd`ljQp?bSqyG=D60}PI diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569685.gpu-51.2674567.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766569685.gpu-51.2674567.0 deleted file mode 100644 index 9956aba98c5b814c60e5e43b87837621b50c77fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48602 zcmb__2Ygi3@;^v(FVbrQDb!FxNFg-oO{EtF6=B)zZjvS2*b<8PL=dH@h=O9lhM*`a zJ_{%!Vi&s)6|hn4hzJ4-BK~L2owN7M-MzTW@1OTR>IdWZ%$+l5`kZrd75(STZ5OY$ zjrgnVxkKak3{0<65^{L_c|$AR{z&h_@p)Qq#O)6ULL~u5$dwn4`h1Si?7W#FPsAO{ zbK`H>OFdq$#V|q(6~D07)_4!0U8wcavfO@Wz~v6v zQ$&5I_*Cd(_1iy@sC~5ywJewW?5K5Pv{oM-@VgCa|9K$r;y}W!vi{Y_ZP$H4)OOdp zY0b+$E|=SH4}0dg`Nu1Og~i?T4Zt_wdgcD11bFcK@{P8)>CU)TI#na1KO)c zVNU~ea-V-zUqqlUzV_EWw%ro6kJekuMqWB1u;md?0N~vY*IBlz(12~Xs3`MNg0+tu zdZTUqS48V2+C^HX0a@a3R=E8x&Vm*(o;Da5iTDTUzmr$$b%e`}HAZAMTQh*x=+Nxg zO55dM(;CHEk=DGT((WvG`2B9LiR3yl^lAkvE51Da;w>Wx??&5Qci7JVhVTYx{j}C) zp+GcfpAmJ3W*aQwbSqnTX(+15!sESfe0DHFKfSQs1Ga4vx~JA%%Rnycz+w>-y|rWX zR#i5xU#9QV(FFPW3pJjwRqQ2d2Wtbhwm!!!doU38h<$VWO584&$6sbbt`|eDQB>HY zyMq3|iH}wtMd*EJhDA9J7_J)p+HdKMpo@_{x_b$+a0X6$@Z!Q zzepRZbq=|`j)5?c&O2qU9s-{rHuTLZURwjNk_7nLiez^%&Kz_MA;fTZU zG<}AZxU9ee%je;z`M z489`oZrcvINH492)|?h7afD4oH;CS#~W1O3&e@&mlgmKolL^uj#3rVy)I9#p-QkN{zU%n8yU>q_1-UoNTWV`Q2B6^rMMC<5tRJiSx0jHxxST8s>m7cJt z1dM*kY`%m-J=3(e&V6t>Gvt^2L<^uHPxl8xKH%~kv5g#2k?ZlJCX9w(G~{~6)X*s= z*V@m7igKNXsCIWS?C}Qt98}r90(X~u#p8JD)f)p$uAh~VLarUbpm(~w62|4Jhp#_8*Eb#60MhN`yO-9{@{GJ$ANPLPA^kqTYxHgbLj} z!|3vlP|k=+l*i1gzn__?tr8qfl;Y~SK-&TN99MB}6nkRdxZ^|SJ3EgM!RR};AU8J! zz*)DA8_WRf9aR9+bavXo*LPT}uzXbkKc7496tfqbBp}+0EN~=1BvOan&}r}j=HuIc zCpyu`&j3&P96(pD8{udAyyh_iMSb2@r4mf2Gh`1(W_wLXqhZXUI3KhT`biN(^Pary z+Rs^-_q1F9!@QcnaUvJbZZn6=;j@>xBaQ+tNeWzk zUN3{VOWw-(WdNxH5hiFS2oA#p1Fn$6DTfK*6uh3rYjYWSTO=Nm*F=y9W`?6F#|KSJ zu|eL1F-vwb$t?SW$U{+23zxatW^VW=0g70|=W&Jt_7X?L zS)L&HRE}eTL1U8>xlgbo{+fgoN4(4%C;=;#1g7GmA6j5tCL?$S;k0y#}ZIPJH|{Bj5R#LVjmwx!YM`FAYGF!-4Lu-5Usp zgrIUOI;Uw3B)^rhMmXrH0QUx#;F6~Zv&j~8nbz;Fk#yFM?|kS1 z+a?J;Q0uR?HlTqiWr9EEe1bLh>c`8{ClGd>Yep}(+5aZ8FV==>9Wi(dc}kG&VT>^B z;Qf0lg$tQ9`RC1jCzH8*HO|ly=|%q~*lCkNQm_mVE!>ikobc{vmotYQwcWCwRIdynqBwNO}$+G%=Tg zZ>y!iWZeJ#0;b;ok;|ZZZyXE-Fy+U=toQNpc9*}*e5og^jM8(0B8x|8V1wL2_zRd( z4y;7{1&YUT=Z7mEU@GeJs)ULvDubcM<(pJYMMX7ATl@v1)OMaqDW*Ji`Lag>_Hc#A z^!e<8O31G`Ok!wF zdY9=r^_aSI*CNtU-DR6l?aoqD+CvczE_~rZ9p|#FN8R%YSftd=zz%C%E_9#l+ZkrK zKb{rWTvkx%<+u-@{^znWbXdF>jk(=+n~mrlq7Bm8nduFM?XGxxYt84=+MX;%c5f{a zJ4&M7ibTMcWWOWa1W9}a z-tTlOK2G~dJuhdcal-{fHlD^ds5+d%D0nxJytYUKDcD@{+TM3CjbXd%DA?`YzF=gw zJ?e)wV&~lpRVyz)l8UueOw7E|14gssQm-yyi)4_?4fenXREHxTC;03xld86sk2$w( zt?MN5d%B|0Zrh6z_;PKc)?4?QC`v1b5(v09A@uuPK`;|RugB>K>#5I5hv`Es`E=~b zTe`9>cGaVA5fTqcPA537;7bL;^h5$sqDjdQTy^T^K8#gE>np6v2B85rBs|;iOqK4? zYrnafE0WcKfYB9ct^=cCHdzK`N0r#?)!Xiw#6qQA5)DJ8bRrWX5wF{XXUW)s<4;v% z(c!HPi9n1F)4YK)i%(W4R3fj3ch~szxxDVAeS*1f z_1!dGB42l?%;s%E$aZZC7(k;Y4e4kTYEjIE2N~2*YJ#cNJ>;yMEb<&26U{S#i z2`y?4B_24}n3GM2YOPN8{&mL7%!0QWQ7o*lfYpNeE$mR&ZBDqtT@bjG zl1mNv@rRzgo$+C#1Qw|PN`_;pBqjDf|H!AGF&FsKG@=(>;P$!>1sQ;}q#oragtzAN zmrw40jpgc2NPNuIwLm*2D%O}%HHm~Z=)+Cwyv01dd(w$$;pt&vM=%9JBsE zF20l*?mJr&g~CM-NI2md1WHT20dqRhn$+lRjjJ(b(k}*o12@g_sMgEu0rh zbWxz@D@Pps^RX*QH#G`8yUn&=!jI8LY56DzdLU#hgc%(4aZ>z{X9m8S&9XxaGwDM_ za*N!59XZA@S7EumywH@aux8d5qgOR%qS7Ht5tZ~HSOlaT-AR+ID0|EG++9^$EN8L# z4he~|c?$^{bU@NCSvZ1TU9Rhn~2;%m= zkvfxUH(OLaZ;7p8D*_&?jn)bs!fVkjg>dq`5YmMlUa#8=R>cc7uc*0!z>+UJw&!nQ zGQUYKC1l6AMHUXl!XwH^PG=PAN>qtYU=^)N&n?;9n6-zjYeS@Cdq@YDI|uQ$kUoJXG~j_ISA`@mpPoXQ^ei;=jqS@k{wZM_*3FysiCP4 zvccf`lsS`S_KKTMf6pw;x^_e-T9~XDo`nmjxRgp5Gp^O*b?h{T+7qZajRq)`O|8QT ze!>^F-^;C5-yFge^-&oyh9Ovi3S+Y2{sluSS2J5!vx7CN+2gl7b$d??9mdr4hY}Ul zbwSV*bfa-KA8wIg`L{rq2csW5^B-q^%{?87e)Mb7U4>B2m1>%cgrcD!uZ^iXm!YAK zTs&tL^GL^ZB3R*(N-J;{cuU^=dd%6Dar~%+#(m62NYN+-+7&uLyis#HGO@({y>b5| zEJb!lXCfF=WL9V8FCRDJ%0Usm|A$yG;J7-SfMc#bp4=+=uy>n?1kyUU;A*%!H`5j7`LqZo+1LUU6t!38i zetG-R@0dZ`BZ1MN&N2QngynozWfK9bYX_#+9D!dv<9o#kb&T0 zVkz~93(vggpuMcJ>{gepT{i-qs9macQ$fL>MPEXEh@LHqltUq-Jm59?or+@9zad|- zOlzxMa;!Hk*l^{~`)xbqf?c&PT7xhYX)Q@FC@fM6WBm@?@FENSe8oh%2>pbnk)Dqo z3JESy4VZ7_@YXx}MfuB{7{8l#SNV-lu%TEZ1O;YIG<1|G<))qCy}ekQ#tRY@+ca`i z&{>X354g5eXH1srrOQrz#Q5vzK~y7u>v>JFnj&vNXnwwumO5{s{S31cHF^>f+KCq4 zklsoIA0O99qM$mVKUwXVf3>Y`J&V&Gm8ck}WdZ8UFqjE!(n*BeBz8V(yJrQV^a=e_ zs%QVm&goyUPKz5=>xlt|Q_3J7%jlKGl=I5oL?$Zd zR-6kdA$R<3eRJma_P&siMPi5{Ct!l5q8OgNY(7(*`y{SVoQUfP`JkI4aa^q&Gn6ys zMEG|6dYV-N=l3Cku@czQ08T({VlTnx^#0B4MowQsM7xn`AST?i#F#ftf4u>VnBSF{ zm|VfbK%XD4VU&~t>n@vc=8{kPk|vyX$&p`dv-%O4g<8JW2vV|k^J!JMXli2JHt_!c zvXCvKKY@vm4gJNC$K^&}RD^>*=twSyI5vg!?C$WdH?mH~FXR%~$=DSGDpAD<2GG>~ z9~P<@vHTR1h&2NgiAcj?ohW4MU2*KO6yBDBnTy#W_6{UOafl7XL`=6tUFyDSG~;=n zK?=_sV229V+Bq@hm2uW2wGjrqI{I2(8aN=A5TyZJ!orbIs4fJ(Za`k%{k032BrhIJ zR0~K*NC;FBv~cnjNbWj+wea#GOzCzSLXct#NAt+H9DKtpYN6wtg6WU={OGUy^|6qjvPeJOw4AFuRkSaB)d^uGc@Gvw@|l7ZJ%g zu++}9g0lj&QGDvoQY=arbpH=ajn=xD&@naI*@R9Vi(-+udXeQCyN1jyVTXFXTmuzp zC({~Un2>n#ZKRF7_D0 z7!sxWjPqWQdHF$}!>V!DZ|iOA_u;=RaLpvfzM(6UhRQbnM>~Awd!?2s{@y z=#mq^Gym;|kwmHpu1&TO`XEgyZ)^VgdzEFgcE7?=gpBQeT@cv^*#SV*^Ct$E#UFCs zSTFNT-}`T8GPqkVgECl*%ZL~ct2>k>6X}|>E~o4)XIIte$xm6-bMm8c3y4++{2c5ZrIQ*_9v08!P75cV`KEql*kG$3YAC~r8HG#fEr2WK# zT!HK6*J>xMSDL$aUn3?-7hbAJQah`aEH|1TrVI!{l zqRB^$+G{WSH)`WuKV_RawF{STx|ZdnTTdi(F}*8z;RwTp1Q5B}jcx6FauM9xR>tAN)^%sZ4_&yN*_kmL-k699R6y1T(i@^V*s$bdH{|ZWkeHv| zX2Vssd6VdK^0hpzL20m{J4d#bl8(<#dYxs7MqEx%A`4eiYKF4p@dNepA7z&CltdDi z5Nm~yrNx3NSXz5Lur-ZahxB=!$=bbF5RF3C#8d=M>Y8>+p_D7B$#cG6$2^yTR}vn2 zE?F43gCC$zTO`0$TK~Hj7Z-I`y91tC>z&G+{{s?QWCui4iOqZoV3q6uYdCo7!N{LX z^DUfA)C$dKe7$|92U43e`OYg$z!a<@Lq$siX)NiGa}~j2(xF-TY;dq}B-JFNN>W5g z-8}O}%>c8yZ%Z&?bxVUqg|PQ=^R1jzYXKN=WqDub?oXdWw2Dp$y=6$hQlY|4Bf4_X zI@vY0E`D?_OCqCuM(xo9bSz2crVdG?JPU1d^ z_&ZM$%h2J@het3Sy-|XTx}m)9oK9)0!#`3>i^jPe!BD_TdGEUhY`uMo0i2`>urI7z`bT#+TNDARxKw}E|5 zA1AFNCXO&$hCMf>o)CD{u~n_6nk<)jb({A+R&f1BE+Y!Aav8V}=Kr$FvHwiFgthuy z=c11htv&|Tq6@mx6N+H_M(nE+l%_2|=*{JWMlw&Jt(%q-O$WG?#22t(v1Dwi6n5zM z&uf-3Nq9r9BqTwMf9g|}`h_-l81?TH<5uQkyX6BE^~zo3j@S&N&5m8tboZ!~P0tFd0A$$J&|^N>WYAwm*#%k)V?=(S8~ zvSHd|%_nhPQclF9E)l_$1cvTNHw?pLcQuSGnoO!PIi}9Iwe%h6!ThP?5?=UIlI4nD zl`Pjs1e$-yxPFgE<+{sV3Wqv2HfR^aBsg+}Q?ALbkrQxT#{_1;bVXoV>6l_AXs5G# zp?lO6#41KY(R?SSfAS!9@{sZakM~rvJ#OURI${ zohcg0g;gd`mUF`rc0?K%Huy?^|2kL}`cOEIATseQ-aQaIObax!64w%4t zS%Tiw=jM}4b6w*jO7*T3pfm`6JW(oUlgr_53OmmHz@LUoOzx-ku`Hglvpe&*vi-D@ zUbKgmpr#SR4t{WeoB`;$#1Y@51#s}8($ATb|C+={C%=_EKA0x;*~?%~FZmqJ_Vo`o zvs3sbu5y+VD!l zuB2t>Ps>n%6fTCAT|Xz6i42DH*+Zt=F}|9xM5@;>ALe=3P?%27l+s$E3vZlm^Uk7gnXFx*^{H^f)c~czo;Vxt55tXEu=ftA zKGXvfx`){w#!7p%$>V1gYuq;I*zmh|ib=2V=JO9-a| z^%8Wy2ZK1Q>w~mZoabq+I=VIOJCE6(Ro4*p!uC`Gs70fWCAFEACw;7aruTmcQrrVa z)5P(kPu4`%Y4>$D6X$OvqG+E{5be{WK9gd$q;t==?f=WfZsE0x*k!4xiEepRWLZf` z_G`GA-wD@Y9)V&FxfzZ-*b_98saaSxU0U+~w`bV)yer|v_ULeUsXgagZYGq%{R^P$`aI}9@C(k^=@~3@Zu`V zh(Q0r{lC~=kbn!SYV)dUP05~_kIz*xbw7S`&bjp9ATBlE6c_g#(Pfgm1pg@Pj;M;o zcfpvGY&yJT0evq{hc`^%I4Nohwjvk)XIP<^3b$2ZLRjm1LK88XV4Oa*kb=JGRFXHC zfj-GVK~_9836TQm(QAhP!MYm9EhI|CNRN;ki=DANf}h?rXd%Nq#xMn(Xf;TI&cTyh z`K+Y(&<#YVXaRycx$fNRTul{y{eUAi7;A^$sP31bSoafBAi4R?H;-U!I3hvyX(`a@ zc=MyUp2ylOaaZah-|KtWF<-fe2-U~L0&vF{Qp;T?>r*-Hvfn!~yxxlyyk>MeZkhQ| zDqG(zzn60BX~XiP2beSQmBbT$!ZK%_GH&&s8gVD(x+O%Xz?E6uByhQz|E12r+gUiA zvy^Z#9ERJqAe{%7e1LH2_XNjrLQ(N><+`B(e!FH(6~@?I5*RnCx!xlT^8@B2n1aU& zcO!B)A7))li*6!9MHds^C4jdj;Y|d}ZTQMQ<>wVB*H?VDx9-c#0u|m&aH6yb;}_6V zf!`dDsZa=*#upF#uwi4SL3c?=alZ!ozC3O0W^ErbfTg9u@C zvk9p=k5`OG!=O&wD0kxRbobc5m>q|m5?xf?jVqx!uN7+fxt;t?*${?z^=(A0f){_a zoF(3>_di?C@Um}L@qBK)b1((Gw0Hlg&b;NYRF-alU;>_ic4`>`-jId z6E%GWQK-8Vc+rm0R+l6j!?&*djk)0URuZUi!6TjueIyI49uB)FiJJ52s8Osj>oo!v)ososW_j-z)F+2V zJZb50!Bhrz&ViI68e1xI8N5Fk#TacH@}~HuYX25?!Tiota=EN+6w` z$Y-#!?@v6w@GFK}bG3q-iRlV)XDBxLy8=$rS+?Gy)iXUi8SVxZHw#jxFun|ZUcw1a z!Tqk*@3s3`VRGDkL@$+Wzw3Vb0-^Ht4NR3ScQ)Cvh&d^JA0SW>l^At{ z6j;6S&b42&O6PYH6ASuT&`IW=1wORZjm(O~v9}(gYj!=$iq=aXBvQpVqlC4-7)?=b z+<;uVdCv%z-RS-hA?u?>V3?fHArdx=fhFJ_746el=i@#Jh@FoOJ-#x#smDP1=)yu} z{`jNbOUphGZ&>S|JT9zWX<(t%@HeB-B=EXnmatUpZ2=c9G%V*Zb+3-&t|& zO9_X?v8Es{aA9kl1yBx7ff63|o&Q!{Cfe7pCE8H5Gl=;XQ=iHQvluBn=)zj8G?=rF zK(RE~RzE2qhXOrC?CX16%kTSpUt$XjE3jn^h8rhEX1K@Qd(M^rV?sFV5h53bFb!XW zfLTO4KC!?Nt-k-=^|w65Vy5gz2~gB_@u?M@#iCi76co{f<$F&vhweQIj1FDfc;E!% zS4ss+@ZVE9+mNlir(3?sD37dHC~rjzU`I^}+L-v7fx?nXkMg|0zKp3ynQ_T@jDSVn z0PeB_Lud`F>|Mm;R|sA?d;T44?_QUR4Z}^6YYvgY;o^QFO$2k zB)Vu?7wE~t_Ari|?ToLImH3wamx#s^U#=C=@DLOn8Iyaf$X}6iVit|O0 z2rET2?=wfnGMzc-IYN&)UHS?p-YUG{;nBo@;Eh4zaDb_I&)~9__=Kb{ z`tv9|dh4Di3i0S=BVe*mWpYZ7RvtgZvUsC65wOVOAuz?d#A6Zv``j)MX7}PXsU=G$ zHQPIwwQ?Mh3yDU!_=TvyjC05OpkLacc3XB7S8k>c5v{w3FM5L^YA5rU~rTh_|tdpW%qk5_31kk?%iqcXbwVe_o&D7!*Uyse>8JEOBg&Sp~XD_3e_T2K*4;%w-+tf zxUS9EtJxZpUncTJkGir3bsv*HNacFn77vxyXX1F?Hd;e;rYUQP>wswzHNF(3>@Qxf zqj(a512d|z71qVAfaxH*2=iPNLh}bcD`yQ=!(X9qz=o<^!u|Oy-SUM*73mfv*5k()1gCpMD{tIOWtPa3 zmwoaz3(WuX8qq6yb`cp1_`0!%bWT|cvFW1^FWAIRRQ~IPjVDSdU%iF_4R|6)fxc(l zy}n9QAxzBaP0EJ)eR%doV_6jWsayl2$V{Z4j;jm$3zKigp0r?cIkVQY-yqsWyNxt^#>-Dfw$-r#NY~kf-W{G-#1yW z#lb1;5H)_Akns?;)R_;yF1+preR#?8;_BNwKEXugMTr^{6;zGUpmR37ZD_u_Na2Ht z$JgIhXCre>%iketMQ%rICA=YmZE0T6z>+r(FoW7-2LX#Z4;p9~8jpy)kxSG| z;N~Oqjk$`Cp+rB={c`dxmV>DAEOqe#&uOLo-DCL6ZpZ|k}C%*Ry zPQ*L#?sNdkzwzS)mdE+@`X)A3r0)Cy0b-6Q4Q}DLKvbhs#RnW!z0q3?Z%YB!c8FQKxgQdJB0mIp&~S~jU>3LvncbD}|HTjHl(TKg`bgQ9W|bCb zR#e`7*M=iZqr5KRP@}XC=?|};&AU^P)p`A4KX=;BZc-@wmAR;Sq|kR*MG^Kt28K5W;o6)vEr?klghd5sQW-Qy~_o6wD7;NAJNE@Lg`g{O~d%v-%?4}FTrE#W$Fu`|Adcs5R1 z<8PgK?|SCRxA}tbMQn`t&_L*gdy#SO{M^=P`r-Jc%$B?@*FY1WJ`<`kCD`+b9W8u^ zwOAi|6+`Ct?LWDZSB-gk{A?U#SnVEoriJFSyTpmu6yKszHLD7s=t2;YB2 zZt zHAZi&KqCN`0l8;D$q5Z(qD#~j7pse+U}v6pFoQ)ctM<_+=urziPo_8M;js$@tn<1B z{8rcnO}2DmS;k@e30GtpA*G8Ur|6rCHJ&7GcKc8JXg#wEha@;!g)9X%VaTr}ihb~6 z3%up!Hx9OYNy9np;Jk2v2*-nyra#W5cb)T{RK&y-!Totd**3=Gso$zRrUZgB>T;ly z6f_lOt#Kbr|7s6&6{>wlxacahH2~#>7dS3%VzMHCO2@~}rQJg8%xsp(!lE1UhYx#L zHTNjcO7(Qd*60aVX`1#u5iJUa9NMb)UeQrdK~)0JYST`?!(823KM=Zbb&1tD28ttm z$!_2-r`EGq75!&vtsX~g8zgF;maEl|OA9LGaTOZ9x&KHeTw{ME@=&;%Azg_gSEWDz zW5sy?upzH)XD?XEnsfe;kYaKjA$>3uV;MY+i&+x-TM3J4?$$9_*kVvJcoUZS znp@@8e_sIR;gbcO86OrORQS*oS4ib;+@rTH9mV+2`4@!`8DKR1VZ8ql zw3P?m4Yd?Y(PCjxS%1xzYzF%=gGDOOucx*= zmv8?+XaR9kDh{r2xloX}Gx5my@8&Wm?K_EtPFkDt*(LBohcX~0UT5amOy(3n;J?o; zxtHm>JN_ghMUyT;s#F`yzm-Q}ZM#WcU^!7vJ_YCZW zMkNF8HL=4xjrJ6Yy~L znMt?O=n@?7-X%;LZhhg{NJi-v2`OwUC}z0s#^aYmfANh&icM9li8r|58+IkHvpP|V zSMs(p!^Ug7w3xQ5Ct@Y>Sg`odj9_TblDqe@?4(wM7QyUfvydC*EdCx8B^N}=MkTgB zals28Fblg`Vxom@ruW*$CwLWdynnbSX4~BLL zabaN!H#T4M_x3-SJk+kG$U_?1EG+L~VK>e-*jae`lhbc~>|Xoh^4A&kW)-@HNS2Ce?07L&6#T`*E|p{3hD`K5nO*ZCN;6&E z>v(h96pg69qpEg$RV}l;R8PhV{P-!ove+@1axT~97*DbM%ac_G#jpJK%+Jgu)vQev zi{2EJIea9`pgLJD*WEX_BjfVp5?E}Y-jH#&A#dvq3?H6WgPoIc7ZA-N!7Kay;Fft) zz@R{e!0}hoMZ3R$dki!DCncn)Bw`047F1z8=4>Qe+UGu#?;pWplGSyHVhl0c8&@b` zI)n6n{EY=l0k@#IpjVMHllObt@NR4yhtyTJF%ww=v6-Gl_Psq`9RH{`D;i|(R>vgnGBN7z=>BYJPHsx7Ii zWre|>;$R+L6{0hQlRPDMQYM_!Nf9@`g}cTtlQ%uA785Sr_9=hJu1|d;R5Su9|2?PA zQWTavd-N4%p$|$xWN0P=f`tSzps!C7c0v2WT^Z~h4T#PdtZ~T{Ff~cozaDvEAM*hF zH6&Q!0SapIda~4rUA^Go-7Gl$R-%eT9HNGCtXtG2IHk%FHaThORZINr2rOm8V=z{J=J(d2bI`vz>QU0L!^}jRWxUi`r94&P*7E`N`#y3fv!V=RyZeG@C7IWecNL1m(d$5^9zb{3< zF)i6`{AG%#B|9ODn-R5mLNYKUz zNWY(VGgf>mv5^&7l4Ps$o#n=T*{aKEhPxn*XcoAr%i$sjG(`z#%q?-NJa*_~b_6@6 z6RwDF2^a6bPZ4<0(%f_=Y41y5p@yL8S=?cgB5Lgo3y-s{o7FWxg=GFuWhN?R6TLsvTtK%~%H_~M2`6Br|2m$1kPNiqhasi3aw zw(l)QYNOZ%G5;`mK(!4j{S;TDJ zdD(=Ewyq^oE3SQm%4ysMb(Sos|Jv&}vV*oxf{KF%kI%#P%XH~~BJar(c+Rk%Cz;ni zvK3K_UVA%~1cDV8i6u4|i5U^8U-&ii+K)+lSPo7fmoOz31@&wkvRxfM z88m>rNO5uM>efW9xJeBAnc#^;hsRWpRlc5Q_Fv4d@h&?D7quZgJha(5)>%3rl@6Uk zC3~{Jdb`Oi;q{Az6%C=4j*v(1k&4|@Xnn#s!4o&%wuiN;-`180M(?y486USV>DYKc zzgY_y;LvsoU{gQ>jyfHt5fH@($O*zINu3`};E`vR@+V zN<^G--M0+}0uW3>9)OFwa&{C6dhd&ePB6h-)PX1!zKgCqbX=D^Y$`t~DN|t4E zj>84U-5pZz`|b6`z;8Wt?7P?QHBgN0-8irH0p>9D=u8WXNI?2y@|G4674hx0-ium8I={m* z5MN4c%s{lmay;}$hlQU5o|#-DS@^Gt(|a?L!+nOf!+L`-S`(naVt(5zl0XZqcnMRWuv4o);A><+&$+TzCmP>n*z`z_p-Q!R?-3SWuwg zUNvI|@6LEt;-a<4z$?`P&=B^ooG30R>Y2~3r`qCM*S{y7n>DkqnPi)uPZW32I%5-@ zX`2+YTd0;2P`&M{KQ=G4;s5d1F-aBE|apoFqmVm-F5P@dp(r-;8V5cST+E5pFL!T5-^PF5VdHkViJ!CLog z7SRgWx}O{KXW)rfx4qIUwoE@}P;53CWTic@pl5#fBK1tnZ2Q}8w$5YfI$aa41Knsd zLH+%UPsy);p7vm`fjvk%e)w_66}Hjc=zDUtPFjQbSvO@b6}I#&a!z91j3*?F=w>vg z0J!E-z?8F)xz~G|c}|ZO6HRy)(sa923fUa5$s=hRQ}Y+ZlzM?(Y5d;a<*)acrun2h zA&XXc%zH!3uD`9#DYaJNANK8YHM<*gUJt^=yD^Jlr(#Wlu_ylYPr#Z9yleF@4uQhu zTR!OaHH#{8deTxDRb-*@5}ku7f%d(@af z7m<&KG|jrJp?Pdy>*`$-C8BLYy`Ju}+-vGGrzWj8VWU&i22MWS{R(rzBFm~LWK4=r z5k|!c`z31_&zig>frWF2&ng7Z1-Jjwm)>UtJ1$H}u%5q7f#A`5GTvsIqir7oMh!*9 z&v>t{eT>H~N8uw#m%_w-o@}dw@-bc5Zn#C0?;5+f4{46KH?0|Odta_opmo)n;(cXe zeu#7v&l^JsHcUYyaFXciBrVgM?3Tttci2*lwnHG+rvi2xL;!{u*P?RT%D59t+prWGxnRDmtJ#%+2?(+LbKR!R0_dRpx%$Yvt3@oSr{Jn7C zO53n|`#<*6s84&>ubLlnc>LM@OWpoR*St~LT4u!U4+lc|0Y}J{9gg~Zj!;?ll#nOl z4rROXx9kNTuiLKw$u+&pULVLR>vRp*}aZ3cPQL8;+`6@ z6I`FmD_>UQk{T5%m+w=dUB|(dwMyD0_}?dFwsj03v`~lV*4mzJLTKHzE?S1$?+m!y zA$y9b7030xWT5)(+nN$;53QS);c{OX^@0 zqt;%lTjX)M++Us>b)Zoe;@sKvJ1S9aFMabk>hXl?L}*} zs&i(EZ9q#}ql4B?t6N-ZcNRJPez(^|a@81m#axvY{XQK0%MikwX7VxqT7 zjNbCf#uZfVI(j%kzVY0pn{9nt6J%elx7N((m}(CO!XB}2ZePCJ(o&}UvkZJ{$YgP!?}N*?Sw?{qxI67#UtAzWkG8}4h4b=H@g1Q_^wd|-u(N^p0zE$ znn=G^>#wy5xxJ2vr_^n5Ssdy7C`d}g^aZM>P{gm~%X?Qb@$1uuRzUH~Ot1p-%jF42 z9Db+iGpxj=PUrbMuBWf~_(-)^Z71X!gSCNLn`qc=4+SEi9aGwe0S<3iVb$V{s;m3a zB7MhaFSjkoqz~z$b<*n60{M=xiRkLFoxQ9>j`%Uo%|ZWnBbYFD%_2+`#s-pTcgP9H zo(s6ji)=+Tb$oB}hCxJT$L3c&X8Te?UZZu_G6>S+2P5HiJ4)TA^RCFD>HFUwJ?%f^ z33zsz`x)DuY@&6zHdMG^pQG4qFAX>y`NDd^u_^V0 zJ^5ht^UL@W3iV9Wjy-zr6f@)>%LUMo*Y^iPKH%~+v5g#2k?T>zMh}NyH01h+Zg*E? za=ko{$VR!Yhp2XUFzoRL{2Wx-zFc=!j^c5wt=;=Llk48?2vW$kBN+6S=^l(g&kb4v zo8-Fppyz{jV)1o7Tis*(S^|&JMrj>%;E+4)aYY?oyE6c0%qfH#G;hRMG1=w9XBK(; z&~jy`{Tppd+S7M*(mH5$K??m2kG;_CjV6n@uw-;TV|LdLDzn2SQAfz_vV*2g8gl8c zZw=~BMDBj}u~D`!BxJ7ER;wAF;wdzPDaV-`+y~Cr&Fx88Bi*GA+q{lMUVAN1s}1~| zY7aXKOj=WcL$pf$iLh;80-G^4+0`J>TO;Wsg1*XQG12iFBu(ew8V=FC?l^ zZc%W$+;+ou+Z}$FJrIoqqY?WQx2LekYz!?IYW>xUvrJpu-I>T2+QJw0Mm!)?=;j$l zmw$wEMogmoUT)==n2E~lLf~kk6j#p$+78I)xQcV5*pru^pZhlRo%cyh^qm`!o0|gQ z_BnGmFu+^85|Id4PiLnce0_(t3d>OyaJ5g}?U}u3+Km9wUSxnH0V0t)?D3m+RbW2; zn-Ug%{50^CF9P&nwHG=ueSULyq80UdGnGm(q0W#!94Ygfjz*1`Ls1#D5&B6HL-Pjb zz4-wP^J?^<1u)F3?euzr_QH_)T&UJTW$lnT?_a{s?(-52&u$%u%i*)lSm|(!=aX95L0i1&OqifqWjJ(Tx5+0Iwl^_qy3`bLr z51N<`26>Yz+Pg8yd|JW@$qW~|?7n~xPG;1{A+5gm_l)rMOf%omi|7-vhR@>+1?>5b zh_fg`@TnZfT!Y5kj0LCI5kD^>#St&`2J*p5C4s59=sORc{+J!0oxO=nJV5Cohc5`> zmKCP5J6uyg-E_f1=K0(E5G;EBSHlSpxxrgjfbE_FyWj10CGV78zWSZBEKsb}m(~!0 zVhAz@KzziFGVhOi9grL`X%&V1f~ePPa60{mW4|%-H%VwDzm2oV?JTwz1R%-bKzG;f z4TQtOF2o-%Wesk2Zhhmo|1k0=^;5`iY^)IudWyllfhD-)DZ*^B1@?aDULQ(l?d<;5 zt85ke6LfE_r`E)P2Bs7W{+RO#*4U+A?D32y?5ej7zu&e_Vh_{?Xst1L3wiR9?O}{C z?BM--N`(uVH2I4^KUkT$dxHiL`RMMY85`?!M~VU_Gq1R&QNIWFNO4V9Z=F%Xg#AYe zjKbd70Jh3qDd)1iAuYZS_N*F5Ecxd7#dB?ouO)hiYJ;_GCwRIdynqBwNO}$+G%=Tg zZ>y!il(eqfo2mD%18Esl@3n%V0H*vnnDsv1YMYqOe5oT6O=R&14Q!A*2>$>x%7K-L zKUeV>ZV5mCB2!U!4I&y*MWr#+xO|g}si>$8GgjZpC~Y^GFa@QU^3>(a9tqgP#U7LQ zub@txY;4;2cec95(xwg1I{K9DONlx}8>D4{(+}<#ILNtAn1@VOGHzi`^m zOo?O-C4AI@jWJ#hmO=OpR)%% zqCmtRbVQ0cbiJ5oRZad@cF^N2_9|(h9tV1U#U$}dxd=*P!$=T9M4!(dD24or!z6~* zq#xXR`T?fyW{n`yQQb8%q1v4VrnHA599;P3j2$|fWj!*lBVdtIHv>DYakQ} z!~NibxaP8gLNCX)kJ$3V2s$j@Yey`yMMo07{j@$>b2Gi6u-z4JZ>{+}Wb!j*jO?sY ziP(`J^%f@rwj}#}@X{X{?tW|BDe(kb;x1X%^D@ScsiPHkw2xQJf6m;s5+d%D0nxJytYUKDcD@{DlgqugJHj}Vz+Snf{`+N z)DLUK&bt??R$hK26>F=Qn9WB^3)peF`Ff&TB!gUTum?V%Ivnvh!Dn}wRJFBy6rNr2 znv=xu$>N%aY}LmS@D18nt*h=eQIu8$B@l3JLg@Fof?y_sUXRle)>EIQ4%3HN@~P&M zZUt$Q=M5BEe6m8J5_#2pa!3_M;a?I?Q0SiOiCExSQds-=$>Yp&ubMy102Cc+;(y4kkg~I_0x%wtf={x~JAdYb2nnM8Z~K1N1IUF#SCm~@FyYc$hCu7DfDAdZTET|r4BHh_V)Rp z17ekJ!(F>)$bX`YUGpcY?D9Fn#h^>Vfr1E@kP4zS<^q2qVbKL{q3ckP0Z2>gQEozbYd(AbSnFiD zx_LJd*_f+qfObq&tTCl(5(#V2HlJUAJM;8%9Rw;oJuK`9Mne(G?Jx=MGr_g;Ti-A3cLYxI?wN(NH^6CJiW_ykBdkjp$*e=P!9A!$XEz7IOyY~_#uUT1~z8dp{w1rl1Ofm`>!L% z80N|=vKQrm>nxFaVIVLI}$Tft;34%pH%F&%P$%?YKT+jWn;C3FHM+%5+jLjQJ z$e;s~e#v59x9;T|nKG(VNSLUM8oC@22Z{)KFL8|tN-i6^irgWvM}{Ci=l`K(BJE~_ z@)c&=c1hstwBcHwLwGH^r4UY@7ecy_!|Qc>!K!$n<`p$J5LohM;qTg6CiAW$`UWBM zehAIMkpX{FFZkK9#>hp|dK*1wPS;~bz~&)rGy>^RF^@vO6!>>Ue6qM-MS8r=vT#pG zT#vvg6V6LF^9y z1b5rq=VQHsxAM+oj5(JUtIR=Iue;FUEK3phSnuGjUL-rB)bNDF#Zp6UA7q2U^(k~F z%j^%MAAi9t%ynL(6D>?e49~&^R9s3WjQRHXmAzP|>{kgYPNM+|WmD^Lf}e2rtjp^% ztF^*Mgre0-lM!PWf)%JRCL8X5zT&Klq4xA!qnbT_%TxF2ujR9ty8cR{qPorvdV+2= zuI9rn5-k4~=<;Cn^%Za2#{8O@0iqxMyZWv?DCbHw%|$}dP>|QgRGrJvP_JB9@fh<+ zGlK*x9C&F3E&y-In>v5A8O}KVk%Y#5Y=n@aQ3|vxbbxrH=5%CYiTib3y$viyR#rj; zV~VV;t}|VB)Y*gz7C7L8#35v_(PQV#NxX>m>SrG?W!fl2i-@d(GYVNffRwwNsO?l? z3*X4&HMGtwW&5VY7AadZwvo(gGyxsJXVOIW9xA~ zS?j;6w~XJ;;{ryFYy2A3;wRv z`yQsm)<@|JP>I#|c^#>9<%;n=W;2)Krm2J}@;j6=hlDPu2FOhXOs=gpyN7?*ViPlH zeaZ+J4O+`!2>RAA`z&!S)cKXMIJQ!*sikjamubtGp*SU15Us%~He?{Um{>}EtyenD zcF$iTQ-fz9;=Pf+N+@8&!R6OK19zJMT(%1Q55i+{7yx&>EDo}Sf=dVJ(qT+ z1#2w%?x?NLEwo@;ElaB&h9a#c>A89Bl)~7jdn$a%Lcik@NrZkv(@4+94uu34s0Pe8 za(L?I!css)90Ve8pGN&=G{i1*rw4^1)b_B z^?++jb;e|=-qpY0Ka9WL|0w*e<~0elB5y!wPL7h6dfdKuF|!k6Zzm+Q6Aio}y_E(& zKCY2OL3KiZvf42}r1yZ2S)6v+bV9{AEdx-eguzT;lTIS!t77M)ihF7hN}td_rF!;P zKDcTV>$G@6!i%P&7|}_m4-fuW;d{n~>+VpvkQM_Br<6f_ykuAzQ_g24Dk|s3oC_%- z-&W_{*O=S8>`tOrB!(Dr0w!20ie+miXEDXudj{bO#fi9%kPo^!635lbF+(|1PK15h z$17MR@T>&JN?=0+I03bZy@YGaCjY|h#^Jk&RJ0rE24ccJON<%3?CCNVG0(f3Ffn4r z!$6-OuVIvw0_!ewMqWJAgEZkygML0~>v0c3=4m-vO-RYw&8Jo2qN#~B;+|ZWg=|M9 zmUkrI%ZuCV(IOv0p# zAt6vj(89@AAi3*2|I-a#rgSIGAu3Vn)|HSUb4$G?V4J=#JoFeHjqPvm%C`L@0Us!@ zEiSKBkIxK7W4tYH>qTLNSP_iABYPUmp1Iok-BY0VJ{&3a`q~a1>KNNQYlPoD7FyZ^ z{;b^W4s8v|4JZ2VS?&8|i7m^aTtp<8XAcGyYw37n;C{x|HFMRS0v;gEdMubETOYc8 z>ko{r*UmGsRX`@s)_s#k4`M#*cM=eN)Yg8Or=VmMW_OYkE-nc*)^@S6fu%Y35y?2P z)ZDa!vlz5deCp0pEY7Fz>C952?dKD^Xq+>lQ^%rMB(7d$xki&n(Vgs2@0V-fp}yL* zh8HFzpmiU=Bl#sdo_K8UjU?TlZ1cZpD_uaJFb)p=D48JK!=R#>46kWywUk=7P&9JImSi=t{>r7WKR=R})c>s2PLu@p-}! zEEHNwj`im}(%62KMHM$KrOy$q4){6PJ4z=tmRwGjTTfj2;}4AfOCBJ2K|jJPic<*c zlt2hN`jTCsM~^0lnBH737Z8J}yk#r>cbM?qhM*hGD74lmdoY7%n zC7FHa^eg*296~Qc$DDe!!QCu4I3X7hRb#;rm?!`z6eh;lur zVI$6eeWoU(_IHVANo~CAr))E)_UBh-)@3>Al`Dx_F}*8z;RwTp1Q5BSItXx}Zxvp$N8b#J(y)Y1;CG3N|)=pLqhAn`tT0bbw1qd;uF4OU9N; zVTX>+J=B~@!d|(O&?;j5Q=h8TFSNnKsDGarw=x&oEx+i4@so3y5ejdiFA~kGvI`-Y zr_ZKv`Lb5*-ODwu#1#8gTM3r6xJg)os~kH#VDvB+tqJkx5Nnq%Xbi*(_c2~p5qRFHxlVj?P+w8Z7{>J>NYo8=| z;ZI4HD}Gh7Trb(ss~qF{WlyPGcexATP{+mw?P8b&M~-mHHQ6;5jfpN}0<&4JfdbQ5 z#}q3;JDuGN-J_-;RxuKa<{PI)?qfWkuwCJK3#!_K*Y3w|OS^uHP}~hJikNfbIj%vY z?-|wqNO<9EiAHi^mC2Lk+^~dQQ@6UG6_8gyO;n2lvI|B#P$wk3^GE^Z%DGCLv=a<; zzz#F04w%4tS%MavDfchFI{A&|@xe5y&t3>~ zddcT#!jFbW*(v-I*SYMK(Be9mM9|=yM~aO8F9SY591As%RIbbW{J>flpcVcEt+>CW>rc z$28~>rC~Krpyh!Izr4K?ORsL0keFU=4{m<|ZhjF5$jfqVI=U8QS z_>1%bqB5(lCgR46Ug+Gh+@(wp|11|m+t)&-S*86GHN*?f2XH-HdF-nGtW3N3CHf95 z(>6qSYQrlCyONfb;*}d8W>v3_FB7V0HP%~hu(en}teBvNtj>^V2z*8CO1AE}Md;3m+*LO19zsO`WcGBhTEs=!Ow z;NPHJ2pi1>wq`7`Wor*f2=QZl%Re*j&MTm`LfvNGY}+l@ny8J}x);0QYJh@ZXPk}q zhv7yn*n0<5AM8_#U``Qw&9P3P_j{%y3@!Wg>&k+T;#cVlG3dxad#t*KF$o6!2R=JA z)0@1w#2Tew!1#eokgs@+R*4C+J}rWzC!I^^b8g%}thMJUiHfZ~O(2CSE?AW%zyv#z znA(Yp_ts@DMak<#E&3_-oHTeIU#L}5ViQ{i^MWmvar`>newxM3M2|NJRNQ-21g&ij zxHterNL+muPq~TFip+9{=;j~G9bpCA%ig4g^nxwAD2BM=OisXG;3))$#yq5NxkQ)g z57%N&#a_9Da4JwQLHBzwh{L)*NK3_ep4O_PaO(cP%=SF^7Ev#3Pbq*}H0oGVdrxVD zXIT4;`)z_0_rTFKas23$HIZ%pxThZz=O()eQMAt}i1wSKK9gd$q;tq#_uowH-jb*& zb{Q&aqFWvnSyob#&)xe|M^-~FeuqfKYIhwRcd#dDBvZ4nYPxjMsH0c1?Wy)(b$fI; zyz%lvV-!3C>-v4>CoHYKO+t$K5r`8ZU`%1K{68Q3(1Ue}jC_|!#O9kicx4IdP>*TQ z&3d>)G}qY1|8 zLklVBiyJPvG@F5b#Xvz;JTwWB0%%^#qo1>`#=G_srDCK<$c@F$SRTO}x;d6J%x3Sa zm;z3;8l*tyD+_GA1!%X#6eT^Vlk3i{&ec@WGq0G&r=>g}sQV=-*8PMONKWf~yaHoG zm3=B3;QDj8{sNq4yaCDDEOA#V`rt9hx9C6mn3%-W$HW3~#}`t|T_)>Od412J(-_{! z4~a%m4Wrv}3(bd8+4|g^Jf(YnTi)nQ=1iQHc%n~O=B!i3J+;32R)+h?M?|Ne!DEHHXSUB8$t=*?14O8pnZ~;W@U|qpi9opxU)iUe>|Evgid$b=lgBL3 zh=T+tN{cXl0X-G?&GDEDg@9>%@rxBp8!!!eT0)BZHP8pvyH`xIu2@_Jt?HcW71_qT z^|7)sV8%gz+hcc$d&~pH))L?KyXS+<>stN^0Sm872Q~_H$XL!(Yv@k*Ui~^d@cj-E zw3sY%!$^nNr&O-R8+~fmcg*4aMPdqv7q29QHWE%RW>ruLU{8ZN5v523*DT{Zj>=*^ z6Aye!#G`_3B|;e8Y(i?z;}zr4FsKtZ%AI&S5AWOBmK}%ApAovyGsczBoYxAq{M`1A zuB^!LK23lZf3=(?-ZP&Rr8B(S4y*L}+<50;3V09QH?M?w%Z>j>cw+oO|6Vw$A%{Iq zb`=gPF2(gfFFVGp|CF&9BsdqX zd9?xu8Bd0Oq41=k!v#|r*f|H9B5sJ)S~rX-J)F*t*Dn%P_%9|@IKe4-|ChhsegRYG zE59U~MZ5cQ5F`^kmjhh|5P07pW^7y@(LH@;xS(~}Y!i>J8;H0G%wEu=SN$V7ywrgKTOX8WRN+IMG0>AtV%E6^xig^ami z=NIS6sMYpC%n0{5Mrgtai@Us2fcsyK`>QfZJ0jue^hLliOgT&_*>d1$1wM(*PN0SHNjH%hp?T>$6*SGu(Go z+zd#W!uT@uc?l;x1^2tz<~ez+FzGu^^kO*}F3Xgz9i#-%+ld5%#V{Z3g!`+U9%$c<+0Z)puY)y<4G!s4n4Y}866i}UISJcIU%L>DSw-@sJa;&fi(XHLpZ zCx~Pb@ELW36j=RX2j@&y=`8;pVPd5-13JmvQ^AL}x{+CtIQG^<e+|tKc=i^BUh@Fo$JibD^ zsmDP1=sY!o`fS`xjUmlHNuyJbmigp??-(u=h`Ct|!ec52} z{j4^c1n5U3us$W?|o#*uugJY*~Zh#z~PG?r}f-Ro7Ka2yZz> zvxs*{J`bD9%MJ{oHLS9C5szOXcuUn=YO}rDC2?`@;JO|N$K$C~2H2te1UCm595^!K z?7J)gah@SUMZQ1)CdyYUAI{D~n_cBDXKMbEvxFzqJSM9}>lWBPr%5v@ADx@mK?y;2 zti0uaOwDhQn5gC(LHQBK5wY}w`Yyh5r|5z%iY{%)ifDKU3XY7)y;bC|opNFhe~{gd zr2$I*B+A9ic}cO=drS&1xY2lQz=yA~(6{DagebbEal8Y3*^BHrKKt9I_G}xUmEa-= zMBvmkom@d!DWcuGbJU+qXO2Hlq{o~teFYP56<+Xgto7ILwqk&Ps(=mf#vpMxz|^}- zxU3~UA!FVf%d^1`{Y@0&nQnxD$v&0IDINLqqrF%bug5>9FhuQz zZl5U|U=3Sm;r_2!^H{U;1dGjMZ4efo&_tVs;Wb!nFEmYHuSUF$Rs0M`bgG!UF^;@e zuJTtH*F$o1-iKcQID-aEfiEe48)ta;tS4TtbkE0-%Lf+mBV;q!k+0&k8GBR`nz5R$Hi4@`W?j_*m^tf`l`|U z%+d_Igvdop(?lT`gOGoDJc6I3@#~#k81#=;(0Y%0JU=YA@ymxsRbvT*`Ii#a;vN8n zYLP0SU_Rm7inJX${JTIYf$$w>4Q|R*X`WRHGqlZCvpwZnWn5Et^=k? z)c8`AvcGt_j^asV&Yj_8D@?tNz9O~)ri17r%!^S7kr@@+vWBX9m1zZRsLE8n0VZZ4 zImFdMvT}NE$&XuDT(eWIA_lOPRiIB9{DDHzS_$oA$u6*IbpBkXE+^RNGelwp7m!K{ z8i?El1sqs$5!oF-t9SGn&(zo-5;msBrU}iL2-H$G4{benH%qrXq!Fr(aI=V-%T=^wV*5L4RTL&DbYS-B`@5 zb(^Y$D;h5auI|t|9YM%S!UGhEVn9p2A9<`#6*h9UN8*cH9|YgQoW^_(++4U4KrY6G zQBen`%R46bf3`K--tZOX?WHY%_zjA-@a%-CJypTRf^*nleYDZ^JSLQNs}Z&sT)|J! z#U|zZCXY?Odj~s2&r4)HL=AQ3gRcv(yFnjbvb+dvFZ`N`%A_mJqJpY18g!Py+lJ<= zixfVXcwD`x^*byo{7d4B+>Y2vctZr=FiUjPy*045W5@z#P*+wbf<>JN4KxgmM?~Jp zCF&(`^O5<+T*b#wqMt4kPAy_Nh-+#Pw5W$e`y{l#0C#f>a!q{;*5u~BzxydBYeys| zs*=VG6Xt51aA_;wIwc1Kt-y{vhaqck_22h1OPEftN%UgE^eX)d;^c=?wmkU8S{9x( zsYP%i-hp?g15o~rA1AOp&h@{3b3e-wy(0lJM^q1P;kQ6kqf^BPthvA2B8FFT6%is@B&0vQej)EpMONqapWIyTPj-_+rMiTSH!0NgA*M+h zt)QOGO}~+)n>R=((VwG3;c$0+#@!0iz47+hOy3Q!N7M;@hgB3Y_X7#G6fTROSab0Y z>^z>5sKT@(Dyk5;?+3OXlC3yQzz?MLS-fiA5F74_4MYozz;GoC zx?$|fv5d3r8W1RQR=lzUCa4gq`=&I6HJzo^c5Pvh2UN&3e9Zz}MS z>6C^M?yIR6eYqFNPBL0brmxtb<+qL4N1B}EKVtyAF@!i zCwvrK=xu1v;j#SF=F5?B}#J!_O7fXNfVio_ibiXlv;m6Bw4LHTEDu_WW_3?g035eT7( z-QjU@OB8STVgXB@^l3y(VDcmzmk_FjI&qe=Va13!4`b*7!ZmrID^hXL)3grMk( z9U*-G6}iQWdX!$;f~U9e$NlbWLDZv0ZE8U48WUo27}Dh6iPheqKdTh8`RSaN1dQ|3 z?L2-Mtrx8^dSe9|0k{mvT>>R1G>nNZQCD27EJeZAt8#h(i&_rImGl4?o+r~A^zhgP z0@iumTz)I;#=~XXS(fp(R`dlT%Lpl53^_&LRIKqNX|sFqm0A6mRcO+hz|ksXD5wcT zekD=tgBM%iEib=uu=k%T$0v05N^IdJ=#O*hUFUo!6)`bIaDlsLY+*buxthooJf;MK zGwO1nlN2-+Wvy|m9=&D*_ruAA`6Rd z$R9rJVb$EDKr7YLmv8ucJ*zaW%_O2l!H`2+_1-Hw3M!~d;Q4*m@=KYkJ2;Ebg{w=f z#xYPF;Y)S{w=a5cMwfE>&z#FT{cQU|qGoHE+Ldu>L4`c7LZ6xO*Ak|c=VlXmC|q@r zu0)ZmQXqh_V!VIYkk^|wPUp9NXSO9sF}aSAJ{XEIk83HLqp)x2n58@p-Xp<8;}C+S z;=QHcHD@&o3;j7n8-|6A5zc`UM6V}y`m8@v2L;T!O_Ubdwk?Y%wSq zya~&E&8>3lzb^pualmwFUNo3mVL`>C%vpF!u7G@~jVq+`HtwZkG8QmCT%V`#Aq|YC zKaBT3g0}L&yP=k1DOxNHDvvJj_Yi~qjlm+77jvJ(xqfd4nCf2Mj;Ix?Tfe5s=oAR2 zbP1pD(+rIr|L*MxTojryViZ}uFibY+mL6lr_-pd0u@2Dp$#tryKKmVvzd6j zDY5bNrDHRgc*h+*UPvaG`f4n7*t*AhPGeTb-HAvRd08D6ABT+fz$VL#@TZ%)FxjZu zS&@zAI|G7SP`-f~goT7YY)0CbD||z+2pQOC)sCniiGi!wVhCfS7olnPW4VQ~ZE=gD)*$ zy6)y{h)B_-OVIeNSPG!GJNJ%d;!vfhA`Z<66x&k7%OX&+OJVXTxT4~2Htw-mf{SsF zGKUXtZ74|g6{{t4uX&!Gx#7KtRFROtIu71wt+&-WpubH#gp%m(T5Iq9ryb{}u=LYu zxdx`6(tP;ZIp$1Z4kW&6Stpj0s46c0$i}yABkh#K=UBD-DVDQY(wja3b2jM$p0d6d zg+HmrSGAet>e7es(Q?(t)Q6+A@ZxAMd+fQ-8EZb1*n%|@J6v*c@YnYD-ofIrd;1dU zYCMMR#+W8CH=J7U=%&8!R$*kf=|{Loc7_3%IM1Xyr4*KCd6h=tnkSDmV)@&>5?=JE zDe$4FKSAEfQjv5zd_R9uJkX!$Mpc@o0E>rgxeBpnNA1!ESFu>E!vKOsqml;qn%Lo; zMth3HK7Px_hnV~Eu|yT_hl&cCB>rJ_gVQsb)C_ViN(}w6Np6tzIWs-j(cEI z!8Q4K+pq1{pINGU16B4yX~<~@R7~PPTEt}htYH_v7U1vhye45WmQ1HxX>N|-vr8PpKp-s+O@Yb>+7pFl%Pd_FtkgE3ky@Yv3cY4-{-L0 z!silEB(BhAVR;V=yUCv5LzBW+G9sTCrVv@*2bUCy``@sEiDNRyLUFGS9k{VG>r1(Q zI3bF@l&~935`ih>_FV2m8wHe|wIt55oBN;BQk z1G_qs1^)H- zhISSwRhmG+BEuKMgS%A<@WSl)kq?FF+>Y%!EB#Rwe#_d$BI=D28xt%|Vnq8qQ$=Yu z{&rC1?7}&L>o<)|Ujq1> z#22L+U5v%lDx~pE5~r}lw2u`JuKtxd@hc_~)grd@U^9n)Uy6QXTC&@izVmdHosh1R z30K5O7!qJTU9vn`*f9qVKft8pQwb}iL-rPXpn(j!XY{+HlEof6=jh9f6}R4~up&c} zY*oIq+_>@|e%y=UHngj_sLSCZ2sA}-4YS|vHAOU*xIO+l@)0|NFG*Yx-x4l_NT~w% zc5UH-=;WJ-V4;Sf=~>)ik|McVRz1*^ZQUgfLdC61Cvx@S`1nJuw4Jxg&1{5!w>Hk+ zM+ZLZz@+VQiH*{hLD@1#Z9Nx!9+VO?#7_iH{{Js+}f8sQ7+38HsNAQ9@@WUTRVK$SG#)rnre(v~>-U zT5;_gR8HeAsIz22#{p-rV+XBK0fCBx29M9f^~-eWe_Va$z4iveTA%_B7J<{pB}|D$K|LFXY{a1DSM?$~DJ~^L{3<4LyX10^3N(x5WKX>7gV(Tx z*EbSYG=!EqLLR+GDt1qy^$Ftyd#~v89P>`M+)MbLe1pvdZrez-XLlnvsC^u3K69%E?lFf=hq zmONUtAT*98M?5~F4Kp}(^@kA?yiKY2E+TbreC?ZuZjCY}V)GNCu0+Hc*L~YyAOOK6 z&-D`!WMpb3YM_hf?kxWq-lOw)CTj_Y!VP30$rT$8KS@a%M+j~)~ta>-u%3#0h% zhMp#iPls6UQ&y!lTUj0Wqy)wufpl@!g~$UkJUDL>cO}a*v8BTW#@!uK@B2Nleo=Q1 z9s7<|y8CVS1&Qn%wMkk-j73lq&=rq+a6K;;H?|TlF4=RTxx`cG_o2rla((tdsl9!E zTLZ<|UdQ4q_b`VcyMz`NzMk~QAtGIPP#z!dGqfEBywUj1am#(4y=GKjrhu!32~{Xyh_c{`kf|4P zAlv1tX^f6;ku$?GFTBc91L-J)Aj zs%SQ7J#W`RDbGD}-`x`!?r({3EhtuSJLcr&<|?>PjNkPavlh#uL@!#4G`vzR01aXP z%83rS?K#0_L*MDMXIyaA%-8RnFqm+=dv^LlTXF7+9+09eSEZ3jQsKt!L5s8CV zrvc#L`&M{KQ=G4;s5d1FZu3IJZ<%W_V+s)_Tmu0JADjqKp|Mj(rAhJUXWo8;jX5`) zN|4xLmI+P~e_tT#EyF9rz`DWsh_y~u9J`*QU$4zt_dk*=VC#Nn%%6cLUfuRmuh=sE zm_f1GWRR8iz}(I`9owmAVr=*2*=(KCGWrtnI8VZLpc`!psK0;lDck3$SNW{{pz6s^ zBpt`U{qJ~N-D$K=rgpVfJ$}|r*-M2jk)c;lVONdqlrW;3(U=0@no9vw&O+wE-SgO4 zn0O1(glD0iZkGxmo8vWkB(-B|{_>bo&$TO!-x=SJJis)~UlLig!eibWU;PlzgL6u) zRe0x3^Tx8fF<0J7)Z*Qk9bl(oO@gr}{`614nhCsX^)C*A!sXo$Pdv$@ik`R8QW#Za zpz#u&gDHV-toCwy*0cVl1Qo9p%9je0r`^(SRR68|W@bot{)fm%Lt4+etD$aeU$4-+ zCh|qwgnB*QZGF0hSZtJkJ7J?!(-clV-u()5!6M75CuB^DPZ37N2|F~b+d8K3D@`Y0 z;oRY~3c-uP?S11(4X!;Uh?6 zVB$Vow$(xTm@aHL+@igm7H#ZKnq&8~kBzd`xs%q()!J&c@xC%Kzr@c`VkQULthtPp z7hjNA;^9;bK*f%WNHG9`IJ~7s@Gw6SfMS)Y3o`LCzptxZk&S!aID<&WanGw0AwUoiL{LCL0qMAIb~l?P+t?Ba~;ek^F?1#NC$_P{cyFWf6{WVY5h?DU2F!6Lsc=*SL5yk1+dJbPNu z9d-t@o%mbUVz)&C@?&&#j&CENoa_50qk_wd<|5VvE#ma8@PMZ8vzt=t(5bqzbG zhphy6Rn5z)*Q-*m(q$E|s+8Y-$YokJtqT74%U3na97Jfrf-S2p_a7m&E3`r_)9JJO z9nPRNMO4S{4ZpZn{q~_>6KWsr3N6#&yf|v@D6Lh7`+d$Bwf?a^miH&z3Txhc-tw!& z?XGpxnv}R54yVr=a?fz`k5>Tm3cBaS0LMRZ_s)R?chO~3eV&lRu3h-d6uP%Eu+6Dn)_*e zwN|d6KN7G`jW~nlF_v(;m8}~cj3}~j^qroQ1`zayJK8>Bxgeo?YTdPTYW#jJbykO#Rf_!^R)fX(QjuN#4wEkKfuWhfuV(3;Q!MYfQE=(^FJtzM8Lew1^wq;?t;#)p0)!ciF0B+<^G9gaO0 zaFrL?ifTH(E2qz3BC~tj+AA%Sj}v5Ht+$p*kZvCs36IlO<}{plMGg($zk9;&x5pFk z{B-9lmYkmnc(`_*mP5cH=Txf)l&M%C;79QRZB=Zuhy6kR0m=#G+PeE3#}1|kE1jT) zP!Fc-yBTm=Lw1`74iZN-r~g>a>>m?Y1Iez2MlCxkK(E zF#1L1d$YG;~!!O3<`rtE5zhQEH z+i4;j<+>@NTAhKA+vE3fP-Xjaon3Mik7H@YmfM+JXZ=EuLauFrfTvvdU<7(@z!caZ z*S5QlWLb&D*I(KGPRkw%JX#y673jc0XUOe{*gRIdAIz9t2sLQlaBRh7mm9xzU7xFH zx$<9p&sjph(s%UG3bZC5g+80x>T-G_$s+#m$+rhGW~cq8GCMRiVhcJQR?xIbLsq)` zil=)Mk-Ijp9BFw&Lgs2+wfdoHZkG{EInLbR`j;%+cqL)o;4HIQY`+tE-LyQd5%6=m zHDoI`XiWuP=TChBhqjwVWR^&Ft>fP9XtI5&zt>Gxsd4CXs;I7<%ZVG_w zek^#10Zurl05;XxX$4>3X0F0=R0Uk=_nAAHy|^F&(OzVNBLO0jI&8HS&oyU0{!@Pv zo#^AIgQt86pnF^;OPD?%_!ohqK5wH^2`1DYw1&dv9>dY77j-Bu18szUQaGl0uV`!y zu`urkxd4WFjqDzG!0HMb&xL9oRMwswcihR&?!A8#xp;OP+Z;Bpwa6K^<#I_<;BvAH zW5hjCw9(&>RDlQ+a?TSRh6ypaJPxNECV*4$mhBpHfRT4V;vsnr1$kg*IGS>N(8LtP z$ou2ThhArrIr9RMhoaszfIitckJ?0j%hSzNm`mIH_ zu)QQf@TnZf+!&1~TmE>49r2wKQXFxY$6o|iDhW)*MIZe93qP_0R9cb9!~>KOw0Q#% zZkb^!yTdhgF#GHz=K0sIM6eimc7ziibb`0609)O~R-e=9NZu)pjH)%A1&SNw8X`~( zLdF1y4?9ujeG!ihk|PGKqL5!4@pxjK&aQQ}n~^`NvO<0*dx_ItYAyCdlEa4XuGQlY zg@j#*KVHfj-0TdU^pTg5e^#!6Hr+_7FxG zR`CAaWx|C_n*5V*Wqr=vy>I0b={YCQC#!ofO7}}lK`Ev@b@{S}{nk*a+u;2x zs1qm0HtmL&y8SqYHf?y9BO5JqG@^8;_n2*1I~5bq6MWGg0dVvfS{ zcMeop%hXc4T7)Rn5+1)Gm|3USj}AO268K*(b>`H5_Ui--O9LH~*q8=tNuVL0Ef6a4 zhphoyxP(JDje1sfNP8&4!G+KLMvF&T*5fM)EK=%5V4FEE7rIEjz!HXg z*X2ZTLR@26L7|u9PAZ>ud;}d9&ov_+vUI6SxL0deX>E=4218ayyuCH&^U0gqZenEb zm+*}2qKKz75wI!Q%iY^QVz}k?60t+zPKzhl6nFEO4j(agWY$;M(N40X$nOt>&9$MI zmLQ3*!29fW#m6b%zI!1%jqgf$JdLeUb=U(D@NOV^ZIT93u({;jm=@^7uzd}Pbi{7w z^ajG^)`$<*h@N*3RINPxNGjG=F)>Rk-p!LijT#cRNCr8aU=O@Nbtvq%gU{|TsA_Zh zIQ{tZjqD_T8%paRu)HpT$7y4EhIVZ;JAV>6#&x{_Ctv#B|oro-NBzQ&7w6{Sk(fA2HcQPxzC;| z-7e?W*~}GLFM-h&X`%z8VK!I>Wk;3R>xa^}e`cZ5*d|0KhDyzdOo&80P6M7PW4B%V z91mWrG$lxk4x4)XE|X7IC{!Y^{GY9lFbda5I6ZPg}&0gTn8AsuZ%&5s(`4&YBh*pX`mwc_$&P;K>iY-KhuoYrnd zpaY_n?U=ikzGm4*#;&2wRd#u8p;FK#A%AfgOGw3$5R@LcHc@tho1M)Ed#+$n!D$IC zY7RwiIMl6sVz5Z;{6gI3mi zhvn*SYe8gVt}YGjn5bA|O4T3|=AiSQp6O?vUdNUMDm*+dQjYGVK~|K#<$7+_*N`_b*A8nUW zZ6fVvTE$B9Ew4!6>$Txpo=tcyx}^|Ko(Dp@pv~iPdcdl9pym}ZHV~NdW#dhixXe#% zPv0P9-Up#MI5OZ*>IFY5))=`+nr~xS%YAP%BT%UWVWSbqfQoqp`lZ0XBjS_AUFRIN zon_&kmbfAdhhpItWhA>j0(B*-#3!(d=A{4Jw#^FG9x}2ck&f*l?H$el1nl6OI;?oK zyb!y?Kf&EL_W78vaHMcS55}Ccas|O0g!MRGHhX!BxY^&g+Srd|N0b^?bRx>J)X>Na z*<%1+~t5>%YV7$}rY&BF

55n~8~6{s*K8}45_h+ov;ieNv*L zy3P%_15Pxq#=|WVEdLbf@?doO+jsoH{F+C)68-4cG;`!ZIajJ_E)t4{g1k1S>Kri* z_1xN14>OOncMib{2VPo%i@=-m=2^%8{>3o zn{Q(&vO9B$U`&y9)pe%BiaMK6!2nDF5JOfVz)^aa9?fNoJ#cbCGsdJ^l!P~B4E=5^S!W8)(N|{4K7gPh}ri`s+ z=IrkM^=pTjLA$Y#fYG331%lAGhS_I{Yog9CkH)dpat$qgEw1i)H8T{|deI7^HCV-l z3h<6EmUTsy{vHmLWgC81RbM|*1D;n;LoBjAwERU7KKZokWu3I82nB}vFYEC zqgbZnO;+Ufq6OGa{6I+-mP7jEpYa3MVk7)mLFxcX>d08`F? zNK{nLEjbrbLVjcSpJC?qF1?EA6^S8+oPY_YisDR_ZOxeCymA2H3dM=IwxAcfITFX! z$}vMZQ%=O=Z|&U5DuG`~V5|f-j{zs3HnEq`@#)o5nBBPbY9bZwMn()V;hrVNZ2i*y zH(12nb|7J5#Egf5K0jW=C@BTjU3Ra1-a~yz6W%=d)K8Y95;9NA(dt7=)@nSh3KtDc ztUK4;Hiw05Zw?~*M97BzV$khyqAx1KK`(S9mp~kwLV9+!tLh)DlhJz(ErFekT``~% zRg8ciP2K-tp-(nlH6EWQs zbzjA2sxqEC2P-_UiybOlYiC83SNcVh)Uhz2%j#LPnAp`ALQ9B54K88gNGMe2fnJY6 z4jO&M8V0$JK|(@8po*Y{ldnK>*SYKZk&~FxyP-O)Z(aHFN;(=Z z?cAPiIVAxfsi-|%QL7uD8H_}ETiVrw!U(Y<7=2sz446H0boDu>LGOJiT;}n1z3w{O znEqWx_?%;)rQPrAlAB%7H72>CME|`@x2w8{Ez6-?L^zOV4FnWxdC;4&lCkx{>(reB z9w5znESMx)7cHOj17mBiVFtDe$mH2t=ixT(nU8u<0-}%F!3Xmcl#IgcPIAJb!RFTQ!JH^v(#vt5rmGZ(M|?*>R1$u#MO&T*Lc7; z@xU{$VmbWFZ=lsgD{C=$XW@v?ps5r?W9a@{dTQ`^mYCQe z;Y4Br!g*0q1??O=Qk-T})`xq3dyY{$exyQaBf(|t7)uiF*LfSBW66R)B%DYVfTUyp zo(Kt&XhGn)u$^zZ2QmNckx@jd2(Asb5c(huDQ|QBK2`D5=d9hY$7n*vcE2u&?1k(A zAnN&(7?;H#a^9FPGvNH8V@w9Wl*^zD7T_`>#>46kWywUk=Az3fJImRX|LobrEb5ta zBYl*JdPL0_l#kaPf?&aADmmt#GojHnEm%~MIffPx%{}-z*gHxmHJV&bmRtLFYiAk# z+aGIpMa8UOFEJ*>~{QB zM*d$C9m#JW{n2ak>;@_2|K1BLT=HL~8|tSLoX&`V4dVJU)6+ z1(xm* ziLTFCh{9P48}aM5+@p-zv6KFZ+IZJb*=9~{UaQbSmXkgs(Z%$x;Ds#&7ZO0^Lh~+= za|lN4;YcuPgIIm-&s*5mE}yJyZOhooY@&s8{UA3@s~o@2htm&IXGs? zYjkQ`;|gMav({@TS^C{XE9Gd}THWG6Zg-AsE+rw`2e~Xu^reInS-7HNBa|tRUCpmA zXO?ibm536S5Nm~yrNx3NSXyg5usMz2)>zk&$y)Q930cURn2NwjUBgZ(lyW6i`Eh3+ z^IZNd@z8V0#K0Z=0Damb0j|>e-@Tw9zq{HUu=pp}Ddzk~Y(%Qa4v44{oB0yJD%k|HlCCyc zL5GVFWMhHec7dlQej#OFxvIC9sK z2AnL-O57(Af9FYJ8G5m6&R(XY(_92D>W1>ZmrGbgk@}A(IFB$v-6j`7LG2JN7>W*q zXbhF~=-h?A-_&G2`{WY(2K3o;p1|x<`XJGCfJ;ey0UHua zMwd!qhca(;)?kv5=Ao5@B#7}(eX3Hw&;}2q{(WNH%2;eS{i5H7-93aEp)GP9G(zdJ z3n7rF&!%wsGFR+}kA5?SDfV$*BA>OmNm!X$FZM)2hPfK^HCn9CS390Y8Y8Gm{etQOr3FCeb>{^Fn_9# zpWuZ*C0VZcRmpO_chK%-T>nxc3$8ny#c-&jV}n*POoAgvIOUq`8fj;4S;7QnPJpOK zfoZ8@ij|<9&gy~gQ9}@`7zsu5?LYMVP{#9iQx%@KqpCf4?LO?bwCcAA#ogedh&eZ& z1+({6W>mi`;f1dy8p(xKCQp`g!xVPy$ia890&+=^s1^le2aI;0PDps?kpjwPPMuO^HDL`oe{CJ{N%qEw>-4s@w`GG$T6&c)5^J6(? zRnb1?Zw(F;`FhbFR)U&F5Igw60kZp{=MqPJlNP{Qf!^;hC;tbDk4}C|d3-QU>b1IH zPA~Z!O-KKxN30ZniR)bMjS%VLI+sMy;G2g_V*OtbTpF{N=A0V0wblZrz_QC|IZ<%c zh4NCqMrh~&-diC~lq7T!n-N5bl!_6<^AjLGU!eXoFqU4L2b0e_%Xbo1G zZ9jun6O~zYH4!&n@K-{vOX<>q$izAaD2GtFV@;KYc8Q;YfmdkVTub@WeG6BjwGgb z#GL(InM*O~R>DO;rKz0;&*KZVN=j^ERv<5srHtcmntpgLI}^twsJQp41X|l{aB%>L zkhuCRo^k`D6`AD@QTwMxyvYi-_uNL*>jhhMQDWkXGdX@=vD*a>jd4idbP3O6hbJ(n zBKvk)LO2zum!SJS5WrzwFQld7JWq4gVXI}`$!yPmB($(SWdLf@sAEd)k_LrmSo_S3 zJBV6w4;)Ps$B#al6S?8oO)e(R>30&MXrECKty3aigJL$N)AP%s>zLT>mZ&IpnJQ|c zTOJizR#K8n&Si~eHDupBA{DFMjd9$;nxK(P&BChb(sAz(<<||?yNlq&_ULeUB-%eq8vxSL4C=9|WNWeMs~w_(uDe79ZA z9$0%nDQ4fXBR^TrO2FGHYI7=TjmVxEkI$s2x*s_)>r#4f5SN;73JQ7-?K0L`gntxr zhE>J#P3t`y*mU@^`SiUw9bPYi6lBFslMpF@UfXHaG1k>M^B$s9jPwY(G1(c@Be?zr*EojR{9YAPz=>9a z6zKf4!q(qeNpG9P6fHndC)b@@ovW#$&(9s#o3VECed>M*igiCB1(G}bJz@=G!vzVd zPfLML$D1F;^*rWgiMvvP*KcKb|5~WvHKyBf zUB*MHY<+G{p3*)4;Db-`YgPt6KzO20Smvx##yzp;#NW(Gsq>(UE3>*u;BqtnZKI`s zuyFV*iHqSd+^z-bJh*>cY0xE$h(d9{2KvBy_liN*6^pB&O=@=jMz%49iwRBSDWLx6 zg#5j{rQB^CC^nb)YkoLXgLz#CC9v?ibl_Nl4jIdNY7Sk=X6?fc{H#ZaW-(dhgpm%h zPpMpsx9#iE{g}hsZV6!uhZnCTgfK?lin)6zrmY>_&6K4L(@Fp*d z$BVyO&J-_gi{oB~SNTyD&+EiH2UEbaA8vCm^Om2Mcw+oO|6Vw$L7O#Bb`=gPE=7km z`<`GXYRF?mq3%-PMLV&!x+K|n+qMtBU@rIx2`XIhu)9UdBqtJs+n9#1jM21(F2kqIH|~lWvSB zha|X&vTY8S%D~P!&=he)tmeAm-{U(y$Bx&nD~M#_zZg*A1gGTvpPt#|38v25JwfOq zpnwJ;ak+GY*j?cQ2xlcl(o@`qLqxyQm7AyqbH;5SD?SFMBjlLeAY| zPW{YI_57!acF|xPTO%Z{n!rY_6tXxUYnEJpH<`JaWA1{c0S z6k=(xjeb%<4h4FO*nf9@t`f7bIWH2nu&@GK)?m1CQe=jE+y}Fpe8Gh9zY-FKuqnO< z0keo!d}4tkntgw}YV*EjG1H87M5BnA@TnD?#iCi76ckbG`4wg`hpyFn0!D|fO+0Xd z@hhbQCHU_yE|1Ap)dN<3KioTV0TG7N0@zVggf=F=W}vX7%&j~x@L6z{jTsl$2BKW# zBj7GOFofo?%HDlI-^Ew%6kSlL=z@(m?Ej2;nyVx%dYWw%Sm-kigs{5>JS~1C z6lk^W>DD(^Z)4CyHW9@L-AX`J8lxNQRRX)9g9k!n&v=CVVUE~jg3kUD- zlqTSYvU4nRwCMZ6{1o5w3ek+ftp#v#(CLKUJJG}_u3X{VP!zb5Qt5p4?tP3IT{kPt zfRO^&K5=!68LYVv_wDMT$Cw|qUqXxMMnKafd)SHfR=8UYF4W~zD{==D?MhOyVC40? znB0}Us>of2Ku;F7N9@Sm@X`nGvJzjTEd+}tzRqSu!$VMTWDM@DB7gbHi5YPH-78oc zV6$98%$!dxHG7Xq;RQDu4WH=v2n&73Z6%^b*EEiIfG>NA9mhR(t>Z!JSqUz3Km<-r z)5#Ttl_J``&&{35bmpV45$REugBB8vaPbRKe;MbF`9ZfX&~9f(@ePTOCLkU0MQ<=f?JlR+kPR@0 z{mH7H8X*-dP&10Pq7M{>Vn}y*uSZoh8O<=D^yp2`-3|nNnn7fG``PnTte9I#4 z3jd~cM8vJEyNksi!uNcS0RciVj#T?jtJCDq{^DwhC^WG$K zh1pfe#USJ#9*^KB>DNj_hcf7_w@je*9`$&Bm~P{kiTi4@gu%NK8WRSp5QjRyaf5$D|KZxn8#*xL_9($IITKHAH8cvWB=0m?lxIdsM@X(>_pYtL)dttg!0vE7|?(xaun!$ z#@*|yG!??coZh4yQ@?*d-0pQ2MZO@{z$h{U>8IoBfd0bdo3Wp;pSz7&>tXK^?V|Bg z;OY*Y-4=kXBs@ToCWO;G;&h)uVRPOkI=#7dBs>VpbUJh>?8m}%=_+a4i zxLSn{=9;$MN4V&kW{RzZH$?CavqU%Dn*(n=aAPHAQ1?h+QRhJe4MXE$kvDRPdI{Wo zWV|s~@iCO>XUlze9c8KJ(*F?6q8<+IlhFPG+|4b{HS{f*lY4VXo8e5>YJ5nTs7hKg zOqi>&!=y#V}v;y1mY%y8edvY7Um;MEbjS16+`W3{<52bwi;2nclcryGWB2~mY z@b0u9%D?gB1g6KiXSaHrS&ryu35YqOrf>_t38EUEsy<-O+)_SV-s@w+6Zs*)gN6;91v9}_$n36!|L1F;{gCCczml-HEse`e(9EcO zeD%&%OrzZX2~mj}rBzUWc>QADorvziOSdrbN(DqY8#+wxCdlAzhjb>1f#_ zt-m9oM1PJBg~Q$P8Fw?tFN+qv!t|ZvGonuDJFKFJxgSWdrEppNutu|->^xTaFQE$4 zj;N?Y;JzQ&dPuh7FabZ1=4ZW5#>ZP(kzkd?#v(x)V#8gr{)npt3|F$CjjL9z9`JD=xj;~pO>xkqg{b1CgSEtQkLOSnrqEh%` zMK-v15gZvY2zFhr6-Poj0~95~6G*-KpSp$_l1>K+84XE>LM%=x7$34wv?qKNT<9Is zp4RHU7cfKez62JAM9&%(`C;-zup)7XgJMXOcw_6~_XaQ_^L(KQS!)#<%G;5D$e8d_ z;E^%YEo}Ix)*-?c!$(5MfV=Q;8+b;_N=dT2#;E5X)#L6iyxP5{{s`Tq)cETU;|cogI=% zUlW;lNbr4L@#JU}7aC1q5qRGbcgrX~Ugz)`W{s`6^Xm8g=#zyYVWVD&%Vs&r~7lk>7z)bgWTNsn6Kc{060509PSZ=To9<+s9S zy?NZivWyRWM_(YajF8gBkW=(cMH^3&HoFVtt-vLtky69kKC?p9G)*kVvJcoUZK znp@@8f3F|rm|jr;5CwJ$I}xK1j3NC%_o3*r5b zpsn2SZm6kPiWZ9)l^f5zbDF_kc1ndsDlg?ehZ{FHwK3KGtb`S+Tfe3$)+rE5=@P#7 znbXd(<3IQ`(JTth7%_^hUI-=|bW4x1WBfILi6x_~?T3|_VK^$$(J-{*=#bz`1^bA1 z${H4v-t`MnE@D!UGq^ZgzPxU_g$C2mW!cC>jNlHx5~v8GW7_Z%u*(*8zn_W6Zi$Vj zF9Vyw#5?Zj@j^1e&{t!s!@k~GonO%C`i)2yd08D6ABT+gz$VL#$&b`bW3o~GcSSbZ z>c}E{YS$-F6HVTl(*4%?&)#S3STE6qGSbmyYr8IkoNz_tlvG-SvdT4CC28UrA{r}6 z9XKpC8_|QO9A3%Zu<5EkFa9=`l{qfR1u!`3%om7_1EsXalHENS(wbd-%Vg$ zzUhC`0{VMJFcrh)LP6rrM3qP1<8vDSmPqKNwJs?yf)_fJ0WtA9Gsk8!r}zOqhHYue zbY0P3M5JiaC1`wBECtZ1hxTMKanSx&#Gws=Vq1!MSp-UUDNG)%ocmF0HiNiPf{SsF za+?=!Z75Fm6{}k|-EU)OZq#`qRU{;^j)ON^>ut3*=x-Abp(J{{*4%sFxp3S!Ed6v& zu7T;NbT7VkjyY491BtI%Hjd^bYKeZQS_eZ6=E zi^rx{B+}J*4BL${O=4^~HQ&*-x13zS$ZlAPaFOiH7+~T&lj@XGSlY#-Gz!n}y|XXN zHoPL?MUR>SAB^}CkXOd%YF%yfSRTGFsF}`CmRQyC};8apeVT@N;WF7wfjGf^H~}s z*Fui1KC6w4eTsD z{mJP!KXwHp`fq2@S}g@SO(aXjGPFL8Il^c1$K3K3`@1QHM%UTJ& zxT5w@MJ=PGSWm_Z{P-z7SM-=nzLe{7Vo$OB!;@7pir=|#^`p!rji^n`5WOiVbNEPB zjOt{$+^VUCH+!CzzJcz{7R_J? z5o80#LEy=E1G5#qJawAWm)E7t6BD~n+7$h@ln%;W<3Ai~c|*e9TTz=|QOgX0J;lL1 zyedRz2q$@R^rVcsq?00Ud<%7rUnYA>NG&EjAMRPm8iDNf>H9?^kn;cM^qGpn=qY8_ zFotRk6ozIXAXrEc1N!CX~XB;pV?gk#;JF2N~PjwSeZ^jPU`ALi3VgcmLCPcVsQAKdf2tgbSRdcwyP<&$x@YveqmspLbF%wKj1@0PY-B~I zB-yNdXS#8_zPafahC8Y`(JXLLm%~L6Xo}z(W}nkzh-gf4w^qtcXGgGII^l}=mT>X@ z`xJq9)P5z6N!rH}Sg0XrdKPz>q=?#ov;Aqdb+=|HTbDuP>cjEzhgxYnZ=c(_1^(S3 zcJ}TX^1$^>A77D4*eGq8gbiKoFaVK4Z{ZNffHxQ;sCJicmN4381d3YtpxM11`S~f`=4R1s*DE{ zTPZw%v|AK*n%i-a+wXWT_W10RUD=sv*qTuBOtd0G#rMO>NOZ%G5;`mK(zu!*KE-U^ zW{Hcot~pXGu6={bY1{>MrYtBwW8KXT+SoQktT<@!_&i*{Oqc#A@}5kA?;VmofW<~< zC9r54KuI82agkVJgOQjKqpzR$2lLwBZA&zx*PbS<1uEb`2{?UR!jxDP)N|~Rm9<`2 zqaS&Z;_bPI+YuzTTs6jiCU_#z<~G!0m9OWS{gKyZe8~j1XBOe%IcT9{U8Dn2>Cnkj zvM00pHvWwzygrw(q9L@*7If=9Qqg+~%}*F7c*T_$a+!BJyFC$%-f3epK2C4avGKYu zJDg>Jxg8Y1Mt}qywc89MAc_x?lU)cuu`L?y3TLpa{>KtfWc5WqTpWGM2Ju5Wx_+B_ z3@zM|$P;K{k}P?&)?=6SWyz5iod^w+BTe*&5fi*ksrW7;bsziMYWHS*%9O|siKr_P zamIDuHsJR|FbR18F6zqJQ6y+#n-}LY!L)ZKN`>#D>kb{);S3qdPYSsPS8LPP!?Ktf zsnvyG$zJ*mtkd+h>`SO=RDsP12fUEP|4Nu6W#o>v_?*vAuY4$(jqzC2p6` ziyn{2^;!L8)^0^zV<@rhU4No;8gm#bccq0zBq04Uc}t6kium@L@5K)#zMd=tu|{HJ z2BIyNWuk%;Xx$!goD8dnhxSLvx69;X%2*xX;jb==Vh8JI77;IWlPE-An-= zm#9JkLzD$igiODf1DT(zrZN6=a|lWRNQbn^t>v?{GQl4wSJAWQK>z^jVHKAC+?VgjDo2%fCo!{?$W-ZF{ ziC(l8>3F4@9~#2`krM^E`8{*k^;Cnpomt$I&dtj5*<&r)-3Ymh)(M;74ChcWyLoDv zCGhj6CCo^ilQ?K~(f|kFx57)B;(SF#y(v-fN9p$!GuPmq0wPSf1_BU1I1!*iV~_4i zlj5SzIr~_L+5O!K5N;H$t^?g@(?I?GgHPEl zN4?5tO-Sd04Kg67xcO^DDHLc;~AU*hK| zF_Q~N^pCLeqOCvCEFMn9095R_h!p)0h{IcI1P}8QekfKMx*!uTQ+dUqpV_!)wX0|u z9QSPKfN`<-2Qm~W2$x6@(lfe^$Rzw?EP)Z)BF%CgS#o1EGR|Bjm~oM}0m=Xj;~kkSF2} zWx4UU?1dh$+phmf?#Ri$`;=?fVepmO6p>@?dYZ-38GvIQE z>?xwYP#Ah+p!)5F_Yi7#t*e&da$g*^a*WpU!vcP{L2aXI6;Agi+%nI;^_(s3Uc&95 zwb$wtdt5HJ-yZf%ck_=|0Q2%YW4KLR}cb@6)JDGAs`>!dX(EcDxx0qxu| zuag0~b=@_a2N3Ayx6d!NZJ9|_cGJ3OjgXg)2yA)8699O(!*!9Z$}?d5w0K_|MX>e} z{bt#A%p%x9+5oMg0a@U1mbm>c&Vu?eo>uD}iTL~Izmrwyb%cwJH7jm)%P z7T9LoM{DG3?X)^2rFLhr!|!){O(a)}pm6J9T^ht{Mh z6o>}xlcVm?G=n9aZe{C6g`$cqXl3Tqy@sG)y0`fX+aU?vN$a4cBbRkxv51M@iZOc2 zDjT=w^DeIpBgh?_E`Q#3#{)#|HCk`2na?rR9t?y%V&B}p0=LWM@fVqptHh8mZd)t*b_I8y3s>d*oOj!U&LzkmS~o3&AU%FC5?;5X)NMNNiX580f6dJ+ejZD} z52U*{+O|sIVcNA?4grVVlkHwmrec9$0L2HiRiVQf3556uC?}L_>kHe~?Pq#$*~9b& zs0Y*a-3+?zVW-0j2Z^Iv)1Q0i{JqS$L?0noG%k(sTu8`-^ePPRq%u6yH}wQNpUoa6 zTs)sm9WGY_sY{mVz2~MrF@l)BY~|rCw)Z7?f32_9%I7F?+e-sZM}e?jaBNCFVNU@V z{eo$H359y5X)_09`Bc~9D4mQ_(emmXTDo+3zO@v z^9dE@x-O#H-NCTO8}M^bW&3j7ZF3Zl<1N=S2bf&{S3(N8b_9doX}Sj^&~t;9z$UrA z)>C7FomhNb&&-*&nG1;4QQAl?Uk47k!yZ@E;k7#hV8)z6s6q2aj1`kz?$(C=UhhN8 zO*`jbYs+0o%XQT9wK^b$euu|iOc#6zm%5mlfclGgGukA@#BiyA9TcgJatG$+|)dqe} zwTB&rCatN!aqZA6XLlbausbCj3T%3TI~1`Oc>JjB(MRG;R8TpQ?`{4ik103bVj@x~ zwdObmVQOJBQRO_I!cJIqw+px3yoTD2K?SoL&UZe^{m@s_>5r<)d0hh<&l*0sY3f_|=$~9x;HC{=0NM0>L9+(-9rW_wM zG5H30AG98H2b0WQ5)MVZZn(&0_XT`#GNV2YY4yE>2JJk^G_!vd(I;XJpT`*r*b5vH zXK{kyQ#p>g292Lr+44O*;&oROq&VV5-ar9ZsU$EJ7yZne=SQ&vv|VE20csd>_<|5_ zSz#)>!!>o8BMWCR&%gLdA{9OVtKo!)+~6%Mz;;ie-S2k0l6Ok&W2XJU0>z3?(HbI9 z3_->Kh>y5Y=KWEx1Ck>qt)h@$81;G$PPhE(OdUr4^AZ}#Z|y90J4@_^0Z4K<(A~9r z1L3f+3-QNGS%aIMu5Y(L#K^zlX@&g8#v0+Erv%&^Sb|HQBFrXR@b;I(uep}a+OG#z zt+4$ep?hmRwI&8MFr`TF$DB{F#=iZJ?t$wGyV9-0942A-j^5bCE`}q0LX@4+ZYM(?CSv*1m8{`hcKfsJ~ zU?t+uRXm2`kF)nM73F`9Xhap2&QRm>O)93MqK?|%n!qS+u#PYVrI_;6<;xxk*uy0r zllQNnj-Oy`n)B~!ok!EA4QqSsW!nykIz$_!wFRdi+%a&RCxOQgVjKx!_A4Z^s(y#h z#AapBxE>7sb>z?MXqEX@mfmG^KTovx)2`84g1?Ue8))u;9eCq(gBnScds6VZ61j2Q zTeX=Ix#|VNM;+K0{+COgHMP6`?sK}l>72||eI^VtKXkY90_ z#L$}b9cMl*Vd`$gOGG-VyGABdyR*=g_E3a_3t#m1&+legk8=`Oq}0v84r^R4bpAP2 zYB1dSFB8EDam{4~glMY;9xKm$E#14TwC7xhQ+5wB!x@Z%cLT|5i!_je%_VQ}$o#$xd%{K{ z9kE-ueZj~yd(;nW#Ll}Hs#acpBo%9`n3!gBYn)}rrR-~jEs{YlH`oIoP#unVoZz#& zOsd*iKF$y5y2VN2_fkoX!?tH6@EC2h)K>#5ID zhv`Es`80EB-deWBgI`y+xH%*_o#42FFBJsS6A3_xCM7?xm#y4~j8#8KcwMt#P0=2n z=69w__nwsADIY!}aZ>SWyr+Ok5c$O5_Ixt`- zv)qlg5TuA+1f(mfdp3Y=2a!(p?!g?un3ho zm)D)NPtf;?_t#C5$akJ8vibf)MBB9qU;vGpG^C?VsO@3~wiWo35O(C+L9G<|F{rkC zy^c}`7*2cp0?+}m%C_OI{qM&Q*E4oi-Kw(7=Lna8E(r$;BUnNzjE14~z_p386Wr{4 z@Z*I4vZ&x?2`y?41s*upn3GM2YOPMzv&#*&% zPU4C~EpRC%ml|-d@gvVOKFoWQ2o|XTN`_;pBqjE)i5}R^T;TTG2^L-87P<}v8Gy8; z9_1#4x8`%%lJIhttNTIXW3H|~+A&eF#+0f_B&@C-G zb0gDNve^7<35l_JeF+(KK+-Q+O!u7eZJ088U>A{z%BX?M5pkf1p!X8jn4sjcp{vLp z0()c#;-t=%?;TIOS-)(#2W(lp3HUl~n3m@dUW;xigp=omkS^r#dfi^IDqg60Ma>Na zmVEJFd9)dm`A_6hLgxJtnu8+){-j>;vtx~si=_27o|u&VIWqz?_Rwdb5oib%^ClHFLWbI_knYvG94#IlfMGoh*6mh@&aZ_P0k{wZM zX!suCVyU6F53<4F`V={nW%gf}P5dvjFgqk3T9}L&o`nmjxRgp5Gjr^T)$BAz-zOr) zX*58gY-$}&@Du*X@d;y?)oSnoVWQPaml0zaf)%JRCL8YmZ|usa8S3_npqf2?%TrhX zt5&O-x}NkQ(TeIiH|Pnv(YTrqw@9%3TcFE>(Y@oZ>%{z;s{3gr^lR$5@}Qh6)if6g zMMFVe8&h>ILqi>FeefIRk#3N%!hx4o;3Dvrym|J|`|XV56FyS*u@ORwMk&y)&;jC& zn$wYqCGLfaO?I;s*<~LSDyGP?b)D(5qs}H&u)qNyBn}~ajUGE^PU1yQI{m*hWx7@_ zBC-n3C}i~jQtob|wo`>Id?RarvUV{`*$zKIA0Se;W^5y!pVBIkML)gI1oIcUitr<# zTNKruH>gxLz+;6NI<_9?lePY9;2!H>aoW;P=zBz*Ca4w-Eh!^E`eMBs#*hYF^z|J}l7;~cb?W#&HYvJL&5Ku2q%wDu|}__OFsh!4@TMUi4CWE2Oy zCcjfrZ2C9kD3)p0t9ie7p#`fi`0kkPSGi!e)>f+)h9a#c>A89Bl)~7wxm~)m&~Mon zM7jw5grI1aZ{+aSJNnSe!-9<8{l8TCjZm_)M*ddunqoCY-hj}Y93?Gv{G-{t z99I&(MhKdTb*6>6tkA)k1JeAj{&BZLA-qF z^DCHgUL#RaIXC89NGUhDv+L&>%iQ36_fD`{$nD%oOJj5?3fr#C3#x z(9MxJu2zm2%9(N^Dv!JOMOF!%cajLkN?-#6I03bZy@a2(H|@ymM*FV`5$#4p12N&A zCB|HK===s2F@GU3F}Z?=fj&Q8!zd{Q)?GI2xyPq=Crx*=twSxI5vg!>=hrskjpw5n|w=4 zU?*cX22`So5e%TI`#&u7*uM08nMCZC3!y~R#bKQ&Wb0jV?6DNy?A15#WrsNU9g!|V z7XvX7(=Ab(R(is4&$o&iw5)uMc1TCC=1(LhYC5PMZV@h|*8KP25K}pDvxuxC`aO}zV zR;{L^@ygC^S++($5a6P++WfLwwfM|nG{)PKY%dBU#EM|_9a+<1_RN*-cTa)d`*5Vx z>&w3OTF2<#ZHN2aqoJie;BT9om7i@$ZaC3@Z`;04Td`$1l#7T2^X$Q(Vl8J@c(f;D z>syj!u~WbUq*;#zlVt0jyVqngwifeV5zxj1!oCpqxjUFrC8iH^NvX@HEKUg^ozzh6FPM)ibdk; zMV4y>PQA2{9qO__(;BEquQsjWg$W60-N)}peu-^e^BPVh>Hc)Re~WFkT;h6dgx1=6 z35T~Rz-6aSOzJCRk0Fd9QL4{4?{oT`=dG-R&d~>nR@OrB&LUC2NmD6?#?<|{zRp{J zv&6(12`3U05YCH=DrjfpNO77iS>JE))6cPL!;)VVN^1))W5-yMaF6B|4`a!KUcVBY zNEU#kWB;BA36f|*;JL7VFa5QW`ETDyY!O_WY$5bPno{1@{9U!C;!UjGZ_aN-HMaY; zMPwgj2LMsepBP*gf5>@bz0CZ3Z@QbwV7LF#GAM)jxQvMLu)0H8GLf#i=yJ-=a&`@> z8LG#kp0DI;BI*$}V^BUmPZ)xQB1_4!{+vBMmN#Ni#mwL7b3}6weh&7I(n*aamy_jI z?++FhG5TBmLGXfpgjW=&5Y#Dw5Onksv)mShDCJMj_ zg~{;*q+0@i(LZv zZ4-TlwR~oHSEsYIUzxvYL6POFTxn>n@jWW4(v%Sq3@K-7xq zUBL@S7%n7$$c5%zAmjUWx(Hv}?vNNrE+|-qrU$5!9n{3s~(n>j6mR7AWnA?FPTT991N2mS4 zvP63%l*qyr6q=zddHl%x8wM~-=qpD=2}_8zLdeo$!4xd5Js#Ma#=~!(InHFQ+GT_+ zWKB#(;H0i;rxZ%Lk~-Yr@DAp=Y>;^9xnyAA4t{_>ZIJ+1Y5nhzpWm*7+8t1Ja+{;f z`5#-JNEO)u5mjO{UjkSqJHQ$a-X68-0@HkdNl2mjjIXy(@jz;G3g3B!37CR4WN1a> zr$(@(!}80CQcOCe6;A^P3rA8-GO8p+l+?|`wHx2YtnRf}5SXyKg~4`tu=jEEt(;VA z0eJQ6v{TI8|4G7&P6)kaNWW5{#7!f*a?m>2H70Z&Rf8pw=T)Fj5J_a?4hz}*1wxG9 zl`MYE$m~*9E9rP8;bW~N+aW9Ha1nxREYRC7@U+A)r0gqKb#or-zJ@uwpUQPan*r(u zyVqxD{ZPq%Lt5W(dp22mi;cbkCreuX!S9u7G2P#o=^naH)3CvpfqjyL8E@z z)tz|)D=O0miKYWwO5zLHuvjv-R0=!Ped6{_OcHumQ6)i)f9g|}`h_-l81?TH<5uQk zyX6;^|8u}-W`s`2b1?%H)_j@m~-Q~;r*I>7}Ygv5xnrVL?gMd%H+v%Zdk%zQGNZDtbn{x z!is{u3r0IoCnUV{NCD-_Sv&dgTn0L}Hqjab)d3S&FH6v(mpVMeG}oUJRPRawN`v6X z6QyD{xft%Iu;a`R{AswrRPC}CmNNx5t1f+sD7flEc`4tqW_$GqnDJ;=kAQ_s3NeZpCDxf^*BvjM zJ&GB{B*p*9O3(hqT++|wI>IHzq#5**6xrQV@d`KCaM1T0+L}4pUp9VI7P zMo-Jf8&pNRD5ab2Eh_W7vRXADFj2bkI;KO9C=IJ|0<8~J%hjt{di4VdiRsn$;PwaL z<`;3042`R>*e<##`C``Gcw@pR%UNYMlujQYDzoZpB5u6og`VoZdkE9RwHnewLJ!L{ ztF(WjhIrBW0Ir8??DaQcW!l#yyqJqXcxuBd47-w+l|p;;Jy!J^mqA2}R%5;823w2u z(+Vv;4##Rw( zZ{%9zwXs^a5;t58P#Ellv+@2g+=vBx?||xqeM&LRDMGI~)(P}}&s2or*>TtGWI@O5 zCiI0EbmX8tR$arG1cUwqpB7+)@2~^yBRSd0d4!Ae~L`YnH7Eif}(TdD+hiG}aDV1hI9kxR2&V$|5_G=@gE*|~gS1qf=V`4vYFuU;!ZL@= zTN1ReJ*5C@(WqldZRa<3{lVI2K9rEyK2wLLiQ`9~tch&()u=0&I8VtWB1QX*f@q%< z^_dj2C7qt=vvwwS>8%JA#V$idO?1nnBFjojGMrO3q~Vn%s6#!bK{xB&Za2oab1o@n|LJ4j+g@!$1m0a%yQ8dDo9vnS_}mmz_an!* zyp$ds#HHq&{QOQs+TQ3cz&{GRBdTI4YMR}fO@}+$5_vcsUOj>1q^K#_iZ<|&mlTRNe}Aex^t^@HC6QTFLr;Kv9@!r zx?h4~-A_n?62mkf8ds6zFuk`B7ZYV{MkWD^+*s4d1Y1{%{@fo!TjQX>>bok@-+6Tc4NHUb+0a!qbCZW6s295>NCA%baz}xYG|W zd!FIm(VplOxH7Ao1THu88?XDC=ko^V6E23saJv?y^Wc&X5H9_m;5bewDju#}H#9(K z*6vKk*ncE2Zd4t;M;PV@%tj}`7(9RG4Qvp}0V5TRmb8t)Ro+mi4m0_8S*WuJ1g za+T{V{Np#aXBMciBf*K%B8*=^PX&H+Jf=b+U>aZikLT4)ra>!pB1mz+2KvBy_lil@ z6^pB&?fiJ_l`K#3vP2Vk3aJ0NA%8D#DfgHMimfI7T$Qg^GOz1~&P1m0x^!TpK!=Rw zJhg_NaP8GI*?}+9g`mY`ksC%j#6G2RE#5cnJukB$@P86hIJ|fzA+(WjdNHelN&tHr z%!w#PBDiMx<#FGYOu@d=m54_Ln<+vV-E2Z?&f^v1(J-hJH_DxOJC`=KwPnX)OgBOo zRd?e`XwGYeT7GVi{QUb8hWB>@y!fl-Eb+>9+FFm{t?aJS=X2wogDK!m9b0u9^OpPe zAUrXCpnor%)R4m-C%Xy<6_=vnbIuSmQKuxL?o!}IJ4Rbwl5EURo6w87;P>|=GKC8s z@s#KzSzz^W*gZ+qI}SB@mNka8?nS8B*OnPWMVSgiI|}wkh;>YwLD?J8;H0G%wEu=SN$V7ywrgKTOW_$UMJC0yR zsZ&2%3XM`NWXwf7zc^1`Uhwb|pFun|ZUcw1a!TpY2J|xTvleGpBE|!DgvP|jPK}rC9 z0`t*MxWCHjF(>4dvZ{I^=Y~nUYq3+kNv?sXx|y*?SX?!Mjan&WalX&%dks_h69&^4 z2$ipIV5)3ckh{~xos=O2DxwmjZjb`2%l`d(C06NNB{8wmnE{<-?y2BITiwX4NF00X zVbx#dy0W76HP;fUVw_RJT3?K&C^v3E9^bL2m}NJ9kjVOI5f~;Xbclq_VqgjQN@l5( zbv`~alt{(S$Lbzmk=@i|pnP;*o-%*D%X!lwhSYwT8Oc-@wL;Py?=@xG>MMza+Nv&E z-^8yRDO$Uo;ora2b#Px+9J_Zo5r)OF+8{1)VQZWPP!3M65*{61_RSAWwA)-qU?|$@ z#C(gXPvwJIjI{s$mQ7e`@IwiTrNL(UNdY+&=qX}v+dW)k0aJ7Y5iBgMz?L-_Zk!aE z;U4#dmM>ReLfB{|L81`W#n&KU7SWDREO11t@BeST+jwnvhXfQc6F#+qvsg4slY$~T z?``d64qfPaq7)svX7RuY#;=qLl;FRoaGD`o>+c@8hedm7qi6vUhSLJrQB#06Ccb8% zu%y(ZJTLJ3yVVHx;<404@x<-Ozg{ni$2EE1VmO0#{Nh zT^g@;F=i|or!WIX3Sj%h)h$-A);`?s5wG6E{GdML30g!q0-7e-BW|p>!rf|cp)RLd zkvo`ZS4`p?KfAtWa`&x77mZW`Jz3a}#*uUE{X4`;d=K43L}Q7sjTO=G5EL94lY6Vk zUpwW*H0ja$6_y5QH-VNAGv|{_tlnc%c)^WEyLV^2%R=9S5>a$b<9G-7vX|I#+_g_Z zMYfI8CMw&QN#N8pom@d!DWaVo(RVM?ndx>yk2ziX3MSqvyx`$jkLtHnXMo#P!1{P& zkT@J*>fMvMtR+4nxBPGF$L#32ZYB!xOgBQnWS`39ls?_?u5&DlSHVHRB8!K>6zdX? zMf~q`yF8fPi`S%?jT`pbx>Ln2z|O zHyEP!BDc?!4X}nSEwLYC&0{}HSZp3^jj-^9CfY0vufbw_p=knpHR5fo;%7LbQ^nkk zapVV7O*_FN?uAYwUqsx>I<9zROQyfKv3yh33e?3#sMreBLP6ETK)iV?zEG~z6KN{M zpgVo#lWmzE`AV)J(uv9n7{&6{ar7Eq(J)R$y+cpr;kA3GDr*X0_bGfi1TTn9{( zsPUyJWqu028y2 z9O7yrSvfuX;%6;bT(fEteTNvpR#t&NW$*`zL~A9qk0ranoHpzJV(M~02`wNJBe;N6 zQqVx;E-d80ii^nZ`00CU!8c5eeJf#OYHYgDe2G9UW%G$@*Q{mfmf2n+R-{{ySdSlH z5S-)@t-NtBm02Q>&3N%g7MN%G2wC*(A~F{6bz=?boT3zB)Ai@8j$|ilzr@BY2vctZr=FiUjPy)|%Ye%fkgs~-;&!J^KC1{#LOBO-6) z67>?e`N(`@uHs`T(a)dFuR6za5Pc#9E$ZRWJ_+qFz}?)!TvOkIHMyhT-}fMswG$E( zRY_xp33D}0xU`jTosxrrR$xb-!;rOicURoW5~dGCiC#>Y*3z#aPJSrGbH|J-EIi37 zB{&i9z`N4{DF4Qf6IdSSy^npjndOM~OF+yK)rDL5EfCe{RPh1#Tz_N{!@GS75h!v_ z5S@zLNL)@=F-t68e^2&-lg!#Ro=SKkKLmKtuz|B+7PtzT9hC4tw?o=XY+H6qSlpJh zQVTRIDl^u0zMW|l|1_c!HA<6^{_y(6ygL1X~K1#os&nu?y@x z{w-03X-8C4A#mRhY&|4fahQM~Nb9qHY)q3ISdn1WEkrRE3F;#@+!Y&$78QfxN*2`d zQ=|{$Z2wyc6gewi*#Q$&1l4_08p4{+r)H1*kU^eSA=B|S3veBg+@v3jTF^5+z=U+} zZA7K;#R?p7?;fWo>UWLy~tpA)_H_s1S=&3g(9_6zvHg z1s8f7+Vl3eHrkmXIVgdJAy2laklk_z(TqaYRE37} zcJyB|CcG55Z2O=m*zi%torEohkA#o`cj4hS@Qjp|l4SRW!S@Vr@x4Tzf+ucJ zjJp>wA>Nt>?M5)o)O{x5iK->A6ho~W)iD;nd+S;Rsq$Q3`j! zjz8p_#SY2LSwtot5`3RmJUJS}g+>!t1m1UzCheDR**C8z(@vT9(GsYg8XM5gNC=9q z*b&0_Uy)n9s7L9g&3XFFKBiF*N@&!mO$}&WV?s<0Lz)~svD(}5?U{LOe)`t?iDsOi zZs+mCXuW8S(Hkq!2*71P?#WPcLc^Ho5_QGJYO5&N9g9xaWl>9u2WTZdz=h|@^aedV zc7cF(UN@KD3j5IO-OjQsU-k zU5O%Br9c2<#d!a)A+H}_vHL7*&bj{)qD@S$Bcu<8V$9=O%H}BSdttz@Rhf}&{V0Kn z#vuet#e199Y|H&DEPPMGVOZE0;T$MI^m;N_BU7WXLZza!CQQA#GfP4T<`I>cgl-ap zg)IgpgEwKBuenuj{r3f6J`R`;&5H(8OK0A2ALB!f`3fIu;|i&~jr;u7w?-KsHc2>9 zI|HNX599rhpshUcZm6YLiWUol$~IfZFJZ7ZEg&idEK+$X_c=5j?VrL__dgO=sBZn5 zCZkgzoYEzHKaV^8Ej#`z7ZS~Q{Kc?=$m)e*vO%}>7(2#a^OsmM&Npe{U@3^DK3&{jiUyY>>8>)TRm|2}qB&^8G>ah4YWUL1^S!N8X zw1;0cddm_;Hk#|mB75r8Cr}ej-dfUKr`qqYGIq3Cs<1;0d<7txueY}AQo;#WL{3Si zJ+yP?JFJp)Si)i@sg+K&gr#O9dhnFPE7==1U3H}MhVd*ox?>s9jloeHzJM_fl+qd} z8mC{+GDNK&r!~-QHqzJ76T)y!PeI@!Szv`V_uUq5z!b?|i7Gl)3{?C#9Ja$dzN7vU zYZpZU?=~j$Ikuht+;c(_knUw@Bnsf;opA}01wABMy*-RYPNKh1qW(10DDdJ@jDA}bj zd6Zf1^k6pbF?bbGEXF;iIec(yLt(P7SS=dY5r=W7}HPbK75oMbEYr{5?{4Ui{&ILiHkq7@NL^jJLPtX7{QCf6qwEt9T@I=mI%KkS)mqjUjL>4045f7 z)+P{(LVWMYSrYfaq=IYm@lI81^*CeiHi;|P3#B2a9Z)fe18EVH@w0~Qx37C9izS`U z5xE#kHl$l=bP0}k?-C{rTRh+M3r6Xc>l8s~42l`9yYcwt&|iGxkYZC6YjR`Oq8aQ; z-t`g|ujGw`jn{T{W7@8Yh?T@+!Gani7`mbPEDy_0j((nK$LwTU$c=Iqe-Db13!-GB z5?hbo_wnL@!4V2=WKyBL>800n|u`bN|`d)m2$QAv; z&@Le^EKK3X=FB>+`5WA$UR30vF4`^9HNW{b{3xgSVcmZ~t+xF)kmHz+waShK!31d0TJbo8K=u$@J#*SBYkk;FbM; zaLc?YU{D}K;P@-)qIokrA7zF=eFH&?N+Na;VnG$gV=hLrrG3s#8~788Nw!OPj7eG; zS14dQgY8u{&h?h7mJg6Zy{ielbXcv;BJ*dyf8a{?Cq2g`?L`SJ)DSd1i#tqGM6G##X(`*fiEk@g*O187 zhvVZ9wbFK;ncJoj{@uzrd+Wa6$^)@-I|v)4ErYP3s~rX)Qs^yws^QuNj1f;tSY(7G z83WN&P?t1bbqzZj1K&}IZKxMrgiOFw&d{j9<=0Xi+RyKC1%v&L!D8Kz_)v^_Sd)gZ zgqO6gEn;+5i1TFJU$QCFVm&}iM%IE z;BLd)9AjR4{oMqNUVC$t1cDV8i6u4|i5X$bzAKM;?OP=@dhPXvwLk?NEC#2KOPCUi zf_gR%S=N^ix_XfpDc+u&w1=q0ma8=EXM!gZ9UfCXR{46K*>5oT$Sx+Zo%RwQ9@<7a z)eMB&Nr)gw- z+`gn^<7WNN?Pq{l?RNPOnWUu{2QG9npPm{%`LoD~H_D63vU_An_Nnq>|XeiFQ5P3j`2j@-V zu4Gxpw{*C`xVuB@eZK>L9MIZB$G$_wZho8nBO-gEc9YfsV-b`Dbj9NyT+fTejhW)b zC3`M3mw1Z&KJ<7*uFoDQwYM+GHc*W1J)Co38*>=Se@qLDNI?2y@|G4674hx0-iyjV z6-{9oh&2)$GZ4+O91s1`Vd3Y1XC~K37QV&t9*3FH9D0CA7e>?L!+nOf!+J?k2A2TR6n+&qj9+=xHr$amSOf1|~>oYbYFzYk=67e`s z!gZhIi=Pryy}|4i`m_n8zm;*jhPQS6>AcVJ@Kc10@h66 zU8{d_2ox^6`$u}PsN&|ML_S6p8ECvj=U_^pTWmMK#Cq1td_|z*wLRR~@RZq}5{yBWc^9ZyKGp1)0j;Ot|*noM(y zIYGdvIjHy<@71-4@wmZId<1C}Ox$P5{x>Ke(}nGZTXgNjW;b^u&9UpX)gx`aPtrQM zTDDdj?<*7YOZ*%qX7Y$ei$<{W;x7_QJe-OFsMv84DFz@Ahqu%S9_A+kP^>a_K_*_N zc+I#d8~0rHHIa;q^Zx+dYcC1_ diff --git a/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766572119.gpu-51.2678290.0 b/stage_1_alignment_llava_ov_4b/tensorboard/events.out.tfevents.1766572119.gpu-51.2678290.0 deleted file mode 100644 index eb24d08f2861b6250fcd73162aa9119eafdf428b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 48602 zcmb__2Ygi3@;_DGtANymKnR^6B&3kgq)P8i1Ob=L?q;)O8(UHk#V!ac_QFF^6vcul zD2gIaMQmV40W0wAsE8Fs{Lh>_XYZN2dvTZFKl<_c!MyL8J7>=HIcIbo{pa|NqgUCA zI(*&o@YH=nvl^9!9o|6xh$?p=+P`ROzScA94n%_CvY;dE%8$hSen+@AzdG!Vy2JTy z{4IOA*XOqDf6^rI_o_jJyZ7B8U)dV9Jf$Pzo>i&kO$7rLpb_Qgb!&&JLxHt>i#Ch6uXfzrarvFZUxz7=)FxL3zx}%OEv_@{mFSpoQ zwV^div|=r@vdZqPa0CKwpNZr~arF9yDl4vDHhtqb!n@jb$E~(~5^sn$NbBec2V)`o zf|xs8Yp{gVt!&-&a7>Yf;_5G+K8K(mzp~3}+aqm>)V^A&mW^E2fyE*wdQXqjTUXh* zZiVr_)@`CNL_Vg8Fwb5FSSj24)2cw`J)7poT4qrrJRr^_=goe{1=gi8# z({_0WTBM)WN6VxI${Z0B(M{t!dumC6_%Y7Su3L5;W5Re_N5Vv5Y$J(whn;Zjxqz#@ z$X8U;gk2dM#t@mMT^g^kHSA1~gS4}?9D?))z)1Mqjw-k5yeo2O`u-7TEPh}X0bi5t ze#-WT1fHmk*9r(Y;$C3)fie{fgn}qOpsmUs&S)^qKR`L5TwA~1@svWQ2RC$~FF-w* zt?y>YZI3t|J~&7m)tdfT-!7HRxXkTJuxMP`-dAO~z1!dKv-4TM8{y*l z?BsB{l1N>uM8DYhtMVzt^#9zm_c_}-2|iLAu66f2D&6*~pwm$%tQQ=cDsRMF21dWE zmM@`D&ou4O&jRh4A)lN})T1HK3IxM`;POJTjT}*t>!}l_O@v=GHHtQ^+b-FfHE75_&?ugeFbNKAeAeb?y5Ngo8QDenam%IIzb2|*9pRVbA;V4J80UJA-9$9Yj`#h z`Re91Q*D!b5|M>kFRgi`+UqfcDaV-`+^#czbq*%1DefwV?H7sFTPxC90Y7W(5l6X6 zYbtPDJLE_DzR3i3RUVOs0-Igt4oB@}-T*3l^pQ9d6;w`S-)%eVG37QcpHPKzi-FVS zwi~wF?g+T-!B{jDi`uK*UQdPD7+NlrG3Lt}rY#Q3g@m^7$9z#Q2o<_{hSB98p_~zu zC~vDZMTYAw~povmkJr`&@AfMwZ&W&PE-uSs7m-){93J4Q@=QiZ#rUCfUo8D;* z@KY7Awa!jE`1%fO6;_}s;H4iw@C>sTmlhI*XfJZWkpPiMANKHg=Q-=TZ1S?e~1n0>wf7Xbll4h9P4B#7EsI z^MRPp0m%`QR#C_=kNJECr)xVqFJa{Gm(WOl4`+qjS!pj1LXyLQ?ylVzj6{T8NIYK3 z8rUm2e?b zCjWt}PN`+?UiH~TKDv9^#>V>H(TbqS%qy;GEZ~JbQe4xf&t1Qt345D?1dPJo!2q_( zT`A|Xw;?U}KVEkJdBl=0x7@tk_FoA*UK^|BJHgWx;RPgc!qRg9p^3W`d|NFA=H`t% zmNNBTHi)Q4_1+>B4r0oWgIVw6^`l}9m@m~}FrkSo9-)B^a);m_U`9Ex67d%*9>dhm z3p+9uwLzkyipplF3Hc@!Q&CaB%`D!=D4jDzr4&=1x_sHAL3^aqYx4dT)Y)^5ON~q(sF=5v2*1I~5bsTqxV3$qptME+KO$m^{1DQTehV%J>3B=IY`2%ed?(GY})e!o3f1^E?+Nerz?Kd)-iPNwdb zjUdud-L*HN+MVU5w1*-bT=*XSbQIA$TpOl!G1D84*j>9Z2avEcZ)S{bt1u*xNGlR`z>R~yfF$px=MDG1%pwrxeoNwk|gmJ_<+-?_&6Dh zJxkeXtUs3E@icZq)!_`qz`KFuwM808!RC_p`O&;V4Eq5UyQ|wDiq_g=0azn`-hEKD z^6?|7SX;%!tpEMmKJ2)RA4hbHWRS}Z_P`HRN1|RQ`0Or|sa)sW`VdP#^{jWXlWp-$qhS_8plpR%Kuj(IKxLByPcp{OBp;8u+36Y4;ZNjr;?3gwK{>P%j9+LqDx&=IdE0Kp^d zWV<#O44_eyhIF(EwK#5IyMsRoVMn1I)QTs7LABlIb5uFNaN2v9fewgQwhedfz^}j7 zGImu?RoUftL@Gg-M1ti}EFqQ0B2aqZ+C4dtWM8#a-ar&n0@%1@5ZrP>=ygOX^W>Qg~}V7v?YD%W`!CW)ME+>e`?k z6BTPrshUK>8uZn5KQCmS-X{`NczRga5sZc+l-pqvbZgW}Ki+kmsgb4U5vd|ukEjcv zUXI9IcC8_Q__)LUOwr}dBuG8O9*TPX(A!`lQ4uaolo`it?U6q|V}|=32`60ipo9~y zL9o2s7c{36tx3&VFw)M9UDYfi72UgT1~%c^%OU2%Y76Iu5?vIk`O12Svbvm4x~X|^ z(>7bX^9g>EHbE;uInV$t`mKb>uk1TtyZ3iXv08 z!kSsv@2fe&M5TPTA}U!Sun0&wx|1eZQTCSWxhLmL@4{kp+Z=+#*u0H|3^^d_mnvqT z7E3%#89gE~Q5m&$Iid~}5%gXX8WWUUHgpxa!(fjLL45oByBE!--E330-Zi!jm@~Xaw3p#XJW6QsCba@u}k8FyiI9EDP7)PPifqhhpIs zWhAFF26ZK>#3!+e)}${Rx8YRQ9&%7Ff$br=E_Vn5cJNJIc05{sh~43z;BK4ye5_aa zt!dUXj5*8ZsmwuGpWEYb)~1O&s?Hs?Lr8W+slnkOTr4%T@z92!xX*58gY-$}&@Du*Te_s5XS*;2u5i07V zGGdHCumTmvRKtB&<>VI`Y9p65s@dbWJawJd_PmFw>kSeW)pcRW8*-y@H6Lz~VEMN| zmj|N*`^~(R`86}#L_hj9S*{`|=SnrrMMBX~kk`gkoy*WryWj0SlzF6oN?75*ODk{! zcuU?mzw>;}IKHY};dpz56pd1#U7-WS7c-|LlS|yY0rVGG~LI)~aeV=3FK zE9e76%GQi+r1R5SrPHfBuVaGw_k3DK_z}=8it5f6QYstZu|fm=E=A7(VT$|?rOY9r3#tKfQ|8t(Yj!{X&y5|KLHkewqe1Hy3Pax- zW}hXlg*rbo9><C2red;Qr~yrwO=@BFY8>t!ez@1 z5$FtUy4G6-1%DQO3GpF%wkTQwg^Y@z&*XP1icSB90>v_&arD!L{b|8ww|slR_Lf|* zmzJkBjX;sslJvr&Vx=(lTI#!PO9wDJ@rOi0d*8+v)>~=d;}aT56jUelC#xOv z!}>RD&*HSzF`^ISv>ZUKj)0lKCY@x+E#v3obazb%N}td_rF!;ftzZ5$>$DhEMew4j zC{A?B>BF1m)VqUm;gAFrUQZk_l2!(>^NK+}rkqz)6Pc)-J8&+fg*+_l*>jlN+q;I4 zMPi5{Ct-r6qImRyLLQ5CtR-BbI1$$o_Cq&E^0-<#W+-RMiFmHA){IpGcS~Tb1hzGR zlTe%7OUOL(*a~JhE?P*WqTOg`AST_jRg6#&P2K-tp#~SPn9d~P4Y?3XL~9(@ zi9xpBmB1cL;T?0@@ZZ=WUT`syE)KDQn2hO`s2v_``wHWEx5Wz2n_`Cw*V^6U$}9VX zNopeuSU2y?kD1u*luL-x04`zSNGMbnfnGNt{~X?DD}xMQLR1S#NJt1&5wvjf6-e$n zzq~8#V@kKxr35Lapd@72+){4|IIjCd&l)-!8(-O$Z+lJx-d0z;xvthUF*6v8^R}{= z4}}q8MKJn~{Dm-k=IRx2S3~c8BwFS3_ZmOmF=J@nq=0({w6q5Ud4>5Uy$s2XB>V4q zy@%zAEi0g0L^M=n4}}zKxw^rC8H}x!mzmg#H0!Zoifp}O{^>0lTeB`Vu~k5(&eo3( zJ3P!s-6{dmN9`Vfc?wEKVRk1q;o_38V#VQkY+%WG1(A#cOI=JWI4eOLC8q8y#p2kg zx2|QWQSC}X7maf!bm~|XizL*GEZ2ygecc*%sMpIiP?4TxTEhnu641Jj-;w$ft8Q*o zb^%HEo(+NLY}2oz&zPo7(Rx@f;qZBaTy`?!Qr{SR3}Fn3QhmmGzo7T;xh#kOhg=PF z_$|x9JB!8wCQYRn8dLY*cNcy!oFyh!UQNV{!~}%%qM{1g**H?1W=qz;5cy{>qjby? zf)>>V!DZ|iOA+qfbw4{Da#~LV^@p5O^-E&8;^+#Qe7#mlCNWxHj2B z=z}z+ysi1Wr^lrItlh8l8bZc)zdS_tLv{cV_56v!W$}lcH`dGiS%1-wOa>3gWl#o7 za2XNfVReVHWFlR2!sV2mHKOW6H~THnmF z9;dD#Sj>9#VX(0fWD?@F0>LZvZIgY5wR~>5(EAQc`>mG?inO0tkSlQA{95gV^-6E; z{b~=Br0F*(lGMd&CCiQG2YG|L`)NbSbcy1}UlLuPvk--|G&W*xlO;Yz?dluO3t3})FmuSG~^b75?Q#iax;`Aj~{>Rirvf-ej$;BCB#}GWNEQr3YONM z2y9Jb$v@?{ElO+R`QI*$684*hpeE(MF_I7KySOi(~`K5vaej# zUG~y*4(9C6`44@EXfr_FVE6g$tRE`XZ@6#Mk~7$3Y28(XjgzI_lJ`j@-g%N-hK6k` zx|iwb2PC+t8_N4$PGS*7>T8#6%3^{#_AdGY6x8nVf}!Xzh{sSVkIo&~aQYDDv;QFB z(PuA|-*8gPC|{AY@XE`WEZuxJeT9%EOL$R`$4Lsl;fgGAMft~;X`|Wa^jl5qh>0W2 zmSN9Lxi<_Rb$nH;sV2*1cJ1;uW(C*%av4!@mCL|=F#nfTzW(UKQLNSH>U-#8M5~WM zwdjJb@`j_>z7hYbB&BJ~54!WhH-BQDz@_)nQljYqmy-AbHX@dcFO|j){W$CMWlR!g ztWhOFjDPA=mHLG?co_BX6XRCqV!P!RJ$U>K4>LlY@1u3l2xZGIgiw(_o5JPGTCv|U z`?=qlVm~ZlS&N&5m8tb|Uo2vptFd0A*X3`IXCcXR_Y>(NBtf@KpCp7{%d{pNJP&Tz z!F0)@wFHd1LC0Kp4?O0rz>t5W6q zrX?F6WL)1akp7$(7yBb;(gb&UhxH1RQksd!Kkm<~FoSP9zc>^|ro zH3hMXkx(>WpV6-jU_3u#ox=03RJ8}MJ%HVocKsHigd1EGG3Unf{YBg6Gpe7E@WR&; zjpV{AlPAl$VF}wZ>*;Q+fPBG2M71c`yI`~fbwbiRj}%a@oIl%q>o*Lv{=;Ta9WaUY zvIKqW!exIm&9zp7>Rl;7X$bsyqEyT#SHRsAcAWWvKaG@`+)wLc=|8ysdFF48euT)^ zi}tV*)HK4_!4D3QGYCDGIO3bK04}RNRLGqCLlPgI{0{Q?V4Bo#_rRQ9>N%S88|*k> zr|?T$=W^qtM7p@nB^fmM=FtkH{|kakbN14jQy&&t1ju*}z#f@T$;{T~9R>oY?>mH*I5H2Yu&7haWV|Ul!6>hNM zpzk@fHFFkD|KU3p#`NDn*dmP4)AET1Rnaa=>E`0x9Y3<@?Q;nxDq=pzLg*2tVKq*m z<$-d(aqL-^UcK^hq7T!ny}|7d!p$$@AQ>80VX<9wQS!yCxv{gw&L`L%GlfskYN9f$ zt|sEflU^uu^T?~19zHG?Lfh9>rdg%^6E(yW&IfQke9?xL#jH&G+D7^gEYr3{cxuBd zkGN8nmC^^^d6ZSXZrentqSaV$xxvAGi|3W*;`0syG;eSuw zi4>`TOEppMajY`x_Lzo{mF6j|o8m7U(fjY-wU4MSLx181r54d{1&9|ApWwtg;JG;^iR|AxX`r>SSAObgH!QMNd z`e3iFfH_6zHOD%E-tU=?Fyudc&MFpkocT1Zia|#K+GEuaqm?Hw6;0m;s6jKarIdu ziXc}U-Ki6`GJSnAqL_ydrivDr&M@9u-+u zQj(`v-S7shA&=fhpjboBz;OqAl14H$3#+C}Z9ldQvF$k|;l%dnaCqb8iN+{+2DWj% z-|k^)?Hm3}6pHx~h!Y`TOk=Qo{_z|7vo4WgFAyF!-(=vGC8$Hara?FB-F~Pv_V)Fp zm;>J(IArVbA^~1rSG%OH){5+z`S{Ft>_7V8CGa>J;x3vgeza7j}d<+dkHE=dW76q?2P3R?DE*LCJb}s%S5Qa6mX)|APqW? zb^LE8E9s5dL71Wi2zd zfa}lU`U`NH@dhMov&3DgD>oG{V8^`lRl?NA#DZ|gmk@Y%DZ2Hkte^42cMR_{6)%Hs z$Mu*GrLy%!1-+HapO0RDydHBVu6T{;6Me!mXPq|gqW0ZxW4PU4S8-)lHw9d7=5yEl zJeh^VZ%bSZhv9ZDNaw*NA0S-%J;4c_P*glzxo&8HCO2KVoH4fM4P~P;^&VlEA227u z6g*bA>r?lGiOd4c-${6)iwW-%z}u4WCIaO)d}W^s@(Y#gEB4&8<5y;Z@^%rNC@sSH z1@u(lHz#5$6auF4#j7`V{fBAL4x#uy&{oup8pSqd(VIk|$WuW5 z&kgx|c}uz1JWy;c@qONYybDX^bbO0|h1aD68wEOKEa#~;^p4z9!tB7mDxt+>ksC%j z#6G2SE#4dV_x3P{xB6`&S2(EQ z*j@Yf)67J<-zN%nmjW-^G1}@LXcT^>EldMbx27?&-`L z!=8|+*w>aDM@5;6KsyTdM~HPynnBqu?(D9paZF&P={k2xEDzwJdI)04*}B(+(59AKCD6<|>{2F`=Rh)?Q~O zSco!6XD9O+tn7P}E2d0hxSy)H?J!*d-J9SvKnB_sbehhx^%m`Us*lNK{6wKQ2U4am zz6^a{=|&;D9O-^md}bP$kSqJK_E+r`PO|D`i#nLe82a zb-!k(`fa&}Xs|Wbh={8uuu&_GEY9dQTYEBtx zI5X-7X+%2Xo;>ALR_T0IVq&E;2Rg~zHQ+;A-N>v+9DD2G{_(ddgshJifnjn&he*UM29|&&qw3tpIv-zhM+JI#wF2 z_a%X1X|S_?Qa}y`dYahtSM4uh7WRILEiA0SmNgh|oD`Yi9`}3W-k-^YaO_t^E(&35 zd<_C-5$*WI0!Os^{(H0$yI9Qhg9H>Y6F#+qvsg4slZGOC_{VZ5bLdunO_ZWT*Etb5 z$@rC0ffD@pme(4xb?%-A>N3iQeM1X~Fq{^^j+!#GF^M$;g(X#9<#~Y?uXo8~M(Z01 zEbmmL^FYglFPqTYZ)@Fj;94P|?G)3?gr!F4?ljwe#746sA_32qKBIPmcw8%tOK z(*HXG#DFGK04B>n+~s`6jI9z_WIhD2 zxVps(*4l@w_doY+<_DD@CYnWbBcN%LJ?h4KE8MLH7wU4V6}f|ncEu#VS=#msCU=d0 zB6QJ6CD2oa?PDCdytZ$yWhK5RB`lWsdRh?;4?)3^F}b&j{1q!FCfKb0dX@&5b%ZDv zGv^B`t=?l&c)^XvJtJQq%0k~iC8Fq>#_xOTyl_1QMw{j;)-xdcv4)2S7N zl_J`N{Gq2ZojLv&LXSIL`U)oAD!ky~*uplAhcmz z(C|Ul%JHRKNHoGFE=2uhoIBPBeNex31K3f#_ILUaJc`+fFM5L^YWKMPrfh&UtY6Fh zi8YTE{z0(VJk|qY;R#K&Sr}e}#r8qd1omnq+E~TUaKviF+>LSMf3@{E%5ppJ$#q1; zt*qlpM7Csl=RGgp$y$M`juQ3Q3e;6W)x$u%c`LC{uGAB0D#W0>#gSv1m>$XelU5Mv zL}dkxV)@9^S7t^)Mb`t&R;|>TAGdu zxfq1}%i|IJB#jtxA9oh&9J7Gdd(;#8VY!WSU%I3ZOBk$?(3mh#g;1mlD40+B_M+t) z|Gd@wVYbG|zlnU&Ev;~dx{pa8q;tLQZ4*Y%VB&a4t|2iuekc4x)=NPevgWjb8f_Yp5#uhgQIbs-DU>z{D&hhqziuRZcrz z^U(7wuGuSB5d+xDD$u74{(wicRzmw&stY{!&7L=zx?Ei6RQ(Zakr=@Rq>_RLB6oQ? z2Uc7}cE`_}!q_cLjdiX|uyHlEP-wnnpq8>Zb=~~6EZy>|L>1{4B-R_i7X;^fMJsQ@ zOJ$bG{qI`*1`EuC^@v{4vx~@Bz}Jm6q;ouJ#HRCJd;TYOqFU4^Y&=my`RX+cXuuOW z3iLe_?)6og3SnYSZ&EhY@7;CUO=VH!7P$sSk?oLvI<7A0FHF4|dtQr!Z!>FM)_`ai zjh6yfcj%ms5M(9c0g7ZXpe5f|uBh`K8@X!OknqK=4}$MtPUAiYZZ2F2AQ$7psHg+e zWEzSqpC)ny)TW z_+aAkxJhqaz+BUDrxCRxwfz8n3GFYy-NN!hQ{RF$xmzyyY%r6x{u*JT ztaV_RFjwP*OI!KYDLEKu1$GoU3|af?wmG|5!t_&#jS171`W3{f52bVryU$?Z$)%?g zsUqHicc+6;{!JVwusqJQ59}*rIik*I5FqA=TEi{;7Kmzes`!BaeDHK0!+S@?%Y^7u zpqK$|?ksks)XxPA6FbiCT%u*%%Uw`YUxolfnHzruzmW(P3 zG%G64Y&!Z$rcs`kaHvr_hV_TnPvqUH$m+cQPWPu<*-Z-0CPXOSq|n@tm?mkof?74i z^DIj@YiAOa=+DuiaJV}$<8B4H`s=;Bn7&&tA%(ufDvFrb@xrVNK_PdF{G0$WhG|$ZUMg0$fKVH{}PTj-0x=2@}$9C8Y4h z${cX-A~-T)5Nv#*9Y;br0~95~6G-RGc4ja`vb+Tmi-x3~LM%=xm>;rGv?qKNTtvXG%thT#Tcgg5yJOhkz2f| zN9m=#BscFk)2Jic5j1MlP6o8DF(D?0Ax#dRT%dVp+ze?Fm<886l;MA*bk@iZ`C5Y19ousNo+hgt@X#b^saNhlZu#_BDe?E^<2+*d~rvW$CN;D##|0`l0v4U ztTpcBoBGseu0qF7gp001X9G}Pc!A^MCMGNLr*wR5y*jpqotal8vasle{NckMR?R&M zv@$1f*Y+n~V3nrI&P22*7;HtIs?!qp{K;}|H8@TIze ztNKjO?N>+tS$1llpKKc?YQEM}J2N3IsF251XqTDCA7R2Zqbrez!j*w^C5v2@0s)K_ ziyuN;>b}4Jl`Ab5I$#sPE!%&QQTua#;g?+aT_^6y2$$PpHeWGy)!P4>GUfFu{ zdn_y*nM-gO7Ir{52TBmVo($H=)M%_wspzbWCk)!clF)}FEGD5l#$jQLLCN4vSmtYP zm0SP)L70yNrbF|h!PHMqA3c=u;l}O?A6nrG>Aa1*E9gJM_)vP5!iQ`ynt=%3{|MU3 z3-5+nilu0=FsNKu_kvOeyNAIdl_ztb!)0xj_h71fQ4gY4sBZn5CZkgzlGY`B#T}bH z$ButaPXfo|FNO_7RxbjR4Z5Yr*fH^%zvPk;-Mr1m48w~O9SuWQj7JT0NbseDeYp1b zOcs-R@`!Q~lY*SV#o6-Zb;~U@nT898wA#uDZjw)+B8WD$;YnZ*to@X~khxi61I|*I=_&J#!6Cm z4ol5O^x!FnSF$&3y6Ub={u^e&QJo@M0E45Rd;wz|D6KVeTQ`21na%s;8fZ4#>ucx< zVYsHJEO>$}u)>-S4LdYrieyYNQ7$@G3{?C#9Ja$dzGHz(YZpZUzvqP?H?i$JB*B&K zG}r-do+(eFY^P#dzPRxr{+`Yay_N0E)oF&YKxkrw?6}U1WPZIOhCB=7>elri*5zAL zLJNqSQgLvF%Y}l(orz=Dyxy2OX&*}@bkaIi)Rw^u9m;^1c%7MJGnrHTfIsfq`6APG zmz5HcqDhya@maAnKz}P3HJ6D)uRe-6bS6-2OA#-Nq&Io=#H?=)v2l+NB)Ay&sCD?^ z)`s#_U$NSF=JwOrnXBnbq>6+D)^YGgYrU=30sU>_A(Uiq*IIk;gR^>F%+gQo`_USh ze#-XaYv-6Vg*lMKs%1tzC(%e;{E?4u+eV9(^Kt&gGxJ%_W`~5woJ~6cPg&oS!oO(l zZ@ZY~^7JS2(Q;*B>cdgxIXT+r_iL2HSYsQYvPNP@7MvXX^WQ!dVe#0b5?;i-6eM7~ zF{Vk(4X4&Sx^>W1D;e1{&Q{3IF#wb2nN+8g!qPmS(kOiDGg)7=Y{M}LFM8Az_;4(c zB=2OYNcL{K?L4MR?-@vRqbki-fWrq(hi`Q;kp}dKo0#SHV!E^Rk0?k+AizNuH?-hMzDA#Zvt$hwrd^Nc1=XA zBpwSE)EL1~s@A$yW)vnOIcrE;c$W${lU;KAucRT$3k(hCJk?RG3!g2Fp>~O zUrNM{CJ90*;eU&kbjZttf8|T>>3Z9B{|T~e-*jae`lhbc~?B;Bp z_ydD}Xp|xoZA7wEOk*dCv7+EdkDM;Ywhfu+usWmXy_9CUu-~tlwn?K2`qsMIO?9<) z73F#|R^TU2>3QPEWbVmamt#D|@-I(T85Hka+4NIpl75yHi{2EJIea9`pgL79zwCOq zh;jM$F$$ME=nWYs8uGT@!2VyH`v*HGgU1rII482-4{n(^1q=#g7#x2kT~zV)<+GUK z|58GVN+Na;VnG$gV@^b}rG0KVbKRXR#JqMKQH&vGSK|r=OlOeZPrR`}Dc}~C6!t4t z=5Q7Uclg;h_8w0Qh{`as1Y$E|9lnbUxTrvF;5ehlC$pJB*&~6`v&e!WM34;_2Z1Nw zP0UvG@`7r2peV1(XNXX1RWccEEaJNc1UYMOY@}Ur2SX$gy>5s}<_R38xqTVdAF~QO?PPE@!BTBQ0 zw}UEY7tRS>ziAw@tKMt&9Sb05Pb0!bizS8RWRw0dVk~oDWrhPp3wCQ`86MwHn6;8Ytn>qCRQuG_sQr*TkC%kbzJ9xv- zC2B>CgdqXe(jmK(Qz-Qv*aD3vSR@%;U3wyT5zq=b} zZ+Y=7p4>bxu~FJ`2phWEVE`hH-ooCGMOrdOtT|s{gd`aQ(R5JXI`-%D>}ZUhO{l_n z=tUPH6Y!KXG%9fUwG@XL_Z2%B?Dq^7>xRUK3e3ZrG=wGW;q}|LFdnR!qwoOIZgJRZ zuj>T2-|<{*Bs$_gb|(7FB~(0X9f?r!{ctjp-SDG?&Pu#w4?Vw&*}9JS8e|FF=y?}@n2Mr#dhwGQ=(*I=MlO^zwL;c=jUVB$N0gJW)lmvnm z7l|b|7|9vY`e+A@dF?wTG~`G%aS7v+=PZ1oJ{>;#3XN1D!z+I-5X!)-?RHHrbHf+h`JIHXI%GfL%|>flaL4C zqOP1BMS_;jEjgVD=EdbisqkHN-J#>U+!0gxNg>zdYW;Uw!3?HGW_k#g>~+8s!*@6I zG+AOg#B!gOwp`wt)qxvT5HR)#v=e7th&&*}gYzbFSF$X#yE$B7+}&aIzTel+Sw7iI z$G-IRvjeuLB>Dx~9IY+JA}9&yiYGj{UKEcTbH$5G_Cjbb@p=M&^ms(B&mOF@_b%&Y zpcva5Ik4nL=IhPz(q{=@Px@o>mKG5e@$I$Vi+-~<V`r!~u0skiPL<$&{L)gD^ zqNK36ZvnfWYE`ylb6+|)Yich#&$cu`6z6F@unEp|4i&Roq?TD6wRb+qjKr`Y;h@!N z1337;6<*Sm;43QXO^JeA9o~18xdw+Mpl}TYAbfB#K!wIWrAm|Hf#$D`V;yEUhKNS& zFzX3U5r1DG=Bvdk!@#=1_=vSmRvf$SJ!dpyt^55K&ByP^S+`$5&s! zKFjuen7*f{c9zyOan?=QONA}p_HEmObu(5+2u5@>8dCsVb17iTSS9_jJQ%Sv@4C@GyiB;#Wc#l~(_`WvQyC%v++k|>O zUE?nMzhO>IW({GZQ_~4fKHmKbbHO6ZswZSjicb+n#R>ai<=;6>;lC<@g>#3`Duhl3 zck@M+E=I7gHYve+{x%JQ4`kmwf@zN43keuC6cs-cy}GV(9=DxlA9ev{raunV4VV=O{UoC+FO_ft438zmRAa52s=P zDt25%ia`j(;Vm_Shxv&h6st^Kkja;^tz0#bjeDN8h?c=|&z3G27fXB~LxEC`;_!VL q*D|QL%ur(EQu+8n7dAh}I?JLL5mne()&i*l4=@#^?=Ss!%KrmbFQOm- From b1429d083b745d847254e42b0ffea5a14b097b23 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 25 Dec 2025 19:54:16 +0400 Subject: [PATCH 06/39] Add AMD/ROCm alignment launcher (alignment_rocm.sh) --- Stage1/alignment_rocm.sh | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 Stage1/alignment_rocm.sh diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh new file mode 100644 index 00000000..a35c6ca0 --- /dev/null +++ b/Stage1/alignment_rocm.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# AMD/ROCm alignment launcher (separate from alignment.sh) +# Adjust SBATCH partition/queue to your cluster settings. + +#SBATCH --job-name=llava_stage1_4b_rocm +#SBATCH --time=72:00:00 +#SBATCH --nodes=1 +#SBATCH -p amd-gpu # TODO: set your AMD GPU partition/queue +#SBATCH --gres=gpu:2 # match quick_start GPUS_PER_NODE=2 +#SBATCH --mem=230G +#SBATCH --ntasks-per-node=2 # one task per GPU +#SBATCH --cpus-per-task=16 +#SBATCH --output=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out + +# ---- ENV SETUP (ROCm) ---- +source ~/.bashrc +conda activate llava-ov-4b-clean + +export ROCM_HOME=${ROCM_HOME:-/opt/rocm} +export PATH="${ROCM_HOME}/bin:${PATH}" +export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH}" +export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1} + +# CUDA-only Apex extensions are disabled on ROCm +export APEX_CUDA_EXT=0 + +# RCCL/NCCL runtime hints (tune as needed) +export NCCL_DEBUG=${NCCL_DEBUG:-WARN} +export NCCL_COLLNET_ENABLE=${NCCL_COLLNET_ENABLE:-0} +export NCCL_P2P_ENABLE=${NCCL_P2P_ENABLE:-1} +# export NCCL_SOCKET_IFNAME=eno1 # uncomment and set to your NIC if needed + +# Go to repo root +cd /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 + +# Required environment variables +AIAK_TRAINING_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 \ +DATA_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset \ +TOKENIZER_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 \ +CHECKPOINT_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ + +echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" +echo "DATA_PATH=${DATA_PATH}" +echo "TOKENIZER_PATH=${TOKENIZER_PATH}" +echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" +echo "SLURM_NODELIST=${SLURM_NODELIST}" + +# Launch quick-start script (uses torchrun and nccl backend which maps to RCCL on ROCm) +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh From fbd311d86fde8bcff4ea7f684ba2cff5b333227a Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 25 Dec 2025 20:04:33 +0400 Subject: [PATCH 07/39] Make ROCm launcher path-relative with conda fallback and quick-start validation --- Stage1/alignment_rocm.sh | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) mode change 100644 => 100755 Stage1/alignment_rocm.sh diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh old mode 100644 new mode 100755 index a35c6ca0..07922211 --- a/Stage1/alignment_rocm.sh +++ b/Stage1/alignment_rocm.sh @@ -12,9 +12,19 @@ #SBATCH --cpus-per-task=16 #SBATCH --output=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out +#!/bin/bash + # ---- ENV SETUP (ROCm) ---- source ~/.bashrc -conda activate llava-ov-4b-clean +# Optional: override with `CONDA_ENV=myenv` before running +CONDA_ENV=${CONDA_ENV:-llava-ov-4b-clean} +if command -v conda >/dev/null 2>&1; then + if conda env list | awk '{print $1}' | grep -qx "$CONDA_ENV"; then + conda activate "$CONDA_ENV" + else + echo "[Warn] Conda env '$CONDA_ENV' not found; continuing without activation" + fi +fi export ROCM_HOME=${ROCM_HOME:-/opt/rocm} export PATH="${ROCM_HOME}/bin:${PATH}" @@ -30,14 +40,18 @@ export NCCL_COLLNET_ENABLE=${NCCL_COLLNET_ENABLE:-0} export NCCL_P2P_ENABLE=${NCCL_P2P_ENABLE:-1} # export NCCL_SOCKET_IFNAME=eno1 # uncomment and set to your NIC if needed +# Resolve repo root relative to this script (Stage1/..) +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd) + # Go to repo root -cd /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 +cd "$REPO_ROOT" || { echo "[Error] Repo root not found: $REPO_ROOT"; exit 1; } # Required environment variables -AIAK_TRAINING_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 \ -DATA_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset \ -TOKENIZER_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 \ -CHECKPOINT_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ +AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH:-$REPO_ROOT} \ +DATA_PATH=${DATA_PATH:-$REPO_ROOT/data/LLaVA-558K-Webdataset} \ +TOKENIZER_PATH=${TOKENIZER_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0} \ +CHECKPOINT_PATH=${CHECKPOINT_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1} \ echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" echo "DATA_PATH=${DATA_PATH}" @@ -46,4 +60,9 @@ echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" echo "SLURM_NODELIST=${SLURM_NODELIST}" # Launch quick-start script (uses torchrun and nccl backend which maps to RCCL on ROCm) -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +QS="$REPO_ROOT/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh" +if [[ ! -f "$QS" ]]; then + echo "[Error] Quick-start script not found: $QS" + exit 1 +fi +bash "$QS" From c9f0be1bf2506d406ed6f931bb35acbda2830cb0 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 5 Jan 2026 00:19:28 +0400 Subject: [PATCH 08/39] Configure for single GPU training (TP=1, PP=1) and fix compilation issues - Updated alignment.sh: Added conda activation, set CUDA paths to 12.1, configured for GPU 7 only - Modified stage_1_alignment_llava_ov_4b.sh: Changed TP from 2 to 1, updated checkpoint path to tp1_pp1 - Fixed Makefile: Added check to skip recompilation if helpers_cpp .so files already exist - Updated utils.py: Added pre-compilation check to avoid unnecessary builds --- Stage1/alignment.sh | 45 ++++++++++++------- aiak_megatron/megatron/core/datasets/Makefile | 10 ++++- aiak_megatron/megatron/core/datasets/utils.py | 17 ++++++- .../stage_1_alignment_llava_ov_4b.sh | 25 +++++++---- 4 files changed, 70 insertions(+), 27 deletions(-) diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index 00733319..d751fa49 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -11,13 +11,18 @@ #SBATCH --output=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out # logs/llava_stage1_4b-.out # ---- ENV SETUP ---- -source ~/.bashrc +source ~/miniconda3/etc/profile.d/conda.sh conda activate llava-ov-4b-clean # conda activate apex_cuda120 +export PATH=/usr/local/cuda-12.1/bin:$PATH +export LD_LIBRARY_PATH=/usr/local/cuda-12.1/lib64:$LD_LIBRARY_PATH export APEX_CUDA_EXT=1 -# Go to repo root -cd /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 +# Go to repo root on CIAI cluster +# cd /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 +# # Go to repo root on 156 machine +cd /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5 + # ============================================================ # Required environment variables: # AIAK_TRAINING_PATH Root directory of the AIAK-Training-LLM project @@ -25,20 +30,28 @@ cd /l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 # TOKENIZER_PATH Hugging Face tokenizer directory # CHECKPOINT_PATH Megatron-formatted checkpoint directory (e.g., mcore TP1/PP1) # SAVE_CKPT_PATH Output directory for saving training checkpoints -export CUDA_HOME=/apps/local/nvidia/cuda-12.0 -export PATH="$CUDA_HOME/bin:$PATH" -export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH +# export CUDA_HOME=/apps/local/nvidia/cuda-12.0 +# export PATH="$CUDA_HOME/bin:$PATH" +# export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH + +# Use CUDA 12.1 libraries but system nvcc (CUDA 10.1) since 12.1 doesn't have nvcc +export CUDA_HOME=/usr +export PATH="/usr/bin:$PATH" +export LD_LIBRARY_PATH="/usr/local/cuda-12.1/lib64:/usr/local/cuda-12.1/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" + -AIAK_TRAINING_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 \ -DATA_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset \ -TOKENIZER_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 \ -CHECKPOINT_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ +# AIAK_TRAINING_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5 \ +# DATA_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset \ +# TOKENIZER_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0 \ +# CHECKPOINT_PATH=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ -echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" -echo "DATA_PATH=${DATA_PATH}" -echo "TOKENIZER_PATH=${TOKENIZER_PATH}" -echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" -echo "SLURM_NODELIST=${SLURM_NODELIST}" +# echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" +# echo "DATA_PATH=${DATA_PATH}" +# echo "TOKENIZER_PATH=${TOKENIZER_PATH}" +# echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" +# echo "SLURM_NODELIST=${SLURM_NODELIST}" +export CUDA_VISIBLE_DEVICES=7 +export GPUS_PER_NODE=1 -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh 1 1 diff --git a/aiak_megatron/megatron/core/datasets/Makefile b/aiak_megatron/megatron/core/datasets/Makefile index e745f523..7e2344c7 100644 --- a/aiak_megatron/megatron/core/datasets/Makefile +++ b/aiak_megatron/megatron/core/datasets/Makefile @@ -7,7 +7,15 @@ LIBEXT = $(shell python3-config --extension-suffix) OUT = $(LIBNAME)$(LIBEXT) SRC = helpers.cpp -default: $(OUT) +# Check if any compiled library exists +EXISTING_SO = $(wildcard helpers_cpp*.so) + +default: +ifneq ($(EXISTING_SO),) + @echo "Using existing compiled library: $(EXISTING_SO)" +else + @$(MAKE) $(OUT) +endif $(OUT): $(SRC) $(CXX) $(CXXFLAGS) $(CPPFLAGS) $< -o $@ diff --git a/aiak_megatron/megatron/core/datasets/utils.py b/aiak_megatron/megatron/core/datasets/utils.py index cb84c89a..abb57fea 100644 --- a/aiak_megatron/megatron/core/datasets/utils.py +++ b/aiak_megatron/megatron/core/datasets/utils.py @@ -22,8 +22,21 @@ def compile_helpers(): """Compile C++ helper functions at runtime. Make sure this is invoked on a single process.""" import os import subprocess - - command = ["make", "-C", os.path.abspath(os.path.dirname(__file__))] + import glob + + # Check if helpers_cpp is already compiled + helpers_dir = os.path.abspath(os.path.dirname(__file__)) + so_files = glob.glob(os.path.join(helpers_dir, "helpers_cpp*.so")) + if so_files: + # Try to import to verify it works + try: + import helpers_cpp + log_single_rank(logger, logging.INFO, f"Using pre-compiled helpers: {so_files[0]}") + return + except ImportError: + pass + + command = ["make", "-C", helpers_dir] if subprocess.run(command).returncode != 0: import sys diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 354e50fc..92daa264 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -1,7 +1,9 @@ -AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5}" #Root of the LLaVA-OneVision training repo -AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" # Megatron-LM fork used (aiak_megatron) inside the training repo. +REPO_ROOT=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5 +echo "$REPO_ROOT" +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" #Root of the LLaVA-OneVision training repo +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" # Megatron-LM fork used (aiak_megatron) inside the training repo. # TP="${1:-1}" # Tensor parallel -TP="${1:-2}" +TP="${1:-1}" PP="${2:-1}" # pipeline parallel # Defaults: TP=1, PP=1. # SEQ_LEN="${3:-32768}" @@ -11,10 +13,17 @@ MBS="${4:-1}" # micro batch size GBS="${5:-8}" # NSTEP="${6:-2500}" # number of training iterations NSTEP="${6:-20}" # number of training iterations -DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} -TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} -CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} - +# DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} +# TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} +# CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} +DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" +TOKENIZER_PATH="${TOKENIZER_PATH:-"$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0"}" +CHECKPOINT_PATH="${CHECKPOINT_PATH:-"$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1"}" +echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" +echo "DATA_PATH=${DATA_PATH}" +echo "TOKENIZER_PATH=${TOKENIZER_PATH}" +echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" +echo "SLURM_NODELIST=${SLURM_NODELIST}" #! /bin/bash # The script needs to be run on at least 1 nodes. @@ -81,7 +90,7 @@ TENSORBOARD_PATH="${SAVE_CKPT_PATH}/tensorboard" mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=2 +GPUS_PER_NODE=${GPUS_PER_NODE:-2} # Change for multinode config MASTER_ADDR=${MASTER_ADDR:-"${list_ip[0]}"} From 8b2f8ce3783e94551e29ffada342ad608a5775a1 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 5 Jan 2026 00:20:57 +0400 Subject: [PATCH 09/39] Add debugging and exploration changes to training pipeline - Added print statements for debugging in dataloader_provider.py - Added print statements in qwen2vl_task_encoder.py for tracing preprocessing - Added debugging prints in llavaov_1_5_provider.py - Added print statements in train.py, megatron_trainer.py, and other training files - These changes were made during codebase exploration and understanding --- .../data/multimodal/dataloader_provider.py | 34 ++-- .../data/multimodal/qwen2vl_task_encoder.py | 151 ++++++++++++++++-- .../llavaov_1_5/llavaov_1_5_provider.py | 65 ++++++-- aiak_training_llm/train.py | 5 +- aiak_training_llm/train/megatron_trainer.py | 16 +- .../train/pretrain/pretrain_llavaov_1_5.py | 51 +++++- .../train/sft/sft_llavaov_1_5_vl.py | 9 +- aiak_training_llm/train/trainer_builder.py | 39 +++-- 8 files changed, 305 insertions(+), 65 deletions(-) diff --git a/aiak_training_llm/data/multimodal/dataloader_provider.py b/aiak_training_llm/data/multimodal/dataloader_provider.py index ba27ac4a..16e7af8b 100644 --- a/aiak_training_llm/data/multimodal/dataloader_provider.py +++ b/aiak_training_llm/data/multimodal/dataloader_provider.py @@ -21,16 +21,21 @@ def get_train_dataset(task_encoder): worker_debug_path=None, worker_log_level=0 ) + #use megatron energon to get the training dataset + # a high performance data loading library for large-scale distributed training + #each gpu gets its shard of the data based on data parallel rank + # Supports packing: Multiple short sequences can be packed into one to maximize GPU utilization + # streaming: can load data on-the-fly without pre-downloading entire dataset into memory train_ds = energon.get_train_dataset( - args.data_path[0], - batch_size=args.micro_batch_size, - task_encoder=task_encoder, - worker_config=worker_config, + args.data_path[0], # Path to dataset (LLaVA-558K-Webdataset) + batch_size=args.micro_batch_size, #Batch size per GPU + task_encoder=task_encoder, # The Qwen2VLTaskEncoder we just created + worker_config=worker_config, #Distributed Config max_samples_per_sequence=None, shuffle_buffer_size=None, - packing_buffer_size=args.packing_batch_size, - handler=print_error_handler, - image_decode="pil", + packing_buffer_size=args.packing_batch_size, #Buffer size for packing sequences + handler=print_error_handler, #Error handler function + image_decode="pil", #Decode images using PIL ) return train_ds @@ -65,11 +70,12 @@ class EnergonDataloader: def __init__(self, dataloader, collator=None): self._dataloader = dataloader self._collator = collator - self._iter = iter(cyclic_iter(dataloader)) + self._iter = iter(cyclic_iter(dataloader)) # Infinite iterator! def __next__(self): - features = self._iter.__next__() + features = self._iter.__next__() #get next batch if self._collator is not None: + # Apply padding using the collator padded = self._collator.tokenizer.pad( {"input_ids": features['tokens']}, padding=self._collator.padding, @@ -77,7 +83,10 @@ def __next__(self): pad_to_multiple_of=self._collator.pad_to_multiple_of, ) paded_length = padded['input_ids'].shape[1] - features['tokens'].shape[1] + + # Update features with padded versions features['tokens'] = padded["input_ids"] + #pad labels and attention mask too features['labels'] = F.pad( features['labels'], (0, paded_length), @@ -85,6 +94,11 @@ def __next__(self): self._collator.label_pad_token_id ) features['attn_mask'] = F.pad(features['attn_mask'], (0, paded_length), "constant", True) + # return a batch + print("Dataloader batch - tokens shape:", features['tokens'].shape) + print("Dataloader batch - labels shape:", features['labels'].shape) + print("Dataloader batch - attn_mask shape:", features['attn_mask'].shape) + print("Dataloader batch - image_grid_thw shape:", features['image_grid_thw'].shape) return features def __iter__(self): @@ -94,7 +108,7 @@ def save_state(self): """ Save the current state of this dataloader """ return self._dataloader.save_state_rank() - +# This means training never runs out of data - it just loops back to the beginning. def cyclic_iter(iter): """ Infinite iteration over an iterator """ while True: diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 9b30a33b..32593197 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -1,4 +1,6 @@ """ Qwen2VLTaskEncoder class.""" +# The Qwen2VLTaskEncoder is responsible for converting raw data samples (images + text conversations) into model-ready tensors. It's called by Megatron Energon during data loading. + import math import re from dataclasses import dataclass, field @@ -113,19 +115,26 @@ class Qwen2VLTaskEncoder(TaskEncoder): def __init__(self, args): super().__init__() if args.training_phase in ['sft']: - self.chat_template = get_chat_template() + self.chat_template = get_chat_template() # Load conversation template + #template for formatting conversations (user/assistant turns) + + # Load HuggingFace processor (tokenizer + image/ video processor from Qwen2-VL) self.processor = AutoProcessor.from_pretrained(self.args.hf_tokenizer_path, trust_remote_code=True) - + print(f"Loaded processor from {self.args.hf_tokenizer_path}") + print(f"Processor config: {self.processor}") + #Resolution parameters for resizing images/videos if args.image_resolution: setattr(self.processor, 'image_resolution', args.image_resolution) - # video + # resolution parameters for resizing images/videos + print("image_resolution:", getattr(self.processor, 'image_resolution', None)) + # Video processing parameters self.frame_min_pixels = args.frame_min_pixels self.frame_max_pixels = args.frame_max_pixels self.video_max_pixels = args.video_max_pixels self.fps = args.fps self.fps_min_frames = args.fps_min_frames self.fps_max_frames = args.fps_max_frames - # image + # Image processing parameters self.min_pixels = args.min_pixels self.max_pixels = args.max_pixels @@ -160,16 +169,38 @@ def _reisize_video(self, vision: VideoData, image_factor=28, frame_factor=2): return video def _resize_image(self, image, size_factor=28): + #calculate optimal size using smart_resize + # constraints: + # 1- width and height must be multiple of size_factor (28) + # 2- Total pixels (height × width) are ≥ min_pixels + # 3- Total pixels (height × width) are ≤ max_pixels + # 4- Aspect ratio is preserved as much as possible + resized_height, resized_width = smart_resize( image.height, image.width, - factor=size_factor, - min_pixels=self.min_pixels, - max_pixels=self.max_pixels, + factor=size_factor, + # why factor of 28? + # Input Image + # ↓ + # Split into 14×14 patches (Patch Embedding) + # ↓ + # 2×2 patch merging (Reduce spatial dimensions by 2) + # ↓ + # Effective patch size = 14 × 2 = 28×28 pixels + # Example: + + # Image size: 1120 × 784 pixels + # Number of patches: (1120/28) × (784/28) = 40 × 28 = 1,120 patches + # Each patch → 1 vision token + min_pixels=self.min_pixels, # e.g., 256*28*28 + max_pixels=self.max_pixels, # e.g., 1280*28*28 ) + print(f"Original image size: {image.width}x{image.height}") image = image.resize((resized_width, resized_height)) + print(f"Resized image to {resized_width}x{resized_height}") - return image + return image # return resized PIL image def _process(self, image, text): """" Process the data to get the model's input """ @@ -226,6 +257,12 @@ def process_sft_vqa(self, context, answer, image): def process_sft_qa(self, messages: list, system: str, raw_video: list, raw_image: list): """ process the data for sft qa """ + # messages: List of conversation turns (user/assistant) + # system: System prompt + # raw_video: List of PIL/VideoData objects (or None) + # raw_image: List of PIL.Image objects (or None) + + #Initialize output containers video_grid_thw = None pixel_values_videos = [] image_grid_thw = None @@ -233,22 +270,43 @@ def process_sft_qa(self, messages: list, system: str, raw_video: list, raw_image video = [] image = [] - + # resize image if raw_image is not None: + # loop through each image and resize for i in raw_image: image.append(self._resize_image(i)) - + # resize video if raw_video is not None: for v in raw_video: video.append(self._reisize_video(v)) + + # input messages: + # [ + # {"from": "human", "value": "\nWhat's in this image?"}, + # {"from": "gpt", "value": "A cat sitting on a mat."} + # ] messages, mm_inputs = self.chat_template.mm_plugin.process_messages( messages, image if image is not None else [], video if raw_video is not None else [], self.processor ) + # output messages: + # [ + # {"from": "human", "value": "<|vision_start|><|image_pad|><|image_pad|>...<|vision_end|>\nWhat's in this image?"}, + # {"from": "gpt", "value": "A cat sitting on a mat."} + # ] + # Output mm_inputs: + # { + # "pixel_values": Tensor([1, 1176, 1176]), # Vision encoder input + # "image_grid_thw": Tensor([[1, 28, 42]]), # 1 temporal, 28 height patches, 42 width patches + # } + + # assert raw_image is not None, f'No image found in {messages}' 确实有纯文本对话 + + #extracting the multi-modal inputs if raw_video is not None: video_grid_thw = mm_inputs["video_grid_thw"] pixel_values_videos = [mm_inputs["pixel_values_videos"]] @@ -257,21 +315,73 @@ def process_sft_qa(self, messages: list, system: str, raw_video: list, raw_image pixel_values_images = [mm_inputs["pixel_values"]] encode_pairs = self.chat_template.encode_multiturn( + # 1. Applies chat template to format conversation: + # example: + # <|im_start|>system + # You are a helpful assistant.<|im_end|> + # <|im_start|>user + # <|vision_start|><|image_pad|><|image_pad|>...<|vision_end|> + # What's in this image?<|im_end|> + # <|im_start|>assistant + # A cat sitting on a mat.<|im_end|> + + # 2. Tokenizes the entire formatted conversation + + # 3. Splits into (source, target) pairs for each turn: + # example: + # encode_pairs = [ + # ( + # [151644, 8948, 198, ...], # source_ids: system + user prompt (don't train) + # [8122, 4758, 6134, ...] # target_ids: assistant response (train on this) + # ) + # ] + + tokenizer=self.tokenizer, messages=messages, system=system, ) + input_ids, target = [], [] for turn_idx, (source_ids, target_ids) in enumerate(encode_pairs): - input_ids += source_ids + target_ids + input_ids += source_ids + target_ids # Append both source and target to input_ids + # input_ids = [151644, 8948, ..., 151645, 8122, 4758, ...] + # ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ + # system + user assistant response + + # Mask source (user prompt), keep target (assistant response) target += [IGNORE_INDEX] * len(source_ids) + target_ids - input_ids = torch.tensor(input_ids) + # target = [-100, -100, ..., -100, 8122, 4758, ...] + # ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ + # Masked (don't train on) Train on this! + # Why mask source_ids? + # We don't want the model to learn to predict the user's input + # only the assistant's response! + + # Convert to Tensors and Create Attention Mask + input_ids = torch.tensor(input_ids) # Shape: [seq_len] + # input_ids: Tensor([151644, 8948, 198, ..., 8122, 4758, ...]) + print("shape of input_ids:", input_ids.shape) target = torch.tensor(target) + # target: Tensor([-100, -100, ..., 8122, 4758, ...]) + + # Create attention mask (all False = attend to all tokens) attn_mask = torch.zeros_like(input_ids).bool() + # attn_mask: Tensor([False, False, False, ..., False, False]) # Shape: [seq_len] + + return input_ids, target, attn_mask, pixel_values_images, image_grid_thw, \ pixel_values_videos, video_grid_thw + # input_ids: Tokenized conversation - Shape: [seq_len] + # target: Labels with masking - Shape: [seq_len] + # attn_mask: Attention mask - Shape: [seq_len] + # pixel_values_images: Image tensors - List of Tensors + # image_grid_thw: Image grid info - Tensor [[1, 28, 42]] + # pixel_values_videos: Video tensors - Empty list + # video_grid_thw: Video grid info - None + def encode_captioning(self, sample: CaptioningSample) -> ImageTaskSample: """Encode CaptioningSample.""" @@ -357,6 +467,7 @@ def encode_vqa4packing(self, sample: VQASample) -> ImageTaskSample: def encode_multi_vid_qa(self, sample: VQASample) -> ImageTaskSample: """Encode sample in Qwen2VL style.""" if self.args.training_phase == constants.TrainingPhase.SFT: + # call main processing function process_sft_qa input_ids, target, attn_mask, imgs, image_grid_thw, video, video_grid_thw = \ self.process_sft_qa(sample.messages, sample.system, sample.video, None) else: @@ -384,12 +495,14 @@ def encode_multi_vid_qa(self, sample: VQASample) -> ImageTaskSample: total_len=len(input_ids), ) - + # Energon automatically calls the appropriate encode method based on sample type. + # For SFT with multi-modal data, it calls: def encode_multi_mix_qa(self, sample: MultiMixQASample) -> ImageTaskSample: """Encode sample in Qwen2VL style.""" if self.args.training_phase == constants.TrainingPhase.SFT: - num_tiles = [] - + num_tiles = [] #store number of tiles for each image/ video after processing + + # call main processing function process_sft_qa input_ids, target, attn_mask, imgs, image_grid_thw, pixel_values_videos, video_grid_thw = \ self.process_sft_qa(sample.messages, sample.system, sample.video, sample.image) if sample.video is not None: @@ -557,12 +670,18 @@ def pack_selected_samples(self, samples: List[Qwen2VLImageTaskSample]) -> List[Q ) @override + # After encoding , multiple encoded samples are batched together def batch(self, samples: List[Union[Qwen2VLImageTaskSample, Qwen2VLImageTaskSamplePacked]]) \ -> Qwen2VLImageTaskBatchPacked: """ Batch samples together """ image_grid_thw, video_grid_thw = self.process_samples_grid(samples) + # Create batch return Qwen2VLImageTaskBatchPacked( - super().batch(samples), + super().batch(samples), # # Stacks tokens, labels, etc. with padding + # Pads tokens to same length → [batch_size, max_seq_len] + # Pads labels to same length → [batch_size, max_seq_len] + # Concatenates imgs → [total_images_in_batch, C, H, W] + # Creates batch attention masks image_grid_thw=image_grid_thw, video_grid_thw=video_grid_thw ) diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index f1d83929..a39fc215 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -17,7 +17,7 @@ from .llavaov_1_5_model import LlavaOnevision1_5 - +# model provider registration @register_model_provider(model_family=[VisionLanguageModelFamilies.LLAVA_OV_1_5]) def rice_vl_model_provider( pre_process: bool = True, @@ -41,32 +41,44 @@ def rice_vl_model_provider( print_rank_0(f'building {args.model_name} model ...') - config = build_transformer_config(args) + config = build_transformer_config(args) #base transformer config with all hyperparams + + language_config = deepcopy(config) # For Qwen2.5 language model + vision_config = deepcopy(config) # For vision encoder (SigLIP) + adapter_config = deepcopy(config) ## For adapter (projection) - language_config = deepcopy(config) - vision_config = deepcopy(config) - adapter_config = deepcopy(config) + # Vision Encoder → Adapter → Language Model + # (SigLIP) (Projection) (Qwen2.5) from aiak_training_llm.models import get_model_family model_family = get_model_family(args.model_name) + # get vision specific config : no. of layers, hidden size, Patch size, Image resolution for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): setattr(vision_config, k, v) + print_rank_0(f'Vision config: {vision_config}') + # get adapter specific config : Projection dimension, Activation function for k, v in asdict(get_adapeter_config(model_family)).items(): setattr(adapter_config, k, v) - # print(vision_config) + print_rank_0(f'Adapter config: {adapter_config}') + + # set special token ids for language model setattr(language_config, "image_token_id", 151655) setattr(language_config, "video_token_id", 151656) + #Handle pipeline parallelism + # FIXME: fix this if model_type is encoder_and_decoder if args.encoder_pipeline_model_parallel_size in [0, None]: + #UNIFIED MODEL (TP=1, PP=1) vision_config.pipeline_model_parallel_size = 1 vision_config.tensor_model_parallel_size = 1 vision_config.sequence_parallel = False vision_config.tp_comm_overlap = False vision_config.context_parallel_size = 1 vision_config.context_parallel_ulysses_degree = 1 - add_encoder = mpu.is_pipeline_first_stage() - add_decoder = True + + add_encoder = mpu.is_pipeline_first_stage() #True on first stage + add_decoder = True #Always add language model decoder else: assert ( args.encoder_pipeline_model_parallel_size == 1 @@ -93,6 +105,23 @@ def rice_vl_model_provider( vision_layer_spec = get_vision_layer_with_spec() language_layer_spec = get_qwen_layer_with_te_spec(language_config) +# # Vision layer spec (Transformer block) +# - MultiheadAttention +# - LayerNorm +# - MLP (feedforward) +# - Residual connections + +# # Language layer spec (Qwen2.5 block) +# - MultiQueryAttention or GroupedQueryAttention +# - RMSNorm +# - SwiGLU MLP +# - RoPE (Rotary Position Embedding) + +# # Adapter spec (projection) +# - Linear layer(s) +# - Optional activation + +#create the model model = LlavaOnevision1_5( language_config=language_config, vision_config=vision_config, @@ -102,19 +131,24 @@ def rice_vl_model_provider( adapter_layer_spec=adapter_layer_spec, language_vocab_size=args.padded_vocab_size, language_max_sequence_length=args.max_position_embeddings, - pre_process=pre_process, - post_process=post_process, - add_encoder=add_encoder, - add_decoder=add_decoder, + pre_process=pre_process, #compute embeddings? + post_process=post_process, #compute output logits/loss? + add_encoder=add_encoder, #add vision encoder? + add_decoder=add_decoder, #add language model decoder? fp16_lm_cross_entropy=args.fp16_lm_cross_entropy, parallel_output=parallel_output, share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights, - language_position_embedding_type=args.position_embedding_type, + language_position_embedding_type=args.position_embedding_type, #"rope" language_rotary_percent=args.rotary_percent, language_rotary_base=args.rotary_base, seq_len_interpolation_factor=args.rotary_seq_len_interpolation_factor, ) + # Vision encoder: SigLIP with 27 layers + # Adapter: Projection network + # Language model: Qwen2.5 with 32 layers + # All distributed training wrappers (TP, PP, DP) + #freeze components if needed if args.trainable_modules != ['all']: train_language_model = "language_model" in args.trainable_modules train_vision_model = "vision_model" in args.trainable_modules @@ -122,5 +156,10 @@ def rice_vl_model_provider( model.freeze(freeze_language_model=not train_language_model, freeze_vision_model=not train_vision_model, freeze_adapter=not train_adapter) + # Stage 0 (Pre-training): Only train adapter + # trainable_modules = ["adapter"] + # → Freeze vision encoder ✓ + # → Freeze language model ✓ + # → Train adapter ✓ return model diff --git a/aiak_training_llm/train.py b/aiak_training_llm/train.py index 2453f3a0..1cd1ba84 100644 --- a/aiak_training_llm/train.py +++ b/aiak_training_llm/train.py @@ -9,10 +9,11 @@ def main(): # parse args args = parse_train_args() - + print("Building model trainer...") # get model trainer trainer = build_model_trainer(args) - + print(trainer) # + print("Starting training...") # start training trainer.train() diff --git a/aiak_training_llm/train/megatron_trainer.py b/aiak_training_llm/train/megatron_trainer.py index b98b62cd..f23f8608 100644 --- a/aiak_training_llm/train/megatron_trainer.py +++ b/aiak_training_llm/train/megatron_trainer.py @@ -43,15 +43,17 @@ def __init__( def train(self): """start training""" + # Mark dataset provider as distributed (multi-GPU) self.train_valid_test_datasets_provider.is_distributed = True pretrain( - train_args=self.train_args, + train_args=self.train_args, #All training arguments (like batch size, lr, etc) train_valid_test_dataset_provider=self.train_valid_test_datasets_provider, - model_provider=self.model_provider, - model_type=self.model_type, - forward_step_func=self.forward_step_func, - process_non_loss_data_func=self.process_non_loss_data_func, - get_embedding_ranks=self.get_embedding_ranks, - get_position_embedding_ranks=self.get_position_embedding_ranks, + # function that provides train/valid/test datasets (preprocessing handled inside) + model_provider=self.model_provider, #function that provides the model architecture + model_type=self.model_type, #type of model being trained (e.g., language model, vision model, etc)(encoder or decoder) + forward_step_func=self.forward_step_func, #function that computes loss for one batch + process_non_loss_data_func=self.process_non_loss_data_func, #Optional function for logging/visualization + get_embedding_ranks=self.get_embedding_ranks, #for distributed embedding tables + get_position_embedding_ranks=self.get_position_embedding_ranks, # for distributed position embeddings ) diff --git a/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py b/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py index 4f065894..c9fe9136 100644 --- a/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py +++ b/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py @@ -58,7 +58,7 @@ def qwen2vl_position_embedding_ranks(pp_ranks): Args: pp_ranks: A list of global ranks that constitute a pipeline group. """ - args = get_args() + args = get_args() #get training arguments # encoder size is also the index to the first rank of the decoder. epp = args.encoder_pipeline_model_parallel_size or 0 @@ -81,9 +81,14 @@ def model_provider(pre_process=True, post_process=True, add_encoder=True, add_de MCoreModel: The returned model """ args = get_args() + # Get model family: "llava-ov-1.5-4b" → "llava_ov_1_5" model_family = get_model_family(args.model_name) + # Lookup the registered provider model_provider = get_model_provider(model_family) + # = MODEL_FAMILY_TO_PROVIDER["llava_ov_1_5"] + # = rice_vl_model_provider assert model_provider is not None, f'model provider for {args.model_name} not found' + # call the provider to build the model and return the model return model_provider(pre_process, post_process, add_encoder, add_decoder) @@ -235,7 +240,7 @@ def loss_func(loss_mask: torch.Tensor, output_tensor: torch.Tensor): loss_reduced_dict ) - +#called by megatron training loop during each training step (called once per micro batch during training) def forward_step(data_iterator, model): """Forward training step. @@ -250,14 +255,28 @@ def forward_step(data_iterator, model): global stimer with stimer(bdata=True): + #get batch from data iterator images, image_grid_thw, pixel_values_videos, video_grid_thw, \ input_ids, position_ids, attention_mask, \ labels, loss_mask, attn_mask_type, packed_seq_params \ = get_batch(data_iterator) + #returns: + # images: Tensor([num_images, C, H, W]) # Processed image pixels + # image_grid_thw: Tensor([num_images, 3]) # Grid dimensions [t, h, w] + # pixel_values_videos: Tensor or None # Video frames (if present) + # video_grid_thw: Tensor or None # Video grid dimensions + # input_ids: Tensor([batch, seq_len]) # Token IDs (with <|image_pad|>) + # position_ids: Tensor([batch, seq_len]) or None # Position indices + # attention_mask: Tensor([batch, seq_len]) # Attention mask (False=attend, True=mask) + # labels: Tensor([batch, seq_len]) # Target labels for loss + # loss_mask: Tensor([batch, seq_len]) # Which tokens to compute loss on + # attn_mask_type: AttnMaskType # Causal or padding_causal + # packed_seq_params: PackedSeqParams or None # For packed sequences timers('batch-generator').stop() with stimer: + # MODEL FORWARD PASS output_tensor = model( images, image_grid_thw, @@ -273,16 +292,40 @@ def forward_step(data_iterator, model): return output_tensor, partial(loss_func, loss_mask) - +# Factory function that creates and returns train/valid/test data loaders for the Megatron trainer. def train_valid_test_dataset_provider(train_val_test_num_samples): """ Provides the datasets used by the trainer """ args = get_args() + # create task encoder task_encoder = Qwen2VLTaskEncoder(args) + # Processes images/videos using processor + # Applies chat template to conversations + # Tokenizes text + # Creates attention masks + # Handles vision token placement (<|image_pad|>, <|video_pad|>) + # Packs sequences efficiently + + #get train dataset train_dataset = get_train_dataset(task_encoder) + + #Purpose: Combines multiple samples into a batch + # Input: List of individual samples (each with different sequence lengths) + # Output: Batched tensors with padding collator = build_sft_data_collator(DataCollatorForSeq2Seq) + # Example: + # Sample 1: [151652, 8932, 1234, ...] # Length 50 + # Sample 2: [151652, 9821, 5567, ...] # Length 120 + # ↓ (collate) + # Batch: [[151652, 8932, 1234, ..., , ], # Padded to 120 + # [151652, 9821, 5567, ..., ...., ....]] # Already 120 + + #create dataloader train_dataloader = get_train_loader(train_dataset, collator) - return train_dataloader, None, None + + return train_dataloader, None, None + # For SFT, typically only training loop is needed + # valid_iterator and test_iterator are set to None @register_model_trainer( diff --git a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py index 5f1cee1e..0105eb5f 100644 --- a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py +++ b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py @@ -249,16 +249,21 @@ def train_valid_test_datasets_provider(train_val_test_num_samples): return train_iter, valid_iter, test_iter - +# This registers the function for model family "llava_ov_1_5" and phase "sft" @register_model_trainer(model_family=[constants.VisionLanguageModelFamilies.LLAVA_OV_1_5], training_phase=constants.TrainingPhase.SFT) def default_pretrain_trainer(train_args): """build trainer""" from aiak_training_llm.train.pretrain import pretrain_llavaov_1_5 + #imports the module containing model/dataset/forward functions for LLaVA-OV-1.5 if train_args.encoder_pipeline_model_parallel_size in [0, None]: - model_type = ModelType.encoder_or_decoder + #check if model use pipeline parallelism with separate encoder and decoder stages + model_type = ModelType.encoder_or_decoder # unified model + print_rank_0("Using unified model type for training.") else: model_type = ModelType.encoder_and_decoder + print_rank_0("Using encoder-and-decoder model type for training.") + #created megatron trainer instance trainer = MegatronTrainer( train_args=train_args, train_valid_test_dataset_provider=pretrain_llavaov_1_5.train_valid_test_dataset_provider, diff --git a/aiak_training_llm/train/trainer_builder.py b/aiak_training_llm/train/trainer_builder.py index a5af5bea..565bf15d 100644 --- a/aiak_training_llm/train/trainer_builder.py +++ b/aiak_training_llm/train/trainer_builder.py @@ -7,7 +7,7 @@ MODEL_FAMILY_TRAINER_FACTORY = {} - +# decorator factory to register model trainers def register_model_trainer(model_family: Union[str, List[str]], training_phase: str, training_func: Callable = None): """ register model training function @@ -19,23 +19,27 @@ def register_model_trainer(model_family: Union[str, List[str]], training_phase: training_phase: need to be consistent with the --training-phase definition in train.arguments trainig_func: training function. """ + # add to the factory def _add_trainer(families, phase, func): - if not isinstance(families, list): + #convert to list if single string + if not isinstance(families, list): families = [families] + #loop through all families for _family in families: - _family = _family.lower() + _family = _family.lower() # make it case-insensitive + #create entry for this family if not exist if _family not in MODEL_FAMILY_TRAINER_FACTORY: MODEL_FAMILY_TRAINER_FACTORY[_family] = {} - + #check for duplicate registration (family+phase) if phase in MODEL_FAMILY_TRAINER_FACTORY[_family]: raise ValueError(f"Cannot register duplicate trainer ({_family} family, {phase} phase)") - + #register the trainer function MODEL_FAMILY_TRAINER_FACTORY[_family][phase] = func def _register_function(fn): _add_trainer(model_family, training_phase, fn) - return fn + return fn #fn is the training function being decorated if training_func is not None: return _add_trainer(model_family, training_phase, training_func) @@ -43,19 +47,32 @@ def _register_function(fn): return _register_function -def build_model_trainer(args): +def build_model_trainer(args): # all args including model name and training phase """create model trainer""" # get model family name - model_family = get_model_family(args.model_name) - + #args.model_name="llava-ov-1.5-4b" in our case + model_family = get_model_family(args.model_name) #map model name to model family + print(f"Model family: {model_family}") + #returns family name like model_family="llava_ov_1_5" in our case # get model family trainer + + #check if this model family has a registered trainer + # MODEL_FAMILY_TRAINER_FACTORY is a nested dictionary ={family:{phase:trainer_func}} if model_family not in MODEL_FAMILY_TRAINER_FACTORY: raise ValueError(f"Not found trainer for {args.model_name} (family: {model_family})") - + # check if this training phase has a registered trainer for this model family + # args.training_phase="sft" in our case + print(f"Training phase: {args.training_phase}") if args.training_phase not in MODEL_FAMILY_TRAINER_FACTORY[model_family]: raise ValueError(f"AIAK not support {args.training_phase} phase for {args.model_name} (family: {model_family})") + #retrieve the trainer function for this model family and training phase from the factory + # MODEL_FAMILY_TRAINER_FACTORY[llava_ov_1_5][sft] -> returns the trainer function trainer = MODEL_FAMILY_TRAINER_FACTORY[model_family][args.training_phase] - return trainer(args) + # This retrieves the default_pretrain_trainer function defined in sft_llavaov_1_5_vl.py + return trainer(args) # Calls default_pretrain_trainer(args) + # returns a trainer object for the specified model and training phase + #trainer(args) calls the trainer function with args. + #trainer function creates a megatron trainer object and returns it. From 1aba2417b58523428f751ec9693cbc41f23ed4b4 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 8 Jan 2026 12:23:35 +0400 Subject: [PATCH 10/39] Integrate FastViT vision encoder replacing RICE-ViT - Add FastViT model implementation (mobileclip_l_384) in aiak_training_llm/models/fastvit/ - Update LlavaOnevision1_5 model to use FastViT encoder - Add FastViT preprocessing in qwen2vl_task_encoder.py - Add --use-fastvit and related command-line arguments - Add checkpoint conversion scripts for FastVLM - Update training configs for 2-GPU setup (TP=2) - Add .gitignore entries for checkpoints and training outputs --- .gitignore | 3 + Stage1/alignment.sh | 11 +- .../megatron/training/wandb_utils.py | 4 +- .../data/multimodal/qwen2vl_task_encoder.py | 105 +- aiak_training_llm/models/fastvit/__init__.py | 10 + aiak_training_llm/models/fastvit/builder.py | 19 + aiak_training_llm/models/fastvit/constants.py | 13 + .../models/fastvit/fastvit_preprocessor.py | 69 + .../models/fastvit/fastvit_vision_model.py | 98 ++ aiak_training_llm/models/fastvit/mm_utils.py | 250 +++ .../models/fastvit/mobileclip/__init__.py | 87 + .../mobileclip/configs/mobileclip_l.json | 20 + .../models/fastvit/mobileclip/mci.py | 1478 +++++++++++++++++ .../models/fastvit/mobileclip_encoder.py | 116 ++ .../models/llavaov_1_5/adapter.py | 2 + .../models/llavaov_1_5/llavaov_1_5_config.py | 3 +- .../models/llavaov_1_5/llavaov_1_5_model.py | 15 +- .../llavaov_1_5/llavaov_1_5_provider.py | 9 + aiak_training_llm/train/arguments.py | 16 + aiak_training_llm/train/training_utils.py | 10 + .../convert/convert_4b_mcore_to_hf.sh | 62 +- .../convert/convert_fastvlm_hf_to_mcore.sh | 89 + .../stage_1_alignment_llava_ov_4b.sh | 11 +- requirements.local.txt | 125 ++ .../config/llava-ov-1.5-4b/fastvit.json | 8 + .../custom/llavaov_1_5/fastvit.py | 96 ++ .../llavaov_1_5/merge_megatron_fastvlm.py | 89 + 27 files changed, 2771 insertions(+), 47 deletions(-) create mode 100644 aiak_training_llm/models/fastvit/__init__.py create mode 100644 aiak_training_llm/models/fastvit/builder.py create mode 100644 aiak_training_llm/models/fastvit/constants.py create mode 100644 aiak_training_llm/models/fastvit/fastvit_preprocessor.py create mode 100644 aiak_training_llm/models/fastvit/fastvit_vision_model.py create mode 100644 aiak_training_llm/models/fastvit/mm_utils.py create mode 100644 aiak_training_llm/models/fastvit/mobileclip/__init__.py create mode 100644 aiak_training_llm/models/fastvit/mobileclip/configs/mobileclip_l.json create mode 100644 aiak_training_llm/models/fastvit/mobileclip/mci.py create mode 100644 aiak_training_llm/models/fastvit/mobileclip_encoder.py create mode 100755 examples/llava_ov_1_5/convert/convert_fastvlm_hf_to_mcore.sh create mode 100644 requirements.local.txt create mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json create mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py create mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py diff --git a/.gitignore b/.gitignore index 00186b4c..11a185b6 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,6 @@ coverage.xml **/.ipynb_checkpoints/ .vscode/ checkpoints/ +stage_1_alignment_llava_ov_4b/ +tmp/ +Stage1/training_output.log diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index d751fa49..bd66fdd8 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -51,7 +51,12 @@ export LD_LIBRARY_PATH="/usr/local/cuda-12.1/lib64:/usr/local/cuda-12.1/targets/ # echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" # echo "SLURM_NODELIST=${SLURM_NODELIST}" -export CUDA_VISIBLE_DEVICES=7 -export GPUS_PER_NODE=1 +# Weights & Biases configuration +export WANDB_API_KEY="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" +export WANDB_PROJECT="llava-ov-1_5" +export WANDB_NAME="fastvit_integration" -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh 1 1 +export CUDA_VISIBLE_DEVICES=0,1 +export GPUS_PER_NODE=2 + +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh 2 1 diff --git a/aiak_megatron/megatron/training/wandb_utils.py b/aiak_megatron/megatron/training/wandb_utils.py index ed3a55aa..ff339613 100644 --- a/aiak_megatron/megatron/training/wandb_utils.py +++ b/aiak_megatron/megatron/training/wandb_utils.py @@ -34,7 +34,9 @@ def on_save_checkpoint_success(checkpoint_path: str, tracker_filename: str, save metadata = {"iteration": iteration} artifact_name, artifact_version = _get_artifact_name_and_version(Path(save_dir), Path(checkpoint_path)) artifact = wandb_writer.Artifact(artifact_name, type="model", metadata=metadata) - artifact.add_reference(f"file://{checkpoint_path}", checksum=False) + # Convert to absolute path for W&B + abs_checkpoint_path = Path(checkpoint_path).resolve() + artifact.add_reference(f"file://{abs_checkpoint_path}", checksum=False) artifact.add_file(tracker_filename) wandb_writer.run.log_artifact(artifact, aliases=[artifact_version]) wandb_tracker_filename = _get_wandb_artifact_tracker_filename(save_dir) diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 32593197..32e9c69d 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -17,6 +17,15 @@ from torchvision.transforms import InterpolationMode from transformers import AutoProcessor from typing_extensions import override +#Fast ViT imports +# Custom Image processor for FastViT +from aiak_training_llm.models.fastvit.fastvit_preprocessor import FastViTImageProcessor +# Utilities for FastViT multimodal image preprocessing +from aiak_training_llm.models.fastvit.mm_utils import ( + expand2square, # pads image to square with background color + process_anyres_image, # handles variable resolution with patches + process_images, #Main dispatcher based on aspect ratio mode +) from aiak_training_llm.data.multimodal import MultiMixQASample from aiak_training_llm.data.multimodal.length_sort_dataset import LengthPoolSortDataset @@ -114,6 +123,7 @@ class Qwen2VLTaskEncoder(TaskEncoder): def __init__(self, args): super().__init__() + self.args = args if args.training_phase in ['sft']: self.chat_template = get_chat_template() # Load conversation template #template for formatting conversations (user/assistant turns) @@ -122,6 +132,15 @@ def __init__(self, args): self.processor = AutoProcessor.from_pretrained(self.args.hf_tokenizer_path, trust_remote_code=True) print(f"Loaded processor from {self.args.hf_tokenizer_path}") print(f"Processor config: {self.processor}") + + # FastViT image processor (following FastVLM repo) + # Use this for vision encoding instead of Qwen2-VL's processor + self.use_fastvit = getattr(args, 'use_fastvit', False) + if self.use_fastvit: + fastvit_image_size = getattr(args, 'fastvit_image_size', 384) + self.fastvit_processor = FastViTImageProcessor(image_size=fastvit_image_size) + print(f"Initialized FastViT processor with image_size={fastvit_image_size}") + #Resolution parameters for resizing images/videos if args.image_resolution: setattr(self.processor, 'image_resolution', args.image_resolution) @@ -169,6 +188,19 @@ def _reisize_video(self, vision: VideoData, image_factor=28, frame_factor=2): return video def _resize_image(self, image, size_factor=28): + """ + Resize image based on vision encoder type. + - For FastViT: Use FastVLM's preprocessing (pad/anyres) + - For Rice/SigLIP: Dynamic resize with smart_resize + """ + if self.use_fastvit: + # FastViT: Use FastVLM's preprocessing approach + # For single image, we'll handle aspect ratio in _process + # Just return the PIL image as-is for now + print(f"FastViT: Keeping original image size {image.width}x{image.height} for aspect ratio handling") + return image + + # Original Rice/SigLIP preprocessing #calculate optimal size using smart_resize # constraints: # 1- width and height must be multiple of size_factor (28) @@ -204,6 +236,59 @@ def _resize_image(self, image, size_factor=28): def _process(self, image, text): """" Process the data to get the model's input """ + + if self.use_fastvit and image is not None: + # FastViT preprocessing using FastVLM's approach + # Tokenize text only + text_inputs = self.processor.tokenizer( + text=text, + padding=True, + return_tensors="pt", + ) + input_ids = text_inputs['input_ids'][0] + attn_mask = text_inputs['attention_mask'][0].logical_not() + + # Process image with FastVLM's preprocessing utilities + # Default to 'pad' aspect ratio (expand to square with padding) + image_aspect_ratio = getattr(self.args, 'image_aspect_ratio', 'pad') + + if image_aspect_ratio == 'pad': + # Expand to square with mean color padding + mean_color = tuple(int(x * 255) for x in self.fastvit_processor.image_mean) + image = expand2square(image, mean_color) + pixel_values = self.fastvit_processor(image) + elif image_aspect_ratio == 'anyres': + # Process with variable resolution (patches) + grid_pinpoints = getattr(self.args, 'image_grid_pinpoints', '[(384, 384), (768, 384), (384, 768), (768, 768)]') + pixel_values = process_anyres_image( + image, + self.fastvit_processor.processor, # CLIPImageProcessor + grid_pinpoints + ) + else: + # Direct resize to target size + pixel_values = self.fastvit_processor(image) + + pixel = [pixel_values] + + # FastViT: Set grid_thw to represent 1 tile (no dynamic grid) + # Format: tensor([[num_tiles, height, width]]) + image_grid_thw = torch.tensor([[1, 1, 1]]) + + # Create target tensor (same as Qwen2-VL path) + target = input_ids.clone() + vision_start_id, img_pad_id, vision_end_id = self.tokenizer.convert_tokens_to_ids([ + VISION_TAGS[0], + IMAGE_TOKEN, + VISION_TAGS[1] + ]) + target[target == vision_start_id] = IGNORE_INDEX + target[target == img_pad_id] = IGNORE_INDEX + target[target == vision_end_id] = IGNORE_INDEX + + return input_ids, target, pixel, image_grid_thw, attn_mask + + # Original Qwen2-VL processing inputs = self.processor( text=text, images=image, @@ -395,11 +480,11 @@ def encode_captioning(self, sample: CaptioningSample) -> ImageTaskSample: text = IMAGE_TOKEN_WITH_TAGS + sample.caption + self.tokenizer.tokenizer.eos_token input_ids, target, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) - num_tiles = [len(image_grid_thw)] + num_tiles = [len(image_grid_thw)] if image_grid_thw is not None else [1] if self.args.enable_discard_sample: assert len(input_ids) <= self.args.seq_length, f"{sample.__key__} input length {len(input_ids)}" - else: + elif image_grid_thw is not None: assert image_grid_thw.prod() / 4 <= self.args.seq_length, f"{sample.__key__} thw {image_grid_thw}" return Qwen2VLImageTaskSample( @@ -443,10 +528,10 @@ def encode_vqa4packing(self, sample: VQASample) -> ImageTaskSample: target[-1] = input_ids[-1] # print(target[-1]) - num_tiles = [len(image_grid_thw)] + num_tiles = [len(image_grid_thw)] if image_grid_thw is not None else [1] if self.args.enable_discard_sample: assert len(input_ids) <= self.args.seq_length, f"{sample.__key__} input length {len(input_ids)}" - else: + elif image_grid_thw is not None: assert image_grid_thw.prod() / 4 <= self.args.seq_length, f"{sample.__key__} grid_thw: {image_grid_thw}" return Qwen2VLImageTaskSample( @@ -475,7 +560,7 @@ def encode_multi_vid_qa(self, sample: VQASample) -> ImageTaskSample: if self.args.enable_discard_sample: assert len(input_ids) <= self.args.seq_length, f"{sample.__key__} input length {len(input_ids)}" - else: + elif video_grid_thw is not None: assert video_grid_thw.prod(dim=-1).sum() / 4 <= self.args.seq_length, \ f"{sample.__key__} grid_thw: {video_grid_thw}" @@ -506,9 +591,9 @@ def encode_multi_mix_qa(self, sample: MultiMixQASample) -> ImageTaskSample: input_ids, target, attn_mask, imgs, image_grid_thw, pixel_values_videos, video_grid_thw = \ self.process_sft_qa(sample.messages, sample.system, sample.video, sample.image) if sample.video is not None: - num_tiles = [len(video_grid_thw)] + num_tiles = [len(video_grid_thw)] if video_grid_thw is not None else [1] elif sample.image is not None: - num_tiles = [len(image_grid_thw)] + num_tiles = [len(image_grid_thw)] if image_grid_thw is not None else [1] else: raise NotImplementedError(f"Unknown training phase {self.args.training_phase}") @@ -518,10 +603,10 @@ def encode_multi_mix_qa(self, sample: MultiMixQASample) -> ImageTaskSample: if self.args.enable_discard_sample: assert len(input_ids) <= self.args.seq_length, f"{sample.__key__} input length {len(input_ids)}" - elif sample.video is not None: + elif sample.video is not None and video_grid_thw is not None: assert video_grid_thw.prod(dim=-1).sum() / 4 <= self.args.seq_length, \ f"{sample.__key__} grid_thw: {video_grid_thw}" - elif sample.image is not None: + elif sample.image is not None and image_grid_thw is not None: assert image_grid_thw.prod(dim=-1).sum() / 4 <= self.args.seq_length, \ f"{sample.__key__} grid_thw: {image_grid_thw}" @@ -620,7 +705,7 @@ def encode_vaq(self, sample: VQASample) -> ImageTaskSample: else: raise NotImplementedError(f"Unknown training phase {self.args.training_phase}") - num_tiles = [len(image_grid_thw)] + num_tiles = [len(image_grid_thw)] if image_grid_thw is not None else [1] if self.args.enable_discard_sample: assert len(input_ids) <= self.args.seq_length, f"{sample.__key__} input length {len(input_ids)}" diff --git a/aiak_training_llm/models/fastvit/__init__.py b/aiak_training_llm/models/fastvit/__init__.py new file mode 100644 index 00000000..04e6a56c --- /dev/null +++ b/aiak_training_llm/models/fastvit/__init__.py @@ -0,0 +1,10 @@ +# +# For licensing see accompanying LICENSE file. +# Copyright (C) 2025 Apple Inc. All Rights Reserved. +# + +from . import mobileclip +from .mobileclip_encoder import MobileCLIPVisionTower +from .fastvit_vision_model import FastViTModel + +__all__ = ['mobileclip', 'MobileCLIPVisionTower', 'FastViTModel'] diff --git a/aiak_training_llm/models/fastvit/builder.py b/aiak_training_llm/models/fastvit/builder.py new file mode 100644 index 00000000..290ecdf7 --- /dev/null +++ b/aiak_training_llm/models/fastvit/builder.py @@ -0,0 +1,19 @@ +import os +from .clip_encoder import CLIPVisionTower, CLIPVisionTowerS2 +from .mobileclip_encoder import MobileCLIPVisionTower + + +def build_vision_tower(vision_tower_cfg, **kwargs): + vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None)) + is_absolute_path_exists = os.path.exists(vision_tower) + use_s2 = getattr(vision_tower_cfg, 's2', False) + + if is_absolute_path_exists or vision_tower.startswith("openai") or vision_tower.startswith("laion") or "ShareGPT4V" in vision_tower: + if use_s2: + return CLIPVisionTowerS2(vision_tower, args=vision_tower_cfg, **kwargs) + else: + return CLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs) + elif "mobileclip" in vision_tower.lower(): + return MobileCLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs) + + raise ValueError(f'Unknown vision tower: {vision_tower}') \ No newline at end of file diff --git a/aiak_training_llm/models/fastvit/constants.py b/aiak_training_llm/models/fastvit/constants.py new file mode 100644 index 00000000..6964799b --- /dev/null +++ b/aiak_training_llm/models/fastvit/constants.py @@ -0,0 +1,13 @@ +CONTROLLER_HEART_BEAT_EXPIRATION = 30 +WORKER_HEART_BEAT_INTERVAL = 15 + +LOGDIR = "." + +# Model Constants +IGNORE_INDEX = -100 +IMAGE_TOKEN_INDEX = -200 +DEFAULT_IMAGE_TOKEN = "" +DEFAULT_IMAGE_PATCH_TOKEN = "" +DEFAULT_IM_START_TOKEN = "" +DEFAULT_IM_END_TOKEN = "" +IMAGE_PLACEHOLDER = "" \ No newline at end of file diff --git a/aiak_training_llm/models/fastvit/fastvit_preprocessor.py b/aiak_training_llm/models/fastvit/fastvit_preprocessor.py new file mode 100644 index 00000000..1a061a73 --- /dev/null +++ b/aiak_training_llm/models/fastvit/fastvit_preprocessor.py @@ -0,0 +1,69 @@ +# +# FastViT Image Preprocessing +# Following Apple FastVLM preprocessing approach +# + +from PIL import Image +import torch +from transformers import CLIPImageProcessor + + +class FastViTImageProcessor: + """ + Image processor for FastViT following FastVLM's approach. + Uses CLIPImageProcessor with specific settings for FastViT. + """ + + def __init__(self, image_size=384): + """ + Initialize FastViT image processor. + + Args: + image_size: Input image size (default 384 as in FastVLM) + """ + self.image_size = image_size + + # Create CLIPImageProcessor with FastViT settings + # Following mobileclip_encoder.py from FastVLM + # Source: ml-fastvlm/llava/model/multimodal_encoder/mobileclip_encoder.py (lines 45-49) + self.image_mean = [0.0, 0.0, 0.0] # No normalization (black padding for 'pad' mode) + self.image_std = [1.0, 1.0, 1.0] + + self.processor = CLIPImageProcessor( + crop_size={"height": image_size, "width": image_size}, + image_mean=self.image_mean, + image_std=self.image_std, + size={"shortest_edge": image_size} + ) + + def preprocess(self, image): + """ + Preprocess a single image for FastViT. + + Args: + image: PIL Image + + Returns: + Preprocessed image tensor + """ + if isinstance(image, Image.Image): + # Process with CLIPImageProcessor + processed = self.processor(images=image, return_tensors="pt") + return processed['pixel_values'][0] # Remove batch dimension + else: + raise ValueError(f"Expected PIL Image, got {type(image)}") + + def __call__(self, images): + """ + Process single image or batch of images. + + Args: + images: PIL Image or list of PIL Images + + Returns: + Tensor of preprocessed images + """ + if isinstance(images, list): + return torch.stack([self.preprocess(img) for img in images]) + else: + return self.preprocess(images).unsqueeze(0) diff --git a/aiak_training_llm/models/fastvit/fastvit_vision_model.py b/aiak_training_llm/models/fastvit/fastvit_vision_model.py new file mode 100644 index 00000000..af68319c --- /dev/null +++ b/aiak_training_llm/models/fastvit/fastvit_vision_model.py @@ -0,0 +1,98 @@ +# +# FastViT Vision Model wrapper for Megatron integration +# Adapted from Apple's FastVLM +# + +import torch +import torch.nn as nn +from megatron.core.transformer import MegatronModule + +from aiak_training_llm.models.fastvit.mobileclip_encoder import MobileCLIPVisionTower + + +class FastViTModel(MegatronModule): + """ + FastViT Vision Model wrapper for Megatron. + + This wraps the MobileCLIPVisionTower (which contains FastViTHD) + to be compatible with Megatron's training framework. + + Args: + config: Megatron TransformerConfig + layer_spec: Module specification (not used for FastViT) + """ + + def __init__(self, config, layer_spec): + # Initialize MegatronModule (will set self.config) + super().__init__(config=config) + + # Create a simple args object for MobileCLIPVisionTower + class Args: + def __init__(self): + self.unfreeze_mm_vision_tower = False + + args = Args() + + # FastViTHD model name with resolution + # Format: mobileclip_l_{resolution} + # The first two parts must match the config file name (mobileclip_l.json) + # The resolution is extracted from the last part + vision_tower_name = "mobileclip_l_384" # Default to 384 + + # Override with config if provided + if hasattr(config, 'vision_tower_name'): + vision_tower_name = config.vision_tower_name + elif hasattr(config, 'img_h') and config.img_w == config.img_h: + # Use square image size from config + vision_tower_name = f"mobileclip_l_{config.img_h}" + + # Initialize the FastViT vision tower + self.vision_tower = MobileCLIPVisionTower( + vision_tower=vision_tower_name, + args=args, + delay_load=False + ) + + self._hidden_size = self.vision_tower.hidden_size + + def forward(self, images, grid_thw=None): + """ + Forward pass through FastViT. + + Args: + images: Image tensor [batch, channels, height, width] + grid_thw: Grid dimensions (not used by FastViT, kept for API compatibility) + + Returns: + Vision features [batch, num_tokens, hidden_size] + window_index: None (FastViT doesn't use windowing) + """ + # MobileCLIPVisionTower expects single images or list of images + # For batch processing, we need to handle it appropriately + if images.dim() == 4: + # Batch of images [B, C, H, W] + image_features = self.vision_tower.forward_images(images) + else: + raise ValueError(f"Unexpected image tensor shape: {images.shape}") + + # Return features and None for window_index (compatibility with Qwen2-VL API) + return image_features, None + + def set_input_tensor(self, input_tensor): + """ + Set input tensor. + + FastViT doesn't use pipeline parallelism, so this is a no-op. + Required for compatibility with Megatron's pipeline parallel interface. + """ + pass + + @property + def hidden_size(self): + """Return the hidden size of the vision model output.""" + return self._hidden_size + + @property + def vision_config(self): + """Return vision tower config.""" + return self.vision_tower.config diff --git a/aiak_training_llm/models/fastvit/mm_utils.py b/aiak_training_llm/models/fastvit/mm_utils.py new file mode 100644 index 00000000..d08dd58d --- /dev/null +++ b/aiak_training_llm/models/fastvit/mm_utils.py @@ -0,0 +1,250 @@ +import PIL +from PIL import Image +PIL.Image.MAX_IMAGE_PIXELS=500000000 +from io import BytesIO +import base64 +import torch +import math +import ast + +from transformers import StoppingCriteria +from aiak_training_llm.models.fastvit.constants import IMAGE_TOKEN_INDEX + + +def select_best_resolution(original_size, possible_resolutions): + """ + Selects the best resolution from a list of possible resolutions based on the original size. + + Args: + original_size (tuple): The original size of the image in the format (width, height). + possible_resolutions (list): A list of possible resolutions in the format [(width1, height1), (width2, height2), ...]. + + Returns: + tuple: The best fit resolution in the format (width, height). + """ + original_width, original_height = original_size + best_fit = None + max_effective_resolution = 0 + min_wasted_resolution = float('inf') + + for width, height in possible_resolutions: + scale = min(width / original_width, height / original_height) + downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale) + effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height) + wasted_resolution = (width * height) - effective_resolution + + if effective_resolution > max_effective_resolution or (effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution): + max_effective_resolution = effective_resolution + min_wasted_resolution = wasted_resolution + best_fit = (width, height) + + return best_fit + + +def resize_and_pad_image(image, target_resolution): + """ + Resize and pad an image to a target resolution while maintaining aspect ratio. + + Args: + image (PIL.Image.Image): The input image. + target_resolution (tuple): The target resolution (width, height) of the image. + + Returns: + PIL.Image.Image: The resized and padded image. + """ + original_width, original_height = image.size + target_width, target_height = target_resolution + + scale_w = target_width / original_width + scale_h = target_height / original_height + + if scale_w < scale_h: + new_width = target_width + new_height = min(math.ceil(original_height * scale_w), target_height) + else: + new_height = target_height + new_width = min(math.ceil(original_width * scale_h), target_width) + + # Resize the image + resized_image = image.resize((new_width, new_height)) + + new_image = Image.new('RGB', (target_width, target_height), (0, 0, 0)) + paste_x = (target_width - new_width) // 2 + paste_y = (target_height - new_height) // 2 + new_image.paste(resized_image, (paste_x, paste_y)) + + return new_image + + +def divide_to_patches(image, patch_size): + """ + Divides an image into patches of a specified size. + + Args: + image (PIL.Image.Image): The input image. + patch_size (int): The size of each patch. + + Returns: + list: A list of PIL.Image.Image objects representing the patches. + """ + patches = [] + width, height = image.size + for i in range(0, height, patch_size): + for j in range(0, width, patch_size): + box = (j, i, j + patch_size, i + patch_size) + patch = image.crop(box) + patches.append(patch) + + return patches + + +def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size): + """ + Calculate the shape of the image patch grid after the preprocessing for images of any resolution. + + Args: + image_size (tuple): The size of the input image in the format (width, height). + grid_pinpoints (str): A string representation of a list of possible resolutions. + patch_size (int): The size of each image patch. + + Returns: + tuple: The shape of the image patch grid in the format (width, height). + """ + if type(grid_pinpoints) is list: + possible_resolutions = grid_pinpoints + else: + possible_resolutions = ast.literal_eval(grid_pinpoints) + width, height = select_best_resolution(image_size, possible_resolutions) + return width // patch_size, height // patch_size + + +def process_anyres_image(image, processor, grid_pinpoints): + """ + Process an image with variable resolutions. + + Args: + image (PIL.Image.Image): The input image to be processed. + processor: The image processor object. + grid_pinpoints (str): A string representation of a list of possible resolutions. + + Returns: + torch.Tensor: A tensor containing the processed image patches. + """ + if type(grid_pinpoints) is list: + possible_resolutions = grid_pinpoints + else: + possible_resolutions = ast.literal_eval(grid_pinpoints) + best_resolution = select_best_resolution(image.size, possible_resolutions) + image_padded = resize_and_pad_image(image, best_resolution) + + patches = divide_to_patches(image_padded, processor.crop_size['height']) + + image_original_resize = image.resize((processor.size['shortest_edge'], processor.size['shortest_edge'])) + + image_patches = [image_original_resize] + patches + image_patches = [processor.preprocess(image_patch, return_tensors='pt')['pixel_values'][0] + for image_patch in image_patches] + return torch.stack(image_patches, dim=0) + + +def load_image_from_base64(image): + return Image.open(BytesIO(base64.b64decode(image))) + + +def expand2square(pil_img, background_color): + width, height = pil_img.size + if width == height: + return pil_img + elif width > height: + result = Image.new(pil_img.mode, (width, width), background_color) + result.paste(pil_img, (0, (width - height) // 2)) + return result + else: + result = Image.new(pil_img.mode, (height, height), background_color) + result.paste(pil_img, ((height - width) // 2, 0)) + return result + + +def process_images(images, image_processor, model_cfg): + image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None) + new_images = [] + if image_aspect_ratio == 'pad': + for image in images: + image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean)) + image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0] + new_images.append(image) + elif image_aspect_ratio == "anyres": + for image in images: + image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints) + new_images.append(image) + else: + return image_processor(images, return_tensors='pt')['pixel_values'] + if all(x.shape == new_images[0].shape for x in new_images): + new_images = torch.stack(new_images, dim=0) + return new_images + + +def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None): + prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('')] + + def insert_separator(X, sep): + return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1] + + input_ids = [] + offset = 0 + if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id: + offset = 1 + input_ids.append(prompt_chunks[0][0]) + + for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)): + input_ids.extend(x[offset:]) + + if return_tensors is not None: + if return_tensors == 'pt': + return torch.tensor(input_ids, dtype=torch.long) + raise ValueError(f'Unsupported tensor type: {return_tensors}') + return input_ids + + +def get_model_name_from_path(model_path): + model_path = model_path.strip("/") + model_paths = model_path.split("/") + if model_paths[-1].startswith('checkpoint-'): + return model_paths[-2] + "_" + model_paths[-1] + else: + return model_paths[-1] + + +class KeywordsStoppingCriteria(StoppingCriteria): + def __init__(self, keywords, tokenizer, input_ids): + self.keywords = keywords + self.keyword_ids = [] + self.max_keyword_len = 0 + for keyword in keywords: + cur_keyword_ids = tokenizer(keyword).input_ids + if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id: + cur_keyword_ids = cur_keyword_ids[1:] + if len(cur_keyword_ids) > self.max_keyword_len: + self.max_keyword_len = len(cur_keyword_ids) + self.keyword_ids.append(torch.tensor(cur_keyword_ids)) + self.tokenizer = tokenizer + self.start_len = input_ids.shape[1] + + def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: + offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len) + self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids] + for keyword_id in self.keyword_ids: + truncated_output_ids = output_ids[0, -keyword_id.shape[0]:] + if torch.equal(truncated_output_ids, keyword_id): + return True + outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0] + for keyword in self.keywords: + if keyword in outputs: + return True + return False + + def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: + outputs = [] + for i in range(output_ids.shape[0]): + outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores)) + return all(outputs) \ No newline at end of file diff --git a/aiak_training_llm/models/fastvit/mobileclip/__init__.py b/aiak_training_llm/models/fastvit/mobileclip/__init__.py new file mode 100644 index 00000000..b4696ec3 --- /dev/null +++ b/aiak_training_llm/models/fastvit/mobileclip/__init__.py @@ -0,0 +1,87 @@ +# +# For licensing see accompanying LICENSE file. +# Copyright (C) 2025 Apple Inc. All Rights Reserved. +# +import os +import json +from typing import Any + +import torch.nn as nn +from timm.models import create_model + +from .mci import GlobalPool2D + + +def load_model_config( + model_name: str, +) -> Any: + # Strip suffixes to model name + model_name = "_".join(model_name.split("_")[0:2]) + + # Config files + root_dir = os.path.dirname(os.path.abspath(__file__)) + configs_dir = os.path.join(root_dir, "configs") + model_cfg_file = os.path.join(configs_dir, model_name + ".json") + + # Get config from yaml file + if not os.path.exists(model_cfg_file): + raise ValueError(f"Unsupported model name: {model_name}") + model_cfg = json.load(open(model_cfg_file, "r")) + + return model_cfg + + +class MCi(nn.Module): + """ + This class implements `MCi Models `_ + """ + + def __init__(self, model_name: str, *args, **kwargs) -> None: + super().__init__() + self.projection_dim = None + if "projection_dim" in kwargs: + self.projection_dim = kwargs.get("projection_dim") + + # Create model + self.model = create_model(model_name, projection_dim=self.projection_dim) + + # Build out projection head. + if self.projection_dim is not None: + if hasattr(self.model, "head"): + self.model.head = MCi._update_image_classifier( + image_classifier=self.model.head, projection_dim=self.projection_dim + ) + + def forward(self, x: Any, *args, **kwargs) -> Any: + """A forward function of the model.""" + x = self.model(x, *args, **kwargs) + return x + + @staticmethod + def _get_in_feature_dimension(image_classifier: nn.Module) -> int: + """Return the input feature dimension to the image classification head.""" + in_features = None + if isinstance(image_classifier, nn.Sequential): + # Classifier that uses nn.Sequential usually has global pooling and + # multiple linear layers. Find the first linear layer and get its + # in_features + for layer in image_classifier: + if isinstance(layer, nn.Linear): + in_features = layer.in_features + break + elif isinstance(image_classifier, nn.Linear): + in_features = image_classifier.in_features + + if in_features is None: + raise NotImplementedError( + f"Cannot get input feature dimension of {image_classifier}." + ) + return in_features + + @staticmethod + def _update_image_classifier( + image_classifier: nn.Module, projection_dim: int, *args, **kwargs + ) -> nn.Module: + in_features = MCi._get_in_feature_dimension(image_classifier) + new_img_classifier = GlobalPool2D(in_dim=in_features, out_dim=projection_dim) + return new_img_classifier \ No newline at end of file diff --git a/aiak_training_llm/models/fastvit/mobileclip/configs/mobileclip_l.json b/aiak_training_llm/models/fastvit/mobileclip/configs/mobileclip_l.json new file mode 100644 index 00000000..c6f98c79 --- /dev/null +++ b/aiak_training_llm/models/fastvit/mobileclip/configs/mobileclip_l.json @@ -0,0 +1,20 @@ +{ + "embed_dim": 768, + "image_cfg": { + "image_size": 1024, + "model_name": "fastvithd", + "embed_dim": 3072, + "patch_size": 64 + }, + "text_cfg": { + "context_length": 77, + "vocab_size": 49408, + "dim": 768, + "ffn_multiplier_per_layer": 4.0, + "n_heads_per_layer": 12, + "n_transformer_layers": 12, + "norm_layer": "layer_norm_fp32", + "causal_masking": false, + "model_name": "base" + } +} \ No newline at end of file diff --git a/aiak_training_llm/models/fastvit/mobileclip/mci.py b/aiak_training_llm/models/fastvit/mobileclip/mci.py new file mode 100644 index 00000000..690cdcb1 --- /dev/null +++ b/aiak_training_llm/models/fastvit/mobileclip/mci.py @@ -0,0 +1,1478 @@ +# +# For licensing see accompanying LICENSE file. +# Copyright (C) 2025 Apple Inc. All Rights Reserved. +# +import copy +from functools import partial +from typing import List, Tuple, Optional, Union, Dict + +import torch +import torch.nn as nn +from torch import Tensor +import torch.nn.functional as F +from torch.nn.init import normal_ + +from timm.models import register_model +from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD +from timm.layers import DropPath, SqueezeExcite + + +def _cfg(url="", **kwargs): + return { + "url": url, + "num_classes": 1000, + "input_size": (3, 256, 256), + "pool_size": None, + "crop_pct": 0.95, + "interpolation": "bicubic", + "mean": IMAGENET_DEFAULT_MEAN, + "std": IMAGENET_DEFAULT_STD, + "classifier": "head", + **kwargs, + } + + +default_cfgs = { + "fastvit_t": _cfg(crop_pct=0.9), + "fastvit_s": _cfg(crop_pct=0.9), + "fastvit_m": _cfg(crop_pct=0.95), +} + + +class SEBlock(nn.Module): + """Squeeze and Excite module. + + Pytorch implementation of `Squeeze-and-Excitation Networks` - + https://arxiv.org/pdf/1709.01507.pdf + """ + + def __init__(self, in_channels: int, rd_ratio: float = 0.0625) -> None: + """Construct a Squeeze and Excite Module. + + Args: + in_channels: Number of input channels. + rd_ratio: Input channel reduction ratio. + """ + super(SEBlock, self).__init__() + self.reduce = nn.Conv2d( + in_channels=in_channels, + out_channels=int(in_channels * rd_ratio), + kernel_size=1, + stride=1, + bias=True, + ) + self.expand = nn.Conv2d( + in_channels=int(in_channels * rd_ratio), + out_channels=in_channels, + kernel_size=1, + stride=1, + bias=True, + ) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + """Apply forward pass.""" + b, c, h, w = inputs.size() + x = F.avg_pool2d(inputs, kernel_size=[h, w]) + x = self.reduce(x) + x = F.relu(x) + x = self.expand(x) + x = torch.sigmoid(x) + x = x.view(-1, c, 1, 1) + return inputs * x + + +class MobileOneBlock(nn.Module): + """MobileOne building block. + + This block has a multi-branched architecture at train-time + and plain-CNN style architecture at inference time + For more details, please refer to our paper: + `An Improved One millisecond Mobile Backbone` - + https://arxiv.org/pdf/2206.04040.pdf + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + dilation: int = 1, + groups: int = 1, + inference_mode: bool = False, + use_se: bool = False, + use_act: bool = True, + use_scale_branch: bool = True, + num_conv_branches: int = 1, + activation: nn.Module = nn.GELU(), + ) -> None: + """Construct a MobileOneBlock module. + + Args: + in_channels: Number of channels in the input. + out_channels: Number of channels produced by the block. + kernel_size: Size of the convolution kernel. + stride: Stride size. + padding: Zero-padding size. + dilation: Kernel dilation factor. + groups: Group number. + inference_mode: If True, instantiates model in inference mode. + use_se: Whether to use SE-ReLU activations. + use_act: Whether to use activation. Default: ``True`` + use_scale_branch: Whether to use scale branch. Default: ``True`` + num_conv_branches: Number of linear conv branches. + """ + super(MobileOneBlock, self).__init__() + self.inference_mode = inference_mode + self.groups = groups + self.stride = stride + self.padding = padding + self.dilation = dilation + self.kernel_size = kernel_size + self.in_channels = in_channels + self.out_channels = out_channels + self.num_conv_branches = num_conv_branches + + # Check if SE-ReLU is requested + if use_se: + self.se = SEBlock(out_channels) + else: + self.se = nn.Identity() + + if use_act: + self.activation = activation + else: + self.activation = nn.Identity() + + if inference_mode: + self.reparam_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=True, + ) + else: + # Re-parameterizable skip connection + # Fallback, sometimes batchnorm tensors + # do not get instantiated correctly on some processes + # when using deepspeed + accelerate + norm_layer = nn.BatchNorm2d(num_features=in_channels) + if norm_layer.weight.shape[0] == 0: + norm_layer.weight = nn.Parameter(torch.zeros(in_channels)) + if norm_layer.bias.shape[0] == 0: + norm_layer.bias = nn.Parameter(torch.zeros(in_channels)) + + self.rbr_skip = ( + norm_layer + if out_channels == in_channels and stride == 1 + else None + ) + + # Re-parameterizable conv branches + if num_conv_branches > 0: + rbr_conv = list() + for _ in range(self.num_conv_branches): + rbr_conv.append( + self._conv_bn(kernel_size=kernel_size, padding=padding) + ) + self.rbr_conv = nn.ModuleList(rbr_conv) + else: + self.rbr_conv = None + + # Re-parameterizable scale branch + self.rbr_scale = None + if not isinstance(kernel_size, int): + kernel_size = kernel_size[0] + if (kernel_size > 1) and use_scale_branch: + self.rbr_scale = self._conv_bn(kernel_size=1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply forward pass.""" + # Inference mode forward pass. + if self.inference_mode: + return self.activation(self.se(self.reparam_conv(x))) + + # Multi-branched train-time forward pass. + # Skip branch output + identity_out = 0 + if self.rbr_skip is not None: + identity_out = self.rbr_skip(x) + + # Scale branch output + scale_out = 0 + if self.rbr_scale is not None: + scale_out = self.rbr_scale(x) + + # Other branches + out = scale_out + identity_out + if self.rbr_conv is not None: + for ix in range(self.num_conv_branches): + out += self.rbr_conv[ix](x) + + return self.activation(self.se(out)) + + def reparameterize(self): + """Following works like `RepVGG: Making VGG-style ConvNets Great Again` - + https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched + architecture used at training time to obtain a plain CNN-like structure + for inference. + """ + if self.inference_mode: + return + kernel, bias = self._get_kernel_bias() + self.reparam_conv = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + groups=self.groups, + bias=True, + ) + self.reparam_conv.weight.data = kernel + self.reparam_conv.bias.data = bias + + # Delete un-used branches + self.__delattr__("rbr_conv") + self.__delattr__("rbr_scale") + if hasattr(self, "rbr_skip"): + self.__delattr__("rbr_skip") + + self.inference_mode = True + + def _get_kernel_bias(self) -> Tuple[torch.Tensor, torch.Tensor]: + """Method to obtain re-parameterized kernel and bias. + Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L83 + + Returns: + Tuple of (kernel, bias) after fusing branches. + """ + # get weights and bias of scale branch + kernel_scale = 0 + bias_scale = 0 + if self.rbr_scale is not None: + kernel_scale, bias_scale = self._fuse_bn_tensor(self.rbr_scale) + # Pad scale branch kernel to match conv branch kernel size. + pad = self.kernel_size // 2 + kernel_scale = torch.nn.functional.pad(kernel_scale, [pad, pad, pad, pad]) + + # get weights and bias of skip branch + kernel_identity = 0 + bias_identity = 0 + if self.rbr_skip is not None: + kernel_identity, bias_identity = self._fuse_bn_tensor(self.rbr_skip) + + # get weights and bias of conv branches + kernel_conv = 0 + bias_conv = 0 + if self.rbr_conv is not None: + for ix in range(self.num_conv_branches): + _kernel, _bias = self._fuse_bn_tensor(self.rbr_conv[ix]) + kernel_conv += _kernel + bias_conv += _bias + + kernel_final = kernel_conv + kernel_scale + kernel_identity + bias_final = bias_conv + bias_scale + bias_identity + return kernel_final, bias_final + + def _fuse_bn_tensor( + self, branch: Union[nn.Sequential, nn.BatchNorm2d] + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Method to fuse batchnorm layer with preceeding conv layer. + Reference: https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py#L95 + + Args: + branch: Sequence of ops to be fused. + + Returns: + Tuple of (kernel, bias) after fusing batchnorm. + """ + if isinstance(branch, nn.Sequential): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + else: + assert isinstance(branch, nn.BatchNorm2d) + if not hasattr(self, "id_tensor"): + input_dim = self.in_channels // self.groups + + kernel_size = self.kernel_size + if isinstance(self.kernel_size, int): + kernel_size = (self.kernel_size, self.kernel_size) + + kernel_value = torch.zeros( + (self.in_channels, input_dim, kernel_size[0], kernel_size[1]), + dtype=branch.weight.dtype, + device=branch.weight.device, + ) + for i in range(self.in_channels): + kernel_value[ + i, i % input_dim, kernel_size[0] // 2, kernel_size[1] // 2 + ] = 1 + self.id_tensor = kernel_value + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def _conv_bn(self, kernel_size: int, padding: int) -> nn.Sequential: + """Helper method to construct conv-batchnorm layers. + + Args: + kernel_size: Size of the convolution kernel. + padding: Zero-padding size. + + Returns: + Conv-BN module. + """ + # Fallback, sometimes batchnorm tensors + # do not get instantiated correctly on some processes + # when using deepspeed + accelerate + norm_layer = nn.BatchNorm2d(num_features=self.out_channels) + if norm_layer.weight.shape[0] == 0: + norm_layer.weight = nn.Parameter(torch.zeros(self.out_channels)) + if norm_layer.bias.shape[0] == 0: + norm_layer.bias = nn.Parameter(torch.zeros(self.out_channels)) + + mod_list = nn.Sequential() + mod_list.add_module( + "conv", + nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=kernel_size, + stride=self.stride, + padding=padding, + groups=self.groups, + bias=False, + ), + ) + mod_list.add_module("bn", norm_layer) + return mod_list + + +class ReparamLargeKernelConv(nn.Module): + """Building Block of RepLKNet + + This class defines overparameterized large kernel conv block + introduced in `RepLKNet `_ + + Reference: https://github.com/DingXiaoH/RepLKNet-pytorch + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int, + groups: int, + small_kernel: int, + inference_mode: bool = False, + use_se: bool = False, + activation: nn.Module = nn.GELU(), + ) -> None: + """Construct a ReparamLargeKernelConv module. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Kernel size of the large kernel conv branch. + stride: Stride size. Default: 1 + groups: Group number. Default: 1 + small_kernel: Kernel size of small kernel conv branch. + inference_mode: If True, instantiates model in inference mode. Default: ``False`` + activation: Activation module. Default: ``nn.GELU`` + """ + super(ReparamLargeKernelConv, self).__init__() + + self.stride = stride + self.groups = groups + self.in_channels = in_channels + self.out_channels = out_channels + self.activation = activation + + self.kernel_size = kernel_size + self.small_kernel = small_kernel + self.padding = kernel_size // 2 + + # Check if SE is requested + if use_se: + self.se = SqueezeExcite(out_channels, rd_ratio=0.25) + else: + self.se = nn.Identity() + + if inference_mode: + self.lkb_reparam = nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + dilation=1, + groups=groups, + bias=True, + ) + else: + self.lkb_origin = self._conv_bn( + kernel_size=kernel_size, padding=self.padding + ) + if small_kernel is not None: + assert ( + small_kernel <= kernel_size + ), "The kernel size for re-param cannot be larger than the large kernel!" + self.small_conv = self._conv_bn( + kernel_size=small_kernel, padding=small_kernel // 2 + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply forward pass.""" + if hasattr(self, "lkb_reparam"): + out = self.lkb_reparam(x) + else: + out = self.lkb_origin(x) + if hasattr(self, "small_conv"): + out += self.small_conv(x) + + return self.activation(self.se(out)) + + def get_kernel_bias(self) -> Tuple[torch.Tensor, torch.Tensor]: + """Method to obtain re-parameterized kernel and bias. + Reference: https://github.com/DingXiaoH/RepLKNet-pytorch + + Returns: + Tuple of (kernel, bias) after fusing branches. + """ + eq_k, eq_b = self._fuse_bn(self.lkb_origin.conv, self.lkb_origin.bn) + if hasattr(self, "small_conv"): + small_k, small_b = self._fuse_bn(self.small_conv.conv, self.small_conv.bn) + eq_b += small_b + eq_k += nn.functional.pad( + small_k, [(self.kernel_size - self.small_kernel) // 2] * 4 + ) + return eq_k, eq_b + + def reparameterize(self) -> None: + """ + Following works like `RepVGG: Making VGG-style ConvNets Great Again` - + https://arxiv.org/pdf/2101.03697.pdf. We re-parameterize multi-branched + architecture used at training time to obtain a plain CNN-like structure + for inference. + """ + eq_k, eq_b = self.get_kernel_bias() + self.lkb_reparam = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + dilation=self.lkb_origin.conv.dilation, + groups=self.groups, + bias=True, + ) + + self.lkb_reparam.weight.data = eq_k + self.lkb_reparam.bias.data = eq_b + self.__delattr__("lkb_origin") + if hasattr(self, "small_conv"): + self.__delattr__("small_conv") + + @staticmethod + def _fuse_bn( + conv: torch.Tensor, bn: nn.BatchNorm2d + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Method to fuse batchnorm layer with conv layer. + + Args: + conv: Convolutional kernel weights. + bn: Batchnorm 2d layer. + + Returns: + Tuple of (kernel, bias) after fusing batchnorm. + """ + kernel = conv.weight + running_mean = bn.running_mean + running_var = bn.running_var + gamma = bn.weight + beta = bn.bias + eps = bn.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def _conv_bn(self, kernel_size: int, padding: int = 0) -> nn.Sequential: + """Helper method to construct conv-batchnorm layers. + + Args: + kernel_size: Size of the convolution kernel. + padding: Zero-padding size. + + Returns: + A nn.Sequential Conv-BN module. + """ + # Fallback, sometimes batchnorm tensors + # do not get instantiated correctly on some processes + # when using deepspeed + accelerate + norm_layer = nn.BatchNorm2d(num_features=self.out_channels) + if norm_layer.weight.shape[0] == 0: + norm_layer.weight = nn.Parameter(torch.zeros(self.out_channels)) + if norm_layer.bias.shape[0] == 0: + norm_layer.bias = nn.Parameter(torch.zeros(self.out_channels)) + + mod_list = nn.Sequential() + mod_list.add_module( + "conv", + nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.out_channels, + kernel_size=kernel_size, + stride=self.stride, + padding=padding, + groups=self.groups, + bias=False, + ), + ) + mod_list.add_module("bn", norm_layer) + return mod_list + + +def convolutional_stem( + in_channels: int, out_channels: int, inference_mode: bool = False, use_scale_branch: bool = True, +) -> nn.Sequential: + """Build convolutional stem with MobileOne blocks. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + inference_mode: Flag to instantiate model in inference mode. Default: ``False`` + + Returns: + nn.Sequential object with stem elements. + """ + return nn.Sequential( + MobileOneBlock( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=3, + stride=2, + padding=1, + groups=1, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + use_scale_branch=use_scale_branch + ), + MobileOneBlock( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=3, + stride=2, + padding=1, + groups=out_channels, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + use_scale_branch=use_scale_branch + ), + MobileOneBlock( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + groups=1, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + use_scale_branch=use_scale_branch + ), + ) + + +class LayerNormChannel(nn.Module): + """ + LayerNorm only for Channel Dimension. + Input: tensor in shape [B, C, H, W] + """ + def __init__(self, num_features, eps=1e-05) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(num_features)) + self.bias = nn.Parameter(torch.zeros(num_features)) + self.eps = eps + + def forward(self, x) -> torch.Tensor: + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight.unsqueeze(-1).unsqueeze(-1) * x \ + + self.bias.unsqueeze(-1).unsqueeze(-1) + return x + + +class MHSA(nn.Module): + """Multi-headed Self Attention module. + + Source modified from: + https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py + """ + + def __init__( + self, + dim: int, + head_dim: int = 32, + qkv_bias: bool = False, + attn_drop: float = 0.0, + proj_drop: float = 0.0, + ) -> None: + """Build MHSA module that can handle 3D or 4D input tensors. + + Args: + dim: Number of embedding dimensions. + head_dim: Number of hidden dimensions per head. Default: ``32`` + qkv_bias: Use bias or not. Default: ``False`` + attn_drop: Dropout rate for attention tensor. + proj_drop: Dropout rate for projection tensor. + """ + super().__init__() + assert dim % head_dim == 0, "dim should be divisible by head_dim" + self.head_dim = head_dim + self.num_heads = dim // head_dim + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + shape = x.shape + B, C, H, W = shape + N = H * W + if len(shape) == 4: + x = torch.flatten(x, start_dim=2).transpose(-2, -1) # (B, N, C) + qkv = ( + self.qkv(x) + .reshape(B, N, 3, self.num_heads, self.head_dim) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + # trick here to make q@k.t more stable + attn = (q * self.scale) @ k.transpose(-2, -1) + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + if len(shape) == 4: + x = x.transpose(-2, -1).reshape(B, C, H, W) + + return x + + +class PatchEmbed(nn.Module): + """Convolutional patch embedding layer.""" + + def __init__( + self, + patch_size: int, + stride: int, + in_channels: int, + embed_dim: int, + inference_mode: bool = False, + use_se: bool = False, + ) -> None: + """Build patch embedding layer. + + Args: + patch_size: Patch size for embedding computation. + stride: Stride for convolutional embedding layer. + in_channels: Number of channels of input tensor. + embed_dim: Number of embedding dimensions. + inference_mode: Flag to instantiate model in inference mode. Default: ``False`` + use_se: If ``True`` SE block will be used. + """ + super().__init__() + block = list() + block.append( + ReparamLargeKernelConv( + in_channels=in_channels, + out_channels=embed_dim, + kernel_size=patch_size, + stride=stride, + groups=in_channels, + small_kernel=3, + inference_mode=inference_mode, + use_se=use_se, + ) + ) + block.append( + MobileOneBlock( + in_channels=embed_dim, + out_channels=embed_dim, + kernel_size=1, + stride=1, + padding=0, + groups=1, + inference_mode=inference_mode, + use_se=False, + num_conv_branches=1, + ) + ) + self.proj = nn.Sequential(*block) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + return x + + +class RepMixer(nn.Module): + """Reparameterizable token mixer. + + For more details, please refer to our paper: + `FastViT: A Fast Hybrid Vision Transformer using Structural Reparameterization `_ + """ + + def __init__( + self, + dim, + kernel_size=3, + use_layer_scale=True, + layer_scale_init_value=1e-5, + inference_mode: bool = False, + ): + """Build RepMixer Module. + + Args: + dim: Input feature map dimension. :math:`C_{in}` from an expected input of size :math:`(B, C_{in}, H, W)`. + kernel_size: Kernel size for spatial mixing. Default: 3 + use_layer_scale: If True, learnable layer scale is used. Default: ``True`` + layer_scale_init_value: Initial value for layer scale. Default: 1e-5 + inference_mode: If True, instantiates model in inference mode. Default: ``False`` + """ + super().__init__() + self.dim = dim + self.kernel_size = kernel_size + self.inference_mode = inference_mode + + if inference_mode: + self.reparam_conv = nn.Conv2d( + in_channels=self.dim, + out_channels=self.dim, + kernel_size=self.kernel_size, + stride=1, + padding=self.kernel_size // 2, + groups=self.dim, + bias=True, + ) + else: + self.norm = MobileOneBlock( + dim, + dim, + kernel_size, + padding=kernel_size // 2, + groups=dim, + use_act=False, + use_scale_branch=False, + num_conv_branches=0, + ) + self.mixer = MobileOneBlock( + dim, + dim, + kernel_size, + padding=kernel_size // 2, + groups=dim, + use_act=False, + ) + self.use_layer_scale = use_layer_scale + if use_layer_scale: + self.layer_scale = nn.Parameter( + layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if hasattr(self, "reparam_conv"): + x = self.reparam_conv(x) + return x + else: + if self.use_layer_scale: + x = x + self.layer_scale * (self.mixer(x) - self.norm(x)) + else: + x = x + self.mixer(x) - self.norm(x) + return x + + def reparameterize(self) -> None: + """Reparameterize mixer and norm into a single + convolutional layer for efficient inference. + """ + if self.inference_mode: + return + + self.mixer.reparameterize() + self.norm.reparameterize() + + if self.use_layer_scale: + w = self.mixer.id_tensor + self.layer_scale.unsqueeze(-1) * ( + self.mixer.reparam_conv.weight - self.norm.reparam_conv.weight + ) + b = torch.squeeze(self.layer_scale) * ( + self.mixer.reparam_conv.bias - self.norm.reparam_conv.bias + ) + else: + w = ( + self.mixer.id_tensor + + self.mixer.reparam_conv.weight + - self.norm.reparam_conv.weight + ) + b = self.mixer.reparam_conv.bias - self.norm.reparam_conv.bias + + self.reparam_conv = nn.Conv2d( + in_channels=self.dim, + out_channels=self.dim, + kernel_size=self.kernel_size, + stride=1, + padding=self.kernel_size // 2, + groups=self.dim, + bias=True, + ) + self.reparam_conv.weight.data = w + self.reparam_conv.bias.data = b + + self.__delattr__("mixer") + self.__delattr__("norm") + if self.use_layer_scale: + self.__delattr__("layer_scale") + + +class ConvFFN(nn.Module): + """Convolutional FFN Module.""" + + def __init__( + self, + in_channels: int, + hidden_channels: Optional[int] = None, + out_channels: Optional[int] = None, + act_layer: nn.Module = nn.GELU, + drop: float = 0.0, + ) -> None: + """Build convolutional FFN module. + + Args: + in_channels: Number of input channels. + hidden_channels: Number of channels after expansion. Default: None + out_channels: Number of output channels. Default: None + act_layer: Activation layer. Default: ``GELU`` + drop: Dropout rate. Default: ``0.0``. + """ + super().__init__() + out_channels = out_channels or in_channels + hidden_channels = hidden_channels or in_channels + self.conv = nn.Sequential() + self.conv.add_module( + "conv", + nn.Conv2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=7, + padding=3, + groups=in_channels, + bias=False, + ), + ) + + # Fallback, sometimes batchnorm tensors + # do not get instantiated correctly on some processes + # when using deepspeed + accelerate + norm_layer = nn.BatchNorm2d(num_features=out_channels) + if norm_layer.weight.shape[0] == 0: + norm_layer.weight = nn.Parameter(torch.zeros(out_channels)) + if norm_layer.bias.shape[0] == 0: + norm_layer.bias = nn.Parameter(torch.zeros(out_channels)) + + self.conv.add_module("bn", norm_layer) + self.fc1 = nn.Conv2d(in_channels, hidden_channels, kernel_size=1) + self.act = act_layer() + self.fc2 = nn.Conv2d(hidden_channels, out_channels, kernel_size=1) + self.drop = nn.Dropout(drop) + self.apply(self._init_weights) + + def _init_weights(self, m: nn.Module) -> None: + if isinstance(m, nn.Conv2d): + normal_(m.weight, std=0.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.conv(x) + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class RepCPE(nn.Module): + """Implementation of conditional positional encoding. + + For more details refer to paper: + `Conditional Positional Encodings for Vision Transformers `_ + + In our implementation, we can reparameterize this module to eliminate a skip connection. + """ + + def __init__( + self, + in_channels: int, + embed_dim: int = 768, + spatial_shape: Union[int, Tuple[int, int]] = (7, 7), + inference_mode=False, + ) -> None: + """Build reparameterizable conditional positional encoding + + Args: + in_channels: Number of input channels. + embed_dim: Number of embedding dimensions. Default: 768 + spatial_shape: Spatial shape of kernel for positional encoding. Default: (7, 7) + inference_mode: Flag to instantiate block in inference mode. Default: ``False`` + """ + super(RepCPE, self).__init__() + if isinstance(spatial_shape, int): + spatial_shape = tuple([spatial_shape] * 2) + assert isinstance(spatial_shape, Tuple), ( + f'"spatial_shape" must by a sequence or int, ' + f"get {type(spatial_shape)} instead." + ) + assert len(spatial_shape) == 2, ( + f'Length of "spatial_shape" should be 2, ' + f"got {len(spatial_shape)} instead." + ) + + self.spatial_shape = spatial_shape + self.embed_dim = embed_dim + self.in_channels = in_channels + self.groups = embed_dim + + if inference_mode: + self.reparam_conv = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.embed_dim, + kernel_size=self.spatial_shape, + stride=1, + padding=int(self.spatial_shape[0] // 2), + groups=self.embed_dim, + bias=True, + ) + else: + self.pe = nn.Conv2d( + in_channels, + embed_dim, + spatial_shape, + 1, + int(spatial_shape[0] // 2), + bias=True, + groups=embed_dim, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if hasattr(self, "reparam_conv"): + x = self.reparam_conv(x) + return x + else: + x = self.pe(x) + x + return x + + def reparameterize(self) -> None: + # Build equivalent Id tensor + input_dim = self.in_channels // self.groups + kernel_value = torch.zeros( + ( + self.in_channels, + input_dim, + self.spatial_shape[0], + self.spatial_shape[1], + ), + dtype=self.pe.weight.dtype, + device=self.pe.weight.device, + ) + for i in range(self.in_channels): + kernel_value[ + i, + i % input_dim, + self.spatial_shape[0] // 2, + self.spatial_shape[1] // 2, + ] = 1 + id_tensor = kernel_value + + # Reparameterize Id tensor and conv + w_final = id_tensor + self.pe.weight + b_final = self.pe.bias + + # Introduce reparam conv + self.reparam_conv = nn.Conv2d( + in_channels=self.in_channels, + out_channels=self.embed_dim, + kernel_size=self.spatial_shape, + stride=1, + padding=int(self.spatial_shape[0] // 2), + groups=self.embed_dim, + bias=True, + ) + self.reparam_conv.weight.data = w_final + self.reparam_conv.bias.data = b_final + + self.__delattr__("pe") + + +class RepMixerBlock(nn.Module): + """Implementation of Metaformer block with RepMixer as token mixer. + + For more details on Metaformer structure, please refer to: + `MetaFormer Is Actually What You Need for Vision `_ + """ + + def __init__( + self, + dim: int, + kernel_size: int = 3, + mlp_ratio: float = 4.0, + act_layer: nn.Module = nn.GELU, + drop: float = 0.0, + drop_path: float = 0.0, + use_layer_scale: bool = True, + layer_scale_init_value: float = 1e-5, + inference_mode: bool = False, + ): + """Build RepMixer Block. + + Args: + dim: Number of embedding dimensions. + kernel_size: Kernel size for repmixer. Default: 3 + mlp_ratio: MLP expansion ratio. Default: 4.0 + act_layer: Activation layer. Default: ``nn.GELU`` + drop: Dropout rate. Default: 0.0 + drop_path: Drop path rate. Default: 0.0 + use_layer_scale: Flag to turn on layer scale. Default: ``True`` + layer_scale_init_value: Layer scale value at initialization. Default: 1e-5 + inference_mode: Flag to instantiate block in inference mode. Default: ``False`` + """ + + super().__init__() + + self.token_mixer = RepMixer( + dim, + kernel_size=kernel_size, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + inference_mode=inference_mode, + ) + + assert mlp_ratio > 0, "MLP ratio should be greater than 0, found: {}".format( + mlp_ratio + ) + mlp_hidden_dim = int(dim * mlp_ratio) + self.convffn = ConvFFN( + in_channels=dim, + hidden_channels=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) + + # Drop Path + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + # Layer Scale + self.use_layer_scale = use_layer_scale + if use_layer_scale: + self.layer_scale = nn.Parameter( + layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True + ) + + def forward(self, x): + if self.use_layer_scale: + x = self.token_mixer(x) + x = x + self.drop_path(self.layer_scale * self.convffn(x)) + else: + x = self.token_mixer(x) + x = x + self.drop_path(self.convffn(x)) + return x + + +class AttentionBlock(nn.Module): + """Implementation of metaformer block with MHSA as token mixer. + + For more details on Metaformer structure, please refer to: + `MetaFormer Is Actually What You Need for Vision `_ + """ + + def __init__( + self, + dim: int, + mlp_ratio: float = 4.0, + act_layer: nn.Module = nn.GELU, + norm_layer: nn.Module = nn.BatchNorm2d, + drop: float = 0.0, + drop_path: float = 0.0, + use_layer_scale: bool = True, + layer_scale_init_value: float = 1e-5, + ): + """Build Attention Block. + + Args: + dim: Number of embedding dimensions. + mlp_ratio: MLP expansion ratio. Default: 4.0 + act_layer: Activation layer. Default: ``nn.GELU`` + norm_layer: Normalization layer. Default: ``nn.BatchNorm2d`` + drop: Dropout rate. Default: 0.0 + drop_path: Drop path rate. Default: 0.0 + use_layer_scale: Flag to turn on layer scale. Default: ``True`` + layer_scale_init_value: Layer scale value at initialization. Default: 1e-5 + """ + + super().__init__() + + # Fallback, sometimes batchnorm tensors + # do not get instantiated correctly on some processes + # when using deepspeed + accelerate + norm_layer_ = norm_layer(num_features=dim) + if norm_layer_.weight.shape[0] == 0: + norm_layer_.weight = nn.Parameter(torch.zeros(dim)) + if norm_layer_.bias.shape[0] == 0: + norm_layer_.bias = nn.Parameter(torch.zeros(dim)) + + self.norm = norm_layer_ + self.token_mixer = MHSA(dim=dim) + + assert mlp_ratio > 0, "MLP ratio should be greater than 0, found: {}".format( + mlp_ratio + ) + mlp_hidden_dim = int(dim * mlp_ratio) + self.convffn = ConvFFN( + in_channels=dim, + hidden_channels=mlp_hidden_dim, + act_layer=act_layer, + drop=drop, + ) + + # Drop path + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + + # Layer Scale + self.use_layer_scale = use_layer_scale + if use_layer_scale: + self.layer_scale_1 = nn.Parameter( + layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True + ) + self.layer_scale_2 = nn.Parameter( + layer_scale_init_value * torch.ones((dim, 1, 1)), requires_grad=True + ) + + def forward(self, x): + if self.use_layer_scale: + x = x + self.drop_path(self.layer_scale_1 * self.token_mixer(self.norm(x))) + x = x + self.drop_path(self.layer_scale_2 * self.convffn(x)) + else: + x = x + self.drop_path(self.token_mixer(self.norm(x))) + x = x + self.drop_path(self.convffn(x)) + return x + + +def basic_blocks( + dim: int, + block_index: int, + num_blocks: List[int], + token_mixer_type: str, + kernel_size: int = 3, + mlp_ratio: float = 4.0, + act_layer: nn.Module = nn.GELU, + norm_layer: nn.Module = nn.BatchNorm2d, + drop_rate: float = 0.0, + drop_path_rate: float = 0.0, + use_layer_scale: bool = True, + layer_scale_init_value: float = 1e-5, + inference_mode=False, +) -> nn.Sequential: + """Build FastViT blocks within a stage. + + Args: + dim: Number of embedding dimensions. + block_index: block index. + num_blocks: List containing number of blocks per stage. + token_mixer_type: Token mixer type. + kernel_size: Kernel size for repmixer. + mlp_ratio: MLP expansion ratio. + act_layer: Activation layer. + norm_layer: Normalization layer. + drop_rate: Dropout rate. + drop_path_rate: Drop path rate. + use_layer_scale: Flag to turn on layer scale regularization. + layer_scale_init_value: Layer scale value at initialization. + inference_mode: Flag to instantiate block in inference mode. + + Returns: + nn.Sequential object of all the blocks within the stage. + """ + blocks = [] + for block_idx in range(num_blocks[block_index]): + block_dpr = ( + drop_path_rate + * (block_idx + sum(num_blocks[:block_index])) + / (sum(num_blocks) - 1) + ) + if token_mixer_type == "repmixer": + blocks.append( + RepMixerBlock( + dim, + kernel_size=kernel_size, + mlp_ratio=mlp_ratio, + act_layer=act_layer, + drop=drop_rate, + drop_path=block_dpr, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + inference_mode=inference_mode, + ) + ) + elif token_mixer_type == "attention": + blocks.append( + AttentionBlock( + dim, + mlp_ratio=mlp_ratio, + act_layer=act_layer, + norm_layer=norm_layer, + drop=drop_rate, + drop_path=block_dpr, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + ) + ) + else: + raise ValueError( + "Token mixer type: {} not supported".format(token_mixer_type) + ) + blocks = nn.Sequential(*blocks) + return blocks + + +class GlobalPool2D(nn.Module): + """This class implements global pooling with linear projection.""" + + def __init__(self, in_dim: int, out_dim: int, *args, **kwargs) -> None: + super().__init__() + scale = in_dim**-0.5 + self.proj = nn.Parameter(scale * torch.randn(size=(in_dim, out_dim))) + self.in_dim = in_dim + self.out_dim = out_dim + + def pool(self, x) -> Tensor: + if x.dim() == 4: + dims = [-2, -1] + elif x.dim() == 5: + dims = [-3, -2, -1] + x = torch.mean(x, dim=dims, keepdim=False) + return x + + def forward(self, x: Tensor, *args, **kwargs) -> Tensor: + # x is of shape [batch, in_dim] + assert ( + x.dim() == 4 + ), "Input should be 4-dimensional (Batch x in_dim x in_height x in_width). Got: {}".format( + x.shape + ) + + # [batch, in_dim, in_height, in_width] --> [batch, in_dim] + x = self.pool(x) + # [batch, in_dim] x [in_dim, out_dim] --> [batch, out_dim] + x = x @ self.proj + return x + + +class FastViT(nn.Module): + """ + This class implements `FastViT architecture `_ + """ + + def __init__( + self, + layers, + token_mixers: Tuple[str, ...], + embed_dims=None, + mlp_ratios=None, + downsamples=None, + se_downsamples=None, + repmixer_kernel_size=3, + norm_layer: nn.Module = nn.BatchNorm2d, + act_layer: nn.Module = nn.GELU, + num_classes=1000, + pos_embs=None, + down_patch_size=7, + down_stride=2, + drop_rate=0.0, + drop_path_rate=0.0, + use_layer_scale=True, + layer_scale_init_value=1e-5, + init_cfg=None, + pretrained=None, + cls_ratio=2.0, + inference_mode=False, + stem_scale_branch=True, + **kwargs, + ) -> None: + + super().__init__() + + self.num_classes = num_classes + if len(layers) == 4: + self.out_indices = [0, 2, 4, 7] + elif len(layers) == 5: + self.out_indices = [0, 2, 4, 7, 10] + else: + raise NotImplementedError("FPN is not implemented for more than 5 stages.") + + if pos_embs is None: + pos_embs = [None] * len(layers) + + if se_downsamples is None: + se_downsamples = [False] * len(layers) + + # Convolutional stem + self.patch_embed = convolutional_stem(3, embed_dims[0], inference_mode, + use_scale_branch=stem_scale_branch) + + # Build the main stages of the network architecture + network = [] + for i in range(len(layers)): + # Add position embeddings if requested + if pos_embs[i] is not None: + network.append( + pos_embs[i]( + embed_dims[i], embed_dims[i], inference_mode=inference_mode + ) + ) + stage = basic_blocks( + embed_dims[i], + i, + layers, + token_mixer_type=token_mixers[i], + kernel_size=repmixer_kernel_size, + mlp_ratio=mlp_ratios[i], + act_layer=act_layer, + norm_layer=norm_layer, + drop_rate=drop_rate, + drop_path_rate=drop_path_rate, + use_layer_scale=use_layer_scale, + layer_scale_init_value=layer_scale_init_value, + inference_mode=inference_mode, + ) + network.append(stage) + if i >= len(layers) - 1: + break + + # Patch merging/downsampling between stages. + if downsamples[i] or embed_dims[i] != embed_dims[i + 1]: + network.append( + PatchEmbed( + patch_size=down_patch_size, + stride=down_stride, + in_channels=embed_dims[i], + embed_dim=embed_dims[i + 1], + inference_mode=inference_mode, + use_se=se_downsamples[i + 1], + ) + ) + self.network = nn.ModuleList(network) + + # Classifier head + self.conv_exp = MobileOneBlock( + in_channels=embed_dims[-1], + out_channels=int(embed_dims[-1] * cls_ratio), + kernel_size=3, + stride=1, + padding=1, + groups=embed_dims[-1], + inference_mode=inference_mode, + use_se=True, + num_conv_branches=1, + ) + self.head = ( + nn.Linear(int(embed_dims[-1] * cls_ratio), num_classes) + if num_classes > 0 + else nn.Identity() + ) + self.apply(self.cls_init_weights) + self.init_cfg = copy.deepcopy(init_cfg) + + def cls_init_weights(self, m: nn.Module) -> None: + """Init. for classification""" + if isinstance(m, nn.Linear): + normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + + def forward_embeddings(self, x: torch.Tensor) -> torch.Tensor: + x = self.patch_embed(x) + return x + + def forward_tokens(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: + for idx, block in enumerate(self.network): + x = block(x) + return x + + def forward(self, x: torch.Tensor, *args, **kwargs) -> Union[Tensor, Dict[str, Tensor]]: + # input embedding + x = self.forward_embeddings(x) + # through backbone + x = self.forward_tokens(x) + # for image classification/embedding + x = self.conv_exp(x) + cls_out = self.head(x) + + out_dict = dict() + if kwargs.get("return_image_embeddings", False): + out_dict.update({"logits": cls_out}) + out_dict.update({"image_embeddings": x}) + return out_dict + else: + return cls_out + + +@register_model +def fastvithd(pretrained=False, **kwargs): + """Instantiate FastViTHD model variant.""" + layers = [2, 12, 24, 4, 2] + embed_dims = [96, 192, 384, 768, 1536] + mlp_ratios = [4, 4, 4, 4, 4] + downsamples = [True, True, True, True, True] + pos_embs = [None, None, None, partial(RepCPE, spatial_shape=(7, 7)), partial(RepCPE, spatial_shape=(7, 7))] + token_mixers = ("repmixer", "repmixer", "repmixer", "attention", "attention") + model = FastViT( + layers, + token_mixers=token_mixers, + embed_dims=embed_dims, + pos_embs=pos_embs, + mlp_ratios=mlp_ratios, + downsamples=downsamples, + norm_layer=LayerNormChannel, + stem_scale_branch=False, + inference_mode=True, + **kwargs, + ) + model.default_cfg = default_cfgs["fastvit_m"] + if pretrained: + raise ValueError("Functionality not implemented.") + return model \ No newline at end of file diff --git a/aiak_training_llm/models/fastvit/mobileclip_encoder.py b/aiak_training_llm/models/fastvit/mobileclip_encoder.py new file mode 100644 index 00000000..4db99252 --- /dev/null +++ b/aiak_training_llm/models/fastvit/mobileclip_encoder.py @@ -0,0 +1,116 @@ +# +# For licensing see accompanying LICENSE file. +# Copyright (C) 2025 Apple Inc. All Rights Reserved. +# +import torch +import torch.nn as nn +import torch.nn.functional as F + +from transformers import CLIPImageProcessor +from aiak_training_llm.models.fastvit import mobileclip + + +class MobileCLIPVisionTower(nn.Module): + def __init__(self, vision_tower, args, delay_load=False): + super().__init__() + + self.is_loaded = False + self.vision_tower_name = vision_tower + self.tune_vision_tower = getattr(args, 'unfreeze_mm_vision_tower', False) + self.input_image_size = int(vision_tower.split("_")[-1]) + + # Delay load is disabled for now + if not delay_load: + self.load_model() + elif getattr(args, 'unfreeze_mm_vision_tower', False): + self.load_model() + else: + model_cfg = mobileclip.load_model_config(self.vision_tower_name) + self.cfg_only = model_cfg + + def load_model(self, device_map=None): + if self.is_loaded: + print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name)) + return + + # Load model config + model_cfg = mobileclip.load_model_config(self.vision_tower_name) + + # Override default image resolution + model_cfg["image_cfg"]["image_size"] = self.input_image_size + + self.cfg_only = model_cfg + + # Build HF CLIPImageProcessor with MobileCLIP parameters + self.image_processor = CLIPImageProcessor(crop_size={"height": model_cfg["image_cfg"]["image_size"], + "width": model_cfg["image_cfg"]["image_size"]}, + image_mean=[0.0, 0.0, 0.0], + image_std=[1.0, 1.0, 1.0], + size={"shortest_edge": model_cfg["image_cfg"]["image_size"]}) + + # Instantiate the image encoder + self.vision_tower = mobileclip.MCi(model_name=model_cfg["image_cfg"]["model_name"], + projection_dim=model_cfg["embed_dim"]) + + if not self.tune_vision_tower: + self.vision_tower.requires_grad_(False) + + self.is_loaded = True + + def feature_select(self, image_forward_outs): + # Features from penultimate layer + image_features = image_forward_outs["image_embeddings"] + + # Reshape 4D tensor to 3D + B, C, H, W = image_features.shape + image_features = image_features.reshape(B, C, H*W) + image_features = image_features.transpose(1, 2) + return image_features + + def forward(self, images): + if self.tune_vision_tower: + return self.forward_images(images) + else: + with torch.no_grad(): + return self.forward_images(images) + + def forward_images(self, images): + if type(images) is list: + image_features = [] + for image in images: + image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), return_image_embeddings=True) + image_feature = self.feature_select(image_forward_out).to(image.dtype) + image_features.append(image_feature) + else: + image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), return_image_embeddings=True) + image_features = self.feature_select(image_forward_outs).to(images.dtype) + + return image_features + + @property + def dummy_feature(self): + return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype) + + @property + def dtype(self): + return next(self.vision_tower.parameters()).dtype + + @property + def device(self): + return next(self.vision_tower.parameters()).device + + @property + def config(self): + return self.cfg_only + + @property + def hidden_size(self): + return self.config["image_cfg"]["embed_dim"] + + @property + def num_patches_per_side(self): + return self.config["image_cfg"]["image_size"] // self.config["image_cfg"]["patch_size"] + + @property + def num_patches(self): + return (self.config["image_cfg"]["image_size"] // self.config["image_cfg"]["patch_size"]) ** 2 \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/adapter.py b/aiak_training_llm/models/llavaov_1_5/adapter.py index 57544bbc..aeb87c59 100644 --- a/aiak_training_llm/models/llavaov_1_5/adapter.py +++ b/aiak_training_llm/models/llavaov_1_5/adapter.py @@ -26,6 +26,7 @@ def __init__(self, ) -> None: super().__init__(config=config) self.hidden_size = input_size * (spatial_merge_size**2) + print(f"[DEBUG] Adapter.__init__: input_size={input_size}, output_size={output_size}, hidden_size={self.hidden_size}") self.layernorm = build_module( submodules.layernorm, @@ -62,6 +63,7 @@ def __init__(self, def forward(self, x: torch.Tensor, window_index: torch.LongTensor = None) -> torch.Tensor: """ Forward pass.""" + print(f"[DEBUG] Adapter.forward: input shape={x.shape}, expected input_size={self.layernorm.weight.shape[0]}, hidden_size={self.hidden_size}") x = self.layernorm(x).view(-1, self.hidden_size) x, _ = self.linear_fc1(x) x = self.activation_func(x) diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py index b88c8ce6..5afef385 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py @@ -191,7 +191,8 @@ def get_vision_config(model_family, model_name): """ get vision config """ config = VisionConfig( num_layers=24, - hidden_size=1024, + # hidden_size=1024, + hidden_size=3072, ffn_hidden_size=4096, num_attention_heads=16, patch_size=14, diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index f063ea37..c00c6239 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -18,6 +18,7 @@ from aiak_training_llm.models.llavaov_1_5.rice_vision_model import ( RiceViTModel, VisionModel) +from aiak_training_llm.models.fastvit import FastViTModel from aiak_training_llm.models.qwen import QwenModel from aiak_training_llm.models.qwen_vl.adapter import Adapter from aiak_training_llm.models.qwen_vl.utils import get_inputs_on_this_cp_rank @@ -161,11 +162,17 @@ def __init__( # define the vision model and the projection from vision model outputs to language model inputs. if self.add_encoder: - # if vision_config.normalization == "RMSNorm": - self.vision_model = RiceViTModel( + # Use FastViT as the vision encoder (following FastVLM repo) + self.vision_model = FastViTModel( vision_config, vision_layer_spec, ) + # Original Rice/SigLIP encoder (commented out): + # if vision_config.normalization == "RMSNorm": + # self.vision_model = RiceViTModel( + # vision_config, + # vision_layer_spec, + # ) # else: # self.vision_model = VisionModel( # vision_config, @@ -173,8 +180,8 @@ def __init__( # ) # Map (intermediate) vision model outputs to the language model input dimension. # from megatron.training import print_rank_0 - # print_rank_0(f"vision_config.hidden_size: {vision_config.hidden_size}") - # print_rank_0(f"language_config.hidden_size: {language_config.hidden_size}") + print(f"[DEBUG] Creating adapter: vision_config.hidden_size={vision_config.hidden_size}") + print(f"[DEBUG] Creating adapter: language_config.hidden_size={language_config.hidden_size}") self.adapter = Adapter( adapter_config, adapter_layer_spec, diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index a39fc215..4800b3b4 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -55,6 +55,13 @@ def rice_vl_model_provider( # get vision specific config : no. of layers, hidden size, Patch size, Image resolution for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): setattr(vision_config, k, v) + + # Add FastViT-specific configuration if enabled + if getattr(args, 'use_fastvit', False): + vision_tower_name = getattr(args, 'vision_tower_name', 'mobileclip_l_384') + setattr(vision_config, 'vision_tower_name', vision_tower_name) + print_rank_0(f'Using FastViT with vision_tower_name: {vision_tower_name}') + print_rank_0(f'Vision config: {vision_config}') # get adapter specific config : Projection dimension, Activation function for k, v in asdict(get_adapeter_config(model_family)).items(): @@ -142,6 +149,8 @@ def rice_vl_model_provider( language_rotary_percent=args.rotary_percent, language_rotary_base=args.rotary_base, seq_len_interpolation_factor=args.rotary_seq_len_interpolation_factor, + # When using FastViT, adapter dimensions change, so allow missing adapter weights + allow_missing_adapter_checkpoint=getattr(args, 'use_fastvit', False), ) # Vision encoder: SigLIP with 27 layers # Adapter: Projection network diff --git a/aiak_training_llm/train/arguments.py b/aiak_training_llm/train/arguments.py index 348734b1..5724b3aa 100644 --- a/aiak_training_llm/train/arguments.py +++ b/aiak_training_llm/train/arguments.py @@ -415,6 +415,22 @@ def _add_extra_multimodal_args(parser): group.add_argument('--fps-max-frames', type=int, default=768, help='The maximum number of frames of the video') + + # FastViT specific arguments (following FastVLM repo) + group.add_argument('--use-fastvit', action='store_true', default=False, + help='Use FastViT vision encoder instead of Rice/SigLIP') + group.add_argument('--fastvit-image-size', type=int, default=384, + help='FastViT input image size (default: 384)') + group.add_argument('--vision-tower-name', type=str, default='mobileclip_l_384', + help='FastViT model variant (mobileclip_l_384, mobileclip_l_448, mobileclip_l_512)') + group.add_argument('--image-aspect-ratio', type=str, default='pad', + choices=['pad', 'anyres', 'square'], + help='Image aspect ratio handling: pad (expand to square with padding), ' + 'anyres (variable resolution with patches), square (direct resize)') + group.add_argument('--image-grid-pinpoints', type=str, + default='[(384, 384), (768, 384), (384, 768), (768, 768)]', + help='Grid pinpoints for anyres image processing') + return parser diff --git a/aiak_training_llm/train/training_utils.py b/aiak_training_llm/train/training_utils.py index e5df5ef3..bee461c2 100644 --- a/aiak_training_llm/train/training_utils.py +++ b/aiak_training_llm/train/training_utils.py @@ -403,6 +403,16 @@ def setup_model_and_optimizer(model_provider_func, optimizer.reload_model_params() print_rank_0(f'Upcycled checkpoint saved to {args.save}') + # When using FastViT, skip checkpoint loading to avoid dimension mismatch + # FastViT outputs 3072 dims vs Qwen2-VL's 1024 dims, causing adapter incompatibility + if getattr(args, 'use_fastvit', False): + print_rank_0(f'[DEBUG] FastViT enabled: use_fastvit={getattr(args, "use_fastvit", False)}') + print_rank_0(f'[DEBUG] Before skip: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') + args.load = None + args.pretrained_checkpoint = None + print_rank_0(f'[DEBUG] After skip: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') + print_rank_0('FastViT enabled: Skipping checkpoint loading, training from scratch') + if (args.load is not None or args.pretrained_checkpoint is not None) and not args.moe_use_upcycling: timers('load-checkpoint', log_level=0).start(barrier=True) diff --git a/examples/llava_ov_1_5/convert/convert_4b_mcore_to_hf.sh b/examples/llava_ov_1_5/convert/convert_4b_mcore_to_hf.sh index e191c976..641a7fb1 100755 --- a/examples/llava_ov_1_5/convert/convert_4b_mcore_to_hf.sh +++ b/examples/llava_ov_1_5/convert/convert_4b_mcore_to_hf.sh @@ -1,43 +1,51 @@ +# Setup paths +# set base training path AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/workspace/LLaVA-OneVision-1.5}" +#megatron path AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" +# conversion script path CONVERT_CHECKPOINT_PATH="$AIAK_TRAINING_PATH/tools/convert_checkpoint" -LOAD=$1 -SAVE=$2 -TP=$3 -PP=$4 +LOAD=$1 # source checkpoint directory (Megatron format) +SAVE=$2 # Destination checkpoint directory (Hugging Face format) +TP=$3 # tensor parallel size +PP=$4 # pipeline parallel size +# Usage: bash script.sh +# create temp directories mkdir -p ./tmp/ -SAVE_LANGUAGE_MODEL=./tmp/language-mcore -SAVE_VISION_MODEL=./tmp/vision-model-mcore -SAVE_ADAPTER=./tmp/adapter-mcore -SAVE_PATCH=./tmp/patch-mcore +SAVE_LANGUAGE_MODEL=./tmp/language-mcore # temp dir for Qwen language model +SAVE_VISION_MODEL=./tmp/vision-model-mcore # temp dir for Rice vision encoder +SAVE_ADAPTER=./tmp/adapter-mcore # temp dir for projection adapter +SAVE_PATCH=./tmp/patch-mcore # temp dir for vision patch embedding +#These temporary directories hold intermediate conversions before final merge. # llama: language expert python $CONVERT_CHECKPOINT_PATH/model.py \ - --load_platform=mcore \ + --load_platform=mcore \ # load from Megatron MCore format --megatron_path $AIAK_MAGATRON_PATH \ - --save_platform=huggingface \ - --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/qwen3.json \ - --tensor_model_parallel_size=$TP \ + --save_platform=huggingface \ #save to Hugging Face format + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/qwen3.json \ #config file for Qwen3 + --tensor_model_parallel_size=$TP \ --pipeline_model_parallel_size=$PP \ - --load_ckpt_path=$LOAD \ - --save_ckpt_path=$SAVE_LANGUAGE_MODEL \ - --safetensors \ - --no_save_optim \ - --no_load_optim + --load_ckpt_path=$LOAD \ #source megatron checkpoint + --save_ckpt_path=$SAVE_LANGUAGE_MODEL \ #output temp directory + --safetensors \ #save as .safetensors format + --no_save_optim \ #do not save optimizer states + --no_load_optim ##do not load optimizer states # vit if [[ $PP -eq 1 ]]; then - LOAD_PATH=$LOAD + LOAD_PATH=$LOAD #if no pipeline parallelism, load directly from source else + # If pipeline parallel, vision is only on rank 0, need to reorganize LOAD_PATH=$LOAD/tmp/ mkdir -p $LOAD_PATH for ((i=0;i<$TP;i++)); do - from=`printf "mp_rank_%02d_000" $i` - to=`printf "mp_rank_%02d" $i` - cp -r $LOAD/$from $LOAD_PATH/$to + from=`printf "mp_rank_%02d_000" $i` # mp_rank_00_000, mp_rank_01_000, etc. + to=`printf "mp_rank_%02d" $i` # mp_rank_00, mp_rank_01, etc. + cp -r $LOAD/$from $LOAD_PATH/$to # Copy first pipeline stage to new location done fi @@ -45,20 +53,21 @@ python $CONVERT_CHECKPOINT_PATH/model.py \ --load_platform=mcore \ --save_platform=huggingface \ --megatron_path $AIAK_MAGATRON_PATH \ - --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/vision-model.json \ + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/vision-model.json \ # Config for Rice vision model --tensor_model_parallel_size=$TP \ - --pipeline_model_parallel_size=1 \ - --load_ckpt_path=$LOAD_PATH \ + --pipeline_model_parallel_size=1 \ # Vision is NOT pipeline parallel (always 1) + --load_ckpt_path=$LOAD_PATH \ # Use reorganized path if PP>1 --save_ckpt_path=$SAVE_VISION_MODEL \ --safetensors \ --no_save_optim \ --no_load_optim if [[ $LOAD != $LOAD_PATH ]]; then - rm -rf $LOAD_PATH + rm -rf $LOAD_PATH # Remove temporary reorganized directory if created fi # adapter +#Extracts adapter/projection layer weights (vision features → language embedding space python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/adapter.py \ --load_platform=mcore \ --save_platform=huggingface \ @@ -70,6 +79,7 @@ python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/adapter.py \ --save_ckpt_path=$SAVE_ADAPTER # vision patch +# Convert Vision Patch Embedding python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/vision_patch.py \ --load_platform=mcore \ --save_platform=huggingface \ @@ -89,7 +99,7 @@ python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/merge_huggingface.py \ --adapter_path $SAVE_ADAPTER \ --save_ckpt_path $SAVE - +# Delete temporary intermediate files, keep only final merged checkpoint. rm -rf $SAVE_LANGUAGE_MODEL rm -rf $SAVE_VISION_MODEL rm -rf $SAVE_ADAPTER diff --git a/examples/llava_ov_1_5/convert/convert_fastvlm_hf_to_mcore.sh b/examples/llava_ov_1_5/convert/convert_fastvlm_hf_to_mcore.sh new file mode 100755 index 00000000..365a8586 --- /dev/null +++ b/examples/llava_ov_1_5/convert/convert_fastvlm_hf_to_mcore.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Activate conda environment +source ~/miniconda3/etc/profile.d/conda.sh +conda activate llava-ov-4b-clean + +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/workspace/LLaVA-OneVision-1.5}" +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" +CONVERT_CHECKPOINT_PATH="$AIAK_TRAINING_PATH/tools/convert_checkpoint" + +LOAD=$1 +SAVE=$2 +TP=$3 +PP=$4 + +mkdir -p ./tmp/ +SAVE_LANGUAGE_MODEL=./tmp/language-mcore +SAVE_VISION_MODEL=./tmp/vision-model-mcore +SAVE_ADAPTER=./tmp/adapter-mcore + +# llm (Now compatible: FastVLM Qwen2-1.5B matches our model config) +python $CONVERT_CHECKPOINT_PATH/model.py \ + --load_platform=huggingface \ + --save_platform=mcore \ + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/qwen3.json \ + --tensor_model_parallel_size=$TP \ + --pipeline_model_parallel_size=$PP \ + --load_ckpt_path=$LOAD \ + --save_ckpt_path=$SAVE_LANGUAGE_MODEL \ + --safetensors \ + --no_save_optim \ + --no_load_optim + +if [ $? -ne 0 ]; then + echo "Language model conversion failed!" + exit 1 +fi + +# fastvit +python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/fastvit.py \ + --load_platform=huggingface \ + --save_platform=mcore \ + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/fastvit.json \ + --tensor_model_parallel_size=$TP \ + --load_ckpt_path=$LOAD \ + --save_ckpt_path=$SAVE_VISION_MODEL \ + --safetensors \ + --no_save_optim \ + --no_load_optim + +if [ $? -ne 0 ]; then + echo "FastViT conversion failed!" + exit 1 +fi + +# adapter +python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/adapter.py \ + --load_platform=huggingface \ + --save_platform=mcore \ + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/adapter.json \ + --tensor_model_parallel_size=$TP \ + --load_ckpt_path=$LOAD \ + --save_ckpt_path=$SAVE_ADAPTER + +if [ $? -ne 0 ]; then + echo "Adapter conversion failed!" + exit 1 +fi + +# merge (all 3 components now compatible!) +python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/merge_megatron.py \ + --megatron_path $AIAK_MAGATRON_PATH \ + --language_model_path $SAVE_LANGUAGE_MODEL/release \ + --vision_model_path $SAVE_VISION_MODEL/release \ + --adapter_path $SAVE_ADAPTER/release \ + --save_ckpt_path $SAVE/release \ + --tensor_model_parallel_size $TP \ + --pipeline_model_parallel_size $PP + +if [ $? -ne 0 ]; then + echo "Merge failed!" + exit 1 +fi + +echo "Conversion completed successfully!" +echo release > $SAVE/latest_checkpointed_iteration.txt +rm -rf $SAVE_LANGUAGE_MODEL +rm -rf $SAVE_VISION_MODEL +rm -rf $SAVE_ADAPTER + diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 92daa264..e5a8e74f 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -18,7 +18,7 @@ NSTEP="${6:-20}" # number of training iterations # CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" TOKENIZER_PATH="${TOKENIZER_PATH:-"$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0"}" -CHECKPOINT_PATH="${CHECKPOINT_PATH:-"$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1"}" +CHECKPOINT_PATH="${CHECKPOINT_PATH:-"$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"}" echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" echo "DATA_PATH=${DATA_PATH}" echo "TOKENIZER_PATH=${TOKENIZER_PATH}" @@ -122,6 +122,13 @@ DATA_ARGS=( --split 100,0,0 --num-workers 16 --chat-template qwen2-vl # will change this chat template + + # FastViT configuration (following FastVLM repo) + --use-fastvit # Enable FastViT vision encoder + --fastvit-image-size 384 # FastViT input resolution (384, 448, or 512) + --vision-tower-name mobileclip_l_384 # FastViT model variant (mobileclip_l_{resolution}) + --image-aspect-ratio pad # Image preprocessing: pad (square with padding), anyres (multi-res), square (direct resize) + # --image-grid-pinpoints "[(384, 384), (768, 384), (384, 768), (768, 768)]" # Uncomment for anyres mode ) TRAINING_ARGS=( @@ -151,7 +158,7 @@ TRAINING_ARGS=( --lr-warmup-fraction 0.002 --initial-loss-scale 65536 --bf16 - --load "$CHECKPOINT_PATH" # load initial checkpoint. + # --load "$REPO_ROOT/checkpoints/llava-fastvithd_1.5b_mcore_tp2_pp1" # Training from scratch --save "$SAVE_CKPT_PATH" # where to save new checkpoints. --save-interval 2000 --ckpt-format torch diff --git a/requirements.local.txt b/requirements.local.txt new file mode 100644 index 00000000..d91a9a81 --- /dev/null +++ b/requirements.local.txt @@ -0,0 +1,125 @@ +absl-py==2.3.1 +accelerate==1.9.0 +AIAK-Training-LLM @ file:///share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5 +aiobotocore==2.26.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.13.2 +aioitertools==0.13.0 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anykeystore==0.2 +async-timeout==5.0.1 +attrs==25.4.0 +av==16.0.1 +botocore==1.41.5 +braceexpand==0.1.7 +build==1.3.0 +cfgv==3.5.0 +click==8.3.1 +cryptacular==1.6.2 +datasets==2.19.2 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.4.0 +einops==0.6.0 +filelock==3.19.1 +frozenlist==1.8.0 +fsspec==2024.3.1 +ftfy==6.1.3 +gitdb==4.0.12 +GitPython==3.1.45 +greenlet==3.3.0 +grpcio==1.76.0 +hf-xet==1.2.0 +huggingface-hub==0.36.0 +hupper==1.12.1 +hydra-core==1.3.2 +identify==2.6.15 +importlib_metadata==8.7.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +Levenshtein==0.27.1 +Markdown==3.10 +megatron-energon==5.0.0 +mkl-service==2.5.2 +mpmath==1.3.0 +multidict==6.7.0 +multiprocess==0.70.16 +networkx==3.3 +ninja==1.13.0 +nltk==3.9.1 +nodeenv==1.9.1 +numpy==2.2.6 +oauthlib==3.3.1 +omegaconf==2.3.0 +packaging==25.0 +pandarallel==1.6.5 +pandas==2.3.3 +PasteDeploy==3.1.0 +pbkdf2==1.3 +pillow==11.3.0 +plaster==1.1.2 +plaster-pastedeploy==1.0.1 +platformdirs==4.5.1 +pre-commit==3.2.2 +propcache==0.4.1 +protobuf==3.20.2 +psutil==7.1.3 +py-cpuinfo==9.0.0 +pyarrow==22.0.0 +pyarrow-hotfix==0.7 +pybind11==3.0.1 +pydantic==2.12.5 +pydantic_core==2.41.5 +pyproject_hooks==1.2.0 +pyramid==2.0.2 +pyramid-mailer==0.15.1 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python3-openid==3.2.0 +pytz==2025.2 +qwen-vl-utils==0.0.14 +RapidFuzz==3.14.2 +regex==2025.11.3 +repoze.sendmail==4.4.1 +requests-oauthlib==2.0.0 +s3fs==2024.3.1 +safetensors==0.7.0 +sentencepiece==0.1.98 +sentry-sdk==2.47.0 +six==1.17.0 +smmap==5.0.2 +SQLAlchemy==2.0.45 +tensorboard==2.20.0 +tensorboard-data-server==0.7.2 +tiktoken==0.5.2 +timm==1.0.3 +tokenizers==0.21.4 +tomli==2.3.0 +tqdm==4.67.1 +transaction==5.0 +transformers==4.53.1 +translationstring==1.4 +triton==3.5.1 +typing-inspection==0.4.2 +typing_extensions==4.15.0 +tzdata==2025.2 +velruse==1.1.1 +venusian==3.1.1 +virtualenv==20.35.4 +wandb==0.21.0 +wcwidth==0.2.14 +webdataset==1.0.2 +WebOb==1.8.9 +Werkzeug==3.1.4 +wrapt==1.16.0 +WTForms==3.2.1 +wtforms-recaptcha==0.3.2 +xxhash==3.6.0 +yarl==1.22.0 +zipp==3.23.0 +zope.deprecation==6.0 +zope.interface==8.1.1 +zope.sqlalchemy==4.1 diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json b/tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json new file mode 100644 index 00000000..48cf997e --- /dev/null +++ b/tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json @@ -0,0 +1,8 @@ +{ + "model_type": "fastvit", + "vision_model_from": "model.vision_tower.vision_tower", + "vision_model_to": "vision_model.vision_tower", + "num_layers": null, + "hidden_size": 3072, + "note": "FastViT/MobileCLIP vision encoder with 629 parameters" +} diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py b/tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py new file mode 100644 index 00000000..c75a0d73 --- /dev/null +++ b/tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py @@ -0,0 +1,96 @@ +""" +Convert FastViT vision encoder weights from HuggingFace to Megatron format. +""" +import argparse +import os +import sys +import torch +from safetensors import safe_open + +def convert_fastvit_weights(load_path, save_path, config): + """ + Convert FastViT weights from HF format to Megatron format. + + HF format: model.vision_tower.vision_tower.model.* + Megatron format: vision_model.vision_tower.model.* + """ + print(f"Loading FastViT weights from {load_path}") + + # Load weights from safetensors + hf_weights = {} + safetensors_path = os.path.join(load_path, "model.safetensors") + + with safe_open(safetensors_path, framework="pt", device="cpu") as f: + for key in f.keys(): + if key.startswith("model.vision_tower.vision_tower"): + hf_weights[key] = f.get_tensor(key) + + print(f"Loaded {len(hf_weights)} FastViT weight tensors") + + # Convert keys from HF to Megatron format + # HF: model.vision_tower.vision_tower.model.* + # Megatron: vision_model.vision_tower.model.* + mcore_weights = {} + + vision_model_from = config.get("vision_model_from", "model.vision_tower.vision_tower") + vision_model_to = config.get("vision_model_to", "vision_model.vision_tower") + + for hf_key, weight in hf_weights.items(): + if hf_key.startswith(vision_model_from): + # Strip the HF prefix and add Megatron prefix + suffix = hf_key[len(vision_model_from):] + mcore_key = vision_model_to + suffix + mcore_weights[mcore_key] = weight + + if len(mcore_weights) <= 5: # Show first 5 conversions + print(f" {hf_key} -> {mcore_key}") + + print(f"Converted {len(mcore_weights)} weight tensors") + + # Save in Megatron format + os.makedirs(os.path.join(save_path, "release"), exist_ok=True) + save_file = os.path.join(save_path, "release", "mp_rank_00", "model_optim_rng.pt") + os.makedirs(os.path.dirname(save_file), exist_ok=True) + + checkpoint = { + 'model': mcore_weights, + 'checkpoint_version': 3.0, + 'args': { + 'tensor_model_parallel_size': 1, + 'pipeline_model_parallel_size': 1, + } + } + + torch.save(checkpoint, save_file) + print(f"Saved Megatron checkpoint to {save_file}") + + return mcore_weights + + +def main(): + parser = argparse.ArgumentParser(description="Convert FastViT weights to Megatron format") + parser.add_argument("--load_platform", type=str, default="huggingface") + parser.add_argument("--save_platform", type=str, default="mcore") + parser.add_argument("--common_config_path", type=str, required=True) + parser.add_argument("--load_ckpt_path", type=str, required=True) + parser.add_argument("--save_ckpt_path", type=str, required=True) + parser.add_argument("--tensor_model_parallel_size", type=int, default=1) + parser.add_argument("--safetensors", action="store_true") + parser.add_argument("--no_save_optim", action="store_true") + parser.add_argument("--no_load_optim", action="store_true") + + args = parser.parse_args() + + # Load config + import json + with open(args.common_config_path, 'r') as f: + config = json.load(f) + + # Convert weights + convert_fastvit_weights(args.load_ckpt_path, args.save_ckpt_path, config) + + print("FastViT conversion completed successfully!") + + +if __name__ == "__main__": + main() diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py b/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py new file mode 100644 index 00000000..880d8d8d --- /dev/null +++ b/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved +# +################################################################################ + +import os +import sys +import json +import torch +import argparse +from os.path import dirname +from copy import deepcopy +from einops import rearrange +from safetensors.torch import load_file, save_file + +SCRIPT_DIR = dirname(os.path.abspath(__file__)) +sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) + +from convert_checkpoint.custom.qwen2_vl.util import ( + load_megatron_checkpoint, + save_megatron_checkpoint, +) + + +def parse_args(title=None): + """Parse all arguments.""" + parser = argparse.ArgumentParser(description='Merger Arguments for FastVLM', allow_abbrev=False) + group = parser.add_argument_group(title='checkpoint') + group.add_argument('--language_model_path', type=str, help="Path to language model."), + group.add_argument('--adapter_path', type=str, help="Path to adapter."), + group.add_argument('--vision_model_path', type=str, default=None, help="Path to vision model (FastViT)."), + group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") + group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") + group.add_argument("--tensor_model_parallel_size", type=int, default=1, help="Tensor parallel size.") + group.add_argument("--pipeline_model_parallel_size", type=int, default=1, help="Pipeline parallel size.") + + return parser.parse_args() + + +def merge_dict(source, destination): + """ merge two dictionaries recursively """ + for key, value in source.items(): + if isinstance(value, dict): + node = destination.setdefault(key, {}) + merge_dict(value, node) + else: + destination[key] = value + + +args = parse_args() +if args.megatron_path is not None: + sys.path.insert(0, args.megatron_path) + +print("===== merge megatron checkpoints (FastVLM with vision model) ======") + +language_model = load_megatron_checkpoint(args.language_model_path) +adapter = load_megatron_checkpoint(args.adapter_path) + +# Load vision model if provided +if args.vision_model_path: + vision_model = load_megatron_checkpoint(args.vision_model_path) + print(f"Loaded FastViT vision model from {args.vision_model_path}") + modules_to_merge = [adapter, vision_model] +else: + print("Warning: No vision model provided. FastViT will be initialized randomly.") + modules_to_merge = [adapter] + +# Merge adapter and vision model into language model +# Merge adapter and vision model into language model +if args.pipeline_model_parallel_size == 1: + for module in modules_to_merge: + assert len(module) == len(language_model) + for i in range(len(module)): + merge_dict(module[i]['model'], language_model[i]['model']) +else: + for module in modules_to_merge: + assert len(module) == len(language_model[0]) + for i in range(len(module)): + merge_dict(module[i]['model'], language_model[0][i]['model']) + +save_megatron_checkpoint(language_model, args.save_ckpt_path) +print(f"Checkpoint saved to {args.save_ckpt_path}") +if args.vision_model_path: + print("Successfully merged: Language Model + Adapter + FastViT Vision Model") +else: + print("Merged: Language Model + Adapter (FastViT will be initialized randomly)") From 2eb9bce8521e1365ac863c7c52d40e25b3bac672 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 9 Jan 2026 13:45:28 +0400 Subject: [PATCH 11/39] Add debugging prints and update configs for FastViT integration - Add debug prints in FastViT forward pass (mci.py, mobileclip_encoder.py, fastvit_vision_model.py) - Update GPU configuration to use 2 GPUs (TP=2) in alignment.sh - Add comprehensive code documentation and comments - Update training configuration for FastViT image processing - Add FastViT preprocessing path in qwen2vl_task_encoder.py --- Stage1/alignment.sh | 6 ++--- .../data/multimodal/qwen2vl_task_encoder.py | 25 ++++++++++++++----- .../data/multimodal/task_encoder.py | 6 +++++ .../models/fastvit/fastvit_preprocessor.py | 2 +- .../models/fastvit/fastvit_vision_model.py | 4 ++- .../models/fastvit/mobileclip/__init__.py | 2 +- .../models/fastvit/mobileclip/mci.py | 8 ++++++ .../models/fastvit/mobileclip_encoder.py | 2 ++ .../models/llavaov_1_5/llavaov_1_5_config.py | 1 - .../models/llavaov_1_5/llavaov_1_5_model.py | 1 + .../train/pretrain/pretrain_qwen2_vl.py | 2 +- .../stage_1_alignment_llava_ov_4b.sh | 8 +++--- 12 files changed, 49 insertions(+), 18 deletions(-) diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index bd66fdd8..723e0e74 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -56,7 +56,7 @@ export WANDB_API_KEY="wandb_v1_5y5JqALBMdHhru8CR1gOLflJlRj_O8BG2XRb0S2x0TJVqW1xA export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="fastvit_integration" -export CUDA_VISIBLE_DEVICES=0,1 -export GPUS_PER_NODE=2 +export CUDA_VISIBLE_DEVICES=0 +export GPUS_PER_NODE=1 -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh 2 1 +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 32e9c69d..db8bbc0e 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -137,7 +137,7 @@ def __init__(self, args): # Use this for vision encoding instead of Qwen2-VL's processor self.use_fastvit = getattr(args, 'use_fastvit', False) if self.use_fastvit: - fastvit_image_size = getattr(args, 'fastvit_image_size', 384) + fastvit_image_size = getattr(args, 'fastvit_image_size', 1024) self.fastvit_processor = FastViTImageProcessor(image_size=fastvit_image_size) print(f"Initialized FastViT processor with image_size={fastvit_image_size}") @@ -236,7 +236,7 @@ def _resize_image(self, image, size_factor=28): def _process(self, image, text): """" Process the data to get the model's input """ - + print("Processing image and text...") if self.use_fastvit and image is not None: # FastViT preprocessing using FastVLM's approach # Tokenize text only @@ -251,12 +251,17 @@ def _process(self, image, text): # Process image with FastVLM's preprocessing utilities # Default to 'pad' aspect ratio (expand to square with padding) image_aspect_ratio = getattr(self.args, 'image_aspect_ratio', 'pad') - + print("image_size:", image.size) if image_aspect_ratio == 'pad': # Expand to square with mean color padding mean_color = tuple(int(x * 255) for x in self.fastvit_processor.image_mean) image = expand2square(image, mean_color) pixel_values = self.fastvit_processor(image) + print("image_aspect_ratio: pad") + print(f"Processed padded image to shape: {pixel_values.shape}") + print("size after pad:", image.size) + + elif image_aspect_ratio == 'anyres': # Process with variable resolution (patches) grid_pinpoints = getattr(self.args, 'image_grid_pinpoints', '[(384, 384), (768, 384), (384, 768), (768, 768)]') @@ -454,7 +459,9 @@ def process_sft_qa(self, messages: list, system: str, raw_video: list, raw_image attn_mask = torch.zeros_like(input_ids).bool() # attn_mask: Tensor([False, False, False, ..., False, False]) # Shape: [seq_len] - + print("pixel_values_images:", pixel_values_images) + print("image_grid_thw:", image_grid_thw) + return input_ids, target, attn_mask, pixel_values_images, image_grid_thw, \ pixel_values_videos, video_grid_thw @@ -520,7 +527,7 @@ def encode_vqa4packing(self, sample: VQASample) -> ImageTaskSample: if text[-1] == '\n': text = text[:-1] pass - + print("image_size in encode_vqa4packing:", sample.image.size) input_ids, _, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) target = torch.ones_like(input_ids) * IGNORE_INDEX answers = self.tokenizer.tokenize(sample.answers) @@ -584,12 +591,18 @@ def encode_multi_vid_qa(self, sample: VQASample) -> ImageTaskSample: # For SFT with multi-modal data, it calls: def encode_multi_mix_qa(self, sample: MultiMixQASample) -> ImageTaskSample: """Encode sample in Qwen2VL style.""" + print("Encoding multi-mix qa") if self.args.training_phase == constants.TrainingPhase.SFT: num_tiles = [] #store number of tiles for each image/ video after processing - + print("calling process_sft_qa for multi-mix sample") # call main processing function process_sft_qa input_ids, target, attn_mask, imgs, image_grid_thw, pixel_values_videos, video_grid_thw = \ self.process_sft_qa(sample.messages, sample.system, sample.video, sample.image) + print("imgs:", imgs) + print("image_grid_thw:", image_grid_thw) + print("pixel_values_images:", pixel_values_images) + print("video_grid_thw:", video_grid_thw) + print("pixel_values_videos:", pixel_values_videos) if sample.video is not None: num_tiles = [len(video_grid_thw)] if video_grid_thw is not None else [1] elif sample.image is not None: diff --git a/aiak_training_llm/data/multimodal/task_encoder.py b/aiak_training_llm/data/multimodal/task_encoder.py index d4f94d4d..56cd49c6 100644 --- a/aiak_training_llm/data/multimodal/task_encoder.py +++ b/aiak_training_llm/data/multimodal/task_encoder.py @@ -161,14 +161,20 @@ def __init__(self): @stateless(restore_seeds=True) def encode_sample(self, sample: Union[CaptioningSample, OCRSample, VQASample, SimilarityInterleavedSample]): + print("encoding samples") + print("sample:", sample) """ Generates an encoded sample from a raw sample. """ if isinstance(sample, CaptioningSample): + print("encode_captioning") yield self.encode_captioning(sample) elif isinstance(sample, VQASample): + print("encode_vqa") yield self.encode_vaq(sample) elif isinstance(sample, MultiVidQASample): + print("encode_multi_vid_qa") yield self.encode_multi_vid_qa(sample) elif isinstance(sample, MultiMixQASample): + print("encode_multi_mix_qa") yield self.encode_multi_mix_qa(sample) elif isinstance(sample, PackedCaptioningSample): # print(f"-------------PackedCaptioningSample---------------") diff --git a/aiak_training_llm/models/fastvit/fastvit_preprocessor.py b/aiak_training_llm/models/fastvit/fastvit_preprocessor.py index 1a061a73..5cf47d71 100644 --- a/aiak_training_llm/models/fastvit/fastvit_preprocessor.py +++ b/aiak_training_llm/models/fastvit/fastvit_preprocessor.py @@ -14,7 +14,7 @@ class FastViTImageProcessor: Uses CLIPImageProcessor with specific settings for FastViT. """ - def __init__(self, image_size=384): + def __init__(self, image_size=1024): """ Initialize FastViT image processor. diff --git a/aiak_training_llm/models/fastvit/fastvit_vision_model.py b/aiak_training_llm/models/fastvit/fastvit_vision_model.py index af68319c..ecd21048 100644 --- a/aiak_training_llm/models/fastvit/fastvit_vision_model.py +++ b/aiak_training_llm/models/fastvit/fastvit_vision_model.py @@ -37,7 +37,7 @@ def __init__(self): # Format: mobileclip_l_{resolution} # The first two parts must match the config file name (mobileclip_l.json) # The resolution is extracted from the last part - vision_tower_name = "mobileclip_l_384" # Default to 384 + vision_tower_name = "mobileclip_l_1024" # Override with config if provided if hasattr(config, 'vision_tower_name'): @@ -67,6 +67,7 @@ def forward(self, images, grid_thw=None): Vision features [batch, num_tokens, hidden_size] window_index: None (FastViT doesn't use windowing) """ + print(f"[FastViTModel] Input images shape: {images.shape}, grid_thw: {grid_thw}") # MobileCLIPVisionTower expects single images or list of images # For batch processing, we need to handle it appropriately if images.dim() == 4: @@ -75,6 +76,7 @@ def forward(self, images, grid_thw=None): else: raise ValueError(f"Unexpected image tensor shape: {images.shape}") + print(f"[FastViTModel] Output features shape: {image_features.shape}") # Return features and None for window_index (compatibility with Qwen2-VL API) return image_features, None diff --git a/aiak_training_llm/models/fastvit/mobileclip/__init__.py b/aiak_training_llm/models/fastvit/mobileclip/__init__.py index b4696ec3..01c48887 100644 --- a/aiak_training_llm/models/fastvit/mobileclip/__init__.py +++ b/aiak_training_llm/models/fastvit/mobileclip/__init__.py @@ -44,7 +44,7 @@ def __init__(self, model_name: str, *args, **kwargs) -> None: # Create model self.model = create_model(model_name, projection_dim=self.projection_dim) - + print(f"Loaded MobileCLIP model: {model_name}") # Build out projection head. if self.projection_dim is not None: if hasattr(self.model, "head"): diff --git a/aiak_training_llm/models/fastvit/mobileclip/mci.py b/aiak_training_llm/models/fastvit/mobileclip/mci.py index 690cdcb1..a97ebc4f 100644 --- a/aiak_training_llm/models/fastvit/mobileclip/mci.py +++ b/aiak_training_llm/models/fastvit/mobileclip/mci.py @@ -1425,12 +1425,19 @@ def cls_init_weights(self, m: nn.Module) -> None: nn.init.constant_(m.bias, 0) def forward_embeddings(self, x: torch.Tensor) -> torch.Tensor: + print(f"[FastViT] Input shape: {x.shape}") x = self.patch_embed(x) + print(f"[FastViT] After convolutional stem: {x.shape}") return x def forward_tokens(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: + stage_idx = 0 for idx, block in enumerate(self.network): x = block(x) + # Print after each stage (skip intermediate pos_emb/downsample layers) + if hasattr(block, '__class__') and 'FastViT' in str(block.__class__.__name__): + stage_idx += 1 + print(f"[FastViT] After Stage {stage_idx}: {x.shape}") return x def forward(self, x: torch.Tensor, *args, **kwargs) -> Union[Tensor, Dict[str, Tensor]]: @@ -1440,6 +1447,7 @@ def forward(self, x: torch.Tensor, *args, **kwargs) -> Union[Tensor, Dict[str, T x = self.forward_tokens(x) # for image classification/embedding x = self.conv_exp(x) + print(f"[FastViT] After expansion layer: {x.shape}") cls_out = self.head(x) out_dict = dict() diff --git a/aiak_training_llm/models/fastvit/mobileclip_encoder.py b/aiak_training_llm/models/fastvit/mobileclip_encoder.py index 4db99252..0a8b38cb 100644 --- a/aiak_training_llm/models/fastvit/mobileclip_encoder.py +++ b/aiak_training_llm/models/fastvit/mobileclip_encoder.py @@ -60,11 +60,13 @@ def load_model(self, device_map=None): def feature_select(self, image_forward_outs): # Features from penultimate layer image_features = image_forward_outs["image_embeddings"] + print(f"[MobileCLIP] Vision tower output (4D): {image_features.shape}") # Reshape 4D tensor to 3D B, C, H, W = image_features.shape image_features = image_features.reshape(B, C, H*W) image_features = image_features.transpose(1, 2) + print(f"[MobileCLIP] Reshaped to 3D (batch, tokens, dim): {image_features.shape}") return image_features def forward(self, images): diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py index 5afef385..4d349260 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py @@ -191,7 +191,6 @@ def get_vision_config(model_family, model_name): """ get vision config """ config = VisionConfig( num_layers=24, - # hidden_size=1024, hidden_size=3072, ffn_hidden_size=4096, num_attention_heads=16, diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index c00c6239..f5ba61b4 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -167,6 +167,7 @@ def __init__( vision_config, vision_layer_spec, ) + print(f'self,vision_model: {self.vision_model}') # Original Rice/SigLIP encoder (commented out): # if vision_config.normalization == "RMSNorm": # self.vision_model = RiceViTModel( diff --git a/aiak_training_llm/train/pretrain/pretrain_qwen2_vl.py b/aiak_training_llm/train/pretrain/pretrain_qwen2_vl.py index ed3c407b..42a420ca 100644 --- a/aiak_training_llm/train/pretrain/pretrain_qwen2_vl.py +++ b/aiak_training_llm/train/pretrain/pretrain_qwen2_vl.py @@ -331,7 +331,7 @@ def forward_step(data_iterator, model): def train_valid_test_dataset_provider(train_val_test_num_samples): """ Provides the datasets used by the trainer """ - + print("pretraining using Qwen2VLTaskEncoder data provider...") task_encoder = Qwen2VLTaskEncoder() train_dataset = get_train_dataset(task_encoder) collator = build_sft_data_collator(DataCollatorForSeq2Seq) diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index e5a8e74f..7bfd2ae4 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -10,7 +10,7 @@ PP="${2:-1}" # pipeline parallel SEQ_LEN="${3:-512}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size -GBS="${5:-8}" +GBS="${5:-1}" # NSTEP="${6:-2500}" # number of training iterations NSTEP="${6:-20}" # number of training iterations # DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} @@ -90,7 +90,7 @@ TENSORBOARD_PATH="${SAVE_CKPT_PATH}/tensorboard" mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=${GPUS_PER_NODE:-2} +GPUS_PER_NODE=${GPUS_PER_NODE:-1} # Change for multinode config MASTER_ADDR=${MASTER_ADDR:-"${list_ip[0]}"} @@ -123,9 +123,9 @@ DATA_ARGS=( --num-workers 16 --chat-template qwen2-vl # will change this chat template - # FastViT configuration (following FastVLM repo) + # FastViT configuration --use-fastvit # Enable FastViT vision encoder - --fastvit-image-size 384 # FastViT input resolution (384, 448, or 512) + --fastvit-image-size 1024 # FastViT input resolution --vision-tower-name mobileclip_l_384 # FastViT model variant (mobileclip_l_{resolution}) --image-aspect-ratio pad # Image preprocessing: pad (square with padding), anyres (multi-res), square (direct resize) # --image-grid-pinpoints "[(384, 384), (768, 384), (384, 768), (768, 768)]" # Uncomment for anyres mode From 46cb9ad0b51a813eae47f287daaa45cb7ee37acd Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 12 Jan 2026 16:26:14 +0400 Subject: [PATCH 12/39] Update configuration and metadata files - Update megatron_core SOURCES.txt - Update stage 1 alignment script configuration --- aiak_megatron/megatron_core.egg-info/SOURCES.txt | 4 ++++ .../quick_start/stage_1_alignment_llava_ov_4b.sh | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/aiak_megatron/megatron_core.egg-info/SOURCES.txt b/aiak_megatron/megatron_core.egg-info/SOURCES.txt index 7be6c918..71f2d906 100644 --- a/aiak_megatron/megatron_core.egg-info/SOURCES.txt +++ b/aiak_megatron/megatron_core.egg-info/SOURCES.txt @@ -115,6 +115,8 @@ megatron/core/export/trtllm/trtllm_weights_converter/distributed_trtllm_model_we megatron/core/export/trtllm/trtllm_weights_converter/single_device_trtllm_model_weights_converter.py megatron/core/extensions/__init__.py megatron/core/extensions/transformer_engine.py +megatron/core/extensions/transformer_engine2.py +megatron/core/extensions/transformer_engine_org.py megatron/core/fusions/__init__.py megatron/core/fusions/fused_bias_dropout.py megatron/core/fusions/fused_bias_geglu.py @@ -123,6 +125,7 @@ megatron/core/fusions/fused_bias_swiglu.py megatron/core/fusions/fused_cross_entropy.py megatron/core/fusions/fused_indices_converter.py megatron/core/fusions/fused_layer_norm.py +megatron/core/fusions/fused_layer_norm_org.py megatron/core/fusions/fused_pad_routing_map.py megatron/core/fusions/fused_softmax.py megatron/core/inference/__init__.py @@ -248,6 +251,7 @@ megatron/core/transformer/__init__.py megatron/core/transformer/attention.py megatron/core/transformer/cuda_graphs.py megatron/core/transformer/dot_product_attention.py +megatron/core/transformer/dot_product_attention_no_te.py megatron/core/transformer/enums.py megatron/core/transformer/identity_op.py megatron/core/transformer/mlp.py diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 7bfd2ae4..a8b89a28 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -7,12 +7,12 @@ TP="${1:-1}" PP="${2:-1}" # pipeline parallel # Defaults: TP=1, PP=1. # SEQ_LEN="${3:-32768}" -SEQ_LEN="${3:-512}" +SEQ_LEN="${3:-1024}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size -GBS="${5:-1}" +GBS="${5:-8}" # NSTEP="${6:-2500}" # number of training iterations -NSTEP="${6:-20}" # number of training iterations +NSTEP="${6:-50}" # number of training iterations # DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} # TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} # CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} From 2da41ab771259df06590ed02af574111eac7490d Mon Sep 17 00:00:00 2001 From: RanaZay Date: Tue, 13 Jan 2026 19:49:54 +0400 Subject: [PATCH 13/39] Fix ROCm compatibility: add PYTHONPATH and local_files_only for tokenizer/processor loading --- Stage1/alignment_rocm.sh | 75 ++++++++++--------- .../data/multimodal/qwen2vl_task_encoder.py | 2 +- .../tokenizer/tokenization_hf.py | 1 + .../train/sft/sft_llavaov_1_5_vl.py | 2 +- aiak_training_llm/train/sft/sft_qwen2_vl.py | 2 +- 5 files changed, 43 insertions(+), 39 deletions(-) diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh index 07922211..91fa23e7 100755 --- a/Stage1/alignment_rocm.sh +++ b/Stage1/alignment_rocm.sh @@ -1,38 +1,29 @@ #!/bin/bash -# AMD/ROCm alignment launcher (separate from alignment.sh) -# Adjust SBATCH partition/queue to your cluster settings. -#SBATCH --job-name=llava_stage1_4b_rocm -#SBATCH --time=72:00:00 +#SBATCH --job-name=llava_stage1_4b_amd #SBATCH --nodes=1 -#SBATCH -p amd-gpu # TODO: set your AMD GPU partition/queue -#SBATCH --gres=gpu:2 # match quick_start GPUS_PER_NODE=2 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=128 +#SBATCH --gres=gpu:2 +#SBATCH --time=72:00:00 #SBATCH --mem=230G -#SBATCH --ntasks-per-node=2 # one task per GPU -#SBATCH --cpus-per-task=16 -#SBATCH --output=/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out +#SBATCH --qos=skqos +#SBATCH --partition=faculty +#SBATCH --output=/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out -#!/bin/bash - -# ---- ENV SETUP (ROCm) ---- +# ---- ENV SETUP (AMD) ---- source ~/.bashrc -# Optional: override with `CONDA_ENV=myenv` before running -CONDA_ENV=${CONDA_ENV:-llava-ov-4b-clean} -if command -v conda >/dev/null 2>&1; then - if conda env list | awk '{print $1}' | grep -qx "$CONDA_ENV"; then - conda activate "$CONDA_ENV" - else - echo "[Warn] Conda env '$CONDA_ENV' not found; continuing without activation" - fi -fi +conda activate mobile_vlm + +export MIOPEN_DISABLE_CACHE=1 +export PYTORCH_TUNABLEOP_ENABLED=0 export ROCM_HOME=${ROCM_HOME:-/opt/rocm} export PATH="${ROCM_HOME}/bin:${PATH}" export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH}" + export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1} -# CUDA-only Apex extensions are disabled on ROCm -export APEX_CUDA_EXT=0 # RCCL/NCCL runtime hints (tune as needed) export NCCL_DEBUG=${NCCL_DEBUG:-WARN} @@ -42,27 +33,39 @@ export NCCL_P2P_ENABLE=${NCCL_P2P_ENABLE:-1} # Resolve repo root relative to this script (Stage1/..) SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd) +REPO_ROOT=/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5 +cd "$REPO_ROOT" || exit 1 + # Go to repo root cd "$REPO_ROOT" || { echo "[Error] Repo root not found: $REPO_ROOT"; exit 1; } +echo "=== ENV CHECK ===" +which conda +which python +python -V +echo "CONDA_DEFAULT_ENV=$CONDA_DEFAULT_ENV" +echo "CONDA_PREFIX=$CONDA_PREFIX" +python -c "import sys; print('sys.executable=', sys.executable)" +python -c "import torch; print('torch=', torch.__version__)" || echo "TORCH NOT FOUND" +pip -V +pip list | grep -E "torch|pytorch" || true +echo "=== END ENV CHECK ===" + # Required environment variables -AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH:-$REPO_ROOT} \ -DATA_PATH=${DATA_PATH:-$REPO_ROOT/data/LLaVA-558K-Webdataset} \ -TOKENIZER_PATH=${TOKENIZER_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0} \ -CHECKPOINT_PATH=${CHECKPOINT_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1} \ +export AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" +export AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" +export DATA_PATH="${DATA_PATH:-$REPO_ROOT/data/LLaVA-558K-Webdataset}" +export TOKENIZER_PATH="${TOKENIZER_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0}" +export CHECKPOINT_PATH="${CHECKPOINT_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1}" +# Add megatron to PYTHONPATH so imports work +export PYTHONPATH="${AIAK_MAGATRON_PATH}:${AIAK_TRAINING_PATH}:${PYTHONPATH}" echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" +echo "AIAK_MAGATRON_PATH=${AIAK_MAGATRON_PATH}" echo "DATA_PATH=${DATA_PATH}" echo "TOKENIZER_PATH=${TOKENIZER_PATH}" echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" echo "SLURM_NODELIST=${SLURM_NODELIST}" - -# Launch quick-start script (uses torchrun and nccl backend which maps to RCCL on ROCm) -QS="$REPO_ROOT/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh" -if [[ ! -f "$QS" ]]; then - echo "[Error] Quick-start script not found: $QS" - exit 1 -fi -bash "$QS" +echo "PYTHONPATH=${PYTHONPATH}" +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh \ No newline at end of file diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index db8bbc0e..2b0326c3 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -129,7 +129,7 @@ def __init__(self, args): #template for formatting conversations (user/assistant turns) # Load HuggingFace processor (tokenizer + image/ video processor from Qwen2-VL) - self.processor = AutoProcessor.from_pretrained(self.args.hf_tokenizer_path, trust_remote_code=True) + self.processor = AutoProcessor.from_pretrained(self.args.hf_tokenizer_path, trust_remote_code=True, local_files_only=True) print(f"Loaded processor from {self.args.hf_tokenizer_path}") print(f"Processor config: {self.processor}") diff --git a/aiak_training_llm/tokenizer/tokenization_hf.py b/aiak_training_llm/tokenizer/tokenization_hf.py index 921b9dfd..2c56cff7 100644 --- a/aiak_training_llm/tokenizer/tokenization_hf.py +++ b/aiak_training_llm/tokenizer/tokenization_hf.py @@ -29,6 +29,7 @@ def __init__(self, split_special_tokens=split_special_tokens, model_max_length=model_max_length, trust_remote_code=True, + local_files_only=True, **kwargs, ) diff --git a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py index 0105eb5f..3a5c1b4c 100644 --- a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py +++ b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py @@ -188,7 +188,7 @@ def train_valid_test_datasets_provider(train_val_test_num_samples): tokenizer = get_tokenizer() - processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True) + processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True, local_files_only=True) if args.image_resolution: setattr(processor, "image_resolution", args.image_resolution) diff --git a/aiak_training_llm/train/sft/sft_qwen2_vl.py b/aiak_training_llm/train/sft/sft_qwen2_vl.py index 8d9aefbe..fdfdb663 100644 --- a/aiak_training_llm/train/sft/sft_qwen2_vl.py +++ b/aiak_training_llm/train/sft/sft_qwen2_vl.py @@ -188,7 +188,7 @@ def train_valid_test_datasets_provider(train_val_test_num_samples): tokenizer = get_tokenizer() - processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True) + processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True, local_files_only=True) if args.image_resolution: setattr(processor, "image_resolution", args.image_resolution) From 6b8eb12766f8c35f6eabf0af7a02ad6c0c4f224a Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 15 Jan 2026 21:44:33 +0400 Subject: [PATCH 14/39] Mobile LLM integration: Add MobileLLM model support and memory optimizations - Add MobileLLM 140M model architecture and configuration - Integrate FastViT vision encoder with configurable image sizes - Optimize training for low-memory GPUs (1 GPU, reduced batch size, increased recomputation) - Add inference script for FastVLM testing - Configure training for 5 sample test runs to validate setup - Update .gitignore to exclude large checkpoint files --- .gitignore | 7 + Stage1/alignment.sh | 7 +- Stage1/inference_fastvlm.py | 114 +++++++ Stage1/test_image.jpg | Bin 0 -> 20456 bytes .../models/llavaov_1_5/llavaov_1_5_model.py | 26 +- .../models/mobilellm/__init__.py | 13 + .../models/mobilellm/mobilellm_config.py | 130 ++++++++ .../models/mobilellm/mobilellm_layer_spec.py | 61 ++++ .../models/mobilellm/mobilellm_model.py | 295 ++++++++++++++++++ .../models/mobilellm/mobilellm_provider.py | 91 ++++++ .../tokenizer/tokenization_hf.py | 6 + aiak_training_llm/utils/constants.py | 1 + .../stage_1_alignment_llava_ov_4b.sh | 11 +- .../config/mobilellm-140m.json | 87 ++++++ 14 files changed, 837 insertions(+), 12 deletions(-) create mode 100644 Stage1/inference_fastvlm.py create mode 100644 Stage1/test_image.jpg create mode 100644 aiak_training_llm/models/mobilellm/__init__.py create mode 100644 aiak_training_llm/models/mobilellm/mobilellm_config.py create mode 100644 aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py create mode 100644 aiak_training_llm/models/mobilellm/mobilellm_model.py create mode 100644 aiak_training_llm/models/mobilellm/mobilellm_provider.py create mode 100644 tools/convert_checkpoint/config/mobilellm-140m.json diff --git a/.gitignore b/.gitignore index 11a185b6..b0c60a4c 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,13 @@ coverage.xml # Translations *.mo +# Model checkpoints and weights +*.safetensors +*.pt +*.bin +**/hf_checkpoint/ +**/megatron_checkpoint/ + # Sphinx documentation /docs/_build/ diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index 723e0e74..1b126ff4 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -56,7 +56,10 @@ export WANDB_API_KEY="wandb_v1_5y5JqALBMdHhru8CR1gOLflJlRj_O8BG2XRb0S2x0TJVqW1xA export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="fastvit_integration" -export CUDA_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=4 export GPUS_PER_NODE=1 +export MASTER_PORT=26000 -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +# bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh + +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh \ No newline at end of file diff --git a/Stage1/inference_fastvlm.py b/Stage1/inference_fastvlm.py new file mode 100644 index 00000000..f70d7d94 --- /dev/null +++ b/Stage1/inference_fastvlm.py @@ -0,0 +1,114 @@ +""" +FastVLM Inference Script +Example run: +python inference_fastvlm.py --checkpoint_path ./stage_1_alignment_llava_ov_4b/iter_0000020 \ + --image_path ./test_image.jpg \ + --prompt "What is in this image?" +""" + +import os +import sys +import torch +from PIL import Image +from argparse import ArgumentParser + +# Add repo root to path +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) +sys.path.insert(0, os.path.join(REPO_ROOT, 'aiak_megatron')) + +from transformers import AutoProcessor +from aiak_training_llm.models.fastvit.fastvit_preprocessor import FastViTImageProcessor +from aiak_training_llm.models.fastvit.mm_utils import expand2square + +# Argument parser +parser = ArgumentParser(description="FastVLM Inference") +parser.add_argument('--checkpoint_path', type=str, + default='/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/stage_1_alignment_llava_ov_4b/iter_0000020', + help='Path to trained checkpoint directory') +parser.add_argument('--tokenizer_path', type=str, + default='/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0', + help='Path to tokenizer') +parser.add_argument('--image_path', type=str, default='test_image.jpg', + help='Path to input image') +parser.add_argument('--prompt', type=str, default='What is in this image?', + help='Text prompt for the model') +parser.add_argument('--image_size', type=int, default=1024, + help='FastViT image size (384 or 1024)') +parser.add_argument('--use_gpu', action='store_true', default=True, + help='Use GPU for inference') +args = parser.parse_args() + +print("=" * 80) +print("FastVLM Inference") +print("=" * 80) +print(f"Checkpoint: {args.checkpoint_path}") +print(f"Tokenizer: {args.tokenizer_path}") +print(f"Image: {args.image_path}") +print(f"Prompt: {args.prompt}") +print(f"Image Size: {args.image_size}") +print("=" * 80) + +# Device setup +device = torch.device("cuda:0" if torch.cuda.is_available() and args.use_gpu else "cpu") +print(f"Using device: {device}") + +# Load tokenizer/processor +print("\nLoading tokenizer and processor...") +processor = AutoProcessor.from_pretrained(args.tokenizer_path, trust_remote_code=True) +tokenizer = processor.tokenizer + +# Initialize FastViT image processor +fastvit_processor = FastViTImageProcessor(image_size=args.image_size) +print(f"FastViT processor initialized with image_size={args.image_size}") + +# Load and preprocess image +print(f"\nLoading image from: {args.image_path}") +if not os.path.exists(args.image_path): + print(f"ERROR: Image file not found: {args.image_path}") + print("Please provide a valid image path using --image_path") + sys.exit(1) + +image = Image.open(args.image_path).convert('RGB') +print(f"Original image size: {image.size}") + +# Preprocess with FastViT (pad to square) +mean_color = tuple(int(x * 255) for x in fastvit_processor.image_mean) +image_padded = expand2square(image, mean_color) +print(f"Padded to square: {image_padded.size}") + +pixel_values = fastvit_processor(image_padded) +print(f"Preprocessed image shape: {pixel_values.shape}") + +# Create prompt with vision tokens +IMAGE_TOKEN = "<|image_pad|>" +VISION_START = "<|vision_start|>" +VISION_END = "<|vision_end|>" + +# Format: <|im_start|>system\n...<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|>...<|vision_end|>\nPROMPT<|im_end|>\n<|im_start|>assistant\n +conversation = f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{VISION_START}{IMAGE_TOKEN}{VISION_END}\n{args.prompt}<|im_end|>\n<|im_start|>assistant\n" + +print("\nTokenizing prompt...") +input_ids = tokenizer(conversation, return_tensors="pt")["input_ids"] +print(f"Input IDs shape: {input_ids.shape}") +print(f"Prompt tokens: {input_ids.shape[1]}") + +# TODO: Load your trained model checkpoint here +# This requires implementing model loading from Megatron checkpoint +print("\n" + "=" * 80) +print("NOTE: Model loading from Megatron checkpoint not yet implemented.") +print("This script currently only demonstrates preprocessing.") +print("\nTo complete inference, you need to:") +print("1. Load the model from checkpoint using Megatron utilities") +print("2. Convert distributed checkpoint to single GPU format") +print("3. Call model.forward() with preprocessed inputs") +print("=" * 80) + +# Placeholder for model inference +print("\nPreprocessed inputs ready:") +print(f" - pixel_values: {pixel_values.shape} ({pixel_values.dtype})") +print(f" - input_ids: {input_ids.shape}") +print(f" - Device: {device}") + +print("\nInference complete (preprocessing only).") + \ No newline at end of file diff --git a/Stage1/test_image.jpg b/Stage1/test_image.jpg new file mode 100644 index 0000000000000000000000000000000000000000..02a0f92f52af7ddb1440824679b45bc2206196b1 GIT binary patch literal 20456 zcmeHv2Ut`~mj6XS5Rr_MQF2g}q~rz>5D<_oSwtjBXmXPpP@?20ISYt@N`@w9P_pEl zGfmFj|Le@o?D*d7j5BX`XZQcThU)(4TXk=pQz!jSodP|Ko(72Iq-CW63=9Ck0Dl1V z7$6DYUc88N5gQi=2j|iyTs(ZTEBKc$<5QB55|YtU(bLgV(a>CH;$pqdaFdaShV35v z%{$zDe0=n*f}#REB3!(DJl}7Eap}?}{LA`q6mz0*3S5!7MHZ`}jwzYTk^$&a-92y=Ootd4RUszmPURmAV+1=Ye zI6OK&`A!!GfcZzV!1q5A_6xd*LAowrVPRt7e5VWJf)m&mP{uKYk}~jk z;ga1CPy1AViIG52!a9$9Kiw4@2zQNZz(Lk_f)=;+*v?Nyp`ljdPL)opyp~l>==CJHD1~EZ2klK=P z3_qxZ%n)yPpI_vg0!Ne#W%cUTjy?%lf*;%^JyzmF1FvOTA$t`4@Y|>%nIFdORkES_ znw$vNU$X8boBcS6qX%wwXh0(P-VPera9FWdKF{*sq#qtuS`g_+5!2J4fgU4;v)WTV zdZho+66zI|^#U3gUF%!UIF%b&;AYQ+{eEZUpM>jQ&qytoL^>AaL2j00uKC@58Q9%%@l+(G zMAu`ZEyqHGLXo3LG;pKvG8!0aL<7spn$p*3wX;kcl%#Eqb`bnDBJU;4YrlMwBGc?m zVIWhM0{$J%`Gp9CqSjPaDOSSkQpIzp@m0 zolBY&*<*~~()3~5tSCwSYiOX*4^+hTLdjnFX%|#EE{Te=>y;{2u+KiUGzO@D3+lRjHT?(ibBG46k?%xk& zOTF-08e(@=>C6{#8x0r`qJcCieZOC_tN=lH+5|7~ONmDV72ONR>CI%y2KsKe%2H&& zzoB8I%+MdRpA??M;M?!_DtDEB;RHHf7;NF6CMKNs-2i1BW7UUDSseSfG}uy8%JCIv z15^z?8n_==P4G*W={eIM`lA6oLI^(^uuJLOl?r4_uR$DvEA(#(5ilV?YSm&gF&c~U}e z@20Kw;X_b(baCGQR!d2~ltKcZ=bwR}9Kp}=IQE2oWS7Gs8R| zoa|2jX3hR5tpYw60sI(N#XJ9Cv};s((15ea`25snI4CsbglfOlOmcRMF`#(L++6*^#87wI=b88de0bG`?o@*w$Bo%! z5r&+YvT70QwUPc}k{w%C;JTD$tHk}Tbihv~Vht|oU%pKcZdvxF_Vs3$NYSE)t6iWU zilmwp1&8mDq5&V562Hc&OJ+4D;i;pLxS;d2M@KC#FrQCDrMyS#-C!sP8l3wFpx^gY zMg!aYZ@%pJ5^tMA zulX+xY87+ZKdllkyM;H$bUtvMv|HC?C+&rMkE@GbT%Cj+cHgzKc)>85tHH1EPoOG8 zlI}>0hFH4~VQW0hd)Tx>LIhTx6rHheGsj=z>n48_7ckoHM9P7^k8`_A zd}FP<0QtpCbnlj0=0SAzL$(hA{Vh~vms!eYM)ZNKc-Lq2&vK6A4z6N`z(xfi-owt3 zj|Q|z1kWmTGHrvi3xMwXN~!!qcs(f%oePeY1B0L=RQYGL<^L;5Czn_GX33X>RiwRSS~E(|hlu0UG};el2>rURueffX;FDmwVkb1eCsFD|iz8KIX{GLKb1B zip=#u66ZG?F0Wl)_`N*wIs5vVUy)SgaPuOM5+J zWBQ$BmQhuWvoDr}4e{4}50h9Gd?%)aGZ)5R1qH9m8!gb7P%f65rkCqjP9zp6Dz1aJ z<^B(@X~_W%U^+WB73{21xwz*>6^;zoE5GD2RP;molO1Vk zY20;*dCelZVvfA#q^o^vYES-AQU3o%gpJLGTEp)Rzq}rCs0KZ)dZL_ENg9EZO4(}A z82=(Wcf4d?TP*Za<+lRGf#8d7h~3DsY!M_5Ye-GPG#qoxS?=q*xaPq^TV6YR*jiU} zx3Jw*9-Y!tflp6Lq_nI^BHk6=E^kAcG(rtUODksC^>?*&E1SMhaoQ?M<2dTW=-F7k z>dg1#*a|N?FyDjG@aD2twSki7ec^98QAA$z6ah;={^-(ck*2qh1hIHTE5lsMEtzaq z>)OsBz+EG-FG))^mQR6wAo`xYvsF@FeGYGf`ZU6i!PQJHucdDKRf^;}Z%A~S-tAB~ zEQbdEe6lz0E*N1FcwFN24pmmIKr-(1wl{(+^uv`0ver<_vVrT&CS^8ixj0zUyF=bW zuj&sEk>xpq`X(>66JhD*y;9_ZKZ}?CkB31;CBX{CYtDnX3e5|R{`oV?5$i5Cf})j9 z*PZg5oRqb+vx@9T68wlWVp&)oRrw+n-ZU0cYy{aJ~hD%OC2(iBHt}a zNUVsf7p&-v1A~yq;y(j~G?UJxkii=RpN=9Mk=X}HmEm@Hxf4$G6mt;n$3hN_X&pI* z57LXXRn+FQNLIc!=(cTN;Z%8}-al9AZ#mJ(BEi;}16fKVcg%BMk}$xuXwP3Mt+&^i zWk|+ygW2wQsLx>n;RA$k-BJUSXrD{jKW%H|=&;4zC*N~+>G&WRK9!-{I}!q#80gR7 zAxhLQQ3Kx($kgqfF$&pT(Ox_8J)2-dm-90$_*;E6lig&|K-iEE8d#e_1AA-PG=L;M-cyccRouhfs@?`=VHQ>Dxr^*B)Oi`Dg6 zb| z>W}XLn5h@ayM0hJp;E|V&CM41b9Iu%+4~D#ILtT#%hasn$q$|4;e4$d!X>90@kuVXIu)lt3DlFSLSvNx5Ji@ek}mnfGTuLr&ICJ?wzh5X9R+{0hB zg*RDhTpS)mLYw-~4_^RlI?Tz4X-A@kb879Y*sW!Az9SJ$^Re5-gAZ(VeIiJPFMCbd zorF*pC3BWx8;lH;Njn&37c!2ZUPzH&$r{is4cBe3qY0$2Z#xj!6jJ-P$1?L1(CYuN z{rEIkFQPv8^|rVnqxStrhY6D8qZiljat8%*t#2>7ARUqh`Ai(~0(ri)0)(pCwK&BG z$x|BJ%uUZ_-67G0Z{s+rb{s2KGs6(jx98<^E#D68lgf;2=ih{{>?upRtt|dDF8Nbl zLK0?%eI9k-#9lKwdpA(6ojTeVw{70WRc%0(au?-C7JoPIy|e91rrFDE_^nKps=2Ov z*rFISej;0E7A{#wDaPlKbF}mZ!H)Hg?*`}JuRm_g5Z+_*ou=z`g_66okQ5eY#w}Oa zF^ho4Tx@{fvplW zFl*7$wD37!x$8je6?S6l6)Dw)+HMraI-h1Jyn*l?yxX<*bOU}A51GN~hb&c3ogdVm zDHxsyLl(~<(@F3(GDze$h-F-_kk3saU#+C(k3lee7S5E@*iejUGzP^6XQnY+G}5>t{E-4QJ% zW-+|jyaa7hj^>g#05!D-j2|~rgg}4(tK9)3=l5Id$@Jxo8!#xzcH&YF8mMv7EIbmH zw#00pdXaG5A%;y-uAn}!t*Iy|OoU}^~MfUP@({Kv@>f+ zh%4Qdl)-V2jg(tW=m9D}JLBaXKMdaliqz~#eGa{V-9HI z?koMv)9s^{I7R>#&u1>-5bBiXV3*_PhXE53lL>k$i}rlVQZ5VsuyOyBk^dKdk`>=R zz8Qb*+_fDIq%Ra&7t$Po){)sn6AgTH+V`yvsd1&-(n>ta-eR5y83?7R{|SfwE8w7y zV~7SAv){ni8Cg+55!E{gNj)%=++F}?k`eF|E{?6vr1!(C4d+?&dSIM0b?+aEbN;*f zlFm$>MJE}c>Zb;&4urM~0ipVBYh}_qLEYKmxOSm;78!lI^~o#Y_9vR3hq{d5L0xW| zE!9L|Lb1*6XX4^t^>OyC0$Cpka;DH33|n>8H}+>ntMXf$H#|(A$dWXL7o~OhUZUW5 zxQX|%_0wL#dhJoG#x$J;GICL9i*e=MP5e8G( zdu$t5mlZ#>P)mYSDC+8Mfi^phYWfvx@wu|P(<_-+ICt{u++f*}v(2lI2!$>$A*=v~-#o{iJ#^5mx}ReG76XWv734;EdfhLz2#iHhtoiAI!!KXFy+%4RR3o`k- zWVbUItQ@~xLtxM3h)BNg81gL1dhctGLp{7luSmF{ADZVJ*@z-ET_2r#?sE^<(op}} zJ0*$XAyQFq@uJ|}SYwRNZJBv5HTnLECoV%QYlx4!Bi94_P z#LV5%KrDbcmX`0XG$Zvzxj5ePL{Wx;J=PE+O#|oHT}y!Fb3HFB56+0c%dy;uMO7|+ zMNH$t0^bmisPB;a5gg`pCBM2fare;-F}%pXlhl((mAuF`)7e7tbIj(JY=4ILjf$yt za~m-)JAKZX&||9Nl#|mpU>ccxeUcvygx^y`0|f>H8g`*KBOdjUEWWX%NEw?Vs9g$4 z0t2poG|;3O1`3n?S7!2FT(-pBgfPu~H=*9W&aKPGLoN4C@XD(^wu)Bp_;84_&eC|Vzvd#{Q`8P*!K?T4uJ zf<7aj7qwdjech$Y4BXxFpV|@DhgZ>~JUItMz)ebq7hue(+l_i{&T((O*S_x{Xu*TY z`B~o;4qAD|BHwZGF_tGrxOipv#rH%!`0GrCRM)g83XNn0> za;o=Rg)I8Q@{MT^HieZIPV%3*2_u^-6W2q%p^8j5y%05W>=bcG)EMqACg`-RZpd?l|4Zx*dZlZNAi7x&!z;N^nW zJx1GwP_S;p?mk$)!Ub~QGZ#hPc*fRe6B}Rrrqg4VaiiV{yo8c2=D$sLo-k`1>=}8r zoqlA2a?b579FyLwQYRvS ze$qf$Kjxw$j&{q@NLF~dnLc|Up^*E96JcL*XbnSWxd{&a9Yq@(UOkpribxeNC54eQ zUw9Twpu9pYG}(`ZCuvoRs}gPiD>TfIBDopMK_}VO-wSWajMey5YDDEdJbi3_hr?1u z!%s0TYQ`TEq8hl69{{&N6HXb=bx%XDZPi-{lPj}J4VE$$6qy0DM^=EmPimB6^j>C4`m5wy^6iu_* zsgw3$v#~{ihVPZKJ74CRGLl$hZG>7=h)v10=Q&lnmEOSr;O7A?PM-2nE5IvI)ZZx* z>F84Q*eL=%SrN}qxijRgTZ3?N=k(R9Bs!frf+1>xGSuVq#P!p!jsg#9_0hm7r&R5L zTj9O+m>s2s&9N%;m=;(IbP&N>Uy4%l>r)un%{1cBzIdKBc6hXY@crVD;pl z44CU-_B(e4KT`vbJF4>(wwHYK_o^Nm50tjkumy<(5a2#0MD`aX$Tg()SuLf=Z(}xJ z=FJ&EViG}mXTPIOPR(t)yn+w%xq7f!204!nP0ALJWjgO_zBM#J5Wv)&P2!3MUZkKH zD$oGMYBEt(PmuOaM+?|u?iY<0EMr};QMRCH6j7934WoZazqNI0vW1S8$wThNw?mHO zOITjrM%qH~Hx3&&u}TNBaS^!l!UL{`MS(p?nZOMMx9CV^S&bGdQ%}UWaf2b72Dl|?0EI8#uTE&G-C2FtRznRR5|uFX zhFZ8^B_4M+-o zK*3t8%h16+Usmo?E-pCqs!-5A#It)YaNAwOdDG#|%8X^NJd`oRh!Q&hCq7$kg=HW> zU0ad$qZ>PDll7NSwFYQFTyECm20NT)^1b&He^1Pcox!pi`lD>=o;}Kq66$W27nNog zjul5#-t$J|l=CM!o;_dq?R_W;a zT7pwYhJkxh6jLFaFzbQ%`l=)A%*j<=19{h+Y1I1t!3Bo;ETH_UHIxAmwBd%DsqEmHKMp|=NO z=<;;un=1Fab>)WamMPvc2ju2yfiFwlaD**3S679)+6x;(fQ$ zm-Dt_kD)yt_NUgwrc|N9C9a}$2ZG6_bBGvjEdK_D^m|Mnf#7r+qr{jq0cg~?(bti< zSp$B4fz1+8t`kfLVtmy9ARD6-JF z!!oSwzfSHL*icn|s^0&#%xZI_OQ8AYgW2f}*UwOOhLs#Uy(G)BGnZ!6G)nzzX`vg1 z7X-d}`$(fns3cH=s`%)JIE2x~odZ&`JT>8mI!i(3oAWbOm{`Y^XuONP*?0Wb=~~$} z!v{OoWBs0xXbByvs1y5`1j`4{6dE^Mi|5t?TMMWH`zDu?w6wK5P=-=u8ZRV%0#M;x z+ICWpeT1axZAS2~x0~zH=#3+>PMXnr@QL*$hatX^t=D@1A5*1Wq( zr~bp)gMPl8Hq+vqpj~n-p0?qaB+iT%xKbuXPRO^62ae5loka00V&e7oL0eso@xK9& zVjJpD65=0~J>2GKtY!>rwE=KA)NF-dadH)1FMhAC1Wiv{L^Fe*j5Pj-%;!WhkEO&Ar!$&?1lbvPUpe13*Ov45 zzgde2B6%-PKT)STbfhS2zDIQ&P~V_Mt*~cq#k!I8S_Bk@{ogtTtY5C znT1@6NS`466`Yy~umr3k;Of%ja+sLJj^~revRq842Vz{OL0__uaN{s*sN1i)a>83@tnQF;R}3t_a=CCAGg#39iBY!6H< z0shT+k2JG9Gvw5;V&rI7RYtfF@j0#t5e$pSqHQsN)%&5bUT<606BA0Tr~L@b*=xvT z5EtXNWH2|yTdoK4tvq=yy$;hQuxavICKSL3pwlq)tvm>4`5>BURb~*rZm%rm-1?6c z6kZP$VtI7UOk>71u<{ zMDc76phXklLg|M!{8EDcoqy-7j_L|kztWLwhK+|ne5z=iR-N+(RLobun2k(OgAtIM zA^%d`Ots_s-4b&sX_=0HsW-v;S=U9`<%jk2zWmD$GgK`h{KT})2;Zs4p1$5QmsB!o zyu!QnB~}R;rz(-xLKVvu-{fOh7mDAJz>sRg+`4T!JnvOc6T)+?p~aB)UN>zJr*dsA zecU9NH(W~e`U#VWp|8G8+DvMGhz5`WrkZKa)*FIh?%$?ihDF_~<%EVQ2o5-%^Uh4c zR3jZlq+{xcPF0hL4#8@6kxR7KJzbT#hY(D$c}=YvXsoyjw&2CJ0#yz@)TfT zxOK12FGY_FmAGW`NU}cs_@RNzh7=&uMDJ9Dby*GL$tEeQQEv#MrYt zpy3#qdlXw&1aB%yQR`EhEQ*RSqTbyBoLO1JxZ9bX$>(Qn=*{C2ms?y~rCq<0e|5HQ zt)bsm9A39Nt13bRT(JOC^$Gq=xr&F1jnHhR_DYAVh(B>{k%BRXlAis2kTI)idd4M%QMT2Bty>>%B`@yGQoGu> zth}%ZtPr4N^=iL>fH$E821HDYR_N%1H|$^C2$?+(p|WokM-UN;I9xAq^pLL+OOg@erUf$wyKt}p9v7;InF!qV*D8`I+%m$V?#2TWSOpw?aY`KX_U+%>I{TXiBmd3T0zYAk zP+PByEA6(4OK>&Ut*$|WOE=25lI`l}^7TG#mj#R5E*=U8IGC&FmOoZsj<9C$S+RV-Xv z>L!{qyhNuN^C02euAGIt_0j&uREV=&?*yd?zhZoHhLp`+dI|(HY!SArH@pGXSz0Nc zEoH}6y%V0x*9Ge*^QH?=uE6I%Or1$RUs2PeB=C_s&>B(JwLZILfDP}l0Q0slKp@HC zbJ${lr>*#@!MaebV2&9<@z~b+xFZoov6PP&Ez|3rrhXpda)o@M=gwrvb zJADsPT35ym)mtAqZS@baz^Tq82Z-*6n^hX(I3-bCyftS1Mc1iYal+lvm!P;*>ccI? zO1e(IUY&ZT{)gdGRIyHuc<)o6M<|J7Lxq-_;RGwZ)H?Sf)vK=i$cyk;)p1LXRz2zf7FC*g#u26$AHjA7?AL=74BD@G~g+rA)u2%GcL4ZFggH{$p9Kg$7sY z5H2NiN7&<~cM{VFHxvo(xO22NP7)1#NJkX)6vLiI89#Z_BIbYY)U`Q+=xfwW2`2b9H5ZTQN< zun(oH;5r)M1@|jnOEyCGxKO*96v=oi!#e zv8G7$rU&wK?T9AIeb=~$(-`_@HZi2Dhu02-jPqaL6nSgpjX`ImA>c@XV`swj^rI z%EGF1T(oo9dRo5?n2lRk`fVCB+cmmnJ!(0VVC8u1PffY^RpLDS9lS@VA9Z&7NPU!D zi(ozLFTe4>m0EH?2HA>78u2MdIYpmCo9*-EkLiKz{=Y?W|8gvQ$k{TluhJ}K7CYwhnAu&<_sD^HV@t9e2P&fO6$B)vd|LiPLO4q zY}{Nk%P>*=DD!EHQrdJ#eY7kNMWObmn$kb0ev~{`21g^~us7j#Z`|uUxYu_LqM=t+ z_i1EA`A;i0=1&TDcJ{<#F#`?;)=|vox}U&Omae4wzZZf%K`{@u_+^SLW9oH0x|`D> zSDYnF{ZX0>n`yU)=J_I4H_6I`!e*cWtk%wA#r2>3pa!RW%sgnmedx~e^+V0>Cx;m zEfs_D^z9~Vp@{$ueQnSM-T3-<3M5KcVI3g!ll$1kgIiy4vk_B?)-M__$S?>m!rr^vK*uE>}t%51X3^OlYtUCK=lRDci4^_ikzvO#f9N%*|T4cSVdPQ5! zJWr0R^_yyDVs>u64!SK-{}=fLQ%g&4+ja9(%z`+mJ`6Zleg*O*S z>e)IzwjEr5)dnh?2>3e`u5tL9KO<|^B3p53U9$j7>*>`?A+s(Ac3$j|J8n#;QEQKtg#j_<#ni6x=nvAHvg&Uv*)^l!n<{DbVU zzm7zHUE1<*>(`zxI~_r(X>ZG}LD?cf!FR%j*@L)IC{}31Tr##fV0nk->05#zolXyd zc05~w%A#_|h!xzFc-oQqDo8W`$&`+{23Sqg z!R)OZZ5X~TffEJ>Qec_}R@hPghW#XjF**Kx{9wmj^BCqrFYncKYVHUpK^1&2rCm~- z{^b&WfRn5lN%qm%V-zf~P^%d;YpY}i4>RIjh=45<@YoQ1=$Ep_itJ{PA~V;U|8qzS zlP-KOTVTFpP#NVT;XBHk5_6|@seYap`LJ%9jgd2QV_s183pcy`bV%V@jR^}cNseme z#JY<5^(x+27;UE0OXH-sMcl145lR4obF)w&=YDy!6jQ0pm%C$C|J!jGXeR!6Cim|| ztxx+|eoUJUT{ZN|4r%Q0QLGZ{&OlXP$MjMM)H+MG>Pfn0+uOw_nrx!K2Ld#N@UmH)uZPJ5)odEwQ-k zpazBU^;yyo*vwn#&IZMeEdZG0E z^r!4^`#%+=q;9tqzeqU68)=im*wT7p81>3uQEGWb|Frk|&Ch$@%Y>B0&#M$9%fp|$9z^9%hOw+qG?b$G zDPnZv(ERRO(mRiEq;D;ZO?TXqcs! zdCM+^Gkk|A!r1DMd1P5rFCa2BzcCH*!){N=-A`?_r0njbFxs8op*&)GYC}dG-F~n` zlecf#QU<88PgotWbA;)|bs|FggiCq4zG%Ij3z22@#1VOxBt?EQ{wzI0 zhjEeG@_R1(e_smp&t~dx917Q&BpM8f%`He0PWhx?PHgP?ghe=DZKMkaXKN0KW%+3K z3xs0v=1yIItKq`^Y`qi{5X9^I4sgl^OYyl%k$3$a;Bxm&K0lc7A*|mXrDnEDhc?*&c|OKJ;}+zE5lpRSzBFWc1!3QzXgsGV+!}k(A&vk1gbsI| zQY{P`_$hBS!Xh3KyP0+4v0-V@(n1jTF%pJKlgXO|4+&QdgT8BoMEn) literal 0 HcmV?d00001 diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index f5ba61b4..ee202c23 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -20,6 +20,7 @@ RiceViTModel, VisionModel) from aiak_training_llm.models.fastvit import FastViTModel from aiak_training_llm.models.qwen import QwenModel +from aiak_training_llm.models.mobilellm import MobileLLMModel from aiak_training_llm.models.qwen_vl.adapter import Adapter from aiak_training_llm.models.qwen_vl.utils import get_inputs_on_this_cp_rank @@ -204,11 +205,10 @@ def __init__( # # This attribute is needed to check if an all-reduce is required # # on the word embeddings inside `finalize_model_grads._allreduce_word_embedding_grads`. if self.add_decoder: - # self.rotary_emb = Qwen2VLRotaryEmbedding( - # dim=language_config.hidden_size // language_config.num_attention_heads, - # theta=language_rotary_base - # ) - self.language_model = QwenModel( + + # Use MobileLLM for smaller, efficient model + print("[INFO] Using MobileLLM-140M as language backbone") + self.language_model = MobileLLMModel( config=language_config, transformer_layer_spec=language_layer_spec, vocab_size=language_vocab_size, @@ -221,6 +221,22 @@ def __init__( rotary_base=language_rotary_base, share_embeddings_and_output_weights=share_embeddings_and_output_weights, ) + + # # Use Qwen model (default for larger models) + # print("[INFO] Using Qwen model as language backbone") + # self.language_model = QwenModel( + # config=language_config, + # transformer_layer_spec=language_layer_spec, + # vocab_size=language_vocab_size, + # max_sequence_length=language_max_sequence_length, + # parallel_output=parallel_output, + # position_embedding_type=language_position_embedding_type, + # rotary_percent=language_rotary_percent, + # pre_process=self.pre_process, + # post_process=self.post_process, + # rotary_base=language_rotary_base, + # share_embeddings_and_output_weights=share_embeddings_and_output_weights, + # ) self.share_embeddings_and_output_weights = ( self.language_model.share_embeddings_and_output_weights ) diff --git a/aiak_training_llm/models/mobilellm/__init__.py b/aiak_training_llm/models/mobilellm/__init__.py new file mode 100644 index 00000000..8b78ca67 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/__init__.py @@ -0,0 +1,13 @@ +"""MobileLLM model family for efficient vision-language models.""" + +from .mobilellm_config import get_mobilellm_config +from .mobilellm_model import MobileLLMModel +from .mobilellm_layer_spec import get_mobilellm_layer_with_te_spec +from .mobilellm_provider import mobilellm_model_provider + +__all__ = [ + "get_mobilellm_config", + "MobileLLMModel", + "get_mobilellm_layer_with_te_spec", + "mobilellm_model_provider", +] diff --git a/aiak_training_llm/models/mobilellm/mobilellm_config.py b/aiak_training_llm/models/mobilellm/mobilellm_config.py new file mode 100644 index 00000000..52892b7d --- /dev/null +++ b/aiak_training_llm/models/mobilellm/mobilellm_config.py @@ -0,0 +1,130 @@ +""" +MobileLLM Configuration + +Loads configuration from facebook/MobileLLM-R1-140M HuggingFace checkpoint. +Reference: https://github.com/facebookresearch/MobileLLM-R1 +""" + +import json +import os +from megatron.core.transformer import TransformerConfig + + +def load_mobilellm_hf_config(checkpoint_dir): + """Load MobileLLM config from HuggingFace config.json""" + config_path = os.path.join(checkpoint_dir, "config.json") + with open(config_path, 'r') as f: + hf_config = json.load(f) + return hf_config + + +def get_mobilellm_config(args): + """ + Create TransformerConfig for MobileLLM-R1-140M from HuggingFace checkpoint. + + Reads the actual config.json from the downloaded checkpoint to ensure + we match the exact architecture of the pretrained model. + """ + # Load HuggingFace config + checkpoint_dir = getattr(args, 'mobilellm_checkpoint_dir', + 'aiak_training_llm/models/mobilellm/hf_checkpoint') + hf_config = load_mobilellm_hf_config(checkpoint_dir) + + # Extract architecture from HuggingFace config + num_layers = hf_config['num_hidden_layers'] # 15 + hidden_size = hf_config['hidden_size'] # 576 + num_attention_heads = hf_config['num_attention_heads'] # 9 + num_key_value_heads = hf_config['num_key_value_heads'] # 3 (GQA) + ffn_hidden_size = hf_config['intermediate_size_mlp'] # 2048 + vocab_size = hf_config['vocab_size'] # 128256 + max_position_embeddings = hf_config['max_position_embeddings'] # 32768 + rope_theta = hf_config['rope_theta'] # 8000000.0 + rms_norm_eps = hf_config['rms_norm_eps'] # 1e-05 + tie_word_embeddings = hf_config['tie_word_embeddings'] # True + + print(f"\n[MobileLLM Config] Loaded from: {checkpoint_dir}/config.json") + print(f" Layers: {num_layers}, Hidden: {hidden_size}, Heads: {num_attention_heads}, KV Heads: {num_key_value_heads}") + print(f" FFN: {ffn_hidden_size}, Vocab: {vocab_size}, Max Seq: {max_position_embeddings}") + print(f" RoPE Theta: {rope_theta}, RMS Epsilon: {rms_norm_eps}, Tied Embeddings: {tie_word_embeddings}\n") + + return TransformerConfig( + # Model architecture from HF config + num_layers=num_layers, + hidden_size=hidden_size, + num_attention_heads=num_attention_heads, + num_query_groups=num_key_value_heads, # GQA + ffn_hidden_size=ffn_hidden_size, + + # Vocabulary and sequence + vocab_size=vocab_size, + max_position_embeddings=max_position_embeddings, + + # Normalization from HF config + normalization="RMSNorm", + norm_epsilon=rms_norm_eps, + layernorm_zero_centered_gamma=False, + layernorm_epsilon=rms_norm_eps, + + # Activation (SwiGLU from HF config hidden_act: "silu") + add_bias_linear=False, + gated_linear_unit=True, + activation_func="swiglu", + bias_activation_fusion=True, + + # Embeddings from HF config + untie_embeddings_and_output_weights=not tie_word_embeddings, + + # Position embeddings (RoPE) from HF config + position_embedding_type="rope", + rotary_base=rope_theta, + rotary_percent=1.0, + rotary_interleaved=False, + + # Attention + kv_channels=hidden_size // num_attention_heads, + attention_dropout=0.0, + + # Initialization + init_method_std=0.02, + apply_query_key_layer_scaling=False, + attention_softmax_in_fp32=True, + + # Precision + params_dtype=getattr(args, 'params_dtype', 'bfloat16'), + bf16=getattr(args, 'bf16', True), + fp16=getattr(args, 'fp16', False), + + # Initialization + use_cpu_initialization=True, + perform_initialization=True, + + # Fusion and optimization + gradient_accumulation_fusion=False, + async_tensor_model_parallel_allreduce=False, + + # Parallelism + sequence_parallel=getattr(args, 'sequence_parallel', False), + + # Memory optimization + use_distributed_optimizer=getattr(args, 'use_distributed_optimizer', True), + ) + + +def print_mobilellm_config(config): + """Print MobileLLM configuration for verification.""" + print("\n" + "="*50) + print("MobileLLM-R1-140M Configuration") + print("="*50) + print(f"Layers: {config.num_layers}") + print(f"Hidden Size: {config.hidden_size}") + print(f"Attention Heads: {config.num_attention_heads}") + print(f"KV Heads (GQA): {config.num_query_groups}") + print(f"FFN Hidden: {config.ffn_hidden_size}") + print(f"Vocab Size: {config.vocab_size}") + print(f"Max Seq Length: {config.max_position_embeddings}") + print(f"RoPE Base: {config.rotary_base}") + print(f"Normalization: {config.normalization}") + print(f"Activation: swiglu (gated_linear_unit={config.gated_linear_unit})") + print(f"Shared Embeddings: {not config.untie_embeddings_and_output_weights}") + print(f"Precision: {config.params_dtype}") + print("="*50 + "\n") diff --git a/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py b/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py new file mode 100644 index 00000000..4ab85fa2 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py @@ -0,0 +1,61 @@ +"""MobileLLM layer specification - Standard LLaMA architecture""" + +from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_layer import TransformerLayer, TransformerLayerSubmodules +from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules +from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.mlp import MLP, MLPSubmodules + +from aiak_training_llm.models.dispatch import multiacc_modules + + +def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: + """ + Get MobileLLM layer specification using Transformer Engine modules. + + MobileLLM-R1-140M uses standard LLaMA architecture: + - Grouped Query Attention (9 heads, 3 KV heads) + - SwiGLU activation (gated_linear_unit=True) + - RMSNorm + - No QK LayerNorm + + Args: + config: Transformer configuration with MobileLLM parameters + + Returns: + ModuleSpec for MobileLLM transformer layer + """ + # Standard dense MLP with SwiGLU + mlp = ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=multiacc_modules.TELayerNormColumnParallelLinear, + linear_fc2=multiacc_modules.TERowParallelLinear, + bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, + ), + ) + + return ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + input_layernorm=IdentityOp, + self_attention=ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=multiacc_modules.TELayerNormColumnParallelLinear, + core_attention=multiacc_modules.DotProductAttention, + linear_proj=multiacc_modules.TERowParallelLinear, + q_layernorm=IdentityOp, # MobileLLM doesn't use QK LayerNorm + k_layernorm=IdentityOp, + apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, + ), + ), + self_attn_bda=multiacc_modules.get_bias_dropout_add, + pre_mlp_layernorm=IdentityOp, + mlp=mlp, + mlp_bda=multiacc_modules.get_bias_dropout_add, + ), + ) diff --git a/aiak_training_llm/models/mobilellm/mobilellm_model.py b/aiak_training_llm/models/mobilellm/mobilellm_model.py new file mode 100644 index 00000000..33410343 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/mobilellm_model.py @@ -0,0 +1,295 @@ +"""MobileLLM Model - Adapted from Qwen model for MobileLLM-R1-140M architecture""" +from collections import OrderedDict +from typing import Dict, Literal, Optional + +from torch import Tensor + +from megatron.core import InferenceParams, tensor_parallel +from megatron.core.config_logger import has_config_logger_enabled, log_config_to_disk +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding +from megatron.core.models.common.embeddings.rotary_pos_embedding import RotaryEmbedding +from megatron.core.models.common.language_module.language_module import LanguageModule +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.transformer.enums import ModelType, AttnMaskType +from megatron.core.transformer.spec_utils import ModuleSpec +from megatron.core.transformer.transformer_block import TransformerBlock +from megatron.core.transformer.transformer_config import TransformerConfig + + +def _load_state_dict_hook_ignore_extra_state(module, incompatible_keys): + """Hook to ignore Transformer Engine _extra_state used for FP8.""" + keys_to_remove = [ + key for key in incompatible_keys.missing_keys + if "input_layernorm._extra_state" in key + or "pre_mlp_layernorm._extra_state" in key + or "output_layernorm._extra_state" in key + or "self_attention.q_layernorm._extra_state" in key + or "self_attention.k_layernorm._extra_state" in key + or "linear_fc1._extra_state" in key + or "linear_fc2._extra_state" in key + ] + + for key in keys_to_remove: + if key in incompatible_keys.missing_keys: + incompatible_keys.missing_keys.remove(key) + + +class MobileLLMModel(LanguageModule): + """MobileLLM Transformer language model. + + Based on facebook/MobileLLM-R1-140M architecture (LLaMA-style): + - 15 layers + - 576 hidden size + - 9 attention heads with 3 KV heads (GQA) + - 2048 FFN hidden size + - RoPE position embeddings + - RMSNorm, SwiGLU activation + - Shared input/output embeddings + + Args: + config (TransformerConfig): Transformer config + transformer_layer_spec (ModuleSpec): Specifies module to use for transformer layers + vocab_size (int): Vocabulary size (128256 for MobileLLM) + max_sequence_length (int): Maximum sequence length (32768 for MobileLLM) + pre_process (bool): Include embedding layer (used with pipeline parallelism) + post_process (bool): Include output layer (used with pipeline parallelism) + fp16_lm_cross_entropy (bool): Use fp16 for cross entropy + parallel_output (bool): Keep outputs split across tensor parallel ranks + share_embeddings_and_output_weights (bool): Share input and output embeddings + position_embedding_type (str): Position embedding type ('rope' for MobileLLM) + rotary_percent (float): Percent of rotary dimension (1.0 for MobileLLM) + rotary_base (int): Base period for RoPE (8000000 for MobileLLM) + rope_scaling (bool): Enable RoPE scaling + rope_scaling_factor (float): RoPE scaling factor + scatter_embedding_sequence_parallel (bool): Scatter embeddings in sequence parallel + seq_len_interpolation_factor (float): Sequence length interpolation factor + """ + + def __init__( + self, + config: TransformerConfig, + transformer_layer_spec: ModuleSpec, + vocab_size: int, + max_sequence_length: int, + pre_process: bool = True, + post_process: bool = True, + fp16_lm_cross_entropy: bool = False, + parallel_output: bool = True, + share_embeddings_and_output_weights: bool = True, + position_embedding_type: Literal['learned_absolute', 'rope'] = 'rope', + rotary_percent: float = 1.0, + rotary_base: int = 8000000, + rope_scaling: bool = False, + rope_scaling_factor: float = 1.0, + scatter_embedding_sequence_parallel: bool = True, + seq_len_interpolation_factor: Optional[float] = None, + ) -> None: + super().__init__(config=config) + + if has_config_logger_enabled(config): + log_config_to_disk(config, locals(), prefix=type(self).__name__) + + self.transformer_layer_spec: ModuleSpec = transformer_layer_spec + self.vocab_size = vocab_size + self.max_sequence_length = max_sequence_length + self.pre_process = pre_process + self.post_process = post_process + self.fp16_lm_cross_entropy = fp16_lm_cross_entropy + self.parallel_output = parallel_output + self.share_embeddings_and_output_weights = share_embeddings_and_output_weights + self.position_embedding_type = position_embedding_type + + # Model type for Megatron pipelining + self.model_type = ModelType.encoder_or_decoder + + # Attributes for TensorRT-LLM export + self.max_position_embeddings = max_sequence_length + self.rotary_percent = rotary_percent + self.rotary_base = rotary_base + self.rotary_scaling = rope_scaling + + # Embedding layer + if self.pre_process: + self.embedding = LanguageModelEmbedding( + config=self.config, + vocab_size=self.vocab_size, + max_sequence_length=self.max_sequence_length, + position_embedding_type=position_embedding_type, + scatter_to_sequence_parallel=scatter_embedding_sequence_parallel, + ) + + # RoPE embeddings + if self.position_embedding_type == 'rope' and not self.config.multi_latent_attention: + self.rotary_pos_emb = RotaryEmbedding( + kv_channels=self.config.kv_channels, + rotary_percent=rotary_percent, + rotary_interleaved=self.config.rotary_interleaved, + seq_len_interpolation_factor=seq_len_interpolation_factor, + rotary_base=rotary_base, + rope_scaling=rope_scaling, + rope_scaling_factor=rope_scaling_factor, + use_cpu_initialization=self.config.use_cpu_initialization, + ) + + # Cache for RoPE tensors + self.rotary_pos_emb_cache = {} + + # Transformer decoder + self.decoder = TransformerBlock( + config=self.config, + spec=transformer_layer_spec, + pre_process=self.pre_process, + post_process=self.post_process, + ) + + # Output layer + if self.post_process: + self.output_layer = tensor_parallel.ColumnParallelLinear( + config.hidden_size, + self.vocab_size, + config=config, + init_method=config.init_method, + bias=False, + skip_bias_add=False, + gather_output=not self.parallel_output, + skip_weight_param_allocation=self.pre_process + and self.share_embeddings_and_output_weights, + ) + + # Register hook for TE compatibility + if self.share_embeddings_and_output_weights and (self.pre_process or self.post_process): + self.register_load_state_dict_post_hook(_load_state_dict_hook_ignore_extra_state) + + def set_input_tensor(self, input_tensor: Tensor) -> None: + """Set input tensor for pipeline parallelism.""" + self.decoder.set_input_tensor(input_tensor) + + def forward( + self, + input_ids: Tensor, + position_ids: Tensor, + attention_mask: Tensor, + attn_mask_type: Optional[AttnMaskType] = None, + decoder_input: Tensor = None, + labels: Tensor = None, + rotary_pos_emb: Tensor = None, + inference_params: InferenceParams = None, + packed_seq_params: PackedSeqParams = None, + extra_block_kwargs: dict = None, + runtime_gather_output: Optional[bool] = None, + ) -> Tensor: + """Forward pass of the model. + + Args: + runtime_gather_output (bool): Gather output at runtime. Default None means + `parallel_output` arg in the constructor will be used. + """ + # If decoder_input is provided (not None), then input_ids and position_ids are ignored. + # Otherwise, apply embedding layer on input_ids and position_ids to get decoder_input. + + # Decoder embedding + if decoder_input is None: + if self.pre_process: + decoder_input = self.embedding(input_ids=input_ids, position_ids=position_ids) + else: + # intermediate stage of pipeline + # decoder will get hidden_states from encoder.input_tensor + decoder_input = None + + # Rotary positional embeddings + if rotary_pos_emb is None and self.position_embedding_type == 'rope': + rotary_seq_len = self.max_sequence_length + if inference_params is not None: + rotary_seq_len = inference_params.max_sequence_length + + # Check cache + cache_key = rotary_seq_len + if cache_key in self.rotary_pos_emb_cache: + rotary_pos_emb = self.rotary_pos_emb_cache[cache_key] + else: + rotary_pos_emb = self.rotary_pos_emb(rotary_seq_len) + self.rotary_pos_emb_cache[cache_key] = rotary_pos_emb + + # Forward through transformer + hidden_states = self.decoder( + hidden_states=decoder_input, + attention_mask=attention_mask, + attn_mask_type=attn_mask_type, + inference_params=inference_params, + rotary_pos_emb=rotary_pos_emb, + packed_seq_params=packed_seq_params, + **(extra_block_kwargs or {}), + ) + + # Output layer + if not self.post_process: + return hidden_states + + # Get logits + output_weight = None + if self.share_embeddings_and_output_weights: + output_weight = self.shared_embedding_or_output_weight() + + logits, _ = self.output_layer( + hidden_states, weight=output_weight, runtime_gather_output=runtime_gather_output + ) + + # If labels are provided, compute and return loss + if labels is None: + # [s b h] => [b s h] + return logits.transpose(0, 1).contiguous() + + loss = self.compute_language_model_loss(labels, logits) + return loss + + def sharded_state_dict( + self, prefix: str = '', sharded_offsets: tuple = (), metadata: dict = None + ) -> ShardedStateDict: + """Provide sharded state dict for distributed checkpointing.""" + sharded_state_dict = {} + + if self.pre_process: + embedding_prefix = f'{prefix}embedding.' + embedding_sharded_state_dict = self.embedding.sharded_state_dict( + prefix=embedding_prefix, sharded_offsets=sharded_offsets, metadata=metadata + ) + sharded_state_dict.update(embedding_sharded_state_dict) + + decoder_prefix = f'{prefix}decoder.' + decoder_sharded_state_dict = self.decoder.sharded_state_dict( + prefix=decoder_prefix, sharded_offsets=sharded_offsets, metadata=metadata + ) + sharded_state_dict.update(decoder_sharded_state_dict) + + if self.post_process: + output_layer_prefix = f'{prefix}output_layer.' + output_layer_key = f'{output_layer_prefix}weight' + if self.share_embeddings_and_output_weights: + if not self.pre_process: + sharded_output_layer_tensor = metadata['embedding'][0] + else: + sharded_embedding_tensor = embedding_sharded_state_dict[ + f'{embedding_prefix}word_embeddings.weight' + ] + sharded_output_layer_tensor = sharded_embedding_tensor + + sharded_state_dict[output_layer_key] = sharded_output_layer_tensor + else: + output_layer_state_dict = self.output_layer.sharded_state_dict( + prefix=output_layer_prefix, + sharded_offsets=sharded_offsets, + metadata=metadata, + ) + sharded_state_dict.update(output_layer_state_dict) + + return sharded_state_dict + + def shared_embedding_or_output_weight(self): + """Get shared embedding/output weight for gradient all-reduce.""" + if self.share_embeddings_and_output_weights: + if self.pre_process: + return self.embedding.word_embeddings.weight + elif self.post_process: + return self.output_layer.weight + return None diff --git a/aiak_training_llm/models/mobilellm/mobilellm_provider.py b/aiak_training_llm/models/mobilellm/mobilellm_provider.py new file mode 100644 index 00000000..d5a38c90 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/mobilellm_provider.py @@ -0,0 +1,91 @@ +"""MobileLLM model provider - Registers MobileLLM with model factory""" +import inspect +from contextlib import nullcontext + +from megatron.core.transformer.spec_utils import import_module + +from aiak_training_llm.utils import get_args, build_transformer_config, print_rank_0 +from aiak_training_llm.utils.constants import LanguageModelFamilies + +from aiak_training_llm.models.factory import register_model_provider + +from .mobilellm_model import MobileLLMModel +from .mobilellm_layer_spec import get_mobilellm_layer_with_te_spec + + +@register_model_provider(model_family=[LanguageModelFamilies.MOBILELLM]) +def mobilellm_model_provider( + pre_process: bool = True, + post_process: bool = True, + parallel_output: bool = True, +) -> MobileLLMModel: + """Build the MobileLLM model. + + Uses facebook/MobileLLM-R1-140M architecture: + - 15 layers, 576 hidden size + - 9 attention heads with 3 KV heads (GQA) + - 2048 FFN hidden size + - 128,256 vocabulary + - 32,768 context length + - RoPE with base 8,000,000 + - RMSNorm, SwiGLU activation + - Shared input/output embeddings + + Args: + pre_process: Include embedding layer + post_process: Include output layer + parallel_output: Keep outputs split across tensor parallel ranks + + Returns: + MobileLLMModel instance + """ + args = get_args() + + print_rank_0('Building MobileLLM model ...') + + config = build_transformer_config(args) + + if args.use_legacy_models: + raise ValueError("Classic Megatron-LM models are not supported.") + + if args.spec is not None: + transformer_layer_spec = import_module(args.spec) + else: + transformer_layer_spec = get_mobilellm_layer_with_te_spec(config) + + build_model_context = nullcontext + build_model_context_args = {} + if args.fp8_param_gather: + try: + from transformer_engine.pytorch import fp8_model_init + + build_model_context = fp8_model_init + build_model_context_args["enabled"] = True + + if "preserve_high_precision_init_val" in inspect.signature(fp8_model_init).parameters: + build_model_context_args["preserve_high_precision_init_val"] = True + except: + raise RuntimeError( + "--fp8-param-gather requires `fp8_model_init` from TransformerEngine, but not found." + ) + + with build_model_context(**build_model_context_args): + model = MobileLLMModel( + config=config, + transformer_layer_spec=transformer_layer_spec, + vocab_size=args.padded_vocab_size, + max_sequence_length=args.max_position_embeddings, + pre_process=pre_process, + post_process=post_process, + fp16_lm_cross_entropy=args.fp16_lm_cross_entropy, + parallel_output=parallel_output, + share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights, + position_embedding_type=args.position_embedding_type, + rotary_percent=args.rotary_percent, + rotary_base=args.rotary_base, + rope_scaling=args.use_rope_scaling, + rope_scaling_factor=args.rope_scaling_factor, + seq_len_interpolation_factor=args.rotary_seq_len_interpolation_factor, + ) + + return model diff --git a/aiak_training_llm/tokenizer/tokenization_hf.py b/aiak_training_llm/tokenizer/tokenization_hf.py index 2c56cff7..29d36f5f 100644 --- a/aiak_training_llm/tokenizer/tokenization_hf.py +++ b/aiak_training_llm/tokenizer/tokenization_hf.py @@ -22,6 +22,12 @@ def __init__(self, **kwargs, ): super().__init__(name_or_path) + # Bypass HF repo validation for local paths + import os + if os.path.isdir(name_or_path) or os.path.isfile(name_or_path): + # For local paths, set token to False to avoid HF Hub validation + kwargs.setdefault('token', False) + self.tokenizer = AutoTokenizer.from_pretrained( name_or_path, use_fast=use_fast_tokenizer, diff --git a/aiak_training_llm/utils/constants.py b/aiak_training_llm/utils/constants.py index c42c3394..04cacb30 100644 --- a/aiak_training_llm/utils/constants.py +++ b/aiak_training_llm/utils/constants.py @@ -73,6 +73,7 @@ class LanguageModelFamilies(_BaseFamilies): QWEN3 = "qwen3" MIXTRAL = "mixtral" DEEPSEEK = "deepseek" + MOBILELLM = "mobilellm" class VideoLanguageModelFamilies(_BaseFamilies): diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index a8b89a28..7139a1aa 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -7,12 +7,12 @@ TP="${1:-1}" PP="${2:-1}" # pipeline parallel # Defaults: TP=1, PP=1. # SEQ_LEN="${3:-32768}" -SEQ_LEN="${3:-1024}" +SEQ_LEN="${3:-512}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size -GBS="${5:-8}" +GBS="${5:-1}" # NSTEP="${6:-2500}" # number of training iterations -NSTEP="${6:-50}" # number of training iterations +NSTEP="${6:-5}" # number of training iterations - reduced to 5 samples for CUDA memory # DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} # TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} # CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} @@ -90,7 +90,7 @@ TENSORBOARD_PATH="${SAVE_CKPT_PATH}/tensorboard" mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=${GPUS_PER_NODE:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-4} # Change for multinode config MASTER_ADDR=${MASTER_ADDR:-"${list_ip[0]}"} @@ -99,6 +99,7 @@ MASTER_PORT=${MASTER_PORT:-"26000"} if [[ $SINGLE_NODE -eq 1 ]]; then DISTRIBUTED_ARGS=( --nproc_per_node "$GPUS_PER_NODE" + --master_port "$MASTER_PORT" ) else DISTRIBUTED_ARGS=( @@ -167,7 +168,7 @@ TRAINING_ARGS=( --ckpt-fully-parallel-load --recompute-granularity full --recompute-method uniform - --recompute-num-layers 4 + --recompute-num-layers 8 ) MODEL_PARALLEL_ARGS=( diff --git a/tools/convert_checkpoint/config/mobilellm-140m.json b/tools/convert_checkpoint/config/mobilellm-140m.json new file mode 100644 index 00000000..9d7156d0 --- /dev/null +++ b/tools/convert_checkpoint/config/mobilellm-140m.json @@ -0,0 +1,87 @@ +{ + "args": { + "common": { + "num_layers": 15, + "hidden_size": 576, + "ffn_hidden_size": 2048, + "num_attention_heads": 9, + "num_key_value_heads": 3, + "max_position_embeddings": 32768, + "vocab_size": 128256 + }, + "huggingface": { + "architectures": [ + "Llama4ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": [128001, 128008, 128009], + "hidden_act": "silu", + "initializer_range": 0.02, + "model_type": "llama4_text", + "rms_norm_eps": 1e-05, + "rope_theta": 8000000.0, + "tie_word_embeddings": true, + "use_cache": true + }, + "mcore": { + "untie_embeddings_and_output_weights": false, + "num_layers_per_virtual_pipeline_stage": null, + "virtual_pipeline_model_parallel_size": null, + "use_rotary_position_embeddings": true, + "add_embedding_padding": true, + "make_vocab_size_divisible_by": 128, + "transpose_mlp_dense": true, + "transpose_query_key_value": true + } + }, + "name_map": { + "huggingface": { + "word_embeddings": "model.embed_tokens", + "transformer": "model", + "layer_prefix": "layers", + "input_layernorm": "input_layernorm", + "attention.query_key_value": [ + "self_attn.q_proj", + "self_attn.k_proj", + "self_attn.v_proj" + ], + "attention.dense": "self_attn.o_proj", + "post_attention_layernorm": "post_attention_layernorm", + "mlp.dense_h_to_4h": [ + "feed_forward.gate_proj", + "feed_forward.up_proj" + ], + "mlp.dense_4h_to_h": "feed_forward.down_proj", + "final_layernorm": "model.norm", + "word_embeddings_for_head": "lm_head" + }, + "mcore": { + "word_embeddings": "embedding.word_embeddings", + "word_position_embeddings": "model.embedding.position_embeddings", + "transformer": "model", + "layer_prefix": "decoder.layers", + "input_layernorm": "self_attention.linear_qkv.layer_norm", + "attention.query_key_value": "self_attention.linear_qkv", + "attention.dense": "self_attention.linear_proj", + "post_attention_layernorm": "mlp.linear_fc1.layer_norm", + "mlp.dense_h_to_4h": "mlp.linear_fc1", + "mlp.dense_4h_to_h": "mlp.linear_fc2", + "final_layernorm": "decoder.final_layernorm", + "word_embeddings_for_head": "output_layer", + "transformer_tpl": "model%d" + } + }, + "tensor_parallel_dim": { + "word_embeddings.weight": 0, + "attention.query_key_value.weight": 0, + "attention.query_key_value.bias": 0, + "attention.dense.weight": 1, + "mlp.dense_h_to_4h.weight": 0, + "mlp.dense_h_to_4h.bias": 0, + "mlp.dense_4h_to_h.weight": 1, + "word_embeddings_for_head.weight": 0 + }, + "torch_dtype": "bfloat16", + "transformers_version": "4.55.0" +} From 4a9753d9600da1ee8bf9509273259ddbe45335de Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 15 Jan 2026 21:47:15 +0400 Subject: [PATCH 15/39] Add MobileLLM checkpoints using Git LFS - Add HuggingFace checkpoint (model.safetensors, tokenizer.json) - Add Megatron checkpoint (TP=2 format) - Configure Git LFS for large model files --- .gitattributes | 3 + .gitignore | 7 - .../mobilellm/hf_checkpoint/.gitattributes | 36 + .../models/mobilellm/hf_checkpoint/LICENSE | 124 + .../models/mobilellm/hf_checkpoint/README.md | 622 +++++ .../hf_checkpoint/chat_template.jinja | 91 + .../mobilellm/hf_checkpoint/config.json | 79 + .../hf_checkpoint/generation_config.json | 11 + .../mobilellm/hf_checkpoint/model.safetensors | 3 + .../hf_checkpoint/special_tokens_map.json | 16 + .../mobilellm/hf_checkpoint/tokenizer.json | 3 + .../hf_checkpoint/tokenizer_config.json | 2065 +++++++++++++++++ .../latest_checkpointed_iteration.txt | 1 + .../release/mp_rank_00/model_optim_rng.pt | 3 + .../release/mp_rank_01/model_optim_rng.pt | 3 + 15 files changed, 3060 insertions(+), 7 deletions(-) create mode 100644 .gitattributes create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/.gitattributes create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/LICENSE create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/README.md create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/chat_template.jinja create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/config.json create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/generation_config.json create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/model.safetensors create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/special_tokens_map.json create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer.json create mode 100644 aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer_config.json create mode 100644 aiak_training_llm/models/mobilellm/megatron_checkpoint/latest_checkpointed_iteration.txt create mode 100644 aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_00/model_optim_rng.pt create mode 100644 aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_01/model_optim_rng.pt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..7c7d6875 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.bin filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index b0c60a4c..11a185b6 100644 --- a/.gitignore +++ b/.gitignore @@ -49,13 +49,6 @@ coverage.xml # Translations *.mo -# Model checkpoints and weights -*.safetensors -*.pt -*.bin -**/hf_checkpoint/ -**/megatron_checkpoint/ - # Sphinx documentation /docs/_build/ diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/.gitattributes b/aiak_training_llm/models/mobilellm/hf_checkpoint/.gitattributes new file mode 100644 index 00000000..52373fe2 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/LICENSE b/aiak_training_llm/models/mobilellm/hf_checkpoint/LICENSE new file mode 100644 index 00000000..65c038ca --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/LICENSE @@ -0,0 +1,124 @@ +FAIR Noncommercial Research License +v1 Last Updated: August 18, 2025 + +“Acceptable Use Policy” means the FAIR Acceptable Use Policy, applicable to Research Materials, that is incorporated into this Agreement. + +“Agreement” means the terms and conditions for use, reproduction, distribution and modification of the Research Materials set forth herein. + + +“Documentation” means the specifications, manuals and documentation accompanying +Research Materials distributed by Meta. + + +“Licensee” or “you” means you, or your employer or any other person or entity (if you are entering into this Agreement on such person or entity’s behalf), of the age required under applicable laws, rules or regulations to provide legal consent and that has legal authority to bind your employer or such other person or entity if you are entering in this Agreement on their behalf. + + +“Meta” or “we” means Meta Platforms Ireland Limited (if you are located in or, if you are an entity, your principal place of business is in the EEA or Switzerland) and Meta Platforms, Inc. (if you are located outside of the EEA or Switzerland). + +“Noncommercial Research Uses” means noncommercial research use cases related to research, development, education, processing, or analysis and in each case, is not primarily intended for commercial advantage or monetary compensation to you or others. + +“Research Materials” means, collectively, Documentation and the models, software and algorithms, including machine-learning model code, trained model weights, inference-enabling code, training-enabling code, fine-tuning enabling code, demonstration materials and other elements of the foregoing distributed by Meta and made available under this Agreement. + +By clicking “I Accept” below or by using or distributing any portion or element of the Research Materials, you agree to be bound by this Agreement. + + +1. License Rights and Redistribution. + + +a. Grant of Rights. You are granted a non-exclusive, worldwide, non-transferable and royalty-free limited license under Meta’s intellectual property or other rights owned by Meta embodied in the Research Materials to use, reproduce, distribute, copy, create derivative works of, and make modifications to the Research Materials. + +b. Redistribution and Use. + i. You will not use the Research Materials or any outputs or results of the Research Materials in connection with any commercial uses or for any uses other than Noncommercial Research Uses; + + +ii. Distribution of Research Materials, and any derivative works thereof, are subject to the terms of this Agreement. If you distribute or make the Research Materials, or any derivative works thereof, available to a third party, you may only do so under the terms of this Agreement. You shall also provide a copy of this Agreement to such third party. + + +iii. If you submit for publication the results of research you perform on, using, or otherwise in connection with Research Materials, you must acknowledge the use of Research Materials in your publication. + + +iv. Your use of the Research Materials must comply with applicable laws and regulations (including Trade Control Laws) and adhere to the FAIR Acceptable Use Policy, which is hereby incorporated by reference into this Agreement. +2. User Support. Your Noncommercial Research Use of the Research Materials is done at your own discretion; Meta does not process any information nor provide any service in relation to such use. Meta is under no obligation to provide any support services for the Research Materials. Any support provided is “as is”, “with all faults”, and without warranty of any kind. + + +3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE RESEARCH MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OF ANY KIND, AND META DISCLAIMS ALL WARRANTIES OF ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING THE RESEARCH MATERIALS AND ASSUME ANY RISKS ASSOCIATED WITH YOUR USE OF THE RESEARCH MATERIALS AND ANY OUTPUT AND RESULTS. + +4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS OR ANY DIRECT OR INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR PUNITIVE DAMAGES, EVEN IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING. + +5. Intellectual Property. + + +a. Subject to Meta’s ownership of Research Materials and derivatives made by or for Meta, with respect to any derivative works and modifications of the Research Materials that are made by you, as between you and Meta, you are and will be the owner of such derivative works and modifications. + +b. If you institute litigation or other proceedings against Meta or any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Research Materials, outputs or results, or any portion of any of the foregoing, constitutes infringement of intellectual property or other rights owned or licensable by you, then any licenses granted to you under this Agreement shall terminate as of the date such litigation or claim is filed or instituted. You will indemnify and hold harmless Meta from and against any claim by any third party arising out of or related to your use or distribution of the Research Materials. + +6. Term and Termination. The term of this Agreement will commence upon your acceptance of this Agreement or access to the Research Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. Meta may terminate this Agreement if you are in breach of any term or condition of this Agreement. Upon termination of this Agreement, you shall delete and cease use of the Research Materials. Sections 3, 4 and 7 shall survive the termination of this Agreement. + +7. Governing Law and Jurisdiction. This Agreement will be governed and construed under the laws of the State of California without regard to choice of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. The courts of California shall have exclusive jurisdiction of any dispute arising out of this Agreement. + + +8. Modifications and Amendments. Meta may modify this Agreement from time to time; provided that they are similar in spirit to the current version of the Agreement, but may differ in detail to address new problems or concerns. All such changes will be effective immediately. Your continued use of the Research Materials after any modification to this Agreement constitutes your agreement to such modification. Except as provided in this Agreement, no modification or addition to any provision of this Agreement will be binding unless it is in writing and signed by an authorized representative of both you and Meta. + + +FAIR Acceptable Use Policy + +The Fundamental AI Research (FAIR) team at Meta seeks to further understanding of new and existing research domains with the mission of advancing the state-of-the-art in artificial intelligence through open research for the benefit of all. + +As part of this mission, Meta makes certain research materials available for noncommercial research use. Meta is committed to promoting the safe and responsible use of such research materials. + +Prohibited Uses + +You agree you will not use, or allow others to use, Research Materials to: + + Violate the law or others’ rights, including to: +Engage in, promote, generate, contribute to, encourage, plan, incite, or further illegal or unlawful activity or content, such as: +Violence or terrorism +Exploitation or harm to children, including the solicitation, creation, acquisition, or dissemination of child exploitative content or failure to report Child Sexual Abuse Material +Human trafficking, exploitation, and sexual violence +The illegal distribution of information or materials to minors, including obscene materials, or failure to employ legally required age-gating in connection with such information or materials. +Sexual solicitation +Any other criminal activity + +Engage in, promote, incite, or facilitate the harassment, abuse, threatening, or bullying of individuals or groups of individuals + +Engage in, promote, incite, or facilitate discrimination or other unlawful or harmful conduct in the provision of employment, employment benefits, credit, housing, other economic benefits, or other essential goods and services + +Engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or related professional practices + +Collect, process, disclose, generate, or infer health, demographic, or other sensitive personal or private information about individuals without rights and consents required by applicable laws + +Engage in or facilitate any action or generate any content that infringes, misappropriates, or otherwise violates any third-party rights, including the outputs or results of any technology using FAIR research materials + +Create, generate, or facilitate the creation of malicious code, malware, computer viruses or do anything else that could disable, overburden, interfere with or impair the proper working, integrity, operation or appearance of a website or computer system + +2. Engage in, promote, incite, facilitate, or assist in the planning or development of activities that present a risk of death or bodily harm to individuals, including use of research artifacts related to the following: + +Military, warfare, nuclear industries or applications, espionage, use for materials or activities that are subject to the International Traffic Arms Regulations (ITAR) maintained by the United States Department of State + +Guns and illegal weapons (including weapon development) + +Illegal drugs and regulated/controlled substances + +Operation of critical infrastructure, transportation technologies, or heavy machinery + +Self-harm or harm to others, including suicide, cutting, and eating disorders + +Any content intended to incite or promote violence, abuse, or any infliction of bodily harm to an individual + +3. Intentionally deceive or mislead others, including use of FAIR Research Materials related to the following: + + Generating, promoting, or furthering fraud or the creation or promotion of disinformation + + Generating, promoting, or furthering defamatory content, including the creation of defamatory statements, images, or other content + +Generating, promoting, or further distributing spam + + Impersonating another individual without consent, authorization, or legal right + +Representing that outputs of FAIR research materials or outputs from technology using FAIR research materials are human-generated + +Generating or facilitating false online engagement, including fake reviews and other means of fake online engagement + +4. Fail to appropriately disclose to end users any known dangers of your Research Materials. + +Please report any violation of this Policy or other problems that could lead to a violation of this Policy by submitting a report here [https://docs.google.com/forms/d/e/1FAIpQLSeb11cryAopJ7LNrC4nxEUXrHY26hfkXQMf_uH-oFgA3WlYZQ/viewform]. diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/README.md b/aiak_training_llm/models/mobilellm/hf_checkpoint/README.md new file mode 100644 index 00000000..ba02dfb9 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/README.md @@ -0,0 +1,622 @@ +--- +license: other +license_name: fair-noncommercial-research +extra_gated_prompt: > + FAIR Noncommercial Research License v1 Last Updated: August 18, 2025 + + “Acceptable Use Policy” means the FAIR Acceptable Use Policy, applicable to + Research Materials, that is incorporated into this Agreement. + + “Agreement” means the terms and conditions for use, reproduction, distribution + and modification of the Research Materials set forth herein. + + + “Documentation” means the specifications, manuals and documentation + accompanying Research Materials distributed by Meta. + + + “Licensee” or “you” means you, or your employer or any other person or entity + (if you are entering into this Agreement on such person or entity’s behalf), + of the age required under applicable laws, rules or regulations to provide + legal consent and that has legal authority to bind your employer or such other + person or entity if you are entering in this Agreement on their behalf. + + + “Meta” or “we” means Meta Platforms Ireland Limited (if you are located in or, + if you are an entity, your principal place of business is in the EEA or + Switzerland) and Meta Platforms, Inc. (if you are located outside of the EEA + or Switzerland). + + “Noncommercial Research Uses” means noncommercial research use cases related + to research, development, education, processing, or analysis and in each case, + is not primarily intended for commercial advantage or monetary compensation to + you or others. + + “Research Materials” means, collectively, Documentation and the models, + software and algorithms, including machine-learning model code, trained model + weights, inference-enabling code, training-enabling code, fine-tuning enabling + code, demonstration materials and other elements of the foregoing distributed + by Meta and made available under this Agreement. + + By clicking “I Accept” below or by using or distributing any portion or + element of the Research Materials, you agree to be bound by this Agreement. + + + 1. License Rights and Redistribution. + + + a. Grant of Rights. You are granted a non-exclusive, worldwide, + non-transferable and royalty-free limited license under Meta’s intellectual + property or other rights owned by Meta embodied in the Research Materials to + use, reproduce, distribute, copy, create derivative works of, and make + modifications to the Research Materials. + + b. Redistribution and Use. i. You will not use the Research Materials or any + outputs or results of the Research Materials in connection with any commercial + uses or for any uses other than Noncommercial Research Uses; + + + ii. Distribution of Research Materials, and any derivative works thereof, are + subject to the terms of this Agreement. If you distribute or make the Research + Materials, or any derivative works thereof, available to a third party, you + may only do so under the terms of this Agreement. You shall also provide a + copy of this Agreement to such third party. + + + iii. If you submit for publication the results of research you perform on, + using, or otherwise in connection with Research Materials, you must + acknowledge the use of Research Materials in your publication. + + + iv. Your use of the Research Materials must comply with applicable laws and + regulations (including Trade Control Laws) and adhere to the FAIR Acceptable + Use Policy, which is hereby incorporated by reference into this Agreement. 2. + User Support. Your Noncommercial Research Use of the Research Materials is + done at your own discretion; Meta does not process any information nor provide + any service in relation to such use. Meta is under no obligation to provide + any support services for the Research Materials. Any support provided is “as + is”, “with all faults”, and without warranty of any kind. + + + 3. Disclaimer of Warranty. UNLESS REQUIRED BY APPLICABLE LAW, THE RESEARCH + MATERIALS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED ON AN “AS IS” + BASIS, WITHOUT WARRANTIES OF ANY KIND, AND META DISCLAIMS ALL WARRANTIES OF + ANY KIND, BOTH EXPRESS AND IMPLIED, INCLUDING, WITHOUT LIMITATION, ANY + WARRANTIES OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A + PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE + APPROPRIATENESS OF USING OR REDISTRIBUTING THE RESEARCH MATERIALS AND ASSUME + ANY RISKS ASSOCIATED WITH YOUR USE OF THE RESEARCH MATERIALS AND ANY OUTPUT + AND RESULTS. + + 4. Limitation of Liability. IN NO EVENT WILL META OR ITS AFFILIATES BE LIABLE + UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS + LIABILITY, OR OTHERWISE, ARISING OUT OF THIS AGREEMENT, FOR ANY LOST PROFITS + OR ANY DIRECT OR INDIRECT, SPECIAL, CONSEQUENTIAL, INCIDENTAL, EXEMPLARY OR + PUNITIVE DAMAGES, EVEN IF META OR ITS AFFILIATES HAVE BEEN ADVISED OF THE + POSSIBILITY OF ANY OF THE FOREGOING. + + 5. Intellectual Property. + + + a. Subject to Meta’s ownership of Research Materials and derivatives made by + or for Meta, with respect to any derivative works and modifications of the + Research Materials that are made by you, as between you and Meta, you are and + will be the owner of such derivative works and modifications. + + b. If you institute litigation or other proceedings against Meta or any entity + (including a cross-claim or counterclaim in a lawsuit) alleging that the + Research Materials, outputs or results, or any portion of any of the + foregoing, constitutes infringement of intellectual property or other rights + owned or licensable by you, then any licenses granted to you under this + Agreement shall terminate as of the date such litigation or claim is filed or + instituted. You will indemnify and hold harmless Meta from and against any + claim by any third party arising out of or related to your use or distribution + of the Research Materials. + + 6. Term and Termination. The term of this Agreement will commence upon your + acceptance of this Agreement or access to the Research Materials and will + continue in full force and effect until terminated in accordance with the + terms and conditions herein. Meta may terminate this Agreement if you are in + breach of any term or condition of this Agreement. Upon termination of this + Agreement, you shall delete and cease use of the Research Materials. Sections + 3, 4 and 7 shall survive the termination of this Agreement. + + 7. Governing Law and Jurisdiction. This Agreement will be governed and + construed under the laws of the State of California without regard to choice + of law principles, and the UN Convention on Contracts for the International + Sale of Goods does not apply to this Agreement. The courts of California shall + have exclusive jurisdiction of any dispute arising out of this Agreement. + + + 8. Modifications and Amendments. Meta may modify this Agreement from time to + time; provided that they are similar in spirit to the current version of the + Agreement, but may differ in detail to address new problems or concerns. All + such changes will be effective immediately. Your continued use of the Research + Materials after any modification to this Agreement constitutes your agreement + to such modification. Except as provided in this Agreement, no modification or + addition to any provision of this Agreement will be binding unless it is in + writing and signed by an authorized representative of both you and Meta. + + + FAIR Acceptable Use Policy + + The Fundamental AI Research (FAIR) team at Meta seeks to further understanding + of new and existing research domains with the mission of advancing the + state-of-the-art in artificial intelligence through open research for the + benefit of all. + + As part of this mission, Meta makes certain research materials available for + noncommercial research use. Meta is committed to promoting the safe and + responsible use of such research materials. + + Prohibited Uses + + You agree you will not use, or allow others to use, Research Materials to: + + Violate the law or others’ rights, including to: Engage in, promote, generate, + contribute to, encourage, plan, incite, or further illegal or unlawful + activity or content, such as: Violence or terrorism Exploitation or harm to + children, including the solicitation, creation, acquisition, or dissemination + of child exploitative content or failure to report Child Sexual Abuse Material + Human trafficking, exploitation, and sexual violence The illegal distribution + of information or materials to minors, including obscene materials, or failure + to employ legally required age-gating in connection with such information or + materials. Sexual solicitation Any other criminal activity + + Engage in, promote, incite, or facilitate the harassment, abuse, threatening, + or bullying of individuals or groups of individuals + + Engage in, promote, incite, or facilitate discrimination or other unlawful or + harmful conduct in the provision of employment, employment benefits, credit, + housing, other economic benefits, or other essential goods and services + + Engage in the unauthorized or unlicensed practice of any profession including, + but not limited to, financial, legal, medical/health, or related professional + practices + + Collect, process, disclose, generate, or infer health, demographic, or other + sensitive personal or private information about individuals without rights and + consents required by applicable laws + + Engage in or facilitate any action or generate any content that infringes, + misappropriates, or otherwise violates any third-party rights, including the + outputs or results of any technology using FAIR research materials + + Create, generate, or facilitate the creation of malicious code, malware, + computer viruses or do anything else that could disable, overburden, interfere + with or impair the proper working, integrity, operation or appearance of a + website or computer system + + 2. Engage in, promote, incite, facilitate, or assist in the planning or + development of activities that present a risk of death or bodily harm to + individuals, including use of research artifacts related to the following: + + Military, warfare, nuclear industries or applications, espionage, use for + materials or activities that are subject to the International Traffic Arms + Regulations (ITAR) maintained by the United States Department of State + + Guns and illegal weapons (including weapon development) + + Illegal drugs and regulated/controlled substances + + Operation of critical infrastructure, transportation technologies, or heavy + machinery + + Self-harm or harm to others, including suicide, cutting, and eating disorders + + Any content intended to incite or promote violence, abuse, or any infliction + of bodily harm to an individual + + 3. Intentionally deceive or mislead others, including use of FAIR Research + Materials related to the following: + + Generating, promoting, or furthering fraud or the creation or promotion of + disinformation + + Generating, promoting, or furthering defamatory content, including the + creation of defamatory statements, images, or other content + + Generating, promoting, or further distributing spam + + Impersonating another individual without consent, authorization, or legal + right + + Representing that outputs of FAIR research materials or outputs from + technology using FAIR research materials are human-generated + + Generating or facilitating false online engagement, including fake reviews and + other means of fake online engagement + + 4. Fail to appropriately disclose to end users any known dangers of your + Research Materials. + + Please report any violation of this Policy or other problems that could lead + to a violation of this Policy by submitting a report here + [https://docs.google.com/forms/d/e/1FAIpQLSeb11cryAopJ7LNrC4nxEUXrHY26hfkXQMf_uH-oFgA3WlYZQ/viewform]. +extra_gated_fields: + First Name: text + Last Name: text + Date of birth: date_picker + Country: country + Affiliation: text + Job title: + type: select + options: + - Student + - Research Graduate + - AI researcher + - AI developer/engineer + - Reporter + - Other + geo: ip_location + By clicking Submit below I accept the terms of the license and acknowledge that the information I provide will be collected stored processed and shared in accordance with the Meta Privacy Policy: checkbox +extra_gated_description: >- + The information you provide will be collected, stored, processed and shared in + accordance with the [Meta Privacy + Policy](https://www.facebook.com/privacy/policy/). +extra_gated_button_content: Submit +extra_gated_heading: >- + Please be sure to provide your full legal name, date of birth, and full + organization name with all corporate identifiers. Avoid the use of acronyms + and special characters. Failure to follow these instructions may prevent you + from accessing this model and others on Hugging Face. You will not have the + ability to edit this form after submission, so please ensure all information + is accurate. +language: +- en +library_name: transformers +tags: +- facebook +- meta +- pytorch +- mobilellm +base_model: +- facebook/MobileLLM-R1-140M-base +--- + +

+ +

+

+ 🤗 Hugging Face   |    📑 Paper    |    💻 Code    +

+ +# Model Details + +We present MobileLLM-R1, a new series of efficient reasoning models in the MobileLLM family. The release includes two categories of models: + +Base models: +- [MobileLLM-R1-140M-base](https://huggingface.co/facebook/MobileLLM-R1-140M-base/) +- [MobileLLM-R1-360M-base](https://huggingface.co/facebook/MobileLLM-R1-360M-base/) +- [MobileLLM-R1-950M-base](https://huggingface.co/facebook/MobileLLM-R1-950M-base/) + +Final models: +- [MobileLLM-R1-140M](https://huggingface.co/facebook/MobileLLM-R1-140M/) +- [MobileLLM-R1-360M](https://huggingface.co/facebook/MobileLLM-R1-360M/) +- [MobileLLM-R1-950M](https://huggingface.co/facebook/MobileLLM-R1-950M/) + +> **Note**: These models are not general-purpose chat models. They are Supervised Fine-Tuned (SFT) models, specifically trained to address mathematical, programming (Python, C++), and scientific problems. + +In addition to the models, we release the complete training recipes and data sources to ensure reproducibility and support further research. + +Remarkably, the MobileLLM-R1 950M, pre-trained on only **~2T high-quality tokens** and with fewer than 5T total training tokens, achieves comparable or superior performance to Qwen3 0.6B, which was trained on 36T tokens, across MATH, GSM8K, MMLU, and LiveCodeBench benchmarks. + +Compared to existing fully open-source models, MobileLLM-R1 950M model achieves **~5× higher accuracy on MATH** compared to the Olmo 1.24B model and **~2× higher accuracy** relative to the SmolLM2 1.7B model, despite being substantially smaller in parameter scale. In addition, MobileLLM-R1 950M outperforms both Olmo 1.24B and SmolLM2 1.7B **by a wide margin on coding benchmarks**, establishing a new state-of-the-art among fully open-source models. + +## News +- Sept 12, 2025: 🚀 MobileLLM-R1 models are released on [HuggingFace](https://huggingface.co/collections/facebook/mobilellm-r1-68c4597b104fac45f28f448e). +- Spet 28, 2025: 🌟 The training code is also available on [GitHub](https://github.com/facebookresearch/MobileLLM-R1). +- Spet 29, 2025: 🎉 The technical report "[MobileLLM-R1: Exploring the Limits of Sub-Billion Language Model Reasoners with Open Training Recipes](https://arxiv.org/pdf/2509.24945)" is also available! Please check it out. + +# Highlights + + +### Pretrained Model +![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/660f893bae89429c07a32cdb/bZCn1kAhxJUl79cKmz24y.jpeg) + +### Token efficiency comparison across pretrained models +![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/660f893bae89429c07a32cdb/s1bodFZmznd9vyBj6hi25.jpeg) + +### Post-trained Model +![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/660f893bae89429c07a32cdb/upEbYrjKnXJc-APYPlN9M.jpeg) + +**Model Architecture**: + +| | # Layers | # Attnetion Heads | # KV Heads | Dim | Hidden Dim | Params | +| --- | --- | --- | --- | --- | --- | --- | +| MobileLLM-R1-140M | 15 | 9 | 3 | 576 | 2048 | 140M | +| MobileLLM-R1-360M | 15 | 16 | 4 | 1024 | 4096 | 359M | +| MobileLLM-R1-950M | 22 | 24 | 6 | 1536 | 6144 | 949M | + +| | Input modalities | Output modalities | Context Length | Vocaburary Size | Shared Embeddings | +| --- | --- | --- | --- | --- | --- | +| [MobileLLM-R1-140M-base](https://huggingface.co/facebook/MobileLLM-R1-140M-base) | Text | Text | 4k | 128k | Yes | +| [MobileLLM-R1-360M-base](https://huggingface.co/facebook/MobileLLM-R1-360M-base) | Text | Text | 4k | 128k | Yes | +| [MobileLLM-R1-950M-base](https://huggingface.co/facebook/MobileLLM-R1-950M-base) | Text | Text | 4k | 128k | Yes | +| [MobileLLM-R1-140M](https://huggingface.co/facebook/MobileLLM-R1-140M) | Text | Text | 32k | 128k | Yes | +| [MobileLLM-R1-360M](https://huggingface.co/facebook/MobileLLM-R1-360M) | Text | Text | 32k | 128k | Yes | +| [MobileLLM-R1-950M](https://huggingface.co/facebook/MobileLLM-R1-950M) | Text | Text | 32k | 128k | Yes | + +# How to use + +To load the pretrained model for further finetuning or evaluation: +```bash +from transformers import AutoModelForCausalLM, AutoTokenizer +tokenizer = AutoTokenizer.from_pretrained("facebook/MobileLLM-R1-950M") +model = AutoModelForCausalLM.from_pretrained("facebook/MobileLLM-R1-950M") +``` + +# Inference examples + +Transformers + +```py +from transformers import pipeline +import torch + +model_id = "facebook/MobileLLM-R1-950M" + +pipe = pipeline( + "text-generation", + model=model_id, + torch_dtype="auto", + device_map="auto", +) + +# Math problem / default scenario +messages = [ + { + "role": "system", + "content": "Please reason step by step, and put your final answer within \\boxed{}." + }, + {"role": "user", "content": "Compute: $1-2+3-4+5- \\dots +99-100$."}, +] + +# C++ coding scenario +messages = [ + { + "role": "system", + "content": ( + "\nYou are a helpful and harmless assistant. You should think step-by-step before responding to the instruction below.\n\n" + "Please use c++ programming language only.\n" + "You must use ```cpp for just the final solution code block with the following format:\n" + "```cpp\n# Your code here\n```\n" + ) + }, + {"role": "user", "content": "Write a C++ program that prints 'Hello, World!'."}, +] + +# Python coding scenario +messages = [ + { + "role": "system", + "content": ( + "\nYou are a helpful and harmless assistant. You should think step-by-step before responding to the instruction below.\n\n" + "Please use python programming language only.\n" + "You must use ```python for just the final solution code block with the following format:\n" + "```python\n# Your code here\n```\n" + ) + }, + {"role": "user", "content": "Write a Python function that returns the square of a number."}, +] + +outputs = pipe( + messages, + max_new_tokens=8192, +) +print(outputs[0]["generated_text"][-1]) +``` + +You can also run inference with vLLM. You only need to register the model architecture Llama4ForCausalLM with the vLLM ModelRegistry. +```bash +from vllm.model_executor.models.llama4 import Llama4ForCausalLM +from vllm.model_executor.models.registry import ModelRegistry +ModelRegistry.register_model("Llama4ForCausalLM", Llama4ForCausalLM) +``` + + +# Evaluation + +## MobileLLM-R1 base model +| Model | Size | MATH500 | GSM8K | MBPP | HumanEval | CommonSense Avg. | MMLU | +| --- | --- | --- | --- | --- | --- | --- | --- | +| | | 4-shot
em | 8-shot
em | 3-shot
pass@1 | 0-shot
pass@1 | 0-shot
accuracy | 5-shot
accuracy | +| | +| *<150M* | | | | | | | | +| SmolLM2-135M-base | 135M | 0.4 | 1.8 | 3.8 | 0.0 | **50.7** | -- | +| **MobileLLM-R1-140M-base** | 140M | **4.6** | **16.3** | **5.4** | **15.9** | 44.3 | -- | +| | +| *150M - 400M* | | | | | | | | +| Gemma-3-270M-pt | 268M | 0.6 | 1.1 | 2.0 | 3.1 | 48.4 | 26.5 | +| SmolLM2-360M-base | 362M | 1.8 | 5.0 | **19.4** | 0.0 | **56.6** | 24.7 | +| **MobileLLM-R1-360M-base** | 359M | **13.4** | **39.4** | **20.8** | **32.9** | 51.0 | **26.8** | +| | +| *400M - 1B* | | | | | | | | +| Qwen2.5-0.5B-base | 494M | 14.8 | 41.8 | 29.6 | 28.1 | 52.3 | 47.5 | +| Qwen3-0.6B-base | 596M | **29.8** | 60.9 | **39.0** | 30.5 | 55.3 | **52.4** | +| **MobileLLM-R1-950M-base** | 949M | 26.8 | **61.6** | **39.2** | **46.3** | **58.6** | 47.4 | +| | +| *> 1B* | | | | | | | | +| Gemma-3-1B-pt | 1.0B | 0.6 | 2.4 | 9.4 | 6.1 | 57.3 | 26.1 | +| LLaMA3.2-1B-base | 1.24B | 1.6 | 6.8 | 26.6 | 17.1 | 58.4 | 32.0 | +| OLMo-2-0425-1B-base | 1.48B | 5.2 | 39.8 | 7.8 | 6.7 | 61.0 | 42.4 | +| Qwen2.5-1.5B-base | 1.54B | 31.0 | 68.4 | 44.6 | 36.6 | 58.7 | 61.2 | +| SmolLM2-1.7B-base | 1.71B | 11.6 | 31.8 | 35.4 | 0.6 | 62.9 | 50.0 | +| Qwen3-1.7B-base | 2.03B | 38.5 | 76.2 | 56.4 | 47.6 | 60.9 | 62.1 | + + +Here, CommonSense Avg. denotes an average of 8 tasks in CommonSense Reasoning benchmarks including ARC-easy, ARC-challenge, BoolQ, PIQA, SIQA, HellaSwag, OBQA, and WinoGrand. Models with fewer than 150M parameters do not yield reliable MMLU scores and are therefore denoted as '—'. + +## MobileLLM-R1 post-trained model + + | Model | Size | MATH500 | GSM8K | AIME'24 | AIME'25 | LiveCodeBench-v6 | + | --- | --- | --- | --- | --- | --- | --- | + | | | 0-shot
pass@1 | 0-shot
pass@1 | 0-shot
pass@1, n=64 | 0-shot
pass@1, n=64 | 0-shot
pass@1, n=16 | + | | + | *<150M* | | | | | | | + | SmolLM2-135M-Instruct | 135M | 3.0 | 2.4 | -- | -- | 0.0 | + | **MobileLLM-R1-140M** | 140M | **6.2** | **4.1** | -- | -- | **1.7** | + | | + | *150M - 400M* | | | | | | | + | Gemma-3-270m-it | 268M | 6.8 | 8.4 | -- | -- | 0.0 | + | SmolLM2-360M-Instruct | 362M | 3.4 | 8.1 | -- | -- | 0.7 | + | **MobileLLM-R1-360M** | 359M | **28.4** | **24.5** | -- | -- | **5.1** | + | | + | *400M - 1B* | | | | | | | + | Qwen2.5-0.5B-Instruct | 494M | 31.2 | 48.1 | 0.1 | 0.3 | 3.6 | + | Qwen3-0.6B | 596M | 73.0 | **79.2** | 11.3 | **17.0** | 14.9 | + | **MobileLLM-R1-950M** | 949M | **74.0** | 67.5 | **15.5** | 16.3 | **19.9** | + | | + | *> 1B* | | | | | | | + | Gemma-3-1B-it | 1.0B | 45.4 | 62.9 | 0.9 | 0.0 | 2.0 | + | LLaMA3.2-1B-Instruct | 1.24B | 24.8 | 38.8 | 1.1 | 0.2 | 4.1 | + | OLMo-2-0425-1B-Instruct | 1.48B | 19.2 | 69.7 | 0.6 | 0.1 | 0.0 | + | OpenReasoning-Nemotron-1.5B | 1.54B | 83.4 | 76.7 | 49.7 | 40.4 | 28.3 | + | DeepSeek-R1-Distill-Qwen-1.5B | 1.54B | 83.2 | 77.3 | 29.1 | 23.4 | 19.9 | + | Qwen2.5-1.5B-Instruct | 1.54B | 54.0 | 70.0 | 2.5 | 0.9 | 7.9 | + | SmolLM2-1.7B-Instruct | 1.71B | 19.2 | 41.8 | 0.3 | 0.1 | 4.4 | + | Qwen3-1.7B | 2.03B | 89.4 | 90.3 | 47.0 | 37.0 | 29.8 | + +For AIME, we evaluate models across 64 runs and report the average accuracy. For LiveCodeBench, results are reported as the average accuracy across 16 runs. Models with fewer than 400M parameters do not produce reliable AIME scores and are therefore denoted as '—'. + + +# Training + +## Training Process +![image/jpeg](https://cdn-uploads.huggingface.co/production/uploads/660f893bae89429c07a32cdb/ThVFzsaaGa4gQ3iha5CKM.jpeg) + +### Training stages and hyperparameter details + +In the pretraining phase, MobileLLM-R1 models are randomly initialized and optimized using the Adam optimizer with hyperparameters (β_1, β_2, ε) = (0.9, 0.95, 1e-8), coupled with a weight decay coefficient of 0.1. The learning rate follows a 2k-step warmup schedule and then decays linearly from its peak to 10\% of the maximum. + +In the mid-training phase, we use Adam optimizer with learning rate linearly decays from its maximum value to zero. We employ knowledge distillation with Llama-3.1-8B-Instruct model as the teacher, where the student is trained via minimizing the KL divergence between its output logits and the teacher logits. + +In the post-training phase, we use the Adam optimizer with zero weight decay. The learning rate warmup ratio is set to 0.03 for general-purpose SFT and 0.1 for reasoning-specific SFT, and it linearly decays from its maximum value to zero. Full training hyperparameters are provided in the table below. + +| Stage | Phase | Tokens / Samples | BS | Sequence Length | Steps | LR | #GPUs | Training Time | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Pre-training | Phase1 | 2T tokens | 16 | 2k | 500k | 4.00E-03 | 16 x 8 | 4-5 days | +| | Phase2 | 2T tokens | 16 | 2k | 500k | 4.00E-03 | 16 x 8 | 4-5 days | +| Mid-training | Phase1 | 100B tokens | 4 | 4k | 50K | 3.60E-04 | 16 x 8 | 1-2 days | +| | Phase2 | 100B tokens | 4 | 4k | 50K | 3.60E-04 | 16 x 8 | 1-2 days | +| Post-training | General SFT | 866K samples | 4 | 4k | 2 epochs | 5.00E-06 | 16 x 8 | ~2h | +| | Reasoning SFT | 6.2M samples | 8 | 32k | 4 epochs | 8.00E-05 | 16 x 8 | ~2.5days | + + +## Data Mix + +### Pre-training + +| Dataset | Rows | Tokens (B) | Phase1 Mix Ratio | Phase2 Mix Ratio | +| --- | --- | --- | --- | --- | +| [StarCoder](https://huggingface.co/datasets/bigcode/starcoderdata) | 206,640,114 | 263.8 | 12.33% | 0.52% | +| [OpenWebMath](https://huggingface.co/datasets/open-web-math/open-web-math) | 6,117,786 | 12.6 | 5.37% | 23.33% | +| [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) | 1,279,107,432 | 1300 | 62.68% | 54.83% | +| [Wiki](https://huggingface.co/datasets/allenai/dolmino-mix-1124/tree/main/data/wiki) | 7,222,303 | 3.7 | 2.40% | 0.14% | +| [Arxiv](https://huggingface.co/datasets/togethercomputer/RedPajama-Data-1T/blob/main/urls/arxiv.txt) | 1,533,917 | 28 | 6.21% | 1.32% | +| [StackExchange](https://data.together.xyz/redpajama-data-1T/v1.0.0/stackexchange/stackexchange.jsonl) | 29,249,120 | 19.6 | 7.60% | 0.86% | +| [Algebraic stack](https://huggingface.co/datasets/EleutherAI/proof-pile-2/tree/main/algebraic-stack) | 3,404,331 | 12.6 | 3.40% | 1.26% | +| [Nemotron science](https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset/blob/main/SFT/science/science.jsonl) | 708,920 | 2 | -- | 0.03% | +| [Nemotron code](https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset/blob/main/SFT/code/code_v1.1.jsonl) | 10,108,883 | 16 | -- | 0.72% | +| [Nemotron math](https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset/blob/main/SFT/math/math_v1.1.jsonl) | 22,066,397 | 15 | -- | 3.01% | +| [Cosmopedia](https://huggingface.co/datasets/HuggingFaceTB/cosmopedia) | 31,064,744 | 25 | -- | 2.70% | +| [Facebook natural reasoning](https://huggingface.co/datasets/facebook/natural_reasoning) | 1,145,824 | 1.8 | -- | 3.18% | +| [FineMath](https://huggingface.co/datasets/HuggingFaceTB/finemath/tree/main/finemath-3plus) | 48,283,984 | 34 | -- | 8.01% | +| [peS2o](https://huggingface.co/datasets/allenai/peS2o) | 38,800,000 | 50 | -- | 0.08% | +| **Total** | | | 100% | 100% | + + + + +### Mid-training + + + | Dataset | Subset | Rows (M) | Phase1 Mix Ratio | Phase2 Mix Ratio | + | --- | --- | --- | --- | --- | + | [Dolmino](https://huggingface.co/datasets/allenai/dolmino-mix-1124) | DCLM Baseline | 606 | 37.03% | 6.51% | + | | FLAN | 57.3 | 4.10% | 0.72% | + | | peS2o | 38.8 | 11.41% | 2.01% | + | | Wiki | 6.17 | 2.66% | 0.47% | + | | StackExchange | 2.48 | 2.12% | 2.00% | + | | Math | 21 | 11.63% | 29.10% | + | Nemotron | [Nemotron-Pretraining-Code-v1](https://huggingface.co/datasets/nvidia/Nemotron-Pretraining-Code-v1) | 882 | 20.69% | 29.10% | + | | [Nemotron-CC-Math-v1](https://huggingface.co/datasets/nvidia/Nemotron-CC-Math-v1) | 144 | 3.45% | 19.40% | + | StarCoder | [StarCoder](https://huggingface.co/datasets/bigcode/starcoderdata) | 206 | 6.90% | 9.70% | + | Benchmark training set | [TriviaQA (train)](https://huggingface.co/datasets/mandarjoshi/trivia_qa/tree/main/rc)
[OBQA (train)](https://huggingface.co/datasets/allenai/openbookqa/blob/main/main/train-00000-of-00001.parquet)
[NaturalQuestions (train)](https://github.com/google-research-datasets/natural-questions/blob/master/nq_open/NQ-open.train.jsonl)
[PIQA (train)](https://github.com/ybisk/ybisk.github.io/blob/master/piqa/data/train.jsonl)
[GSM8K (train)](https://huggingface.co/datasets/openai/gsm8k/blob/main/main/train-00000-of-00001.parquet)
[BoolQ (train)](https://huggingface.co/datasets/google/boolq/blob/main/data/train-00000-of-00001.parquet)
[ARC-Easy (train)](https://huggingface.co/datasets/allenai/ai2_arc/blob/main/ARC-Easy/train-00000-of-00001.parquet)
[ARC-Challenge (train)](https://huggingface.co/datasets/allenai/ai2_arc/blob/main/ARC-Challenge/train-00000-of-00001.parquet) | ~0.01 | 0 | 0.97% | + | Total | | | 100.00% | 100.00% | + +### Post-training + | Phase | Dataset | Rows | + | --- | --- | --- | + | General SFT | [Tulu-3-sft-olmo-2-mixture-0225](https://huggingface.co/datasets/allenai/tulu-3-sft-olmo-2-mixture-0225) | 866K samples | + | Reasoning SFT | [OpenMathReasoning](https://huggingface.co/datasets/nvidia/OpenMathReasoning) | 3.2M samples | + | | [OpenScienceReasoning-2](https://huggingface.co/datasets/nvidia/OpenScienceReasoning-2) | 803K samples | + | | [OpenCodeReasoning-2](https://huggingface.co/datasets/nvidia/OpenCodeReasoning-2) | 2.16M samples | + +## Data Mix + +### Pre-training + +| Dataset | Rows | Tokens (B) | Phase1 Mix Ratio | Phase2 Mix Ratio | +| --- | --- | --- | --- | --- | +| [StarCoder](https://huggingface.co/datasets/bigcode/starcoderdata) | 206,640,114 | 263.8 | 12.33% | 0.52% | +| [OpenWebMath](https://huggingface.co/datasets/open-web-math/open-web-math) | 6,117,786 | 12.6 | 5.37% | 23.33% | +| [FineWeb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) | 1,279,107,432 | 1300 | 62.68% | 54.83% | +| [Wiki](https://huggingface.co/datasets/allenai/dolmino-mix-1124/tree/main/data/wiki) | 7,222,303 | 3.7 | 2.40% | 0.14% | +| [Arxiv](https://huggingface.co/datasets/togethercomputer/RedPajama-Data-1T/blob/main/urls/arxiv.txt) | 1,533,917 | 28 | 6.21% | 1.32% | +| [StackExchange](https://data.together.xyz/redpajama-data-1T/v1.0.0/stackexchange/stackexchange.jsonl) | 29,249,120 | 19.6 | 7.60% | 0.86% | +| [Algebraic stack](https://huggingface.co/datasets/EleutherAI/proof-pile-2/tree/main/algebraic-stack) | 3,404,331 | 12.6 | 3.40% | 1.26% | +| [Nemotron science](https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset/blob/main/SFT/science/science.jsonl) | 708,920 | 2 | -- | 0.03% | +| [Nemotron code](https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset/blob/main/SFT/code/code_v1.1.jsonl) | 10,108,883 | 16 | -- | 0.72% | +| [Nemotron math](https://huggingface.co/datasets/nvidia/Llama-Nemotron-Post-Training-Dataset/blob/main/SFT/math/math_v1.1.jsonl) | 22,066,397 | 15 | -- | 3.01% | +| [Cosmopedia](https://huggingface.co/datasets/HuggingFaceTB/cosmopedia) | 31,064,744 | 25 | -- | 2.70% | +| [Facebook natural reasoning](https://huggingface.co/datasets/facebook/natural_reasoning) | 1,145,824 | 1.8 | -- | 3.18% | +| [FineMath](https://huggingface.co/datasets/HuggingFaceTB/finemath/tree/main/finemath-3plus) | 48,283,984 | 34 | -- | 8.01% | +| [peS2o](https://huggingface.co/datasets/allenai/peS2o) | 38,800,000 | 50 | -- | 0.08% | +| **Total** | | | 100% | 100% | + +### Mid-training + + | Dataset | Subset | Rows (M) | Phase1 Mix Ratio | Phase2 Mix Ratio | + | --- | --- | --- | --- | --- | + | [Dolmino](https://huggingface.co/datasets/allenai/dolmino-mix-1124) | DCLM Baseline | 606 | 37.03% | 6.51% | + | | FLAN | 57.3 | 4.10% | 0.72% | + | | peS2o | 38.8 | 11.41% | 2.01% | + | | Wiki | 6.17 | 2.66% | 0.47% | + | | StackExchange | 2.48 | 2.12% | 2.00% | + | | Math | 21 | 11.63% | 29.10% | + | Nemotron | [Nemotron-Pretraining-Code-v1](https://huggingface.co/datasets/nvidia/Nemotron-Pretraining-Code-v1) | 882 | 20.69% | 29.10% | + | | [Nemotron-CC-Math-v1](https://huggingface.co/datasets/nvidia/Nemotron-CC-Math-v1) | 144 | 3.45% | 19.40% | + | StarCoder | [StarCoder](https://huggingface.co/datasets/bigcode/starcoderdata) | 206 | 6.90% | 9.70% | + | Benchmark training set | [TriviaQA (train)](https://huggingface.co/datasets/mandarjoshi/trivia_qa/tree/main/rc)
[OBQA (train)](https://huggingface.co/datasets/allenai/openbookqa/blob/main/main/train-00000-of-00001.parquet)
[NaturalQuestions (train)](https://github.com/google-research-datasets/natural-questions/blob/master/nq_open/NQ-open.train.jsonl)
[PIQA (train)](https://github.com/ybisk/ybisk.github.io/blob/master/piqa/data/train.jsonl)
[GSM8K (train)](https://huggingface.co/datasets/openai/gsm8k/blob/main/main/train-00000-of-00001.parquet)
[BoolQ (train)](https://huggingface.co/datasets/google/boolq/blob/main/data/train-00000-of-00001.parquet)
[ARC-Easy (train)](https://huggingface.co/datasets/allenai/ai2_arc/blob/main/ARC-Easy/train-00000-of-00001.parquet)
[ARC-Challenge (train)](https://huggingface.co/datasets/allenai/ai2_arc/blob/main/ARC-Challenge/train-00000-of-00001.parquet) | ~0.01 | 0 | 0.97% | + | Total | | | 100.00% | 100.00% | + +### Post-training + | Phase | Dataset | Rows | + | --- | --- | --- | + | General SFT | [Tulu-3-sft-olmo-2-mixture-0225](https://huggingface.co/datasets/allenai/tulu-3-sft-olmo-2-mixture-0225) | 866K samples | + | Reasoning SFT | [OpenMathReasoning](https://huggingface.co/datasets/nvidia/OpenMathReasoning) | 3.2M samples | + | | [OpenScienceReasoning-2](https://huggingface.co/datasets/nvidia/OpenScienceReasoning-2) | 803K samples | + | | [OpenCodeReasoning-2](https://huggingface.co/datasets/nvidia/OpenCodeReasoning-2) | 2.16M samples | + +# Citation + +If you find our model useful for your research, please consider citing: + + @article{zhao2025mobilellm-r1, + title={MobileLLM-R1: Exploring the Limits of Sub-Billion Language Model Reasoners with Open Training Recipes}, + author={Zhao, Changsheng and Chang, Ernie and Liu, Zechun and Chang, Chia-Jung and Wen, Wei and Lai, Chen and Cao, Sheng, and Tian, Yuandong and Krishnamoorthi, Raghuraman and Shi, Yangyang and Chandra, Vikas}, + journal={arXiv preprint arXiv:2509.24945}, + year={2025} + } + +# Contact +Changsheng Zhao, Meta Inc (cszhao at meta dot com) + +Ernie Chang, Meta Inc (erniecyc at meta dot com) + +Zechun Liu, Meta Inc (zechunliu at meta dot com) + +# License + +MobileLLM-R1 is FAIR NC licensed as of now diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/chat_template.jinja b/aiak_training_llm/models/mobilellm/hf_checkpoint/chat_template.jinja new file mode 100644 index 00000000..626d4453 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/chat_template.jinja @@ -0,0 +1,91 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "Please reason step by step, and put your final answer within \\boxed{}." %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/config.json b/aiak_training_llm/models/mobilellm/hf_checkpoint/config.json new file mode 100644 index 00000000..b843e8f1 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/config.json @@ -0,0 +1,79 @@ +{ + "architectures": [ + "Llama4ForCausalLM" + ], + "attention_bias": false, + "attention_chunk_size": 32768, + "attention_dropout": 0.0, + "attn_scale": 0.1, + "attn_temperature_tuning": false, + "bos_token_id": 128000, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "floor_scale": 8192, + "for_llm_compressor": false, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 576, + "initializer_range": 0.02, + "interleave_moe_layer_step": 0, + "intermediate_size": 8192, + "intermediate_size_mlp": 2048, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "model_type": "llama4_text", + "moe_layers": [], + "no_rope_layers": [ + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "num_attention_heads": 9, + "num_experts_per_tok": 0, + "num_hidden_layers": 15, + "num_key_value_heads": 3, + "num_local_experts": 0, + "output_router_logits": false, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 8000000.0, + "router_aux_loss_coef": 0.001, + "router_jitter_noise": 0.0, + "tie_word_embeddings": true, + "torch_dtype": "float32", + "transformers_version": "4.55.0", + "use_cache": true, + "use_qk_norm": true, + "vocab_size": 128256 +} diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/generation_config.json b/aiak_training_llm/models/mobilellm/hf_checkpoint/generation_config.json new file mode 100644 index 00000000..6ce6220e --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "cache_implementation": "dynamic", + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "transformers_version": "4.55.0" +} diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/model.safetensors b/aiak_training_llm/models/mobilellm/hf_checkpoint/model.safetensors new file mode 100644 index 00000000..873e833b --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b5236e401a4c13800a32406fcc38e4ae3ca76e75856838d05cb5db9a364ffc9 +size 561009704 diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/special_tokens_map.json b/aiak_training_llm/models/mobilellm/hf_checkpoint/special_tokens_map.json new file mode 100644 index 00000000..02ee80b6 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/special_tokens_map.json @@ -0,0 +1,16 @@ +{ + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer.json b/aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer.json new file mode 100644 index 00000000..c3f018bd --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8462b5b8e7d9c25fb5581c3e4560717862d728887b4bb7cdf54f7cea3e44547e +size 17210103 diff --git a/aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer_config.json b/aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer_config.json new file mode 100644 index 00000000..a1e89513 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/hf_checkpoint/tokenizer_config.json @@ -0,0 +1,2065 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 32768, + "padding_side": "right", + "tokenizer_class": "PreTrainedTokenizerFast" +} diff --git a/aiak_training_llm/models/mobilellm/megatron_checkpoint/latest_checkpointed_iteration.txt b/aiak_training_llm/models/mobilellm/megatron_checkpoint/latest_checkpointed_iteration.txt new file mode 100644 index 00000000..30b81dbf --- /dev/null +++ b/aiak_training_llm/models/mobilellm/megatron_checkpoint/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +release \ No newline at end of file diff --git a/aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_00/model_optim_rng.pt b/aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_00/model_optim_rng.pt new file mode 100644 index 00000000..2323f7a4 --- /dev/null +++ b/aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_00/model_optim_rng.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a7eedc547baf39ba6ef5ece0252644805c62d7fa5eb4c012c53886f5e79574f +size 280575327 diff --git a/aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_01/model_optim_rng.pt b/aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_01/model_optim_rng.pt new file mode 100644 index 00000000..f0a4e2aa --- /dev/null +++ b/aiak_training_llm/models/mobilellm/megatron_checkpoint/release/mp_rank_01/model_optim_rng.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27710a67afe58ef0af22162535dca2907c9e9f552c838a6b10bee85b6f65759 +size 280575327 From 055ae655233b852090890120dbb0f736bf6d7573 Mon Sep 17 00:00:00 2001 From: Rena Zayed Date: Fri, 16 Jan 2026 15:35:17 +0400 Subject: [PATCH 16/39] Add MobileLLM-140M integration with LLaVA-OneVision - Add MobileLLM config and layer specs - Update model provider and layer specs to support MobileLLM backbone - Add stage 1 alignment script for MobileLLM-140M - Add comprehensive integration documentation - Update alignment.sh to support MobileLLM training option --- Stage1/alignment.sh | 10 +- .../models/llavaov_1_5/__init__.py | 14 +- .../llavaov_1_5/llavaov_1_5_layer_spec.py | 50 ++++ .../models/llavaov_1_5/llavaov_1_5_model.py | 68 ++--- .../llavaov_1_5/llavaov_1_5_provider.py | 27 +- .../llavaov_1_5/llavaov_mobilellm_config.py | 165 ++++++++++++ docs/MOBILELLM_INTEGRATION.md | 253 ++++++++++++++++++ .../stage_1_alignment_mobilellm_140m.sh | 217 +++++++++++++++ 8 files changed, 765 insertions(+), 39 deletions(-) create mode 100644 aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py create mode 100644 docs/MOBILELLM_INTEGRATION.md create mode 100644 examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index 1b126ff4..66b7b366 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -54,12 +54,16 @@ export LD_LIBRARY_PATH="/usr/local/cuda-12.1/lib64:/usr/local/cuda-12.1/targets/ # Weights & Biases configuration export WANDB_API_KEY="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" export WANDB_PROJECT="llava-ov-1_5" -export WANDB_NAME="fastvit_integration" +export WANDB_NAME="mobilellm_integration" export CUDA_VISIBLE_DEVICES=4 export GPUS_PER_NODE=1 export MASTER_PORT=26000 -# bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +# Choose which backbone to use for training: +# ============================================================ +# Option 1: MobileLLM-R1-140M (140M params, efficient for mobile/edge) +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh \ No newline at end of file +# Option 2: Original Qwen2.5-4B backbone (4B params) +# bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/__init__.py b/aiak_training_llm/models/llavaov_1_5/__init__.py index 521aff6d..4300efc7 100644 --- a/aiak_training_llm/models/llavaov_1_5/__init__.py +++ b/aiak_training_llm/models/llavaov_1_5/__init__.py @@ -1,3 +1,13 @@ -"""qwen model""" +"""LLaVA-OneVision-1.5 model""" -from .llavaov_1_5_model import LlavaOnevision1_5 \ No newline at end of file +from .llavaov_1_5_model import LlavaOnevision1_5 +from .llavaov_mobilellm_config import ( + get_llava_ov_mobilellm_140m_config, + get_llava_ov_mobilellm_140m_fastvit_config, +) + +__all__ = [ + "LlavaOnevision1_5", + "get_llava_ov_mobilellm_140m_config", + "get_llava_ov_mobilellm_140m_fastvit_config", +] \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py index 591797a1..89acd916 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py @@ -140,6 +140,56 @@ def get_qwen_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: ) ) + +def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: + """ + Get MobileLLM layer specification using Transformer Engine modules. + + MobileLLM-R1-140M uses standard LLaMA-style architecture: + - Grouped Query Attention (9 heads, 3 KV heads) + - SwiGLU activation (gated_linear_unit=True) + - RMSNorm + - No QK LayerNorm + + Args: + config: Transformer configuration with MobileLLM parameters + + Returns: + ModuleSpec for MobileLLM transformer layer + """ + # Standard dense MLP with SwiGLU (no MoE for MobileLLM) + mlp = ModuleSpec( + module=MLP, + submodules=MLPSubmodules( + linear_fc1=multiacc_modules.TELayerNormColumnParallelLinear, + linear_fc2=multiacc_modules.TERowParallelLinear, + bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, + ), + ) + + return ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + input_layernorm=IdentityOp, + self_attention=ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=multiacc_modules.TELayerNormColumnParallelLinear, + core_attention=multiacc_modules.DotProductAttention, + linear_proj=multiacc_modules.TERowParallelLinear, + q_layernorm=IdentityOp, # MobileLLM doesn't use QK LayerNorm + k_layernorm=IdentityOp, + apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, + ), + ), + self_attn_bda=multiacc_modules.get_bias_dropout_add, + pre_mlp_layernorm=IdentityOp, + mlp=mlp, + mlp_bda=multiacc_modules.get_bias_dropout_add, + ), + ) + # def get_qwen_layer_with_spec(qk_layernorm: bool = False) -> ModuleSpec: # """ # Use this spec for an implementation using transformer, local or multi-accel engine diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index ee202c23..bb1fd958 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -205,38 +205,44 @@ def __init__( # # This attribute is needed to check if an all-reduce is required # # on the word embeddings inside `finalize_model_grads._allreduce_word_embedding_grads`. if self.add_decoder: - - # Use MobileLLM for smaller, efficient model - print("[INFO] Using MobileLLM-140M as language backbone") - self.language_model = MobileLLMModel( - config=language_config, - transformer_layer_spec=language_layer_spec, - vocab_size=language_vocab_size, - max_sequence_length=language_max_sequence_length, - parallel_output=parallel_output, - position_embedding_type=language_position_embedding_type, - rotary_percent=language_rotary_percent, - pre_process=self.pre_process, - post_process=self.post_process, - rotary_base=language_rotary_base, - share_embeddings_and_output_weights=share_embeddings_and_output_weights, + # Dynamically select language model based on configuration + # MobileLLM: 15 layers, 576 hidden size + # Qwen2.5: 32+ layers, 3584+ hidden size + is_mobilellm = ( + language_config.num_layers <= 20 and + language_config.hidden_size <= 1024 ) - - # # Use Qwen model (default for larger models) - # print("[INFO] Using Qwen model as language backbone") - # self.language_model = QwenModel( - # config=language_config, - # transformer_layer_spec=language_layer_spec, - # vocab_size=language_vocab_size, - # max_sequence_length=language_max_sequence_length, - # parallel_output=parallel_output, - # position_embedding_type=language_position_embedding_type, - # rotary_percent=language_rotary_percent, - # pre_process=self.pre_process, - # post_process=self.post_process, - # rotary_base=language_rotary_base, - # share_embeddings_and_output_weights=share_embeddings_and_output_weights, - # ) + + if is_mobilellm: + print("[INFO] Using MobileLLM as language backbone") + self.language_model = MobileLLMModel( + config=language_config, + transformer_layer_spec=language_layer_spec, + vocab_size=language_vocab_size, + max_sequence_length=language_max_sequence_length, + parallel_output=parallel_output, + position_embedding_type=language_position_embedding_type, + rotary_percent=language_rotary_percent, + pre_process=self.pre_process, + post_process=self.post_process, + rotary_base=language_rotary_base, + share_embeddings_and_output_weights=share_embeddings_and_output_weights, + ) + else: + print("[INFO] Using Qwen model as language backbone") + self.language_model = QwenModel( + config=language_config, + transformer_layer_spec=language_layer_spec, + vocab_size=language_vocab_size, + max_sequence_length=language_max_sequence_length, + parallel_output=parallel_output, + position_embedding_type=language_position_embedding_type, + rotary_percent=language_rotary_percent, + pre_process=self.pre_process, + post_process=self.post_process, + rotary_base=language_rotary_base, + share_embeddings_and_output_weights=share_embeddings_and_output_weights, + ) self.share_embeddings_and_output_weights = ( self.language_model.share_embeddings_and_output_weights ) diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index 4800b3b4..65708d0d 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -1,4 +1,4 @@ -"""qwen model provider""" +"""LLaVA-OneVision-1.5 model provider - supports Qwen and MobileLLM backbones""" from copy import deepcopy from dataclasses import asdict @@ -6,9 +6,11 @@ from aiak_training_llm.models.factory import register_model_provider from aiak_training_llm.models.llavaov_1_5.llavaov_1_5_layer_spec import ( get_adapeter_layer_with_spec, get_qwen_layer_with_te_spec, - get_vision_layer_with_spec) + get_mobilellm_layer_with_te_spec, get_vision_layer_with_spec) from aiak_training_llm.models.llavaov_1_5.llavaov_1_5_config import ( get_adapeter_config, get_vision_config) +from aiak_training_llm.models.llavaov_1_5.llavaov_mobilellm_config import ( + get_language_config as get_mobilellm_language_config) from aiak_training_llm.utils import (build_transformer_config, get_args, print_rank_0) from aiak_training_llm.utils.constants import VisionLanguageModelFamilies @@ -52,6 +54,18 @@ def rice_vl_model_provider( from aiak_training_llm.models import get_model_family model_family = get_model_family(args.model_name) + + # Detect if using MobileLLM backbone + use_mobilellm = "mobilellm" in args.model_name.lower() + + if use_mobilellm: + print_rank_0(f'Using MobileLLM-R1-140M as language backbone') + # Load MobileLLM configuration + mobilellm_config = get_mobilellm_language_config(args.model_name) + for k, v in asdict(mobilellm_config).items(): + setattr(language_config, k, v) + print_rank_0(f'MobileLLM language config: {language_config}') + # get vision specific config : no. of layers, hidden size, Patch size, Image resolution for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): setattr(vision_config, k, v) @@ -110,7 +124,14 @@ def rice_vl_model_provider( else: adapter_layer_spec = get_adapeter_layer_with_spec() vision_layer_spec = get_vision_layer_with_spec() - language_layer_spec = get_qwen_layer_with_te_spec(language_config) + + # Choose language layer spec based on backbone + if use_mobilellm: + print_rank_0("Using MobileLLM layer specification") + language_layer_spec = get_mobilellm_layer_with_te_spec(language_config) + else: + print_rank_0("Using Qwen layer specification") + language_layer_spec = get_qwen_layer_with_te_spec(language_config) # # Vision layer spec (Transformer block) # - MultiheadAttention diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py b/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py new file mode 100644 index 00000000..d8e060c1 --- /dev/null +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py @@ -0,0 +1,165 @@ +""" +LLaVA-OneVision-1.5 with MobileLLM backbone configuration + +This module configures LLaVA-OneVision-1.5 to use MobileLLM-R1-140M as the language model +instead of Qwen. It maintains the vision encoder (SigLIP or FastViT) and adapter layers +while replacing the language model architecture. +""" + +from dataclasses import dataclass +from aiak_training_llm.models.factory import register_model_config +from aiak_training_llm.utils.constants import VisionLanguageModelFamilies + + +@dataclass +class MobileLLMLanguageConfig: + """Language model configuration for MobileLLM-R1-140M""" + # Architecture from facebook/MobileLLM-R1-140M + num_layers: int = 15 + hidden_size: int = 576 + num_attention_heads: int = 9 + num_query_groups: int = 3 # GQA: 3 KV heads + ffn_hidden_size: int = 2048 + + # Vocabulary and sequence + vocab_size: int = 128256 + max_position_embeddings: int = 32768 + + # Normalization + normalization: str = "RMSNorm" + norm_epsilon: float = 1e-05 + layernorm_zero_centered_gamma: bool = False + + # Activation + add_bias_linear: bool = False + gated_linear_unit: bool = True + activation_func: str = "swiglu" + bias_activation_fusion: bool = True + + # Embeddings + untie_embeddings_and_output_weights: bool = False # True means tied (shared) + + # Position embeddings + position_embedding_type: str = "rope" + rotary_base: int = 8000000 + rotary_percent: float = 1.0 + rotary_interleaved: bool = False + + # Attention + attention_dropout: float = 0.0 + + # Initialization + init_method_std: float = 0.02 + apply_query_key_layer_scaling: bool = False + attention_softmax_in_fp32: bool = True + + +@dataclass +class VisionConfig: + """Vision encoder configuration (can use SigLIP or FastViT)""" + num_layers: int = 27 + hidden_size: int = 1152 + num_attention_heads: int = 16 + ffn_hidden_size: int = 4304 + patch_size: int = 14 + image_resolution: int = 384 + + +@dataclass +class AdapterConfig: + """Adapter/Projection layer configuration""" + adapter_dim: int = 2048 # Projection dimension + adapter_act: str = "gelu" # Activation function + + +@register_model_config( + model_family=VisionLanguageModelFamilies.LLAVA_OV_1_5, + model_arch="llava-ov-mobilellm-140m" +) +def get_llava_ov_mobilellm_140m_config(): + """ + Configuration for LLaVA-OneVision-1.5 with MobileLLM-R1-140M backbone + + Returns a dict with language, vision, and adapter configurations + """ + return { + "language": MobileLLMLanguageConfig(), + "vision": VisionConfig(), + "adapter": AdapterConfig(), + "model_type": "llava_ov_mobilellm" + } + + +@register_model_config( + model_family=VisionLanguageModelFamilies.LLAVA_OV_1_5, + model_arch="llava-ov-mobilellm-140m-fastvit" +) +def get_llava_ov_mobilellm_140m_fastvit_config(): + """ + Configuration for LLaVA-OneVision-1.5 with MobileLLM-R1-140M backbone and FastViT + + Uses FastViT (MobileCLIP) as the vision encoder for efficiency + """ + # FastViT configuration (MobileCLIP-L-384) + vision_config = VisionConfig( + num_layers=12, # FastViT is shallower + hidden_size=768, # FastViT hidden dimension + num_attention_heads=12, + ffn_hidden_size=3072, + patch_size=16, + image_resolution=384 + ) + + return { + "language": MobileLLMLanguageConfig(), + "vision": vision_config, + "adapter": AdapterConfig(), + "model_type": "llava_ov_mobilellm_fastvit" + } + + +def get_vision_config(model_family: str, model_name: str): + """ + Get vision encoder configuration based on model name + + Args: + model_family: Model family (llava_ov_1_5) + model_name: Specific model architecture name + + Returns: + VisionConfig dataclass instance + """ + config = get_llava_ov_mobilellm_140m_config() + + if "fastvit" in model_name.lower() or "mobilellm" in model_name.lower(): + config = get_llava_ov_mobilellm_140m_fastvit_config() + + return config["vision"] + + +def get_adapter_config(model_family: str): + """ + Get adapter/projection configuration + + Args: + model_family: Model family (llava_ov_1_5) + + Returns: + AdapterConfig dataclass instance + """ + config = get_llava_ov_mobilellm_140m_config() + return config["adapter"] + + +def get_language_config(model_name: str): + """ + Get language model configuration for MobileLLM + + Args: + model_name: Specific model architecture name + + Returns: + MobileLLMLanguageConfig dataclass instance + """ + config = get_llava_ov_mobilellm_140m_config() + return config["language"] diff --git a/docs/MOBILELLM_INTEGRATION.md b/docs/MOBILELLM_INTEGRATION.md new file mode 100644 index 00000000..cc0222df --- /dev/null +++ b/docs/MOBILELLM_INTEGRATION.md @@ -0,0 +1,253 @@ +# MobileLLM Integration for LLaVA-OneVision-1.5 + +This document describes the integration of Facebook's MobileLLM-R1-140M as the language backbone for LLaVA-OneVision-1.5, replacing the original Qwen2.5 model. + +## Overview + +**MobileLLM-R1-140M** is a compact 140M parameter language model from Facebook Research optimized for on-device deployment with strong reasoning capabilities. + +### Architecture Comparison + +| Component | Original (Qwen2.5-4B) | MobileLLM-R1-140M | +|-----------|----------------------|-------------------| +| Parameters | 4B | 140M (35x smaller) | +| Layers | 32 | 15 | +| Hidden Size | 3584 | 576 | +| Attention Heads | 28 | 9 | +| KV Heads (GQA) | 4 | 3 | +| FFN Size | 18944 | 2048 | +| Vocab Size | 151,936 | 128,256 | +| Context Length | 32,768 | 32,768 | +| RoPE Base | 1,000,000 | 8,000,000 | + +### Key Features +- **Grouped Query Attention (GQA)**: 9 query heads, 3 KV heads +- **SwiGLU Activation**: Gated linear unit for better performance +- **RMSNorm**: Fast layer normalization +- **Shared Embeddings**: Input/output embeddings are tied +- **RoPE**: Rotary position embeddings with base 8M + +## Files Modified/Created + +### Core Integration Files + +1. **aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py** (NEW) + - MobileLLM configuration for LLaVA-OneVision + - Registered model architectures: + - `llava-ov-mobilellm-140m` (with SigLIP) + - `llava-ov-mobilellm-140m-fastvit` (with FastViT) + +2. **aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py** (MODIFIED) + - Added `get_mobilellm_layer_with_te_spec()` function + - Configures MobileLLM-style transformer layers with: + - GQA attention + - SwiGLU MLP + - RMSNorm + - No QK LayerNorm + +3. **aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py** (MODIFIED) + - Auto-detects MobileLLM from model name + - Routes to appropriate language config and layer spec + - Supports both Qwen and MobileLLM backbones + +4. **aiak_training_llm/models/llavaov_1_5/__init__.py** (MODIFIED) + - Exports MobileLLM configuration functions + +### Training Script + +5. **examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh** (NEW) + - Stage 1 alignment training script + - Configured for MobileLLM-R1-140M + FastViT + - Trains only the adapter (vision + language frozen) + +### Existing MobileLLM Files (from your previous work) + +6. **aiak_training_llm/models/mobilellm/** (EXISTING) + - `mobilellm_config.py`: Standalone MobileLLM config + - `mobilellm_model.py`: MobileLLM model class + - `mobilellm_layer_spec.py`: Layer specifications + - `mobilellm_provider.py`: Model provider registration + + **Note**: These files are for standalone MobileLLM LM training. The LLaVA integration uses the `llavaov_1_5/llavaov_mobilellm_config.py` instead. + +## Configuration Details + +### Model Architecture (llava-ov-mobilellm-140m-fastvit) + +```python +Language Model (MobileLLM-R1-140M): +- num_layers: 15 +- hidden_size: 576 +- num_attention_heads: 9 +- num_query_groups: 3 (GQA) +- ffn_hidden_size: 2048 +- vocab_size: 128,256 +- max_position_embeddings: 32,768 +- rotary_base: 8,000,000 +- normalization: RMSNorm (epsilon=1e-05) +- activation: swiglu +- tied_embeddings: True + +Vision Encoder (FastViT/MobileCLIP-S-384): +- image_resolution: 384x384 +- patch_size: 16 +- hidden_size: 768 +- num_layers: 12 +- num_attention_heads: 12 + +Adapter (Projection): +- adapter_dim: 2048 +- activation: gelu +``` + +## Training Setup + +### Prerequisites + +1. **Tokenizer**: Use HuggingFace tokenizer from `facebook/MobileLLM-R1-140M` +2. **Data**: LLaVA-558K dataset in WebDataset format +3. **Checkpoints** (Optional): + - MobileLLM pretrained weights converted to Megatron format + - FastViT pretrained weights + +### Stage 1: Alignment Training + +Train only the projection adapter while keeping vision encoder and language model frozen. + +```bash +cd /path/to/LLaVA-OneVision-1.5 + +# Set environment variables +export WANDB_API_KEY="your_wandb_key" +export WANDB_PROJECT="llava-ov-mobilellm" +export WANDB_NAME="stage1-alignment" + +# Run training +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +``` + +#### Training Parameters + +- **Model**: `llava-ov-mobilellm-140m-fastvit` +- **Trainable**: Adapter only +- **Frozen**: Vision encoder + Language model +- **Sequence Length**: 512 tokens +- **Batch Size**: Global=1, Micro=1 +- **Learning Rate**: 1e-4 → 1e-6 (cosine) +- **Training Steps**: 2,500 (adjust as needed) +- **Precision**: BF16 + +### Running on Your Cluster + +Update these paths in the script: + +```bash +# Data directory with WebDataset shards +DATA_PATH="/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset" + +# MobileLLM tokenizer (HuggingFace) +TOKENIZER_PATH="facebook/MobileLLM-R1-140M" + +# (Optional) Checkpoint path if loading pretrained weights +# CHECKPOINT_PATH="/path/to/mobilellm_mcore_checkpoint" +``` + +## Model Registration + +The MobileLLM integration uses the factory pattern for model registration: + +```python +# In llavaov_mobilellm_config.py +@register_model_config( + model_family=VisionLanguageModelFamilies.LLAVA_OV_1_5, + model_arch="llava-ov-mobilellm-140m-fastvit" +) +``` + +When you specify `--model-name llava-ov-mobilellm-140m-fastvit`, the system automatically: +1. Routes to LLaVA-OneVision-1.5 model family +2. Loads MobileLLM language config +3. Configures FastViT vision encoder +4. Sets up appropriate layer specs + +## Key Differences from Qwen Integration + +### 1. Configuration +- **Qwen**: Uses `llavaov_1_5_config.py` → Qwen-specific configs +- **MobileLLM**: Uses `llavaov_mobilellm_config.py` → MobileLLM configs + +### 2. Layer Specification +- **Qwen**: `get_qwen_layer_with_te_spec()` → QK LayerNorm enabled +- **MobileLLM**: `get_mobilellm_layer_with_te_spec()` → No QK LayerNorm + +### 3. Tokenizer +- **Qwen**: `Qwen/Qwen2.5-4B-Instruct` (151,936 vocab) +- **MobileLLM**: `facebook/MobileLLM-R1-140M` (128,256 vocab) + +### 4. RoPE Base +- **Qwen**: 1,000,000 +- **MobileLLM**: 8,000,000 (8x larger for better long-context) + +### 5. Chat Template +- **Qwen**: Uses Qwen2-VL chat template +- **MobileLLM**: TODO - needs custom chat template for vision tokens + +## Verification Checklist + +After integration, verify: + +- [ ] Model name detection: Check logs for "Using MobileLLM-R1-140M as language backbone" +- [ ] Config loading: Verify 15 layers, 576 hidden size in logs +- [ ] Layer spec: Confirm "Using MobileLLM layer specification" +- [ ] Tokenizer: Vocab size should be 128,256 +- [ ] Training: Adapter parameters update, language/vision frozen +- [ ] Memory: 140M model should use significantly less VRAM than 4B + +## Expected Benefits + +1. **Efficiency**: 35x fewer parameters → faster inference, lower memory +2. **On-device Deployment**: Fits on mobile/edge devices +3. **Long Context**: 32k context with efficient RoPE +4. **Reasoning**: R1 series has strong math/reasoning capabilities + +## Troubleshooting + +### Error: "Unknown model family or arch: llava-ov-mobilellm-140m-fastvit" + +**Solution**: Ensure `llavaov_mobilellm_config.py` is imported. Check that decorators are registered before model initialization. + +### Error: "Cannot load checkpoint" + +**Solution**: If starting from scratch, comment out `--load` in the training script. For loading MobileLLM weights, they must be converted to Megatron format first. + +### Memory Issues + +**Solution**: +- Reduce sequence length: `SEQ_LEN=256` +- Enable gradient checkpointing: Already enabled in script +- Reduce recompute layers: `--recompute-num-layers 2` + +### Tokenizer Mismatch + +**Solution**: Ensure using `facebook/MobileLLM-R1-140M` tokenizer, not Qwen tokenizer. + +## Next Steps + +1. **Stage 1 Completion**: Train adapter on full LLaVA-558K dataset +2. **Stage 2 (Optional)**: Fine-tune full model on instruction data +3. **Evaluation**: Test on VQA benchmarks +4. **Export**: Convert to ONNX/TensorRT for deployment + +## References + +- [MobileLLM-R1 GitHub](https://github.com/facebookresearch/MobileLLM-R1) +- [MobileLLM-R1-140M on HuggingFace](https://huggingface.co/facebook/MobileLLM-R1-140M) +- [LLaVA-OneVision Paper](https://arxiv.org/abs/2408.03326) +- [FastViT/MobileCLIP Paper](https://arxiv.org/abs/2303.15378) + +## Contact + +For issues specific to this integration, check: +1. Model logs in `stage_1_alignment_mobilellm_140m/` +2. WandB dashboard if configured +3. TensorBoard logs in `stage_1_alignment_mobilellm_140m/tensorboard/` diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh new file mode 100644 index 00000000..d29f659d --- /dev/null +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -0,0 +1,217 @@ +#!/bin/bash +# Stage 1 Alignment Training for LLaVA-OneVision-1.5 with MobileLLM-R1-140M +# This script trains the projection adapter while keeping vision encoder and language model frozen + +REPO_ROOT=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5 +echo "$REPO_ROOT" + +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" + +# Model parallelism configuration +TP="${1:-1}" # Tensor parallel +PP="${2:-1}" # Pipeline parallel +SEQ_LEN="${3:-512}" # Sequence length (reduced for testing) +MBS="${4:-1}" # Micro batch size +GBS="${5:-1}" # Global batch size +NSTEP="${6:-5}" # Number of training iterations (reduced for testing) + +# Data paths - UPDATE THESE FOR YOUR SETUP +DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" +# MobileLLM tokenizer path - use facebook/MobileLLM-R1-140M from HuggingFace +TOKENIZER_PATH="${TOKENIZER_PATH:-"facebook/MobileLLM-R1-140M"}" +# Checkpoint path - for MobileLLM converted to Megatron format +# CHECKPOINT_PATH="${CHECKPOINT_PATH:-"$REPO_ROOT/checkpoints/mobilellm-140m_mcore_tp1_pp1"}" + +echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" +echo "DATA_PATH=${DATA_PATH}" +echo "TOKENIZER_PATH=${TOKENIZER_PATH}" + +# Multi-node configuration +declare -a list_ip=( + "localhost" +) + +CURRENT_IP=$(hostname -I | awk '{print $1}') +if [ -z "$CURRENT_IP" ]; then + CURRENT_IP=$(hostname -i 2>/dev/null | awk '{print $1}') +fi + +SINGLE_NODE=0 +if [[ ${#list_ip[@]} -eq 1 && ( "${list_ip[0]}" == "localhost" || "${list_ip[0]}" == "127.0.0.1" ) ]]; then + SINGLE_NODE=1 +fi + +NNODES=${#list_ip[@]} +MASTER_ADDR=${list_ip[0]} + +if [[ $SINGLE_NODE -eq 1 ]]; then + NNODES=1 + MASTER_ADDR=127.0.0.1 + NODE_RANK=0 + echo "--- Single-node mode ---" +else + NODE_RANK=-1 + for i in "${!list_ip[@]}"; do + if [[ "${list_ip[$i]}" == "${CURRENT_IP}" ]]; then + NODE_RANK=$i + break + fi + done + + if [ "$NODE_RANK" -eq -1 ]; then + echo "Error: Current IP ($CURRENT_IP) not found in the IP list." + exit 1 + fi + echo "--- Running on ${NNODES} nodes ---" +fi + +echo "MASTER_ADDR: ${MASTER_ADDR}" +echo "Current Node IP: ${CURRENT_IP}" +echo "Current Node Rank: ${NODE_RANK}" +echo "Node Size: ${NNODES}" + +# Output directories +SAVE_CKPT_PATH=$(basename "$0" .sh) +TENSORBOARD_PATH="${SAVE_CKPT_PATH}/tensorboard" + +mkdir -p "$SAVE_CKPT_PATH" +mkdir -p "$TENSORBOARD_PATH" +mkdir -p "$SAVE_CKPT_PATH/dataloader" + +GPUS_PER_NODE=${GPUS_PER_NODE:-1} +MASTER_PORT=${MASTER_PORT:-26000} + +if [[ $SINGLE_NODE -eq 1 ]]; then + DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --master_port "$MASTER_PORT" + ) +else + DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --nnodes "$NNODES" + --node_rank "$NODE_RANK" + --master_addr "$MASTER_ADDR" + --master_port "$MASTER_PORT" + ) +fi + +# ======================================== +# MODEL CONFIGURATION - MobileLLM Backbone +# ======================================== +MODEL_ARGS=( + --model-name llava-ov-mobilellm-140m-fastvit +) + +# ======================================== +# DATA CONFIGURATION +# ======================================== +DATA_ARGS=( + --tokenizer-type HFTokenizer + --hf-tokenizer-path "$TOKENIZER_PATH" + --data-path "$DATA_PATH" + --dataloader-type external + --split 100,0,0 + --num-workers 16 + + # FastViT vision encoder configuration + --use-fastvit + --fastvit-image-size 384 # MobileLLM is efficient, use 384 + --vision-tower-name mobileclip_s_384 # Use smaller FastViT for efficiency + --image-aspect-ratio pad +) + +# ======================================== +# TRAINING CONFIGURATION +# ======================================== +TRAINING_ARGS=( + --training-phase sft + --trainable-modules adapter # Only train adapter in Stage 1 + --no-gradient-accumulation-fusion + --seq-length "${SEQ_LEN}" + --no-rope-fusion + --training-rice-vl-max-answer-length 512 + --transformer-impl local + --max-position-embeddings 32768 # MobileLLM supports 32k context + --init-method-std 0.02 + --micro-batch-size "${MBS}" + --global-batch-size "${GBS}" + --lr 1.0e-4 + --min-lr 1.0e-6 + --clip-grad 1.0 + --weight-decay 0 + --optimizer adam + --adam-beta1 0.9 + --adam-beta2 0.99 + --adam-eps 1e-05 + --norm-epsilon 1e-05 # MobileLLM RMSNorm epsilon + --train-iters "$NSTEP" + --lr-decay-iters "$NSTEP" + --lr-decay-style cosine + --lr-warmup-fraction 0.002 + --initial-loss-scale 65536 + --bf16 + --save "$SAVE_CKPT_PATH" + --save-interval 2000 + --ckpt-format torch + --dataloader-save "${SAVE_CKPT_PATH}/dataloader" + --ckpt-fully-parallel-load + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 4 # Fewer layers to recompute for 140M model +) + +# ======================================== +# MODEL PARALLELISM CONFIGURATION +# ======================================== +MODEL_PARALLEL_ARGS=( + --attention-backend local + --pipeline-model-parallel-size "${PP}" + --tensor-model-parallel-size "${TP}" + --use-distributed-optimizer + --distributed-backend nccl +) + +# ======================================== +# LOGGING CONFIGURATION +# ======================================== +LOGGING_ARGS=( + --log-interval 1 + --tensorboard-dir "${TENSORBOARD_PATH}" + --log-timers-to-tensorboard +) + +if [ -n "${WANDB_API_KEY}" ]; then + LOGGING_ARGS+=( + --wandb-project "${WANDB_PROJECT:-llava-ov-mobilellm}" + --wandb-exp-name "${WANDB_NAME:-stage1-mobilellm-140m}" + ) +fi + +TM=$(date "+%Y-%m-%d_%H:%M:%S") +logfile="${SAVE_CKPT_PATH}/run_${TM}_tp${TP}_pp${PP}_seqlen${SEQ_LEN}_mbs${MBS}_gbs${GBS}_${NSTEP}steps.log" + +export OFFLINE_PACKED_DATA='1' +export OFFLINE_PACKING_VQA='1' + +# ======================================== +# RUN TRAINING +# ======================================== +echo "=========================================" +echo "Starting Stage 1 Training" +echo "Model: LLaVA-OneVision-1.5 + MobileLLM-R1-140M" +echo "Vision: FastViT (MobileCLIP)" +echo "Training: Adapter only (vision + language frozen)" +echo "=========================================" + +torchrun "${DISTRIBUTED_ARGS[@]}" \ + "$AIAK_TRAINING_PATH/aiak_training_llm/train.py" \ + "${MODEL_ARGS[@]}" \ + "${DATA_ARGS[@]}" \ + "${TRAINING_ARGS[@]}" \ + "${MODEL_PARALLEL_ARGS[@]}" \ + "${LOGGING_ARGS[@]}" \ + 2>&1 | tee "$logfile" + +echo "Training completed. Logs saved to: $logfile" From a53826d072ad4a20e876d31aabc58ba667536e53 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sat, 14 Feb 2026 17:54:20 +0400 Subject: [PATCH 17/39] feat: Add MobileLLM-R1-140M integration with FastViT - Implement MobileLLM-R1-140M (140M params) with GQA, SwiGLU, RMSNorm - Fix QK LayerNorm configuration (was disabled, now enabled) - Add FastViT/MobileCLIP vision encoder support - Add local _is_te_min_version() implementation to fix import error - Update training scripts for MobileLLM experiments - Verify model architecture matches official config.json --- Stage1/alignment.sh | 4 +- Stage1/alignment_rocm.sh | 14 +++- .../megatron_core.egg-info/SOURCES.txt | 18 +++++ aiak_megatron/pretrain_vlm.py | 5 +- aiak_megatron/setup.py | 2 +- aiak_training_llm/data/chat_templete.py | 70 +++++++++++++++++ .../data/multimodal/qwen2vl_task_encoder.py | 52 +++++++++++-- .../models/llavaov_1_5/__init__.py | 6 +- .../models/llavaov_1_5/llavaov_1_5_config.py | 39 ++++++++++ .../llavaov_1_5/llavaov_1_5_provider.py | 49 ++++++++---- .../llavaov_1_5/llavaov_mobilellm_config.py | 76 +++++++++++++------ .../models/mobilellm/mobilellm_layer_spec.py | 23 +++++- .../tokenizer/tokenization_hf.py | 7 -- aiak_training_llm/tokenizer/tokenizer.py | 3 +- aiak_training_llm/train/arguments.py | 4 + .../train/sft/sft_llavaov_1_5_vl.py | 2 +- aiak_training_llm/train/sft/sft_qwen2_vl.py | 2 +- aiak_training_llm/utils/initialize.py | 1 + .../stage_1_alignment_llava_ov_4b.sh | 2 +- .../stage_1_alignment_mobilellm_140m.sh | 28 +++---- 20 files changed, 325 insertions(+), 82 deletions(-) diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index 66b7b366..d2878bae 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -56,8 +56,8 @@ export WANDB_API_KEY="wandb_v1_5y5JqALBMdHhru8CR1gOLflJlRj_O8BG2XRb0S2x0TJVqW1xA export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="mobilellm_integration" -export CUDA_VISIBLE_DEVICES=4 -export GPUS_PER_NODE=1 +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export GPUS_PER_NODE=4 export MASTER_PORT=26000 # Choose which backbone to use for training: diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh index 91fa23e7..da2bdf45 100755 --- a/Stage1/alignment_rocm.sh +++ b/Stage1/alignment_rocm.sh @@ -22,7 +22,11 @@ export ROCM_HOME=${ROCM_HOME:-/opt/rocm} export PATH="${ROCM_HOME}/bin:${PATH}" export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH}" -export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1} +export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0} + +# Force HuggingFace offline mode to use local files only +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 # RCCL/NCCL runtime hints (tune as needed) @@ -68,4 +72,10 @@ echo "TOKENIZER_PATH=${TOKENIZER_PATH}" echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" echo "SLURM_NODELIST=${SLURM_NODELIST}" echo "PYTHONPATH=${PYTHONPATH}" -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh \ No newline at end of file + +# Weights & Biases configuration +export WANDB_API_KEY="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" +export WANDB_PROJECT="llava-ov-1_5" +export WANDB_NAME="fastvit_integration" +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh + 2,1 Top diff --git a/aiak_megatron/megatron_core.egg-info/SOURCES.txt b/aiak_megatron/megatron_core.egg-info/SOURCES.txt index 71f2d906..d13b8e94 100644 --- a/aiak_megatron/megatron_core.egg-info/SOURCES.txt +++ b/aiak_megatron/megatron_core.egg-info/SOURCES.txt @@ -277,6 +277,24 @@ megatron/core/transformer/moe/router.py megatron/core/transformer/moe/shared_experts.py megatron/core/transformer/moe/token_dispatcher.py megatron/core/transformer/moe/upcycling_utils.py +megatron/legacy/indexer.py +megatron/training/__init__.py +megatron/training/activations.py +megatron/training/arguments.py +megatron/training/async_utils.py +megatron/training/checkpointing.py +megatron/training/checkpointing_org.py +megatron/training/dist_signal_handler.py +megatron/training/ft_integration.py +megatron/training/global_vars.py +megatron/training/initialize.py +megatron/training/log_handler.py +megatron/training/one_logger_utils.py +megatron/training/theoretical_memory_usage.py +megatron/training/training.py +megatron/training/utils.py +megatron/training/wandb_utils.py +megatron/training/yaml_arguments.py megatron_core.egg-info/PKG-INFO megatron_core.egg-info/SOURCES.txt megatron_core.egg-info/dependency_links.txt diff --git a/aiak_megatron/pretrain_vlm.py b/aiak_megatron/pretrain_vlm.py index f5cdbf33..a169d791 100644 --- a/aiak_megatron/pretrain_vlm.py +++ b/aiak_megatron/pretrain_vlm.py @@ -129,7 +129,10 @@ def model_provider( # TODO: Make these configurable via input .yaml config. vision_transformer_config = deepcopy(language_transformer_config) - vision_transformer_config.num_layers = args.encoder_num_layers + # Only set num_layers from args for ViT models (like Qwen2-VL), not for FastViT + # FastViT determines its own layer count from architecture config + if not getattr(args, 'use_fastvit', False): + vision_transformer_config.num_layers = args.encoder_num_layers vision_transformer_config.first_pipeline_num_layers = None vision_transformer_config.last_pipeline_num_layers = None vision_transformer_config.vision_model_type = vision_model_type diff --git a/aiak_megatron/setup.py b/aiak_megatron/setup.py index 4f8d121c..c00cd7bc 100644 --- a/aiak_megatron/setup.py +++ b/aiak_megatron/setup.py @@ -105,7 +105,7 @@ def req_file(filename, folder="requirements"): 'Natural Language :: English', 'Operating System :: OS Independent', ], - packages=setuptools.find_namespace_packages(include=["megatron.core", "megatron.core.*"]), + packages=setuptools.find_namespace_packages(include=["megatron.core", "megatron.core.*","megatron.training","megatron.legacy"]), ext_modules=[ Extension( "megatron.core.datasets.helpers_cpp", diff --git a/aiak_training_llm/data/chat_templete.py b/aiak_training_llm/data/chat_templete.py index 82262cc4..136aee3f 100644 --- a/aiak_training_llm/data/chat_templete.py +++ b/aiak_training_llm/data/chat_templete.py @@ -98,6 +98,7 @@ class ChatTemplate: efficient_eos: bool = False replace_eos: bool = False mm_plugin: Optional[MMPlugin] = None + use_tokenizer_template: bool = False # If True, use tokenizer's built-in chat template def __post_init__(self): if self.format_user is None: @@ -163,6 +164,11 @@ def _encode( Turn 0: prefix + system + query resp Turn t: sep + query resp """ + # Use tokenizer's built-in chat template if enabled + if self.use_tokenizer_template: + return self._encode_with_tokenizer_template(tokenizer, messages, system) + + # Original custom template encoding system = system or self.default_system encoded_messages = [] for i, message in enumerate(messages): @@ -187,6 +193,58 @@ def _encode( return encoded_messages + def _encode_with_tokenizer_template( + self, + tokenizer: "AutoTokenizerFromHF", + messages: Sequence[Dict[str, str]], + system: Optional[str], + ) -> List[List[int]]: + """ + Use tokenizer's built-in apply_chat_template for encoding. + This is useful for models like MobileLLM-R1 that have their own chat template. + """ + # Prepare messages with system prompt if provided + formatted_messages = [] + if system: + formatted_messages.append({"role": "system", "content": system}) + + # Add all user/assistant messages + for msg in messages: + formatted_messages.append(msg) + + # Apply tokenizer's chat template + # Split into pairs: (user+system, assistant), (user, assistant), ... + encoded_messages = [] + + # Process each turn separately to get proper tokenization per turn + for i in range(0, len(messages), 2): + # User message (with system if first turn) + user_msgs = [] + if i == 0 and system: + user_msgs.append({"role": "system", "content": system}) + user_msgs.append(messages[i]) + + # Tokenize user turn + user_text = tokenizer.apply_chat_template( + user_msgs, + tokenize=False, + add_generation_prompt=True # Add assistant prompt + ) + user_ids = tokenizer.encode(user_text, add_special_tokens=False) + encoded_messages.append(user_ids) + + # Assistant message + if i + 1 < len(messages): + # Just the content, no special formatting + assistant_text = messages[i + 1]["content"] + # Add EOS if not efficient_eos + if not self.efficient_eos: + assistant_text += tokenizer.eos_token + assistant_ids = tokenizer.encode(assistant_text, add_special_tokens=False) + encoded_messages.append(assistant_ids) + + return encoded_messages + def _convert_elements_to_ids( self, tokenizer: "AutoTokenizerFromHF", @@ -279,6 +337,7 @@ def _register_chat_template( efficient_eos: bool = False, replace_eos: bool = False, mm_plugin: Optional[MMPlugin] = None, + use_tokenizer_template: bool = False, # New parameter for auto template ) -> None: """ Registers a chat template. @@ -320,6 +379,7 @@ def _register_chat_template( efficient_eos=efficient_eos, replace_eos=replace_eos, mm_plugin=mm_plugin, + use_tokenizer_template=use_tokenizer_template, # Pass the new parameter ) @@ -459,3 +519,13 @@ def get_support_templates() -> List[str]: format_user=StringFormatter(slots=["<｜User｜>{{content}}<｜Assistant｜>"]), format_prefix=EmptyFormatter(slots=[{"bos_token"}]), ) + + +# Auto template - uses tokenizer's built-in chat template +# Perfect for MobileLLM-R1 and other models with embedded templates +_register_chat_template( + name="auto", + cls=ChatTemplate, + use_tokenizer_template=True, + efficient_eos=True, +) diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 2b0326c3..214f8828 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -128,18 +128,58 @@ def __init__(self, args): self.chat_template = get_chat_template() # Load conversation template #template for formatting conversations (user/assistant turns) + # OLD APPROACH (commented out - didn't work for MobileLLM): # Load HuggingFace processor (tokenizer + image/ video processor from Qwen2-VL) - self.processor = AutoProcessor.from_pretrained(self.args.hf_tokenizer_path, trust_remote_code=True, local_files_only=True) - print(f"Loaded processor from {self.args.hf_tokenizer_path}") - print(f"Processor config: {self.processor}") + # self.processor = AutoProcessor.from_pretrained(self.args.hf_tokenizer_path, trust_remote_code=True, local_files_only=False) + # print(f"Loaded processor from {self.args.hf_tokenizer_path}") + # print(f"Processor config: {self.processor}") + # NEW APPROACH: Handle FastVLM (MobileLLM + FastViT) vs Qwen2-VL separately # FastViT image processor (following FastVLM repo) # Use this for vision encoding instead of Qwen2-VL's processor self.use_fastvit = getattr(args, 'use_fastvit', False) + if self.use_fastvit: + # For FastVLM: Load only the language model tokenizer (e.g., MobileLLM) + # No need for a full processor since FastViT handles vision separately + from transformers import AutoTokenizer + self.tokenizer_hf = AutoTokenizer.from_pretrained( + self.args.hf_tokenizer_path, + trust_remote_code=True, + local_files_only=False + ) + + # Set padding token if not present (required for batch processing) + if self.tokenizer_hf.pad_token is None: + self.tokenizer_hf.pad_token = self.tokenizer_hf.eos_token + print(f"Set pad_token to eos_token: {self.tokenizer_hf.eos_token}") + + # Create a wrapper that properly delegates all methods to the tokenizer + class TokenizerWrapper: + def __init__(self, tokenizer): + self.tokenizer = tokenizer + + def __getattr__(self, name): + # Delegate all other attributes/methods to the underlying tokenizer + return getattr(self.tokenizer, name) + + self.processor = TokenizerWrapper(self.tokenizer_hf) + print(f"Loaded tokenizer (FastVLM mode) from {self.args.hf_tokenizer_path}") + + # Initialize FastViT image processor fastvit_image_size = getattr(args, 'fastvit_image_size', 1024) self.fastvit_processor = FastViTImageProcessor(image_size=fastvit_image_size) print(f"Initialized FastViT processor with image_size={fastvit_image_size}") + else: + # For Qwen2-VL: Load full processor (tokenizer + image/video processor) + self.processor = AutoProcessor.from_pretrained( + self.args.hf_tokenizer_path, + trust_remote_code=True, + local_files_only=False + ) + print(f"Loaded Qwen2-VL processor from {self.args.hf_tokenizer_path}") + + print(f"Processor config: {self.processor}") #Resolution parameters for resizing images/videos if args.image_resolution: @@ -484,7 +524,8 @@ def encode_captioning(self, sample: CaptioningSample) -> ImageTaskSample: # assert self.args.training_phase == constants.TrainingPhase.PRETRAIN, "Only support PRETRAIN phase" - text = IMAGE_TOKEN_WITH_TAGS + sample.caption + self.tokenizer.tokenizer.eos_token + # text = IMAGE_TOKEN_WITH_TAGS + sample.caption + self.tokenizer.tokenizer.eos_token + text = IMAGE_TOKEN_WITH_TAGS + sample.caption + self.tokenizer.eos_token input_ids, target, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) num_tiles = [len(image_grid_thw)] if image_grid_thw is not None else [1] @@ -650,7 +691,8 @@ def encode_vaq(self, sample: VQASample) -> ImageTaskSample: ) else: text = IMAGE_TOKEN_WITH_TAGS + sample.answers - text = text + self.tokenizer.tokenizer.eos_token + # text = text + self.tokenizer.tokenizer.eos_token + text = text + self.tokenizer.eos_token input_ids, target, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) elif self.args.training_phase == constants.TrainingPhase.SFT: diff --git a/aiak_training_llm/models/llavaov_1_5/__init__.py b/aiak_training_llm/models/llavaov_1_5/__init__.py index 4300efc7..d9563d5f 100644 --- a/aiak_training_llm/models/llavaov_1_5/__init__.py +++ b/aiak_training_llm/models/llavaov_1_5/__init__.py @@ -1,11 +1,7 @@ """LLaVA-OneVision-1.5 model""" from .llavaov_1_5_model import LlavaOnevision1_5 -from .llavaov_mobilellm_config import ( - get_llava_ov_mobilellm_140m_config, - get_llava_ov_mobilellm_140m_fastvit_config, -) - +from . import llavaov_1_5_config __all__ = [ "LlavaOnevision1_5", "get_llava_ov_mobilellm_140m_config", diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py index 4d349260..00afdeca 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py @@ -157,6 +157,45 @@ def llava_one_vision_1_5_14b(): ) +@register_model_config(model_family=VisionLanguageModelFamilies.LLAVA_OV_1_5, model_arch="llava-ov-mobilellm-140m") +def llava_ov_mobilellm_140m(): + """ + LLaVA-OneVision-1.5 with MobileLLM-R1-140M backbone + + Integrates Facebook's MobileLLM-R1-140M (140M params) as the language model + replacing Qwen2.5. Architecture specs from: + https://github.com/facebookresearch/MobileLLM-R1 + """ + return LlavaOnevision1_5Config( + # Core architecture from MobileLLM-R1-140M + num_layers=15, # 15 transformer layers (vs 36 in Qwen2.5-3B) + hidden_size=576, # 576 hidden dimension (vs 2048 in Qwen2.5-3B) + ffn_hidden_size=2048, # 2048 FFN dimension + num_attention_heads=9, # 9 attention heads + + # Grouped Query Attention (GQA) - efficiency optimization + group_query_attention=True, # Enable GQA + num_query_groups=3, # 3 KV heads for GQA (9 Q heads / 3 KV heads) + + # Vocabulary - MobileLLM uses Llama3 tokenizer (128,256 tokens) + vocab_size_in_config_file=128256, # Llama3 vocab (vs 151936 in Qwen) + make_vocab_size_divisible_by=128, # Keep same for efficiency + + # Embeddings - MobileLLM ties embeddings (shared input/output weights) + untie_embeddings_and_output_weights=False, # Tied embeddings (saves params) + + # Position embeddings - RoPE with extended base for long context + rotary_base=8000000, # 8M RoPE base (vs 1M in Qwen, supports 32k context) + + # Bias settings - MobileLLM uses no bias + add_qkv_bias=False, # No bias in attention (efficiency) + qk_layernorm=True, # MobileLLM-R1 uses QK layernorm (use_qk_norm: true in config.json) + + # Normalization - standard RMSNorm + norm_epsilon=1e-05, # 1e-5 epsilon (vs 1e-6 in Qwen) + ) + + @dataclass class VisionConfig: """configuration for vision model diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index 65708d0d..01f287b4 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -9,8 +9,7 @@ get_mobilellm_layer_with_te_spec, get_vision_layer_with_spec) from aiak_training_llm.models.llavaov_1_5.llavaov_1_5_config import ( get_adapeter_config, get_vision_config) -from aiak_training_llm.models.llavaov_1_5.llavaov_mobilellm_config import ( - get_language_config as get_mobilellm_language_config) +# MobileLLM config is now in llavaov_1_5_config.py and applied via args from aiak_training_llm.utils import (build_transformer_config, get_args, print_rank_0) from aiak_training_llm.utils.constants import VisionLanguageModelFamilies @@ -42,45 +41,59 @@ def rice_vl_model_provider( args = get_args() print_rank_0(f'building {args.model_name} model ...') + print_rank_0(f'[DEBUG PROVIDER] Model name: {args.model_name}') + print_rank_0(f'[DEBUG PROVIDER] Args num_layers: {args.num_layers}') + print_rank_0(f'[DEBUG PROVIDER] Args hidden_size: {args.hidden_size}') + print_rank_0(f'[DEBUG PROVIDER] Args vocab_size: {args.vocab_size_in_config_file}') config = build_transformer_config(args) #base transformer config with all hyperparams + print_rank_0(f'[DEBUG PROVIDER] TransformerConfig built with num_layers: {config.num_layers}, hidden_size: {config.hidden_size}') language_config = deepcopy(config) # For Qwen2.5 language model vision_config = deepcopy(config) # For vision encoder (SigLIP) adapter_config = deepcopy(config) ## For adapter (projection) + print_rank_0(f'[DEBUG PROVIDER] Initial language_config: layers={language_config.num_layers}, hidden={language_config.hidden_size}') # Vision Encoder → Adapter → Language Model # (SigLIP) (Projection) (Qwen2.5) from aiak_training_llm.models import get_model_family model_family = get_model_family(args.model_name) + print_rank_0(f'[DEBUG PROVIDER] Model family: {model_family}') # Detect if using MobileLLM backbone use_mobilellm = "mobilellm" in args.model_name.lower() + print_rank_0(f'[DEBUG PROVIDER] use_mobilellm flag: {use_mobilellm}') if use_mobilellm: - print_rank_0(f'Using MobileLLM-R1-140M as language backbone') - # Load MobileLLM configuration - mobilellm_config = get_mobilellm_language_config(args.model_name) - for k, v in asdict(mobilellm_config).items(): - setattr(language_config, k, v) - print_rank_0(f'MobileLLM language config: {language_config}') + print_rank_0(f'[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone') + print_rank_0(f'[DEBUG PROVIDER] Language config BEFORE override: layers={language_config.num_layers}, ' + f'hidden={language_config.hidden_size}, heads={language_config.num_attention_heads}') + # MobileLLM params are already in language_config from args! + # No need to load separately - they were applied in _validate_extra_model_args + print_rank_0(f'[DEBUG PROVIDER] Language config AFTER (should be same): layers={language_config.num_layers}, ' + f'hidden={language_config.hidden_size}, heads={language_config.num_attention_heads}, ' + f'query_groups={language_config.num_query_groups}') + else: + print_rank_0(f'[DEBUG PROVIDER] Using Qwen2.5 as language backbone') # get vision specific config : no. of layers, hidden size, Patch size, Image resolution + print_rank_0(f'[DEBUG PROVIDER] Loading vision config...') for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): setattr(vision_config, k, v) + print_rank_0(f'[DEBUG PROVIDER] Vision config: layers={vision_config.num_layers}, hidden={vision_config.hidden_size}') # Add FastViT-specific configuration if enabled if getattr(args, 'use_fastvit', False): vision_tower_name = getattr(args, 'vision_tower_name', 'mobileclip_l_384') setattr(vision_config, 'vision_tower_name', vision_tower_name) - print_rank_0(f'Using FastViT with vision_tower_name: {vision_tower_name}') + print_rank_0(f'[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: {vision_tower_name}') - print_rank_0(f'Vision config: {vision_config}') # get adapter specific config : Projection dimension, Activation function + print_rank_0(f'[DEBUG PROVIDER] Loading adapter config...') for k, v in asdict(get_adapeter_config(model_family)).items(): setattr(adapter_config, k, v) - print_rank_0(f'Adapter config: {adapter_config}') + print_rank_0(f'[DEBUG PROVIDER] Adapter config loaded') # set special token ids for language model setattr(language_config, "image_token_id", 151655) @@ -121,16 +134,21 @@ def rice_vl_model_provider( if args.spec is not None: language_layer_spec = import_module(args.spec) + print_rank_0(f'[DEBUG PROVIDER] Using custom spec: {args.spec}') else: + print_rank_0(f'[DEBUG PROVIDER] Building layer specs...') adapter_layer_spec = get_adapeter_layer_with_spec() vision_layer_spec = get_vision_layer_with_spec() # Choose language layer spec based on backbone if use_mobilellm: - print_rank_0("Using MobileLLM layer specification") + print_rank_0(f'[DEBUG PROVIDER] ✓ Using MobileLLM layer specification') + print_rank_0(f'[DEBUG PROVIDER] MobileLLM layer config: layers={language_config.num_layers}, ' + f'hidden={language_config.hidden_size}, heads={language_config.num_attention_heads}') language_layer_spec = get_mobilellm_layer_with_te_spec(language_config) + print_rank_0(f'[DEBUG PROVIDER] MobileLLM layer spec created successfully') else: - print_rank_0("Using Qwen layer specification") + print_rank_0(f'[DEBUG PROVIDER] Using Qwen layer specification') language_layer_spec = get_qwen_layer_with_te_spec(language_config) # # Vision layer spec (Transformer block) @@ -150,6 +168,10 @@ def rice_vl_model_provider( # - Optional activation #create the model + print_rank_0(f'[DEBUG PROVIDER] Creating LlavaOnevision1_5 model...') + print_rank_0(f'[DEBUG PROVIDER] Final language_config: layers={language_config.num_layers}, ' + f'hidden={language_config.hidden_size}, vocab={args.padded_vocab_size}, ' + f'rotary_base={args.rotary_base}') model = LlavaOnevision1_5( language_config=language_config, vision_config=vision_config, @@ -173,6 +195,7 @@ def rice_vl_model_provider( # When using FastViT, adapter dimensions change, so allow missing adapter weights allow_missing_adapter_checkpoint=getattr(args, 'use_fastvit', False), ) + print_rank_0(f'[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully!') # Vision encoder: SigLIP with 27 layers # Adapter: Projection network # Language model: Qwen2.5 with 32 layers diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py b/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py index d8e060c1..3da5df87 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py @@ -6,11 +6,60 @@ while replacing the language model architecture. """ +import torch from dataclasses import dataclass from aiak_training_llm.models.factory import register_model_config from aiak_training_llm.utils.constants import VisionLanguageModelFamilies +@dataclass +class LlavaOvMobileLLMConfig: + """Unified config for LLaVA-OV with MobileLLM backbone (matching LlavaOnevision1_5Config structure)""" + # Core architecture + num_layers: int = 15 + hidden_size: int = 576 + ffn_hidden_size: int = 2048 + num_attention_heads: int = 9 + + # GQA configuration + group_query_attention: bool = True + num_query_groups: int = 3 + + # Position embeddings + position_embedding_type: str = "rope" + add_position_embedding: bool = False + rotary_interleaved: bool = False + rotary_base: int = 8000000 + + # Normalization + normalization: str = "RMSNorm" + norm_epsilon: float = 1e-05 + + # Activation + swiglu: bool = True + + # Dropout + attention_dropout: float = 0.0 + hidden_dropout: float = 0.0 + + # Bias settings + add_bias_linear: bool = False + add_qkv_bias: bool = False + qk_layernorm: bool = False + + # Embeddings + untie_embeddings_and_output_weights: bool = False + + # Vocabulary + vocab_size_in_config_file: int = 128256 + make_vocab_size_divisible_by: int = 128 + + # Optional + kv_channels: int = None + num_experts: int = None + moe_ffn_hidden_size: int = None + + @dataclass class MobileLLMLanguageConfig: """Language model configuration for MobileLLM-R1-140M""" @@ -79,15 +128,8 @@ class AdapterConfig: def get_llava_ov_mobilellm_140m_config(): """ Configuration for LLaVA-OneVision-1.5 with MobileLLM-R1-140M backbone - - Returns a dict with language, vision, and adapter configurations """ - return { - "language": MobileLLMLanguageConfig(), - "vision": VisionConfig(), - "adapter": AdapterConfig(), - "model_type": "llava_ov_mobilellm" - } + return LlavaOvMobileLLMConfig() @register_model_config( @@ -100,22 +142,8 @@ def get_llava_ov_mobilellm_140m_fastvit_config(): Uses FastViT (MobileCLIP) as the vision encoder for efficiency """ - # FastViT configuration (MobileCLIP-L-384) - vision_config = VisionConfig( - num_layers=12, # FastViT is shallower - hidden_size=768, # FastViT hidden dimension - num_attention_heads=12, - ffn_hidden_size=3072, - patch_size=16, - image_resolution=384 - ) - - return { - "language": MobileLLMLanguageConfig(), - "vision": vision_config, - "adapter": AdapterConfig(), - "model_type": "llava_ov_mobilellm_fastvit" - } + # Use default MobileLLM config (FastViT vision config can be handled separately) + return LlavaOvMobileLLMConfig() def get_vision_config(model_family: str, model_name: str): diff --git a/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py b/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py index 4ab85fa2..6ddb5fc3 100644 --- a/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py +++ b/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py @@ -11,15 +11,26 @@ from aiak_training_llm.models.dispatch import multiacc_modules +def _is_te_min_version(version: str) -> bool: + """Check if Transformer Engine version is at least the specified version.""" + try: + import transformer_engine + from packaging import version as pkg_version + te_version = getattr(transformer_engine, '__version__', '0.0.0') + return pkg_version.parse(te_version) >= pkg_version.parse(version) + except (ImportError, AttributeError): + return False + + def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: """ Get MobileLLM layer specification using Transformer Engine modules. - MobileLLM-R1-140M uses standard LLaMA architecture: + MobileLLM-R1-140M architecture: - Grouped Query Attention (9 heads, 3 KV heads) - SwiGLU activation (gated_linear_unit=True) - RMSNorm - - No QK LayerNorm + - QK LayerNorm (use_qk_norm: true in official config.json) Args: config: Transformer configuration with MobileLLM parameters @@ -27,6 +38,10 @@ def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: Returns: ModuleSpec for MobileLLM transformer layer """ + # TENorm significantly harms convergence when used for QKLayerNorm if TE Version < 1.9; + # we instead use the Apex implementation. + qk_norm = multiacc_modules.TENorm if _is_te_min_version("1.9.0") else multiacc_modules.LocalNorm + # Standard dense MLP with SwiGLU mlp = ModuleSpec( module=MLP, @@ -48,8 +63,8 @@ def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: linear_qkv=multiacc_modules.TELayerNormColumnParallelLinear, core_attention=multiacc_modules.DotProductAttention, linear_proj=multiacc_modules.TERowParallelLinear, - q_layernorm=IdentityOp, # MobileLLM doesn't use QK LayerNorm - k_layernorm=IdentityOp, + q_layernorm=qk_norm if config.qk_layernorm else IdentityOp, + k_layernorm=qk_norm if config.qk_layernorm else IdentityOp, apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, ), ), diff --git a/aiak_training_llm/tokenizer/tokenization_hf.py b/aiak_training_llm/tokenizer/tokenization_hf.py index 29d36f5f..921b9dfd 100644 --- a/aiak_training_llm/tokenizer/tokenization_hf.py +++ b/aiak_training_llm/tokenizer/tokenization_hf.py @@ -22,12 +22,6 @@ def __init__(self, **kwargs, ): super().__init__(name_or_path) - # Bypass HF repo validation for local paths - import os - if os.path.isdir(name_or_path) or os.path.isfile(name_or_path): - # For local paths, set token to False to avoid HF Hub validation - kwargs.setdefault('token', False) - self.tokenizer = AutoTokenizer.from_pretrained( name_or_path, use_fast=use_fast_tokenizer, @@ -35,7 +29,6 @@ def __init__(self, split_special_tokens=split_special_tokens, model_max_length=model_max_length, trust_remote_code=True, - local_files_only=True, **kwargs, ) diff --git a/aiak_training_llm/tokenizer/tokenizer.py b/aiak_training_llm/tokenizer/tokenizer.py index 0e5f4031..3c7bb865 100644 --- a/aiak_training_llm/tokenizer/tokenizer.py +++ b/aiak_training_llm/tokenizer/tokenizer.py @@ -77,7 +77,8 @@ def build_tokenizer(args, chat_template: Optional["ChatTemplate"] = None) -> Opt if args.training_phase == constants.TrainingPhase.PRETRAIN: if tokenizer.eos is None: if args.model_family == constants.LanguageModelFamilies.QWEN: - tokenizer.eos = tokenizer.tokenizer.eod_id + # tokenizer.eos = tokenizer.tokenizer.eod_id + tokenizer.eos = tokenizer.eod_id elif chat_template is not None: # update the tokenizer with chat template diff --git a/aiak_training_llm/train/arguments.py b/aiak_training_llm/train/arguments.py index 5724b3aa..fc57272e 100644 --- a/aiak_training_llm/train/arguments.py +++ b/aiak_training_llm/train/arguments.py @@ -452,6 +452,8 @@ def _validate_extra_model_args(args): if model_config is not None: # the structural configuration of model will be overwritten, such as num_layers, hidden_states.. print_rank_0(f'-------------- Configure model to {args.model_name} --------------', args.rank) + print_rank_0(f'[DEBUG] Model config type: {type(model_config).__name__}', args.rank) + print_rank_0(f'[DEBUG] Is MobileLLM: {"mobilellm" in args.model_name.lower()}', args.rank) for field in fields(model_config.__class__): assert hasattr(args, field.name), f"The model config field ({field.name}) is not defined in args." @@ -460,6 +462,8 @@ def _validate_extra_model_args(args): print_rank_0(f" {key} = {value} ", args.rank) print_rank_0('---------------- End of configuration ----------------', args.rank) + print_rank_0(f'[DEBUG] Args now has: num_layers={args.num_layers}, hidden_size={args.hidden_size}, ' + f'num_attention_heads={args.num_attention_heads}, vocab_size={args.vocab_size_in_config_file}', args.rank) if args.enable_fa_within_mla: args.attention_backend = AttnBackend.flash diff --git a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py index 3a5c1b4c..0354a598 100644 --- a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py +++ b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py @@ -188,7 +188,7 @@ def train_valid_test_datasets_provider(train_val_test_num_samples): tokenizer = get_tokenizer() - processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True, local_files_only=True) + processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True, local_files_only=False) if args.image_resolution: setattr(processor, "image_resolution", args.image_resolution) diff --git a/aiak_training_llm/train/sft/sft_qwen2_vl.py b/aiak_training_llm/train/sft/sft_qwen2_vl.py index fdfdb663..02b9acec 100644 --- a/aiak_training_llm/train/sft/sft_qwen2_vl.py +++ b/aiak_training_llm/train/sft/sft_qwen2_vl.py @@ -188,7 +188,7 @@ def train_valid_test_datasets_provider(train_val_test_num_samples): tokenizer = get_tokenizer() - processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True, local_files_only=True) + processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True, local_files_only=False) if args.image_resolution: setattr(processor, "image_resolution", args.image_resolution) diff --git a/aiak_training_llm/utils/initialize.py b/aiak_training_llm/utils/initialize.py index 40e29448..e77fa032 100644 --- a/aiak_training_llm/utils/initialize.py +++ b/aiak_training_llm/utils/initialize.py @@ -46,6 +46,7 @@ def parse_arguments( ignore_unknown_args=False, ): """Parse arguments.""" + print("parse arguments function called") args = parse_args(extra_args_provider, ignore_unknown_args) # Prep for checkpoint conversion. diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 7139a1aa..993d5090 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -10,7 +10,7 @@ PP="${2:-1}" # pipeline parallel SEQ_LEN="${3:-512}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size -GBS="${5:-1}" +GBS="${5:-2}" # NSTEP="${6:-2500}" # number of training iterations NSTEP="${6:-5}" # number of training iterations - reduced to 5 samples for CUDA memory # DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index d29f659d..74621543 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -13,8 +13,8 @@ TP="${1:-1}" # Tensor parallel PP="${2:-1}" # Pipeline parallel SEQ_LEN="${3:-512}" # Sequence length (reduced for testing) MBS="${4:-1}" # Micro batch size -GBS="${5:-1}" # Global batch size -NSTEP="${6:-5}" # Number of training iterations (reduced for testing) +GBS="${5:-4}" # Global batch size (4 examples for testing) +NSTEP="${6:-1}" # Number of training iterations (1 step with 4 examples) # Data paths - UPDATE THESE FOR YOUR SETUP DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" @@ -79,7 +79,7 @@ mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=${GPUS_PER_NODE:-1} +GPUS_PER_NODE=${GPUS_PER_NODE:-4} MASTER_PORT=${MASTER_PORT:-26000} if [[ $SINGLE_NODE -eq 1 ]]; then @@ -101,7 +101,7 @@ fi # MODEL CONFIGURATION - MobileLLM Backbone # ======================================== MODEL_ARGS=( - --model-name llava-ov-mobilellm-140m-fastvit + --model-name llava-ov-mobilellm-140m ) # ======================================== @@ -112,22 +112,23 @@ DATA_ARGS=( --hf-tokenizer-path "$TOKENIZER_PATH" --data-path "$DATA_PATH" --dataloader-type external - --split 100,0,0 + --split 100,0,0 # Data Splitting part --num-workers 16 # FastViT vision encoder configuration --use-fastvit - --fastvit-image-size 384 # MobileLLM is efficient, use 384 - --vision-tower-name mobileclip_s_384 # Use smaller FastViT for efficiency - --image-aspect-ratio pad + --fastvit-image-size 384 # MobileLLM is efficient, use 384 (for testing) + --vision-tower-name mobileclip_l_384 # MobileCLIP-L with 384 resolution + --image-aspect-ratio pad # Pad images to square aspect ratio ) # ======================================== # TRAINING CONFIGURATION # ======================================== TRAINING_ARGS=( - --training-phase sft - --trainable-modules adapter # Only train adapter in Stage 1 + --training-phase sft # supervised fine-tuning + --chat-template llama3 # MobileLLM uses Llama3 tokenizer + --trainable-modules language_model adapter # Train language model + adapter (vision frozen) --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" --no-rope-fusion @@ -153,13 +154,14 @@ TRAINING_ARGS=( --initial-loss-scale 65536 --bf16 --save "$SAVE_CKPT_PATH" - --save-interval 2000 + # --save-interval 2000 + --save-interval 1 --ckpt-format torch --dataloader-save "${SAVE_CKPT_PATH}/dataloader" --ckpt-fully-parallel-load --recompute-granularity full --recompute-method uniform - --recompute-num-layers 4 # Fewer layers to recompute for 140M model + --recompute-num-layers 3 # Must divide evenly into 15 layers (15/3=5 chunks) ) # ======================================== @@ -213,5 +215,3 @@ torchrun "${DISTRIBUTED_ARGS[@]}" \ "${MODEL_PARALLEL_ARGS[@]}" \ "${LOGGING_ARGS[@]}" \ 2>&1 | tee "$logfile" - -echo "Training completed. Logs saved to: $logfile" From 5fbe2d427209883b27c90b9a85dd5504aa993c82 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sat, 14 Feb 2026 20:03:13 +0400 Subject: [PATCH 18/39] Add MobileLLM-140M + FastViT checkpoint conversion and merge scripts - Add convert_fastvit_hf_to_mcore.sh: Convert FastViT from HF to Megatron format - Add convert_mobilellm_hf_to_mcore.sh: Convert MobileLLM-R1-140M to Megatron format - Add merge_mobilellm_fastvit.sh: Merge language and vision checkpoints - Update stage_1_alignment_mobilellm_140m.sh: Training config with merged checkpoint - Add test_inference_mobilellm.py: Inference script for FastVLM - Fix nvcc path resolution for multiple CUDA installations - Update training_utils.py: FastViT checkpoint handling Successfully tested: - Checkpoint conversion: 629 vision + 152 language tensors - Merged checkpoint: 781 keys, 774MB - Training verified: Loss 11.87, 36.6 tokens/sec/GPU on A100-40GB --- Stage1/test_inference_mobilellm.py | 211 ++++++++++++++++++ .../megatron/legacy/fused_kernels/__init__.py | 14 +- aiak_training_llm/train/training_utils.py | 25 ++- .../convert/convert_fastvit_hf_to_mcore.sh | 70 ++++++ .../convert/convert_mobilellm_hf_to_mcore.sh | 56 +++++ .../convert/merge_mobilellm_fastvit.sh | 83 +++++++ .../stage_1_alignment_mobilellm_140m.sh | 15 +- 7 files changed, 461 insertions(+), 13 deletions(-) create mode 100644 Stage1/test_inference_mobilellm.py create mode 100755 examples/llava_ov_1_5/convert/convert_fastvit_hf_to_mcore.sh create mode 100755 examples/llava_ov_1_5/convert/convert_mobilellm_hf_to_mcore.sh create mode 100755 examples/llava_ov_1_5/convert/merge_mobilellm_fastvit.sh diff --git a/Stage1/test_inference_mobilellm.py b/Stage1/test_inference_mobilellm.py new file mode 100644 index 00000000..940e4311 --- /dev/null +++ b/Stage1/test_inference_mobilellm.py @@ -0,0 +1,211 @@ +""" +Test FastVLM inference with MobileLLM-140M + FastViT +Uses the same model setup as training but loads from checkpoint for inference. +""" +import os +import sys +import torch +import torch.nn.functional as F +from PIL import Image +from argparse import ArgumentParser + +# Add paths +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, REPO_ROOT) +sys.path.insert(0, os.path.join(REPO_ROOT, 'aiak_megatron')) +sys.path.insert(0, os.path.join(REPO_ROOT, 'aiak_training_llm')) + +from transformers import AutoTokenizer +from aiak_training_llm.models.fastvit.fastvit_preprocessor import FastViTImageProcessor +from aiak_training_llm.models.fastvit.mm_utils import expand2square +from aiak_training_llm.models.llavaov_1_5.llavaov_1_5_config import llava_ov_mobilellm_140m +from aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model import LlavaOnevision1_5 +from aiak_training_llm.models.mobilellm.mobilellm_layer_spec import get_mobilellm_layer_with_te_spec +from aiak_training_llm.models.qwen.layer_spec import get_adapeter_layer_with_spec + +def main(): + parser = ArgumentParser(description="FastVLM + MobileLLM Inference Test") + parser.add_argument('--checkpoint', type=str, + default='checkpoints/mobilellm-fastvit-merged-tp1-pp1', + help='Path to merged checkpoint') + parser.add_argument('--tokenizer', type=str, + default='checkpoints/MobileLLM-R1-140M', + help='Path to tokenizer') + parser.add_argument('--image', type=str, default='test_image.jpg', + help='Path to test image') + parser.add_argument('--prompt', type=str, default='What is in this image?', + help='Text prompt') + parser.add_argument('--image_size', type=int, default=1024, + help='FastViT image size') + parser.add_argument('--max_new_tokens', type=int, default=100, + help='Maximum number of tokens to generate') + parser.add_argument('--temperature', type=float, default=0.7, + help='Sampling temperature') + args = parser.parse_args() + + print("=" * 80) + print("FastVLM + MobileLLM-140M Inference Test") + print("=" * 80) + print(f"Checkpoint: {args.checkpoint}") + print(f"Tokenizer: {args.tokenizer}") + print(f"Image: {args.image}") + print(f"Image Size: {args.image_size}") + print("=" * 80) + + # Check checkpoint exists + checkpoint_path = os.path.join(REPO_ROOT, args.checkpoint) + if not os.path.exists(checkpoint_path): + print(f"\nERROR: Checkpoint not found: {checkpoint_path}") + print("Please run training first or provide a valid checkpoint path.") + return + + # Load tokenizer + print("\n[1/4] Loading tokenizer...") + tokenizer_path = os.path.join(REPO_ROOT, args.tokenizer) + tokenizer = AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True) + print(f"✓ Loaded tokenizer from {tokenizer_path}") + print(f" Vocab size: {len(tokenizer)}") + + # Load and preprocess image + print("\n[2/4] Loading and preprocessing image...") + image_path = os.path.join(os.path.dirname(__file__), args.image) if not os.path.isabs(args.image) else args.image + if not os.path.exists(image_path): + print(f"ERROR: Image not found: {image_path}") + return + + image = Image.open(image_path).convert('RGB') + print(f" Original size: {image.size}") + + # Initialize FastViT processor + fastvit_processor = FastViTImageProcessor(image_size=args.image_size) + mean_color = tuple(int(x * 255) for x in fastvit_processor.image_mean) + image_padded = expand2square(image, mean_color) + pixel_values = fastvit_processor(image_padded).unsqueeze(0) # Add batch dim + print(f" Padded to square: {image_padded.size}") + print(f" Preprocessed shape: {pixel_values.shape}") + + # Create prompt + print("\n[3/4] Tokenizing prompt...") + IMAGE_TOKEN = "<|image_pad|>" + VISION_START = "<|vision_start|>" + VISION_END = "<|vision_end|>" + + conversation = ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + f"<|im_start|>user\n{VISION_START}{IMAGE_TOKEN}{VISION_END}\n{args.prompt}<|im_end|>\n" + "<|im_start|>assistant\n" + ) + + input_ids = tokenizer(conversation, return_tensors="pt")["input_ids"] + print(f" Prompt: {args.prompt}") + print(f" Input IDs shape: {input_ids.shape}") + + # Initialize model + print("\n[4/4] Loading model and running inference...") + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + print(f" Using device: {device}") + + # Get model configs + print(" Building model...") + language_config, vision_config, adapter_config = llava_ov_mobilellm_140m() + + # Get layer specs (FastViT doesn't use layer spec, uses nn.Module directly) + language_layer_spec = get_mobilellm_layer_with_te_spec() + vision_layer_spec = None # FastViT uses direct nn.Module + adapter_layer_spec = get_adapeter_layer_with_spec() + + # Build model + model = LlavaOnevision1_5( + language_config=language_config, + vision_config=vision_config, + adapter_config=adapter_config, + language_layer_spec=language_layer_spec, + vision_layer_spec=vision_layer_spec, + adapter_layer_spec=adapter_layer_spec, + language_vocab_size=128256, + language_max_sequence_length=512, + pre_process=True, + post_process=True, + fp16_lm_cross_entropy=False, + parallel_output=False, + share_embeddings_and_output_weights=True + ) + + # Load checkpoint + print(f" Loading weights from {checkpoint_path}...") + checkpoint_file = os.path.join(checkpoint_path, 'release', 'mp_rank_00', 'model_optim_rng.pt') + if os.path.exists(checkpoint_file): + checkpoint = torch.load(checkpoint_file, map_location='cpu', weights_only=False) + # Remove 'module.' prefix if present + state_dict = {} + for k, v in checkpoint['model'].items(): + new_key = k.replace('module.', '') if k.startswith('module.') else k + state_dict[new_key] = v + + missing_keys, unexpected_keys = model.load_state_dict(state_dict, strict=False) + print(f" ✓ Checkpoint loaded") + print(f" Missing keys: {len(missing_keys)} (adapter weights expected)") + print(f" Unexpected keys: {len(unexpected_keys)}") + else: + print(f" WARNING: Checkpoint file not found: {checkpoint_file}") + print(" Using randomly initialized weights") + + model = model.to(device) + model.eval() + print(" ✓ Model ready") + + # Move inputs to device + pixel_values = pixel_values.to(device) + input_ids = input_ids.to(device) + + # Generate + print("\n" + "=" * 80) + print("Generating response...") + print("=" * 80) + + with torch.no_grad(): + generated_ids = input_ids.clone() + + for step in range(args.max_new_tokens): + # Forward pass + outputs = model( + input_ids=generated_ids, + image=pixel_values, + labels=None + ) + + # Get logits for next token + logits = outputs[0] # [batch, seq_len, vocab_size] + next_token_logits = logits[:, -1, :] / args.temperature + + # Sample next token + probs = F.softmax(next_token_logits, dim=-1) + next_token = torch.multinomial(probs, num_samples=1) + + # Append to sequence + generated_ids = torch.cat([generated_ids, next_token], dim=1) + + # Check for EOS + if next_token.item() in [tokenizer.eos_token_id, 128009]: # llama3 eos + break + + # Print progress + if (step + 1) % 10 == 0: + print(f" Generated {step + 1} tokens...") + + # Decode output + output_ids = generated_ids[0, input_ids.shape[1]:] # Remove prompt + output_text = tokenizer.decode(output_ids, skip_special_tokens=True) + + print("\n" + "=" * 80) + print("RESULTS") + print("=" * 80) + print(f"Prompt: {args.prompt}") + print(f"\nGenerated ({output_ids.shape[0]} tokens):") + print(output_text) + print("=" * 80) + + print("\n✓ Inference complete!") + +if __name__ == "__main__": + main() diff --git a/aiak_megatron/megatron/legacy/fused_kernels/__init__.py b/aiak_megatron/megatron/legacy/fused_kernels/__init__.py index 74592952..7901600b 100644 --- a/aiak_megatron/megatron/legacy/fused_kernels/__init__.py +++ b/aiak_megatron/megatron/legacy/fused_kernels/__init__.py @@ -56,8 +56,20 @@ def _cpp_extention_load_helper(name, sources, extra_cuda_flags): def _get_cuda_bare_metal_version(cuda_dir): + # Handle case where cuda_dir might be None or nvcc not in expected location + if cuda_dir is None: + cuda_dir = "/usr/local/cuda" + + nvcc_path = cuda_dir + "/bin/nvcc" + if not os.path.exists(nvcc_path): + # Try alternate locations + for alt_path in ["/usr/local/cuda/bin/nvcc", "/usr/local/cuda-12.8/bin/nvcc", "/usr/local/cuda-12.1/bin/nvcc", "/usr/bin/nvcc"]: + if os.path.exists(alt_path): + nvcc_path = alt_path + break + raw_output = subprocess.check_output( - [cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True + [nvcc_path, "-V"], universal_newlines=True ) output = raw_output.split() release_idx = output.index("release") + 1 diff --git a/aiak_training_llm/train/training_utils.py b/aiak_training_llm/train/training_utils.py index bee461c2..497f197d 100644 --- a/aiak_training_llm/train/training_utils.py +++ b/aiak_training_llm/train/training_utils.py @@ -403,15 +403,26 @@ def setup_model_and_optimizer(model_provider_func, optimizer.reload_model_params() print_rank_0(f'Upcycled checkpoint saved to {args.save}') - # When using FastViT, skip checkpoint loading to avoid dimension mismatch - # FastViT outputs 3072 dims vs Qwen2-VL's 1024 dims, causing adapter incompatibility + # When using FastViT with pretrained checkpoint, we need special handling + # The checkpoint may contain incompatible language model weights (e.g., Qwen2-1.5B vs MobileLLM-140M) + # So we only load the vision tower weights if pretrained_checkpoint is specified if getattr(args, 'use_fastvit', False): print_rank_0(f'[DEBUG] FastViT enabled: use_fastvit={getattr(args, "use_fastvit", False)}') - print_rank_0(f'[DEBUG] Before skip: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') - args.load = None - args.pretrained_checkpoint = None - print_rank_0(f'[DEBUG] After skip: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') - print_rank_0('FastViT enabled: Skipping checkpoint loading, training from scratch') + print_rank_0(f'[DEBUG] Before handling: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') + + if args.pretrained_checkpoint is not None: + # We have a pretrained checkpoint - load only vision tower from HF checkpoint + print_rank_0(f'FastViT enabled: Will load vision tower from pretrained checkpoint: {args.pretrained_checkpoint}') + # Keep args.pretrained_checkpoint for vision tower loading + # Clear args.load to prevent Megatron checkpoint loading + args.load = None + else: + # No pretrained checkpoint - train vision from scratch + args.load = None + args.pretrained_checkpoint = None + print_rank_0('FastViT enabled: No pretrained checkpoint, training vision from scratch') + + print_rank_0(f'[DEBUG] After handling: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') if (args.load is not None or args.pretrained_checkpoint is not None) and not args.moe_use_upcycling: timers('load-checkpoint', log_level=0).start(barrier=True) diff --git a/examples/llava_ov_1_5/convert/convert_fastvit_hf_to_mcore.sh b/examples/llava_ov_1_5/convert/convert_fastvit_hf_to_mcore.sh new file mode 100755 index 00000000..44dc35d5 --- /dev/null +++ b/examples/llava_ov_1_5/convert/convert_fastvit_hf_to_mcore.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Convert FastViT vision tower from HuggingFace checkpoint to Megatron format +# This script extracts ONLY the vision tower weights, ignoring the language model + +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/workspace/LLaVA-OneVision-1.5}" +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" +CONVERT_CHECKPOINT_PATH="$AIAK_TRAINING_PATH/tools/convert_checkpoint" + +LOAD=$1 # HF checkpoint path (e.g., checkpoints/llava-fastvithd_1.5b_stage3) +SAVE=$2 # Megatron output path (e.g., checkpoints/fastvit_vision_mcore_tp1) +TP=${3:-1} # Tensor parallel size (default: 1) +PP=${4:-1} # Pipeline parallel size (default: 1) + +if [ -z "$LOAD" ] || [ -z "$SAVE" ]; then + echo "Usage: $0 [TP] [PP]" + echo "Example: $0 checkpoints/llava-fastvithd_1.5b_stage3 checkpoints/fastvit_vision_mcore_tp1 1 1" + exit 1 +fi + +# Convert to absolute paths if relative +if [[ "$LOAD" != /* ]]; then + LOAD="$AIAK_TRAINING_PATH/checkpoints/$LOAD" +fi +if [[ "$SAVE" != /* ]]; then + SAVE="$AIAK_TRAINING_PATH/checkpoints/$SAVE" +fi + +SAVE_VISION_MODEL=./tmp/vision-model-mcore-fastvit + +echo "==========================================" +echo "Converting FastViT vision tower" +echo "==========================================" +echo "Source (HF): $LOAD" +echo "Target (Megatron): $SAVE" +echo "Tensor Parallel: $TP" +echo "Pipeline Parallel: $PP" +echo "==========================================" + +# Extract vision tower weights from HuggingFace checkpoint +python $CONVERT_CHECKPOINT_PATH/custom/llavaov_1_5/fastvit.py \ + --load_platform=huggingface \ + --save_platform=mcore \ + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/llava-ov-1.5-4b/fastvit.json \ + --tensor_model_parallel_size=$TP \ + --load_ckpt_path=$LOAD \ + --save_ckpt_path=$SAVE_VISION_MODEL \ + --safetensors \ + --no_save_optim \ + --no_load_optim + +# Copy the converted weights to final destination +mkdir -p $SAVE +cp -r $SAVE_VISION_MODEL/release $SAVE/ + +# Create iteration marker +echo "release" > $SAVE/latest_checkpointed_iteration.txt + +# Cleanup +rm -rf $SAVE_VISION_MODEL + +echo "==========================================" +echo "✓ FastViT vision tower conversion complete!" +echo "Output: $SAVE/release" +echo "==========================================" +echo "" +echo "To use this checkpoint with MobileLLM-140M training:" +echo " 1. Set: PRETRAINED_VISION_CHECKPOINT=\"$SAVE\"" +echo " 2. Add flag: --load \$PRETRAINED_VISION_CHECKPOINT" +echo " 3. Add flag: --no-load-strict (to skip language model weights)" +echo "" diff --git a/examples/llava_ov_1_5/convert/convert_mobilellm_hf_to_mcore.sh b/examples/llava_ov_1_5/convert/convert_mobilellm_hf_to_mcore.sh new file mode 100755 index 00000000..3eed8306 --- /dev/null +++ b/examples/llava_ov_1_5/convert/convert_mobilellm_hf_to_mcore.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Convert MobileLLM-R1-140M from HuggingFace checkpoint to Megatron format + +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/workspace/LLaVA-OneVision-1.5}" +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-${AIAK_TRAINING_PATH%/}/aiak_megatron}" +CONVERT_CHECKPOINT_PATH="$AIAK_TRAINING_PATH/tools/convert_checkpoint" + +LOAD=$1 # HF checkpoint path (e.g., checkpoints/MobileLLM-R1-140M) +SAVE=$2 # Megatron output path (e.g., checkpoints/MobileLLM-R1-140M-mcore-tp1-pp1) +TP=${3:-1} # Tensor parallel size (default: 1) +PP=${4:-1} # Pipeline parallel size (default: 1) + +if [ -z "$LOAD" ] || [ -z "$SAVE" ]; then + echo "Usage: $0 [TP] [PP]" + echo "Example: $0 checkpoints/MobileLLM-R1-140M checkpoints/MobileLLM-R1-140M-mcore-tp1-pp1 1 1" + exit 1 +fi + +SAVE_LANGUAGE_MODEL=./tmp/mobilellm-language-mcore + +echo "==========================================" +echo "Converting MobileLLM-R1-140M" +echo "==========================================" +echo "Source (HF): $LOAD" +echo "Target (Megatron): $SAVE" +echo "Tensor Parallel: $TP" +echo "Pipeline Parallel: $PP" +echo "==========================================" + +# Convert language model +python $CONVERT_CHECKPOINT_PATH/model.py \ + --load_platform=huggingface \ + --save_platform=mcore \ + --common_config_path=$CONVERT_CHECKPOINT_PATH/config/mobilellm-140m.json \ + --tensor_model_parallel_size=$TP \ + --pipeline_model_parallel_size=$PP \ + --load_ckpt_path=$LOAD \ + --save_ckpt_path=$SAVE_LANGUAGE_MODEL \ + --safetensors \ + --no_save_optim \ + --no_load_optim + +# Copy the converted weights to final destination +mkdir -p $SAVE +cp -r $SAVE_LANGUAGE_MODEL/release $SAVE/ + +# Create iteration marker +echo "release" > $SAVE/latest_checkpointed_iteration.txt + +# Cleanup +rm -rf $SAVE_LANGUAGE_MODEL + +echo "==========================================" +echo "✓ MobileLLM-R1-140M conversion complete!" +echo "Output: $SAVE/release" +echo "==========================================" diff --git a/examples/llava_ov_1_5/convert/merge_mobilellm_fastvit.sh b/examples/llava_ov_1_5/convert/merge_mobilellm_fastvit.sh new file mode 100755 index 00000000..455d8a3c --- /dev/null +++ b/examples/llava_ov_1_5/convert/merge_mobilellm_fastvit.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Merge MobileLLM-R1-140M language model with FastViT vision encoder +# This creates a complete multimodal checkpoint ready for training + +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-/workspace/LLaVA-OneVision-1.5}" +CONVERT_CHECKPOINT_PATH="$AIAK_TRAINING_PATH/tools/convert_checkpoint" + +LANGUAGE_MODEL_PATH=$1 # MobileLLM language model (e.g., checkpoints/MobileLLM-R1-140M-mcore-tp1-pp1) +VISION_MODEL_PATH=$2 # FastViT vision encoder (e.g., checkpoints/llava-fastvithd_1.5b_stage3_mcore_tp1_pp1) +SAVE_PATH=$3 # Output merged checkpoint path +TP=${4:-1} # Tensor parallel size +PP=${5:-1} # Pipeline parallel size + +if [ -z "$LANGUAGE_MODEL_PATH" ] || [ -z "$VISION_MODEL_PATH" ] || [ -z "$SAVE_PATH" ]; then + echo "Usage: $0 [TP] [PP]" + echo "Example: $0 checkpoints/MobileLLM-R1-140M-mcore-tp1-pp1 checkpoints/llava-fastvithd_1.5b_stage3_mcore_tp1_pp1 checkpoints/mobilellm-fastvit-merged-tp1-pp1 1 1" + exit 1 +fi + +echo "==========================================" +echo "Merging MobileLLM + FastViT" +echo "==========================================" +echo "Language Model: $LANGUAGE_MODEL_PATH" +echo "Vision Model: $VISION_MODEL_PATH" +echo "Output: $SAVE_PATH" +echo "Tensor Parallel: $TP" +echo "Pipeline Parallel: $PP" +echo "==========================================" + +# Merge the checkpoints +# Note: We're only merging language model + vision model +# The adapter will be randomly initialized during training +echo "Merging language model and vision model..." +echo "Note: Adapter will be randomly initialized during training" + +# Create a simple Python script to merge just language and vision models +python -c " +import torch +import sys +import os + +# Load language model checkpoint +lang_ckpt = torch.load('$LANGUAGE_MODEL_PATH/release/mp_rank_00/model_optim_rng.pt', map_location='cpu', weights_only=False) +print(f'Loaded language model: {len(lang_ckpt[\"model\"])} keys') + +# Load vision model checkpoint +vis_ckpt = torch.load('$VISION_MODEL_PATH/release/mp_rank_00/model_optim_rng.pt', map_location='cpu', weights_only=False) +print(f'Loaded vision model: {len(vis_ckpt[\"model\"])} keys') + +# Merge vision model into language model +merged_model = lang_ckpt['model'].copy() +vision_key_count = 0 +for k, v in vis_ckpt['model'].items(): + merged_model[k] = v + vision_key_count += 1 + if vision_key_count <= 5: + print(f' Added: {k}') + +print(f'Merged model: {len(merged_model)} keys ({vision_key_count} vision keys)') + +# Save merged checkpoint +os.makedirs('$SAVE_PATH/release/mp_rank_00', exist_ok=True) +merged_ckpt = { + 'model': merged_model, + 'checkpoint_version': 3.0, + 'args': lang_ckpt.get('args', {}), + 'iteration': 0 +} +torch.save(merged_ckpt, '$SAVE_PATH/release/mp_rank_00/model_optim_rng.pt') +print(f'Saved merged checkpoint to $SAVE_PATH/release/mp_rank_00/model_optim_rng.pt') +" + +# Create iteration marker +echo "release" > $SAVE_PATH/latest_checkpointed_iteration.txt + +echo "==========================================" +echo "✓ Merge complete!" +echo "Output: $SAVE_PATH" +echo "==========================================" +echo "" +echo "To use this checkpoint in training:" +echo " Set: --pretrained-checkpoint \"$SAVE_PATH\"" +echo "" diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index 74621543..366d83a1 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -20,12 +20,13 @@ NSTEP="${6:-1}" # Number of training iterations (1 step with 4 examples) DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" # MobileLLM tokenizer path - use facebook/MobileLLM-R1-140M from HuggingFace TOKENIZER_PATH="${TOKENIZER_PATH:-"facebook/MobileLLM-R1-140M"}" -# Checkpoint path - for MobileLLM converted to Megatron format -# CHECKPOINT_PATH="${CHECKPOINT_PATH:-"$REPO_ROOT/checkpoints/mobilellm-140m_mcore_tp1_pp1"}" +# Pretrained checkpoint (MobileLLM-R1-140M + FastViT merged) +PRETRAINED_CHECKPOINT="${PRETRAINED_CHECKPOINT:-"$REPO_ROOT/checkpoints/mobilellm-fastvit-merged-tp1-pp1"}" echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" echo "DATA_PATH=${DATA_PATH}" echo "TOKENIZER_PATH=${TOKENIZER_PATH}" +echo "PRETRAINED_CHECKPOINT=${PRETRAINED_CHECKPOINT}" # Multi-node configuration declare -a list_ip=( @@ -115,11 +116,15 @@ DATA_ARGS=( --split 100,0,0 # Data Splitting part --num-workers 16 - # FastViT vision encoder configuration + # FastViT vision encoder configuration + # Using pretrained FastViT + MobileLLM-R1-140M (merged checkpoint) --use-fastvit - --fastvit-image-size 384 # MobileLLM is efficient, use 384 (for testing) - --vision-tower-name mobileclip_l_384 # MobileCLIP-L with 384 resolution + --fastvit-image-size 1024 # Match pretrained checkpoint resolution + --vision-tower-name mobileclip_l_1024 # MobileCLIP-L with 1024 resolution (from checkpoint) --image-aspect-ratio pad # Pad images to square aspect ratio + + # Load pretrained checkpoint (language model + vision encoder merged) + --pretrained-checkpoint "$PRETRAINED_CHECKPOINT" ) # ======================================== From b01f2d99f24f83005080e1597c61690134a2168c Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 16 Feb 2026 14:44:34 +0400 Subject: [PATCH 19/39] Fix vision config loading and tokenizer local path handling - Load FastViT config from mobileclip_l.json to get correct architecture values - Print full config objects for language, vision, and adapter - Add local_files_only parameter for HF tokenizer when loading from filesystem - Fixes vision config showing incorrect language model values --- .../llavaov_1_5/llavaov_1_5_provider.py | 54 ++++++++++++++++--- .../models/mobilellm/mobilellm_model.py | 2 +- .../tokenizer/tokenization_hf.py | 4 ++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index 01f287b4..9624cfe1 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -77,23 +77,65 @@ def rice_vl_model_provider( else: print_rank_0(f'[DEBUG PROVIDER] Using Qwen2.5 as language backbone') + print_rank_0(f'[DEBUG PROVIDER] ========== LANGUAGE CONFIG ==========') + print_rank_0(f'{language_config}') + print_rank_0(f'[DEBUG PROVIDER] ======================================') + # get vision specific config : no. of layers, hidden size, Patch size, Image resolution print_rank_0(f'[DEBUG PROVIDER] Loading vision config...') - for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): - setattr(vision_config, k, v) - print_rank_0(f'[DEBUG PROVIDER] Vision config: layers={vision_config.num_layers}, hidden={vision_config.hidden_size}') - # Add FastViT-specific configuration if enabled + # Check if using FastViT - it has its own config system if getattr(args, 'use_fastvit', False): - vision_tower_name = getattr(args, 'vision_tower_name', 'mobileclip_l_384') + # FastViT loads config from mobileclip_l.json - set TransformerConfig to match + import json + import os + + vision_tower_name = getattr(args, 'vision_tower_name', 'mobileclip_l_1024') setattr(vision_config, 'vision_tower_name', vision_tower_name) + + # Parse resolution from vision_tower_name (e.g., "mobileclip_l_1024" -> 1024) + resolution = int(vision_tower_name.split('_')[-1]) if '_' in vision_tower_name else 1024 + + # Load the actual mobileclip JSON config + json_config_path = os.path.join( + os.path.dirname(__file__), + '../fastvit/mobileclip/configs/mobileclip_l.json' + ) + with open(json_config_path, 'r') as f: + mobileclip_config = json.load(f) + + # Extract image_cfg from the JSON + image_cfg = mobileclip_config['image_cfg'] + + # Set vision_config values from mobileclip_l.json + setattr(vision_config, 'num_layers', 24) # RepMixer layers in main stage + setattr(vision_config, 'hidden_size', image_cfg['embed_dim']) # 3072 + setattr(vision_config, 'patch_size', image_cfg['patch_size']) # 64 + setattr(vision_config, 'image_size', resolution) # from vision_tower_name (1024) + setattr(vision_config, 'num_attention_heads', image_cfg['embed_dim'] // 64) # 48 + print_rank_0(f'[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: {vision_tower_name}') + print_rank_0(f'[DEBUG PROVIDER] FastViT config loaded from {json_config_path}') + print_rank_0(f'[DEBUG PROVIDER] image_cfg: embed_dim={image_cfg["embed_dim"]}, patch_size={image_cfg["patch_size"]}, model={image_cfg["model_name"]}') + print_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ==========') + print_rank_0(f'{vision_config}') + print_rank_0(f'[DEBUG PROVIDER] ================================================') + else: + # For SigLIP/Rice models, use generic vision config + for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): + setattr(vision_config, k, v) + print_rank_0(f'[DEBUG PROVIDER] Vision config (SigLIP/Rice): layers={vision_config.num_layers}, hidden={vision_config.hidden_size}') + print_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (SigLIP/Rice) ==========') + print_rank_0(f'{vision_config}') + print_rank_0(f'[DEBUG PROVIDER] ====================================================') # get adapter specific config : Projection dimension, Activation function print_rank_0(f'[DEBUG PROVIDER] Loading adapter config...') for k, v in asdict(get_adapeter_config(model_family)).items(): setattr(adapter_config, k, v) - print_rank_0(f'[DEBUG PROVIDER] Adapter config loaded') + print_rank_0(f'[DEBUG PROVIDER] ========== ADAPTER CONFIG ==========') + print_rank_0(f'{adapter_config}') + print_rank_0(f'[DEBUG PROVIDER] ======================================') # set special token ids for language model setattr(language_config, "image_token_id", 151655) diff --git a/aiak_training_llm/models/mobilellm/mobilellm_model.py b/aiak_training_llm/models/mobilellm/mobilellm_model.py index 33410343..49def08c 100644 --- a/aiak_training_llm/models/mobilellm/mobilellm_model.py +++ b/aiak_training_llm/models/mobilellm/mobilellm_model.py @@ -147,7 +147,7 @@ def __init__( if self.post_process: self.output_layer = tensor_parallel.ColumnParallelLinear( config.hidden_size, - self.vocab_size, + self.vocab_size, config=config, init_method=config.init_method, bias=False, diff --git a/aiak_training_llm/tokenizer/tokenization_hf.py b/aiak_training_llm/tokenizer/tokenization_hf.py index 921b9dfd..5fcbda6c 100644 --- a/aiak_training_llm/tokenizer/tokenization_hf.py +++ b/aiak_training_llm/tokenizer/tokenization_hf.py @@ -22,6 +22,9 @@ def __init__(self, **kwargs, ): super().__init__(name_or_path) + # Add local_files_only=True if path is a local directory (starts with / or ./) + import os + is_local_path = os.path.exists(name_or_path) self.tokenizer = AutoTokenizer.from_pretrained( name_or_path, use_fast=use_fast_tokenizer, @@ -29,6 +32,7 @@ def __init__(self, split_special_tokens=split_special_tokens, model_max_length=model_max_length, trust_remote_code=True, + local_files_only=is_local_path, **kwargs, ) From 5e16565a66fa48108e0f1451b59ceed911ae7c20 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 16 Feb 2026 15:18:37 +0400 Subject: [PATCH 20/39] Improve local path detection for tokenizer loading --- aiak_training_llm/tokenizer/tokenization_hf.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/aiak_training_llm/tokenizer/tokenization_hf.py b/aiak_training_llm/tokenizer/tokenization_hf.py index 5fcbda6c..c17ee1d8 100644 --- a/aiak_training_llm/tokenizer/tokenization_hf.py +++ b/aiak_training_llm/tokenizer/tokenization_hf.py @@ -22,9 +22,15 @@ def __init__(self, **kwargs, ): super().__init__(name_or_path) - # Add local_files_only=True if path is a local directory (starts with / or ./) + # Add local_files_only=True if path is a local directory (absolute or relative path) import os - is_local_path = os.path.exists(name_or_path) + # Detect local path: starts with / or ./ or ../ or contains path separator + is_local_path = ( + os.path.isabs(name_or_path) or + name_or_path.startswith('./') or + name_or_path.startswith('../') or + os.path.exists(name_or_path) + ) self.tokenizer = AutoTokenizer.from_pretrained( name_or_path, use_fast=use_fast_tokenizer, From fcadc4c190f0bbc307bf24e154a02d2b5f9b1b62 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 16 Feb 2026 16:03:09 +0400 Subject: [PATCH 21/39] Skip fused CUDA kernels compilation on ROCm/AMD platforms --- .gitattributes | 1 + aiak_megatron/megatron/legacy/fused_kernels/__init__.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.gitattributes b/.gitattributes index 7c7d6875..f0ee6826 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ *.bin filter=lfs diff=lfs merge=lfs -text *.safetensors filter=lfs diff=lfs merge=lfs -text *.pt filter=lfs diff=lfs merge=lfs -text +checkpoints/LLaVA-OneVision-1.5-4B-stage0/** filter=lfs diff=lfs merge=lfs -text diff --git a/aiak_megatron/megatron/legacy/fused_kernels/__init__.py b/aiak_megatron/megatron/legacy/fused_kernels/__init__.py index 7901600b..a9f77ba9 100644 --- a/aiak_megatron/megatron/legacy/fused_kernels/__init__.py +++ b/aiak_megatron/megatron/legacy/fused_kernels/__init__.py @@ -15,6 +15,12 @@ def load(args): + + # Skip fused kernels on ROCm (AMD GPUs) + import torch + if torch.version.hip is not None: + print("Skipping fused kernels compilation on ROCm/AMD platform") + return # Check if cuda 11 is installed for compute capability 8.0 cc_flag = [] From b16cd6c72718b396033e1bd74520eb493a35ced2 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 16 Feb 2026 17:01:05 +0400 Subject: [PATCH 22/39] Skip fused CUDA kernels compilation on ROCm/AMD platforms and add model printing --- aiak_training_llm/train/training_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/aiak_training_llm/train/training_utils.py b/aiak_training_llm/train/training_utils.py index 497f197d..849f6358 100644 --- a/aiak_training_llm/train/training_utils.py +++ b/aiak_training_llm/train/training_utils.py @@ -244,6 +244,16 @@ def pretrain( timers('model-and-optimizer-setup').stop() print_datetime('after model, optimizer, and learning rate scheduler are built') config = get_model_config(model[0]) + + # Print full model structure + print_rank_0('FULL MODEL STRUCTURE:') + if isinstance(model, list): + for idx, m in enumerate(model): + print_rank_0(f'\n--- Model {idx} (pipeline rank {idx}) ---') + print_rank_0(m) + else: + print_rank_0(model) + print_rank_0('=' * 80) # Data stuff. timers('train/valid/test-data-iterators-setup', log_level=0).start(barrier=True) @@ -360,6 +370,7 @@ def setup_model_and_optimizer(model_provider_func, timers = get_timers() model = get_model(model_provider_func, model_type) + print(model) unwrapped_model = unwrap_model(model) kwargs = {} From fc435567ad0bb6d2088f6d02a044975ddac3dd84 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 23 Feb 2026 10:27:24 +0400 Subject: [PATCH 23/39] Fix ROCm training issues: add None checks for debug prints and parameter trainability status logging --- Stage1/alignment_rocm.sh | 19 ++++++++------- .../models/llavaov_1_5/llavaov_1_5_model.py | 6 +++++ aiak_training_llm/train/training_utils.py | 24 +++++++++++++++++++ .../stage_1_alignment_mobilellm_140m.sh | 2 +- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh index da2bdf45..d6bccd05 100755 --- a/Stage1/alignment_rocm.sh +++ b/Stage1/alignment_rocm.sh @@ -22,13 +22,6 @@ export ROCM_HOME=${ROCM_HOME:-/opt/rocm} export PATH="${ROCM_HOME}/bin:${PATH}" export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH}" -export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0} - -# Force HuggingFace offline mode to use local files only -export HF_HUB_OFFLINE=1 -export TRANSFORMERS_OFFLINE=1 - - # RCCL/NCCL runtime hints (tune as needed) export NCCL_DEBUG=${NCCL_DEBUG:-WARN} export NCCL_COLLNET_ENABLE=${NCCL_COLLNET_ENABLE:-0} @@ -61,7 +54,7 @@ export AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" export AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" export DATA_PATH="${DATA_PATH:-$REPO_ROOT/data/LLaVA-558K-Webdataset}" export TOKENIZER_PATH="${TOKENIZER_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0}" -export CHECKPOINT_PATH="${CHECKPOINT_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1}" +export CHECKPOINT_PATH="${CHECKPOINT_PATH:-$REPO_ROOT/checkpoints/mobilellm-fastvit-merged-tp1-pp1}" # Add megatron to PYTHONPATH so imports work export PYTHONPATH="${AIAK_MAGATRON_PATH}:${AIAK_TRAINING_PATH}:${PYTHONPATH}" @@ -77,5 +70,13 @@ echo "PYTHONPATH=${PYTHONPATH}" export WANDB_API_KEY="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" export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="fastvit_integration" + +# Fix HIP_VISIBLE_DEVICES to use both GPUs +export HIP_VISIBLE_DEVICES=0,1 + +# Set number of GPUs to match SLURM allocation (2 GPUs) +export GPUS_PER_NODE=2 + +# Run training with global batch size matching 2 GPUs bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh - 2,1 Top + 14,1 Top diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index bb1fd958..963db996 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -342,6 +342,12 @@ def forward( # ) # print_rank_0(input_ids) # print_rank_0(position_ids) + print("training forward step: input_ids shape {}, images shape {}, image_grid_thw shape {}, attn_mask_type {}, position_ids shape {}".format( + input_ids.shape if input_ids is not None else None, + images.shape if images is not None else None, + image_grid_thw.shape if image_grid_thw is not None else None, + attn_mask_type, + position_ids.shape if position_ids is not None else None)) use_inference_kv_cache = ( inference_params is not None and "image_tokens_count" in inference_params.key_value_memory_dict diff --git a/aiak_training_llm/train/training_utils.py b/aiak_training_llm/train/training_utils.py index 849f6358..1eadc24b 100644 --- a/aiak_training_llm/train/training_utils.py +++ b/aiak_training_llm/train/training_utils.py @@ -246,7 +246,9 @@ def pretrain( config = get_model_config(model[0]) # Print full model structure + print_rank_0('=' * 80) print_rank_0('FULL MODEL STRUCTURE:') + print_rank_0('=' * 80) if isinstance(model, list): for idx, m in enumerate(model): print_rank_0(f'\n--- Model {idx} (pipeline rank {idx}) ---') @@ -254,6 +256,28 @@ def pretrain( else: print_rank_0(model) print_rank_0('=' * 80) + + # Print parameter trainability status + print_rank_0('\n' + '=' * 80) + print_rank_0('PARAMETER TRAINABILITY STATUS:') + print_rank_0('=' * 80) + model_to_check = model[0] if isinstance(model, list) else model + trainable_params = 0 + frozen_params = 0 + for name, param in model_to_check.named_parameters(): + status = "TRAINABLE" if param.requires_grad else "FROZEN" + print_rank_0(f"{status:12s} | {name:80s} | shape: {str(tuple(param.shape)):30s} | dtype: {param.dtype}") + if param.requires_grad: + trainable_params += param.numel() + else: + frozen_params += param.numel() + + print_rank_0('=' * 80) + print_rank_0(f'Total trainable parameters: {trainable_params:,} ({trainable_params/1e6:.2f}M)') + print_rank_0(f'Total frozen parameters: {frozen_params:,} ({frozen_params/1e6:.2f}M)') + print_rank_0(f'Total parameters: {trainable_params + frozen_params:,} ({(trainable_params + frozen_params)/1e6:.2f}M)') + print_rank_0(f'Trainable percentage: {100 * trainable_params / (trainable_params + frozen_params):.2f}%') + print_rank_0('=' * 80) # Data stuff. timers('train/valid/test-data-iterators-setup', log_level=0).start(barrier=True) diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index 366d83a1..7b820236 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -133,7 +133,7 @@ DATA_ARGS=( TRAINING_ARGS=( --training-phase sft # supervised fine-tuning --chat-template llama3 # MobileLLM uses Llama3 tokenizer - --trainable-modules language_model adapter # Train language model + adapter (vision frozen) + --trainable-modules language_model adapter # Stage 1: Train language model + adapter (vision frozen) --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" --no-rope-fusion From dcc698fee08c8d94f113702c6671f5e4150d5865 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Mon, 23 Feb 2026 11:20:25 +0400 Subject: [PATCH 24/39] Fix ROCm compatibility: skip CUDA arch version checks on AMD GPUs --- aiak_megatron/megatron/training/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aiak_megatron/megatron/training/utils.py b/aiak_megatron/megatron/training/utils.py index 61b18609..97199e9e 100644 --- a/aiak_megatron/megatron/training/utils.py +++ b/aiak_megatron/megatron/training/utils.py @@ -381,6 +381,9 @@ def print_rank_last(message): def get_device_arch_version(): """Returns GPU arch version (8: Ampere, 9: Hopper, 10: Blackwell, ...)""" + # On AMD/ROCm, return a high version to skip CUDA-specific checks + if torch.version.hip is not None: + return 99 # Return high value to skip CUDA_DEVICE_MAX_CONNECTIONS checks return torch.cuda.get_device_properties(torch.device("cuda:0")).major From 457531ae0022883a40b0520deb7ec56bab23cf0e Mon Sep 17 00:00:00 2001 From: RanaZay Date: Wed, 11 Mar 2026 10:56:01 +0400 Subject: [PATCH 25/39] Fix multimodal token alignment and add unified debug instrumentation --- Stage1/alignment.sh | 2 +- Stage1/alignment_rocm.sh | 29 +-- aiak_megatron/megatron/training/training.py | 15 ++ .../data/multimodal/qwen2vl_task_encoder.py | 59 ++++-- .../models/llavaov_1_5/llavaov_1_5_model.py | 61 +++++- .../llavaov_1_5/llavaov_1_5_provider.py | 38 +++- aiak_training_llm/tokenizer/special_tokens.py | 30 +++ aiak_training_llm/tokenizer/tokenizer.py | 21 +++ .../train/pretrain/pretrain_llavaov_1_5.py | 173 +++++++++++++++++- .../train/sft/sft_llavaov_1_5_vl.py | 15 ++ aiak_training_llm/train/training_utils.py | 14 ++ .../stage_1_alignment_llava_ov_4b.sh | 4 +- .../stage_1_alignment_mobilellm_140m.sh | 2 +- 13 files changed, 412 insertions(+), 51 deletions(-) create mode 100644 aiak_training_llm/tokenizer/special_tokens.py diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index d2878bae..fefc904b 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -56,7 +56,7 @@ export WANDB_API_KEY="wandb_v1_5y5JqALBMdHhru8CR1gOLflJlRj_O8BG2XRb0S2x0TJVqW1xA export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="mobilellm_integration" -export CUDA_VISIBLE_DEVICES=0,1,2,3 +export CUDA_VISIBLE_DEVICES=2,3,4,5 export GPUS_PER_NODE=4 export MASTER_PORT=26000 diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh index d6bccd05..7ca3b796 100755 --- a/Stage1/alignment_rocm.sh +++ b/Stage1/alignment_rocm.sh @@ -4,7 +4,7 @@ #SBATCH --nodes=1 #SBATCH --ntasks-per-node=1 #SBATCH --cpus-per-task=128 -#SBATCH --gres=gpu:2 +#SBATCH --gres=gpu:4 #SBATCH --time=72:00:00 #SBATCH --mem=230G #SBATCH --qos=skqos @@ -22,6 +22,14 @@ export ROCM_HOME=${ROCM_HOME:-/opt/rocm} export PATH="${ROCM_HOME}/bin:${PATH}" export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH}" +export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3} + +# Force HuggingFace offline mode to use local files only +# Commented out to allow downloading MobileLLM tokenizer on first run +# export HF_HUB_OFFLINE=1 +# export TRANSFORMERS_OFFLINE=1 + + # RCCL/NCCL runtime hints (tune as needed) export NCCL_DEBUG=${NCCL_DEBUG:-WARN} export NCCL_COLLNET_ENABLE=${NCCL_COLLNET_ENABLE:-0} @@ -53,8 +61,9 @@ echo "=== END ENV CHECK ===" export AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" export AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" export DATA_PATH="${DATA_PATH:-$REPO_ROOT/data/LLaVA-558K-Webdataset}" -export TOKENIZER_PATH="${TOKENIZER_PATH:-$REPO_ROOT/checkpoints/LLaVA-OneVision-1.5-4B-stage0}" -export CHECKPOINT_PATH="${CHECKPOINT_PATH:-$REPO_ROOT/checkpoints/mobilellm-fastvit-merged-tp1-pp1}" +# Use MobileLLM tokenizer and checkpoint (let stage_1_alignment_mobilellm_140m.sh set defaults) +export TOKENIZER_PATH="${TOKENIZER_PATH:-facebook/MobileLLM-R1-140M}" +export PRETRAINED_CHECKPOINT="${PRETRAINED_CHECKPOINT:-$REPO_ROOT/checkpoints/mobilellm-fastvit-merged-tp1-pp1}" # Add megatron to PYTHONPATH so imports work export PYTHONPATH="${AIAK_MAGATRON_PATH}:${AIAK_TRAINING_PATH}:${PYTHONPATH}" @@ -62,7 +71,7 @@ echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" echo "AIAK_MAGATRON_PATH=${AIAK_MAGATRON_PATH}" echo "DATA_PATH=${DATA_PATH}" echo "TOKENIZER_PATH=${TOKENIZER_PATH}" -echo "CHECKPOINT_PATH=${CHECKPOINT_PATH}" +echo "PRETRAINED_CHECKPOINT=${PRETRAINED_CHECKPOINT}" echo "SLURM_NODELIST=${SLURM_NODELIST}" echo "PYTHONPATH=${PYTHONPATH}" @@ -70,13 +79,5 @@ echo "PYTHONPATH=${PYTHONPATH}" export WANDB_API_KEY="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" export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="fastvit_integration" - -# Fix HIP_VISIBLE_DEVICES to use both GPUs -export HIP_VISIBLE_DEVICES=0,1 - -# Set number of GPUs to match SLURM allocation (2 GPUs) -export GPUS_PER_NODE=2 - -# Run training with global batch size matching 2 GPUs -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh - 14,1 Top +# bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh \ No newline at end of file diff --git a/aiak_megatron/megatron/training/training.py b/aiak_megatron/megatron/training/training.py index 0a6d4b82..78f1e4d4 100644 --- a/aiak_megatron/megatron/training/training.py +++ b/aiak_megatron/megatron/training/training.py @@ -413,6 +413,21 @@ def pretrain( iteration = 0 if args.do_train and args.train_iters > 0: + # Print parameter trainability status + print_rank_0('\n' + '=' * 80) + print_rank_0("training with the following parameter status:") + print_rank_0('=' * 80) + model_to_check = model[0] if isinstance(model, list) else model + trainable_params = 0 + frozen_params = 0 + for name, param in model_to_check.named_parameters(): + status = "TRAINABLE" if param.requires_grad else "FROZEN" + print_rank_0(f"{status:12s} | {name:80s} | shape: {str(tuple(param.shape)):30s} | dtype: {param.dtype}") + if param.requires_grad: + trainable_params += param.numel() + else: + frozen_params += param.numel() + iteration, num_floating_point_operations_so_far = train( forward_step_func, model, optimizer, opt_param_scheduler, diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 214f8828..1f7c5aae 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -29,7 +29,8 @@ from aiak_training_llm.data.multimodal import MultiMixQASample from aiak_training_llm.data.multimodal.length_sort_dataset import LengthPoolSortDataset -from aiak_training_llm.utils import constants, get_chat_template +from aiak_training_llm.utils import constants, get_chat_template, get_tokenizer +from aiak_training_llm.tokenizer.special_tokens import ensure_multimodal_special_tokens from .task_encoder import (ImageTaskBatchPacked, ImageTaskSample, ImageTaskSamplePacked, TaskEncoder) @@ -143,11 +144,19 @@ def __init__(self, args): # For FastVLM: Load only the language model tokenizer (e.g., MobileLLM) # No need for a full processor since FastViT handles vision separately from transformers import AutoTokenizer - self.tokenizer_hf = AutoTokenizer.from_pretrained( - self.args.hf_tokenizer_path, - trust_remote_code=True, - local_files_only=False - ) + shared_tokenizer = get_tokenizer() + if shared_tokenizer is not None and hasattr(shared_tokenizer, "hf_tokenizer"): + self.tokenizer_hf = shared_tokenizer.hf_tokenizer() + print("Using shared training tokenizer in FastVLM mode") + else: + self.tokenizer_hf = AutoTokenizer.from_pretrained( + self.args.hf_tokenizer_path, + trust_remote_code=True, + local_files_only=False, + ) + + added_mm_tokens = ensure_multimodal_special_tokens(self.tokenizer_hf) + print(f"Ensured multimodal special tokens for FastVLM tokenizer; added={added_mm_tokens}") # Set padding token if not present (required for batch processing) if self.tokenizer_hf.pad_token is None: @@ -196,6 +205,7 @@ def __getattr__(self, name): # Image processing parameters self.min_pixels = args.min_pixels self.max_pixels = args.max_pixels + self._logged_multimodal_token_debug_once = False def _reisize_video(self, vision: VideoData, image_factor=28, frame_factor=2): """ Resize video: frame number, height, width """ @@ -287,6 +297,29 @@ def _process(self, image, text): ) input_ids = text_inputs['input_ids'][0] attn_mask = text_inputs['attention_mask'][0].logical_not() + fastvit_tokenizer = self.processor.tokenizer + vision_start_id, img_pad_id, vision_end_id = fastvit_tokenizer.convert_tokens_to_ids([ + VISION_TAGS[0], + IMAGE_TOKEN, + VISION_TAGS[1] + ]) + if vision_start_id is None or img_pad_id is None or vision_end_id is None: + vocab = fastvit_tokenizer.get_vocab() + if img_pad_id is None: + img_pad_id = vocab.get(IMAGE_TOKEN) + if vision_start_id is None: + vision_start_id = vocab.get(VISION_TAGS[0]) + if vision_end_id is None: + vision_end_id = vocab.get(VISION_TAGS[1]) + if not self._logged_multimodal_token_debug_once: + img_count = int((input_ids == img_pad_id).sum().item()) if img_pad_id is not None else 0 + vstart_count = int((input_ids == vision_start_id).sum().item()) if vision_start_id is not None else 0 + print( + f"[DEBUG PREPROCESS TOKENS] image_token='{IMAGE_TOKEN}' id={img_pad_id}, " + f"vision_start='{VISION_TAGS[0]}' id={vision_start_id}, vision_end='{VISION_TAGS[1]}' id={vision_end_id}, " + f"counts_in_input_ids: image={img_count}, vision_start={vstart_count}" + ) + self._logged_multimodal_token_debug_once = True # Process image with FastVLM's preprocessing utilities # Default to 'pad' aspect ratio (expand to square with padding) @@ -322,11 +355,6 @@ def _process(self, image, text): # Create target tensor (same as Qwen2-VL path) target = input_ids.clone() - vision_start_id, img_pad_id, vision_end_id = self.tokenizer.convert_tokens_to_ids([ - VISION_TAGS[0], - IMAGE_TOKEN, - VISION_TAGS[1] - ]) target[target == vision_start_id] = IGNORE_INDEX target[target == img_pad_id] = IGNORE_INDEX target[target == vision_end_id] = IGNORE_INDEX @@ -354,6 +382,15 @@ def _process(self, image, text): IMAGE_TOKEN, VISION_TAGS[1] ]) + if not self._logged_multimodal_token_debug_once: + img_count = int((input_ids == img_pad_id).sum().item()) if img_pad_id is not None else 0 + vstart_count = int((input_ids == vision_start_id).sum().item()) if vision_start_id is not None else 0 + print( + f"[DEBUG PREPROCESS TOKENS] image_token='{IMAGE_TOKEN}' id={img_pad_id}, " + f"vision_start='{VISION_TAGS[0]}' id={vision_start_id}, vision_end='{VISION_TAGS[1]}' id={vision_end_id}, " + f"counts_in_input_ids: image={img_count}, vision_start={vstart_count}" + ) + self._logged_multimodal_token_debug_once = True target[target == vision_start_id] = IGNORE_INDEX target[target == img_pad_id] = IGNORE_INDEX target[target == vision_end_id] = IGNORE_INDEX diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index 963db996..8090dfee 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -23,6 +23,7 @@ from aiak_training_llm.models.mobilellm import MobileLLMModel from aiak_training_llm.models.qwen_vl.adapter import Adapter from aiak_training_llm.models.qwen_vl.utils import get_inputs_on_this_cp_rank +from aiak_training_llm.utils import print_rank_0 def _rotate_half(x): @@ -160,6 +161,7 @@ def __init__( self.vision_model = None self.adapter = None self.language_model = None + self._big_debug_forward_step = 0 # define the vision model and the projection from vision model outputs to language model inputs. if self.add_encoder: @@ -342,12 +344,17 @@ def forward( # ) # print_rank_0(input_ids) # print_rank_0(position_ids) - print("training forward step: input_ids shape {}, images shape {}, image_grid_thw shape {}, attn_mask_type {}, position_ids shape {}".format( - input_ids.shape if input_ids is not None else None, - images.shape if images is not None else None, - image_grid_thw.shape if image_grid_thw is not None else None, - attn_mask_type, - position_ids.shape if position_ids is not None else None)) + self._big_debug_forward_step += 1 + debug_step = self._big_debug_forward_step + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"input_ids={tuple(input_ids.shape) if input_ids is not None else None}, " + f"images={tuple(images.shape) if images is not None else None}, " + f"image_grid_thw={tuple(image_grid_thw.shape) if image_grid_thw is not None else None}, " + f"labels={tuple(labels.shape) if labels is not None else None}, " + f"attn_mask_type={attn_mask_type}, " + f"position_ids={tuple(position_ids.shape) if position_ids is not None else None}" + ) use_inference_kv_cache = ( inference_params is not None and "image_tokens_count" in inference_params.key_value_memory_dict @@ -360,9 +367,25 @@ def forward( if images is not None: image_embeddings, window_index = self.vision_model(images, \ grid_thw=image_grid_thw) # [img_len, h_vision] + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"vision_out shape={tuple(image_embeddings.shape)}, dtype={image_embeddings.dtype}, " + f"requires_grad={image_embeddings.requires_grad}, window_index_shape=" + f"{tuple(window_index.shape) if window_index is not None else None}" + ) image_embeddings = self.adapter(image_embeddings, window_index) + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"adapter_out shape={tuple(image_embeddings.shape)}, dtype={image_embeddings.dtype}, " + f"requires_grad={image_embeddings.requires_grad}" + ) n_image_tokens = (input_ids == self.config.image_token_id).sum().item() n_image_features = image_embeddings.shape[0] + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"image_token_id={self.config.image_token_id}, token_count={n_image_tokens}, " + f"feature_count={n_image_features}" + ) if n_image_tokens != n_image_features: # raise ValueError( # f"Image features {n_image_features} != image tokens {n_image_tokens}" @@ -430,13 +453,19 @@ def forward( language_embeddings = self.language_model.embedding( input_ids=input_ids, position_ids=None ) # [text_seq_len, b, h_language] + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"language_embeddings shape={tuple(language_embeddings.shape)}, dtype={language_embeddings.dtype}, " + f"requires_grad={language_embeddings.requires_grad}" + ) # If running inference, we can skip image token computation if they were computed already # earlier for this sample. if use_inference_kv_cache or (images is None and pixel_values_videos is None): combined_embeddings = language_embeddings else: - if images is not None and self.config.image_token_id in input_ids: + combined_embeddings = language_embeddings + if images is not None and (input_ids == self.config.image_token_id).any(): image_token_id = self.config.image_token_id images_mask = ( (input_ids == image_token_id).transpose(0, 1) @@ -445,9 +474,15 @@ def forward( .to(language_embeddings.device) ) image_embeddings = image_embeddings.to(language_embeddings.device, language_embeddings.dtype) - combined_embeddings = language_embeddings.masked_scatter(images_mask, image_embeddings) + combined_embeddings = combined_embeddings.masked_scatter(images_mask, image_embeddings) + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"image_inserted mask_true={int(images_mask.sum().item())}, " + f"combined_embeddings shape={tuple(combined_embeddings.shape)}, " + f"requires_grad={combined_embeddings.requires_grad}" + ) - if pixel_values_videos is not None and self.config.video_token_id in input_ids: + if pixel_values_videos is not None and (input_ids == self.config.video_token_id).any(): video_token_id = self.config.video_token_id videos_mask = ( (input_ids == video_token_id).transpose(0, 1) @@ -456,7 +491,7 @@ def forward( .to(language_embeddings.device) ) video_embeddings = video_embeddings.to(language_embeddings.device, language_embeddings.dtype) - combined_embeddings = language_embeddings.masked_scatter(videos_mask, video_embeddings) + combined_embeddings = combined_embeddings.masked_scatter(videos_mask, video_embeddings) if self.config.context_parallel_size > 1: combined_embeddings = get_inputs_on_this_cp_rank(combined_embeddings) @@ -481,6 +516,12 @@ def forward( extra_block_kwargs={}, ) + print_rank_0( + f"[BIG DEBUG][FORWARD][STEP {debug_step}] " + f"model_output shape={tuple(output.shape) if hasattr(output, 'shape') else None}, " + f"dtype={getattr(output, 'dtype', None)}, requires_grad={getattr(output, 'requires_grad', None)}" + ) + return output diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index 9624cfe1..c440bc92 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -11,7 +11,7 @@ get_adapeter_config, get_vision_config) # MobileLLM config is now in llavaov_1_5_config.py and applied via args from aiak_training_llm.utils import (build_transformer_config, get_args, - print_rank_0) + print_rank_0, get_tokenizer) from aiak_training_llm.utils.constants import VisionLanguageModelFamilies from megatron.core import mpu from megatron.core.transformer.spec_utils import import_module @@ -137,9 +137,39 @@ def rice_vl_model_provider( print_rank_0(f'{adapter_config}') print_rank_0(f'[DEBUG PROVIDER] ======================================') - # set special token ids for language model - setattr(language_config, "image_token_id", 151655) - setattr(language_config, "video_token_id", 151656) + # set special token ids for language model using the shared runtime tokenizer + image_token_id = 151655 + video_token_id = 151656 + try: + tokenizer = get_tokenizer() + image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>") + video_token_id = tokenizer.convert_tokens_to_ids("<|video_pad|>") + + if image_token_id is None or video_token_id is None: + vocab = getattr(tokenizer, "vocab", None) + if isinstance(vocab, dict): + if image_token_id is None: + image_token_id = vocab.get("<|image_pad|>") + if video_token_id is None: + video_token_id = vocab.get("<|video_pad|>") + + if image_token_id is None: + image_token_id = 151655 + if video_token_id is None: + video_token_id = 151656 + + print_rank_0( + f"[DEBUG PROVIDER] Resolved vision token ids from tokenizer: " + f"image_token_id={image_token_id}, video_token_id={video_token_id}" + ) + except Exception as e: + print_rank_0( + f"[WARN PROVIDER] Failed to resolve vision token ids from tokenizer " + f"({e}); fallback to defaults image_token_id={image_token_id}, video_token_id={video_token_id}" + ) + + setattr(language_config, "image_token_id", int(image_token_id)) + setattr(language_config, "video_token_id", int(video_token_id)) #Handle pipeline parallelism diff --git a/aiak_training_llm/tokenizer/special_tokens.py b/aiak_training_llm/tokenizer/special_tokens.py new file mode 100644 index 00000000..a2de87ca --- /dev/null +++ b/aiak_training_llm/tokenizer/special_tokens.py @@ -0,0 +1,30 @@ +"""Shared multimodal special-token helpers.""" + +from typing import List, Optional + +MM_SPECIAL_TOKENS: List[str] = [ + "<|vision_start|>", + "<|vision_end|>", + "<|image_pad|>", + "<|video_pad|>", +] + + +def ensure_multimodal_special_tokens(hf_tokenizer) -> int: + """Ensure multimodal tokens exist in tokenizer; returns number of newly added tokens.""" + missing = [token for token in MM_SPECIAL_TOKENS if hf_tokenizer.convert_tokens_to_ids(token) is None] + if not missing: + return 0 + return hf_tokenizer.add_special_tokens( + {"additional_special_tokens": missing}, + replace_additional_special_tokens=False, + ) + + +def get_mm_token_id(hf_tokenizer, token: str) -> Optional[int]: + """Resolve token id robustly from convert() then vocab fallback.""" + token_id = hf_tokenizer.convert_tokens_to_ids(token) + if token_id is None: + vocab = hf_tokenizer.get_vocab() + token_id = vocab.get(token) + return token_id diff --git a/aiak_training_llm/tokenizer/tokenizer.py b/aiak_training_llm/tokenizer/tokenizer.py index 3c7bb865..97b00ead 100644 --- a/aiak_training_llm/tokenizer/tokenizer.py +++ b/aiak_training_llm/tokenizer/tokenizer.py @@ -11,6 +11,7 @@ from aiak_training_llm.utils import constants, print_rank_0 from .tokenization_hf import AutoTokenizerFromHF +from .special_tokens import ensure_multimodal_special_tokens, MM_SPECIAL_TOKENS if TYPE_CHECKING: @@ -66,6 +67,26 @@ def build_tokenizer(args, chat_template: Optional["ChatTemplate"] = None) -> Opt model_max_length=args.seq_length, split_special_tokens=args.split_special_tokens) + if ( + args.model_family in constants.VisionLanguageModelFamilies.names() + or args.model_family in constants.VideoLanguageModelFamilies.names() + ): + added_mm_tokens = ensure_multimodal_special_tokens(tokenizer.hf_tokenizer()) + print_rank_0( + f"INFO: ensured multimodal special tokens {MM_SPECIAL_TOKENS}; added={added_mm_tokens}", + args.rank, + ) + current_tokenizer_vocab_size = tokenizer.vocab_size + if getattr(args, "vocab_size_in_config_file", None) is not None and \ + current_tokenizer_vocab_size > args.vocab_size_in_config_file: + print_rank_0( + f"WARNING: tokenizer vocab size increased from config size " + f"{args.vocab_size_in_config_file} to {current_tokenizer_vocab_size}; " + f"updating args.vocab_size_in_config_file to avoid embedding index OOB.", + args.rank, + ) + args.vocab_size_in_config_file = current_tokenizer_vocab_size + if args.additional_special_tokens is not None: added_tokens = tokenizer.add_special_tokens( dict(additional_special_tokens=args.additional_special_tokens), diff --git a/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py b/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py index c9fe9136..b8624b5c 100644 --- a/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py +++ b/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py @@ -30,10 +30,118 @@ stimer = StragglerDetector() -# TODO: get token id from tokenizer -image_token_id = 151655 -video_token_id = 151656 -vision_start_token_id = 151652 +# Resolve vision token ids lazily from the active tokenizer (works for Qwen and MobileLLM tokenizers) +image_token_id = None +video_token_id = None +vision_start_token_id = None +_logged_token_presence_once = False +_big_debug_step = 0 + + +def _log_big_batch_debug( + step_id: int, + tokens: torch.Tensor, + labels: torch.Tensor, + loss_mask: torch.Tensor, + attn_mask: Optional[torch.Tensor], + imgs: Optional[torch.Tensor], + image_grid_thw: Optional[torch.Tensor], + cu_lengths: torch.Tensor, + max_lengths: torch.Tensor, + has_image: bool, + has_video: bool, +): + sample_idx = 0 + sample_tokens = tokens[sample_idx] + sample_labels = labels[sample_idx] + sample_loss_mask = loss_mask[sample_idx] + + image_token_count = int((tokens == image_token_id).sum().item()) if image_token_id is not None else 0 + video_token_count = int((tokens == video_token_id).sum().item()) if video_token_id is not None else 0 + vision_start_count = int((tokens == vision_start_token_id).sum().item()) if vision_start_token_id is not None else 0 + sample_valid_label_count = int((sample_labels != -100).sum().item()) + + sample_decoded = "" + try: + tokenizer = get_tokenizer() + ids_for_decode = sample_tokens[:128].tolist() + if hasattr(tokenizer, "decode"): + sample_decoded = tokenizer.decode(ids_for_decode) + elif hasattr(tokenizer, "detokenize"): + sample_decoded = tokenizer.detokenize(ids_for_decode) + except Exception as exc: + sample_decoded = f"" + + img_stats = "None" + if imgs is not None: + img_stats = ( + f"shape={tuple(imgs.shape)}, dtype={imgs.dtype}, " + f"min={float(imgs.min().item()):.4f}, max={float(imgs.max().item()):.4f}, " + f"mean={float(imgs.mean().item()):.4f}" + ) + + attn_stats = "None" + if attn_mask is not None: + attn_stats = ( + f"shape={tuple(attn_mask.shape)}, dtype={attn_mask.dtype}, " + f"masked={(attn_mask == True).sum().item()}, unmasked={(attn_mask == False).sum().item()}" + ) + + print_rank_0("\n" + "=" * 120) + print_rank_0(f"[BIG DEBUG][BATCH][STEP {step_id}]") + print_rank_0( + f"tokens.shape={tuple(tokens.shape)} dtype={tokens.dtype} | " + f"labels.shape={tuple(labels.shape)} dtype={labels.dtype} | " + f"loss_mask.shape={tuple(loss_mask.shape)} dtype={loss_mask.dtype}" + ) + print_rank_0( + f"token_ids: image={image_token_id}, video={video_token_id}, vision_start={vision_start_token_id} | " + f"counts: image={image_token_count}, video={video_token_count}, vision_start={vision_start_count}" + ) + print_rank_0( + f"has_image={has_image}, has_video={has_video} | imgs={img_stats} | " + f"image_grid_thw={tuple(image_grid_thw.shape) if image_grid_thw is not None else None}" + ) + print_rank_0( + f"attn_mask={attn_stats} | cu_lengths.shape={tuple(cu_lengths.shape)} values={cu_lengths[0].tolist()} | " + f"max_lengths.shape={tuple(max_lengths.shape)} values={max_lengths[0].tolist()}" + ) + print_rank_0( + f"sample[{sample_idx}] first_64_tokens={sample_tokens[:64].tolist()} | " + f"first_64_labels={sample_labels[:64].tolist()}" + ) + print_rank_0( + f"sample[{sample_idx}] valid_label_count={sample_valid_label_count} | " + f"loss_mask_active={(sample_loss_mask == 1).sum().item()} | decoded_prefix={sample_decoded}" + ) + print_rank_0("=" * 120) + + +def _ensure_vision_token_ids(): + """Resolve multimodal special token ids once from the active tokenizer.""" + global image_token_id, video_token_id, vision_start_token_id + if image_token_id is not None and video_token_id is not None and vision_start_token_id is not None: + return + + tokenizer = get_tokenizer() + image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>") + video_token_id = tokenizer.convert_tokens_to_ids("<|video_pad|>") + vision_start_token_id = tokenizer.convert_tokens_to_ids("<|vision_start|>") + + if image_token_id is None or video_token_id is None or vision_start_token_id is None: + vocab = getattr(tokenizer, "vocab", None) + if isinstance(vocab, dict): + if image_token_id is None: + image_token_id = vocab.get("<|image_pad|>") + if video_token_id is None: + video_token_id = vocab.get("<|video_pad|>") + if vision_start_token_id is None: + vision_start_token_id = vocab.get("<|vision_start|>") + + print_rank_0( + f"[DEBUG TOKEN IDS] image_token_id={image_token_id}, " + f"video_token_id={video_token_id}, vision_start_token_id={vision_start_token_id}" + ) def qwen2vl_embedding_ranks(pp_ranks): @@ -94,9 +202,25 @@ def model_provider(pre_process=True, post_process=True, add_encoder=True, add_de def get_batch(data_iterator): """Generate a batch""" + global _logged_token_presence_once, _big_debug_step args = get_args() + _ensure_vision_token_ids() + + print("=" * 80) + print("[DEBUG GET_BATCH - pretrain_llavaov_1_5.py]") + if data_iterator is not None and mpu.get_tensor_model_parallel_rank() == 0: data = next(data_iterator) + + print(f" Data from iterator is dict: {isinstance(data, dict)}") + if isinstance(data, dict): + print(f" Data keys: {list(data.keys())}") + print(f" 'imgs' in data: {'imgs' in data}") + print(f" 'image_grid_thw' in data: {'image_grid_thw' in data}") + if 'imgs' in data: + print(f" data['imgs'] is None: {data['imgs'] is None}") + print(f" data['imgs'] shape: {data['imgs'].shape if data['imgs'] is not None else 'None'}") + if isinstance(data.get('tokens'), torch.Tensor): orig_dtype = data['tokens'].dtype if data['tokens'].dtype != torch.long: @@ -112,14 +236,26 @@ def get_batch(data_iterator): else: data = None + print(f" Broadcasting data...") tokens = tensor_parallel.broadcast_data(["tokens"], data, torch.int64)["tokens"] labels = tensor_parallel.broadcast_data(["labels"], data, torch.int64)["labels"] attn_mask = tensor_parallel.broadcast_data(["attn_mask"], data, torch.bool)["attn_mask"] cu_lengths = tensor_parallel.broadcast_data(["cu_lengths"], data, torch.int32)["cu_lengths"] max_lengths = tensor_parallel.broadcast_data(["max_lengths"], data, torch.int32)["max_lengths"] - - has_video = video_token_id in tokens - has_image = image_token_id in tokens + + has_video = False # Video path intentionally disabled for this training setup. + has_image = image_token_id is not None and image_token_id in tokens + if not _logged_token_presence_once: + image_token_count = int((tokens == image_token_id).sum().item()) if image_token_id is not None else 0 + vision_start_count = int((tokens == vision_start_token_id).sum().item()) if vision_start_token_id is not None else 0 + print_rank_0( + f"[DEBUG TRAIN TOKENS] image_token_id={image_token_id} count={image_token_count}, " + f"vision_start_token_id={vision_start_token_id} count={vision_start_count}, " + f"first_32_ids={tokens[0, :32].tolist()}" + ) + _logged_token_presence_once = True + print(f" has_image token in batch: {has_image}") + print(f" has_video token in batch: {has_video}") thw = None video_grid_thw = None imgs = None @@ -127,6 +263,10 @@ def get_batch(data_iterator): if has_image: imgs = tensor_parallel.broadcast_data(["imgs"], data, torch.float32)["imgs"] thw = tensor_parallel.broadcast_data(["image_grid_thw"], data, torch.int32)["image_grid_thw"] + print(f" Broadcasted imgs: {imgs.shape if imgs is not None else 'None'}") + print(f" Broadcasted image_grid_thw: {thw.shape if thw is not None else 'None'}") + else: + print(" No image tokens in this packed batch (imgs not forwarded to model).") if has_video: pixel_values_videos = tensor_parallel.broadcast_data( ["pixel_values_videos"], @@ -137,8 +277,10 @@ def get_batch(data_iterator): data, torch.int32)["video_grid_thw"] + print("=" * 80) + packed_seq_params = None - is_video = video_token_id in tokens + is_video = False attn_mask_type = AttnMaskType.padding_causal if attn_mask.any() else AttnMaskType.causal @@ -167,6 +309,21 @@ def get_batch(data_iterator): labels = get_inputs_on_this_cp_rank(labels.transpose(0, 1)).transpose(0, 1) loss_mask = get_inputs_on_this_cp_rank(loss_mask.transpose(0, 1)).transpose(0, 1) + _big_debug_step += 1 + _log_big_batch_debug( + step_id=_big_debug_step, + tokens=tokens, + labels=labels, + loss_mask=loss_mask, + attn_mask=attn_mask, + imgs=imgs, + image_grid_thw=thw, + cu_lengths=cu_lengths, + max_lengths=max_lengths, + has_image=bool(has_image), + has_video=bool(has_video), + ) + # TODO attn_mask_type = AttnMaskType.causal attn_mask = None diff --git a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py index 0354a598..f2214069 100644 --- a/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py +++ b/aiak_training_llm/train/sft/sft_llavaov_1_5_vl.py @@ -67,6 +67,16 @@ def get_batch(data_iterator): else: data = None + print("=" * 80) + print("[DEBUG GET_BATCH] Checking data from iterator:") + print(f" data is None: {data is None}") + if data is not None: + print(f" data keys: {data.keys() if isinstance(data, dict) else 'not a dict'}") + if isinstance(data, dict) and 'images' in data: + print(f" data['images'] is None: {data['images'] is None}") + print(f" data['images'] shape: {data['images'].shape if data['images'] is not None else 'None'}") + print("=" * 80) + data_i = tensor_parallel.broadcast_data([ "input_ids", # "position_ids", @@ -78,6 +88,11 @@ def get_batch(data_iterator): ], data, torch.int64) data_f = tensor_parallel.broadcast_data(["images"], data, torch.float32) + print("[DEBUG GET_BATCH] After broadcast:") + print(f" data_f['images'] is None: {data_f['images'] is None}") + print(f" data_f['images'] shape: {data_f['images'].shape if data_f['images'] is not None else 'None'}") + print("=" * 80) + # slice batch along sequence dimension for context parallelism assert mpu.get_context_parallel_world_size() == 1, "not implemented" diff --git a/aiak_training_llm/train/training_utils.py b/aiak_training_llm/train/training_utils.py index 1eadc24b..10211a19 100644 --- a/aiak_training_llm/train/training_utils.py +++ b/aiak_training_llm/train/training_utils.py @@ -313,6 +313,20 @@ def pretrain( iteration = 0 if args.do_train and args.train_iters > 0: + print_rank_0('\n' + '=' * 80) + print_rank_0("training with the following parameter status:") + print_rank_0('=' * 80) + model_to_check = model[0] if isinstance(model, list) else model + trainable_params = 0 + frozen_params = 0 + for name, param in model_to_check.named_parameters(): + status = "TRAINABLE" if param.requires_grad else "FROZEN" + print_rank_0(f"{status:12s} | {name:80s} | shape: {str(tuple(param.shape)):30s} | dtype: {param.dtype}") + if param.requires_grad: + trainable_params += param.numel() + else: + frozen_params += param.numel() + iteration, num_floating_point_operations_so_far = train( forward_step_func=forward_step_func, model=model, diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 993d5090..7824f6ab 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -10,9 +10,9 @@ PP="${2:-1}" # pipeline parallel SEQ_LEN="${3:-512}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size -GBS="${5:-2}" +GBS="${5:-4}" # NSTEP="${6:-2500}" # number of training iterations -NSTEP="${6:-5}" # number of training iterations - reduced to 5 samples for CUDA memory +NSTEP="${6:-1}" # number of training iterations - reduced to 5 samples for CUDA memory # DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} # TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} # CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index 7b820236..941e35a5 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -133,7 +133,7 @@ DATA_ARGS=( TRAINING_ARGS=( --training-phase sft # supervised fine-tuning --chat-template llama3 # MobileLLM uses Llama3 tokenizer - --trainable-modules language_model adapter # Stage 1: Train language model + adapter (vision frozen) + --trainable-modules adapter # Stage 1: Train only adapter (vision and language model fully frozen) --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" --no-rope-fusion From d55cac07531a5d33387067072040c10bc7fb62a8 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Thu, 26 Mar 2026 13:32:38 +0400 Subject: [PATCH 26/39] Stage alignment updates and remove convert/data preprocess tools --- Stage1/alignment.sh | 4 +- Stage1/alignment_rocm.sh | 15 +- .../stage_1_alignment_llava_ov_4b.sh | 5 +- .../stage_1_alignment_mobilellm_140m.sh | 10 +- tools/convert_checkpoint/__init__.py | 11 - .../convert_checkpoint/abstact_checkpoint.py | 53 - tools/convert_checkpoint/abstact_config.py | 49 - tools/convert_checkpoint/arguments.py | 177 -- tools/convert_checkpoint/common_checkpoint.py | 621 ----- tools/convert_checkpoint/common_config.py | 72 - .../config/llava-ov-1.5-14b/adapter.json | 8 - .../config/llava-ov-1.5-14b/qwen3.json | 110 - .../config/llava-ov-1.5-14b/vision-model.json | 66 - .../config/llava-ov-1.5-14b/vision-patch.json | 7 - .../config/llava-ov-1.5-30b-a3b/adapter.json | 8 - .../config/llava-ov-1.5-30b-a3b/qwen3.json | 122 - .../llava-ov-1.5-30b-a3b/vision-model.json | 66 - .../llava-ov-1.5-30b-a3b/vision-patch.json | 7 - .../config/llava-ov-1.5-3b/adapter.json | 8 - .../config/llava-ov-1.5-3b/qwen2_5.json | 99 - .../config/llava-ov-1.5-3b/vision-model.json | 66 - .../config/llava-ov-1.5-3b/vision-patch.json | 7 - .../config/llava-ov-1.5-4b/adapter.json | 8 - .../config/llava-ov-1.5-4b/fastvit.json | 8 - .../config/llava-ov-1.5-4b/qwen3.json | 110 - .../config/llava-ov-1.5-4b/vision-model.json | 66 - .../config/llava-ov-1.5-4b/vision-patch.json | 7 - .../config/llava-ov-1.5-8b/adapter.json | 8 - .../config/llava-ov-1.5-8b/qwen3.json | 110 - .../config/llava-ov-1.5-8b/vision-model.json | 66 - .../config/llava-ov-1.5-8b/vision-patch.json | 7 - .../config/mobilellm-140m.json | 87 - .../convert_checkpoint/config/qwen-1.8b.json | 131 - tools/convert_checkpoint/config/qwen-14b.json | 131 - tools/convert_checkpoint/config/qwen-72b.json | 132 - tools/convert_checkpoint/config/qwen-7b.json | 131 - .../config/qwen1.5-0.5b.json | 96 - .../config/qwen1.5-1.8b.json | 96 - .../config/qwen1.5-14b.json | 96 - .../config/qwen1.5-32b.json | 96 - .../convert_checkpoint/config/qwen1.5-4b.json | 96 - .../config/qwen1.5-72b.json | 96 - .../convert_checkpoint/config/qwen1.5-7b.json | 96 - .../convert_checkpoint/config/qwen2-0.5b.json | 90 - .../convert_checkpoint/config/qwen2-1.5b.json | 91 - .../convert_checkpoint/config/qwen2-72b.json | 94 - tools/convert_checkpoint/config/qwen2-7b.json | 97 - .../config/qwen2-vl-2b/adapter.json | 8 - .../config/qwen2-vl-2b/qwen2.json | 95 - .../config/qwen2-vl-2b/vision-model.json | 66 - .../config/qwen2-vl-2b/vision-patch.json | 3 - .../config/qwen2-vl-72b/adapter.json | 8 - .../config/qwen2-vl-72b/qwen2.json | 96 - .../config/qwen2-vl-72b/vision-model.json | 66 - .../config/qwen2-vl-72b/vision-patch.json | 3 - .../config/qwen2-vl-7b/adapter.json | 8 - .../config/qwen2-vl-7b/qwen2.json | 96 - .../config/qwen2-vl-7b/vision-model.json | 66 - .../config/qwen2-vl-7b/vision-patch.json | 3 - .../config/qwen2.5-0.5b.json | 90 - .../config/qwen2.5-1.5b.json | 91 - .../config/qwen2.5-14b.json | 97 - .../config/qwen2.5-32b.json | 97 - .../convert_checkpoint/config/qwen2.5-3b.json | 91 - .../config/qwen2.5-72b.json | 94 - .../convert_checkpoint/config/qwen2.5-7b.json | 97 - .../config/qwen2_5-vl-32b/adapter.json | 7 - .../config/qwen2_5-vl-32b/qwen2_5.json | 101 - .../config/qwen2_5-vl-32b/vision-model.json | 80 - .../config/qwen2_5-vl-32b/vision-patch.json | 3 - .../config/qwen2_5-vl-3b/adapter.json | 7 - .../config/qwen2_5-vl-3b/qwen2_5.json | 99 - .../config/qwen2_5-vl-3b/vision-model.json | 80 - .../config/qwen2_5-vl-3b/vision-patch.json | 3 - .../config/qwen2_5-vl-72b/adapter.json | 7 - .../config/qwen2_5-vl-72b/qwen2_5.json | 96 - .../config/qwen2_5-vl-72b/vision-model.json | 80 - .../config/qwen2_5-vl-72b/vision-patch.json | 3 - .../config/qwen2_5-vl-7b/adapter.json | 7 - .../config/qwen2_5-vl-7b/qwen2_5.json | 101 - .../config/qwen2_5-vl-7b/vision-model.json | 80 - .../config/qwen2_5-vl-7b/vision-patch.json | 3 - tools/convert_checkpoint/config/qwen3-4b.json | 111 - .../custom/cogvlm/__init__.py | 0 .../custom/cogvlm/adapter.py | 103 - .../custom/cogvlm/merge_huggingface.py | 68 - .../custom/cogvlm/merge_megatron.py | 70 - .../convert_checkpoint/custom/cogvlm/util.py | 91 - .../custom/cogvlm/vision_patch.py | 62 - .../custom/llavaov_1_5/__init__.py | 0 .../custom/llavaov_1_5/adapter.py | 83 - .../custom/llavaov_1_5/fastvit.py | 96 - .../custom/llavaov_1_5/merge_huggingface.py | 66 - .../custom/llavaov_1_5/merge_megatron.py | 76 - .../llavaov_1_5/merge_megatron_fastvlm.py | 89 - .../custom/llavaov_1_5/util.py | 371 --- .../custom/llavaov_1_5/vision_patch.py | 77 - .../custom/llavaov_1_5_30b_a3b/__init__.py | 0 .../custom/llavaov_1_5_30b_a3b/adapter.py | 65 - .../llavaov_1_5_30b_a3b/merge_huggingface.py | 66 - .../llavaov_1_5_30b_a3b/merge_megatron.py | 76 - .../merge_megatron_qwen3_30b_a3b.py | 98 - .../custom/llavaov_1_5_30b_a3b/util.py | 359 --- .../llavaov_1_5_30b_a3b/vision_patch.py | 62 - .../custom/qwen2_vl/__init__.py | 0 .../custom/qwen2_vl/adapter.py | 65 - .../custom/qwen2_vl/merge_huggingface.py | 66 - .../custom/qwen2_vl/merge_megatron.py | 76 - .../custom/qwen2_vl/util.py | 112 - .../custom/qwen2_vl/vision_patch.py | 61 - .../huggingface_checkpoint.py | 999 ------- .../convert_checkpoint/huggingface_config.py | 49 - tools/convert_checkpoint/mcore_checkpoint.py | 2431 ----------------- tools/convert_checkpoint/mcore_config.py | 96 - .../convert_checkpoint/megatron_checkpoint.py | 880 ------ tools/convert_checkpoint/megatron_config.py | 94 - .../convert_checkpoint/megatron_optimizer.py | 844 ------ tools/convert_checkpoint/model.py | 264 -- tools/convert_checkpoint/optim.py | 207 -- tools/convert_checkpoint/utils.py | 416 --- .../data_preprocess/convert_to_webdataset.py | 195 -- .../offline_packing/S2_hashbacket.ipynb | 312 --- .../configs/s1_config_emova.yaml | 48 - .../configs/s1_config_emova_3000tk.yaml | 48 - .../s1_config_vqa_pretrain_5M_16k.yaml | 50 - .../configs/s1_config_vqa_pretrain_5M_8k.yaml | 50 - .../convert_packedsample_to_wds.py | 228 -- .../offline_packing/convert_pairs.py | 124 - .../offline_packing/hashbacket.py | 2223 --------------- .../offline_packing/readme.ipynb | 128 - .../offline_packing/s1_get_tokenlens_v2.py | 717 ----- .../s1_get_tokenlens_v3-sft.py | 733 ----- .../offline_packing/s2_data_depict.ipynb | 846 ------ .../s2_prepare_rawsamples-emova.py | 396 --- ...2_prepare_rawsamples-vqa_5500k-16k-fast.py | 489 ---- .../offline_packing/s3_test_emova.sh | 1 - .../preprocess_pretrain_data.py | 444 --- tools/data_preprocess/preprocess_sft_data.py | 200 -- 138 files changed, 14 insertions(+), 21536 deletions(-) delete mode 100644 tools/convert_checkpoint/__init__.py delete mode 100644 tools/convert_checkpoint/abstact_checkpoint.py delete mode 100644 tools/convert_checkpoint/abstact_config.py delete mode 100644 tools/convert_checkpoint/arguments.py delete mode 100644 tools/convert_checkpoint/common_checkpoint.py delete mode 100644 tools/convert_checkpoint/common_config.py delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-14b/adapter.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-14b/qwen3.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/adapter.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/qwen3.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-3b/adapter.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-3b/qwen2_5.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-4b/adapter.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-4b/qwen3.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-8b/adapter.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-8b/qwen3.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/mobilellm-140m.json delete mode 100644 tools/convert_checkpoint/config/qwen-1.8b.json delete mode 100644 tools/convert_checkpoint/config/qwen-14b.json delete mode 100644 tools/convert_checkpoint/config/qwen-72b.json delete mode 100644 tools/convert_checkpoint/config/qwen-7b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-0.5b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-1.8b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-14b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-32b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-4b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-72b.json delete mode 100644 tools/convert_checkpoint/config/qwen1.5-7b.json delete mode 100644 tools/convert_checkpoint/config/qwen2-0.5b.json delete mode 100644 tools/convert_checkpoint/config/qwen2-1.5b.json delete mode 100644 tools/convert_checkpoint/config/qwen2-72b.json delete mode 100644 tools/convert_checkpoint/config/qwen2-7b.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-2b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-2b/qwen2.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-2b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-2b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-72b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-72b/qwen2.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-72b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-72b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-7b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-7b/qwen2.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-7b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2-vl-7b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-0.5b.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-1.5b.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-14b.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-32b.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-3b.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-72b.json delete mode 100644 tools/convert_checkpoint/config/qwen2.5-7b.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-32b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-32b/qwen2_5.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-3b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-3b/qwen2_5.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-72b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-72b/qwen2_5.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-7b/adapter.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-7b/qwen2_5.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-model.json delete mode 100644 tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-patch.json delete mode 100644 tools/convert_checkpoint/config/qwen3-4b.json delete mode 100644 tools/convert_checkpoint/custom/cogvlm/__init__.py delete mode 100644 tools/convert_checkpoint/custom/cogvlm/adapter.py delete mode 100644 tools/convert_checkpoint/custom/cogvlm/merge_huggingface.py delete mode 100644 tools/convert_checkpoint/custom/cogvlm/merge_megatron.py delete mode 100644 tools/convert_checkpoint/custom/cogvlm/util.py delete mode 100644 tools/convert_checkpoint/custom/cogvlm/vision_patch.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/__init__.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/adapter.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/merge_huggingface.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/util.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5/vision_patch.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/__init__.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/adapter.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_huggingface.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron_qwen3_30b_a3b.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/util.py delete mode 100644 tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/vision_patch.py delete mode 100644 tools/convert_checkpoint/custom/qwen2_vl/__init__.py delete mode 100644 tools/convert_checkpoint/custom/qwen2_vl/adapter.py delete mode 100644 tools/convert_checkpoint/custom/qwen2_vl/merge_huggingface.py delete mode 100644 tools/convert_checkpoint/custom/qwen2_vl/merge_megatron.py delete mode 100644 tools/convert_checkpoint/custom/qwen2_vl/util.py delete mode 100644 tools/convert_checkpoint/custom/qwen2_vl/vision_patch.py delete mode 100644 tools/convert_checkpoint/huggingface_checkpoint.py delete mode 100644 tools/convert_checkpoint/huggingface_config.py delete mode 100644 tools/convert_checkpoint/mcore_checkpoint.py delete mode 100644 tools/convert_checkpoint/mcore_config.py delete mode 100644 tools/convert_checkpoint/megatron_checkpoint.py delete mode 100644 tools/convert_checkpoint/megatron_config.py delete mode 100644 tools/convert_checkpoint/megatron_optimizer.py delete mode 100644 tools/convert_checkpoint/model.py delete mode 100644 tools/convert_checkpoint/optim.py delete mode 100644 tools/convert_checkpoint/utils.py delete mode 100644 tools/data_preprocess/convert_to_webdataset.py delete mode 100644 tools/data_preprocess/offline_packing/S2_hashbacket.ipynb delete mode 100644 tools/data_preprocess/offline_packing/configs/s1_config_emova.yaml delete mode 100644 tools/data_preprocess/offline_packing/configs/s1_config_emova_3000tk.yaml delete mode 100644 tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_16k.yaml delete mode 100644 tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_8k.yaml delete mode 100644 tools/data_preprocess/offline_packing/convert_packedsample_to_wds.py delete mode 100644 tools/data_preprocess/offline_packing/convert_pairs.py delete mode 100644 tools/data_preprocess/offline_packing/hashbacket.py delete mode 100644 tools/data_preprocess/offline_packing/readme.ipynb delete mode 100644 tools/data_preprocess/offline_packing/s1_get_tokenlens_v2.py delete mode 100644 tools/data_preprocess/offline_packing/s1_get_tokenlens_v3-sft.py delete mode 100644 tools/data_preprocess/offline_packing/s2_data_depict.ipynb delete mode 100644 tools/data_preprocess/offline_packing/s2_prepare_rawsamples-emova.py delete mode 100644 tools/data_preprocess/offline_packing/s2_prepare_rawsamples-vqa_5500k-16k-fast.py delete mode 100644 tools/data_preprocess/offline_packing/s3_test_emova.sh delete mode 100644 tools/data_preprocess/preprocess_pretrain_data.py delete mode 100644 tools/data_preprocess/preprocess_sft_data.py diff --git a/Stage1/alignment.sh b/Stage1/alignment.sh index fefc904b..6f8fd25c 100755 --- a/Stage1/alignment.sh +++ b/Stage1/alignment.sh @@ -56,8 +56,8 @@ export WANDB_API_KEY="wandb_v1_5y5JqALBMdHhru8CR1gOLflJlRj_O8BG2XRb0S2x0TJVqW1xA export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="mobilellm_integration" -export CUDA_VISIBLE_DEVICES=2,3,4,5 -export GPUS_PER_NODE=4 +export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 +export GPUS_PER_NODE=8 export MASTER_PORT=26000 # Choose which backbone to use for training: diff --git a/Stage1/alignment_rocm.sh b/Stage1/alignment_rocm.sh index 7ca3b796..4657de3a 100755 --- a/Stage1/alignment_rocm.sh +++ b/Stage1/alignment_rocm.sh @@ -4,9 +4,7 @@ #SBATCH --nodes=1 #SBATCH --ntasks-per-node=1 #SBATCH --cpus-per-task=128 -#SBATCH --gres=gpu:4 -#SBATCH --time=72:00:00 -#SBATCH --mem=230G +#SBATCH --gres=gpu:8 #SBATCH --qos=skqos #SBATCH --partition=faculty #SBATCH --output=/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5/Stage1/logs/%x-%j.out @@ -22,12 +20,11 @@ export ROCM_HOME=${ROCM_HOME:-/opt/rocm} export PATH="${ROCM_HOME}/bin:${PATH}" export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH}" -export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3} +export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} # Force HuggingFace offline mode to use local files only -# Commented out to allow downloading MobileLLM tokenizer on first run -# export HF_HUB_OFFLINE=1 -# export TRANSFORMERS_OFFLINE=1 +#export HF_HUB_OFFLINE=1 +#export TRANSFORMERS_OFFLINE=1 # RCCL/NCCL runtime hints (tune as needed) @@ -35,7 +32,7 @@ export NCCL_DEBUG=${NCCL_DEBUG:-WARN} export NCCL_COLLNET_ENABLE=${NCCL_COLLNET_ENABLE:-0} export NCCL_P2P_ENABLE=${NCCL_P2P_ENABLE:-1} # export NCCL_SOCKET_IFNAME=eno1 # uncomment and set to your NIC if needed - + # Resolve repo root relative to this script (Stage1/..) SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO_ROOT=/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5 @@ -71,7 +68,6 @@ echo "AIAK_TRAINING_PATH=${AIAK_TRAINING_PATH}" echo "AIAK_MAGATRON_PATH=${AIAK_MAGATRON_PATH}" echo "DATA_PATH=${DATA_PATH}" echo "TOKENIZER_PATH=${TOKENIZER_PATH}" -echo "PRETRAINED_CHECKPOINT=${PRETRAINED_CHECKPOINT}" echo "SLURM_NODELIST=${SLURM_NODELIST}" echo "PYTHONPATH=${PYTHONPATH}" @@ -79,5 +75,4 @@ echo "PYTHONPATH=${PYTHONPATH}" export WANDB_API_KEY="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" export WANDB_PROJECT="llava-ov-1_5" export WANDB_NAME="fastvit_integration" -# bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh \ No newline at end of file diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh index 7824f6ab..a31b198b 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh @@ -6,13 +6,12 @@ AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" # Megatron- TP="${1:-1}" PP="${2:-1}" # pipeline parallel # Defaults: TP=1, PP=1. -# SEQ_LEN="${3:-32768}" -SEQ_LEN="${3:-512}" +SEQ_LEN="${3:-32768}" MBS="${4:-1}" # micro batch size # GBS="${5:-8}" # global batch size GBS="${5:-4}" # NSTEP="${6:-2500}" # number of training iterations -NSTEP="${6:-1}" # number of training iterations - reduced to 5 samples for CUDA memory +NSTEP="${6:-5}" # number of training iterations - reduced to 5 samples for CUDA memory # DATA_PATH=${DATA_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset"} # TOKENIZER_PATH=${TOKENIZER_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0"} # CHECKPOINT_PATH=${CHECKPOINT_PATH:-"/l/users/rana.zayed/new_fastvlm/LLaVA-OneVision-1.5/checkpoints/LLaVA-OneVision-1.5-4B-stage0_mcore_tp2_pp1"} diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index 941e35a5..16bce8e3 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -11,10 +11,10 @@ AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" # Model parallelism configuration TP="${1:-1}" # Tensor parallel PP="${2:-1}" # Pipeline parallel -SEQ_LEN="${3:-512}" # Sequence length (reduced for testing) +SEQ_LEN="${3:-32768}" # Sequence length (reduced for testing) MBS="${4:-1}" # Micro batch size -GBS="${5:-4}" # Global batch size (4 examples for testing) -NSTEP="${6:-1}" # Number of training iterations (1 step with 4 examples) +GBS="${5:-8}" # Global batch size (4 examples for testing) +NSTEP="${6:-20}" # Number of training iterations (1 step with 4 examples) # Data paths - UPDATE THESE FOR YOUR SETUP DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" @@ -80,7 +80,7 @@ mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=${GPUS_PER_NODE:-4} +GPUS_PER_NODE=${GPUS_PER_NODE:-8} MASTER_PORT=${MASTER_PORT:-26000} if [[ $SINGLE_NODE -eq 1 ]]; then @@ -137,7 +137,7 @@ TRAINING_ARGS=( --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" --no-rope-fusion - --training-rice-vl-max-answer-length 512 + --training-rice-vl-max-answer-length 32768 --transformer-impl local --max-position-embeddings 32768 # MobileLLM supports 32k context --init-method-std 0.02 diff --git a/tools/convert_checkpoint/__init__.py b/tools/convert_checkpoint/__init__.py deleted file mode 100644 index 74b5730a..00000000 --- a/tools/convert_checkpoint/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -""" convert_checkpoint """ -from .common_config import CommonConfig - -from .megatron_checkpoint import MegatronCheckpoint -from .megatron_config import MegatronConfig - -from .huggingface_checkpoint import HuggingFaceCheckpoint -from .huggingface_config import HuggingFaceConfig - -from .mcore_checkpoint import McoreCheckpoint -from .mcore_config import McoreConfig \ No newline at end of file diff --git a/tools/convert_checkpoint/abstact_checkpoint.py b/tools/convert_checkpoint/abstact_checkpoint.py deleted file mode 100644 index 7e3cae0d..00000000 --- a/tools/convert_checkpoint/abstact_checkpoint.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import torch -from abc import ABC, abstractmethod - - -class AbstractCheckpoint(ABC): - """ - AbstractCheckpoint - """ - - def __init__(self, num_layers): - self.num_layers = num_layers - self.state_dict = {} - - @staticmethod - @abstractmethod - def convert_from_common(*args, **kwargs): - """ - return checkpoints converted from common checkpoint - """ - raise NotImplementedError() - - @abstractmethod - def convert_to_common(self, *args, **kwargs): - """ - convert checkpoints to common checkpoint - """ - raise NotImplementedError() - - def set_dtype(self, dtype): - """ set dtype """ - self.dtype = dtype - - def load(self, ckpt_path): - """ - load checkpoint - """ - self.state_dict = torch.load(ckpt_path) - - def save(self, ckpt_path): - """ - save checkpoint - """ - torch.save(self.state_dict, ckpt_path) - - \ No newline at end of file diff --git a/tools/convert_checkpoint/abstact_config.py b/tools/convert_checkpoint/abstact_config.py deleted file mode 100644 index 86f7d2e6..00000000 --- a/tools/convert_checkpoint/abstact_config.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import torch -import json -from abc import ABC, abstractmethod -from convert_checkpoint.utils import get_element_from_dict_by_path - - -class AbstractConfig(ABC): - """ - AbstractConfig - """ - - def __init__(self): - self.data = {} - - @staticmethod - @abstractmethod - def convert_from_common(): - """ - return config converted from common config - """ - raise NotImplementedError() - - def update(self, config): - """ update data by given config(dict) """ - self.data.update(config) - - def get(self, *args, **kwargs): - """ return args """ - return self.data.get(*args, **kwargs) - - def load(self, config_path): - """ - load config - """ - raise NotImplementedError() - - def save(self, config_path): - """ - save config - """ - raise NotImplementedError() \ No newline at end of file diff --git a/tools/convert_checkpoint/arguments.py b/tools/convert_checkpoint/arguments.py deleted file mode 100644 index be8e398c..00000000 --- a/tools/convert_checkpoint/arguments.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -"""argparse""" - -import argparse - -_GLOBAL_ARGS = None - - -def set_args(args): - global _GLOBAL_ARGS - _GLOBAL_ARGS = args - - -def parse_args(title=None): - global _GLOBAL_ARGS - if _GLOBAL_ARGS is not None: - return _GLOBAL_ARGS - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Aiak-Tool Arguments', - allow_abbrev=False) - _add_checkpoint_args(parser) - _add_common_args(parser) - _add_huggingface_args(parser) - _add_megatron_args(parser) - - args = parser.parse_args() - if args.convert_to_fp8: - assert args.load_platform == 'mcore' and args.save_platform == 'huggingface', \ - "convert_to_fp8 only support mcore to huggingface" - if title is None: - _GLOBAL_ARGS = args - return _GLOBAL_ARGS - else: - for group in parser._action_groups: - if group.title == title: - group_dict={item.dest: getattr(args, item.dest, None) for item in group._group_actions} - _GLOBAL_ARGS = argparse.Namespace(**group_dict) - return _GLOBAL_ARGS - _GLOBAL_ARGS = argparse.Namespace() - return _GLOBAL_ARGS - - -def _add_checkpoint_args(parser): - group = parser.add_argument_group(title='checkpoint') - - group.add_argument('--load_platform', type=str, default=None, - choices=['huggingface', 'megatron', 'mcore']) - group.add_argument('--save_platform', type=str, default=None, - choices=['huggingface', 'megatron', 'mcore']) - group.add_argument('--load_ckpt_path', type=str, default=None, - help='path to load checkpoint') - group.add_argument('--save_ckpt_path', type=str, default=None, - help='path to save checkpoint') - group.add_argument('--common_config_path', type=str, default=None, - help='path to common config') - group.add_argument("--megatron_path", type=str, default=None, - help="Base directory of Megatron repository") - group.add_argument('--no_load_optim', action='store_true', - help='do not convert optimizer') - group.add_argument('--no_save_optim', action='store_true', - help='do not save optimizer') - group.add_argument('--model_type_custom', type=str, default=None, - help='custom model type') - group.add_argument('--safetensors', action='store_true', - help='Use [safetensors](https://huggingface.co/docs/safetensors).') - group.add_argument('--convert_to_fp8', action='store_true', - help='Convert float16 weights to fp8') - group.add_argument('--quant_method', type=str, default='te', choices=['te', 'pt', 'aiak'], - help='The quantization method to use. Choices: [te, pt, aiak].') - group.add_argument('--amax_epsilon', type=float, default=0.0, - help=("Epsilon value added to the amax calculation to avoid division by zero " - "when converting to FP8. Only used in Transformer Engine FP8 conversion.") - ) - group.add_argument('--force_pow_2_scales', action='store_true', - help=("Force power of 2 scales, only used in Transformer Engine FP8 conversion.") - ) - group.add_argument('--pretrain_as_fp8', action='store_true', - help='Run pretrain as fp8, only used for hf to mcore when ' - 'saved checkpoint is bf16 and pretrain as fp8') - - group.add_argument('--fp8_quant_transfer_type', type=str, default="float32", choices=["float32", "bfloat16"], - help='The transfer dtype when convert from hf fp8 to mcore fp8') - - -def _add_common_args(parser): - group = parser.add_argument_group(title='common') - - group.add_argument('--torch_dtype', type=str, choices=["float16", "float32", "bfloat16"], - help='target torch dtype') - group.add_argument('--vocab_size', type=int, default=None, help='vocab size') - group.add_argument('--vpp-scheduler', type=str, default=None, - choices=["dualpipev"], - help='By default, the original 1F1B scheduling method is used. When selecting DualPipeV, ' - 'the effect can be referred to at https://hackmd.io/@ufotalent/r1lVXsa9Jg') - group.add_argument('--num-virtual-stages-per-pipeline-rank', type=int, default=None, - help='Number of virtual pipeline stages per pipeline parallelism rank') - group.add_argument('--decoder-first-pipeline-num-layers', - type=int, default=None, - help=('The number of transformer layers on the first pipeline stage of the decoder. ' - 'Default None is even split of transformer layers across all pipeline stages')) - group.add_argument('--decoder-last-pipeline-num-layers', - type=int, default=None, - help=('The number of transformer layers on the last pipeline stage of the decoder. ' - 'Default None is even split of transformer layers across all pipeline stages')) - -def _add_megatron_args(parser): - """ - Add MegaTron related parameters to the parser. - - Args: - parser (ArgumentParser, str): ArgumentParser object or parameter string, used to add MegaTron related parameters. - - Returns: - None, void: No return value, directly modify the passed ArgumentParser object. - """ - group = parser.add_argument_group(title='megatron') - - group.add_argument('--use_distributed_optimizer', action='store_true', - help='use distributed optimizer') - group.add_argument('--tensor_model_parallel_size', type=int, default=1, - help='target tensor model parallel size') - group.add_argument('--pipeline_model_parallel_size', type=int, default=1, - help='target pipeline model parallel size') - group.add_argument('--data_parallel_size', type=int, default=1, - help='target data parallel size') - group.add_argument('--expert_parallel_size', type=int, default=None, - help='target expert parallel size') - group.add_argument('--expert_tensor_parallel_size', type=int, default=None, - help='Degree of expert model parallelism. Default is None, ' - 'which will be set to the value of --tensor-model-paralle-size.') - group.add_argument('--pad_vocab_size_to', type=int, default=None, - help='Pad the vocab size to this value.' - 'This value must be greater than the initial size of the tokenizer' - ', needs to be divisible by TP size and `make-vocab-size-divisible-by`.') - group.add_argument('--custom_pipeline_layers', type=str, default=None, - help='add by aiak for pp layer imbalance.For example 19,20,20,21.' - '19 for stage0 layers, 20 for stage1 layers...') - group.add_argument('--num_layers_per_virtual_pipeline_stage', type=int, default=None, - help='Number of layers per virtual pipeline stage') - group.add_argument('--transformer_impl', default='transformer_engine', - choices=['local', 'transformer_engine'], - help='Which Transformer implementation to use when load or save mcore checkpoint.' - 'Only support `transformer_engine` now.') - group.add_argument('--num_experts', type=int, default=None, - help='Number of Experts in MoE (None means no MoE)') - group.add_argument('--checkpoint-format', type=str, default=None, - help='hf checkpoint format end with safetensors') - group.add_argument('--max_workers', type=int, default=1, - help='thread for checkpoint converting') - group.add_argument('--no-te', action='store_true', - help='do not use transformer engine') - group.add_argument('--moe-grouped-gemm', action='store_true', - help='use grouped gemm in moe') - group.add_argument('--resume-convert', action='store_true', - help='resume checkpoint converting when failed') - group.add_argument('--cache-path', type=str, default=None, - help='cache path used during conversion') - group.add_argument('--layer-for-test', type=str, default=None, - help='get specific layer from checkpoint for test') - group.add_argument('--num-experts-for-test', type=int, default=None, - help='Number of Experts in MoE for test') - group.add_argument('--sub-num-layers-for-save', type=int, default=None, - help='number of layers for saving each time') - group.add_argument('--save-sub-checkpoint-by-pp', action='store_true', - help='save sub checkpoints by pipeline parallel') - - -def _add_huggingface_args(parser): - group = parser.add_argument_group(title='huggingface') - pass diff --git a/tools/convert_checkpoint/common_checkpoint.py b/tools/convert_checkpoint/common_checkpoint.py deleted file mode 100644 index ebbe6678..00000000 --- a/tools/convert_checkpoint/common_checkpoint.py +++ /dev/null @@ -1,621 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -from convert_checkpoint.abstact_checkpoint import AbstractCheckpoint -from convert_checkpoint.utils import get_element_from_dict_by_path -from convert_checkpoint.arguments import parse_args - - -""" Name of common checkpoint layers, matching key of name_map in {model}.json """ - -WORD_EMBEDDINGS = "word_embeddings" -WORD_POSITION_EMBEDDINGS = "word_position_embeddings" -WORD_BLOCK_POSITION_EMBEDDINGS = "word_block_position_embeddings" -LAYER_PREFIX = "layer_prefix" -MTP_LAYER_PREFIX = "mtp_layer_prefix" -TRANSFORMER = "transformer" -INPUT_LAYERNORM = "input_layernorm" -ROTARY_EMB_INV_FREQ = "rotary_emb.inv_freq" -ATTENTION_ROTARY_EMB_INV_FREQ = "attention.rotary_emb.inv_freq" -ATTENTION_QUERY_KEY_VALUE = "attention.query_key_value" -ATTENTION_QKV_MAP = "attention.qkv_map" -ATTENTION_DENSE = "attention.dense" -POST_ATTENTION_LAYERNORM = "post_attention_layernorm" -MOE_GATE = "moe.gate" -MOE_GATE_BIAS = "moe.gate.bias" -MOE_MLP = "moe.mlp" -MOE_EXPERT = "moe.expert" -MOE_GROUPED_GEMM_EXPERT = "moe.groupedgemm.expert" -MOE_SHARED_EXPERT = "moe.shared_expert" -MLP_DENSE_H_TO_4H = "mlp.dense_h_to_4h" -MLP_DENSE_4H_TO_H = "mlp.dense_4h_to_h" -MOE_SHARED_EXPERT_DENSE_H_TO_4H = "mlp.shared_expert.dense_h_to_4h" -POST_MLP_LAYERNORM = "post_mlp_layernorm" -FINAL_LAYERNORM = "final_layernorm" -WORD_EMBEDDINGS_FOR_HEAD = "word_embeddings_for_head" - -WORD_EMBEDDINGS_TPL = "word_embeddings_tpl" -WORD_POSITION_EMBEDDINGS_TPL = "word_position_embeddings_tpl" -WORD_BLOCK_POSITION_EMBEDDINGS_TPL = "word_block_position_embeddings_tpl" -TRANSFORMER_TPL = "transformer_tpl" -WORD_EMBEDDINGS_FOR_HEAD_TPL = "word_embeddings_for_head_tpl" - -MTP_WORD_EMBEDDING = "mtp_word_embeddings" -MTP_ENORM = "mtp_enorm" -MTP_HNORM = "mtp_hnorm" -MTP_EH_PROJ = "mtp_eh_proj" -MTP_SHARED_HEAD_NORM = "mtp_shared_head_norm" -MTP_SHARED_HEAD_HEAD = "mtp_shared_head_head" - - -class CommonCheckpoint(AbstractCheckpoint): - """ - CommonCheckpoint - """ - def __init__(self, num_layers): - super().__init__(num_layers) - self.other_args = {} - self.args = parse_args() - - @staticmethod - def convert_from_common(*args, **kwargs): - raise NotImplementedError() - - def convert_to_common(self, *args, **kwargs): - raise NotImplementedError() - - def convert(self, ckpt_class, *args, **kwargs): - return ckpt_class.convert_from_common(self, *args, **kwargs) - - def set_word_embedding(self, weight): - self._set(WORD_EMBEDDINGS, "weight", weight) - - def clear_word_embedding(self): - self._clear(WORD_EMBEDDINGS, "weight") - - def get_word_embedding(self): - return self._get(WORD_EMBEDDINGS, "weight") - - def set_word_position_embedding(self, weight): - self._set(WORD_POSITION_EMBEDDINGS, "weight", weight) - - def clear_word_position_embedding(self): - self._clear(WORD_POSITION_EMBEDDINGS, "weight") - - def get_word_position_embedding(self): - return self._get(WORD_POSITION_EMBEDDINGS, "weight") - - def set_word_block_position_embedding(self, weight): - self._set(WORD_BLOCK_POSITION_EMBEDDINGS, "weight", weight) - - def clear_word_block_position_embedding(self): - self._clear(WORD_BLOCK_POSITION_EMBEDDINGS, "weight") - - def get_word_block_position_embedding(self): - return self._get(WORD_BLOCK_POSITION_EMBEDDINGS, "weight") - - def set_layer_input_layernorm(self, index, weight, bias, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{INPUT_LAYERNORM}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - - def clear_layer_input_layernorm(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{INPUT_LAYERNORM}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - - def get_layer_input_layernorm_weight(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{INPUT_LAYERNORM}" - if one_layer_weights is None: - return self._get(path, "weight") - else: - return one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - - def get_layer_input_layernorm_bias(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{INPUT_LAYERNORM}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_attention_rotary_emb_inv_freq(self, index, inv_freq, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_ROTARY_EMB_INV_FREQ}" - if one_layer_weights is None: - self._set(path, "value", inv_freq) - else: - one_layer_weights[path] = inv_freq - - def clear_layer_attention_rotary_emb_inv_freq(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_ROTARY_EMB_INV_FREQ}" - if one_layer_weights is None: - self._clear(path, "value") - else: - one_layer_weights[path] = None - - def get_layer_attention_rotary_emb_inv_freq(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_ROTARY_EMB_INV_FREQ}" - if one_layer_weights is None: - return self._get(path, "value") - else: - return one_layer_weights[path] - - # ====attention qkv a b===== - def set_layer_attention_by_name(self, name, index, weight, bias, one_layer_weights=None, weight_scale_inv=None): - """ - Sets the attention weights and biases of the Transformer layer by name. - Args: - name (str): The name of the attention parameter to be set, which can be one of "query_weight", "key_weight", or "value_weight". - index (int): The index number of the Transformer layer, counted from 0. - weight (Tensor): The weight matrix of the attention parameter, with a shape of (hidden_size, hidden_size). - bias (Tensor, optional): The bias vector of the attention parameter, with a shape of (hidden_size,), defaults to None. - """ - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{name}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - if weight_scale_inv is not None: - self._set(path, "weight_scale_inv", weight_scale_inv) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - if weight_scale_inv is not None: - one_layer_weights[path + ".weight_scale_inv"] = weight_scale_inv - - def clear_layer_attention_by_name(self, name, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{name}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - self._clear(path, "weight_scale_inv") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - one_layer_weights[path + ".weight_scale_inv"] = None - - def get_layer_attention_weight_by_name(self, name, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{name}" - if one_layer_weights is None: - weight = self._get(path, "weight") - weight_scale_inv = self._get(path, "weight_scale_inv") - else: - weight = one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - weight_scale_inv = one_layer_weights[path + ".weight_scale_inv"] \ - if path + ".weight_scale_inv" in one_layer_weights else None - return weight, weight_scale_inv - - def get_layer_attention_bias_by_name(self, name, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{name}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - # ====attention qkv a b end===== - - def set_layer_attention_query_key_value(self, index, weight, bias, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_QUERY_KEY_VALUE}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - - def clear_layer_attention_query_key_value(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_QUERY_KEY_VALUE}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - - def get_layer_attention_query_key_value_weight(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_QUERY_KEY_VALUE}" - if one_layer_weights is None: - return self._get(path, "weight") - else: - return one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - - def get_layer_attention_query_key_value_bias(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_QUERY_KEY_VALUE}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_attention_dense(self, index, weight, bias, one_layer_weights=None, weight_scale_inv=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_DENSE}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - if weight_scale_inv is not None: - self._set(path, "weight_scale_inv", weight_scale_inv) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - if weight_scale_inv is not None: - one_layer_weights[path + ".weight_scale_inv"] = weight_scale_inv - - def clear_layer_attention_dense(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_DENSE}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - self._clear(path, "weight_scale_inv") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - one_layer_weights[path + ".weight_scale_inv"] = None - - def get_layer_attention_dense_weight(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_DENSE}" - if one_layer_weights is None: - weight = self._get(path, "weight") - weight_scale_inv = self._get(path, "weight_scale_inv") - else: - weight = one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - weight_scale_inv = one_layer_weights[path + ".weight_scale_inv"] \ - if path + ".weight_scale_inv" in one_layer_weights else None - return weight, weight_scale_inv - - def get_layer_attention_dense_bias(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{ATTENTION_DENSE}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_post_attention_layernorm(self, index, weight, bias, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_ATTENTION_LAYERNORM}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - - def clear_layer_post_attention_layernorm(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_ATTENTION_LAYERNORM}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - - def get_layer_post_attention_layernorm_weight(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_ATTENTION_LAYERNORM}" - if one_layer_weights is None: - return self._get(path, "weight") - else: - return one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - - def get_layer_post_attention_layernorm_bias(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_ATTENTION_LAYERNORM}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_mlp_dense_h_to_4h(self, index, weight, bias, is_moe_mlp=False, expert_id=None, is_shared=False, - one_layer_weights=None, weight_scale_inv=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_H_TO_4H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_H_TO_4H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_H_TO_4H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_H_TO_4H}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - if weight_scale_inv is not None: - self._set(path, "weight_scale_inv", weight_scale_inv) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - if weight_scale_inv is not None: - one_layer_weights[path + ".weight_scale_inv"] = weight_scale_inv - - def clear_layer_mlp_dense_h_to_4h(self, index, is_moe_mlp = False, expert_id = None, is_shared = False, - one_layer_weights=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_H_TO_4H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_H_TO_4H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_H_TO_4H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_H_TO_4H}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - self._clear(path, "weight_scale_inv") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - one_layer_weights[path + ".weight_scale_inv"] = None - - def get_layer_mlp_dense_h_to_4h_weight(self, index, is_moe_mlp = False, expert_id = None, is_shared = False, - one_layer_weights=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_H_TO_4H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_H_TO_4H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_H_TO_4H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_H_TO_4H}" - if one_layer_weights is None: - weight = self._get(path, "weight") - weight_scale_inv = self._get(path, "weight_scale_inv") - else: - weight = one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - weight_scale_inv = one_layer_weights[path + ".weight_scale_inv"] \ - if path + ".weight_scale_inv" in one_layer_weights else None - return weight, weight_scale_inv - - def get_layer_mlp_dense_h_to_4h_bias(self, index, is_moe_mlp = False, expert_id = None, is_shared = False, - one_layer_weights=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_H_TO_4H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_H_TO_4H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_H_TO_4H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_H_TO_4H}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_mlp_dense_4h_to_h(self, index, weight, bias, is_moe_mlp=False, expert_id=None, is_shared=False, - one_layer_weights=None, weight_scale_inv=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_4H_TO_H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_4H_TO_H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_4H_TO_H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_4H_TO_H}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - if weight_scale_inv is not None: - self._set(path, "weight_scale_inv", weight_scale_inv) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - if weight_scale_inv is not None: - one_layer_weights[path + ".weight_scale_inv"] = weight_scale_inv - - def clear_layer_mlp_dense_4h_to_h(self, index, is_moe_mlp = False, expert_id = None, is_shared = False, - one_layer_weights=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_4H_TO_H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_4H_TO_H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_4H_TO_H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_4H_TO_H}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - self._clear(path, "weight_scale_inv") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - one_layer_weights[path + ".weight_scale_inv"] = None - - def get_layer_mlp_dense_4h_to_h_weight(self, index, is_moe_mlp = False, expert_id = None, is_shared = False, - one_layer_weights=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_4H_TO_H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_4H_TO_H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_4H_TO_H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_4H_TO_H}" - if one_layer_weights is None: - weight = self._get(path, "weight") - weight_scale_inv = self._get(path, "weight_scale_inv") - else: - weight = one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - weight_scale_inv = one_layer_weights[path + ".weight_scale_inv"] \ - if path + ".weight_scale_inv" in one_layer_weights else None - return weight, weight_scale_inv - - def get_layer_mlp_dense_4h_to_h_bias(self, index, is_moe_mlp = False, expert_id = None, is_shared = False, - one_layer_weights=None): - if is_moe_mlp: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_MLP}.{MLP_DENSE_4H_TO_H}" - elif is_shared: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_SHARED_EXPERT}.{MLP_DENSE_4H_TO_H}" - elif expert_id is None: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MLP_DENSE_4H_TO_H}" - else: - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_EXPERT}.{expert_id}.{MLP_DENSE_4H_TO_H}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_post_mlp_layernorm(self, index, weight, bias, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_MLP_LAYERNORM}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - - def clear_layer_post_mlp_layernorm(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_MLP_LAYERNORM}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - - def get_layer_post_mlp_layernorm_weight(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_MLP_LAYERNORM}" - if one_layer_weights is None: - return self._get(path, "weight") - else: - return one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - - def get_layer_post_mlp_layernorm_bias(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{POST_MLP_LAYERNORM}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_moe_gate(self, index, weight, bias, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_GATE}" - if one_layer_weights is None: - self._set(path, "weight", weight) - self._set(path, "bias", bias) - else: - one_layer_weights[path + ".weight"] = weight - one_layer_weights[path + ".bias"] = bias - - def clear_layer_moe_gate(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_GATE}" - if one_layer_weights is None: - self._clear(path, "weight") - self._clear(path, "bias") - else: - one_layer_weights[path + ".weight"] = None - one_layer_weights[path + ".bias"] = None - - def get_layer_moe_gate_weight(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_GATE}" - if one_layer_weights is None: - return self._get(path, "weight") - else: - return one_layer_weights[path + ".weight"] if path + ".weight" in one_layer_weights else None - - def get_layer_moe_gate_bias(self, index, one_layer_weights=None): - path = f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MOE_GATE}" - if one_layer_weights is None: - return self._get(path, "bias") - else: - return one_layer_weights[path + ".bias"] if path + ".bias" in one_layer_weights else None - - def set_layer_mtp_weight(self, index, mtp_word_embedding, mtp_enorm, mtp_hnorm, mtp_eh_proj, \ - mtp_shared_head_norm, mtp_shared_head_head, one_layer_weights=None): - if one_layer_weights is None: - self._set(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_WORD_EMBEDDING}", "weight", mtp_word_embedding) - self._set(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_ENORM}", "weight", mtp_enorm) - self._set(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_HNORM}", "weight", mtp_hnorm) - self._set(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_EH_PROJ}", "weight", mtp_eh_proj) - self._set(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_NORM}", "weight", mtp_shared_head_norm) - self._set(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_HEAD}", "weight", mtp_shared_head_head) - else: - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_WORD_EMBEDDING}.weight"] = mtp_word_embedding - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_ENORM}.weight"] = mtp_enorm - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_HNORM}.weight"] = mtp_hnorm - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_EH_PROJ}.weight"] = mtp_eh_proj - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_NORM}.weight"] = mtp_shared_head_norm - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_HEAD}.weight"] = mtp_shared_head_head - - def clear_layer_mtp_weight(self, index, one_layer_weights=None): - if one_layer_weights is None: - self._clear(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_WORD_EMBEDDING}", "weight") - self._clear(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_ENORM}", "weight") - self._clear(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_HNORM}", "weight") - self._clear(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_EH_PROJ}", "weight") - self._clear(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_NORM}", "weight") - self._clear(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_HEAD}", "weight") - else: - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_WORD_EMBEDDING}.weight"] = None - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_ENORM}.weight"] = None - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_HNORM}.weight"] = None - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_EH_PROJ}.weight"] = None - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_NORM}.weight"] = None - one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_HEAD}.weight"] = None - - def get_layer_mtp_weight(self, index, one_layer_weights=None): - if one_layer_weights is None: - mtp_word_embedding = self._get(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_WORD_EMBEDDING}", "weight") - mtp_enorm = self._get(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_ENORM}", "weight") - mtp_hnorm = self._get(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_HNORM}", "weight") - mtp_eh_proj = self._get(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_EH_PROJ}", "weight") - mtp_shared_head_norm = self._get(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_NORM}", "weight") - mtp_shared_head_head = self._get(f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_HEAD}", "weight") - else: - mtp_word_embedding = one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_WORD_EMBEDDING}.weight"] - mtp_enorm = one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_ENORM}.weight"] - mtp_hnorm = one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_HNORM}.weight"] - mtp_eh_proj = one_layer_weights[f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_EH_PROJ}.weight"] - mtp_shared_head_norm = one_layer_weights[ - f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_NORM}.weight"] - mtp_shared_head_head = one_layer_weights[ - f"{TRANSFORMER}.{LAYER_PREFIX}.{index}.{MTP_SHARED_HEAD_HEAD}.weight"] - - return (mtp_word_embedding, mtp_enorm, mtp_hnorm, mtp_eh_proj), (mtp_shared_head_norm, mtp_shared_head_head) - - def set_final_layernorm(self, weight, bias): - self._set(FINAL_LAYERNORM, "weight", weight) - self._set(FINAL_LAYERNORM, "bias", bias) - - def clear_final_layernorm(self): - self._clear(FINAL_LAYERNORM, "weight") - self._clear(FINAL_LAYERNORM, "bias") - - def get_final_layernorm_weight(self): - return self._get(FINAL_LAYERNORM, "weight") - - def get_final_layernorm_bias(self): - return self._get(FINAL_LAYERNORM, "bias") - - def set_word_embeddings_for_head(self, weight): - self._set(WORD_EMBEDDINGS_FOR_HEAD, "weight", weight) - - def clear_word_embeddings_for_head(self): - self._clear(WORD_EMBEDDINGS_FOR_HEAD, "weight") - - def get_word_embeddings_for_head_weight(self): - return self._get(WORD_EMBEDDINGS_FOR_HEAD, "weight") - - def _set(self, path, key, val): - if val is not None: - element = get_element_from_dict_by_path( - self.state_dict, path - ) - element[key] = val - - def _clear(self, path, key): - element = get_element_from_dict_by_path( - self.state_dict, path - ) - element[key] = None - - def _get(self, path, key): - element = get_element_from_dict_by_path( - self.state_dict, path - ) - return element[key] if key in element else None - - def has_optimizer(self): - return "optimizer" in self.state_dict diff --git a/tools/convert_checkpoint/common_config.py b/tools/convert_checkpoint/common_config.py deleted file mode 100644 index 63525d55..00000000 --- a/tools/convert_checkpoint/common_config.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import torch -import json - -from convert_checkpoint.abstact_config import AbstractConfig - - -class CommonConfig(AbstractConfig): - """ - CommonConfig - """ - - def __init__(self): - super().__init__() - - @staticmethod - def convert_from_common(*args, **kwargs): - """ - return config converted from common config - """ - raise NotImplementedError() - - def convert(self, config_class, *args, **kwargs): - """ - convert common config to config_class - """ - return config_class.convert_from_common(self, *args, **kwargs) - - def get_args(self, platform='common'): - """ return args of platform """ - return self.get("args").get(platform, {}) - - def update_args(self, args, platform='common'): - """ update args for platform """ - res = {k: v for k, v in args.items() if v is not None} - self.get("args").setdefault(platform, {}) - self.get("args").get(platform).update(res) - if res.get("torch_dtype") is not None: - self.update({"torch_dtype": res.get("torch_dtype")}) - - def get_name_map(self, platform): - """ return args of platform """ - return self.get("name_map").get(platform, {}) - - def get_dtype(self): - """ - return data type accoding to config - """ - target_params_dtype = self.get("torch_dtype") - if target_params_dtype == "float16": - return torch.float16 - elif target_params_dtype == "bfloat16": - return torch.bfloat16 - elif target_params_dtype == "float32": - return torch.float32 - else: - raise TypeError("Unsupported dtype {}".format(target_params_dtype)) - - def load(self, config_path): - """ - load config - """ - with open(config_path, 'r', encoding='utf-8') as f: - self.data = json.loads(f.read()) - diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-14b/adapter.json b/tools/convert_checkpoint/config/llava-ov-1.5-14b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-14b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-14b/qwen3.json b/tools/convert_checkpoint/config/llava-ov-1.5-14b/qwen3.json deleted file mode 100644 index 3a185876..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-14b/qwen3.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 40, - "hidden_size": 5120, - "ffn_hidden_size": 17408, - "num_attention_heads": 40, - "num_key_value_heads": 8, - "max_position_embeddings": 40960, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen3ForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "head_dim": 128, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 17408, - "max_position_embeddings": 40960, - "max_window_layers": 40, - "model_type": "qwen3", - "num_attention_heads": 40, - "num_hidden_layers": 40, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 1000000, - "sliding_window": null, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.51.0", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 151936 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.qkv_map": { - "attention.q_a_layernorm": "self_attn.q_norm", - "attention.kv_a_layernorm": "self_attn.k_norm" - }, - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.qkv_map": { - "attention.q_a_layernorm": {"name": "self_attention.q_layernorm", "is_layernorm": false}, - "attention.kv_a_layernorm": {"name": "self_attention.k_layernorm", "is_layernorm": false} - }, - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-model.json b/tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-model.json deleted file mode 100644 index 20d5d8c1..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 1024, - "ffn_hidden_size": 4096, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 24, - "embed_dim": 1024, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "float32", - "transformers_version": "4.53.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-patch.json b/tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-patch.json deleted file mode 100644 index e94b135c..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-14b/vision-patch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight", - "vision_model.class_embedding": "visual.class_embedding", - "vision_model.class_pos_emb": "visual.class_pos_emb", - "vision_model.pre_layernorm.weight": "visual.pre_layernorm.weight", - "vision_model.pre_layernorm.bias": "visual.pre_layernorm.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/adapter.json b/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/qwen3.json b/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/qwen3.json deleted file mode 100644 index 10b0a5e0..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/qwen3.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 48, - "hidden_size": 2048, - "ffn_hidden_size": 6144, - "num_attention_heads": 32, - "num_key_value_heads": 4, - "max_position_embeddings": 40960, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen3MoeForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 151643, - "decoder_sparse_step": 1, - "eos_token_id": 151645, - "head_dim": 128, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "intermediate_size": 6144, - "max_position_embeddings": 40960, - "max_window_layers": 48, - "mlp_only_layers": [], - "model_type": "qwen3_moe", - "moe_intermediate_size": 768, - "norm_topk_prob": true, - "num_attention_heads": 32, - "num_experts": 128, - "num_experts_per_tok": 8, - "num_hidden_layers": 48, - "num_key_value_heads": 4, - "output_router_logits": false, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 1000000.0, - "router_aux_loss_coef": 0.001, - "sliding_window": null, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.51.0", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 151936 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.qkv_map": { - "attention.q_a_layernorm": "self_attn.q_norm", - "attention.kv_a_layernorm": "self_attn.k_norm" - }, - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "moe.gate": "mlp.gate", - "moe.expert": "mlp.experts", - "mlp.dense_h_to_4h": [ - "gate_proj", - "up_proj" - ], - "mlp.dense_4h_to_h": "down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.qkv_map": { - "attention.q_a_layernorm": {"name": "self_attention.q_layernorm", "is_layernorm": false}, - "attention.kv_a_layernorm": {"name": "self_attention.k_layernorm", "is_layernorm": false} - }, - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "pre_mlp_layernorm", - "moe.gate": "mlp.router", - "moe.expert": "mlp.experts.local_experts", - "mlp.dense_h_to_4h": "linear_fc1", - "mlp.dense_4h_to_h": "linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-model.json b/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-model.json deleted file mode 100644 index 20d5d8c1..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 1024, - "ffn_hidden_size": 4096, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 24, - "embed_dim": 1024, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "float32", - "transformers_version": "4.53.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-patch.json b/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-patch.json deleted file mode 100644 index e94b135c..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-30b-a3b/vision-patch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight", - "vision_model.class_embedding": "visual.class_embedding", - "vision_model.class_pos_emb": "visual.class_pos_emb", - "vision_model.pre_layernorm.weight": "visual.pre_layernorm.weight", - "vision_model.pre_layernorm.bias": "visual.pre_layernorm.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-3b/adapter.json b/tools/convert_checkpoint/config/llava-ov-1.5-3b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-3b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-3b/qwen2_5.json b/tools/convert_checkpoint/config/llava-ov-1.5-3b/qwen2_5.json deleted file mode 100644 index 3478b0ed..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-3b/qwen2_5.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 36, - "hidden_size": 2048, - "ffn_hidden_size": 11008, - "num_attention_heads": 16, - "num_key_value_heads": 2, - "max_position_embeddings": 128000, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2_5_VLForConditionalGeneration" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "vision_start_token_id": 151652, - "vision_end_token_id": 151653, - "vision_token_id": 151654, - "image_token_id": 151655, - "video_token_id": 151656, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "intermediate_size": 11008, - "max_position_embeddings": 128000, - "max_window_layers": 70, - "num_attention_heads": 16, - "num_hidden_layers": 36, - "num_key_value_heads": 2, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-model.json b/tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-model.json deleted file mode 100644 index 20d5d8c1..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 1024, - "ffn_hidden_size": 4096, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 24, - "embed_dim": 1024, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "float32", - "transformers_version": "4.53.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-patch.json b/tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-patch.json deleted file mode 100644 index e94b135c..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-3b/vision-patch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight", - "vision_model.class_embedding": "visual.class_embedding", - "vision_model.class_pos_emb": "visual.class_pos_emb", - "vision_model.pre_layernorm.weight": "visual.pre_layernorm.weight", - "vision_model.pre_layernorm.bias": "visual.pre_layernorm.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-4b/adapter.json b/tools/convert_checkpoint/config/llava-ov-1.5-4b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-4b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json b/tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json deleted file mode 100644 index 48cf997e..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-4b/fastvit.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model_type": "fastvit", - "vision_model_from": "model.vision_tower.vision_tower", - "vision_model_to": "vision_model.vision_tower", - "num_layers": null, - "hidden_size": 3072, - "note": "FastViT/MobileCLIP vision encoder with 629 parameters" -} diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-4b/qwen3.json b/tools/convert_checkpoint/config/llava-ov-1.5-4b/qwen3.json deleted file mode 100644 index 03138d9a..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-4b/qwen3.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 36, - "hidden_size": 2560, - "ffn_hidden_size": 9728, - "num_attention_heads": 32, - "num_key_value_heads": 8, - "max_position_embeddings": 40960, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen3ForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "head_dim": 128, - "hidden_act": "silu", - "hidden_size": 2560, - "initializer_range": 0.02, - "intermediate_size": 9728, - "max_position_embeddings": 40960, - "max_window_layers": 36, - "model_type": "qwen3", - "num_attention_heads": 32, - "num_hidden_layers": 36, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 1000000, - "sliding_window": null, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "transformers_version": "4.51.0", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 151936 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.qkv_map": { - "attention.q_a_layernorm": "self_attn.q_norm", - "attention.kv_a_layernorm": "self_attn.k_norm" - }, - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.qkv_map": { - "attention.q_a_layernorm": {"name": "self_attention.q_layernorm", "is_layernorm": false}, - "attention.kv_a_layernorm": {"name": "self_attention.k_layernorm", "is_layernorm": false} - }, - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-model.json b/tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-model.json deleted file mode 100644 index 20d5d8c1..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 1024, - "ffn_hidden_size": 4096, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 24, - "embed_dim": 1024, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "float32", - "transformers_version": "4.53.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-patch.json b/tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-patch.json deleted file mode 100644 index e94b135c..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-4b/vision-patch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight", - "vision_model.class_embedding": "visual.class_embedding", - "vision_model.class_pos_emb": "visual.class_pos_emb", - "vision_model.pre_layernorm.weight": "visual.pre_layernorm.weight", - "vision_model.pre_layernorm.bias": "visual.pre_layernorm.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-8b/adapter.json b/tools/convert_checkpoint/config/llava-ov-1.5-8b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-8b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-8b/qwen3.json b/tools/convert_checkpoint/config/llava-ov-1.5-8b/qwen3.json deleted file mode 100644 index 771b7e87..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-8b/qwen3.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 36, - "hidden_size": 4096, - "ffn_hidden_size": 12288, - "num_attention_heads": 32, - "num_key_value_heads": 8, - "max_position_embeddings": 40960, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen3ForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "head_dim": 128, - "hidden_act": "silu", - "hidden_size": 4096, - "initializer_range": 0.02, - "intermediate_size": 12288, - "max_position_embeddings": 40960, - "max_window_layers": 36, - "model_type": "qwen3", - "num_attention_heads": 32, - "num_hidden_layers": 36, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 1000000, - "sliding_window": null, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.51.0", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 151936 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.qkv_map": { - "attention.q_a_layernorm": "self_attn.q_norm", - "attention.kv_a_layernorm": "self_attn.k_norm" - }, - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.qkv_map": { - "attention.q_a_layernorm": {"name": "self_attention.q_layernorm", "is_layernorm": false}, - "attention.kv_a_layernorm": {"name": "self_attention.k_layernorm", "is_layernorm": false} - }, - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-model.json b/tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-model.json deleted file mode 100644 index 20d5d8c1..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 1024, - "ffn_hidden_size": 4096, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 24, - "embed_dim": 1024, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "float32", - "transformers_version": "4.53.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-patch.json b/tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-patch.json deleted file mode 100644 index e94b135c..00000000 --- a/tools/convert_checkpoint/config/llava-ov-1.5-8b/vision-patch.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight", - "vision_model.class_embedding": "visual.class_embedding", - "vision_model.class_pos_emb": "visual.class_pos_emb", - "vision_model.pre_layernorm.weight": "visual.pre_layernorm.weight", - "vision_model.pre_layernorm.bias": "visual.pre_layernorm.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/mobilellm-140m.json b/tools/convert_checkpoint/config/mobilellm-140m.json deleted file mode 100644 index 9d7156d0..00000000 --- a/tools/convert_checkpoint/config/mobilellm-140m.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 15, - "hidden_size": 576, - "ffn_hidden_size": 2048, - "num_attention_heads": 9, - "num_key_value_heads": 3, - "max_position_embeddings": 32768, - "vocab_size": 128256 - }, - "huggingface": { - "architectures": [ - "Llama4ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 128000, - "eos_token_id": [128001, 128008, 128009], - "hidden_act": "silu", - "initializer_range": 0.02, - "model_type": "llama4_text", - "rms_norm_eps": 1e-05, - "rope_theta": 8000000.0, - "tie_word_embeddings": true, - "use_cache": true - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "feed_forward.gate_proj", - "feed_forward.up_proj" - ], - "mlp.dense_4h_to_h": "feed_forward.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.55.0" -} diff --git a/tools/convert_checkpoint/config/qwen-1.8b.json b/tools/convert_checkpoint/config/qwen-1.8b.json deleted file mode 100644 index b13a3907..00000000 --- a/tools/convert_checkpoint/config/qwen-1.8b.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 2048, - "ffn_hidden_size": 5504, - "max_sequence_length": 2048, - "num_attention_heads": 16, - "max_position_embeddings": 8192, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "QWenLMHeadModel" - ], - "auto_map": { - "AutoConfig": "configuration_qwen.QWenConfig", - "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel" - }, - "attn_dropout_prob": 0.0, - "bf16": false, - "emb_dropout_prob": 0.0, - "fp16": false, - "fp32": false, - "hidden_size": 2048, - "intermediate_size": 11008, - "initializer_range": 0.02, - "kv_channels": 128, - "layer_norm_epsilon": 1e-06, - "max_position_embeddings": 8192, - "model_type": "qwen", - "no_bias": true, - "num_attention_heads": 16, - "num_hidden_layers": 24, - "onnx_safe": null, - "rotary_emb_base": 10000, - "rotary_pct": 1.0, - "scale_attn_weights": true, - "seq_length": 8192, - "tie_word_embeddings": false, - "tokenizer_class": "QWenTokenizer", - "use_cache": true, - "use_dynamic_ntk": true, - "use_flash_attn": "auto", - "use_logn_attn": true - }, - "megatron": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "transformer.wte", - "transformer": "transformer", - "layer_prefix": "h", - "input_layernorm": "ln_1", - "attention.query_key_value": "attn.c_attn", - "attention.dense": "attn.c_proj", - "post_attention_layernorm": "ln_2", - "mlp.dense_h_to_4h": [ - "mlp.w2", - "mlp.w1" - ], - "mlp.dense_4h_to_h": "mlp.c_proj", - "final_layernorm": "transformer.ln_f", - "word_embeddings_for_head": "lm_head" - }, - "megatron": { - "word_embeddings": "model.language_model.embedding.word_embeddings", - "word_position_embeddings": "model.language_model.embedding.position_embeddings", - "transformer": "model.language_model.encoder", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": "self_attention.query_key_value", - "attention.dense": "self_attention.dense", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": "mlp.dense_h_to_4h", - "mlp.dense_4h_to_h": "mlp.dense_4h_to_h", - "final_layernorm": "final_layernorm", - "word_embeddings_for_head": "model.language_model.output_layer", - "word_embeddings_tpl": "model%d.language_model.embedding.word_embeddings", - "word_position_embeddings_tpl": "model%d.language_model.embedding.position_embeddings", - "transformer_tpl": "model%d.language_model.encoder", - "word_embeddings_for_head_tpl": "model%d.language_model.output_layer" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.32.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen-14b.json b/tools/convert_checkpoint/config/qwen-14b.json deleted file mode 100644 index 6254dd93..00000000 --- a/tools/convert_checkpoint/config/qwen-14b.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 40, - "hidden_size": 5120, - "ffn_hidden_size": 13696, - "max_sequence_length": 2048, - "num_attention_heads": 40, - "max_position_embeddings": 8192, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "QWenLMHeadModel" - ], - "auto_map": { - "AutoConfig": "configuration_qwen.QWenConfig", - "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel" - }, - "attn_dropout_prob": 0.0, - "bf16": false, - "emb_dropout_prob": 0.0, - "fp16": false, - "fp32": false, - "hidden_size": 5120, - "intermediate_size": 27392, - "initializer_range": 0.02, - "kv_channels": 128, - "layer_norm_epsilon": 1e-06, - "max_position_embeddings": 8192, - "model_type": "qwen", - "no_bias": true, - "num_attention_heads": 40, - "num_hidden_layers": 40, - "onnx_safe": null, - "rotary_emb_base": 10000, - "rotary_pct": 1.0, - "scale_attn_weights": true, - "seq_length": 2048, - "tie_word_embeddings": false, - "tokenizer_class": "QWenTokenizer", - "use_cache": true, - "use_dynamic_ntk": true, - "use_flash_attn": "auto", - "use_logn_attn": true - }, - "megatron": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "transformer.wte", - "transformer": "transformer", - "layer_prefix": "h", - "input_layernorm": "ln_1", - "attention.query_key_value": "attn.c_attn", - "attention.dense": "attn.c_proj", - "post_attention_layernorm": "ln_2", - "mlp.dense_h_to_4h": [ - "mlp.w2", - "mlp.w1" - ], - "mlp.dense_4h_to_h": "mlp.c_proj", - "final_layernorm": "transformer.ln_f", - "word_embeddings_for_head": "lm_head" - }, - "megatron": { - "word_embeddings": "model.language_model.embedding.word_embeddings", - "word_position_embeddings": "model.language_model.embedding.position_embeddings", - "transformer": "model.language_model.encoder", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": "self_attention.query_key_value", - "attention.dense": "self_attention.dense", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": "mlp.dense_h_to_4h", - "mlp.dense_4h_to_h": "mlp.dense_4h_to_h", - "final_layernorm": "final_layernorm", - "word_embeddings_for_head": "model.language_model.output_layer", - "word_embeddings_tpl": "model%d.language_model.embedding.word_embeddings", - "word_position_embeddings_tpl": "model%d.language_model.embedding.position_embeddings", - "transformer_tpl": "model%d.language_model.encoder", - "word_embeddings_for_head_tpl": "model%d.language_model.output_layer" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.32.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen-72b.json b/tools/convert_checkpoint/config/qwen-72b.json deleted file mode 100644 index 92261fde..00000000 --- a/tools/convert_checkpoint/config/qwen-72b.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 80, - "hidden_size": 8192, - "ffn_hidden_size": 24576, - "max_sequence_length": 32768, - "num_attention_heads": 64, - "max_position_embeddings": 32768, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "QWenLMHeadModel" - ], - "auto_map": { - "AutoConfig": "configuration_qwen.QWenConfig", - "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel" - }, - "attn_dropout_prob": 0.0, - "bf16": false, - "emb_dropout_prob": 0.0, - "fp16": false, - "fp32": false, - "hidden_size": 8192, - "intermediate_size": 49152, - "initializer_range": 0.02, - "kv_channels": 128, - "layer_norm_epsilon": 1e-06, - "max_position_embeddings": 32768, - "model_type": "qwen", - "no_bias": true, - "num_attention_heads": 64, - "num_hidden_layers": 80, - "onnx_safe": null, - "rope_theta": 1000000, - "rotary_emb_base": 1000000, - "rotary_pct": 1.0, - "scale_attn_weights": true, - "seq_length": 32768, - "tie_word_embeddings": false, - "tokenizer_class": "QWenTokenizer", - "use_cache": true, - "use_dynamic_ntk": false, - "use_flash_attn": "auto", - "use_logn_attn": false - }, - "megatron": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "transformer.wte", - "transformer": "transformer", - "layer_prefix": "h", - "input_layernorm": "ln_1", - "attention.query_key_value": "attn.c_attn", - "attention.dense": "attn.c_proj", - "post_attention_layernorm": "ln_2", - "mlp.dense_h_to_4h": [ - "mlp.w2", - "mlp.w1" - ], - "mlp.dense_4h_to_h": "mlp.c_proj", - "final_layernorm": "transformer.ln_f", - "word_embeddings_for_head": "lm_head" - }, - "megatron": { - "word_embeddings": "model.language_model.embedding.word_embeddings", - "word_position_embeddings": "model.language_model.embedding.position_embeddings", - "transformer": "model.language_model.encoder", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": "self_attention.query_key_value", - "attention.dense": "self_attention.dense", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": "mlp.dense_h_to_4h", - "mlp.dense_4h_to_h": "mlp.dense_4h_to_h", - "final_layernorm": "final_layernorm", - "word_embeddings_for_head": "model.language_model.output_layer", - "word_embeddings_tpl": "model%d.language_model.embedding.word_embeddings", - "word_position_embeddings_tpl": "model%d.language_model.embedding.position_embeddings", - "transformer_tpl": "model%d.language_model.encoder", - "word_embeddings_for_head_tpl": "model%d.language_model.output_layer" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.32.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen-7b.json b/tools/convert_checkpoint/config/qwen-7b.json deleted file mode 100644 index f865da94..00000000 --- a/tools/convert_checkpoint/config/qwen-7b.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 4096, - "ffn_hidden_size": 11008, - "max_sequence_length": 2048, - "num_attention_heads": 32, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "QWenLMHeadModel" - ], - "auto_map": { - "AutoConfig": "configuration_qwen.QWenConfig", - "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel" - }, - "attn_dropout_prob": 0.0, - "bf16": false, - "emb_dropout_prob": 0.0, - "fp16": false, - "fp32": false, - "hidden_size": 4096, - "intermediate_size": 22016, - "initializer_range": 0.02, - "kv_channels": 128, - "layer_norm_epsilon": 1e-06, - "max_position_embeddings": 32768, - "model_type": "qwen", - "no_bias": true, - "num_attention_heads": 32, - "num_hidden_layers": 32, - "onnx_safe": null, - "rotary_emb_base": 10000, - "rotary_pct": 1.0, - "scale_attn_weights": true, - "seq_length": 8192, - "tie_word_embeddings": false, - "tokenizer_class": "QWenTokenizer", - "use_cache": true, - "use_dynamic_ntk": true, - "use_flash_attn": "auto", - "use_logn_attn": true - }, - "megatron": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "transformer.wte", - "transformer": "transformer", - "layer_prefix": "h", - "input_layernorm": "ln_1", - "attention.query_key_value": "attn.c_attn", - "attention.dense": "attn.c_proj", - "post_attention_layernorm": "ln_2", - "mlp.dense_h_to_4h": [ - "mlp.w2", - "mlp.w1" - ], - "mlp.dense_4h_to_h": "mlp.c_proj", - "final_layernorm": "transformer.ln_f", - "word_embeddings_for_head": "lm_head" - }, - "megatron": { - "word_embeddings": "model.language_model.embedding.word_embeddings", - "word_position_embeddings": "model.language_model.embedding.position_embeddings", - "transformer": "model.language_model.encoder", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": "self_attention.query_key_value", - "attention.dense": "self_attention.dense", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": "mlp.dense_h_to_4h", - "mlp.dense_4h_to_h": "mlp.dense_4h_to_h", - "final_layernorm": "final_layernorm", - "word_embeddings_for_head": "model.language_model.output_layer", - "word_embeddings_tpl": "model%d.language_model.embedding.word_embeddings", - "word_position_embeddings_tpl": "model%d.language_model.embedding.position_embeddings", - "transformer_tpl": "model%d.language_model.encoder", - "word_embeddings_for_head_tpl": "model%d.language_model.output_layer" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.32.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-0.5b.json b/tools/convert_checkpoint/config/qwen1.5-0.5b.json deleted file mode 100644 index 2329253b..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-0.5b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 1024, - "ffn_hidden_size": 2816, - "num_attention_heads": 16, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 1024, - "initializer_range": 0.02, - "intermediate_size": 2816, - "max_position_embeddings": 32768, - "max_window_layers": 21, - "model_type": "qwen2", - "num_attention_heads": 16, - "num_hidden_layers": 24, - "num_key_value_heads": 16, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-1.8b.json b/tools/convert_checkpoint/config/qwen1.5-1.8b.json deleted file mode 100644 index 12641e6d..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-1.8b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 2048, - "ffn_hidden_size": 5504, - "num_attention_heads": 16, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "intermediate_size": 5504, - "max_position_embeddings": 32768, - "max_window_layers": 21, - "model_type": "qwen2", - "num_attention_heads": 16, - "num_hidden_layers": 24, - "num_key_value_heads": 16, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-14b.json b/tools/convert_checkpoint/config/qwen1.5-14b.json deleted file mode 100644 index 09fcd26f..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-14b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 40, - "hidden_size": 5120, - "ffn_hidden_size": 13696, - "num_attention_heads": 40, - "max_position_embeddings": 32768, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 13696, - "max_position_embeddings": 32768, - "max_window_layers": 35, - "model_type": "qwen2", - "num_attention_heads": 40, - "num_hidden_layers": 40, - "num_key_value_heads": 40, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-32b.json b/tools/convert_checkpoint/config/qwen1.5-32b.json deleted file mode 100644 index ba71bf9e..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-32b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 64, - "hidden_size": 5120, - "ffn_hidden_size": 27392, - "num_attention_heads": 40, - "num_key_value_heads": 8, - "max_position_embeddings": 32768, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 27392, - "max_position_embeddings": 32768, - "max_window_layers": 35, - "model_type": "qwen2", - "num_attention_heads": 40, - "num_hidden_layers": 64, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-4b.json b/tools/convert_checkpoint/config/qwen1.5-4b.json deleted file mode 100644 index 92d18fb5..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-4b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 40, - "hidden_size": 2560, - "ffn_hidden_size": 6912, - "num_attention_heads": 20, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 2560, - "initializer_range": 0.02, - "intermediate_size": 6912, - "max_position_embeddings": 32768, - "max_window_layers": 21, - "model_type": "qwen2", - "num_attention_heads": 20, - "num_hidden_layers": 40, - "num_key_value_heads": 20, - "rms_norm_eps": 1e-06, - "rope_theta": 5000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-72b.json b/tools/convert_checkpoint/config/qwen1.5-72b.json deleted file mode 100644 index e410273b..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-72b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 80, - "hidden_size": 8192, - "ffn_hidden_size": 24576, - "num_attention_heads": 64, - "max_position_embeddings": 32768, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 8192, - "initializer_range": 0.02, - "intermediate_size": 24576, - "max_position_embeddings": 32768, - "max_window_layers": 28, - "model_type": "qwen2", - "num_attention_heads": 64, - "num_hidden_layers": 80, - "num_key_value_heads": 64, - "rms_norm_eps": 1e-05, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen1.5-7b.json b/tools/convert_checkpoint/config/qwen1.5-7b.json deleted file mode 100644 index cc7955c1..00000000 --- a/tools/convert_checkpoint/config/qwen1.5-7b.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 4096, - "ffn_hidden_size": 11008, - "num_attention_heads": 32, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 4096, - "initializer_range": 0.02, - "intermediate_size": 11008, - "max_position_embeddings": 32768, - "max_window_layers": 28, - "model_type": "qwen2", - "num_attention_heads": 32, - "num_hidden_layers": 32, - "num_key_value_heads": 32, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-0.5b.json b/tools/convert_checkpoint/config/qwen2-0.5b.json deleted file mode 100644 index 9c0d37d1..00000000 --- a/tools/convert_checkpoint/config/qwen2-0.5b.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 896, - "ffn_hidden_size": 4864, - "num_attention_heads": 14, - "num_key_value_heads": 2, - "max_position_embeddings": 131072, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_window_layers": 24, - "model_type": "qwen2", - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": true, - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-1.5b.json b/tools/convert_checkpoint/config/qwen2-1.5b.json deleted file mode 100644 index 088be82f..00000000 --- a/tools/convert_checkpoint/config/qwen2-1.5b.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 1536, - "ffn_hidden_size": 8960, - "num_attention_heads": 12, - "num_key_value_heads": 2, - "max_position_embeddings": 131072, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_position_embeddings": 131072, - "max_window_layers": 28, - "model_type": "qwen2", - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": true, - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-72b.json b/tools/convert_checkpoint/config/qwen2-72b.json deleted file mode 100644 index 1b27b681..00000000 --- a/tools/convert_checkpoint/config/qwen2-72b.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 80, - "hidden_size": 8192, - "ffn_hidden_size": 29568, - "num_attention_heads": 64, - "num_key_value_heads": 8, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_position_embeddings": 131072, - "max_window_layers": 80, - "model_type": "qwen2", - "rms_norm_eps": 1e-05, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 152064 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-7b.json b/tools/convert_checkpoint/config/qwen2-7b.json deleted file mode 100644 index 2efc8b97..00000000 --- a/tools/convert_checkpoint/config/qwen2-7b.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 3584, - "ffn_hidden_size": 18944, - "num_attention_heads": 28, - "num_key_value_heads": 4, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 3584, - "initializer_range": 0.02, - "intermediate_size": 18944, - "max_position_embeddings": 131072, - "max_window_layers": 28, - "model_type": "qwen2", - "num_attention_heads": 28, - "num_hidden_layers": 28, - "num_key_value_heads": 4, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-2b/adapter.json b/tools/convert_checkpoint/config/qwen2-vl-2b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-2b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-2b/qwen2.json b/tools/convert_checkpoint/config/qwen2-vl-2b/qwen2.json deleted file mode 100644 index 6077060f..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-2b/qwen2.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 1536, - "ffn_hidden_size": 8960, - "num_attention_heads": 12, - "num_key_value_heads": 2, - "max_position_embeddings": 131072, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 1536, - "initializer_range": 0.02, - "intermediate_size": 8960, - "max_position_embeddings": 131072, - "max_window_layers": 28, - "model_type": "qwen2", - "num_attention_heads": 12, - "num_hidden_layers": 12, - "num_key_value_heads": 2, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-2b/vision-model.json b/tools/convert_checkpoint/config/qwen2-vl-2b/vision-model.json deleted file mode 100644 index e2b962f9..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-2b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 5120, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "embed_dim": 1280, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-2b/vision-patch.json b/tools/convert_checkpoint/config/qwen2-vl-2b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-2b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-72b/adapter.json b/tools/convert_checkpoint/config/qwen2-vl-72b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-72b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-72b/qwen2.json b/tools/convert_checkpoint/config/qwen2-vl-72b/qwen2.json deleted file mode 100644 index c96ce6ed..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-72b/qwen2.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 80, - "hidden_size": 8192, - "ffn_hidden_size": 29568, - "num_attention_heads": 64, - "num_key_value_heads": 8, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 8192, - "initializer_range": 0.02, - "intermediate_size": 29568, - "max_position_embeddings": 131072, - "max_window_layers": 80, - "model_type": "qwen2", - "num_attention_heads": 64, - "num_hidden_layers": 80, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-72b/vision-model.json b/tools/convert_checkpoint/config/qwen2-vl-72b/vision-model.json deleted file mode 100644 index e2b962f9..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-72b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 5120, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "embed_dim": 1280, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-72b/vision-patch.json b/tools/convert_checkpoint/config/qwen2-vl-72b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-72b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-7b/adapter.json b/tools/convert_checkpoint/config/qwen2-vl-7b/adapter.json deleted file mode 100644 index 8f819a44..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-7b/adapter.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.layernorm.bias": "visual.merger.ln_q.bias", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-7b/qwen2.json b/tools/convert_checkpoint/config/qwen2-vl-7b/qwen2.json deleted file mode 100644 index 291b563b..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-7b/qwen2.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 3584, - "ffn_hidden_size": 18944, - "num_attention_heads": 28, - "num_key_value_heads": 4, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 3584, - "initializer_range": 0.02, - "intermediate_size": 18944, - "max_position_embeddings": 131072, - "max_window_layers": 28, - "model_type": "qwen2", - "num_attention_heads": 28, - "num_hidden_layers": 28, - "num_key_value_heads": 4, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-7b/vision-model.json b/tools/convert_checkpoint/config/qwen2-vl-7b/vision-model.json deleted file mode 100644 index e2b962f9..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-7b/vision-model.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 5120, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "embed_dim": 1280, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": false, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": "mlp.fc1", - "mlp.dense_4h_to_h": "mlp.fc2" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2-vl-7b/vision-patch.json b/tools/convert_checkpoint/config/qwen2-vl-7b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2-vl-7b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-0.5b.json b/tools/convert_checkpoint/config/qwen2.5-0.5b.json deleted file mode 100644 index 5e50871f..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-0.5b.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 24, - "hidden_size": 896, - "ffn_hidden_size": 4864, - "num_attention_heads": 14, - "num_key_value_heads": 2, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_window_layers": 24, - "model_type": "qwen2", - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": true, - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-1.5b.json b/tools/convert_checkpoint/config/qwen2.5-1.5b.json deleted file mode 100644 index ef1d9a5e..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-1.5b.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 1536, - "ffn_hidden_size": 8960, - "num_attention_heads": 12, - "num_key_value_heads": 2, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_position_embeddings": 32768, - "max_window_layers": 28, - "model_type": "qwen2", - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": true, - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-14b.json b/tools/convert_checkpoint/config/qwen2.5-14b.json deleted file mode 100644 index 4069ec6a..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-14b.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 48, - "hidden_size": 5120, - "ffn_hidden_size": 13824, - "num_attention_heads": 40, - "num_key_value_heads": 8, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 13824, - "max_position_embeddings": 131072, - "max_window_layers": 48, - "model_type": "qwen2", - "num_attention_heads": 40, - "num_hidden_layers": 48, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-05, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-32b.json b/tools/convert_checkpoint/config/qwen2.5-32b.json deleted file mode 100644 index b86ba200..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-32b.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 64, - "hidden_size": 5120, - "ffn_hidden_size": 27648, - "num_attention_heads": 40, - "num_key_value_heads": 8, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 27648, - "max_position_embeddings": 131072, - "max_window_layers": 64, - "model_type": "qwen2", - "num_attention_heads": 40, - "num_hidden_layers": 64, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-05, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-3b.json b/tools/convert_checkpoint/config/qwen2.5-3b.json deleted file mode 100644 index 2dc9b003..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-3b.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 36, - "hidden_size": 2048, - "ffn_hidden_size": 11008, - "num_attention_heads": 16, - "num_key_value_heads": 2, - "max_position_embeddings": 32768, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_position_embeddings": 32768, - "max_window_layers": 36, - "model_type": "qwen2", - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": true, - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-72b.json b/tools/convert_checkpoint/config/qwen2.5-72b.json deleted file mode 100644 index 9d12835b..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-72b.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 80, - "hidden_size": 8192, - "ffn_hidden_size": 29568, - "num_attention_heads": 64, - "num_key_value_heads": 8, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "initializer_range": 0.02, - "max_position_embeddings": 131072, - "max_window_layers": 80, - "model_type": "qwen2", - "rms_norm_eps": 1e-05, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 152064 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2.5-7b.json b/tools/convert_checkpoint/config/qwen2.5-7b.json deleted file mode 100644 index c8c58034..00000000 --- a/tools/convert_checkpoint/config/qwen2.5-7b.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 3584, - "ffn_hidden_size": 18944, - "num_attention_heads": 28, - "num_key_value_heads": 4, - "max_position_embeddings": 131072, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2ForCausalLM" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 3584, - "initializer_range": 0.02, - "intermediate_size": 18944, - "max_position_embeddings": 131072, - "max_window_layers": 28, - "model_type": "qwen2", - "num_attention_heads": 28, - "num_hidden_layers": 28, - "num_key_value_heads": 4, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 131072, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-32b/adapter.json b/tools/convert_checkpoint/config/qwen2_5-vl-32b/adapter.json deleted file mode 100644 index 0ba7269e..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-32b/adapter.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-32b/qwen2_5.json b/tools/convert_checkpoint/config/qwen2_5-vl-32b/qwen2_5.json deleted file mode 100644 index 29485a17..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-32b/qwen2_5.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 64, - "hidden_size": 5120, - "ffn_hidden_size": 27648, - "num_attention_heads": 40, - "num_key_value_heads": 8, - "max_position_embeddings": 128000, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2_5_VLForConditionalGeneration" - ], - "attention_dropout": 0.0, - "pad_token_id": 151643, - "eos_token_id": 151645, - "vision_start_token_id": 151652, - "vision_end_token_id": 151653, - "vision_token_id": 151654, - "image_token_id": 151655, - "video_token_id": 151656, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 27648, - "max_position_embeddings": 128000, - "max_window_layers": 64, - "model_type": "qwen2_5_vl", - "num_attention_heads": 40, - "num_hidden_layers": 64, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.49.0" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-model.json b/tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-model.json deleted file mode 100644 index cceffa2f..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-model.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 3456, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2, - "hidden_act": "silu", - "hidden_size": 1280, - "intermediate_size": 3420, - "out_hidden_size": 5120, - "window_size": 112, - "fullatt_block_indexes": [ - 7, - 15, - 23, - 31 - ], - "tokens_per_second": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-patch.json b/tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-32b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-3b/adapter.json b/tools/convert_checkpoint/config/qwen2_5-vl-3b/adapter.json deleted file mode 100644 index 0ba7269e..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-3b/adapter.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-3b/qwen2_5.json b/tools/convert_checkpoint/config/qwen2_5-vl-3b/qwen2_5.json deleted file mode 100644 index 3478b0ed..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-3b/qwen2_5.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 36, - "hidden_size": 2048, - "ffn_hidden_size": 11008, - "num_attention_heads": 16, - "num_key_value_heads": 2, - "max_position_embeddings": 128000, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen2_5_VLForConditionalGeneration" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "vision_start_token_id": 151652, - "vision_end_token_id": 151653, - "vision_token_id": 151654, - "image_token_id": 151655, - "video_token_id": 151656, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "intermediate_size": 11008, - "max_position_embeddings": 128000, - "max_window_layers": 70, - "num_attention_heads": 16, - "num_hidden_layers": 36, - "num_key_value_heads": 2, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": false, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-model.json b/tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-model.json deleted file mode 100644 index 9d1761e3..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-model.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 3420, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2, - "hidden_act": "silu", - "hidden_size": 1280, - "intermediate_size": 3420, - "out_hidden_size": 2048, - "window_size": 112, - "fullatt_block_indexes": [ - 7, - 15, - 23, - 31 - ], - "tokens_per_second": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-patch.json b/tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-3b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-72b/adapter.json b/tools/convert_checkpoint/config/qwen2_5-vl-72b/adapter.json deleted file mode 100644 index 0ba7269e..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-72b/adapter.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-72b/qwen2_5.json b/tools/convert_checkpoint/config/qwen2_5-vl-72b/qwen2_5.json deleted file mode 100644 index 67cba2ea..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-72b/qwen2_5.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 80, - "hidden_size": 8192, - "ffn_hidden_size": 29568, - "num_attention_heads": 64, - "num_key_value_heads": 8, - "max_position_embeddings": 128000, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2_5_VLForConditionalGeneration" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151643, - "hidden_act": "silu", - "hidden_size": 8192, - "initializer_range": 0.02, - "intermediate_size": 29568, - "max_position_embeddings": 128000, - "max_window_layers": 80, - "model_type": "qwen2_5_vl", - "num_attention_heads": 64, - "num_hidden_layers": 80, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-model.json b/tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-model.json deleted file mode 100644 index aaf6c3c2..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-model.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 3456, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2, - "hidden_act": "silu", - "hidden_size": 1280, - "intermediate_size": 3420, - "out_hidden_size": 8192, - "window_size": 112, - "fullatt_block_indexes": [ - 7, - 15, - 23, - 31 - ], - "tokens_per_second": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-patch.json b/tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-72b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-7b/adapter.json b/tools/convert_checkpoint/config/qwen2_5-vl-7b/adapter.json deleted file mode 100644 index 0ba7269e..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-7b/adapter.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "adapter.layernorm.weight": "visual.merger.ln_q.weight", - "adapter.linear_fc1.weight": "visual.merger.mlp.0.weight", - "adapter.linear_fc1.bias": "visual.merger.mlp.0.bias", - "adapter.linear_fc2.weight": "visual.merger.mlp.2.weight", - "adapter.linear_fc2.bias": "visual.merger.mlp.2.bias" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-7b/qwen2_5.json b/tools/convert_checkpoint/config/qwen2_5-vl-7b/qwen2_5.json deleted file mode 100644 index e4b091db..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-7b/qwen2_5.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 28, - "hidden_size": 3584, - "ffn_hidden_size": 18944, - "num_attention_heads": 28, - "num_key_value_heads": 4, - "max_position_embeddings": 128000, - "vocab_size": 152064 - }, - "huggingface": { - "architectures": [ - "Qwen2_5_VLForConditionalGeneration" - ], - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "vision_start_token_id": 151652, - "vision_end_token_id": 151653, - "vision_token_id": 151654, - "image_token_id": 151655, - "video_token_id": 151656, - "hidden_act": "silu", - "hidden_size": 3584, - "initializer_range": 0.02, - "intermediate_size": 18944, - "max_position_embeddings": 128000, - "max_window_layers": 28, - "model_type": "qwen2_5_vl", - "num_attention_heads": 28, - "num_hidden_layers": 28, - "num_key_value_heads": 4, - "rms_norm_eps": 1e-06, - "rope_theta": 1000000.0, - "sliding_window": 32768, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "use_cache": true, - "use_sliding_window": false - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "language_model.embedding.word_embeddings", - "transformer": "model", - "layer_prefix": "language_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "language_model.decoder.final_layernorm", - "word_embeddings_for_head": "language_model.output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.37.2" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-model.json b/tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-model.json deleted file mode 100644 index ec16cc5c..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-model.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "args": { - "common": { - "num_layers": 32, - "hidden_size": 1280, - "ffn_hidden_size": 3420, - "num_attention_heads": 16, - "num_key_value_heads": 16 - }, - "huggingface": { - "depth": 32, - "mlp_ratio": 4, - "num_heads": 16, - "in_chans": 3, - "patch_size": 14, - "spatial_merge_size": 2, - "spatial_patch_size": 14, - "temporal_patch_size": 2, - "hidden_act": "silu", - "hidden_size": 1280, - "intermediate_size": 3420, - "out_hidden_size": 3584, - "window_size": 112, - "fullatt_block_indexes": [ - 7, - 15, - 23, - 31 - ], - "tokens_per_second": 2 - }, - "mcore": { - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "tensor_model_parallel_size": 1, - "pipeline_model_parallel_size": 1, - "data_parallel_size": 1, - "use_rotary_position_embeddings": true, - "use_distributed_optimizer": false, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "transformer": "visual", - "layer_prefix": "blocks", - "input_layernorm": "norm1", - "attention.query_key_value": "attn.qkv", - "attention.dense": "attn.proj", - "post_attention_layernorm": "norm2", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj" - }, - "mcore": { - "transformer": "model", - "layer_prefix": "vision_model.decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.23.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-patch.json b/tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-patch.json deleted file mode 100644 index 5b8e0212..00000000 --- a/tools/convert_checkpoint/config/qwen2_5-vl-7b/vision-patch.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "vision_model.patch_embed.proj.weight": "visual.patch_embed.proj.weight" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/config/qwen3-4b.json b/tools/convert_checkpoint/config/qwen3-4b.json deleted file mode 100644 index c6c0cdbb..00000000 --- a/tools/convert_checkpoint/config/qwen3-4b.json +++ /dev/null @@ -1,111 +0,0 @@ - -{ - "args": { - "common": { - "num_layers": 36, - "hidden_size": 2560, - "ffn_hidden_size": 9728, - "num_attention_heads": 32, - "num_key_value_heads": 8, - "max_position_embeddings": 40960, - "vocab_size": 151936 - }, - "huggingface": { - "architectures": [ - "Qwen3ForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 151643, - "eos_token_id": 151645, - "head_dim": 128, - "hidden_act": "silu", - "hidden_size": 2560, - "initializer_range": 0.02, - "intermediate_size": 9728, - "max_position_embeddings": 40960, - "max_window_layers": 36, - "model_type": "qwen3", - "num_attention_heads": 32, - "num_hidden_layers": 36, - "num_key_value_heads": 8, - "rms_norm_eps": 1e-06, - "rope_scaling": null, - "rope_theta": 1000000, - "sliding_window": null, - "tie_word_embeddings": true, - "torch_dtype": "bfloat16", - "transformers_version": "4.51.0", - "use_cache": true, - "use_sliding_window": false, - "vocab_size": 151936 - }, - "mcore": { - "untie_embeddings_and_output_weights": true, - "num_layers_per_virtual_pipeline_stage": null, - "virtual_pipeline_model_parallel_size": null, - "use_rotary_position_embeddings": true, - "add_embedding_padding": true, - "make_vocab_size_divisible_by": 128, - "transpose_mlp_dense": true, - "transpose_query_key_value": true - } - }, - "name_map": { - "huggingface": { - "word_embeddings": "model.embed_tokens", - "transformer": "model", - "layer_prefix": "layers", - "input_layernorm": "input_layernorm", - "attention.query_key_value": [ - "self_attn.q_proj", - "self_attn.k_proj", - "self_attn.v_proj" - ], - "attention.qkv_map": { - "attention.q_a_layernorm": "self_attn.q_norm", - "attention.kv_a_layernorm": "self_attn.k_norm" - }, - "attention.dense": "self_attn.o_proj", - "post_attention_layernorm": "post_attention_layernorm", - "mlp.dense_h_to_4h": [ - "mlp.gate_proj", - "mlp.up_proj" - ], - "mlp.dense_4h_to_h": "mlp.down_proj", - "final_layernorm": "model.norm", - "word_embeddings_for_head": "lm_head" - }, - "mcore": { - "word_embeddings": "embedding.word_embeddings", - "word_position_embeddings": "model.embedding.position_embeddings", - "transformer": "model", - "layer_prefix": "decoder.layers", - "input_layernorm": "self_attention.linear_qkv.layer_norm", - "attention.query_key_value": "self_attention.linear_qkv", - "attention.qkv_map": { - "attention.q_a_layernorm": {"name": "self_attention.q_layernorm", "is_layernorm": false}, - "attention.kv_a_layernorm": {"name": "self_attention.k_layernorm", "is_layernorm": false} - }, - "attention.dense": "self_attention.linear_proj", - "post_attention_layernorm": "mlp.linear_fc1.layer_norm", - "mlp.dense_h_to_4h": "mlp.linear_fc1", - "mlp.dense_4h_to_h": "mlp.linear_fc2", - "final_layernorm": "decoder.final_layernorm", - "word_embeddings_for_head": "output_layer", - "transformer_tpl": "model%d" - } - }, - "tensor_parallel_dim": { - "word_embeddings.weight": 0, - "attention.query_key_value.weight": 0, - "attention.query_key_value.bias": 0, - "attention.dense.weight": 1, - "mlp.dense_h_to_4h.weight": 0, - "mlp.dense_h_to_4h.bias": 0, - "mlp.dense_4h_to_h.weight": 1, - "word_embeddings_for_head.weight": 0 - }, - "torch_dtype": "bfloat16", - "transformers_version": "4.40.1" -} \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/cogvlm/__init__.py b/tools/convert_checkpoint/custom/cogvlm/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tools/convert_checkpoint/custom/cogvlm/adapter.py b/tools/convert_checkpoint/custom/cogvlm/adapter.py deleted file mode 100644 index bd526b05..00000000 --- a/tools/convert_checkpoint/custom/cogvlm/adapter.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import io -import sys -import json -import torch -from os.path import dirname -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.cogvlm.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - print(" ====== convert adapters from Megatron Core to HuggingFace ======") - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - target = {} - source = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - tp = len(state_dict) - state_dict = [state_dict[t]['model'] for t in range(tp)] - for k, v in state_dict[0].items(): - if k.startswith("adapter") and isinstance(v, torch.Tensor): - if k == "adapter.mlp.linear_fc1.weight": - source[k] = torch.cat(tuple(state_dict[t][k] for t in range(tp)), dim = 0) - elif k == "adapter.mlp.linear_fc2.weight": - source[k] = torch.cat(tuple(state_dict[t][k] for t in range(tp)), dim = 1) - else: - source[k] = state_dict[0][k].clone() - print(f" > {k}") - for k1, k2 in name_map.items(): # mcore -> huggingface - if 'adapter.mlp.linear_fc1.weight' == k1: - source[k1] = rearrange(source[k1], "(TP N D) H -> (N TP D) H", - N = 2, TP = tp, D = 14336 // tp, H = 4096) - if isinstance(k2, str): - target[k2] = source[k1] - elif isinstance(k2, list): - weights = torch.chunk(source[k1], len(k2)) - for i, k in enumerate(k2): - target[k] = weights[i].clone() - else: - raise ValueError - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert adapters from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): # mcore -> huggingface - if isinstance(k2, str): - target[k1] = source[k2] - elif isinstance(k2, list): - target[k1] = torch.cat([source[i] for i in k2]) - else: - raise ValueError - if 'adapter.mlp.linear_fc1.weight' == k1: - target[k1] = rearrange(target[k1], "(N TP D) H -> (TP N D) H", - N = 2, TP = tp, D = 14336 // tp, H = 4096) - state_dict = [{'model': {}} for i in range(tp)] - for k, v in target.items(): - for t in range(tp): - if k == "adapter.mlp.linear_fc1.weight": - state_dict[t]['model'][k] = v.chunk(tp, dim = 0)[t].clone() - elif k == "adapter.mlp.linear_fc2.weight": - state_dict[t]['model'][k] = v.chunk(tp, dim = 1)[t].clone() - else: - state_dict[t]['model'][k] = v.clone() - - if "linear" in k: - a = io.BytesIO() - torch.save(None, a) - state_dict[t]['model'][k.replace('.weight', '._extra_state')] = a - print(f" > {k}") - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/custom/cogvlm/merge_huggingface.py b/tools/convert_checkpoint/custom/cogvlm/merge_huggingface.py deleted file mode 100644 index ed1e28f9..00000000 --- a/tools/convert_checkpoint/custom/cogvlm/merge_huggingface.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.cogvlm.util import ( - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -def parse_args(): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_expert_path', type=str, help="Path to language expert model."), - group.add_argument('--vision_expert_path', type=str, help="Path to vision expert model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge huggingface checkpoints ======") - -language_expert = load_huggingface_checkpoint(args.language_expert_path) -vision_expert = load_huggingface_checkpoint(args.vision_expert_path) -vision_model = load_huggingface_checkpoint(args.vision_model_path) -adapter = load_huggingface_checkpoint(args.adapter_path) -patch = load_huggingface_checkpoint(args.vision_patch) - -for module in [vision_expert, vision_model, adapter, patch]: - merge_dict(module, language_expert) - -save_huggingface_checkpoint(language_expert, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/cogvlm/merge_megatron.py b/tools/convert_checkpoint/custom/cogvlm/merge_megatron.py deleted file mode 100644 index ebdaffab..00000000 --- a/tools/convert_checkpoint/custom/cogvlm/merge_megatron.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.cogvlm.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, -) - - -def parse_args(title=None): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_expert_path', type=str, help="Path to language expert model."), - group.add_argument('--vision_expert_path', type=str, help="Path to vision expert model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge megatron checkpoints ======") - -language_expert = load_megatron_checkpoint(args.language_expert_path) -vision_expert = load_megatron_checkpoint(args.vision_expert_path) -vision_model = load_megatron_checkpoint(args.vision_model_path) -adapter = load_megatron_checkpoint(args.adapter_path) -patch = load_megatron_checkpoint(args.vision_patch) - -for module in [vision_expert, vision_model, adapter, patch]: - assert len(module) == len(language_expert) - for i in range(len(module)): - merge_dict(module[i]['model'], language_expert[i]['model']) - -save_megatron_checkpoint(language_expert, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/cogvlm/util.py b/tools/convert_checkpoint/custom/cogvlm/util.py deleted file mode 100644 index 2389a36f..00000000 --- a/tools/convert_checkpoint/custom/cogvlm/util.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import torch -import json -from os.path import dirname -from safetensors.torch import load_file, save_file -from huggingface_hub import split_torch_state_dict_into_shards -from transformers.modeling_utils import SAFE_WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_NAME - - -def merge_transformers_sharded_states(path, num_checkpoints): - """ - Merge sharded checkpoints from transformers into a single checkpoint. - - Args: - path (str): the path to the sharded checkpoints - num_checkpoints (int): the number of checkpoints to merge - """ - state_dict = {} - for i in range(1, num_checkpoints + 1): - checkpoint_path = os.path.join(path, f"model-{i:05d}-of-{num_checkpoints:05d}.safetensors") - current_chunk = load_file(checkpoint_path) - state_dict.update(current_chunk) - return state_dict - -def load_huggingface_checkpoint(load_path): - """ load ckpt """ - state_dict = {} - sub_dirs = [x for x in os.listdir(load_path) if x.endswith("safetensors")] - if len(sub_dirs) == 1: - checkpoint_name = "model.safetensors" - state_dict = load_file(os.path.join(load_path, checkpoint_name), device="cpu") - else: - num_checkpoints = len(sub_dirs) - state_dict = merge_transformers_sharded_states(load_path, num_checkpoints) - return state_dict - - -def save_huggingface_checkpoint(state_dict, save_path): - """ save ckpt """ - os.makedirs(save_path, exist_ok=True) - checkpoint_path = os.path.join(save_path, "model.safetensors") - - state_dict_split = split_torch_state_dict_into_shards(state_dict) - for shard_file, tensors in state_dict_split.filename_to_tensors.items(): - shard = {} - for tensor in tensors: - shard[tensor] = state_dict[tensor].contiguous() - del state_dict[tensor] - shard_path = os.path.join(save_path, shard_file) - save_file(shard, shard_path, metadata={"format": "pt"}) - print(f"Saving HuggingFace shard to: {shard_path}") - - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = os.path.join(save_path, SAFE_WEIGHTS_INDEX_NAME) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_megatron_checkpoint(load_path): - """ load ckpt """ - state_dict = [] - sub_dirs = [x for x in os.listdir(load_path) if x.startswith("mp_rank")] - for t in range(len(sub_dirs)): - checkpoint_name = f"mp_rank_{t:02d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu') - state_dict.append(ckpt) - return state_dict - - -def save_megatron_checkpoint(state_dict, save_path): - """ save ckpt """ - for t in range(len(state_dict)): - sub_dir_name = f"mp_rank_{t:02d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/cogvlm/vision_patch.py b/tools/convert_checkpoint/custom/cogvlm/vision_patch.py deleted file mode 100644 index 7b580f8a..00000000 --- a/tools/convert_checkpoint/custom/cogvlm/vision_patch.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.cogvlm.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -assert args.pipeline_model_parallel_size == 1, "NotImplementedError" -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert vision patch from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert vision patch from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/__init__.py b/tools/convert_checkpoint/custom/llavaov_1_5/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/adapter.py b/tools/convert_checkpoint/custom/llavaov_1_5/adapter.py deleted file mode 100644 index d6c5bb12..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/adapter.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import io -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert adapter from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0]['model'] if args.pipeline_model_parallel_size == 1 else state_dict[0][0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert adapter from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - for k in ['adapter.linear_fc1._extra_state', 'adapter.linear_fc2._extra_state']: - extra_state = io.BytesIO() - torch.save(None, extra_state) - target[k] = extra_state - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -elif (args.load_platform, args.save_platform) == ('mcore', 'mcore'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert adapter from Megatron Core to Megatron Core ======") - tp = args.tensor_model_parallel_size - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - if args.pipeline_model_parallel_size == 1: - source = state_dict[0]['model'] - target = deepcopy(source) - else: - source = state_dict[0][0]['model'] - target = deepcopy(source) - - # Create the new state dict structure - new_state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(new_state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py b/tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py deleted file mode 100644 index c75a0d73..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/fastvit.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -Convert FastViT vision encoder weights from HuggingFace to Megatron format. -""" -import argparse -import os -import sys -import torch -from safetensors import safe_open - -def convert_fastvit_weights(load_path, save_path, config): - """ - Convert FastViT weights from HF format to Megatron format. - - HF format: model.vision_tower.vision_tower.model.* - Megatron format: vision_model.vision_tower.model.* - """ - print(f"Loading FastViT weights from {load_path}") - - # Load weights from safetensors - hf_weights = {} - safetensors_path = os.path.join(load_path, "model.safetensors") - - with safe_open(safetensors_path, framework="pt", device="cpu") as f: - for key in f.keys(): - if key.startswith("model.vision_tower.vision_tower"): - hf_weights[key] = f.get_tensor(key) - - print(f"Loaded {len(hf_weights)} FastViT weight tensors") - - # Convert keys from HF to Megatron format - # HF: model.vision_tower.vision_tower.model.* - # Megatron: vision_model.vision_tower.model.* - mcore_weights = {} - - vision_model_from = config.get("vision_model_from", "model.vision_tower.vision_tower") - vision_model_to = config.get("vision_model_to", "vision_model.vision_tower") - - for hf_key, weight in hf_weights.items(): - if hf_key.startswith(vision_model_from): - # Strip the HF prefix and add Megatron prefix - suffix = hf_key[len(vision_model_from):] - mcore_key = vision_model_to + suffix - mcore_weights[mcore_key] = weight - - if len(mcore_weights) <= 5: # Show first 5 conversions - print(f" {hf_key} -> {mcore_key}") - - print(f"Converted {len(mcore_weights)} weight tensors") - - # Save in Megatron format - os.makedirs(os.path.join(save_path, "release"), exist_ok=True) - save_file = os.path.join(save_path, "release", "mp_rank_00", "model_optim_rng.pt") - os.makedirs(os.path.dirname(save_file), exist_ok=True) - - checkpoint = { - 'model': mcore_weights, - 'checkpoint_version': 3.0, - 'args': { - 'tensor_model_parallel_size': 1, - 'pipeline_model_parallel_size': 1, - } - } - - torch.save(checkpoint, save_file) - print(f"Saved Megatron checkpoint to {save_file}") - - return mcore_weights - - -def main(): - parser = argparse.ArgumentParser(description="Convert FastViT weights to Megatron format") - parser.add_argument("--load_platform", type=str, default="huggingface") - parser.add_argument("--save_platform", type=str, default="mcore") - parser.add_argument("--common_config_path", type=str, required=True) - parser.add_argument("--load_ckpt_path", type=str, required=True) - parser.add_argument("--save_ckpt_path", type=str, required=True) - parser.add_argument("--tensor_model_parallel_size", type=int, default=1) - parser.add_argument("--safetensors", action="store_true") - parser.add_argument("--no_save_optim", action="store_true") - parser.add_argument("--no_load_optim", action="store_true") - - args = parser.parse_args() - - # Load config - import json - with open(args.common_config_path, 'r') as f: - config = json.load(f) - - # Convert weights - convert_fastvit_weights(args.load_ckpt_path, args.save_ckpt_path, config) - - print("FastViT conversion completed successfully!") - - -if __name__ == "__main__": - main() diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/merge_huggingface.py b/tools/convert_checkpoint/custom/llavaov_1_5/merge_huggingface.py deleted file mode 100644 index 641747f5..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/merge_huggingface.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -def parse_args(): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language expert model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge huggingface checkpoints ======") - -language_model = load_huggingface_checkpoint(args.language_model_path) -vision_model = load_huggingface_checkpoint(args.vision_model_path) -adapter = load_huggingface_checkpoint(args.adapter_path) -patch = load_huggingface_checkpoint(args.vision_patch) - -for module in [vision_model, adapter, patch]: - merge_dict(module, language_model) - -save_huggingface_checkpoint(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron.py b/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron.py deleted file mode 100644 index 718fbf60..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, -) - - -def parse_args(title=None): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - group.add_argument("--tensor_model_parallel_size", type=int, default=1, help="Tensor parallel size.") - group.add_argument("--pipeline_model_parallel_size", type=int, default=1, help="Pipeline parallel size.") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge megatron checkpoints ======") - -language_model = load_megatron_checkpoint(args.language_model_path) -vision_model = load_megatron_checkpoint(args.vision_model_path) -adapter = load_megatron_checkpoint(args.adapter_path) -patch = load_megatron_checkpoint(args.vision_patch) - -if args.pipeline_model_parallel_size == 1: - for module in [vision_model, adapter, patch]: - assert len(module) == len(language_model) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[i]['model']) -else: - for module in [vision_model, adapter, patch]: - assert len(module) == len(language_model[0]) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[0][i]['model']) - -save_megatron_checkpoint(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py b/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py deleted file mode 100644 index 880d8d8d..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/merge_megatron_fastvlm.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, -) - - -def parse_args(title=None): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments for FastVLM', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language model."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument('--vision_model_path', type=str, default=None, help="Path to vision model (FastViT)."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - group.add_argument("--tensor_model_parallel_size", type=int, default=1, help="Tensor parallel size.") - group.add_argument("--pipeline_model_parallel_size", type=int, default=1, help="Pipeline parallel size.") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge megatron checkpoints (FastVLM with vision model) ======") - -language_model = load_megatron_checkpoint(args.language_model_path) -adapter = load_megatron_checkpoint(args.adapter_path) - -# Load vision model if provided -if args.vision_model_path: - vision_model = load_megatron_checkpoint(args.vision_model_path) - print(f"Loaded FastViT vision model from {args.vision_model_path}") - modules_to_merge = [adapter, vision_model] -else: - print("Warning: No vision model provided. FastViT will be initialized randomly.") - modules_to_merge = [adapter] - -# Merge adapter and vision model into language model -# Merge adapter and vision model into language model -if args.pipeline_model_parallel_size == 1: - for module in modules_to_merge: - assert len(module) == len(language_model) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[i]['model']) -else: - for module in modules_to_merge: - assert len(module) == len(language_model[0]) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[0][i]['model']) - -save_megatron_checkpoint(language_model, args.save_ckpt_path) -print(f"Checkpoint saved to {args.save_ckpt_path}") -if args.vision_model_path: - print("Successfully merged: Language Model + Adapter + FastViT Vision Model") -else: - print("Merged: Language Model + Adapter (FastViT will be initialized randomly)") diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/util.py b/tools/convert_checkpoint/custom/llavaov_1_5/util.py deleted file mode 100644 index 466a38b9..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/util.py +++ /dev/null @@ -1,371 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import torch -import json -from os.path import dirname -from safetensors.torch import load_file, save_file -from huggingface_hub import split_torch_state_dict_into_shards -from transformers.modeling_utils import SAFE_WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_NAME - - -def merge_transformers_sharded_states(path, num_checkpoints): - """ - Merge sharded checkpoints from transformers into a single checkpoint. - - Args: - path (str): the path to the sharded checkpoints - num_checkpoints (int): the number of checkpoints to merge - """ - state_dict = {} - for i in range(1, num_checkpoints + 1): - checkpoint_path = os.path.join(path, f"model-{i:05d}-of-{num_checkpoints:05d}.safetensors") - current_chunk = load_file(checkpoint_path) - state_dict.update(current_chunk) - return state_dict - -def load_huggingface_checkpoint(load_path): - """ load ckpt """ - state_dict = {} - sub_dirs = [x for x in os.listdir(load_path) if x.endswith("safetensors")] - if len(sub_dirs) == 1: - checkpoint_name = "model.safetensors" - state_dict = load_file(os.path.join(load_path, checkpoint_name), device="cpu") - else: - num_checkpoints = len(sub_dirs) - state_dict = merge_transformers_sharded_states(load_path, num_checkpoints) - return state_dict - - -def save_huggingface_checkpoint(state_dict, save_path): - """ save ckpt """ - os.makedirs(save_path, exist_ok=True) - checkpoint_path = os.path.join(save_path, "model.safetensors") - - state_dict_split = split_torch_state_dict_into_shards(state_dict) - for shard_file, tensors in state_dict_split.filename_to_tensors.items(): - shard = {} - for tensor in tensors: - shard[tensor] = state_dict[tensor].contiguous() - del state_dict[tensor] - shard_path = os.path.join(save_path, shard_file) - save_file(shard, shard_path, metadata={"format": "pt"}) - print(f"Saving HuggingFace shard to: {shard_path}") - - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = os.path.join(save_path, SAFE_WEIGHTS_INDEX_NAME) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_megatron_checkpoint(load_path): - """ load ckpt """ - state_dict = [] - sub_dirs = sorted([x for x in os.listdir(load_path) if x.startswith("mp_rank")]) - last_dir = sub_dirs[-1].split('_') - if len(last_dir) == 4: - tp = int(last_dir[-2]) + 1 - pp = int(last_dir[-1]) + 1 - for p in range(pp): - state_dict.append([]) - for t in range(tp): - checkpoint_name = f"mp_rank_{t:02d}_{p:03d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu', weights_only=False) - state_dict[p].append(ckpt) - return state_dict - else: - for t in range(len(sub_dirs)): - checkpoint_name = f"mp_rank_{t:02d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu', weights_only=False) - state_dict.append(ckpt) - return state_dict - - -def save_megatron_checkpoint(state_dict, save_path): - """ save ckpt """ - if isinstance(state_dict[0], list): - for p in range(len(state_dict)): - for t in range(len(state_dict[p])): - sub_dir_name = f"mp_rank_{t:02d}_{p:03d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[p][t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - else: - for t in range(len(state_dict)): - sub_dir_name = f"mp_rank_{t:02d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - -import os -import re -import torch -from typing import Dict, List, Tuple - - -_MP3D_DIR_RE = re.compile(r"^mp_rank_(\d+)_(\d+)_(\d+)$") -_MP2D_DIR_RE = re.compile(r"^mp_rank_(\d+)_(\d+)$") -_MP1D_DIR_RE = re.compile(r"^mp_rank_(\d+)$") -_SHARD_FILE = "model_optim_rng.pt" - - -def _scan_mp_dirs(load_path: str) -> Tuple[Dict[Tuple[int, int, int], str], List[int], List[int], List[int]]: - """ - Scan load_path for 3D mp_rank directories and return: - - dir_map: mapping from (tp, pp, ep) -> directory name (not full path) - - t_vals, p_vals, e_vals: sorted unique indices found for tp, pp, ep. - - Raises: - FileNotFoundError: if load_path does not exist. - ValueError: if no 3D mp_rank_*_*_* directories are found. - """ - if not os.path.isdir(load_path): - raise FileNotFoundError(f"Directory not found: {load_path}") - - sub_dirs = [d for d in os.listdir(load_path) if os.path.isdir(os.path.join(load_path, d))] - dir_map_3d: Dict[Tuple[int, int, int], str] = {} - t_set, p_set, e_set = set(), set(), set() - - # Prefer strict 3D directories - for d in sub_dirs: - m = _MP3D_DIR_RE.match(d) - if m: - t, p, e = (int(m.group(1)), int(m.group(2)), int(m.group(3))) - dir_map_3d[(t, p, e)] = d - t_set.add(t) - p_set.add(p) - e_set.add(e) - - if not dir_map_3d: - # Helpful diagnostics if the tree is not 3D - has_2d = any(_MP2D_DIR_RE.match(d) for d in sub_dirs) - has_1d = any(_MP1D_DIR_RE.match(d) for d in sub_dirs) - if has_2d or has_1d: - raise ValueError( - "Expected 3D mp_rank_{tp}_{pp}_{ep} directory layout, but found 1D/2D.\n" - "Use the original load_megatron_checkpoint for 1D/2D or convert your checkpoints." - ) - raise ValueError(f"No mp_rank_* directories found in: {load_path}") - - t_vals = sorted(t_set) - p_vals = sorted(p_set) - e_vals = sorted(e_set) - return dir_map_3d, t_vals, p_vals, e_vals - - -def load_megatron_checkpoint_tp_pp_ep(load_path: str): - """ - Load Megatron checkpoints organized in 3D layout: mp_rank_{tp}_{pp}_{ep}/model_optim_rng.pt - - Returns: - A 3-level nested list organized as state_dict[pp_idx][tp_idx][ep_idx], - where indices follow the ascending order of discovered pp, tp, ep values. - - Each leaf is the object loaded from torch.load(...): - typically a dict with keys like 'model', 'optimizer', etc. - - Notes: - - All shards are loaded onto CPU (map_location='cpu'). - - This function is strict for 3D; it will error if only 1D/2D layout is found. - - Directory numeric widths (zero-padding) are not assumed; we reuse the actual directory names found. - """ - dir_map_3d, t_vals, p_vals, e_vals = _scan_mp_dirs(load_path) - - # Build nested structure: [pp][tp][ep] - state_dict: List[List[List[dict]]] = [] - for p in p_vals: - pp_list: List[List[dict]] = [] - for t in t_vals: - tp_list: List[dict] = [] - for e in e_vals: - sub_dir = dir_map_3d.get((t, p, e)) - if sub_dir is None: - raise FileNotFoundError( - f"Missing shard directory for (tp={t}, pp={p}, ep={e}) under {load_path}" - ) - checkpoint_path = os.path.join(load_path, sub_dir, _SHARD_FILE) - if not os.path.isfile(checkpoint_path): - raise FileNotFoundError(f"Shard file not found: {checkpoint_path}") - ckpt = torch.load(checkpoint_path, map_location='cpu', weights_only=False) - tp_list.append(ckpt) - pp_list.append(tp_list) - state_dict.append(pp_list) - - return state_dict - - -def save_megatron_checkpoint_tp_pp_ep(state_dict, save_path: str, pad_tp: int = 2, pad_pp: int = 3, pad_ep: int = 3): - """ - Save a 3D Megatron checkpoint layout to save_path as: - mp_rank_{tp:0{pad_tp}d}_{pp:0{pad_pp}d}_{ep:0{pad_ep}d}/model_optim_rng.pt - - Args: - state_dict: 3-level nested list organized as state_dict[pp_idx][tp_idx][ep_idx]. - Each leaf element is the checkpoint object to be torch.save'd. - save_path: Target root directory. - pad_tp: Zero-padding width for 'tp' in directory names (default 2). - pad_pp: Zero-padding width for 'pp' in directory names (default 3). - pad_ep: Zero-padding width for 'ep' in directory names (default 3). - - Raises: - ValueError: if state_dict is not a 3-level nested list. - """ - if not isinstance(state_dict, list) or not state_dict: - raise ValueError("state_dict must be a non-empty 3-level nested list: [pp][tp][ep].") - - if not isinstance(state_dict[0], list) or not state_dict[0]: - raise ValueError("state_dict must be a 3-level nested list: [pp][tp][ep].") - if not isinstance(state_dict[0][0], list) or not state_dict[0][0]: - raise ValueError("state_dict must be a 3-level nested list: [pp][tp][ep].") - - os.makedirs(save_path, exist_ok=True) - - for p_idx, pp_list in enumerate(state_dict): - if not isinstance(pp_list, list): - raise ValueError(f"Expected state_dict[{p_idx}] to be a list (tp dimension).") - for t_idx, tp_list in enumerate(pp_list): - if not isinstance(tp_list, list): - raise ValueError(f"Expected state_dict[{p_idx}][{t_idx}] to be a list (ep dimension).") - for e_idx, ckpt in enumerate(tp_list): - sub_dir_name = f"mp_rank_{t_idx:0{pad_tp}d}_{p_idx:0{pad_pp}d}_{e_idx:0{pad_ep}d}" - full_dir = os.path.join(save_path, sub_dir_name) - os.makedirs(full_dir, exist_ok=True) - checkpoint_path = os.path.join(full_dir, _SHARD_FILE) - torch.save(ckpt, checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - - - -def _scan_mp_dirs_2d(load_path: str) -> Tuple[Dict[Tuple[int, int], str], List[int], List[int]]: - """ - Scan load_path for 2D mp_rank directories and return: - - dir_map: mapping from (tp, ep) -> directory name (not full path) - - t_vals, e_vals: sorted unique indices found for tp, ep. - - Raises: - FileNotFoundError: if load_path does not exist. - ValueError: if no 2D mp_rank_*_* directories are found. - """ - if not os.path.isdir(load_path): - raise FileNotFoundError(f"Directory not found: {load_path}") - - sub_dirs = [d for d in os.listdir(load_path) if os.path.isdir(os.path.join(load_path, d))] - dir_map_2d: Dict[Tuple[int, int], str] = {} - t_set, e_set = set(), set() - - # Look for 2D directories - for d in sub_dirs: - m = _MP2D_DIR_RE.match(d) - if m: - t, e = (int(m.group(1)), int(m.group(2))) - dir_map_2d[(t, e)] = d - t_set.add(t) - e_set.add(e) - - if not dir_map_2d: - raise ValueError(f"No mp_rank_*_* directories found in: {load_path}") - - t_vals = sorted(t_set) - e_vals = sorted(e_set) - return dir_map_2d, t_vals, e_vals - - -def load_megatron_checkpoint_tp_ep(load_path: str): - """ - Load Megatron checkpoints organized in 2D layout: mp_rank_{tp}_{ep}/model_optim_rng.pt - - Returns: - A 2-level nested list organized as state_dict[tp_idx][ep_idx], - where indices follow the ascending order of discovered tp, ep values. - - Each leaf is the object loaded from torch.load(...): - typically a dict with keys like 'model', 'optimizer', etc. - - Notes: - - All shards are loaded onto CPU (map_location='cpu'). - - This function is for 2D layout (tensor parallel + expert parallel). - - Directory numeric widths (zero-padding) are not assumed; we reuse the actual directory names found. - """ - dir_map_2d, t_vals, e_vals = _scan_mp_dirs_2d(load_path) - - # Build nested structure: [tp][ep] - state_dict: List[List[dict]] = [] - for t in t_vals: - tp_list: List[dict] = [] - for e in e_vals: - sub_dir = dir_map_2d.get((t, e)) - if sub_dir is None: - raise FileNotFoundError( - f"Missing shard directory for (tp={t}, ep={e}) under {load_path}" - ) - checkpoint_path = os.path.join(load_path, sub_dir, _SHARD_FILE) - if not os.path.isfile(checkpoint_path): - raise FileNotFoundError(f"Shard file not found: {checkpoint_path}") - - print(f"Loading checkpoint from: {checkpoint_path}") - ckpt = torch.load(checkpoint_path, map_location='cpu', weights_only=False) - tp_list.append(ckpt) - state_dict.append(tp_list) - - print(f"Loaded {len(t_vals)} tensor parallel ranks and {len(e_vals)} expert parallel ranks") - return state_dict - - -def save_megatron_checkpoint_tp_ep(state_dict, save_path: str, pad_tp: int = 2, pad_ep: int = 3): - """ - Save a 2D Megatron checkpoint layout to save_path as: - mp_rank_{tp:0{pad_tp}d}_{ep:0{pad_ep}d}/model_optim_rng.pt - - Args: - state_dict: 2-level nested list organized as state_dict[tp_idx][ep_idx]. - Each leaf element is the checkpoint object to be torch.save'd. - save_path: Target root directory. - pad_tp: Zero-padding width for 'tp' in directory names (default 2). - pad_ep: Zero-padding width for 'ep' in directory names (default 3). - - Raises: - ValueError: if state_dict is not a 2-level nested list. - """ - if not isinstance(state_dict, list) or not state_dict: - raise ValueError("state_dict must be a non-empty 2-level nested list: [tp][ep].") - - if not isinstance(state_dict[0], list) or not state_dict[0]: - raise ValueError("state_dict must be a 2-level nested list: [tp][ep].") - - os.makedirs(save_path, exist_ok=True) - - for t_idx, tp_list in enumerate(state_dict): - if not isinstance(tp_list, list): - raise ValueError(f"Expected state_dict[{t_idx}] to be a list (ep dimension).") - for e_idx, ckpt in enumerate(tp_list): - sub_dir_name = f"mp_rank_{t_idx:0{pad_tp}d}_{e_idx:0{pad_ep}d}" - full_dir = os.path.join(save_path, sub_dir_name) - os.makedirs(full_dir, exist_ok=True) - checkpoint_path = os.path.join(full_dir, _SHARD_FILE) - torch.save(ckpt, checkpoint_path) - print(f"Saved Megatron shard to: {checkpoint_path}") - - -# # 使用示例 -# if __name__ == "__main__": -# # 加载checkpoint -# load_path = "/path/to/your/checkpoint/dir" -# state_dict = load_megatron_checkpoint_tp_ep(load_path) - -# # 保存checkpoint -# save_path = "/path/to/save/checkpoint/dir" -# save_megatron_checkpoint_tp_ep(state_dict, save_path) diff --git a/tools/convert_checkpoint/custom/llavaov_1_5/vision_patch.py b/tools/convert_checkpoint/custom/llavaov_1_5/vision_patch.py deleted file mode 100644 index 3c27b076..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5/vision_patch.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert vision patch from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0]['model'] if args.pipeline_model_parallel_size == 1 else state_dict[0][0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert vision patch from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -elif (args.load_platform, args.save_platform) == ('mcore', 'mcore'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert vision patch from Megatron Core to Megatron Core ======") - tp = args.tensor_model_parallel_size - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0]['model'] if args.pipeline_model_parallel_size == 1 else state_dict[0][0]['model'] - - target = {} - for k in source.keys(): - target[k] = source[k] - print(f" > {k}") - - # Create state dict for each tensor parallel rank - target_state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(target_state_dict, os.path.join(args.save_ckpt_path, 'release')) -else: - raise NotImplementedError diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/__init__.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/adapter.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/adapter.py deleted file mode 100644 index a2f10b01..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/adapter.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import io -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert adapter from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0][0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert adapter from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - for k in ['adapter.linear_fc1._extra_state', 'adapter.linear_fc2._extra_state']: - extra_state = io.BytesIO() - torch.save(None, extra_state) - target[k] = extra_state - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_huggingface.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_huggingface.py deleted file mode 100644 index 641747f5..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_huggingface.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -def parse_args(): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language expert model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge huggingface checkpoints ======") - -language_model = load_huggingface_checkpoint(args.language_model_path) -vision_model = load_huggingface_checkpoint(args.vision_model_path) -adapter = load_huggingface_checkpoint(args.adapter_path) -patch = load_huggingface_checkpoint(args.vision_patch) - -for module in [vision_model, adapter, patch]: - merge_dict(module, language_model) - -save_huggingface_checkpoint(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron.py deleted file mode 100644 index 718fbf60..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, -) - - -def parse_args(title=None): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - group.add_argument("--tensor_model_parallel_size", type=int, default=1, help="Tensor parallel size.") - group.add_argument("--pipeline_model_parallel_size", type=int, default=1, help="Pipeline parallel size.") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge megatron checkpoints ======") - -language_model = load_megatron_checkpoint(args.language_model_path) -vision_model = load_megatron_checkpoint(args.vision_model_path) -adapter = load_megatron_checkpoint(args.adapter_path) -patch = load_megatron_checkpoint(args.vision_patch) - -if args.pipeline_model_parallel_size == 1: - for module in [vision_model, adapter, patch]: - assert len(module) == len(language_model) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[i]['model']) -else: - for module in [vision_model, adapter, patch]: - assert len(module) == len(language_model[0]) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[0][i]['model']) - -save_megatron_checkpoint(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron_qwen3_30b_a3b.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron_qwen3_30b_a3b.py deleted file mode 100644 index 39bd36d2..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/merge_megatron_qwen3_30b_a3b.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.llavaov_1_5_30b_a3b.util import ( - load_megatron_checkpoint, - load_megatron_checkpoint_tp_ep, - save_megatron_checkpoint_tp_ep -) - -def parse_args(title=None): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - group.add_argument("--tensor_model_parallel_size", type=int, default=1, help="Tensor parallel size.") - group.add_argument("--pipeline_model_parallel_size", type=int, default=1, help="Pipeline parallel size.") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge megatron checkpoints ======") - - -# 仅 LLM 使用 2D 装载，其他模块使用原来的 1D（仅 TP）装载 -language_model = load_megatron_checkpoint_tp_ep(args.language_model_path) -vision_model = load_megatron_checkpoint(args.vision_model_path) -adapter = load_megatron_checkpoint(args.adapter_path) -patch = load_megatron_checkpoint(args.vision_patch) - -# 解析 LLM 的 2D 维度：state_dict[tp][ep] -tp_size = len(language_model) -assert tp_size > 0, "language_model(tp) 维度为空" -ep_size = len(language_model[0]) -assert ep_size > 0, "language_model(ep) 维度为空" - -def merge_dict(src_dict, dst_dict): - """将src_dict的内容合并到dst_dict中""" - for key, value in src_dict.items(): - if key in dst_dict: - if isinstance(value, dict) and isinstance(dst_dict[key], dict): - merge_dict(value, dst_dict[key]) - # 如果key已存在且不是字典，跳过以避免覆盖 - else: - dst_dict[key] = value - -# 合并：把每个 TP 分片的模块权重，广播到该 TP 分片的所有 EP 分片 -for module_name, module in [("vision", vision_model), ("adapter", adapter), ("patch", patch)]: - assert isinstance(module, list), f"{module_name} 模块应为 1D TP 分片列表" - assert len(module) == tp_size, ( - f"{module_name} 的 TP 分片数({len(module)}) 与 LLM 的 TP 分片数({tp_size}) 不一致" - ) - for t in range(tp_size): - src = module[t] - assert 'model' in src, f"{module_name}[{t}] 缺少 'model' 键" - for e in range(ep_size): - dst = language_model[t][e] - assert 'model' in dst, f"LLM[tp={t}][ep={e}] 缺少 'model' 键" - merge_dict(src['model'], dst['model']) - -# 保存为 2D Megatron 布局 -save_megatron_checkpoint_tp_ep(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/util.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/util.py deleted file mode 100644 index 0798f4e4..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/util.py +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import json -import os -import re -import sys -from os.path import dirname -from typing import Dict, List, Tuple - -import torch -from huggingface_hub import split_torch_state_dict_into_shards -from safetensors.torch import load_file, save_file -from transformers.modeling_utils import (SAFE_WEIGHTS_INDEX_NAME, - SAFE_WEIGHTS_NAME) - - -def merge_transformers_sharded_states(path, num_checkpoints): - """ - Merge sharded checkpoints from transformers into a single checkpoint. - - Args: - path (str): the path to the sharded checkpoints - num_checkpoints (int): the number of checkpoints to merge - """ - state_dict = {} - for i in range(1, num_checkpoints + 1): - checkpoint_path = os.path.join(path, f"model-{i:05d}-of-{num_checkpoints:05d}.safetensors") - current_chunk = load_file(checkpoint_path) - state_dict.update(current_chunk) - return state_dict - -def load_huggingface_checkpoint(load_path): - """ load ckpt """ - state_dict = {} - sub_dirs = [x for x in os.listdir(load_path) if x.endswith("safetensors")] - if len(sub_dirs) == 1: - checkpoint_name = "model.safetensors" - state_dict = load_file(os.path.join(load_path, checkpoint_name), device="cpu") - else: - num_checkpoints = len(sub_dirs) - state_dict = merge_transformers_sharded_states(load_path, num_checkpoints) - return state_dict - - -def save_huggingface_checkpoint(state_dict, save_path): - """ save ckpt """ - os.makedirs(save_path, exist_ok=True) - checkpoint_path = os.path.join(save_path, "model.safetensors") - - state_dict_split = split_torch_state_dict_into_shards(state_dict) - for shard_file, tensors in state_dict_split.filename_to_tensors.items(): - shard = {} - for tensor in tensors: - shard[tensor] = state_dict[tensor].contiguous() - del state_dict[tensor] - shard_path = os.path.join(save_path, shard_file) - save_file(shard, shard_path, metadata={"format": "pt"}) - print(f"Saving HuggingFace shard to: {shard_path}") - - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = os.path.join(save_path, SAFE_WEIGHTS_INDEX_NAME) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_megatron_checkpoint(load_path): - """ load ckpt """ - state_dict = [] - sub_dirs = sorted([x for x in os.listdir(load_path) if x.startswith("mp_rank")]) - last_dir = sub_dirs[-1].split('_') - if len(last_dir) == 4: - tp = int(last_dir[-2]) + 1 - pp = int(last_dir[-1]) + 1 - for p in range(pp): - state_dict.append([]) - for t in range(tp): - checkpoint_name = f"mp_rank_{t:02d}_{p:03d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu', weights_only=False) - state_dict[p].append(ckpt) - return state_dict - else: - for t in range(len(sub_dirs)): - checkpoint_name = f"mp_rank_{t:02d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu', weights_only=False) - state_dict.append(ckpt) - return state_dict - - -def save_megatron_checkpoint(state_dict, save_path): - """ save ckpt """ - if isinstance(state_dict[0], list): - for p in range(len(state_dict)): - for t in range(len(state_dict[p])): - sub_dir_name = f"mp_rank_{t:02d}_{p:03d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[p][t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - else: - for t in range(len(state_dict)): - sub_dir_name = f"mp_rank_{t:02d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - - -_MP3D_DIR_RE = re.compile(r"^mp_rank_(\d+)_(\d+)_(\d+)$") -_MP2D_DIR_RE = re.compile(r"^mp_rank_(\d+)_(\d+)$") -_MP1D_DIR_RE = re.compile(r"^mp_rank_(\d+)$") -_SHARD_FILE = "model_optim_rng.pt" - - -def _scan_mp_dirs(load_path: str) -> Tuple[Dict[Tuple[int, int, int], str], List[int], List[int], List[int]]: - """ - Scan load_path for 3D mp_rank directories and return: - - dir_map: mapping from (tp, pp, ep) -> directory name (not full path) - - t_vals, p_vals, e_vals: sorted unique indices found for tp, pp, ep. - - Raises: - FileNotFoundError: if load_path does not exist. - ValueError: if no 3D mp_rank_*_*_* directories are found. - """ - if not os.path.isdir(load_path): - raise FileNotFoundError(f"Directory not found: {load_path}") - - sub_dirs = [d for d in os.listdir(load_path) if os.path.isdir(os.path.join(load_path, d))] - dir_map_3d: Dict[Tuple[int, int, int], str] = {} - t_set, p_set, e_set = set(), set(), set() - - # Prefer strict 3D directories - for d in sub_dirs: - m = _MP3D_DIR_RE.match(d) - if m: - t, p, e = (int(m.group(1)), int(m.group(2)), int(m.group(3))) - dir_map_3d[(t, p, e)] = d - t_set.add(t) - p_set.add(p) - e_set.add(e) - - if not dir_map_3d: - # Helpful diagnostics if the tree is not 3D - has_2d = any(_MP2D_DIR_RE.match(d) for d in sub_dirs) - has_1d = any(_MP1D_DIR_RE.match(d) for d in sub_dirs) - if has_2d or has_1d: - raise ValueError( - "Expected 3D mp_rank_{tp}_{pp}_{ep} directory layout, but found 1D/2D.\n" - "Use the original load_megatron_checkpoint for 1D/2D or convert your checkpoints." - ) - raise ValueError(f"No mp_rank_* directories found in: {load_path}") - - t_vals = sorted(t_set) - p_vals = sorted(p_set) - e_vals = sorted(e_set) - return dir_map_3d, t_vals, p_vals, e_vals - - -def load_megatron_checkpoint_tp_pp_ep(load_path: str): - """ - Load Megatron checkpoints organized in 3D layout: mp_rank_{tp}_{pp}_{ep}/model_optim_rng.pt - - Returns: - A 3-level nested list organized as state_dict[pp_idx][tp_idx][ep_idx], - where indices follow the ascending order of discovered pp, tp, ep values. - - Each leaf is the object loaded from torch.load(...): - typically a dict with keys like 'model', 'optimizer', etc. - - Notes: - - All shards are loaded onto CPU (map_location='cpu'). - - This function is strict for 3D; it will error if only 1D/2D layout is found. - - Directory numeric widths (zero-padding) are not assumed; we reuse the actual directory names found. - """ - dir_map_3d, t_vals, p_vals, e_vals = _scan_mp_dirs(load_path) - - # Build nested structure: [pp][tp][ep] - state_dict: List[List[List[dict]]] = [] - for p in p_vals: - pp_list: List[List[dict]] = [] - for t in t_vals: - tp_list: List[dict] = [] - for e in e_vals: - sub_dir = dir_map_3d.get((t, p, e)) - if sub_dir is None: - raise FileNotFoundError( - f"Missing shard directory for (tp={t}, pp={p}, ep={e}) under {load_path}" - ) - checkpoint_path = os.path.join(load_path, sub_dir, _SHARD_FILE) - if not os.path.isfile(checkpoint_path): - raise FileNotFoundError(f"Shard file not found: {checkpoint_path}") - ckpt = torch.load(checkpoint_path, map_location='cpu', weights_only=False) - tp_list.append(ckpt) - pp_list.append(tp_list) - state_dict.append(pp_list) - - return state_dict - - -def save_megatron_checkpoint_tp_pp_ep(state_dict, save_path: str, pad_tp: int = 2, pad_pp: int = 3, pad_ep: int = 3): - """ - Save a 3D Megatron checkpoint layout to save_path as: - mp_rank_{tp:0{pad_tp}d}_{pp:0{pad_pp}d}_{ep:0{pad_ep}d}/model_optim_rng.pt - - Args: - state_dict: 3-level nested list organized as state_dict[pp_idx][tp_idx][ep_idx]. - Each leaf element is the checkpoint object to be torch.save'd. - save_path: Target root directory. - pad_tp: Zero-padding width for 'tp' in directory names (default 2). - pad_pp: Zero-padding width for 'pp' in directory names (default 3). - pad_ep: Zero-padding width for 'ep' in directory names (default 3). - - Raises: - ValueError: if state_dict is not a 3-level nested list. - """ - if not isinstance(state_dict, list) or not state_dict: - raise ValueError("state_dict must be a non-empty 3-level nested list: [pp][tp][ep].") - - if not isinstance(state_dict[0], list) or not state_dict[0]: - raise ValueError("state_dict must be a 3-level nested list: [pp][tp][ep].") - if not isinstance(state_dict[0][0], list) or not state_dict[0][0]: - raise ValueError("state_dict must be a 3-level nested list: [pp][tp][ep].") - - os.makedirs(save_path, exist_ok=True) - - for p_idx, pp_list in enumerate(state_dict): - if not isinstance(pp_list, list): - raise ValueError(f"Expected state_dict[{p_idx}] to be a list (tp dimension).") - for t_idx, tp_list in enumerate(pp_list): - if not isinstance(tp_list, list): - raise ValueError(f"Expected state_dict[{p_idx}][{t_idx}] to be a list (ep dimension).") - for e_idx, ckpt in enumerate(tp_list): - sub_dir_name = f"mp_rank_{t_idx:0{pad_tp}d}_{p_idx:0{pad_pp}d}_{e_idx:0{pad_ep}d}" - full_dir = os.path.join(save_path, sub_dir_name) - os.makedirs(full_dir, exist_ok=True) - checkpoint_path = os.path.join(full_dir, _SHARD_FILE) - torch.save(ckpt, checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - - - -def _scan_mp_dirs_2d(load_path: str) -> Tuple[Dict[Tuple[int, int], str], List[int], List[int]]: - """ - Scan load_path for 2D mp_rank directories and return: - - dir_map: mapping from (tp, ep) -> directory name (not full path) - - t_vals, e_vals: sorted unique indices found for tp, ep. - - Raises: - FileNotFoundError: if load_path does not exist. - ValueError: if no 2D mp_rank_*_* directories are found. - """ - if not os.path.isdir(load_path): - raise FileNotFoundError(f"Directory not found: {load_path}") - - sub_dirs = [d for d in os.listdir(load_path) if os.path.isdir(os.path.join(load_path, d))] - dir_map_2d: Dict[Tuple[int, int], str] = {} - t_set, e_set = set(), set() - - # Look for 2D directories - for d in sub_dirs: - m = _MP2D_DIR_RE.match(d) - if m: - t, e = (int(m.group(1)), int(m.group(2))) - dir_map_2d[(t, e)] = d - t_set.add(t) - e_set.add(e) - - if not dir_map_2d: - raise ValueError(f"No mp_rank_*_* directories found in: {load_path}") - - t_vals = sorted(t_set) - e_vals = sorted(e_set) - return dir_map_2d, t_vals, e_vals - - -def load_megatron_checkpoint_tp_ep(load_path: str): - """ - Load Megatron checkpoints organized in 2D layout: mp_rank_{tp}_{ep}/model_optim_rng.pt - - Returns: - A 2-level nested list organized as state_dict[tp_idx][ep_idx], - where indices follow the ascending order of discovered tp, ep values. - - Each leaf is the object loaded from torch.load(...): - typically a dict with keys like 'model', 'optimizer', etc. - - Notes: - - All shards are loaded onto CPU (map_location='cpu'). - - This function is for 2D layout (tensor parallel + expert parallel). - - Directory numeric widths (zero-padding) are not assumed; we reuse the actual directory names found. - """ - dir_map_2d, t_vals, e_vals = _scan_mp_dirs_2d(load_path) - - # Build nested structure: [tp][ep] - state_dict: List[List[dict]] = [] - for t in t_vals: - tp_list: List[dict] = [] - for e in e_vals: - sub_dir = dir_map_2d.get((t, e)) - if sub_dir is None: - raise FileNotFoundError( - f"Missing shard directory for (tp={t}, ep={e}) under {load_path}" - ) - checkpoint_path = os.path.join(load_path, sub_dir, _SHARD_FILE) - if not os.path.isfile(checkpoint_path): - raise FileNotFoundError(f"Shard file not found: {checkpoint_path}") - - print(f"Loading checkpoint from: {checkpoint_path}") - ckpt = torch.load(checkpoint_path, map_location='cpu', weights_only=False) - tp_list.append(ckpt) - state_dict.append(tp_list) - - print(f"Loaded {len(t_vals)} tensor parallel ranks and {len(e_vals)} expert parallel ranks") - return state_dict - - -def save_megatron_checkpoint_tp_ep(state_dict, save_path: str, pad_tp: int = 2, pad_ep: int = 3): - """ - Save a 2D Megatron checkpoint layout to save_path as: - mp_rank_{tp:0{pad_tp}d}_{ep:0{pad_ep}d}/model_optim_rng.pt - - Args: - state_dict: 2-level nested list organized as state_dict[tp_idx][ep_idx]. - Each leaf element is the checkpoint object to be torch.save'd. - save_path: Target root directory. - pad_tp: Zero-padding width for 'tp' in directory names (default 2). - pad_ep: Zero-padding width for 'ep' in directory names (default 3). - - Raises: - ValueError: if state_dict is not a 2-level nested list. - """ - if not isinstance(state_dict, list) or not state_dict: - raise ValueError("state_dict must be a non-empty 2-level nested list: [tp][ep].") - - if not isinstance(state_dict[0], list) or not state_dict[0]: - raise ValueError("state_dict must be a 2-level nested list: [tp][ep].") - - os.makedirs(save_path, exist_ok=True) - - for t_idx, tp_list in enumerate(state_dict): - if not isinstance(tp_list, list): - raise ValueError(f"Expected state_dict[{t_idx}] to be a list (ep dimension).") - for e_idx, ckpt in enumerate(tp_list): - sub_dir_name = f"mp_rank_{t_idx:0{pad_tp}d}_{e_idx:0{pad_ep}d}" - full_dir = os.path.join(save_path, sub_dir_name) - os.makedirs(full_dir, exist_ok=True) - checkpoint_path = os.path.join(full_dir, _SHARD_FILE) - torch.save(ckpt, checkpoint_path) - print(f"Saved Megatron shard to: {checkpoint_path}") \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/vision_patch.py b/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/vision_patch.py deleted file mode 100644 index 6c26f79d..00000000 --- a/tools/convert_checkpoint/custom/llavaov_1_5_30b_a3b/vision_patch.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert vision patch from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - # source = state_dict[0]['model'] if args.pipeline_model_parallel_size == 1 else state_dict[0][0]['model'] - source = state_dict[0][0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert vision patch from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/custom/qwen2_vl/__init__.py b/tools/convert_checkpoint/custom/qwen2_vl/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tools/convert_checkpoint/custom/qwen2_vl/adapter.py b/tools/convert_checkpoint/custom/qwen2_vl/adapter.py deleted file mode 100644 index 06031412..00000000 --- a/tools/convert_checkpoint/custom/qwen2_vl/adapter.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import io -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert adapter from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0]['model'] if args.pipeline_model_parallel_size == 1 else state_dict[0][0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert adapter from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - for k in ['adapter.linear_fc1._extra_state', 'adapter.linear_fc2._extra_state']: - extra_state = io.BytesIO() - torch.save(None, extra_state) - target[k] = extra_state - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/custom/qwen2_vl/merge_huggingface.py b/tools/convert_checkpoint/custom/qwen2_vl/merge_huggingface.py deleted file mode 100644 index 641747f5..00000000 --- a/tools/convert_checkpoint/custom/qwen2_vl/merge_huggingface.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_huggingface_checkpoint, - save_huggingface_checkpoint, -) - - -def parse_args(): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language expert model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge huggingface checkpoints ======") - -language_model = load_huggingface_checkpoint(args.language_model_path) -vision_model = load_huggingface_checkpoint(args.vision_model_path) -adapter = load_huggingface_checkpoint(args.adapter_path) -patch = load_huggingface_checkpoint(args.vision_patch) - -for module in [vision_model, adapter, patch]: - merge_dict(module, language_model) - -save_huggingface_checkpoint(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/qwen2_vl/merge_megatron.py b/tools/convert_checkpoint/custom/qwen2_vl/merge_megatron.py deleted file mode 100644 index 718fbf60..00000000 --- a/tools/convert_checkpoint/custom/qwen2_vl/merge_megatron.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -import argparse -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, -) - - -def parse_args(title=None): - """Parse all arguments.""" - parser = argparse.ArgumentParser(description='Merger Arguments', allow_abbrev=False) - group = parser.add_argument_group(title='checkpoint') - group.add_argument('--language_model_path', type=str, help="Path to language model."), - group.add_argument('--vision_model_path', type=str, help="Path to vision model."), - group.add_argument('--vision_patch', type=str, help="Path to vision patch."), - group.add_argument('--adapter_path', type=str, help="Path to adapter."), - group.add_argument("--save_ckpt_path", type=str, help="Path to save checkpoint.") - group.add_argument("--megatron_path", type=str, help="Base directory of Megatron repository") - group.add_argument("--tensor_model_parallel_size", type=int, default=1, help="Tensor parallel size.") - group.add_argument("--pipeline_model_parallel_size", type=int, default=1, help="Pipeline parallel size.") - - return parser.parse_args() - - -def merge_dict(source, destination): - """ merge two dictionaries recursively """ - for key, value in source.items(): - if isinstance(value, dict): - node = destination.setdefault(key, {}) - merge_dict(value, node) - else: - destination[key] = value - - -args = parse_args() -if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - -print("===== merge megatron checkpoints ======") - -language_model = load_megatron_checkpoint(args.language_model_path) -vision_model = load_megatron_checkpoint(args.vision_model_path) -adapter = load_megatron_checkpoint(args.adapter_path) -patch = load_megatron_checkpoint(args.vision_patch) - -if args.pipeline_model_parallel_size == 1: - for module in [vision_model, adapter, patch]: - assert len(module) == len(language_model) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[i]['model']) -else: - for module in [vision_model, adapter, patch]: - assert len(module) == len(language_model[0]) - for i in range(len(module)): - merge_dict(module[i]['model'], language_model[0][i]['model']) - -save_megatron_checkpoint(language_model, args.save_ckpt_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/qwen2_vl/util.py b/tools/convert_checkpoint/custom/qwen2_vl/util.py deleted file mode 100644 index 37a063cd..00000000 --- a/tools/convert_checkpoint/custom/qwen2_vl/util.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import torch -import json -from os.path import dirname -from safetensors.torch import load_file, save_file -from huggingface_hub import split_torch_state_dict_into_shards -from transformers.modeling_utils import SAFE_WEIGHTS_INDEX_NAME, SAFE_WEIGHTS_NAME - - -def merge_transformers_sharded_states(path, num_checkpoints): - """ - Merge sharded checkpoints from transformers into a single checkpoint. - - Args: - path (str): the path to the sharded checkpoints - num_checkpoints (int): the number of checkpoints to merge - """ - state_dict = {} - for i in range(1, num_checkpoints + 1): - checkpoint_path = os.path.join(path, f"model-{i:05d}-of-{num_checkpoints:05d}.safetensors") - current_chunk = load_file(checkpoint_path) - state_dict.update(current_chunk) - return state_dict - -def load_huggingface_checkpoint(load_path): - """ load ckpt """ - state_dict = {} - sub_dirs = [x for x in os.listdir(load_path) if x.endswith("safetensors")] - if len(sub_dirs) == 1: - checkpoint_name = "model.safetensors" - state_dict = load_file(os.path.join(load_path, checkpoint_name), device="cpu") - else: - num_checkpoints = len(sub_dirs) - state_dict = merge_transformers_sharded_states(load_path, num_checkpoints) - return state_dict - - -def save_huggingface_checkpoint(state_dict, save_path): - """ save ckpt """ - os.makedirs(save_path, exist_ok=True) - checkpoint_path = os.path.join(save_path, "model.safetensors") - - state_dict_split = split_torch_state_dict_into_shards(state_dict) - for shard_file, tensors in state_dict_split.filename_to_tensors.items(): - shard = {} - for tensor in tensors: - shard[tensor] = state_dict[tensor].contiguous() - del state_dict[tensor] - shard_path = os.path.join(save_path, shard_file) - save_file(shard, shard_path, metadata={"format": "pt"}) - print(f"Saving HuggingFace shard to: {shard_path}") - - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = os.path.join(save_path, SAFE_WEIGHTS_INDEX_NAME) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - - -def load_megatron_checkpoint(load_path): - """ load ckpt """ - state_dict = [] - sub_dirs = sorted([x for x in os.listdir(load_path) if x.startswith("mp_rank")]) - last_dir = sub_dirs[-1].split('_') - if len(last_dir) == 4: - tp = int(last_dir[-2]) + 1 - pp = int(last_dir[-1]) + 1 - for p in range(pp): - state_dict.append([]) - for t in range(tp): - checkpoint_name = f"mp_rank_{t:02d}_{p:03d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu', weights_only=False) - state_dict[p].append(ckpt) - return state_dict - else: - for t in range(len(sub_dirs)): - checkpoint_name = f"mp_rank_{t:02d}/model_optim_rng.pt" - ckpt = torch.load(os.path.join(load_path, checkpoint_name), map_location='cpu', weights_only=False) - state_dict.append(ckpt) - return state_dict - - -def save_megatron_checkpoint(state_dict, save_path): - """ save ckpt """ - if isinstance(state_dict[0], list): - for p in range(len(state_dict)): - for t in range(len(state_dict[p])): - sub_dir_name = f"mp_rank_{t:02d}_{p:03d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[p][t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") - else: - for t in range(len(state_dict)): - sub_dir_name = f"mp_rank_{t:02d}" - os.makedirs(os.path.join(save_path, sub_dir_name), exist_ok=True) - checkpoint_path = os.path.join(save_path, sub_dir_name, "model_optim_rng.pt") - torch.save(state_dict[t], checkpoint_path) - print(f"Saving Megatron shard to: {checkpoint_path}") \ No newline at end of file diff --git a/tools/convert_checkpoint/custom/qwen2_vl/vision_patch.py b/tools/convert_checkpoint/custom/qwen2_vl/vision_patch.py deleted file mode 100644 index d5b2d9d4..00000000 --- a/tools/convert_checkpoint/custom/qwen2_vl/vision_patch.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import sys -import json -import torch -from os.path import dirname -from copy import deepcopy -from einops import rearrange -from safetensors.torch import load_file, save_file - -SCRIPT_DIR = dirname(os.path.abspath(__file__)) -sys.path.append(dirname(dirname(dirname(SCRIPT_DIR)))) - -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.custom.qwen2_vl.util import ( - load_megatron_checkpoint, - save_megatron_checkpoint, - load_huggingface_checkpoint, - save_huggingface_checkpoint -) - - -args = parse_args() -name_map = {} # megatron -> huggingface -with open(args.common_config_path, 'r', encoding='utf-8') as f: - name_map = json.loads(f.read()) - -if (args.load_platform, args.save_platform) == ('mcore', 'huggingface'): - """ megatron to huggingface """ - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - print(" ====== convert vision patch from Megatron Core to HuggingFace ======") - target = {} - state_dict = load_megatron_checkpoint(args.load_ckpt_path) - source = state_dict[0]['model'] if args.pipeline_model_parallel_size == 1 else state_dict[0][0]['model'] - for k1, k2 in name_map.items(): - target[k2] = source[k1] - save_huggingface_checkpoint(target, args.save_ckpt_path) - -elif (args.load_platform, args.save_platform) == ('huggingface', 'mcore'): - """ huggingface to megatron """ - print(" ====== convert vision patch from HuggingFace to Megatron Core ======") - tp = args.tensor_model_parallel_size - source = load_huggingface_checkpoint(args.load_ckpt_path) - target = {} - for k1, k2 in name_map.items(): - target[k1] = source[k2] - print(f" > {k1}") - state_dict = [{'model': deepcopy(target)} for i in range(tp)] - save_megatron_checkpoint(state_dict, os.path.join(args.save_ckpt_path, 'release')) - -else: - raise NotImplementedError - diff --git a/tools/convert_checkpoint/huggingface_checkpoint.py b/tools/convert_checkpoint/huggingface_checkpoint.py deleted file mode 100644 index 19f44576..00000000 --- a/tools/convert_checkpoint/huggingface_checkpoint.py +++ /dev/null @@ -1,999 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import torch -import json -import resource - -import concurrent.futures -from convert_checkpoint.abstact_checkpoint import AbstractCheckpoint -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.common_checkpoint import CommonCheckpoint - -from convert_checkpoint.utils import ( - transpose_shape0, - touch_file, - check_all_done, - get_done_keys, - make_hf_sub_checkpoints, -) - -from convert_checkpoint import utils -from convert_checkpoint.common_checkpoint import ( - WORD_EMBEDDINGS, - WORD_POSITION_EMBEDDINGS, - WORD_BLOCK_POSITION_EMBEDDINGS, - TRANSFORMER, - LAYER_PREFIX, - INPUT_LAYERNORM, - ROTARY_EMB_INV_FREQ, - ATTENTION_ROTARY_EMB_INV_FREQ, - ATTENTION_QUERY_KEY_VALUE, - ATTENTION_QKV_MAP, - ATTENTION_DENSE, - POST_ATTENTION_LAYERNORM, - MOE_GATE, - MOE_GATE_BIAS, - MOE_MLP, - MOE_EXPERT, - MOE_SHARED_EXPERT, - MLP_DENSE_H_TO_4H, - MLP_DENSE_4H_TO_H, - POST_MLP_LAYERNORM, - FINAL_LAYERNORM, - WORD_EMBEDDINGS_FOR_HEAD, - MTP_WORD_EMBEDDING, - MTP_ENORM, - MTP_HNORM, - MTP_EH_PROJ, - MTP_SHARED_HEAD_NORM, - MTP_SHARED_HEAD_HEAD, -) - - -def merge_transformers_sharded_states(path, num_checkpoints, load_safe=False): - """ - Merge sharded checkpoints from transformers into a single checkpoint. - - Args: - path (str): the path to the sharded checkpoints - num_checkpoints (int): the number of checkpoints to merge - """ - if load_safe: - from safetensors.torch import load_file - args = parse_args() - state_dict = {} - current_chunks = [None] * (num_checkpoints + 1) - def load_files(i): - if load_safe: - if args.checkpoint_format is not None: - checkpoint_path = os.path.join(path, args.checkpoint_format.format(i=i, num_checkpoints=num_checkpoints)) - else: - checkpoint_path = os.path.join(path, f"model-{i:05d}-of-{num_checkpoints:05d}.safetensors") - current_chunks[i] = load_file(checkpoint_path, device="cpu") - else: - if args.checkpoint_format is not None: - checkpoint_path = os.path.join(path, args.checkpoint_format.format(i=i, num_checkpoints=num_checkpoints)) - else: - checkpoint_path = os.path.join(path, f"pytorch_model-{i:05d}-of-{num_checkpoints:05d}.bin") - current_chunks[i] = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - print(f"Loaded huggingface checkpoint: {checkpoint_path}") - if args.max_workers is None: - for i in range(1, num_checkpoints + 1): - load_files(i) - else: - with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor: - for i in range(1, num_checkpoints + 1): - executor.submit(load_files, i) - executor.shutdown(wait=True) - for i in range(1, num_checkpoints + 1): - state_dict.update(current_chunks[i]) - return state_dict - - -class HuggingFaceCheckpoint(AbstractCheckpoint): - """ - HuggingFaceCheckpoint - """ - - def __init__(self, num_layers): - super().__init__(num_layers) - self.optim_dict = None - self.sched_dict = None - - def update_tensor(self, key, value): - if value is not None: - self.state_dict[key] = value - - @staticmethod - def convert_from_common(c_ckpt, c_config): - """ - Convert HuggingFace checkpoint to common checkpoint. - - Args: - c_cpkt: CommonCheckpoint - c_config: CommonConfig - """ - - print("==================== Common -> HuggingFace ====================") - - name_map = c_config.get("name_map")["huggingface"] - cargs = c_config.get_args("common") - hargs = c_config.get_args("huggingface") - args = parse_args() - cache_path = args.cache_path - if args.load_platform == "megatron": - margs = c_config.get_args("megatron") - else: - margs = c_config.get_args("mcore") - dtype = c_config.get_dtype() - num_nextn_predict_layers = hargs.get("num_nextn_predict_layers", 0) - ori_num_layers = cargs["num_layers"] - num_layers = ori_num_layers + num_nextn_predict_layers - first_k_dense_replace = margs.get("first_k_dense_replace", None) - sub_num_layers_for_save = args.sub_num_layers_for_save - if utils.LOADED_LAYERS is None and sub_num_layers_for_save is None and \ - args.max_workers is not None and args.max_workers > 1: - args.sub_num_layers_for_save = 5 - sub_num_layers_for_save = args.sub_num_layers_for_save - separate_dtype = c_config.get("separate_dtype") - - if args.num_experts is not None: - if MOE_MLP in name_map: - moe_mlp = name_map[MOE_MLP] - moe_expert = name_map[MOE_EXPERT] - if MOE_SHARED_EXPERT in name_map: - moe_shared_expert = name_map[MOE_SHARED_EXPERT] - - h_ckpt = HuggingFaceCheckpoint(num_layers) - h_ckpt.set_dtype(dtype) - - # 1.1 word_embeddings - if WORD_EMBEDDINGS in name_map: - name = name_map[WORD_EMBEDDINGS] + ".weight" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name, c_ckpt.get_word_embedding()) - c_ckpt.set_word_embedding(None) - - # 1.2 add_position_embedding: - if margs.get("add_position_embedding", False): - name = name_map[WORD_POSITION_EMBEDDINGS] + ".weight" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name, c_ckpt.get_word_position_embedding()) - c_ckpt.set_word_position_embedding(None) - - if margs.get("add_block_position_embedding", False): - name = name_map[WORD_BLOCK_POSITION_EMBEDDINGS] + ".weight" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name, c_ckpt.get_word_block_position_embedding()) - c_ckpt.set_word_block_position_embedding(None) - - # 2. Transformer - hidden_size = cargs["hidden_size"] - heads = cargs["num_attention_heads"] - use_rotary_position_embeddings = margs.get("use_rotary_position_embeddings", False) - hidden_size_per_head = hidden_size // heads - - transformer = name_map[TRANSFORMER] - layer_prefix = name_map[LAYER_PREFIX] - - if utils.LOADED_LAYERS is not None or sub_num_layers_for_save is not None: - save_option = dict() - save_option["save_optim"] = False - save_option["save_safe"] = args.safetensors - - if utils.LOADED_LAYERS is not None: - cur_sub_num_layer = min(utils.LOADED_LAYERS) - sub_h_ckpts = {} - sub_h_ckpts[cur_sub_num_layer] = HuggingFaceCheckpoint(num_layers) - elif sub_num_layers_for_save is not None: - cur_sub_num_layer = 0 - sub_h_ckpt_count = (num_layers + sub_num_layers_for_save - 1) // sub_num_layers_for_save - sub_h_ckpts = {index : HuggingFaceCheckpoint(num_layers) for index in range(sub_h_ckpt_count)} - - if utils.LOADED_STATE_DICT is not None: - del utils.LOADED_STATE_DICT - for layer_id in range(num_layers): - if utils.LOADED_LAYERS is not None and layer_id not in utils.LOADED_LAYERS: - continue - if utils.LOADED_LAYERS is None and layer_id > 0 and sub_num_layers_for_save is not None \ - and layer_id % sub_num_layers_for_save == 0: - sub_h_ckpts[cur_sub_num_layer].state_dict = h_ckpt.state_dict - h_ckpt.state_dict = {} - cur_sub_num_layer += 1 - - if cache_path is None: - one_layer_weights = None - else: - one_layer_weights = torch.load(f"{cache_path}/checkpoint_{layer_id}.pt", - map_location="cpu", weights_only=False) - # 2.1 input_layernorm - if INPUT_LAYERNORM in name_map: - name = name_map[INPUT_LAYERNORM] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name + ".weight", c_ckpt.get_layer_input_layernorm_weight( - layer_id, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(name + ".bias", c_ckpt.get_layer_input_layernorm_bias( - layer_id, one_layer_weights=one_layer_weights)) - c_ckpt.clear_layer_input_layernorm(layer_id, one_layer_weights=one_layer_weights) - - # 2.2 rotary_emb.inv_freq - if ATTENTION_ROTARY_EMB_INV_FREQ in name_map: - assert use_rotary_position_embeddings == True, \ - f"args.use_rotary_position_embeddings is required to be set to True since we capture the rotary_emb op" - name = name_map[ATTENTION_ROTARY_EMB_INV_FREQ] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name, c_ckpt.get_layer_attention_rotary_emb_inv_freq( - layer_id, one_layer_weights=one_layer_weights)) - c_ckpt.clear_layer_attention_rotary_emb_inv_freq(layer_id, one_layer_weights=one_layer_weights) - if ROTARY_EMB_INV_FREQ in name_map: - assert use_rotary_position_embeddings == True, \ - f"args.use_rotary_position_embeddings is required to be set to True since we capture the rotary_emb op" - name = name_map[ROTARY_EMB_INV_FREQ] - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name, c_ckpt.get_layer_attention_rotary_emb_inv_freq( - layer_id, one_layer_weights=one_layer_weights)) - c_ckpt.clear_layer_attention_rotary_emb_inv_freq(layer_id, one_layer_weights=one_layer_weights) - - # 2.3 self attention query_key_value - if ATTENTION_QUERY_KEY_VALUE in name_map: - name = name_map[ATTENTION_QUERY_KEY_VALUE] - names = name if isinstance(name, list) else [name] - weight = c_ckpt.get_layer_attention_query_key_value_weight( - layer_id, one_layer_weights=one_layer_weights) - bias = c_ckpt.get_layer_attention_query_key_value_bias(layer_id, one_layer_weights=one_layer_weights) - - # transpose weight in shape[0] for llama - transpose_query_key_value = margs.get("transpose_query_key_value", False) - num_key_value_heads = cargs.get("num_key_value_heads", heads) - assert heads % num_key_value_heads == 0 - num_repeats = heads // num_key_value_heads - num_splits = num_repeats + 2 # repeats*Q + K + V - if not transpose_query_key_value: - assert len(names) == 1 - weight = [weight] - bias = [bias] - else: - weight = transpose_shape0(weight, num_key_value_heads, num_splits) - weight = list(torch.chunk(weight, num_splits, dim=0)) - q, k, v = torch.cat(weight[:-2]), weight[-2], weight[-1] - q = transpose_shape0(q, num_repeats, num_key_value_heads) - if len(names) == 1: - weight = [torch.cat((q, k, v), dim=0)] - else: - assert len(names) == 3 - weight = [q, k, v] - - - if bias is not None: - bias = transpose_shape0(bias, num_key_value_heads, num_splits) - bias = list(torch.chunk(bias, num_splits, dim=0)) - q, k, v = torch.cat(bias[:-2]), bias[-2], bias[-1] - q = transpose_shape0(q, num_repeats, num_key_value_heads) - if len(names) == 1: - bias = [torch.cat((q, k, v), dim=0)] - else: - assert len(names) == 3 - bias = [q, k, v] - - for i in range(len(names)): - item = f"{transformer}.{layer_prefix}.{layer_id}.{names[i]}" - print(f"> update_tensor: {item}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(item + ".weight", weight[i]) - if bias is not None: - h_ckpt.update_tensor(item + ".bias", bias[i]) - c_ckpt.clear_layer_attention_query_key_value(layer_id, one_layer_weights=one_layer_weights) - if ATTENTION_QKV_MAP in name_map: - name = name_map[ATTENTION_QKV_MAP] - names = name if isinstance(name, list) else [name] - for sub_key, name_hf in name_map[ATTENTION_QKV_MAP].items(): - item = f"{transformer}.{layer_prefix}.{layer_id}.{name_hf}" - weight, weight_scale_inv = c_ckpt.get_layer_attention_weight_by_name( - sub_key, layer_id, one_layer_weights=one_layer_weights) - if weight is None: - continue - bias = c_ckpt.get_layer_attention_bias_by_name( - sub_key, layer_id, one_layer_weights=one_layer_weights) - print(f"> update_tensor: {item}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(item + ".weight", weight) - if bias is not None: - h_ckpt.update_tensor(item + ".bias", bias) - if weight_scale_inv is not None: - h_ckpt.update_tensor(item + ".weight_scale_inv", weight_scale_inv) - c_ckpt.clear_layer_attention_by_name(sub_key, layer_id, one_layer_weights=one_layer_weights) - - # 2.4 self attention dense - name = name_map[ATTENTION_DENSE] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - weight, weight_scale_inv = c_ckpt.get_layer_attention_dense_weight( - layer_id, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(name + ".weight", weight) - h_ckpt.update_tensor(name + ".bias", c_ckpt.get_layer_attention_dense_bias( - layer_id, one_layer_weights=one_layer_weights)) - if weight_scale_inv is not None: - h_ckpt.update_tensor(name + ".weight_scale_inv", weight_scale_inv) - c_ckpt.clear_layer_attention_dense(layer_id, one_layer_weights=one_layer_weights) - - # 2.5 post attention layernorm - name = name_map[POST_ATTENTION_LAYERNORM] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name + ".weight", c_ckpt.get_layer_post_attention_layernorm_weight( - layer_id, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(name + ".bias", c_ckpt.get_layer_post_attention_layernorm_bias( - layer_id, one_layer_weights=one_layer_weights)) - c_ckpt.clear_layer_post_attention_layernorm(layer_id, one_layer_weights=one_layer_weights) - - # 2.6 mlp dense h_to_4h - name = name_map[MLP_DENSE_H_TO_4H] - names = name if isinstance(name, list) else [name] - def update_ffn_tensor_node(item_prefix, weight, bias, weight_scale_inv=None): - if weight is None: - return - weight = torch.chunk(weight, len(names), dim=0) - if bias is not None: - bias = torch.chunk(bias, len(names), dim=0) - if weight_scale_inv is not None: - weight_scale_inv = torch.chunk(weight_scale_inv, len(names), dim=0) - - for i in range(len(names)): - item = f"{item_prefix}.{names[i]}" - print(f"> update_tensor: {item}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(item + ".weight", weight[i]) - if bias is not None: - h_ckpt.update_tensor(item + ".bias", bias[i]) - if weight_scale_inv is not None: - h_ckpt.update_tensor(item + ".weight_scale_inv", weight_scale_inv[i]) - if args.num_experts is None: - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_h_to_4h_weight( - layer_id, one_layer_weights=one_layer_weights) - bias = c_ckpt.get_layer_mlp_dense_h_to_4h_bias(layer_id, one_layer_weights=one_layer_weights) - update_ffn_tensor_node(f"{transformer}.{layer_prefix}.{layer_id}", - weight, bias, weight_scale_inv=weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_h_to_4h(layer_id, one_layer_weights=one_layer_weights) - else: - if first_k_dense_replace is not None and layer_id < first_k_dense_replace: - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_h_to_4h_weight( - layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights) - bias = c_ckpt.get_layer_mlp_dense_h_to_4h_bias( - layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights) - update_ffn_tensor_node(f"{transformer}.{layer_prefix}.{layer_id}.{moe_mlp}", - weight, bias, weight_scale_inv=weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_h_to_4h(layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights) - else: - name = name_map[MOE_GATE] - hf_name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {hf_name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - weight = c_ckpt.get_layer_moe_gate_weight(layer_id, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(hf_name + ".weight", weight) - if MOE_GATE_BIAS in name_map: - bias_name = name_map[MOE_GATE_BIAS] - bias_name = f"{transformer}.{layer_prefix}.{layer_id}.{bias_name}" - else: - bias_name = hf_name + ".bias" - bias = c_ckpt.get_layer_moe_gate_bias(layer_id, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(bias_name, bias) - c_ckpt.clear_layer_moe_gate(layer_id, one_layer_weights=one_layer_weights) - - if MOE_SHARED_EXPERT in name_map: - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_h_to_4h_weight( - layer_id, is_shared=True, one_layer_weights=one_layer_weights) - bias = c_ckpt.get_layer_mlp_dense_h_to_4h_bias( - layer_id, is_shared=True, one_layer_weights=one_layer_weights) - update_ffn_tensor_node(f"{transformer}.{layer_prefix}.{layer_id}.{moe_shared_expert}", - weight, bias, weight_scale_inv=weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_h_to_4h( - layer_id, is_shared=True, one_layer_weights=one_layer_weights) - for expert_id in range(0, args.num_experts): - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_h_to_4h_weight( - layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights) - bias = c_ckpt.get_layer_mlp_dense_h_to_4h_bias( - layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights) - update_ffn_tensor_node(f"{transformer}.{layer_prefix}.{layer_id}.{moe_expert}.{expert_id}", - weight, bias, weight_scale_inv=weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_h_to_4h( - layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights) - - # 2.7 mlp dense 4h_to_h - if args.num_experts is None: - name = name_map[MLP_DENSE_4H_TO_H] - hf_name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {hf_name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_4h_to_h_weight( - layer_id, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(hf_name + ".weight", weight) - h_ckpt.update_tensor(hf_name + ".bias", c_ckpt.get_layer_mlp_dense_4h_to_h_bias( - layer_id, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(hf_name + ".weight_scale_inv", weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_4h_to_h(layer_id, one_layer_weights=one_layer_weights) - else: - name = name_map[MLP_DENSE_4H_TO_H] - if first_k_dense_replace is not None and layer_id < first_k_dense_replace: - hf_name = f"{transformer}.{layer_prefix}.{layer_id}.{moe_mlp}.{name}" - print(f"> update_tensor: {hf_name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_4h_to_h_weight( - layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(hf_name + ".weight", weight) - h_ckpt.update_tensor(hf_name + ".bias", c_ckpt.get_layer_mlp_dense_4h_to_h_bias( - layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(hf_name + ".weight_scale_inv", weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_4h_to_h(layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights) - else: - if MOE_SHARED_EXPERT in name_map: - hf_name = f"{transformer}.{layer_prefix}.{layer_id}.{moe_shared_expert}.{name}" - print(f"> update_tensor: {hf_name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_4h_to_h_weight( - layer_id, is_shared=True, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(hf_name + ".weight", weight) - h_ckpt.update_tensor(hf_name + ".bias", c_ckpt.get_layer_mlp_dense_4h_to_h_bias( - layer_id, is_shared=True, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(hf_name + ".weight_scale_inv", weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_4h_to_h( - layer_id, is_shared=True, one_layer_weights=one_layer_weights) - for expert_id in range(0, args.num_experts): - hf_name = f"{transformer}.{layer_prefix}.{layer_id}.{moe_expert}.{expert_id}.{name}" - print(f"> update_tensor: {hf_name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - weight, weight_scale_inv = c_ckpt.get_layer_mlp_dense_4h_to_h_weight( - layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(hf_name + ".weight", weight) - h_ckpt.update_tensor(hf_name + ".bias", c_ckpt.get_layer_mlp_dense_4h_to_h_bias( - layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(hf_name + ".weight_scale_inv", weight_scale_inv) - c_ckpt.clear_layer_mlp_dense_4h_to_h( - layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights) - - # 2.8 post mlp layernorm - if POST_MLP_LAYERNORM in name_map: - name = name_map[POST_MLP_LAYERNORM] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name + ".weight", c_ckpt.get_layer_post_mlp_layernorm_weight( - layer_id, one_layer_weights=one_layer_weights)) - h_ckpt.update_tensor(name + ".bias", c_ckpt.get_layer_post_mlp_layernorm_bias( - layer_id, one_layer_weights=one_layer_weights)) - c_ckpt.clear_layer_post_mlp_layernorm(layer_id, one_layer_weights=one_layer_weights) - - if MTP_WORD_EMBEDDING in name_map and layer_id >= ori_num_layers: - (mtp_word_embedding, mtp_enorm, mtp_hnorm, mtp_eh_proj), \ - (mtp_shared_head_norm, mtp_shared_head_head) = c_ckpt.get_layer_mtp_weight( - layer_id, one_layer_weights=one_layer_weights) - h_ckpt.update_tensor(f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_WORD_EMBEDDING]}.weight", - mtp_word_embedding) - h_ckpt.update_tensor(f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_ENORM]}.weight", mtp_enorm) - h_ckpt.update_tensor(f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_HNORM]}.weight", mtp_hnorm) - h_ckpt.update_tensor(f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_EH_PROJ]}.weight", - mtp_eh_proj) - h_ckpt.update_tensor(f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_SHARED_HEAD_NORM]}.weight", - mtp_shared_head_norm.clone()) - h_ckpt.update_tensor(f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_SHARED_HEAD_HEAD]}.weight", - mtp_shared_head_head.clone()) - print(f"> update_tensor mtp, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}. "\ - f"mtp_word_embedding: {mtp_word_embedding.shape}, "\ - f"mtp_enorm: {mtp_enorm.shape}, mtp_hnorm: {mtp_hnorm.shape}, "\ - f"mtp_eh_proj: {mtp_eh_proj.shape}, mtp_shared_head_norm: {mtp_shared_head_norm.shape}, "\ - f"mtp_shared_head_head: {mtp_shared_head_head.shape}") - c_ckpt.clear_layer_mtp_weight(layer_id, one_layer_weights=one_layer_weights) - - # 2.9 final_layernorm - if FINAL_LAYERNORM in name_map: - name = name_map[FINAL_LAYERNORM] - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(f"{name}.weight", c_ckpt.get_final_layernorm_weight()) - h_ckpt.update_tensor(f"{name}.bias", c_ckpt.get_final_layernorm_bias()) - c_ckpt.clear_final_layernorm() - - # 3 word embedding for head - if WORD_EMBEDDINGS_FOR_HEAD in name_map: - name = name_map[WORD_EMBEDDINGS_FOR_HEAD] + ".weight" - print(f"> update_tensor: {name}, max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - h_ckpt.update_tensor(name, c_ckpt.get_word_embeddings_for_head_weight()) - c_ckpt.clear_word_embeddings_for_head() - - # 4 optimizer - if c_ckpt.has_optimizer(): - h_ckpt.optim_dict = c_ckpt.state_dict["optimizer"]["optimizer"] - h_ckpt.sched_dict = c_ckpt.state_dict["optimizer"]["opt_param_scheduler"] # todo - - if utils.LOADED_LAYERS is not None: - sub_h_ckpts[cur_sub_num_layer].state_dict = h_ckpt.state_dict - h_ckpt.state_dict = {} - elif sub_num_layers_for_save is not None: - sub_h_ckpts[cur_sub_num_layer].state_dict = h_ckpt.state_dict - h_ckpt.state_dict = {} - def save_sub_ckpt(sub_num_layer): - if utils.LOADED_MIN_E is None: - sub_dir_name= sub_num_layer - else: - sub_dir_name = sub_num_layer * 1000 + utils.LOADED_MIN_E - sub_h_ckpts[sub_num_layer].save( - f"{args.save_ckpt_path}/sub_checkpoint/{sub_dir_name}", None, **save_option) - if args.max_workers is not None and args.max_workers > 1: - futures = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor: - for sub_num_layer in sorted(sub_h_ckpts.keys()): - futures.append(executor.submit(save_sub_ckpt, sub_num_layer)) - - concurrent.futures.wait(futures) - for future in futures: - try: - result = future.result() - except Exception as e: - print(f"An error occurred: {e}") - raise e - elif utils.LOADED_LAYERS is not None or sub_num_layers_for_save is not None: - for sub_num_layer in sorted(sub_h_ckpts.keys()): - if utils.LOADED_MIN_E is None: - sub_dir_name= sub_num_layer - else: - sub_dir_name = sub_num_layer * 1000 + utils.LOADED_MIN_E - sub_h_ckpts[sub_num_layer].save( - f"{args.save_ckpt_path}/sub_checkpoint/{sub_dir_name}", None, **save_option) - if utils.LOADED_LAYERS is None and sub_num_layers_for_save is not None: - make_hf_sub_checkpoints(args.save_ckpt_path) - - del c_ckpt.state_dict - return h_ckpt - - def convert_to_common(self, config): - """ - Convert HuggingFace checkpoint to common checkpoint. - - Args: - config: CommonConfig - """ - - print("==================== HuggingFace -> Common ====================") - - name_map = config.get_name_map("huggingface") - cargs = config.get_args("common") - args = parse_args() - hargs = config.get_args("huggingface") - if args.save_platform == "megatron": - margs = config.get_args("megatron") - else: - margs = config.get_args("mcore") - cache_path = args.cache_path - if cache_path is not None: - os.makedirs(cache_path, exist_ok=True) - num_nextn_predict_layers = hargs.get("num_nextn_predict_layers", 0) - ori_num_layers = cargs["num_layers"] - num_layers = ori_num_layers + num_nextn_predict_layers - - c_ckpt = CommonCheckpoint(num_layers) - c_ckpt.set_dtype(self.dtype) - - # 1.1 word_embeddings - if WORD_EMBEDDINGS in name_map: - name = name_map[WORD_EMBEDDINGS] + ".weight" - word_embeddings_weight = self.state_dict[name] - c_ckpt.set_word_embedding(word_embeddings_weight) - print(f"> word_embeddings, shape: {word_embeddings_weight.shape}") - - # 1.2 add_position_embedding: - if margs.get("add_position_embedding", False): - name = name_map[WORD_POSITION_EMBEDDINGS] + ".weight" - weight = self.state_dict[name] - c_ckpt.set_word_position_embedding(weight) - print(f"> add_position_embedding, shape: {weight.shape}") - - if margs.get("add_block_position_embedding", False): - name = name_map[WORD_BLOCK_POSITION_EMBEDDINGS] + ".weight" - weight = self.state_dict[name] - c_ckpt.set_word_block_position_embedding(weight) - print(f"> add_block_position_embedding, shape: {weight.shape}") - - # 2. Transformer - hidden_size = cargs["hidden_size"] - heads = cargs["num_attention_heads"] - use_rotary_position_embeddings = margs.get("use_rotary_position_embeddings", False) - hidden_size_per_head = hidden_size // heads - - transformer = name_map[TRANSFORMER] - layer_prefix = name_map[LAYER_PREFIX] - - def set_one_layer_weight(layer_id): - if cache_path is None: - one_layer_weights = None - else: - if os.path.exists(f"{cache_path}/checkpoint_{layer_id}.pt") and \ - os.path.exists(f"{cache_path}/checkpoint_{layer_id}.done"): - return - if os.path.exists(f"{cache_path}/checkpoint_{layer_id}.done"): - os.remove(f"{cache_path}/checkpoint_{layer_id}.done") - if os.path.exists(f"{cache_path}/checkpoint_{layer_id}.pt"): - os.remove(f"{cache_path}/checkpoint_{layer_id}.pt") - one_layer_weights = {} - # 2.1 input_layernorm - if INPUT_LAYERNORM in name_map: - name = name_map[INPUT_LAYERNORM] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - weight = self.state_dict[name + ".weight"] - bias = self.state_dict.get(name + ".bias") - c_ckpt.set_layer_input_layernorm(layer_id, weight, bias, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} {name} weight: {weight.shape}") - - # 2.2 rotary_emb.inv_freq - if ROTARY_EMB_INV_FREQ in name_map: - assert use_rotary_position_embeddings == True, \ - f"args.use_rotary_position_embeddings is required to be set to True since we capture the rotary_emb op" - name = name_map[ROTARY_EMB_INV_FREQ] - param = self.state_dict[name] - c_ckpt.set_layer_attention_rotary_emb_inv_freq(layer_id, param, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} {name}: param shape: {param.shape}") - if ATTENTION_ROTARY_EMB_INV_FREQ in name_map: - assert use_rotary_position_embeddings == True, \ - f"args.use_rotary_position_embeddings is required to be set to True since we capture the rotary_emb op" - name = name_map[ATTENTION_ROTARY_EMB_INV_FREQ] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - if name in self.state_dict: - param = self.state_dict[name] - else: - dim = hidden_size // heads - param = 1.0 / (hargs.get("rotary_base", 10000) ** (torch.arange(0, dim, 2).float() / dim)) - c_ckpt.set_layer_attention_rotary_emb_inv_freq(layer_id, param, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} {name}: param shape: {param.shape}") - - # 2.3 self attention query_key_value - if ATTENTION_QUERY_KEY_VALUE in name_map: - transpose_query_key_value = margs.get("transpose_query_key_value", False) - name = name_map[ATTENTION_QUERY_KEY_VALUE] - names = name if isinstance(name, list) else [name] - weight = [] - bias = [] - for item in names: - item = f"{transformer}.{layer_prefix}.{layer_id}.{item}" - weight.append(self.state_dict[item + ".weight"]) - bias.append(self.state_dict.get(item + ".bias")) - - if not transpose_query_key_value: - assert len(names) == 1 - weight = torch.cat(weight, dim=0) - bias = torch.cat(bias, dim=0) if None not in bias else None - else: - num_key_value_heads = cargs.get("num_key_value_heads", heads) - assert heads % num_key_value_heads == 0 - num_repeats = heads // num_key_value_heads - num_splits = num_repeats + 2 - - if len(names) == 1: - weight = list(torch.chunk(weight[0], num_splits, dim=0)) - q, k, v = torch.cat(weight[:-2]), weight[-2], weight[-1] - else: - assert len(names) == 3 - q, k, v = weight[0], weight[1], weight[2] - q = transpose_shape0(q, num_key_value_heads, num_repeats) - weight = torch.cat([q, k, v], dim=0) - weight = transpose_shape0(weight, num_splits, num_key_value_heads) - - if None in bias: - assert all(x is None for x in bias) - bias = None - else: - if len(names) == 1: - bias = list(torch.chunk(bias[0], num_splits, dim=0)) - q, k, v = torch.cat(bias[:-2]), bias[-2], bias[-1] - else: - assert len(names) == 3 - q, k, v = bias[0], bias[1], bias[2] - q = transpose_shape0(q, num_key_value_heads, num_repeats) - bias = torch.cat([q, k, v], dim=0) - bias = transpose_shape0(bias, num_splits, num_key_value_heads) - - c_ckpt.set_layer_attention_query_key_value( - layer_id, weight, bias, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} attention query key value weight: {weight.shape}") - if ATTENTION_QKV_MAP in name_map: - for key, name_hf in name_map[ATTENTION_QKV_MAP].items(): - item = f"{transformer}.{layer_prefix}.{layer_id}.{name_hf}" - weight = self.state_dict[item + ".weight"] - bias = self.state_dict.get(item + ".bias") - scale_key = item + ".weight_scale_inv" - weight_scale_inv = self.state_dict[scale_key] if scale_key in self.state_dict else None - c_ckpt.set_layer_attention_by_name( - key, layer_id, weight, bias, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - - # 2.4 self attention dense - name = name_map[ATTENTION_DENSE] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - weight = self.state_dict[name + ".weight"] - bias = self.state_dict.get(name + ".bias") - weight_scale_inv = self.state_dict[name + ".weight_scale_inv"] \ - if name + ".weight_scale_inv" in self.state_dict else None - c_ckpt.set_layer_attention_dense(layer_id, weight, bias, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id} attention dense weight: {weight.shape}") - - # 2.5 post attention layernorm - name = name_map[POST_ATTENTION_LAYERNORM] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - weight = self.state_dict[name + ".weight"] - bias = self.state_dict.get(name + ".bias") - c_ckpt.set_layer_post_attention_layernorm( - layer_id, weight, bias, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} post attention layernorm weight: {weight.shape}") - - if args.num_experts is not None: - if MOE_MLP in name_map: - moe_mlp = name_map[MOE_MLP] - moe_expert = name_map[MOE_EXPERT] - if MOE_SHARED_EXPERT in name_map: - moe_shared_expert = name_map[MOE_SHARED_EXPERT] - - def set_mlp_weight(is_moe_mlp = False, expert_id = None, is_shared = False): - # 2.6 mlp dense h_to_4h - name = name_map[MLP_DENSE_H_TO_4H] - names = name if isinstance(name, list) else [name] - weight = [] - bias = [] - weight_scale_inv = None - for item in names: - if is_moe_mlp: - item = f"{transformer}.{layer_prefix}.{layer_id}.{moe_mlp}.{item}" - elif is_shared: - item = f"{transformer}.{layer_prefix}.{layer_id}.{moe_shared_expert}.{item}" - elif expert_id is None: - item = f"{transformer}.{layer_prefix}.{layer_id}.{item}" - else: - item = f"{transformer}.{layer_prefix}.{layer_id}.{moe_expert}.{expert_id}.{item}" - weight.append(self.state_dict[item + ".weight"]) - bias.append(self.state_dict.get(item + ".bias")) - if item + ".weight_scale_inv" in self.state_dict: - if weight_scale_inv is None: - weight_scale_inv = [] - weight_scale_inv.append(self.state_dict[item + ".weight_scale_inv"]) - - weight = torch.cat(weight, dim=0) - bias = torch.cat(bias, dim=0) if None not in bias else None - weight_scale_inv = torch.cat(weight_scale_inv, dim=0) if weight_scale_inv is not None else None - if is_moe_mlp: - c_ckpt.set_layer_mlp_dense_h_to_4h( - layer_id, weight, bias, is_moe_mlp=True, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id}, mlp dense h_to_4h weight: {weight.shape}") - elif is_shared: - c_ckpt.set_layer_mlp_dense_h_to_4h( - layer_id, weight, bias, is_shared=True, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id}, shared_expert mlp dense h_to_4h weight: {weight.shape}") - elif expert_id is None: - c_ckpt.set_layer_mlp_dense_h_to_4h( - layer_id, weight, bias, one_layer_weights=one_layer_weights, weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id} mlp dense h_to_4h weight: {weight.shape}") - else: - c_ckpt.set_layer_mlp_dense_h_to_4h( - layer_id, weight, bias, expert_id=expert_id, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id}, expert_id {expert_id} mlp dense h_to_4h weight: {weight.shape}") - - # 2.7 mlp dense 4h_to_h - name = name_map[MLP_DENSE_4H_TO_H] - if is_moe_mlp: - name = f"{transformer}.{layer_prefix}.{layer_id}.{moe_mlp}.{name}" - elif is_shared: - name = f"{transformer}.{layer_prefix}.{layer_id}.{moe_shared_expert}.{name}" - elif expert_id is None: - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - else: - name = f"{transformer}.{layer_prefix}.{layer_id}.{moe_expert}.{expert_id}.{name}" - weight = self.state_dict[name + ".weight"] - bias = self.state_dict.get(name + ".bias") - weight_scale_inv = self.state_dict[name + ".weight_scale_inv"] \ - if name + ".weight_scale_inv" in self.state_dict else None - if is_moe_mlp: - c_ckpt.set_layer_mlp_dense_4h_to_h( - layer_id, weight, bias, is_moe_mlp=True, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id} moe mlp dense 4h_to_h weight: {weight.shape}") - elif is_shared: - c_ckpt.set_layer_mlp_dense_4h_to_h( - layer_id, weight, bias, is_shared=True, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id} shared_expert mlp dense 4h_to_h weight: {weight.shape}") - elif expert_id is None: - c_ckpt.set_layer_mlp_dense_4h_to_h( - layer_id, weight, bias, one_layer_weights=one_layer_weights, weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id} mlp dense 4h_to_h weight: {weight.shape}") - else: - c_ckpt.set_layer_mlp_dense_4h_to_h( - layer_id, weight, bias, expert_id=expert_id, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - print(f"> layer {layer_id}, expert_id {expert_id} mlp dense 4h_to_h weight: {weight.shape}") - - # 2.8 post mlp layernorm - if POST_MLP_LAYERNORM in name_map: - name = name_map[POST_MLP_LAYERNORM] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - weight = self.state_dict[name + ".weight"] - bias = self.state_dict.get(name + ".bias") - c_ckpt.set_layer_post_mlp_layernorm( - layer_id, weight, bias, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} post mlp layernorm weight: {weight.shape}") - - first_k_dense_replace = hargs.get("first_k_dense_replace", None) - if args.num_experts is None: - set_mlp_weight() - elif first_k_dense_replace is not None and layer_id < first_k_dense_replace: - set_mlp_weight(is_moe_mlp=True) - else: - # moe gate - name = name_map[MOE_GATE] - name = f"{transformer}.{layer_prefix}.{layer_id}.{name}" - if MOE_GATE_BIAS in name_map: - bias_name = name_map[MOE_GATE_BIAS] - bias_name = f"{transformer}.{layer_prefix}.{layer_id}.{bias_name}" - else: - bias_name = name + ".bias" - weight = self.state_dict[name + ".weight"] - bias = self.state_dict.get(bias_name) - c_ckpt.set_layer_moe_gate(layer_id, weight, bias, one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} moe gate weight: {weight.shape}") - if bias is not None: - print(f"> layer {layer_id} moe gate bias: {bias.shape}") - - for expert_id in range(0, args.num_experts): - set_mlp_weight(expert_id=expert_id) - - if MOE_SHARED_EXPERT in name_map: - set_mlp_weight(expert_id=None, is_shared=True) - - if MTP_WORD_EMBEDDING in name_map and layer_id >= ori_num_layers: - mtp_word_embedding = self.state_dict[ - f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_WORD_EMBEDDING]}.weight"] - mtp_enorm = self.state_dict[ - f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_ENORM]}.weight"] - mtp_hnorm = self.state_dict[ - f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_HNORM]}.weight"] - mtp_eh_proj = self.state_dict[ - f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_EH_PROJ]}.weight"] - mtp_shared_head_norm = self.state_dict[ - f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_SHARED_HEAD_NORM]}.weight"] - mtp_shared_head_head = self.state_dict[ - f"{transformer}.{layer_prefix}.{layer_id}.{name_map[MTP_SHARED_HEAD_HEAD]}.weight"] - c_ckpt.set_layer_mtp_weight(layer_id, mtp_word_embedding, mtp_enorm, mtp_hnorm, mtp_eh_proj, \ - mtp_shared_head_norm, mtp_shared_head_head, \ - one_layer_weights=one_layer_weights) - print(f"> layer {layer_id} mtp. mtp_word_embedding: {mtp_word_embedding.shape}, "\ - f"mtp_enorm: {mtp_enorm.shape}, mtp_hnorm: {mtp_hnorm.shape}, "\ - f"mtp_eh_proj: {mtp_eh_proj.shape}, mtp_shared_head_norm: {mtp_shared_head_norm.shape}, "\ - f"mtp_shared_head_head: {mtp_shared_head_head.shape}") - - if cache_path is not None: - torch.save(one_layer_weights, f"{cache_path}/checkpoint_{layer_id}.pt") - done_file_name = f"{cache_path}/checkpoint_{layer_id}.done" - with open(done_file_name, 'w'): - os.utime(done_file_name, None) - - if args.max_workers > 1: - futures = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor: - for one_layer_id in range(num_layers): - futures.append(executor.submit(set_one_layer_weight, one_layer_id)) - - concurrent.futures.wait(futures) - for future in futures: - try: - result = future.result() - except Exception as e: - print(f"An error occurred: {e}") - raise e - else: - for one_layer_id in range(num_layers): - set_one_layer_weight(one_layer_id) - - # 2.9 final_layernorm - if FINAL_LAYERNORM in name_map: - name = name_map[FINAL_LAYERNORM] - weight = self.state_dict[f"{name}.weight"] - bias = self.state_dict.get(f"{name}.bias") - c_ckpt.set_final_layernorm(weight, bias) - print(f"> {name} final_layernorm weight: {weight.shape}") - - # 3 word embedding for head - if WORD_EMBEDDINGS_FOR_HEAD in name_map: - name = name_map[WORD_EMBEDDINGS_FOR_HEAD] + ".weight" - weight = self.state_dict.get(name, word_embeddings_weight) - c_ckpt.set_word_embeddings_for_head(weight) - print(f"> {name} word embedding for head weight: {weight.shape}") - - # 4 optimizer - if self.has_optimizer(): - opt_param_scheduler = self.sched_dict # todo - c_ckpt.state_dict["optimizer"] = { - "optimizer": self.optim_dict, - "opt_param_scheduler": opt_param_scheduler - } - return c_ckpt - - def has_optimizer(self): - """ has optimizer """ - return self.optim_dict is not None and self.sched_dict is not None - - def load(self, load_path, load_safe=False): - """ load ckpt """ - if load_safe: - from safetensors.torch import load_file - sub_dirs = [x for x in os.listdir(load_path) if x.endswith("safetensors")] - if len(sub_dirs) == 1: - checkpoint_name = "model.safetensors" - self.state_dict = load_file(os.path.join(load_path, checkpoint_name), device="cpu") - else: - num_checkpoints = len(sub_dirs) - self.state_dict = merge_transformers_sharded_states(load_path, num_checkpoints, True) - else: - sub_dirs = [x for x in os.listdir(load_path) if x.startswith("pytorch_model")] - if len(sub_dirs) == 1: - checkpoint_name = "pytorch_model.bin" - self.state_dict = torch.load(os.path.join(load_path, checkpoint_name), - map_location="cpu", weights_only=False) - else: - num_checkpoints = len(sub_dirs) - 1 - self.state_dict = merge_transformers_sharded_states(load_path, num_checkpoints) - - - def print_memory_usage(self, desc): - import psutil - process = psutil.Process(os.getpid()) - mem = process.memory_info().rss / 1024**2 # 转为 MB - print(f"{desc}内存占用: {mem:.2f} MB") - - def save(self, save_path, h_config=None, save_optim=True, save_safe=False): - """ save ckpt """ - from huggingface_hub import split_torch_state_dict_into_shards - from transformers.modeling_utils import SAFE_WEIGHTS_INDEX_NAME - from safetensors.torch import save_file - if not save_safe: - print("Warning: --safetensors is required!!!") - os.makedirs(save_path, exist_ok=True) - state_dict_split = split_torch_state_dict_into_shards(self.state_dict) - self.print_memory_usage(f"before save {save_path}") - has_safetensor_file = False - for shard_file, tensors in state_dict_split.filename_to_tensors.items(): - shard = {} - for tensor in tensors: - shard[tensor] = self.state_dict[tensor].contiguous() - del self.state_dict[tensor] - shard_path = os.path.join(save_path, shard_file) - save_file(shard, shard_path, metadata={"format": "pt"}) - has_safetensor_file = True - print(f"Saving HuggingFace shard to: {shard_path}") - self.print_memory_usage(f"after save {save_path}") - - args = parse_args() - if state_dict_split.is_sharded: - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = SAFE_WEIGHTS_INDEX_NAME - save_index_file = os.path.join(save_path, save_index_file) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - elif has_safetensor_file and (args.save_sub_checkpoint_by_pp or args.sub_num_layers_for_save is not None): - for key in state_dict_split.tensor_to_filename.keys(): - if state_dict_split.tensor_to_filename[key] == "model.safetensors": - state_dict_split.tensor_to_filename[key] = "model-00001-of-00001.safetensors" - index = { - "metadata": state_dict_split.metadata, - "weight_map": state_dict_split.tensor_to_filename, - } - save_index_file = SAFE_WEIGHTS_INDEX_NAME - save_index_file = os.path.join(save_path, save_index_file) - with open(save_index_file, "w", encoding="utf-8") as f: - content = json.dumps(index, indent=2, sort_keys=True) + "\n" - f.write(content) - os.rename(os.path.join(save_path, 'model.safetensors'), \ - os.path.join(save_path, 'model-00001-of-00001.safetensors')) - - if h_config is not None: - h_config.save(save_path) \ No newline at end of file diff --git a/tools/convert_checkpoint/huggingface_config.py b/tools/convert_checkpoint/huggingface_config.py deleted file mode 100644 index ed840a19..00000000 --- a/tools/convert_checkpoint/huggingface_config.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import json - -import os -from convert_checkpoint.abstact_checkpoint import AbstractCheckpoint -from convert_checkpoint.abstact_config import AbstractConfig -from convert_checkpoint.common_config import CommonConfig -from pprint import pprint - - -class HuggingFaceConfig(AbstractConfig): - """ - HuggingFaceConfig - """ - - def __init__(self): - super().__init__() - - @staticmethod - def convert_from_common(c_config): - """ - return HuggingFace config converted from Common config. - - Args: - cc_config: CommonConfig - """ - config = HuggingFaceConfig() - config.update(c_config.get_args("common")) - config.update(c_config.get_args("huggingface")) - return config - - def save(self, save_path): - """ - save config - """ - os.makedirs(save_path, exist_ok=True) - config_path = os.path.join(save_path, "config.json") - with open(config_path, 'w', encoding='utf-8') as f: - json.dump(self.data, f, ensure_ascii=False, indent=4) - print(f"Saving HuggingFace config to {config_path}") - pprint(self.data) - \ No newline at end of file diff --git a/tools/convert_checkpoint/mcore_checkpoint.py b/tools/convert_checkpoint/mcore_checkpoint.py deleted file mode 100644 index 53d7ac9b..00000000 --- a/tools/convert_checkpoint/mcore_checkpoint.py +++ /dev/null @@ -1,2431 +0,0 @@ -""" Mcore_checkpoint converter for aiak megatron. """ -import io -import os -from typing import Tuple, Literal -import shutil -import torch -import types -from tqdm import tqdm - -import resource - -import concurrent.futures -from convert_checkpoint.arguments import parse_args -from convert_checkpoint.abstact_checkpoint import AbstractCheckpoint -from convert_checkpoint.common_checkpoint import CommonCheckpoint -from convert_checkpoint.megatron_optimizer import MegatronOptimizer, merge_optimizer_by_pp_tp, merge_optimizer_by_dp -from convert_checkpoint.utils import ( - check_path_in_dict, - get_element_from_dict_by_path, - add_embedding_padding, cut_embedding_padding, - transpose_shape0, - partition_balanced, - custom_partition_imbalanced, - uneven_vpp_partition, - touch_file, - check_all_done, - get_done_keys, - ceil_div, -) - -from convert_checkpoint import utils - -from convert_checkpoint.common_checkpoint import ( - WORD_EMBEDDINGS, - WORD_POSITION_EMBEDDINGS, - WORD_BLOCK_POSITION_EMBEDDINGS, - TRANSFORMER, - LAYER_PREFIX, - MTP_LAYER_PREFIX, - INPUT_LAYERNORM, - ATTENTION_ROTARY_EMB_INV_FREQ, - ATTENTION_QUERY_KEY_VALUE, - ATTENTION_QKV_MAP, - ATTENTION_DENSE, - POST_ATTENTION_LAYERNORM, - MOE_GATE, - MOE_GATE_BIAS, - MOE_MLP, - MOE_EXPERT, - MOE_GROUPED_GEMM_EXPERT, - MOE_SHARED_EXPERT, - MLP_DENSE_H_TO_4H, - MLP_DENSE_4H_TO_H, - POST_MLP_LAYERNORM, - FINAL_LAYERNORM, - WORD_EMBEDDINGS_FOR_HEAD, - WORD_EMBEDDINGS_TPL, - WORD_POSITION_EMBEDDINGS_TPL, - WORD_BLOCK_POSITION_EMBEDDINGS_TPL, - TRANSFORMER_TPL, - WORD_EMBEDDINGS_FOR_HEAD_TPL, - MTP_WORD_EMBEDDING, - MTP_ENORM, - MTP_HNORM, - MTP_EH_PROJ, - MTP_SHARED_HEAD_NORM, - MTP_SHARED_HEAD_HEAD, -) - - -def get_sharded_states(args, tp_size, pp_size, pp_rank): - """ - Get sharded checkpoints from NVIDIA Mcore-LM checkpoint based on the provided tensor parallel size, pipeline - parallel size and pipeline parallel rank. - - Args: - args (argparse.Namespace): the arguments to the script - tp_size (int): the tensor parallel size - pp_size (int): the pipeline parallel size - pp_rank (int): the pipeline parallel rank - """ - tp_state_dicts = [] - for i in range(tp_size): - sub_dir_name = f"mp_rank_{i:02d}" if pp_size == 1 else f"mp_rank_{i:02d}_{pp_rank:03d}" - checkpoint_name = os.listdir(os.path.join(args.load_path, sub_dir_name))[0] - checkpoint_path = os.path.join(args.load_path, sub_dir_name, checkpoint_name) - state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - tp_state_dicts.append(state_dict) - return tp_state_dicts - - -class McoreCheckpoint(AbstractCheckpoint): - """ - McoreCheckpoint - """ - - def __init__(self, num_layers, load_path=None): - super().__init__(num_layers) - self.version = 3.0 - self.iteration = 0 - self.args = parse_args() - self.rng_state = None - self.load_path = load_path - - def init_pipeline_size(self, pp, tp, dp, ep, tensor_parallel_dim, stage, num_layers_for_test=None, - custom_pipeline_layers=None, etp=None): - "Initialize tp pp dp" - self.pp = pp - self.tp = tp - self.dp = dp - self.ep = ep - self.etp = etp - self.tensor_parallel_dim = tensor_parallel_dim - self.num_stages = stage or 1 - args = parse_args() - if args.decoder_first_pipeline_num_layers is not None and args.decoder_last_pipeline_num_layers is not None: - assert (self.num_layers - args.decoder_first_pipeline_num_layers - \ - args.decoder_last_pipeline_num_layers) // (self.pp - 2) % self.num_stages == 0 - elif num_layers_for_test is None and custom_pipeline_layers is None: - assert self.num_layers // self.pp % self.num_stages == 0 - - def get_state_dict(self, p, t, e=None): - if self.load_path is None: - if utils.LOADED_STATE_DICT is None: - return None - if t == 0 and e is None: - if p in utils.LOADED_STATE_DICT: - return utils.LOADED_STATE_DICT[p] - else: - return None - elif t == 0 and e is not None: - if p in utils.LOADED_STATE_DICT and e in utils.LOADED_STATE_DICT[p]: - return utils.LOADED_STATE_DICT[p][e] - else: - return None - raise Exception("get_state_dict failed. No load_path specified.") - checkpoint_name = "model_optim_rng.pt" - if e is None or self.ep == 1: - sub_dir_name = f"mp_rank_{t:02d}" if self.pp == 1 \ - else f"mp_rank_{t:02d}_{p:03d}" - checkpoint_path = os.path.join(self.load_path, sub_dir_name, checkpoint_name) - print(f"load checkpoint: {checkpoint_path}") - return torch.load(checkpoint_path, map_location="cpu", weights_only=False) - else: - sub_dir_name = f"mp_rank_{t:02d}_{e:03d}" if self.pp == 1 \ - else f"mp_rank_{t:02d}_{p:03d}_{e:03d}" - checkpoint_path = os.path.join(self.load_path, sub_dir_name, checkpoint_name) - print(f"load checkpoint: {checkpoint_path}") - return torch.load(checkpoint_path, map_location="cpu", weights_only=False) - - def init_optimizer(self, use_distributed_optimizer): - assert self.pp > 0 and self.tp > 0 - self.use_distributed_optimizer = use_distributed_optimizer - self.optim_state_dict = [] - for p in range(self.pp): - self.optim_state_dict.append([]) - if self.ep is None: - for t in range(self.tp): - opt = MegatronOptimizer.generate_optimizer(self, self.num_layers // self.pp, p) - self.optim_state_dict[p].append(opt) - else: - for e in range(self.ep): - self.optim_state_dict[p].append([]) - for t in range(self.tp): - opt = MegatronOptimizer.generate_optimizer(self, self.num_layers // self.pp, p) - self.optim_state_dict[p][e].append(opt) - - def init_layers(self): - self.layers = [] - for p in range(self.pp): - self.layers.append(self.get_layers(p)) - - def init_named_parameters_shape(self): - self.named_parameters_shape = self._get_named_parameters_shape() - self.named_parameters_shape_by_p = [None] * self.pp - self.named_parameters_shape_by_t = [None] * self.tp - self.named_parameters_shape_by_pt = [] - for p in range(self.pp): - self.named_parameters_shape_by_p[p] = self._get_named_parameters_shape(pp_rank=p) - for t in range(self.tp): - self.named_parameters_shape_by_t[t] = self._get_named_parameters_shape(tp_rank=t) - for p in range(self.pp): - self.named_parameters_shape_by_pt.append([None] * self.tp) - for t in range(self.tp): - self.named_parameters_shape_by_pt[p][t] = self._get_named_parameters_shape(p, t) - - @staticmethod - def get_virtual_partition(dualpipev, stage_index, p, pp, num_layers_in_rank, m_ckpt): - if dualpipev: - if stage_index == 0: - virtual_p = p - else: - virtual_p = pp * stage_index + (pp - 1 - p) - else: - virtual_p = p + pp * stage_index - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = m_ckpt.get_transformer_name(stage_index) - return virtual_p, layer_offset, transformer - - @staticmethod - def convert_from_common(c_ckpt, c_config, save_path=None, m_config=None, save_optim=True): - """ - Convert common checkpoint to mcore checkpoint. - - Args: - c_ckpt: CommonCheckpoint - c_config: CommonConfig - """ - - print("\n==================== Common -> Mcore ====================") - - name_map = c_config.get("name_map")["mcore"] - cargs = c_config.get_args("common") - hargs = c_config.get_args("huggingface") - margs = c_config.get_args("mcore") - dtype = c_config.get_dtype() - tensor_parallel_dim = c_config.get("tensor_parallel_dim") - separate_dtype = c_config.get("separate_dtype") - - args = parse_args() - cache_path = args.cache_path - tp = args.tensor_model_parallel_size - pp = args.pipeline_model_parallel_size - dp = args.data_parallel_size - ep = args.expert_parallel_size - etp = args.expert_tensor_parallel_size if hasattr(args, 'expert_tensor_parallel_size') else None - dualpipev = args.vpp_scheduler == 'dualpipev' - custom_pipeline_layers = args.custom_pipeline_layers - num_experts = args.num_experts - if num_experts is not None: - assert num_experts > 0, "num_experts must be greater than zero" - if ep is None: - ep = 1 # if ep is not set, will process the model as no MoE - num_experts_for_test = args.num_experts_for_test - num_nextn_predict_layers = hargs.get("num_nextn_predict_layers", 0) - ori_num_layers = cargs["num_layers"] - num_layers = ori_num_layers + num_nextn_predict_layers - hidden_size = cargs["hidden_size"] - use_distributed_optimizer = margs["use_distributed_optimizer"] - num_layers_per_stage = margs["num_layers_per_virtual_pipeline_stage"] - if num_layers_per_stage: - stage = num_layers // pp // num_layers_per_stage - else: - stage = args.num_virtual_stages_per_pipeline_rank or 1 - num_layers_in_first_pipeline_stage = args.decoder_first_pipeline_num_layers - num_layers_in_last_pipeline_stage = args.decoder_last_pipeline_num_layers - if num_layers_in_first_pipeline_stage is not None or num_layers_in_last_pipeline_stage is not None: - assert args.num_virtual_stages_per_pipeline_rank is not None, "num_virtual_stages_per_pipeline_rank is required" - ignore_tp_keys = margs.get("ignore_tp_keys", []) - pretrain_as_fp8 = args.pretrain_as_fp8 - - # get specific layer for test - layer_for_test = args.layer_for_test - num_layers_for_test = None - if layer_for_test is not None: - splits = [] - if layer_for_test.find(',') != -1: - splits = [int(s) for s in layer_for_test.split(',')] - else: - splits = [int(layer_for_test)] - num_layers_for_test = len(splits) - layer_for_test_map = {} - for index, layer_id in enumerate(splits): - layer_for_test_map[layer_id] = index - - m_ckpt = McoreCheckpoint(num_layers) - m_ckpt.set_dtype(dtype) - m_ckpt.init_pipeline_size(pp, tp, dp, ep, tensor_parallel_dim, stage, num_layers_for_test=num_layers_for_test, - custom_pipeline_layers=custom_pipeline_layers, etp=etp) - m_ckpt.set_name_map(name_map) - - m_ckpt.iteration = c_ckpt.other_args.get("iteration", m_ckpt.iteration) - m_ckpt.version = c_ckpt.other_args.get("version", m_ckpt.version) - m_ckpt.args = c_ckpt.other_args.get("args", m_ckpt.args) - m_ckpt.rng_state = c_ckpt.other_args.get("rng_state", m_ckpt.rng_state) - - if c_ckpt.has_optimizer() and ep is None: - # 1. optimizer - m_ckpt.init_layers() - m_ckpt.init_named_parameters_shape() - m_ckpt.init_optimizer(use_distributed_optimizer) - opt = MegatronOptimizer.generate_optimizer(m_ckpt, c_ckpt.num_layers) - opt.load(c_ckpt.state_dict["optimizer"]) - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - opt.add_embedding_padding(divisible_by, vocab_size, tp, hidden_size, padded_vocab_size) - if m_ckpt.pp == 1 and not margs.get("untie_embeddings_and_output_weights", False): - opt.remove_word_embedding_for_head() - opt.build_param_map(m_ckpt.get_named_parameters_shape()) - if stage > 1: - opt.interleave(stage, pp) - opts = opt.chunk_by_pp_tp(pp, tp, margs) - for p in range(pp): - for t in range(tp): - named_parameters_shape = m_ckpt.get_named_parameters_shape(p, t) - opts[p][t].build_param_map(named_parameters_shape) - m_ckpt.optim_state_dict = opts - else: - m_ckpt.optim_state_dict = None - print("> optimizer empty") - - layer_prefix = name_map[LAYER_PREFIX] - if num_nextn_predict_layers > 0: - min_mtp_layer_id = ori_num_layers - mtp_layer_prefix = name_map[MTP_LAYER_PREFIX] - else: - min_mtp_layer_id = None - if custom_pipeline_layers is not None: - assert num_layers_in_first_pipeline_stage is None and num_layers_in_last_pipeline_stage is None, \ - "custom_pipeline_layers need not num_layers_in_first_pipeline_stage or in_last_pipeline_stage" - num_layers_in_rank, _ = custom_partition_imbalanced(ori_num_layers, pp * stage, custom_pipeline_layers) - num_layers_in_rank[-1] += num_nextn_predict_layers - elif num_layers_in_first_pipeline_stage is not None or num_layers_in_last_pipeline_stage is not None: - num_layers_in_rank = uneven_vpp_partition(num_layers, pp, stage, num_layers_in_first_pipeline_stage, num_layers_in_last_pipeline_stage) - num_layers_in_rank[-1] += num_nextn_predict_layers - else: - num_layers_in_rank, _ = partition_balanced(num_layers, pp * stage) - first_k_dense_replace = margs.get("first_k_dense_replace", None) - - done_dir = os.path.join(save_path, "dones") - need_check_dones = False - if args.resume_convert and os.path.exists(done_dir): - need_check_dones = True - done_keys = get_done_keys(done_dir, pp, ep) - else: - if os.path.exists(save_path): - shutil.rmtree(save_path) - release_dir, save_margs = m_ckpt.pre_save(save_path, m_config) - if layer_for_test is not None: - save_margs.num_layers = len(layer_for_test_map) - num_nextn_predict_layers - os.makedirs(done_dir, exist_ok=True) - - if ep is not None: - post_attention_layernorm = name_map[POST_ATTENTION_LAYERNORM] - if MOE_MLP in name_map: - moe_mlp = name_map[MOE_MLP] - moe_expert = name_map[MOE_EXPERT] - if MOE_GROUPED_GEMM_EXPERT in name_map: - moe_grouped_gemm_expert = name_map[MOE_GROUPED_GEMM_EXPERT] - if MOE_SHARED_EXPERT in name_map: - moe_shared_expert = name_map[MOE_SHARED_EXPERT] - if num_experts_for_test is None: - experts_ids = [x for x in range(num_experts)] - chunks = [experts_ids[x:x + num_experts // ep] - for x in range(0, len(experts_ids), num_experts // ep)] # ep_id -> [expert_ids] - else: - experts_ids = [x for x in range(num_experts_for_test)] - chunks = [experts_ids[x:x + num_experts_for_test // ep] - for x in range(0, len(experts_ids), num_experts_for_test // ep)] # ep_id -> [expert_ids] - - expert_local_mapping = {} - expert_ep_mapping = {} - ep_expert_mapping = {} - for ep_id, chunk in enumerate(chunks): - ep_expert_mapping[ep_id] = chunk # ep_id -> [expert_ids] - for idx, ele in enumerate(chunk): - expert_local_mapping[ele] = idx # expert_id -> local_ep_id - expert_ep_mapping[ele] = ep_id # expert_id -> ep_id - print(f"expert_local_mapping: {expert_local_mapping}") - print(f"expert_ep_mapping: {expert_ep_mapping}") - print(f"ep_expert_mapping: {ep_expert_mapping}") - - etp_to_tp_mapping = None - if etp is not None: - assert tp % (etp * ep) == 0 or (etp * ep) % tp == 0, f"tp: {tp}, etp: {etp}, ep: {ep}, tp % (etp * ep) != 0" - etp_to_tp_mapping = {} - v_tp = tp - if tp < etp * ep: - v_tp = etp * ep - for t in range(v_tp): - etp_id = t % etp - ep_id = (t // etp) % ep - if ep_id not in etp_to_tp_mapping: - etp_to_tp_mapping[ep_id] = {} - etp_to_tp_mapping[ep_id][etp_id] = t % tp - print(f"{etp_to_tp_mapping=}") - - def _convert(layer_name, state_dict_node, p, ep_id, tp_transpose_shape=None, sub_key=None, - one_layer_weights_map=None, etp_transpose_shape=None): - if layer_name not in name_map: - print(f"> p: {p}. {layer_name}: Not found.") - return - no_extra = False - if sub_key is not None: - no_extra = False if "extra" in name_map[layer_name][sub_key] and \ - name_map[layer_name][sub_key]["extra"] else True - - _get_weight_func = { - INPUT_LAYERNORM: c_ckpt.get_layer_input_layernorm_weight, - ATTENTION_QUERY_KEY_VALUE: c_ckpt.get_layer_attention_query_key_value_weight, - ATTENTION_QKV_MAP: c_ckpt.get_layer_attention_weight_by_name, - ATTENTION_DENSE: c_ckpt.get_layer_attention_dense_weight, - POST_ATTENTION_LAYERNORM: c_ckpt.get_layer_post_attention_layernorm_weight, - MOE_GATE: c_ckpt.get_layer_moe_gate_weight, - MLP_DENSE_H_TO_4H: c_ckpt.get_layer_mlp_dense_h_to_4h_weight, - MLP_DENSE_4H_TO_H: c_ckpt.get_layer_mlp_dense_4h_to_h_weight, - POST_MLP_LAYERNORM: c_ckpt.get_layer_post_mlp_layernorm_weight, - }[layer_name] - _weight_bias_dict = { - MOE_GATE: MOE_GATE_BIAS - } - _get_bias_func = { - INPUT_LAYERNORM: c_ckpt.get_layer_input_layernorm_bias, - ATTENTION_QUERY_KEY_VALUE: c_ckpt.get_layer_attention_query_key_value_bias, - ATTENTION_QKV_MAP: c_ckpt.get_layer_attention_bias_by_name, - ATTENTION_DENSE: c_ckpt.get_layer_attention_dense_bias, - POST_ATTENTION_LAYERNORM: c_ckpt.get_layer_post_attention_layernorm_bias, - MOE_GATE: c_ckpt.get_layer_moe_gate_bias, - MLP_DENSE_H_TO_4H: c_ckpt.get_layer_mlp_dense_h_to_4h_bias, - MLP_DENSE_4H_TO_H: c_ckpt.get_layer_mlp_dense_4h_to_h_bias, - POST_MLP_LAYERNORM: c_ckpt.get_layer_post_mlp_layernorm_bias, - }[layer_name] - _get_layer_norm_weight_func = { - INPUT_LAYERNORM: lambda x, one_layer_weights=None: None, - ATTENTION_QKV_MAP: lambda x, one_layer_weights=None: None, - ATTENTION_DENSE: lambda x, one_layer_weights=None: None, - POST_ATTENTION_LAYERNORM: lambda x, one_layer_weights=None: None, - MOE_GATE: lambda x, one_layer_weights=None: None, - MLP_DENSE_4H_TO_H: lambda x, one_layer_weights=None: None, - ATTENTION_QUERY_KEY_VALUE: lambda x, one_layer_weights=None: None, - MLP_DENSE_H_TO_4H: lambda x, one_layer_weights=None: None, - POST_MLP_LAYERNORM: lambda x, one_layer_weights=None: None, - }[layer_name] - _get_layer_norm_bias_func = { - INPUT_LAYERNORM: lambda x, one_layer_weights=None: None, - ATTENTION_QKV_MAP: lambda x, one_layer_weights=None: None, - ATTENTION_DENSE: lambda x, one_layer_weights=None: None, - POST_ATTENTION_LAYERNORM: lambda x, one_layer_weights=None: None, - MOE_GATE: lambda x, one_layer_weights=None: None, - MLP_DENSE_4H_TO_H: lambda x, one_layer_weights=None: None, - ATTENTION_QUERY_KEY_VALUE: lambda x, one_layer_weights=None: None, - MLP_DENSE_H_TO_4H: lambda x, one_layer_weights=None: None, - POST_MLP_LAYERNORM: lambda x, one_layer_weights=None: None, - }[layer_name] - a = io.BytesIO() - torch.save(None, a) - _get_layer_extra_state_func = { - INPUT_LAYERNORM: lambda x: None, - ATTENTION_QKV_MAP: lambda x: None if no_extra else a, - ATTENTION_DENSE: lambda x: a, - POST_ATTENTION_LAYERNORM: lambda x: None, - MLP_DENSE_4H_TO_H: lambda x: a, - ATTENTION_QUERY_KEY_VALUE: lambda x: a, - MLP_DENSE_H_TO_4H: lambda x: a, - POST_MLP_LAYERNORM: lambda x: None, - MOE_GATE: lambda x: None, - }[layer_name] - # input_layernorm - if not args.no_te: - if layer_name == INPUT_LAYERNORM: - _get_weight_func = lambda x, one_layer_weights=None: None - _get_bias_func = lambda x, one_layer_weights=None: None - if layer_name == ATTENTION_QUERY_KEY_VALUE: - _get_layer_norm_weight_func = c_ckpt.get_layer_input_layernorm_weight - _get_layer_norm_bias_func = c_ckpt.get_layer_input_layernorm_bias - # post_attention_layernorm - if (not args.no_te) and ep is None: - if layer_name == POST_ATTENTION_LAYERNORM: - _get_weight_func = lambda x, one_layer_weights=None: None - _get_bias_func = lambda x, one_layer_weights=None: None - if layer_name == MLP_DENSE_H_TO_4H: - _get_layer_norm_weight_func = c_ckpt.get_layer_post_attention_layernorm_weight - _get_layer_norm_bias_func = c_ckpt.get_layer_post_attention_layernorm_bias - - weight_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.weight") - bias_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.bias") - if sub_key is not None and \ - ("dim" not in name_map[layer_name][sub_key] or not name_map[layer_name][sub_key]["dim"]): - weight_chunk_dim = None - bias_chunk_dim = None - - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, m_ckpt) - if layer_for_test is not None: - cur_new_layer_index = 0 - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - new_layer_index = layer_index - if layer_for_test is not None: - if layer_id in layer_for_test_map: - new_layer_index = cur_new_layer_index - cur_new_layer_index += 1 - else: - continue - if min_mtp_layer_id is not None and layer_id >= min_mtp_layer_id: - cur_layer_prefix = mtp_layer_prefix - new_layer_index = layer_id - min_mtp_layer_id - else: - cur_layer_prefix = layer_prefix - one_layer_weights = one_layer_weights_map[layer_id] if one_layer_weights_map is not None else None - - def update_tensor_per_node(state_dict_node, sub_name, transpose_shape = None, weight_chunk_dim=None, \ - is_moe_mlp = False, expert_id = None, is_shared = False): - weight_name = None - bias_name = None - weight_scale_inv_name = None - weight_scale_inv = None - etp_to_tp = None - # weight - if is_moe_mlp: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_mlp}.{sub_name}" - weight = _get_weight_func(layer_id, is_moe_mlp=True, one_layer_weights=one_layer_weights) - elif is_shared: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_shared_expert}.{sub_name}" - weight = _get_weight_func(layer_id, is_shared=True, one_layer_weights=one_layer_weights) - elif expert_id is not None: - local_eid = expert_local_mapping[expert_id] - if args.moe_grouped_gemm: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_grouped_gemm_expert}.{sub_name}" - weight_name = f"{name}.weight{local_eid}" - bias_name = f"{name}.bias{local_eid}" - else: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_expert}.{local_eid}.{sub_name}" - weight = _get_weight_func(layer_id, expert_id=expert_id, one_layer_weights=one_layer_weights) - etp_to_tp = etp_to_tp_mapping[ep_id] if etp is not None else None - else: - name = f"{cur_layer_prefix}.{new_layer_index}.{sub_name}" - if sub_key is not None: - if layer_name == ATTENTION_QKV_MAP and \ - ("is_layernorm" not in name_map[layer_name][sub_key] or \ - not name_map[layer_name][sub_key]["is_layernorm"]): - weight = _get_weight_func(sub_key, layer_id, one_layer_weights=one_layer_weights) - else: - weight = _get_weight_func(sub_key, layer_id, one_layer_weights=one_layer_weights) - if "is_layernorm" in name_map[layer_name][sub_key] and name_map[layer_name][sub_key]["is_layernorm"]: - weight_name = name + ".layer_norm_weight" - else: - if layer_name == ATTENTION_DENSE: - weight = _get_weight_func(layer_id, one_layer_weights=one_layer_weights) - else: - weight = _get_weight_func(layer_id, one_layer_weights=one_layer_weights) - if layer_name in _weight_bias_dict and _weight_bias_dict[layer_name] in name_map: - bias_name = f"{cur_layer_prefix}.{new_layer_index}.{name_map[_weight_bias_dict[layer_name]]}" - if layer_name == MOE_GATE and num_experts_for_test is not None: - weight = weight[:num_experts_for_test] - if weight is not None and layer_name in [ATTENTION_QKV_MAP, ATTENTION_DENSE, \ - MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H]: - weight, weight_scale_inv = weight - if weight_name is not None: - if weight_scale_inv is not None or pretrain_as_fp8: - for key in ignore_tp_keys: - if key in weight_name: - weight_chunk_dim = None - m_ckpt.update_tensor(state_dict_node, weight, transformer, weight_name, - dim=weight_chunk_dim, etp_to_tp=etp_to_tp, - weight_scale_inv=weight_scale_inv, transpose_shape=transpose_shape) - else: - if weight_scale_inv is not None or pretrain_as_fp8: - for key in ignore_tp_keys: - if key in name + ".weight": - weight_chunk_dim = None - m_ckpt.update_tensor(state_dict_node, weight, transformer, name + ".weight", - dim=weight_chunk_dim, etp_to_tp=etp_to_tp, - weight_scale_inv=weight_scale_inv, transpose_shape=transpose_shape) - # bias - if sub_key is not None: - bias = _get_bias_func(sub_key, layer_id, one_layer_weights=one_layer_weights) - else: - bias = _get_bias_func(layer_id, one_layer_weights=one_layer_weights) - if layer_name == MOE_GATE and num_experts_for_test is not None: - bias = bias[:num_experts_for_test] - if transpose_shape is not None and bias is not None: - bias = transpose_shape0(bias, *transpose_shape) - if separate_dtype is not None and layer_name in separate_dtype: - bias_type = bias.dtype - else: - bias_type = None - if bias_name is not None: - m_ckpt.update_tensor(state_dict_node, bias, transformer, bias_name, - bias_chunk_dim, dtype=bias_type, etp_to_tp=etp_to_tp) - else: - m_ckpt.update_tensor(state_dict_node, bias, transformer, name + ".bias", - bias_chunk_dim, dtype=bias_type, etp_to_tp=etp_to_tp) - # layer norm - weight = _get_layer_norm_weight_func(layer_id, one_layer_weights=one_layer_weights) - bias = _get_layer_norm_bias_func(layer_id, one_layer_weights=one_layer_weights) - if transpose_shape is not None: - pass # todo - m_ckpt.update_tensor(state_dict_node, weight, transformer, name + ".layer_norm_weight", - etp_to_tp=etp_to_tp) - m_ckpt.update_tensor(state_dict_node, bias, transformer, name + ".layer_norm_bias", - etp_to_tp=etp_to_tp) - # _extra_state - if expert_id is None or layer_name not in [MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H] or \ - (args.moe_grouped_gemm and local_eid == 0): - state = _get_layer_extra_state_func(layer_id) - m_ckpt.update_tensor(state_dict_node, state, transformer, name + "._extra_state", None) - - if ep_id is not None and layer_name in [MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H]: - if first_k_dense_replace is not None and layer_id < first_k_dense_replace: - update_tensor_per_node(state_dict_node, name_map[layer_name], is_moe_mlp=True, - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - else: - for expert_id in ep_expert_mapping[ep_id]: - update_tensor_per_node(state_dict_node, name_map[layer_name], expert_id=expert_id, - transpose_shape=etp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - if MOE_SHARED_EXPERT in name_map: - update_tensor_per_node(state_dict_node, name_map[layer_name], is_shared=True, - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - elif layer_name == MOE_GATE: - if first_k_dense_replace is None or layer_id >= first_k_dense_replace: - update_tensor_per_node(state_dict_node, name_map[layer_name], - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - else: - if sub_key is not None: - update_tensor_per_node(state_dict_node, name_map[layer_name][sub_key]["name"], - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - else: - update_tensor_per_node(state_dict_node, name_map[layer_name], - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - - if ep_id is None: - print(f"> p: {p}. {layer_name} chunk weight dim {weight_chunk_dim}, bias dim {bias_chunk_dim}") - else: - print(f"> p: {p}, ep_id: {ep_id}. {layer_name} chunk weight dim {weight_chunk_dim}, bias dim {bias_chunk_dim}") - - def convert_one_p(p, ep_id=None, last_p=None, one_layer_weights_map=None): - if need_check_dones and (p, ep_id) in done_keys: - print(f"> p: {p}, ep_id: {ep_id} already converted. pass...") - return - - if cache_path is None: - one_layer_weights_map = None - elif last_p is None or last_p != p: - if one_layer_weights_map is None: - one_layer_weights_map = {} - one_layer_weights_map.clear() - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, m_ckpt) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if layer_for_test is not None: - if layer_id in layer_for_test_map: - one_layer_weights_map[layer_id] = torch.load(f"{cache_path}/checkpoint_{layer_id}.pt", - map_location="cpu", weights_only=False) - else: - continue - else: - one_layer_weights_map[layer_id] = torch.load(f"{cache_path}/checkpoint_{layer_id}.pt", - map_location="cpu", weights_only=False) - print(f"> load cached weights of layer {layer_id}: {cache_path}/checkpoint_{layer_id}.pt") - - state_dict_node = {} - if etp is None: - for t in range(tp): - state_dict_node[t] = {} - else: - for t in etp_to_tp_mapping[ep_id].values(): - state_dict_node[t] = {} - - if p == 0: - # 1.1 word embeddings with paddding - weight = c_ckpt.get_word_embedding() - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - if padded_vocab_size is None: - assert vocab_size == weight.shape[0] - weight = add_embedding_padding(weight, divisible_by, vocab_size, tp, padded_vocab_size) - - if WORD_EMBEDDINGS in name_map: - name = m_ckpt.get_word_embedding_name() - chunk_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS}.weight") - transformer = m_ckpt.get_transformer_name(0) - m_ckpt.update_tensor(state_dict_node, weight, transformer, name + ".weight", chunk_dim) - print(f"> p: {p}. {name} weight {weight.shape}") - - # 1.2 position embedding - if margs.get("add_position_embedding", False): - name = m_ckpt.get_position_embedding_name() - weight = c_ckpt.get_word_position_embedding() - chunk_dim = tensor_parallel_dim.get(f"{WORD_POSITION_EMBEDDINGS}.weight") - m_ckpt.update_tensor(state_dict_node, weight, name, "weight", chunk_dim) - print(f"> p: {p}. {name} weight {weight.shape}") - - if margs.get("add_block_position_embedding", False): - name = m_ckpt.get_block_position_embedding_name() - weight = c_ckpt.get_word_block_position_embedding() - chunk_dim = tensor_parallel_dim.get(f"{WORD_BLOCK_POSITION_EMBEDDINGS}.weight") - m_ckpt.update_tensor(state_dict_node, weight, name, "weight", chunk_dim) - print(f"> {name} weight {weight.shape}") - - # 2.1 input_layernorm - _convert(INPUT_LAYERNORM, state_dict_node, p=p, ep_id=ep_id, one_layer_weights_map=one_layer_weights_map) - - # 2.2 rotary_emb.inv_freqs - if ATTENTION_ROTARY_EMB_INV_FREQ in name_map: - chunk_dim = tensor_parallel_dim.get(ATTENTION_ROTARY_EMB_INV_FREQ) - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, m_ckpt) - if layer_for_test is not None: - cur_new_layer_index = 0 - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - new_layer_index = layer_index - if layer_for_test is not None: - if layer_id in layer_for_test_map: - new_layer_index = cur_new_layer_index - cur_new_layer_index += 1 - else: - continue - if min_mtp_layer_id is not None and layer_id >= min_mtp_layer_id: - cur_layer_prefix = mtp_layer_prefix - else: - cur_layer_prefix = layer_prefix - name = f"{cur_layer_prefix}.{new_layer_index}.{name_map[ATTENTION_ROTARY_EMB_INV_FREQ]}" - one_layer_weights = one_layer_weights_map[layer_id] \ - if one_layer_weights_map is not None else None - inv_freq = c_ckpt.get_layer_attention_rotary_emb_inv_freq( - layer_id, one_layer_weights=one_layer_weights) - m_ckpt.update_tensor(state_dict_node, inv_freq, transformer, name, chunk_dim) - print(f"> p: {p}. rotary_emb.inv_freqs chunk dim {chunk_dim}") - - # 2.3 self attention query_key_value - if ATTENTION_QUERY_KEY_VALUE in name_map: - _convert(ATTENTION_QUERY_KEY_VALUE, state_dict_node, p=p, ep_id=ep_id, - one_layer_weights_map=one_layer_weights_map) - if ATTENTION_QKV_MAP in name_map: - for sub_key, dict_mcore in name_map[ATTENTION_QKV_MAP].items(): - _convert(ATTENTION_QKV_MAP, state_dict_node, p=p, ep_id=ep_id, sub_key=sub_key, - one_layer_weights_map=one_layer_weights_map) - - # 2.4 self attention dense - _convert(ATTENTION_DENSE, state_dict_node, p=p, ep_id=ep_id, one_layer_weights_map=one_layer_weights_map) - - # 2.5 post attention layernorm - _convert(POST_ATTENTION_LAYERNORM, state_dict_node, p=p, ep_id=ep_id, one_layer_weights_map=one_layer_weights_map) - - # 2.6 mlp dense h_to_4h - if ep is not None: - _convert(MOE_GATE, state_dict_node, p=p, ep_id=ep_id, one_layer_weights_map=one_layer_weights_map) - if margs.get("transpose_mlp_dense", False): - if etp is None: - etp_transpose_shape = (2, tp) - else: - etp_transpose_shape = (2, etp) - _convert(MLP_DENSE_H_TO_4H, state_dict_node, p=p, ep_id=ep_id, tp_transpose_shape=(2, tp), - etp_transpose_shape=etp_transpose_shape, one_layer_weights_map=one_layer_weights_map) - else: - _convert(MLP_DENSE_H_TO_4H, state_dict_node, p=p, ep_id=ep_id, - one_layer_weights_map=one_layer_weights_map) - - # 2.7 mlp dense 4h_to_h - _convert(MLP_DENSE_4H_TO_H, state_dict_node, p=p, ep_id=ep_id, one_layer_weights_map=one_layer_weights_map) - - # 2.8 post mlp layernorm - _convert(POST_MLP_LAYERNORM, state_dict_node, p=p, ep_id=ep_id, one_layer_weights_map=one_layer_weights_map) - add_final_layer = False - if not dualpipev and p == pp - 1: - add_final_layer = True - elif dualpipev and p == 0: - add_final_layer = True - if add_final_layer: - # 2.9 final_layernorm - if FINAL_LAYERNORM in name_map: - transformer = m_ckpt.get_transformer_name(m_ckpt.num_stages - 1) - name = name_map[FINAL_LAYERNORM] - # weight - chunk_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.weight") - weight = c_ckpt.get_final_layernorm_weight() - m_ckpt.update_tensor(state_dict_node, weight, transformer, name + ".weight", chunk_dim) - # bias - chunk_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.bias") - bias = c_ckpt.get_final_layernorm_bias() - m_ckpt.update_tensor(state_dict_node, bias, transformer, name + ".bias", chunk_dim) - print(f"> p: {p}. {name} weight {weight.shape}") - - # 3 word embedding for head - if WORD_EMBEDDINGS_FOR_HEAD in name_map: - if margs.get("untie_embeddings_and_output_weights", False) or m_ckpt.pp > 1: - chunk_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS_FOR_HEAD}.weight") - name = m_ckpt.get_word_embedding_for_head_name() - weight = c_ckpt.get_word_embeddings_for_head_weight() - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - orig_vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - weight = add_embedding_padding(weight, divisible_by, orig_vocab_size, tp, padded_vocab_size) - m_ckpt.update_tensor(state_dict_node, weight, transformer, name + ".weight", chunk_dim) - print(f"> p: {p}. {name} weight {weight.shape}") - - if MTP_WORD_EMBEDDING in name_map: - virtual_p = pp * m_ckpt.num_stages - 1 - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = m_ckpt.get_transformer_name(m_ckpt.num_stages - 1) - new_layer_index = 0 - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if layer_id < ori_num_layers: - continue - if layer_for_test is not None and layer_id not in layer_for_test_map: - continue - - one_layer_weights = one_layer_weights_map[layer_id] \ - if one_layer_weights_map is not None else None - (mtp_word_embedding, mtp_enorm, mtp_hnorm, mtp_eh_proj), \ - (mtp_shared_head_norm, mtp_shared_head_head) = c_ckpt.get_layer_mtp_weight( - layer_id, one_layer_weights=one_layer_weights) - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - if padded_vocab_size is None: - assert vocab_size == mtp_word_embedding.shape[0] - mtp_word_embedding = add_embedding_padding(mtp_word_embedding, divisible_by, - vocab_size, tp, padded_vocab_size) - - name = f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_WORD_EMBEDDING]}.weight" - m_ckpt.update_tensor(state_dict_node, mtp_word_embedding, transformer, \ - name, tensor_parallel_dim.get(f"{MTP_WORD_EMBEDDING}.weight")) - name = f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_ENORM]}.weight" - m_ckpt.update_tensor(state_dict_node, mtp_enorm, transformer, \ - name, tensor_parallel_dim.get(f"{MTP_ENORM}.weight")) - name = f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_HNORM]}.weight" - m_ckpt.update_tensor(state_dict_node, mtp_hnorm, transformer, \ - name, tensor_parallel_dim.get(f"{MTP_HNORM}.weight")) - name = f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_EH_PROJ]}.weight" - m_ckpt.update_tensor(state_dict_node, mtp_eh_proj, transformer, \ - name, tensor_parallel_dim.get(f"{MTP_EH_PROJ}.weight")) - name = f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_SHARED_HEAD_NORM]}.weight" - m_ckpt.update_tensor(state_dict_node, mtp_shared_head_norm, transformer, \ - name, tensor_parallel_dim.get(f"{MTP_SHARED_HEAD_NORM}.weight")) - #name = f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_SHARED_HEAD_HEAD]}.weight" - # m_ckpt.update_tensor(state_dict_node, mtp_shared_head_head, transformer, \ - # name, tensor_parallel_dim.get(f"{MTP_SHARED_HEAD_HEAD}.weight")) - print(f"> p: {p}. layer_index: {new_layer_index}, "\ - f"mtp_word_embedding: {mtp_word_embedding.shape}, "\ - f"mtp_enorm: {mtp_enorm.shape}, mtp_hnorm: {mtp_hnorm.shape}, "\ - f"mtp_eh_proj: {mtp_eh_proj.shape}, mtp_shared_head_norm: {mtp_shared_head_norm.shape}") - new_layer_index += 1 - - saved_models_str = {} - for stage_index in range(stage): - saved_layer_ids = [] - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, m_ckpt) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if layer_for_test is not None: - if layer_id in layer_for_test_map: - saved_layer_ids.append(str(layer_id)) - else: - saved_layer_ids.append(str(layer_id)) - saved_models_str[transformer] = ", ".join(saved_layer_ids) - - for t in state_dict_node.keys(): - if ep_id is None: - m_ckpt.save_model_file( - release_dir, save_margs, p, t, None, state_dict_node[t], - m_ckpt.optim_state_dict[p][t] if m_ckpt.optim_state_dict is not None else None, saved_models_str) - else: - m_ckpt.save_model_file( - release_dir, save_margs, p, t, ep_id, state_dict_node[t], - m_ckpt.optim_state_dict[p][ep_id][t] if m_ckpt.optim_state_dict is not None else None, - saved_models_str) - - # optimizer - if use_distributed_optimizer and save_optim: - for t in range(tp): - if ep_id is None: - m_ckpt.save_optimizer(release_dir, p, t, None, state_dict_node[t]) - else: - m_ckpt.save_optimizer(release_dir, p, t, ep_id, state_dict_node[t][ep_id]) - touch_file(done_dir=done_dir, p_id=p, ep_id=ep_id) - - if cache_path is None: - one_layer_weights_map = None - else: - one_layer_weights_map = {} - if args.max_workers > 1: - futures = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor: - for p in range(pp): - if ep is None: - futures.append(executor.submit(convert_one_p, p=p, ep_id=None, - one_layer_weights_map=one_layer_weights_map)) - else: - for ep_id in range(ep): - futures.append(executor.submit(convert_one_p, p=p, ep_id=ep_id, - one_layer_weights_map=one_layer_weights_map)) - - concurrent.futures.wait(futures) - for future in futures: - try: - result = future.result() - except Exception as e: - print(f"An error occurred: {e}") - raise e - else: - last_p = None - for p in range(pp): - if ep is None: - convert_one_p(p=p, ep_id=None, last_p=last_p, one_layer_weights_map=one_layer_weights_map) - last_p = p - else: - for ep_id in range(ep): - convert_one_p(p=p, ep_id=ep_id, last_p=last_p, one_layer_weights_map=one_layer_weights_map) - last_p = p - checked_done = check_all_done(done_dir, pp, ep) - assert checked_done, f"{done_dir} is not complete. please retry it again" - shutil.rmtree(done_dir) - - m_ckpt.debug("Finish common -> mcore") - return m_ckpt - - def update_tensor(self, state_dict_node, source, layer, key, dim=None, dtype=None, etp_to_tp=None, - weight_scale_inv=None, transpose_shape = None): - """ - Update a tensor, which can be a torch.Tensor or other types of objects. If it is a torch.Tensor, - it will be split into multiple parts and updated; otherwise, it will be updated directly. - If dim is not None, the source tensor is divided into multiple parts, - and each part is updated to the corresponding layer and key in state_dict_node. - - Args: - state_dict_node (List[Dict]): A list containing multiple elements, - each of which is a dictionary used to store model parameters. - Each element is a dictionary that stores model parameters in the format - {layer1: {param1: value1}, layer2: {param2: value2}}. - source (Union[torch.Tensor, Any]): The source tensor or other types of objects that need to be updated. - If it is a torch.Tensor, it will be split into multiple parts and updated; otherwise, - it will be updated directly. - layer (str): The name of the layer to be updated. - key (str): The key name that needs to be updated. - dim (Optional[int], optional): If the source tensor is 3D or higher, - the dimensions of the split need to be specified. The default is None, - which means no split is performed. Defaults to None - - Returns: - None. - """ - dtype = self.dtype if dtype is None else dtype - args = parse_args() - - if args.fp8_quant_transfer_type == "bfloat16": - transfer_dtype = torch.bfloat16 - else: - transfer_dtype = torch.float32 - - def get_quantizer_with_weight_scale_inv(weight, weight_scale_inv): - from transformer_engine.pytorch.tensor.float8_blockwise_tensor \ - import Float8BlockwiseQTensor - from transformer_engine.pytorch.constants import TE_DType - return Float8BlockwiseQTensor( - rowwise_data=weight, - rowwise_scale_inv=weight_scale_inv, - columnwise_data=None, - columnwise_scale_inv=None, - fp8_dtype=TE_DType[torch.float8_e4m3fn], - quantizer=None, - is_2D_scaled=True, - shape=weight.shape, - dtype=torch.bfloat16 - ) - - def chunk_fp8_weight(weight, weight_scale_inv, cur_tp, transpose_shape=None): - weight_bf16 = McoreCheckpoint.per_block_dequant_from_fp8(weight, weight_scale_inv, dtype=transfer_dtype) - if transpose_shape is not None: - weight_bf16 = transpose_shape0(weight_bf16, *transpose_shape) - weight_bf16_s = torch.chunk(weight_bf16, cur_tp, dim=dim) - weight_s = [] - weight_scale_inv_s = [] - for w_bf16 in weight_bf16_s: - weight, weight_scale_inv = McoreCheckpoint.per_block_cast_to_fp8( - w_bf16, method=args.quant_method, amax_epsilon=args.amax_epsilon, - force_pow_2_scales=args.force_pow_2_scales) - weight_s.append(weight) - weight_scale_inv_s.append(weight_scale_inv) - return weight_s, weight_scale_inv_s - - if isinstance(source, torch.Tensor): - shape = source.shape - tp_shapes = [] - if etp_to_tp is None: - cur_tp = self.tp - else: - cur_tp = self.etp - need_chunk = dim is not None and cur_tp > 1 - if need_chunk: - if weight_scale_inv is None: - if transpose_shape is not None: - source = transpose_shape0(source, *transpose_shape) - source = torch.chunk(source, cur_tp, dim=dim) - for s in source: - tp_shapes.append(s.shape) - else: - source, weight_scale_inv = chunk_fp8_weight( - source, weight_scale_inv, cur_tp, transpose_shape=transpose_shape) - for s in source: - tp_shapes.append(s.shape) - for inv in weight_scale_inv: - tp_shapes.append(inv.shape) - elif weight_scale_inv is not None: - weight_bf16 = McoreCheckpoint.per_block_dequant_from_fp8(source, weight_scale_inv, dtype=transfer_dtype) - source, weight_scale_inv = McoreCheckpoint.per_block_cast_to_fp8( - weight_bf16, method=args.quant_method, amax_epsilon=args.amax_epsilon, - force_pow_2_scales=args.force_pow_2_scales) - - if etp_to_tp is None: - for t in state_dict_node.keys(): - element = get_element_from_dict_by_path( - state_dict_node[t], layer - ) - if weight_scale_inv is None: - element[key] = (source if not need_chunk else source[t].clone()) - else: - element[key] = get_quantizer_with_weight_scale_inv( - weight=(source if not need_chunk else source[t].clone()), - weight_scale_inv=(weight_scale_inv if not need_chunk else weight_scale_inv[t].clone()) - ) - else: - for et in range(self.etp): - t = etp_to_tp[et] - element = get_element_from_dict_by_path( - state_dict_node[t], layer - ) - if weight_scale_inv is None: - element[key] = (source if not need_chunk else source[et].clone()) - else: - element[key] = get_quantizer_with_weight_scale_inv( - weight = (source if not need_chunk else source[et].clone()), - weight_scale_inv=(weight_scale_inv if not need_chunk else weight_scale_inv[et].clone()), - ) - - print(f"update_tensor: {key}, {shape=}, {tp_shapes=}") - elif source is not None: - for t in state_dict_node.keys(): - element = get_element_from_dict_by_path( - state_dict_node[t], layer - ) - element[key] = source - print(f"update_tensor: {key}") - - def is_fp8_q_tensor(self, state_dict, layer, key): - from transformer_engine.pytorch.tensor.float8_blockwise_tensor \ - import Float8BlockwiseQTensor - element = get_element_from_dict_by_path( - state_dict[0], layer - ) - if key not in element: - return False - return isinstance(element[key], Float8BlockwiseQTensor) - - def get_tensor(self, state_dict, layer, key, dim=None, etp_to_tp=None, transpose_shape=None): - args = parse_args() - if args.pretrain_as_fp8: - from transformer_engine.pytorch.tensor.float8_blockwise_tensor \ - import Float8BlockwiseQTensor - - def cat_fp8_weight(tp_weight_dict, tp_weight_scale_inv, transpose_shape=None): - weight_bf16_s = [] - if args.fp8_quant_transfer_type == "bfloat16": - transfer_dtype = torch.bfloat16 - else: - transfer_dtype = torch.float32 - for i in range(len(tp_weight_dict)): - weight = tp_weight_dict[i] - weight_scale_inv = tp_weight_scale_inv[i] - weight_bf16 = McoreCheckpoint.per_block_dequant_from_fp8(weight, weight_scale_inv, dtype=transfer_dtype) - weight_bf16_s.append(weight_bf16) - weight_bf16 = torch.cat(weight_bf16_s, dim=dim) - - if transpose_shape is not None: - weight_bf16 = transpose_shape0(weight_bf16, *transpose_shape) - - weight, weight_scale_inv = McoreCheckpoint.per_block_cast_to_fp8( - weight_bf16, method=args.quant_method, amax_epsilon=args.amax_epsilon, - force_pow_2_scales=args.force_pow_2_scales) - - return weight, weight_scale_inv - - def get_weight_dict(cur_tp): - tp_weight_dict = [] - tp_weight_scale_inv = [] - for t in range(cur_tp): - element = get_element_from_dict_by_path( - state_dict[t], layer - ) - if key not in element: - return None, None - if args.pretrain_as_fp8 and isinstance(element[key], Float8BlockwiseQTensor): - tp_weight_dict.append(element[key]._rowwise_data) - tp_weight_scale_inv.append(element[key]._rowwise_scale_inv) - else: - tp_weight_dict.append(element[key]) - return tp_weight_dict, tp_weight_scale_inv - - if dim is not None: - if etp_to_tp is None: - cur_tp = self.tp - else: - cur_tp = self.etp - tp_weight_dict, tp_weight_scale_inv = get_weight_dict(cur_tp) - if tp_weight_dict is None: - return None, None - - if tp_weight_scale_inv is None or len(tp_weight_scale_inv) == 0: - source = torch.cat(tp_weight_dict, dim=dim) - weight_scale_inv = None - if transpose_shape is not None: - source = transpose_shape0(source, *transpose_shape) - else: - weight_len = 0 - scale_inv_len = 0 - for weight in tp_weight_dict: - weight_len += weight.shape[0] - for scale_inv in tp_weight_scale_inv: - scale_inv_len += scale_inv.shape[0] - source, weight_scale_inv = cat_fp8_weight( - tp_weight_dict, tp_weight_scale_inv, transpose_shape=transpose_shape) - else: - element = get_element_from_dict_by_path( - state_dict[0], layer - ) - if key not in element: - return None, None - if args.pretrain_as_fp8 and isinstance(element[key], Float8BlockwiseQTensor): - source = element[key]._rowwise_data - weight_scale_inv = element[key]._rowwise_scale_inv - else: - source = element[key] - weight_scale_inv = None - if source is not None: - print(f"get_tensor: {key}, {source.shape}") - return source, weight_scale_inv - - @staticmethod - def per_block_dequant_from_fp8(fp8_blocks: torch.Tensor, - scales: torch.Tensor, - orig_shape: Tuple[int, int] | None = None, - dtype: torch.dtype = torch.bfloat16) -> torch.Tensor: - m_full, n_full = fp8_blocks.shape - if orig_shape is None: - m, n = m_full, n_full - else: - m, n = orig_shape - # sanity check - if m > m_full or n > n_full: - raise ValueError("orig_shape is larger than fp8 tensor shape") - - m_pad = ceil_div(m, 128) * 128 - n_pad = ceil_div(n, 128) * 128 - - if m_pad == m_full and n_pad == n_full: - fp8_padded = fp8_blocks - else: - fp8_padded = torch.zeros((m_pad, n_pad), dtype=fp8_blocks.dtype, device=fp8_blocks.device) - fp8_padded[:m, :n] = fp8_blocks - - fp8_view = fp8_padded.view(-1, 128, n_pad // 128, 128) - scale_expanded = scales.view(-1, 1, scales.size(1), 1) - x_recon_view = fp8_view.to(scale_expanded.dtype) * scale_expanded - x_recon = x_recon_view.view(m_pad, n_pad)[:m, :n].contiguous() - x_recon = x_recon.to(dtype) - - return x_recon - - @staticmethod - def per_block_cast_to_fp8( - x: torch.Tensor, - method: Literal["te", "pt", "aiak"] = 'te', - fp8_dtype: torch.dtype = torch.float8_e4m3fn, - **kwargs, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Args: - x: 2d tensor, the model parameter what will be block-wise quantize to fp8. - method: one of the ["te", "pt", "aiak"], means using TransformerEngine, naive PyTorch, aiak-fp8-quantizer - to do the quantization respectively. Defaults to "te". - fp8_dtype: the dtype of the output fp8 tensor. Defaults to torch.float8_e4m3fn. - **kwargs: kwargs pass to the `te` method. Take no effect in other quantization methods. Belows are the args: - `amax_epsilon`: defaults to 0. - `force_pow_2_scales` defaults to True. - Returns: - x_scaled: 2d tensor, the quantized tensor. - weight_scale_inv: 2d tensor, the scale_inv of the quantized tensor. - """ - - # Always do the quantization on device - x = x.cuda() - - if method == "pt": - assert x.dim() == 2 - m, n = x.shape - x_padded = torch.zeros((ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device) - x_padded[:m, :n] = x - x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) - x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) - x_scaled = (x_view * (448.0 / x_amax)).to(fp8_dtype) - return x_scaled.view_as(x_padded)[:m, :n].contiguous().cpu(), \ - (x_amax / 448.0).view(x_view.size(0), x_view.size(2)).cpu() - - elif method == "aiak": - import aiak_fp8_quantizer - weight, weight_scale_inv, *_ = aiak_fp8_quantizer.per_block_cast_to_fp8_fprop_vector(x) - return weight.view(fp8_dtype).cpu(), weight_scale_inv.cpu() - - elif method == "te": - from transformer_engine.pytorch.tensor.float8_blockwise_tensor import Float8BlockQuantizer - from transformer_engine.pytorch.constants import TE_DType - quantizer = Float8BlockQuantizer( - fp8_dtype=TE_DType[fp8_dtype], - rowwise=True, - columnwise=False, - amax_epsilon=kwargs.get("amax_epsilon", 0.0), - force_pow_2_scales=kwargs.get("force_pow_2_scales", True), - block_scaling_dim=2, - ) - xq = quantizer(x) - fp8_weight = xq._rowwise_data.view(fp8_dtype).cpu() # `_rowwise_data` is torch.uint8 - fp8_scale_inv = xq._rowwise_scale_inv.cpu() - return fp8_weight, fp8_scale_inv - else: - raise ValueError(f"invalid quantization method: {method}") - - def convert_to_common(self, c_config): - """ - Convert Mcore checkpoint to common checkpoint. - Args: - c_config: CommonConfig - """ - - print("\n==================== Mcore -> Common ====================") - - tensor_parallel_dim = self.tensor_parallel_dim - name_map = c_config.get_name_map("mcore") - hargs = c_config.get_args("huggingface") - cargs = c_config.get_args("common") - margs = c_config.get_args("mcore") - num_nextn_predict_layers = hargs.get("num_nextn_predict_layers", 0) - ori_num_layers = cargs["num_layers"] - num_layers = ori_num_layers + num_nextn_predict_layers - hidden_size = cargs["hidden_size"] - num_attention_heads = cargs["num_attention_heads"] - args = parse_args() - custom_pipeline_layers = args.custom_pipeline_layers - cache_path = args.cache_path - if cache_path is not None: - os.makedirs(cache_path, exist_ok=True) - tp = args.tensor_model_parallel_size - ep = args.expert_parallel_size - pp = args.pipeline_model_parallel_size - etp = args.expert_tensor_parallel_size if hasattr(args, 'expert_tensor_parallel_size') else None - dualpipev = args.vpp_scheduler == 'dualpipev' - num_experts = args.num_experts - convert_to_fp8 = args.convert_to_fp8 - if convert_to_fp8: - assert "fp8_keys" in margs, "fp8_keys should be defined when convert_to_fp8 is True" - fp8_keys = margs["fp8_keys"] - else: - fp8_keys = None - ignore_tp_keys = margs.get("ignore_tp_keys", []) - pretrain_as_fp8 = args.pretrain_as_fp8 - if num_experts is not None: - assert num_experts > 0, "num_experts must be greater than zero" - if ep is None: - ep = 1 # if ep is not set, will process the model as no MoE - - c_ckpt = CommonCheckpoint(num_layers) - c_ckpt.set_dtype(self.dtype) - - layer_prefix = name_map[LAYER_PREFIX] - - num_layers_in_first_pipeline_stage = args.decoder_first_pipeline_num_layers - num_layers_in_last_pipeline_stage = args.decoder_last_pipeline_num_layers - if num_layers_in_first_pipeline_stage is not None or num_layers_in_last_pipeline_stage is not None: - assert args.num_virtual_stages_per_pipeline_rank is not None, "num_virtual_stages_per_pipeline_rank is required" - num_layers_per_stage = margs["num_layers_per_virtual_pipeline_stage"] - if num_layers_per_stage: - stage = num_layers // pp // num_layers_per_stage - else: - stage = args.num_virtual_stages_per_pipeline_rank or 1 - first_k_dense_replace = margs.get("first_k_dense_replace", None) - if custom_pipeline_layers is not None: - assert num_layers_in_first_pipeline_stage is None and num_layers_in_last_pipeline_stage is None, \ - "custom_pipeline_layers need not num_layers_in_first_pipeline_stage or in_last_pipeline_stage" - num_layers_in_rank, _ = custom_partition_imbalanced( - ori_num_layers, self.pp * stage, custom_pipeline_layers) - num_layers_in_rank[-1] += num_nextn_predict_layers - elif num_layers_in_first_pipeline_stage is not None or num_layers_in_last_pipeline_stage is not None: - num_layers_in_rank = uneven_vpp_partition(num_layers, pp, stage, num_layers_in_first_pipeline_stage, num_layers_in_last_pipeline_stage) - num_layers_in_rank[-1] += num_nextn_predict_layers - else: - num_layers_in_rank, _ = partition_balanced(num_layers, self.pp * self.num_stages) - - # get specific layer for test - layer_for_test = args.layer_for_test - if layer_for_test is not None: - splits = [] - if layer_for_test.find(',') != -1: - splits = [int(s) for s in layer_for_test.split(',')] - else: - splits = [int(layer_for_test)] - layer_for_test_map = {} - for index, layer_id in enumerate(splits): - layer_for_test_map[layer_id] = index - - if num_nextn_predict_layers > 0: - min_mtp_layer_id = ori_num_layers - mtp_layer_prefix = name_map[MTP_LAYER_PREFIX] - else: - min_mtp_layer_id = None - - if ep is not None: - post_attention_layernorm = name_map[POST_ATTENTION_LAYERNORM] - if MOE_MLP in name_map: - moe_mlp = name_map[MOE_MLP] - moe_expert = name_map[MOE_EXPERT] - if MOE_GROUPED_GEMM_EXPERT in name_map: - moe_grouped_gemm_expert = name_map[MOE_GROUPED_GEMM_EXPERT] - if MOE_SHARED_EXPERT in name_map: - moe_shared_expert = name_map[MOE_SHARED_EXPERT] - - experts_ids = [x for x in range(num_experts)] - chunks = [experts_ids[x:x + num_experts // ep] - for x in range(0, len(experts_ids), num_experts // ep)] # ep_id -> [expert_ids] - - expert_local_mapping = {} - expert_ep_mapping = {} - ep_expert_mapping = {} - for ep_id, chunk in enumerate(chunks): - ep_expert_mapping[ep_id] = chunk # ep_id -> [expert_ids] - for idx, ele in enumerate(chunk): - expert_local_mapping[ele] = idx # expert_id -> local_ep_id - expert_ep_mapping[ele] = ep_id # expert_id -> ep_id - print(f"expert_local_mapping: {expert_local_mapping}") - print(f"expert_ep_mapping: {expert_ep_mapping}") - print(f"ep_expert_mapping: {ep_expert_mapping}") - - etp_to_tp_mapping = None - if etp is not None: - assert tp % (etp * ep) == 0 or (etp * ep) % tp == 0, f"tp: {tp}, etp: {etp}, ep: {ep}, tp % (etp * ep) != 0" - etp_to_tp_mapping = {} - v_tp = tp - if tp < etp * ep: - v_tp = etp * ep - for t in range (v_tp): - etp_id = t % etp - ep_id = (t // etp) % ep - if ep_id not in etp_to_tp_mapping: - etp_to_tp_mapping[ep_id] = {} - etp_to_tp_mapping[ep_id][etp_id] = t % tp - print(f"{etp_to_tp_mapping=}") - - if args.save_sub_checkpoint_by_pp and utils.LOADED_STATE_DICT is not None: - utils.LOADED_LAYERS = set() - if num_experts is not None: - utils.LOADED_MIN_E = min(e for p in utils.LOADED_STATE_DICT for e in utils.LOADED_STATE_DICT[p]) - - def _convert(layer_name, p, state_dict_node, ep_id=None, tp_transpose_shape=None, sub_key=None, - expert_state_dict=None, one_layer_weights_map=None, etp_transpose_shape=None): - if one_layer_weights_map is not None and len(one_layer_weights_map) == 0: - return - _set_func = { - INPUT_LAYERNORM: c_ckpt.set_layer_input_layernorm, - ATTENTION_QUERY_KEY_VALUE: c_ckpt.set_layer_attention_query_key_value, - ATTENTION_QKV_MAP: c_ckpt.set_layer_attention_by_name, - ATTENTION_DENSE: c_ckpt.set_layer_attention_dense, - POST_ATTENTION_LAYERNORM: c_ckpt.set_layer_post_attention_layernorm, - MOE_GATE: c_ckpt.set_layer_moe_gate, - MLP_DENSE_H_TO_4H: c_ckpt.set_layer_mlp_dense_h_to_4h, - MLP_DENSE_4H_TO_H: c_ckpt.set_layer_mlp_dense_4h_to_h, - POST_MLP_LAYERNORM: c_ckpt.set_layer_post_mlp_layernorm, - }[layer_name] - _weight_bias_dict = { - MOE_GATE: MOE_GATE_BIAS - } - _set_layernorm_func = { - INPUT_LAYERNORM: lambda x, y, z, one_layer_weights=None: None, - ATTENTION_QUERY_KEY_VALUE: lambda x, y, z, one_layer_weights=None: None, - ATTENTION_QKV_MAP: lambda x, y, z, one_layer_weights=None: None, - ATTENTION_DENSE: lambda x, y, z, one_layer_weights=None: None, - POST_ATTENTION_LAYERNORM: lambda x, y, z, one_layer_weights=None: None, - MOE_GATE: lambda x, y, z, one_layer_weights=None: None, - MLP_DENSE_H_TO_4H: lambda x, y, z, one_layer_weights=None: None, - MLP_DENSE_4H_TO_H: lambda x, y, z, one_layer_weights=None: None, - POST_MLP_LAYERNORM: lambda x, y, z, one_layer_weights=None: None, - }[layer_name] - # input_layernorm - if not args.no_te: - if layer_name == INPUT_LAYERNORM: - _set_func = lambda x, y, z, one_layer_weights=None: None - if layer_name == ATTENTION_QUERY_KEY_VALUE: - _set_layernorm_func = c_ckpt.set_layer_input_layernorm - # post_attention_layernorm - if (not args.no_te) and ep is None: - if layer_name == POST_ATTENTION_LAYERNORM: - _set_func = lambda x, y, z, one_layer_weights=None: None - if layer_name == MLP_DENSE_H_TO_4H: - _set_layernorm_func = c_ckpt.set_layer_post_attention_layernorm - - weight_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.weight") - bias_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.bias") - if sub_key is not None and \ - ("dim" not in name_map[layer_name][sub_key] or not name_map[layer_name][sub_key]["dim"]): - weight_chunk_dim = None - bias_chunk_dim = None - - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - - if layer_for_test is not None: - cur_new_layer_index = 0 - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - new_layer_index = layer_index - if layer_for_test is not None: - if layer_id in layer_for_test_map: - new_layer_index = cur_new_layer_index - cur_new_layer_index += 1 - else: - continue - if min_mtp_layer_id is not None and layer_id >= min_mtp_layer_id: - cur_layer_prefix = mtp_layer_prefix - new_layer_index = layer_id - min_mtp_layer_id - else: - cur_layer_prefix = layer_prefix - one_layer_weights = one_layer_weights_map[layer_id] \ - if one_layer_weights_map is not None else None - - def get_tensor_per_node(transformer, sub_name, is_moe_mlp = False, expert_id = None, is_shared = False, - transpose_shape=None, weight_chunk_dim=None): - weight_name = None - bias_name = None - weight_scale_inv_name = None - etp_to_tp = None - cur_state_dict = state_dict_node if expert_id is None else expert_state_dict - if is_moe_mlp: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_mlp}.{sub_name}" - elif is_shared: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_shared_expert}.{sub_name}" - elif expert_id is not None: - local_eid = expert_local_mapping[expert_id] - if args.moe_grouped_gemm: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_grouped_gemm_expert}.{sub_name}" - weight_name = f"{name}.weight{local_eid}" - bias_name = f"{name}.bias{local_eid}" - else: - name = f"{cur_layer_prefix}.{new_layer_index}.{moe_expert}.{local_eid}.{sub_name}" - etp_to_tp = etp_to_tp_mapping[ep_id] if etp is not None else None - else: - name = f"{cur_layer_prefix}.{new_layer_index}.{sub_name}" - if sub_key is not None and "is_layernorm" in name_map[layer_name][sub_key] and name_map[layer_name][sub_key]["is_layernorm"]: - weight_name = name + ".layer_norm_weight" - if layer_name in _weight_bias_dict and _weight_bias_dict[layer_name] in name_map: - bias_name = f"{cur_layer_prefix}.{new_layer_index}.{name_map[_weight_bias_dict[layer_name]]}" - - if weight_name is None: - weight_name = name + ".weight" - if pretrain_as_fp8: - for key in ignore_tp_keys: - if key in weight_name: - weight_chunk_dim = None - weight, weight_scale_inv = self.get_tensor( - cur_state_dict, transformer, weight_name, dim=weight_chunk_dim, etp_to_tp=etp_to_tp) - need_to_convert_fp8 = False - if convert_to_fp8: - for fp8_key in fp8_keys: - if fp8_key in weight_name: - need_to_convert_fp8 = True - - if weight is None and utils.LOADED_LAYERS is not None: - return - if bias_name is not None: - bias, _ = self.get_tensor(cur_state_dict, transformer, bias_name, bias_chunk_dim, - etp_to_tp=etp_to_tp) - else: - bias, _ = self.get_tensor(cur_state_dict, transformer, name + ".bias", bias_chunk_dim, - etp_to_tp=etp_to_tp) - layernorm_weight, _ = self.get_tensor(cur_state_dict, transformer, name + ".layer_norm_weight", - etp_to_tp=etp_to_tp) - layernorm_bias, _ = self.get_tensor(cur_state_dict, transformer, name + ".layer_norm_bias", - etp_to_tp=etp_to_tp) - if transpose_shape is not None: - weight = transpose_shape0(weight, *transpose_shape) - if bias is not None: - bias = transpose_shape0(bias, *transpose_shape) - if weight_scale_inv is not None: - weight_scale_inv = transpose_shape0(weight_scale_inv, *transpose_shape) - - if weight_scale_inv is None and need_to_convert_fp8: - weight, weight_scale_inv = McoreCheckpoint.per_block_cast_to_fp8( - weight, method=args.quant_method, amax_epsilon=args.amax_epsilon, - force_pow_2_scales=args.force_pow_2_scales) - weight = weight.cpu() - weight_scale_inv = weight_scale_inv.cpu() - - if layer_name in [MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H]: - if is_moe_mlp: - _set_func(layer_id, weight, bias, is_moe_mlp = is_moe_mlp, - one_layer_weights=one_layer_weights, weight_scale_inv=weight_scale_inv) - elif is_shared: - _set_func(layer_id, weight, bias, is_shared = is_shared, - one_layer_weights=one_layer_weights, weight_scale_inv=weight_scale_inv) - elif expert_id is not None: - _set_func(layer_id, weight, bias, expert_id = expert_id, - one_layer_weights=one_layer_weights, weight_scale_inv=weight_scale_inv) - else: - _set_func(layer_id, weight, bias, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - _set_layernorm_func(layer_id, layernorm_weight, layernorm_bias, - one_layer_weights=one_layer_weights) - else: - if sub_key is not None: - if layer_name == ATTENTION_QKV_MAP and \ - ("is_layernorm" not in name_map[layer_name][sub_key] or \ - not name_map[layer_name][sub_key]["is_layernorm"]): - _set_func(sub_key, layer_id, weight, bias, one_layer_weights=one_layer_weights, - weight_scale_inv=weight_scale_inv) - else: - _set_func(sub_key, layer_id, weight, bias, one_layer_weights=one_layer_weights) - else: - if layer_name == ATTENTION_DENSE: - _set_func(layer_id, weight, bias, - one_layer_weights=one_layer_weights, weight_scale_inv=weight_scale_inv) - else: - _set_func(layer_id, weight, bias, one_layer_weights=one_layer_weights) - _set_layernorm_func(layer_id, layernorm_weight, layernorm_bias, - one_layer_weights=one_layer_weights) - if ep_id is not None and layer_name in [MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H]: - if first_k_dense_replace is not None and layer_id < first_k_dense_replace: - get_tensor_per_node(transformer, name_map[layer_name], is_moe_mlp=True, - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - else: - for expert_id in ep_expert_mapping[ep_id]: - get_tensor_per_node(transformer, name_map[layer_name], expert_id=expert_id, - transpose_shape=etp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - if MOE_SHARED_EXPERT in name_map: - get_tensor_per_node(transformer, name_map[layer_name], is_shared=True, - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - elif layer_name == MOE_GATE: - if first_k_dense_replace is None or layer_id >= first_k_dense_replace: - get_tensor_per_node(transformer, name_map[layer_name], transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - else: - if sub_key is not None: - get_tensor_per_node(transformer, name_map[layer_name][sub_key]["name"], - transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - else: - get_tensor_per_node(transformer, name_map[layer_name], transpose_shape=tp_transpose_shape, - weight_chunk_dim=weight_chunk_dim) - - if ep_id is None: - if sub_key is None: - print(f"> p: {p}. {layer_name} chunk weight dim {weight_chunk_dim}, bias dim {bias_chunk_dim}," - f"max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - else: - print(f"> p: {p}. {layer_name}:{sub_key} chunk weight dim {weight_chunk_dim}, bias dim {bias_chunk_dim}," - f"max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - else: - if sub_key is None: - print(f"> p: {p}, ep_id: {ep_id}. {layer_name} chunk weight dim {weight_chunk_dim}," - f"bias dim {bias_chunk_dim}," - f"max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - else: - print(f"> p: {p}, ep_id: {ep_id}. {layer_name}:{sub_key} chunk weight dim {weight_chunk_dim}," - f"bias dim {bias_chunk_dim}," - f"max_memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}") - - def convert_one_p(p): - if self.load_path is None and utils.LOADED_STATE_DICT is not None and \ - p not in utils.LOADED_STATE_DICT.keys(): - return - if utils.LOADED_LAYERS is not None: - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - utils.LOADED_LAYERS.add(layer_id) - - if cache_path is None: - one_layer_weights_map = None - else: - one_layer_weights_map = {} - is_done = True - for stage_index in range(stage): - if not is_done: - break - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if not os.path.exists(f"{cache_path}/checkpoint_{layer_id}.pt") or \ - not os.path.exists(f"{cache_path}/checkpoint_{layer_id}.done"): - is_done = False - if not is_done: - break - if not is_done: - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if os.path.exists(f"{cache_path}/checkpoint_{layer_id}.done"): - os.remove(f"{cache_path}/checkpoint_{layer_id}.done") - if os.path.exists(f"{cache_path}/checkpoint_{layer_id}.pt"): - os.remove(f"{cache_path}/checkpoint_{layer_id}.pt") - one_layer_weights_map[layer_id] = {} - if is_done and p != 0 and p != self.pp-1: - return - - state_dict_node = {} - if etp is None: - for t in range(self.tp): - key_e = 0 - if self.load_path is None and utils.LOADED_STATE_DICT is not None and self.ep is not None: - key_e = list(utils.LOADED_STATE_DICT[p].keys())[0] - state_dict_node[t] = self.get_state_dict(p, t, e=None if self.ep is None else key_e) - else: - t_to_e_map = {} - for ep_id in etp_to_tp_mapping: - for t in etp_to_tp_mapping[ep_id].values(): - t_to_e_map[t] = ep_id - for t in range(self.tp): - key_e = t_to_e_map[t] - state_dict_node[t] = self.get_state_dict(p, t, e=None if self.ep is None else key_e) - - if p == 0: - # 1.1 word_embeddings - if WORD_EMBEDDINGS in name_map: - parallel_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS}.weight") - name = self.get_word_embedding_name() - transformer = self.get_transformer_name(0) - weight, _ = self.get_tensor(state_dict_node, transformer, name + '.weight', parallel_dim) - if margs.get("add_embedding_padding", False) and weight is not None: - orig_vocab_size = cargs["vocab_size"] - weight = cut_embedding_padding(weight, orig_vocab_size) - c_ckpt.set_word_embedding(weight) - print(f"> p: {p}. word_embeddings weight: {weight.shape}") \ - if weight is not None else None - - # 1.2 position embedding - if margs.get("add_position_embedding", False): - name = self.get_position_embedding_name() - parallel_dim = tensor_parallel_dim.get(f"{WORD_POSITION_EMBEDDINGS}.weight") - weight, _ = self.get_tensor(state_dict_node, transformer, name + '.weight', parallel_dim) - c_ckpt.set_word_position_embedding(weight) - print(f"> p: {p}. add position embedding weight: {weight.shape}") \ - if weight is not None else None - - if margs.get("add_block_position_embedding", False): - name = self.get_block_position_embedding_name() - parallel_dim = tensor_parallel_dim.get(f"{WORD_BLOCK_POSITION_EMBEDDINGS}.weight") - weight, _ = self.get_tensor(state_dict_node, transformer, name + '.weight', parallel_dim) - c_ckpt.set_word_block_position_embedding(weight) - print(f"> p: {p}. add block position embedding weight: {weight.shape}") \ - if weight is not None else None - - # 2. transformer layers - - # 2.1 input_layernorm - if ep is None: - if INPUT_LAYERNORM in name_map: - _convert(INPUT_LAYERNORM, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - else: - _convert(INPUT_LAYERNORM, p, state_dict_node, ep_id=0, one_layer_weights_map=one_layer_weights_map) - - # 2.2 rotary_emb.inv_freqs - if ATTENTION_ROTARY_EMB_INV_FREQ in name_map: - chunk_dim = tensor_parallel_dim.get(ATTENTION_ROTARY_EMB_INV_FREQ) - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - if layer_for_test is not None: - cur_new_layer_index = 0 - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - new_layer_index = layer_index - if layer_for_test is not None: - if layer_id in layer_for_test_map: - new_layer_index = cur_new_layer_index - cur_new_layer_index += 1 - else: - continue - if min_mtp_layer_id is not None and layer_id >= min_mtp_layer_id: - cur_layer_prefix = mtp_layer_prefix - else: - cur_layer_prefix = layer_prefix - - if one_layer_weights_map is None or layer_id in one_layer_weights_map: - one_layer_weights = one_layer_weights_map[layer_id] \ - if one_layer_weights_map is not None else None - - name = f"{cur_layer_prefix}.{new_layer_index}.{name_map[ATTENTION_ROTARY_EMB_INV_FREQ]}" - inv_freq, _ = self.get_tensor(state_dict_node, transformer, name, chunk_dim) - c_ckpt.set_layer_attention_rotary_emb_inv_freq( - layer_id, inv_freq, one_layer_weights=one_layer_weights) - print(f"> p: {p}. rotary_emb.inv_freqs chunk dim {chunk_dim}") - elif margs.get('use_rotary_position_embeddings', False): - dim = hidden_size // num_attention_heads - inv_freq = 1.0 / (margs.get('rotary_base', 10000) ** (torch.arange(0, dim, 2).float() / dim)) - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if one_layer_weights_map is None or layer_id in one_layer_weights_map: - one_layer_weights = one_layer_weights_map[layer_id] \ - if one_layer_weights_map is not None else None - c_ckpt.set_layer_attention_rotary_emb_inv_freq( - layer_id, inv_freq, one_layer_weights=one_layer_weights) - print(f"> p: {p}. rotary_emb.inv_freqs, created by dim {dim}") - - # 2.3 self attention query_key_value - if ATTENTION_QUERY_KEY_VALUE in name_map: - if ep is None: - _convert(ATTENTION_QUERY_KEY_VALUE, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - else: - _convert(ATTENTION_QUERY_KEY_VALUE, p, state_dict_node, ep_id=0, - one_layer_weights_map=one_layer_weights_map) - if ATTENTION_QKV_MAP in name_map: - for sub_key, dict_mcore in name_map[ATTENTION_QKV_MAP].items(): - if ep is None: - _convert(ATTENTION_QKV_MAP, p, state_dict_node, sub_key=sub_key, - one_layer_weights_map=one_layer_weights_map) - else: - _convert(ATTENTION_QKV_MAP, p, state_dict_node, ep_id=0, sub_key=sub_key, - one_layer_weights_map=one_layer_weights_map) - - # 2.4 self attention dense - if ep is None: - _convert(ATTENTION_DENSE, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - else: - _convert(ATTENTION_DENSE, p, state_dict_node, ep_id=0, one_layer_weights_map=one_layer_weights_map) - - # 2.5 post attention layernorm - if ep is None: - _convert(POST_ATTENTION_LAYERNORM, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - else: - _convert(POST_ATTENTION_LAYERNORM, p, state_dict_node, ep_id=0, one_layer_weights_map=one_layer_weights_map) - - if ep is None: - # 2.6 mlp dense h_to_4h - if margs.get("transpose_mlp_dense", False): - _convert(MLP_DENSE_H_TO_4H, p, state_dict_node, tp_transpose_shape=(self.tp, 2), - one_layer_weights_map=one_layer_weights_map) - else: - _convert(MLP_DENSE_H_TO_4H, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - # 2.7 mlp dense 4h_to_h - _convert(MLP_DENSE_4H_TO_H, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - - # 2.8 post mlp layernorm - if POST_MLP_LAYERNORM in name_map: - _convert(POST_MLP_LAYERNORM, p, state_dict_node, one_layer_weights_map=one_layer_weights_map) - elif one_layer_weights_map is None or len(one_layer_weights_map) > 0: - _convert(MOE_GATE, p, state_dict_node, ep_id=0, one_layer_weights_map=one_layer_weights_map) - for ep_id in range(ep): - if self.load_path is None and utils.LOADED_STATE_DICT is not None and \ - ep_id not in utils.LOADED_STATE_DICT[p].keys(): - continue - if self.etp is None: - expert_state_dict = {} - if ep_id == 0: - expert_state_dict = state_dict_node - else: - for t in range(self.tp): - expert_state_dict[t] = self.get_state_dict(p, t, e=ep_id) - else: - expert_state_dict = {} - for et in etp_to_tp_mapping[ep_id]: - t = etp_to_tp_mapping[ep_id][et] - expert_state_dict[et] = self.get_state_dict(p, t, e=ep_id) - # 2.6 mlp dense h_to_4h - if margs.get("transpose_mlp_dense", False): - etp_transpose_shape=(self.tp, 2) - if self.etp is not None: - etp_transpose_shape=(self.etp, 2) - _convert(MLP_DENSE_H_TO_4H, p, state_dict_node, tp_transpose_shape=(self.tp, 2), - etp_transpose_shape=etp_transpose_shape, ep_id=ep_id, - expert_state_dict=expert_state_dict, one_layer_weights_map=one_layer_weights_map) - else: - _convert(MLP_DENSE_H_TO_4H, p, state_dict_node, ep_id=ep_id, - expert_state_dict=expert_state_dict, one_layer_weights_map=one_layer_weights_map) - # 2.7 mlp dense 4h_to_h - _convert(MLP_DENSE_4H_TO_H, p, state_dict_node, ep_id=ep_id, expert_state_dict=expert_state_dict, - one_layer_weights_map=one_layer_weights_map) - - # 2.8 post mlp layernorm - if POST_MLP_LAYERNORM in name_map: - _convert(POST_MLP_LAYERNORM, p, state_dict_node, ep_id=0, one_layer_weights_map=one_layer_weights_map) - add_final_layer = False - if not dualpipev and p == pp - 1: - add_final_layer = True - elif dualpipev and p == 0: - add_final_layer = True - if add_final_layer: - # 2.9 final_layernorm - if FINAL_LAYERNORM in name_map: - name = name_map[FINAL_LAYERNORM] - transformer = self.get_transformer_name(self.num_stages - 1) - - parallel_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.weight") - weight, _ = self.get_tensor(state_dict_node, transformer, name + ".weight", parallel_dim) - - parallel_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.bias") - bias, _ = self.get_tensor(state_dict_node, transformer, name + ".bias", parallel_dim) - - c_ckpt.set_final_layernorm(weight, bias) - print(f"> p: {p}. final_layernorm weight {weight.shape}") if weight is not None else None - - # 3 word embedding for head - if WORD_EMBEDDINGS_FOR_HEAD in name_map: - if margs.get("untie_embeddings_and_output_weights", False) or self.pp > 1: - parallel_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS_FOR_HEAD}.weight") - name = self.get_word_embedding_for_head_name() - else: - parallel_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS}.weight") - name = self.get_word_embedding_name() - weight, _ = self.get_tensor(state_dict_node, transformer, name + '.weight', parallel_dim) - if margs.get("add_embedding_padding", False) and weight is not None: - orig_vocab_size = cargs["vocab_size"] - weight = cut_embedding_padding(weight, orig_vocab_size) - c_ckpt.set_word_embeddings_for_head(weight) - print(f"> p: {p}. word embedding for head weight {weight.shape}") if weight is not None else None - - if MTP_WORD_EMBEDDING in name_map: - virtual_p = pp * self.num_stages - 1 - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = self.get_transformer_name(self.num_stages - 1) - new_layer_index = 0 - has_mtp = True - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - if layer_id < ori_num_layers: - continue - if layer_for_test is not None and layer_id not in layer_for_test_map: - continue - - mtp_word_embedding, _ = self.get_tensor( - state_dict_node, transformer, \ - f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_WORD_EMBEDDING]}.weight", \ - tensor_parallel_dim.get(f"{MTP_WORD_EMBEDDING}.weight")) - if mtp_word_embedding is None: - has_mtp = False - break - if margs.get("add_embedding_padding", False): - orig_vocab_size = cargs["vocab_size"] - mtp_word_embedding = cut_embedding_padding(mtp_word_embedding, orig_vocab_size) - mtp_enorm, _ = self.get_tensor( - state_dict_node, transformer, \ - f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_ENORM]}.weight", \ - tensor_parallel_dim.get(f"{MTP_ENORM}.weight")) - mtp_hnorm, _ = self.get_tensor( - state_dict_node, transformer, \ - f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_HNORM]}.weight", \ - tensor_parallel_dim.get(f"{MTP_HNORM}.weight")) - mtp_eh_proj, _ = self.get_tensor( - state_dict_node, transformer, \ - f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_EH_PROJ]}.weight", \ - tensor_parallel_dim.get(f"{MTP_EH_PROJ}.weight")) - mtp_shared_head_norm, _ = self.get_tensor( - state_dict_node, transformer, \ - f"{mtp_layer_prefix}.{new_layer_index}.{name_map[MTP_SHARED_HEAD_NORM]}.weight", \ - tensor_parallel_dim.get(f"{MTP_SHARED_HEAD_NORM}.weight")) - - assert WORD_EMBEDDINGS_FOR_HEAD in name_map, \ - f"{WORD_EMBEDDINGS_FOR_HEAD} is needed in name_map" - mtp_shared_head_head = c_ckpt.get_word_embeddings_for_head_weight() - new_layer_index += 1 - if has_mtp and one_layer_weights_map is None or layer_id in one_layer_weights_map: - one_layer_weights = one_layer_weights_map[layer_id] \ - if one_layer_weights_map is not None else None - c_ckpt.set_layer_mtp_weight(layer_id, mtp_word_embedding, mtp_enorm, mtp_hnorm, mtp_eh_proj,\ - mtp_shared_head_norm, mtp_shared_head_head, \ - one_layer_weights=one_layer_weights) - print(f"> p: {p}. mtp_word_embedding: {mtp_word_embedding.shape}, "\ - f"mtp_enorm: {mtp_enorm.shape}, mtp_hnorm: {mtp_hnorm.shape}, "\ - f"mtp_eh_proj: {mtp_eh_proj.shape}, mtp_shared_head_norm: {mtp_shared_head_norm.shape}, "\ - f"mtp_shared_head_head: {mtp_shared_head_head.shape}") - if cache_path is not None and one_layer_weights_map is not None and len(one_layer_weights_map) != 0: - for stage_index in range(stage): - virtual_p, layer_offset, transformer = McoreCheckpoint.get_virtual_partition( - dualpipev, stage_index, p, pp, num_layers_in_rank, self) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - torch.save(one_layer_weights_map[layer_id], f"{cache_path}/checkpoint_{layer_id}.pt") - done_file_name = f"{cache_path}/checkpoint_{layer_id}.done" - with open(done_file_name, 'w'): - os.utime(done_file_name, None) - - if args.max_workers > 1: - futures = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=args.max_workers) as executor: - for p in range(self.pp): - futures.append(executor.submit(convert_one_p, p=p)) - concurrent.futures.wait(futures) - for future in futures: - try: - result = future.result() - except Exception as e: - print(f"An error occurred: {e}") - raise e - else: - for p in range(self.pp): - convert_one_p(p=p) - - # 4. optimizer - if ep is not None: - print("Optimizer is not supported in moe") - elif self.has_optimizer(): - opt = merge_optimizer_by_pp_tp(self.optim_state_dict, margs) - opt.build_param_map(self.get_named_parameters_shape()) - if margs.get("add_embedding_padding", False): - orig_vocab_size = cargs["vocab_size"] - opt.cut_embedding_padding(orig_vocab_size) - if self.num_stages > 1: - opt.interleave(self.pp, self.num_stages) - if self.pp == 1 and not margs.get("untie_embeddings_and_output_weights", False): - opt.add_word_embedding_for_head() - c_ckpt.state_dict["optimizer"] = opt.to_dict() - print("> optimizer params: ", opt.get_param_num()) - else: - print("> optimizer empty") - - # 5.others - c_ckpt.other_args["iteration"] = self.iteration - c_ckpt.other_args["version"] = self.version - c_ckpt.other_args["args"] = self.args - c_ckpt.other_args["rng_state"] = self.rng_state - - return c_ckpt - - def set_name_map(self, name_map): - """ set name_map """ - self.name_map = name_map - - def get_word_embedding_name(self): - """ get word_embedding name """ - #if self.num_stages > 1: - # if WORD_EMBEDDINGS_TPL not in self.name_map: - # return None - # return self.name_map[WORD_EMBEDDINGS_TPL] % 0 - #else: - return self.name_map.get(WORD_EMBEDDINGS) - - def get_position_embedding_name(self): - """ get position_embedding name """ - #if self.num_stages > 1 : - # if WORD_POSITION_EMBEDDINGS_TPL not in self.name_map: - # return None - # return self.name_map[WORD_POSITION_EMBEDDINGS_TPL] % 0 - #else: - return self.name_map.get(WORD_POSITION_EMBEDDINGS) - - def get_block_position_embedding_name(self): - """ get block_position_embedding name """ - #if self.num_stages > 1 : - # if WORD_BLOCK_POSITION_EMBEDDINGS_TPL not in self.name_map: - # return None - # return self.name_map[WORD_BLOCK_POSITION_EMBEDDINGS_TPL] % 0 - #else: - return self.name_map.get(WORD_BLOCK_POSITION_EMBEDDINGS) - - def get_transformer_name(self, stage_index): - """ get transformer name """ - if self.num_stages > 1: - return self.name_map[TRANSFORMER_TPL] % stage_index - else: - return self.name_map[TRANSFORMER] - - def get_word_embedding_for_head_name(self): - """ get word_embedding for head name """ - #if self.num_stages > 1: - # return self.name_map[WORD_EMBEDDINGS_FOR_HEAD_TPL] % (self.num_stages - 1) - #else: - return self.name_map[WORD_EMBEDDINGS_FOR_HEAD] - - def has_optimizer(self): - """ whether has optimizer """ - if self.optim_state_dict is None: - return False - for p in range(self.pp): - for t in range(self.tp): - if self.ep is None: - if self.optim_state_dict[p][t].empty(): - return False - else: - for ep_id in range(self.ep): - if self.optim_state_dict[p][ep_id][t].empty(): - return False - return True - - def has_bias(self, pp_rank=None): - for p in range(self.pp) if pp_rank is None else [pp_rank]: - for prefix, name, _ in self.layers[p]: - if name.endswith("bias"): - return True - return False - - def has_word_embeddings(self, p=None): - """ whether has word embeddings """ - if WORD_EMBEDDINGS not in self.name_map: - return False - p = 0 if p is None else p - for prefix, name, _ in self.layers[p]: - if prefix == self.get_word_embedding_name() and "weight" == name: - return True - return False - def has_position_embeddings(self, p=None): - """ whether has position embeddings """ - if WORD_POSITION_EMBEDDINGS not in self.name_map: - return False - p = 0 if p is None else p - for prefix, name, _ in self.layers[p]: - if prefix == self.get_position_embedding_name() and "weight" == name: - return True - return False - - def has_block_position_embeddings(self, p=None): - """ whether has block position embeddings """ - if WORD_BLOCK_POSITION_EMBEDDINGS not in self.name_map: - return False - p = 0 if p is None else p - for prefix, name, _ in self.layers[p]: - if prefix == self.get_block_position_embedding_name() and "weight" == name: - return True - return False - - def has_word_embeddings_for_head(self, p=None): - if WORD_EMBEDDINGS_FOR_HEAD not in self.name_map: - return False - p = self.pp-1 if p is None else p - for prefix, name, _ in self.layers[p]: - if self.get_word_embedding_for_head_name() == prefix and "weight" == name: - return True - return False - - def has_final_layernorm(self, key, p=None): - if FINAL_LAYERNORM not in self.name_map: - return False - p = self.pp-1 if p is None else p - for prefix, name, _ in self.layers[p]: - if f"{self.name_map[FINAL_LAYERNORM]}.{key}" == name: - return True - return False - - def get_layers(self, pp_rank=None): - """ get all layers """ - layers = [] - tensor_parallel_dim = self.tensor_parallel_dim - - # embedding - if pp_rank == 0 or pp_rank is None: - keys = (WORD_EMBEDDINGS, \ - WORD_POSITION_EMBEDDINGS, \ - WORD_BLOCK_POSITION_EMBEDDINGS) - names = [self.get_word_embedding_name(), \ - self.get_position_embedding_name(), \ - self.get_block_position_embedding_name()] - if self.ep is None: - state_dict = self.get_state_dict(0, 0) - else: - state_dict = self.get_state_dict(0, 0, 0) - for i in range(3): - if names[i] is not None and check_path_in_dict(state_dict, names[i]): - chunk_dim = tensor_parallel_dim.get(f"{keys[i]}.weight", -1) - layers.append((names[i], "weight", chunk_dim)) - - # transformers - if self.ep is None: - if ATTENTION_QUERY_KEY_VALUE in self.name_map: - TRANSFORMER_LAYERS = [INPUT_LAYERNORM, ATTENTION_QUERY_KEY_VALUE, ATTENTION_DENSE, \ - POST_ATTENTION_LAYERNORM, MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H, ] - else: - TRANSFORMER_LAYERS = [INPUT_LAYERNORM, ATTENTION_QKV_MAP, ATTENTION_DENSE, \ - POST_ATTENTION_LAYERNORM, MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H, ] - else: - if ATTENTION_QUERY_KEY_VALUE in self.name_map: - TRANSFORMER_LAYERS = [INPUT_LAYERNORM, ATTENTION_QUERY_KEY_VALUE, ATTENTION_DENSE, \ - POST_ATTENTION_LAYERNORM, MOE_GATE, MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H, ] - else: - TRANSFORMER_LAYERS = [INPUT_LAYERNORM, ATTENTION_QKV_MAP, ATTENTION_DENSE, \ - POST_ATTENTION_LAYERNORM, MOE_GATE, MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H, ] - - num_layers_per_pp = self.num_layers // self.pp - num_layers_per_stage = num_layers_per_pp // self.num_stages - for p in range(self.pp) if pp_rank is None else [pp_rank]: - layer_prefix = self.name_map.get(LAYER_PREFIX) - if self.ep is None: - state_dict = self.get_state_dict(p, 0) - else: - state_dict = self.get_state_dict(p, 0, 0) - for layer_id in range(num_layers_per_pp): - layer_prefix = self.name_map.get(LAYER_PREFIX) - stage_index = layer_id // num_layers_per_stage - layer_index = layer_id % num_layers_per_stage - transformer_name = self.get_transformer_name(stage_index) - transformer = get_element_from_dict_by_path(state_dict, transformer_name) - for layer in TRANSFORMER_LAYERS: - for w in ("weight", "bias"): - layer_name = f"{layer_prefix}.{layer_index}.{self.name_map[layer]}.{w}" - if layer_name in transformer.keys(): - chunk_dim = tensor_parallel_dim.get(f"{layer}.{w}", -1) - layers.append((transformer_name, layer_name, chunk_dim)) - for layer in [ATTENTION_ROTARY_EMB_INV_FREQ]: - if layer not in self.name_map: - continue - layer_name = f"{layer_prefix}.{layer_index}.{self.name_map[layer]}" - if layer_name in transformer.keys(): - chunk_dim = self.tensor_parallel_dim.get(f"{layer}", -1) - layers.append((transformer_name, layer_name, chunk_dim)) - - transformer_name = self.get_transformer_name(self.num_stages - 1) - transformer = get_element_from_dict_by_path(state_dict, transformer_name) - for layer in [FINAL_LAYERNORM] if FINAL_LAYERNORM in self.name_map else []: - for w in ("weight", "bias"): - layer_name = f"{self.name_map[layer]}.{w}" - if layer_name in transformer.keys(): - chunk_dim = tensor_parallel_dim.get(f"{layer}.{w}", -1) - layers.append((transformer_name, layer_name, chunk_dim)) - - # emebdding for head - if WORD_EMBEDDINGS_FOR_HEAD in self.name_map: - if pp_rank == self.pp-1 or pp_rank is None: - layer = self.get_word_embedding_for_head_name() - chunk_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS_FOR_HEAD}.weight", -1) - layers.append((layer, "weight", chunk_dim)) - - return layers - - def get_transformer_layers(self, pp_rank, layer_index): - """ get transformer layers """ - layers = [] - pp_rank = 0 if pp_rank is None else pp_rank - transformer_name = self.get_transformer_name(0) - for meta in self.get_named_parameters_shape(pp_rank): - layer_name = meta[0] - key = f"{transformer_name}.{self.name_map[LAYER_PREFIX]}.{layer_index}." - if key in layer_name: - layers.append(meta) - # print(pp_rank, layer_index, layers) - return layers - - def get_named_parameters_shape(self, p=None, t=None): - if p == None and t == None: - return self.named_parameters_shape - if p != None and t != None: - return self.named_parameters_shape_by_pt[p][t] - if p != None: - return self.named_parameters_shape_by_p[p] - if t != None: - return self.named_parameters_shape_by_t[t] - - def _get_named_parameters_shape(self, pp_rank=None, tp_rank=None): - """ return list of (layer_name, tensor_shape, parallel_dim) """ - result = [] - for p in range(self.pp) if pp_rank is None else [pp_rank]: - if self.ep is None: - state_dict = self.get_state_dict(p, 0) - else: - state_dict = self.get_state_dict(p, 0, 0) - for layer, key, parallel_dim in self.layers[p]: - if key.endswith("weight") or key.endswith("bias"): - element = get_element_from_dict_by_path(state_dict, layer) - if key in element: - assert element[key] is not None, f"key {key}" - shape = element[key].shape - if tp_rank is None and parallel_dim >= 0: - shape = list(shape) - shape[parallel_dim] *= self.tp - shape = torch.Size(shape) - result.append((f"{layer}.{key}", shape, parallel_dim)) - return result - - def load(self, load_path, m_config, name_map, load_optimizer=True): - """ - Load mcore checkpoint from checkpoints folder. - - Args: - load_path (str): the path to checkpoint - m_config: mcore m_config loaded from ckpt - """ - - args = parse_args() - tp = args.tensor_model_parallel_size - pp = args.pipeline_model_parallel_size - dp = args.data_parallel_size - ep = args.expert_parallel_size - etp = args.expert_tensor_parallel_size if hasattr(args, 'expert_tensor_parallel_size') else None - dtype = m_config.get("dtype") - tensor_parallel_dim = m_config.get("tensor_parallel_dim") - num_layers_per_stage = m_config.get('num_layers_per_virtual_pipeline_stage') - custom_pipeline_layers = m_config.get("custom_pipeline_layers") - stage = args.num_virtual_stages_per_pipeline_rank - if stage is None: - stage = None if num_layers_per_stage is None else (self.num_layers // pp // num_layers_per_stage) - use_distributed_optimizer = m_config.get("use_distributed_optimizer", False) - self.set_dtype(dtype) - self.init_pipeline_size(pp, tp, dp, ep, tensor_parallel_dim, stage, - custom_pipeline_layers=custom_pipeline_layers, etp=etp) - self.set_name_map(name_map) - - key_t = 0 - key_p = 0 - key_e = 0 - if self.load_path is None and utils.LOADED_STATE_DICT is not None: - key_p = list(utils.LOADED_STATE_DICT.keys())[0] - if self.ep is not None: - key_e = list(utils.LOADED_STATE_DICT[key_p].keys())[0] - - if self.ep is None: - state_dict = self.get_state_dict(key_p, key_t) - else: - state_dict = self.get_state_dict(key_p, key_t, key_e) - self.iteration = state_dict.get('iteration', 0) - self.version = state_dict['checkpoint_version'] - self.args = state_dict['args'] - self.rng_state = state_dict.get('rng_state', None) - - self.optim_state_dict = None - # optimizer - if load_optimizer: - self.init_layers() - self.init_named_parameters_shape() - self.init_optimizer(use_distributed_optimizer) - if use_distributed_optimizer: - for p in range(self.pp): - for t in range(self.tp): - opts = [] - named_parameters_shape = self.get_named_parameters_shape(p, t) - if self.ep is None: - for d in range(self.dp): - if self.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}" - checkpoint_dir = os.path.join(load_path, checkpoint_dir) - checkpoint_path = os.path.join(checkpoint_dir, "distrib_optim.pt") - optim_state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - opt = MegatronOptimizer.generate_optimizer(self, self.num_layers // self.pp, p) - opt.load(optim_state_dict) - opts.append(opt) - opt.debug(f"tp/pp/dp rank: {t}/{p}/{d}, load from: {checkpoint_path}") - self.optim_state_dict[p][t] = merge_optimizer_by_dp(opts, named_parameters_shape) - self.optim_state_dict[p][t].debug(f"merge by dp {self.dp} in pp/tp rank {p}/{t}") - else: - for e in range(self.ep): - for d in range(self.dp): - if self.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}_{e:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}_{e:03d}" - checkpoint_dir = os.path.join(load_path, checkpoint_dir) - checkpoint_path = os.path.join(checkpoint_dir, "distrib_optim.pt") - optim_state_dict = torch.load( - checkpoint_path, map_location="cpu", weights_only=False) - opt = MegatronOptimizer.generate_optimizer(self, self.num_layers // self.pp, p) - opt.load(optim_state_dict) - opts.append(opt) - opt.debug(f"tp/pp/ep/dp rank: {t}/{p}/{e}{d}, load from: {checkpoint_path}") - self.optim_state_dict[p][e][t] = merge_optimizer_by_dp(opts, named_parameters_shape) - self.optim_state_dict[p][e][t].debug(f"merge by dp {self.dp} in pp/tp/ep rank {p}/{t}/{e}") - - else: - for p in range(self.pp): - for t in range(self.tp): - if self.ep is None: - state_dict = self.get_state_dict(p, t) - if "optimizer" in state_dict: - named_parameters_shape = self.get_named_parameters_shape(p, t) - self.optim_state_dict[p][t].load(state_dict, named_parameters_shape) - else: - for e in range(self.ep): - state_dict = self.get_state_dict(p, t, e) - if "optimizer" in state_dict: - named_parameters_shape = self.get_named_parameters_shape(p, t) - self.optim_state_dict[p][e][t].load(state_dict, named_parameters_shape) - self.debug("==================== mcore checkpoint loaded ================================") - - def pre_save(self, save_path, m_config=None): - """ - Before saving the model, delete the old save directory, - create a new save directory, and update the tracking file. - If 'm_config' is not provided, the current 'mcore' configuration will be used. - - Args: - save_path (str): Path where the model should be saved. - m_config (Optional[dict], optional): Optional `mcore` configuration dictionary, default to None. - - Returns: - tuple(str, dict): Returns a tuple containing two elements: the first is the new saved directory path, - and the second is the updated `mcore` configuration dictionary. - """ - os.makedirs(save_path, exist_ok=True) - # Saving the tracker file - tracker_filepath = os.path.join(save_path, "latest_checkpointed_iteration.txt") - with open(tracker_filepath, "w") as f: - f.write(str(self.iteration or "release")) - - # create `release` dir in args.load_path - folder_name = f"iter_{self.iteration:07d}" if self.iteration > 0 else "release" - release_dir = os.path.join(save_path, folder_name) - os.makedirs(release_dir, exist_ok=True) - - # mcore config - margs = self.args - if m_config is not None: - for k, v in m_config.data.items(): - setattr(margs, k, v) - print(f"Saving mcore args {margs}") - return release_dir, margs - - def save_model_file(self, release_dir, margs, p, t, e, state_dict_node, optim_state_dict_node, saved_models_str): - """ - Save the model file, including model parameters, optimizer state, and random seed. - If the number of iterations is None, use mp_rank as the directory name; otherwise, - use mp_rank and epoch as the directory name. - - Args: - release_dir (str): The path of the release directory. - margs (Optional[Namespace], optional): Namespace object of command line parameters, default is None. - p (int): process number mp_rank. - t (int): task number mp_rank. - e (Optional[int], optional): The number of epochs, default to None. - state_dict_node (Dict[str, Any]): Model parameter dictionary. - optim_state_dict_node (Dict[str, Any]): Optimizer state dictionary. - - Returns: - None. - - Raises: - None. - """ - state_dict_node["checkpoint_version"] = self.version - if e is None or self.ep == 1: - checkpoint_dir = ( - f"mp_rank_{t:02d}" - if self.pp == 1 - else f"mp_rank_{t:02d}_{p:03d}" - ) - else: - checkpoint_dir = ( - f"mp_rank_{t:02d}_{e:03d}" - if self.pp == 1 - else f"mp_rank_{t:02d}_{p:03d}_{e:03d}" - ) - - checkpoint_name = "model_optim_rng.pt" - if optim_state_dict_node is not None: - state_dict_node.update(optim_state_dict_node.to_dict()) - if margs is not None: - state_dict_node['args'] = margs - if self.rng_state is not None: - state_dict_node['rng_state'] = self.rng_state - state_dict_node["iteration"] = self.iteration - checkpoint_dir = os.path.join(release_dir, checkpoint_dir) - os.makedirs(checkpoint_dir, exist_ok=True) - checkpoint_path = os.path.join(checkpoint_dir, checkpoint_name) - torch.save(state_dict_node, checkpoint_path) - print(f"Saving mcore checkpoint {state_dict_node.keys()} to: {checkpoint_path}, {saved_models_str}") - - def save_optimizer(self, release_dir, p, t, e, optim_state_dict_node): - """ - Save the optimizer state. - If the current epoch is None, the name is in the mp_rank_t_d format; otherwise, - the name is in the mp_rank_t_e_d format. - Each distributed process saves its optimizer state to a different directory. - - Args: - release_dir (str) The directory path where the optimizer state should be saved. - p (int) The current process number starts from 0. - t (int) The current task number starts from 0. - e (Optional[int]) The current epoch is optional, and the default is None. - optim_state_dict_node (DistributedOptimizerStateDictNode) - DistributedOptimizerStateDictNode object containing the optimizer state. - - Returns: - None. - """ - chunk_optimers = optim_state_dict_node.chunk_by_dp(self.dp, self.num_stages) - for d in range(self.dp): - if e is None or self.ep == 1: - if self.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}" - else: - if self.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}_{e:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}_{e:03d}" - - checkpoint_dir = os.path.join(release_dir, checkpoint_dir) - os.makedirs(checkpoint_dir, exist_ok=True) - checkpoint_path = os.path.join(checkpoint_dir, "distrib_optim.pt") - torch.save( - chunk_optimers[d].to_dict(), - checkpoint_path, - ) - if e is None or self.ep == 1: - chunk_optimers[d].debug(f"tp/pp/dp rank {t}/{p}/{d}, saved to: {checkpoint_path}") - else: - chunk_optimers[d].debug(f"tp/pp/ep/dp rank {t}/{p}/{e}/{d}, saved to: {checkpoint_path}") - - def debug(self, title): - """ debbug """ - print(f"\n【Mcore】{title}") - if self.ep is None: - print(f"-> tp/pp/dp size: {self.tp}/{self.pp}/{self.dp}") - else: - print(f"-> tp/pp/ep/dp size: {self.tp}/{self.pp}/{self.ep}/{self.dp}") - if self.has_optimizer(): - for t in range(self.tp): - for p in range(self.pp): - if self.ep is None: - self.optim_state_dict[p][t].debug(f"tp/pp rank {t}/{p}") - else: - for e in range(self.ep): - self.optim_state_dict[p][e][t].debug(f"tp/pp/ep rank {t}/{p}/{e}") - - print("\n") - -if __name__ == "__main__": - pass diff --git a/tools/convert_checkpoint/mcore_config.py b/tools/convert_checkpoint/mcore_config.py deleted file mode 100644 index a8ebc58e..00000000 --- a/tools/convert_checkpoint/mcore_config.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import torch -import types -from tqdm import tqdm -import pprint - -from convert_checkpoint.abstact_config import AbstractConfig -from convert_checkpoint.common_config import CommonConfig - - -class McoreConfig(AbstractConfig): - """ - McoreConfig - """ - - def __init__(self): - super().__init__() - - @staticmethod - def convert_from_common(c_config): - """ - return mcore config converted from common config - """ - config = McoreConfig() - cargs = c_config.get_args("common") - margs = c_config.get_args("mcore") - config.update(cargs) - config.update(margs) - - num_layers_per_stage = margs["num_layers_per_virtual_pipeline_stage"] - if num_layers_per_stage is not None: - num_layers = cargs["num_layers"] - pp = margs["pipeline_model_parallel_size"] - stage = num_layers // pp // num_layers_per_stage - config.update({"virtual_pipeline_model_parallel_size": stage}) - return config - - def load(self, load_path): - """ load config """ - if load_path is None: - return - else: - sub_dirs = os.listdir(load_path) - possible_sub_dirs = ["mp_rank_00", "mp_rank_00_000", "mp_rank_00_000_000"] - rank0_checkpoint_path = None - for sub_dir in possible_sub_dirs: - if sub_dir in sub_dirs: - rank0_checkpoint_name = "model_optim_rng.pt" - rank0_checkpoint_path = os.path.join(load_path, sub_dir, rank0_checkpoint_name) - break - print(f"Loading Mcore config from: {rank0_checkpoint_path}") - - rank0_state_dict = torch.load(rank0_checkpoint_path, map_location="cpu", weights_only=False) - if "args" not in rank0_state_dict: - raise ValueError( - "Megatron-LM checkpoint does not contain arguments. This utility only supports Megatron-LM checkpoints" - " containing all the megatron arguments. This is because it loads all config related to model" - " architecture, the tensor and pipeline model parallel size from the checkpoint insead of user having to" - " manually specify all the details. Please save Megatron-LM checkpoint along with all the megatron" - " arguments to use this utility." - ) - self.data = vars(rank0_state_dict["args"]) - - def save(self, save_path): - """ save config """ - - release_dir = os.path.join(save_path, "release") - os.makedirs(release_dir, exist_ok=True) - - tp = self.get("tensor_model_parallel_size") - pp = self.get("pipeline_model_parallel_size") - pbar = tqdm(range(tp*pp), desc='Saving Megatron-LM Config') - for p in range(pp): - for t in range(tp): - sub_dir_name = f"mp_rank_{t:02d}" if pp == 1 \ - else f"mp_rank_{t:02d}_{p:03d}" - checkpoint_name = os.listdir(os.path.join(release_dir, sub_dir_name))[0] - checkpoint_path = os.path.join(release_dir, sub_dir_name, checkpoint_name) - tp_state_dict = torch.load(checkpoint_path, map_location="cpu", weights_only=False) - tp_state_dict["args"] = self.to_namespace() - torch.save(tp_state_dict, checkpoint_path) - pbar.update(1) - - def to_namespace(self): - margs = types.SimpleNamespace() - for k, v in self.data.items(): - setattr(margs, k, v) - return margs \ No newline at end of file diff --git a/tools/convert_checkpoint/megatron_checkpoint.py b/tools/convert_checkpoint/megatron_checkpoint.py deleted file mode 100644 index befb224d..00000000 --- a/tools/convert_checkpoint/megatron_checkpoint.py +++ /dev/null @@ -1,880 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import shutil -import torch -import types -from tqdm import tqdm - -from convert_checkpoint.abstact_checkpoint import AbstractCheckpoint -from convert_checkpoint.common_checkpoint import CommonCheckpoint -from convert_checkpoint.megatron_optimizer import MegatronOptimizer, merge_optimizer_by_pp_tp, merge_optimizer_by_dp -from convert_checkpoint.utils import ( - check_path_in_dict, - get_element_from_dict_by_path, - add_embedding_padding, cut_embedding_padding, - transpose_shape0, - partition_balanced -) - -from convert_checkpoint.common_checkpoint import ( - WORD_EMBEDDINGS, - WORD_POSITION_EMBEDDINGS, - WORD_BLOCK_POSITION_EMBEDDINGS, - TRANSFORMER, - LAYER_PREFIX, - INPUT_LAYERNORM, - ATTENTION_ROTARY_EMB_INV_FREQ, - ATTENTION_QUERY_KEY_VALUE, - ATTENTION_DENSE, - POST_ATTENTION_LAYERNORM, - MLP_DENSE_H_TO_4H, - MLP_DENSE_4H_TO_H, - FINAL_LAYERNORM, - WORD_EMBEDDINGS_FOR_HEAD, - WORD_EMBEDDINGS_TPL, - WORD_POSITION_EMBEDDINGS_TPL, - WORD_BLOCK_POSITION_EMBEDDINGS_TPL, - TRANSFORMER_TPL, - WORD_EMBEDDINGS_FOR_HEAD_TPL, -) - - -def get_sharded_states(args, tp_size, pp_size, pp_rank): - """ - Get sharded checkpoints from NVIDIA Megatron-LM checkpoint based on the provided tensor parallel size, pipeline - parallel size and pipeline parallel rank. - - Args: - args (argparse.Namespace): the arguments to the script - tp_size (int): the tensor parallel size - pp_size (int): the pipeline parallel size - pp_rank (int): the pipeline parallel rank - """ - tp_state_dicts = [] - for i in range(tp_size): - sub_dir_name = f"mp_rank_{i:02d}" if pp_size == 1 else f"mp_rank_{i:02d}_{pp_rank:03d}" - checkpoint_name = os.listdir(os.path.join(args.load_path, sub_dir_name))[0] - checkpoint_path = os.path.join(args.load_path, sub_dir_name, checkpoint_name) - state_dict = torch.load(checkpoint_path, map_location="cpu") - tp_state_dicts.append(state_dict) - return tp_state_dicts - - -class MegatronCheckpoint(AbstractCheckpoint): - """ - MegatronCheckpoint - """ - - def __init__(self, num_layers): - super().__init__(num_layers) - self.version = 3.0 - self.iteration = 0 - self.args = types.SimpleNamespace() - self.rng_state = None - - def init_pipeline_size(self, pp, tp, dp, tensor_parallel_dim, stage): - "Initialize tp pp dp" - self.pp = pp - self.tp = tp - self.dp = dp - self.state_dict = [] - self.tensor_parallel_dim = tensor_parallel_dim - self.num_stages = stage or 1 - assert self.num_layers // self.pp % self.num_stages == 0 - for p in range(pp): - self.state_dict.append([]) - for t in range(tp): - self.state_dict[p].append({}) - - def init_optimizer(self, use_distributed_optimizer): - assert self.pp > 0 and self.tp > 0 - self.use_distributed_optimizer = use_distributed_optimizer - self.optim_state_dict = [] - for p in range(self.pp): - self.optim_state_dict.append([]) - for t in range(self.tp): - opt = MegatronOptimizer.generate_optimizer(self, self.num_layers // self.pp, p) - self.optim_state_dict[p].append(opt) - - def init_layers(self): - self.layers = [] - for p in range(self.pp): - self.layers.append(self.get_layers(p)) - - def init_named_parameters_shape(self): - self.named_parameters_shape = self._get_named_parameters_shape() - self.named_parameters_shape_by_p = [None] * self.pp - self.named_parameters_shape_by_t = [None] * self.tp - self.named_parameters_shape_by_pt = [] - for p in range(self.pp): - self.named_parameters_shape_by_p[p] = self._get_named_parameters_shape(pp_rank=p) - for t in range(self.tp): - self.named_parameters_shape_by_t[t] = self._get_named_parameters_shape(tp_rank=t) - for p in range(self.pp): - self.named_parameters_shape_by_pt.append([None] * self.tp) - for t in range(self.tp): - self.named_parameters_shape_by_pt[p][t] = self._get_named_parameters_shape(p, t) - - @staticmethod - def convert_from_common(c_ckpt, c_config): - """ - Convert common checkpoint to megatron checkpoint. - - Args: - c_ckpt: CommonCheckpoint - c_config: CommonConfig - """ - - print("\n==================== Common -> Megatron ====================") - - name_map = c_config.get("name_map")["megatron"] - cargs = c_config.get_args("common") - margs = c_config.get_args("megatron") - dtype = c_config.get_dtype() - tensor_parallel_dim = c_config.get("tensor_parallel_dim") - - tp = margs["tensor_model_parallel_size"] - pp = margs["pipeline_model_parallel_size"] - dp = margs["data_parallel_size"] - num_layers = cargs["num_layers"] - hidden_size = cargs["hidden_size"] - use_distributed_optimizer = margs["use_distributed_optimizer"] - num_layers_per_stage = margs["num_layers_per_virtual_pipeline_stage"] or (num_layers // pp) - stage = num_layers // pp // num_layers_per_stage - - m_ckpt = MegatronCheckpoint(num_layers) - m_ckpt.set_dtype(dtype) - m_ckpt.init_pipeline_size(pp, tp, dp, tensor_parallel_dim, stage) - m_ckpt.set_name_map(name_map) - - layer_prefix = name_map[LAYER_PREFIX] - num_layers_in_rank, _ = partition_balanced(num_layers, pp * stage) - - def _convert(layer_name, transpose_shape=None): - _get_weight_func = { - INPUT_LAYERNORM: c_ckpt.get_layer_input_layernorm_weight, - ATTENTION_QUERY_KEY_VALUE: c_ckpt.get_layer_attention_query_key_value_weight, - ATTENTION_DENSE: c_ckpt.get_layer_attention_dense_weight, - POST_ATTENTION_LAYERNORM: c_ckpt.get_layer_post_attention_layernorm_weight, - MLP_DENSE_H_TO_4H: c_ckpt.get_layer_mlp_dense_h_to_4h_weight, - MLP_DENSE_4H_TO_H: c_ckpt.get_layer_mlp_dense_4h_to_h_weight - }[layer_name] - _get_bias_func = { - INPUT_LAYERNORM: c_ckpt.get_layer_input_layernorm_bias, - ATTENTION_QUERY_KEY_VALUE: c_ckpt.get_layer_attention_query_key_value_bias, - ATTENTION_DENSE: c_ckpt.get_layer_attention_dense_bias, - POST_ATTENTION_LAYERNORM: c_ckpt.get_layer_post_attention_layernorm_bias, - MLP_DENSE_H_TO_4H: c_ckpt.get_layer_mlp_dense_h_to_4h_bias, - MLP_DENSE_4H_TO_H: c_ckpt.get_layer_mlp_dense_4h_to_h_bias - }[layer_name] - weight_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.weight") - bias_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.bias") - for virtual_p in range(pp * stage): - p = virtual_p % pp - stage_index = virtual_p // pp - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = m_ckpt.get_transformer_name(stage_index) - for layer_index in range(num_layers_in_rank[virtual_p]): - name = f"{layer_prefix}.{layer_index}.{name_map[layer_name]}" - layer_id = layer_index + layer_offset - # weight - weight = _get_weight_func(layer_id) - if transpose_shape is not None: - weight = transpose_shape0(weight, *transpose_shape) - m_ckpt.update_tensor(p, weight, transformer, name + ".weight", weight_chunk_dim) - # bias - bias = _get_bias_func(layer_id) - if transpose_shape is not None and bias is not None: - bias = transpose_shape0(bias, *transpose_shape) - m_ckpt.update_tensor(p, bias, transformer, name + ".bias", bias_chunk_dim) - print(f"> {layer_name} chunk weight dim {weight_chunk_dim}, bias dim {bias_chunk_dim}") - - # 1.1 word embeddings with paddding - weight = c_ckpt.get_word_embedding() - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - if padded_vocab_size is None: - assert vocab_size == weight.shape[0] - weight = add_embedding_padding(weight, divisible_by, vocab_size, tp, padded_vocab_size) - - name = m_ckpt.get_word_embedding_name() - chunk_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS}.weight") - m_ckpt.update_tensor(0, weight, name, "weight", chunk_dim) - print(f"> {name} weight {weight.shape}") - - # 1.2 position embedding - if margs.get("add_position_embedding", False): - name = m_ckpt.get_position_embedding_name() - weight = c_ckpt.get_word_position_embedding() - chunk_dim = tensor_parallel_dim.get(f"{WORD_POSITION_EMBEDDINGS}.weight") - m_ckpt.update_tensor(0, weight, name, "weight", chunk_dim) - print(f"> {name} weight {weight.shape}") - - if margs.get("add_block_position_embedding", False): - name = m_ckpt.get_block_position_embedding_name() - weight = c_ckpt.get_word_block_position_embedding() - chunk_dim = tensor_parallel_dim.get(f"{WORD_BLOCK_POSITION_EMBEDDINGS}.weight") - m_ckpt.update_tensor(0, weight, name, "weight", chunk_dim) - print(f"> {name} weight {weight.shape}") - - # 2. transformer layers - # 2.1 input_layernorm - _convert(INPUT_LAYERNORM) - - # 2.2 rotary_emb.inv_freqs - if ATTENTION_ROTARY_EMB_INV_FREQ in name_map: - chunk_dim = tensor_parallel_dim.get(ATTENTION_ROTARY_EMB_INV_FREQ) - for virtual_p in range(pp * stage): - p = virtual_p % pp - stage_index = virtual_p // pp - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = m_ckpt.get_transformer_name(stage_index) - for layer_index in range(num_layers_in_rank[virtual_p]): - name = f"{layer_prefix}.{layer_index}.{name_map[ATTENTION_ROTARY_EMB_INV_FREQ]}" - layer_id = layer_index + layer_offset - inv_freq = c_ckpt.get_layer_attention_rotary_emb_inv_freq(layer_id) - m_ckpt.update_tensor(p, inv_freq, transformer, name, chunk_dim) - print(f"> rotary_emb.inv_freqs chunk dim {chunk_dim}") - - # 2.3 self attention query_key_value - _convert(ATTENTION_QUERY_KEY_VALUE) - - # 2.4 self attention dense - _convert(ATTENTION_DENSE) - - # 2.5 post attention layernorm - _convert(POST_ATTENTION_LAYERNORM) - - # 2.6 mlp dense h_to_4h - if margs.get("transpose_mlp_dense", False): - _convert(MLP_DENSE_H_TO_4H, transpose_shape=(2, tp)) - else: - _convert(MLP_DENSE_H_TO_4H) - - # 2.7 mlp dense 4h_to_h - _convert(MLP_DENSE_4H_TO_H) - - # 2.8 final_layernorm - transformer = m_ckpt.get_transformer_name(m_ckpt.num_stages - 1) - name = name_map[FINAL_LAYERNORM] - # weight - chunk_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.weight") - weight = c_ckpt.get_final_layernorm_weight() - m_ckpt.update_tensor(pp-1, weight, transformer, name + ".weight", chunk_dim) - # bias - chunk_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.bias") - bias = c_ckpt.get_final_layernorm_bias() - m_ckpt.update_tensor(pp-1, bias, transformer, name + ".bias", chunk_dim) - print(f"> {name} weight {weight.shape}") - - # 3 word embedding for head - if margs.get("untie_embeddings_and_output_weights", False) or m_ckpt.pp > 1: - chunk_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS_FOR_HEAD}.weight") - name = m_ckpt.get_word_embedding_for_head_name() - weight = c_ckpt.get_word_embeddings_for_head_weight() - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - orig_vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - weight = add_embedding_padding(weight, divisible_by, orig_vocab_size, tp, padded_vocab_size) - m_ckpt.update_tensor(pp - 1, weight, name, "weight", chunk_dim) - print(f"> {name} weight {weight.shape}") - - # 4. optimizer - m_ckpt.init_layers() - m_ckpt.init_named_parameters_shape() - m_ckpt.init_optimizer(use_distributed_optimizer) - if c_ckpt.has_optimizer(): - opt = MegatronOptimizer.generate_optimizer(m_ckpt, c_ckpt.num_layers) - opt.load(c_ckpt.state_dict["optimizer"]) - if margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - vocab_size = cargs["vocab_size"] - padded_vocab_size = margs.get("pad_vocab_size_to") - opt.add_embedding_padding(divisible_by, vocab_size, tp, hidden_size, padded_vocab_size) - if m_ckpt.pp == 1 and not margs.get("untie_embeddings_and_output_weights", False): - opt.remove_word_embedding_for_head() - opt.build_param_map(m_ckpt.get_named_parameters_shape()) - if stage > 1: - opt.interleave(stage, pp) - opts = opt.chunk_by_pp_tp(pp, tp, margs) - for p in range(pp): - for t in range(tp): - named_parameters_shape = m_ckpt.get_named_parameters_shape(p, t) - opts[p][t].build_param_map(named_parameters_shape) - m_ckpt.optim_state_dict = opts - else: - print("> optimizer empty") - - # 5.others - m_ckpt.iteration = c_ckpt.other_args.get("iteration", m_ckpt.iteration) - m_ckpt.version = c_ckpt.other_args.get("version", m_ckpt.version) - m_ckpt.args = c_ckpt.other_args.get("args", m_ckpt.args) - m_ckpt.rng_state = c_ckpt.other_args.get("rng_state", m_ckpt.rng_state) - - m_ckpt.debug("Finish common -> megatron") - return m_ckpt - - def update_tensor(self, pp, source, layer, key, dim=None): - if source is not None: - if dim is not None: - source = torch.chunk(source, self.tp, dim=dim) - for t in range(self.tp): - element = get_element_from_dict_by_path( - self.state_dict[pp][t], layer - ) - element[key] = (source if dim is None else source[t]) \ - .clone().to(self.dtype) - - def get_tensor(self, pp, layer, key, dim=None): - if dim is not None: - tp_state_dict = [] - for t in range(self.tp): - element = get_element_from_dict_by_path( - self.state_dict[pp][t], layer - ) - if key not in element: - return None - tp_state_dict.append(element[key]) - return torch.cat(tp_state_dict, dim=dim) - else: - element = get_element_from_dict_by_path( - self.state_dict[pp][0], layer - ) - return element[key] if key in element else None - - def convert_to_common(self, c_config): - """ - Convert Megatron checkpoint to common checkpoint. - Args: - c_config: CommonConfig - """ - - print("\n==================== Megatron -> Common ====================") - - tensor_parallel_dim = self.tensor_parallel_dim - name_map = c_config.get_name_map("megatron") - cargs = c_config.get_args("common") - margs = c_config.get_args("megatron") - num_layers = cargs["num_layers"] - hidden_size = cargs["hidden_size"] - num_attention_heads = cargs["num_attention_heads"] - num_layers_per_stage = self.num_layers // self.pp // self.num_stages - - c_ckpt = CommonCheckpoint(num_layers) - c_ckpt.set_dtype(self.dtype) - - layer_prefix = name_map[LAYER_PREFIX] - num_layers_in_rank, _ = partition_balanced(num_layers, self.pp * self.num_stages) - - def _convert(layer_name, transpose_shape=None): - _set_func = { - INPUT_LAYERNORM: c_ckpt.set_layer_input_layernorm, - ATTENTION_QUERY_KEY_VALUE: c_ckpt.set_layer_attention_query_key_value, - ATTENTION_DENSE: c_ckpt.set_layer_attention_dense, - POST_ATTENTION_LAYERNORM: c_ckpt.set_layer_post_attention_layernorm, - MLP_DENSE_H_TO_4H: c_ckpt.set_layer_mlp_dense_h_to_4h, - MLP_DENSE_4H_TO_H: c_ckpt.set_layer_mlp_dense_4h_to_h - }[layer_name] - weight_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.weight") - bias_chunk_dim = tensor_parallel_dim.get(f"{layer_name}.bias") - for virtual_p in range(self.pp * self.num_stages): - p = virtual_p % self.pp - stage_index = virtual_p // self.pp - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = self.get_transformer_name(stage_index) - for layer_index in range(num_layers_in_rank[virtual_p]): - name = f"{layer_prefix}.{layer_index}.{name_map[layer_name]}" - layer_id = layer_index + layer_offset - weight = self.get_tensor(p, transformer, name + ".weight", weight_chunk_dim) - bias = self.get_tensor(p, transformer, name + ".bias", bias_chunk_dim) - if transpose_shape is not None: - weight = transpose_shape0(weight, *transpose_shape) - if bias is not None: - bias = transpose_shape0(bias, *transpose_shape) - _set_func(layer_id, weight, bias) - print(f"> {layer_name} chunk weight dim {weight_chunk_dim}, bias dim {bias_chunk_dim}") - - # 1.1 word_embeddings - parallel_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS}.weight") - name = self.get_word_embedding_name() - weight = self.get_tensor(0, name, 'weight', parallel_dim) - if margs.get("add_embedding_padding", False): - orig_vocab_size = cargs["vocab_size"] - weight = cut_embedding_padding(weight, orig_vocab_size) - c_ckpt.set_word_embedding(weight) - print(f"> word_embeddings weight: {weight.shape}") - - # 1.2 position embedding - if margs.get("add_position_embedding", False): - name = self.get_position_embedding_name() - parallel_dim = tensor_parallel_dim.get(f"{WORD_POSITION_EMBEDDINGS}.weight") - weight = self.get_tensor(0, name, 'weight', parallel_dim) - c_ckpt.set_word_position_embedding(weight) - print(f"add position embedding weight: {weight.shape}") - - if margs.get("add_block_position_embedding", False): - name = self.get_block_position_embedding_name() - parallel_dim = tensor_parallel_dim.get(f"{WORD_BLOCK_POSITION_EMBEDDINGS}.weight") - weight = self.get_tensor(0, name, 'weight', parallel_dim) - c_ckpt.set_word_block_position_embedding(weight) - print(f"> add block position embedding weight: {weight.shape}") - - # 2. transformer layers - - # 2.1 input_layernorm - _convert(INPUT_LAYERNORM) - - # 2.2 rotary_emb.inv_freqs - if ATTENTION_ROTARY_EMB_INV_FREQ in name_map: - chunk_dim = tensor_parallel_dim.get(ATTENTION_ROTARY_EMB_INV_FREQ) - for virtual_p in range(self.pp * self.num_stages): - p = virtual_p % self.pp - stage_index = virtual_p // self.pp - layer_offset = sum(num_layers_in_rank[:virtual_p]) - transformer = self.get_transformer_name(stage_index) - for layer_index in range(num_layers_in_rank[virtual_p]): - name = f"{layer_prefix}.{layer_index}.{name_map[ATTENTION_ROTARY_EMB_INV_FREQ]}" - layer_id = layer_index + layer_offset - inv_freq = self.get_tensor(p, transformer, name, chunk_dim) - c_ckpt.set_layer_attention_rotary_emb_inv_freq(layer_id, inv_freq) - print(f"> rotary_emb.inv_freqs chunk dim {chunk_dim}") - elif margs.get('use_rotary_position_embeddings', False): - dim = hidden_size // num_attention_heads - inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim)) - for virtual_p in range(self.pp * self.num_stages): - p = virtual_p % self.pp - stage_index = virtual_p // self.pp - layer_offset = sum(num_layers_in_rank[:virtual_p]) - for layer_index in range(num_layers_in_rank[virtual_p]): - layer_id = layer_index + layer_offset - c_ckpt.set_layer_attention_rotary_emb_inv_freq(layer_id, inv_freq) - print(f"> rotary_emb.inv_freqs, created by dim {dim}") - - - # 2.3 self attention query_key_value - _convert(ATTENTION_QUERY_KEY_VALUE) - - # 2.4 self attention dense - _convert(ATTENTION_DENSE) - - # 2.5 post attention layernorm - _convert(POST_ATTENTION_LAYERNORM) - - # 2.6 mlp dense h_to_4h - if margs.get("transpose_mlp_dense", False): - _convert(MLP_DENSE_H_TO_4H, transpose_shape=(self.tp, 2)) - else: - _convert(MLP_DENSE_H_TO_4H) - - # 2.7 mlp dense 4h_to_h - _convert(MLP_DENSE_4H_TO_H) - - # 2.8 final_layernorm - transformer = self.get_transformer_name(self.num_stages - 1) - name = name_map[FINAL_LAYERNORM] - - parallel_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.weight") - weight = self.get_tensor(self.pp-1, transformer, name + ".weight", parallel_dim) - - parallel_dim = tensor_parallel_dim.get(f"{FINAL_LAYERNORM}.bias") - bias = self.get_tensor(self.pp-1, transformer, name + ".bias", parallel_dim) - - c_ckpt.set_final_layernorm(weight, bias) - print(f"> final_layernorm weight {weight.shape}") - - # 3 word embedding for head - if margs.get("untie_embeddings_and_output_weights", False) or self.pp > 1: - parallel_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS_FOR_HEAD}.weight") - name = self.get_word_embedding_for_head_name() - else: - parallel_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS}.weight") - name = self.get_word_embedding_name() - weight = self.get_tensor(self.pp - 1, name, 'weight', parallel_dim) - if margs.get("add_embedding_padding", False): - orig_vocab_size = cargs["vocab_size"] - weight = cut_embedding_padding(weight, orig_vocab_size) - c_ckpt.set_word_embeddings_for_head(weight) - print(f"> word embedding for head weight {weight.shape}") - - # 4. optimizer - if self.has_optimizer(): - opt = merge_optimizer_by_pp_tp(self.optim_state_dict, margs) - opt.build_param_map(self.get_named_parameters_shape()) - if margs.get("add_embedding_padding", False): - orig_vocab_size = cargs["vocab_size"] - opt.cut_embedding_padding(orig_vocab_size) - if self.num_stages > 1: - opt.interleave(self.pp, self.num_stages) - if self.pp == 1 and not margs.get("untie_embeddings_and_output_weights", False): - opt.add_word_embedding_for_head() - c_ckpt.state_dict["optimizer"] = opt.to_dict() - print("> optimizer params: ", opt.get_param_num()) - else: - print("> optimizer empty") - - - # 5.others - c_ckpt.other_args["iteration"] = self.iteration - c_ckpt.other_args["version"] = self.version - c_ckpt.other_args["args"] = self.args - c_ckpt.other_args["rng_state"] = self.rng_state - - return c_ckpt - - def set_name_map(self, name_map): - """ set name_map """ - self.name_map = name_map - - def get_word_embedding_name(self): - """ get word_embedding name """ - if self.num_stages > 1: - if WORD_EMBEDDINGS_TPL not in self.name_map: - return None - return self.name_map[WORD_EMBEDDINGS_TPL] % 0 - else: - return self.name_map.get(WORD_EMBEDDINGS) - - def get_position_embedding_name(self): - """ get position_embedding name """ - if self.num_stages > 1 : - if WORD_POSITION_EMBEDDINGS_TPL not in self.name_map: - return None - return self.name_map[WORD_POSITION_EMBEDDINGS_TPL] % 0 - else: - return self.name_map.get(WORD_POSITION_EMBEDDINGS) - - def get_block_position_embedding_name(self): - """ get block_position_embedding name """ - if self.num_stages > 1 : - if WORD_BLOCK_POSITION_EMBEDDINGS_TPL not in self.name_map: - return None - return self.name_map[WORD_BLOCK_POSITION_EMBEDDINGS_TPL] % 0 - else: - return self.name_map.get(WORD_BLOCK_POSITION_EMBEDDINGS) - - def get_transformer_name(self, stage_index): - """ get transformer name """ - if self.num_stages > 1: - return self.name_map[TRANSFORMER_TPL] % stage_index - else: - return self.name_map[TRANSFORMER] - - def get_word_embedding_for_head_name(self): - """ get word_embedding for head name """ - if self.num_stages > 1: - return self.name_map[WORD_EMBEDDINGS_FOR_HEAD_TPL] % (self.num_stages - 1) - else: - return self.name_map[WORD_EMBEDDINGS_FOR_HEAD] - - def has_optimizer(self): - """ whether has optimizer """ - for p in range(self.pp): - for t in range(self.tp): - if self.optim_state_dict[p][t].empty(): - return False - return True - - def has_bias(self, pp_rank=None): - for p in range(self.pp) if pp_rank is None else [pp_rank]: - for prefix, name, _ in self.layers[p]: - if name.endswith("bias"): - return True - return False - - def has_word_embeddings(self, p=None): - """ whether has word embeddings """ - p = 0 if p is None else p - for prefix, name, _ in self.layers[p]: - if prefix == self.get_word_embedding_name() and "weight" == name: - return True - return False - def has_position_embeddings(self, p=None): - """ whether has position embeddings """ - p = 0 if p is None else p - for prefix, name, _ in self.layers[p]: - if prefix == self.get_position_embedding_name() and "weight" == name: - return True - return False - - def has_block_position_embeddings(self, p=None): - """ whether has block position embeddings """ - p = 0 if p is None else p - for prefix, name, _ in self.layers[p]: - if prefix == self.get_block_position_embedding_name() and "weight" == name: - return True - return False - - def has_word_embeddings_for_head(self, p=None): - p = self.pp-1 if p is None else p - for prefix, name, _ in self.layers[p]: - if self.get_word_embedding_for_head_name() == prefix and "weight" == name: - return True - return False - - def has_final_layernorm(self, key, p=None): - p = self.pp-1 if p is None else p - for prefix, name, _ in self.layers[p]: - if f"{self.name_map[FINAL_LAYERNORM]}.{key}" == name: - return True - return False - - def get_layers(self, pp_rank=None): - """ get all layers """ - layers = [] - tensor_parallel_dim = self.tensor_parallel_dim - - # embedding - if pp_rank == 0 or pp_rank is None: - keys = (WORD_EMBEDDINGS, \ - WORD_POSITION_EMBEDDINGS, \ - WORD_BLOCK_POSITION_EMBEDDINGS) - names = [self.get_word_embedding_name(), \ - self.get_position_embedding_name(), \ - self.get_block_position_embedding_name()] - for i in range(3): - if names[i] is not None and check_path_in_dict(self.state_dict[0][0], names[i]): - chunk_dim = tensor_parallel_dim.get(f"{keys[i]}.weight", -1) - layers.append((names[i], "weight", chunk_dim)) - - # transformers - TRANSFORMER_LAYERS = [INPUT_LAYERNORM, ATTENTION_QUERY_KEY_VALUE, ATTENTION_DENSE, \ - POST_ATTENTION_LAYERNORM, MLP_DENSE_H_TO_4H, MLP_DENSE_4H_TO_H, ] - num_layers_per_pp = self.num_layers // self.pp - num_layers_per_stage = num_layers_per_pp // self.num_stages - for p in range(self.pp) if pp_rank is None else [pp_rank]: - layer_prefix = self.name_map.get(LAYER_PREFIX) - for layer_id in range(num_layers_per_pp): - layer_prefix = self.name_map.get(LAYER_PREFIX) - stage_index = layer_id // num_layers_per_stage - layer_index = layer_id % num_layers_per_stage - transformer_name = self.get_transformer_name(stage_index) - transformer = get_element_from_dict_by_path(self.state_dict[p][0], transformer_name) - for layer in TRANSFORMER_LAYERS: - for w in ("weight", "bias"): - layer_name = f"{layer_prefix}.{layer_index}.{self.name_map[layer]}.{w}" - if layer_name in transformer.keys(): - chunk_dim = tensor_parallel_dim.get(f"{layer}.{w}", -1) - layers.append((transformer_name, layer_name, chunk_dim)) - for layer in [ATTENTION_ROTARY_EMB_INV_FREQ]: - if layer not in self.name_map: - continue - layer_name = f"{layer_prefix}.{layer_index}.{self.name_map[layer]}" - if layer_name in transformer.keys(): - chunk_dim = self.tensor_parallel_dim.get(f"{layer}", -1) - layers.append((transformer_name, layer_name, chunk_dim)) - - transformer_name = self.get_transformer_name(self.num_stages - 1) - transformer = get_element_from_dict_by_path(self.state_dict[p][0], transformer_name) - for layer in [FINAL_LAYERNORM]: - for w in ("weight", "bias"): - layer_name = f"{self.name_map[layer]}.{w}" - if layer_name in transformer.keys(): - chunk_dim = tensor_parallel_dim.get(f"{layer}.{w}", -1) - layers.append((transformer_name, layer_name, chunk_dim)) - - # emebdding for head - if pp_rank == self.pp-1 or pp_rank is None: - layer = self.get_word_embedding_for_head_name() - chunk_dim = tensor_parallel_dim.get(f"{WORD_EMBEDDINGS_FOR_HEAD}.weight", -1) - layers.append((layer, "weight", chunk_dim)) - - return layers - - def get_transformer_layers(self, pp_rank, layer_index): - """ get transformer layers """ - layers = [] - pp_rank = 0 if pp_rank is None else pp_rank - transformer_name = self.get_transformer_name(0) - for meta in self.get_named_parameters_shape(pp_rank): - layer_name = meta[0] - key = f"{transformer_name}.{self.name_map[LAYER_PREFIX]}.{layer_index}." - if key in layer_name: - layers.append(meta) - # print(pp_rank, layer_index, layers) - return layers - - def get_named_parameters_shape(self, p=None, t=None): - if p == None and t == None: - return self.named_parameters_shape - if p != None and t != None: - return self.named_parameters_shape_by_pt[p][t] - if p != None: - return self.named_parameters_shape_by_p[p] - if t != None: - return self.named_parameters_shape_by_t[t] - - def _get_named_parameters_shape(self, pp_rank=None, tp_rank=None): - """ return list of (layer_name, tensor_shape, parallel_dim) """ - result = [] - for p in range(self.pp) if pp_rank is None else [pp_rank]: - for layer, key, parallel_dim in self.layers[p]: - if key.endswith("weight") or key.endswith("bias"): - element = get_element_from_dict_by_path( - self.state_dict[p][0], layer - ) - if key in element: - shape = element[key].shape - if tp_rank is None and parallel_dim >= 0: - shape = list(shape) - shape[parallel_dim] *= self.tp - shape = torch.Size(shape) - result.append((f"{layer}.{key}", shape, parallel_dim)) - return result - - def load(self, load_path, m_config, name_map, load_optimizer=True): - """ - Load megatron checkpoint from checkpoints folder. - - Args: - load_path (str): the path to checkpoint - m_config: megatron m_config loaded from ckpt - """ - - tp = m_config.get("tensor_model_parallel_size") - pp = m_config.get("pipeline_model_parallel_size") - dp = m_config.get("data_parallel_size") - dtype = m_config.get("dtype") - tensor_parallel_dim = m_config.get("tensor_parallel_dim") - num_layers_per_stage = m_config.get('num_layers_per_virtual_pipeline_stage') - stage = None if num_layers_per_stage is None else (self.num_layers // pp // num_layers_per_stage) - use_distributed_optimizer = m_config.get("use_distributed_optimizer", False) - self.set_dtype(dtype) - self.init_pipeline_size(pp, tp, dp, tensor_parallel_dim, stage) - self.set_name_map(name_map) - - # weight and bias - pbar = tqdm(range(self.tp * self.pp), desc='Loading Megatron-LM Checkpoint', leave=False) - for p in range(self.pp): - for t in range(self.tp): - sub_dir_name = f"mp_rank_{t:02d}" if self.pp == 1 \ - else f"mp_rank_{t:02d}_{p:03d}" - checkpoint_name = os.listdir(os.path.join(load_path, sub_dir_name))[0] - checkpoint_path = os.path.join(load_path, sub_dir_name, checkpoint_name) - self.state_dict[p][t] = torch.load(checkpoint_path, map_location="cpu") - pbar.update(1) - self.iteration = self.state_dict[0][0].get('iteration', 0) - self.version = self.state_dict[0][0]['checkpoint_version'] - self.args = self.state_dict[0][0]['args'] - self.rng_state = self.state_dict[0][0].get('rng_state', None) - self.init_layers() - self.init_named_parameters_shape() - self.init_optimizer(use_distributed_optimizer) - - # optimizer - if load_optimizer: - if use_distributed_optimizer: - for p in range(self.pp): - for t in range(self.tp): - opts = [] - named_parameters_shape = self.get_named_parameters_shape(p, t) - for d in range(self.dp): - if self.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}" - checkpoint_dir = os.path.join(load_path, checkpoint_dir) - checkpoint_path = os.path.join(checkpoint_dir, "optim.pt") - optim_state_dict = torch.load(checkpoint_path, map_location="cpu") - opt = MegatronOptimizer.generate_optimizer(self, self.num_layers // self.pp, p) - opt.load(optim_state_dict) - opts.append(opt) - opt.debug(f"tp/pp/dp rank: {t}/{p}/{d}, load from: {checkpoint_path}") - self.optim_state_dict[p][t] = merge_optimizer_by_dp(opts, named_parameters_shape) - self.optim_state_dict[p][t].debug(f"merge by dp {self.dp} in pp/tp rank {p}/{t}") - - else: - for p in range(self.pp): - for t in range(self.tp): - if "optimizer" in self.state_dict[p][t]: - named_parameters_shape = self.get_named_parameters_shape(p, t) - self.optim_state_dict[p][t].load(self.state_dict[p][t], named_parameters_shape) - self.debug("==================== megatron checkpoint loaded ================================") - - def save(self, save_path, m_config=None, save_optim=True): - """ save megatron checkpoint """ - os.makedirs(save_path, exist_ok=True) - # Saving the tracker file - tracker_filepath = os.path.join(save_path, "latest_checkpointed_iteration.txt") - with open(tracker_filepath, "w") as f: - f.write(str(self.iteration or "release")) - - # create `release` dir in args.load_path - folder_name = f"iter_{self.iteration:07d}" if self.iteration > 0 else "release" - release_dir = os.path.join(save_path, folder_name) - os.makedirs(release_dir, exist_ok=True) - - # megatron config - margs = self.args - if m_config is not None: - for k, v in m_config.data.items(): - setattr(margs, k, v) - print(f"Saving megatron args {margs}") - - # weight and bias - for p in range(self.pp): - for t in range(self.tp): - self.state_dict[p][t]["checkpoint_version"] = self.version - checkpoint_dir = ( - f"mp_rank_{t:02d}" - if self.pp == 1 - else f"mp_rank_{t:02d}_{p:03d}" - ) - - if self.use_distributed_optimizer: - checkpoint_name = "model_rng.pt" - else: - self.state_dict[p][t].update(self.optim_state_dict[p][t].to_dict()) - checkpoint_name = "model_optim_rng.pt" - if margs is not None: - self.state_dict[p][t]['args'] = margs - if self.rng_state is not None: - self.state_dict[p][t]['rng_state'] = self.rng_state - self.state_dict[p][t]["iteration"] = self.iteration - checkpoint_dir = os.path.join(release_dir, checkpoint_dir) - os.makedirs(checkpoint_dir, exist_ok=True) - checkpoint_path = os.path.join(checkpoint_dir, checkpoint_name) - torch.save(self.state_dict[p][t], checkpoint_path) - print(f"Saving megatron checkpoint {self.state_dict[p][t].keys()} to: {checkpoint_path}") - - # optimizer - if self.use_distributed_optimizer and save_optim: - for p in range(self.pp): - for t in range(self.tp): - chunk_optimers = self.optim_state_dict[p][t].chunk_by_dp(self.dp, self.num_stages) - for d in range(self.dp): - if self.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}" - - checkpoint_dir = os.path.join(release_dir, checkpoint_dir) - os.makedirs(checkpoint_dir, exist_ok=True) - checkpoint_path = os.path.join(checkpoint_dir, "optim.pt") - torch.save( - chunk_optimers[d].to_dict(), - checkpoint_path, - ) - chunk_optimers[d].debug(f"tp/pp/dp rank {t}/{p}/{d}, saved to: {checkpoint_path}") - - def debug(self, title): - """ debbug """ - print(f"\n【Megatron】{title}") - print(f"-> tp/pp/dp size: {self.tp}/{self.pp}/{self.dp}") - if self.has_optimizer(): - for t in range(self.tp): - for p in range(self.pp): - self.optim_state_dict[p][t].debug(f"tp/pp rank {t}/{p}") - print("\n") - -if __name__ == "__main__": - pass diff --git a/tools/convert_checkpoint/megatron_config.py b/tools/convert_checkpoint/megatron_config.py deleted file mode 100644 index 1938c09f..00000000 --- a/tools/convert_checkpoint/megatron_config.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os -import torch -import types -from tqdm import tqdm -import pprint - -from convert_checkpoint.abstact_config import AbstractConfig -from convert_checkpoint.common_config import CommonConfig - - -class MegatronConfig(AbstractConfig): - """ - MegatronConfig - """ - - def __init__(self): - super().__init__() - - @staticmethod - def convert_from_common(c_config): - """ - return megatron config converted from common config - """ - config = MegatronConfig() - cargs = c_config.get_args("common") - margs = c_config.get_args("megatron") - config.update(cargs) - config.update(margs) - - num_layers_per_stage = margs["num_layers_per_virtual_pipeline_stage"] - if num_layers_per_stage is not None: - num_layers = cargs["num_layers"] - pp = margs["pipeline_model_parallel_size"] - stage = num_layers // pp // num_layers_per_stage - config.update({"virtual_pipeline_model_parallel_size": stage}) - return config - - def load(self, load_path): - """ load config """ - sub_dirs = os.listdir(load_path) - possible_sub_dirs = ["mp_rank_00", "mp_rank_00_000"] - rank0_checkpoint_name = None - rank0_checkpoint_path = None - for sub_dir in possible_sub_dirs: - if sub_dir in sub_dirs: - rank0_checkpoint_name = os.listdir(os.path.join(load_path, sub_dir))[0] - rank0_checkpoint_path = os.path.join(load_path, sub_dir, rank0_checkpoint_name) - break - print(f"Loading Megatron-LM config from: {rank0_checkpoint_path}") - - rank0_state_dict = torch.load(rank0_checkpoint_path, map_location="cpu") - if "args" not in rank0_state_dict: - raise ValueError( - "Megatron-LM checkpoint does not contain arguments. This utility only supports Megatron-LM checkpoints" - " containing all the megatron arguments. This is because it loads all config related to model" - " architecture, the tensor and pipeline model parallel size from the checkpoint insead of user having to" - " manually specify all the details. Please save Megatron-LM checkpoint along with all the megatron" - " arguments to use this utility." - ) - self.data = vars(rank0_state_dict["args"]) - - def save(self, save_path): - """ save config """ - - release_dir = os.path.join(save_path, "release") - os.makedirs(release_dir, exist_ok=True) - - tp = self.get("tensor_model_parallel_size") - pp = self.get("pipeline_model_parallel_size") - pbar = tqdm(range(tp*pp), desc='Saving Megatron-LM Config') - for p in range(pp): - for t in range(tp): - sub_dir_name = f"mp_rank_{t:02d}" if pp == 1 \ - else f"mp_rank_{t:02d}_{p:03d}" - checkpoint_name = os.listdir(os.path.join(release_dir, sub_dir_name))[0] - checkpoint_path = os.path.join(release_dir, sub_dir_name, checkpoint_name) - tp_state_dict = torch.load(checkpoint_path, map_location="cpu") - tp_state_dict["args"] = self.to_namespace() - torch.save(tp_state_dict, checkpoint_path) - pbar.update(1) - - def to_namespace(self): - margs = types.SimpleNamespace() - for k, v in self.data.items(): - setattr(margs, k, v) - return margs diff --git a/tools/convert_checkpoint/megatron_optimizer.py b/tools/convert_checkpoint/megatron_optimizer.py deleted file mode 100644 index 87c68ffd..00000000 --- a/tools/convert_checkpoint/megatron_optimizer.py +++ /dev/null @@ -1,844 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import torch -import numpy -import math -from copy import deepcopy - -from convert_checkpoint.utils import add_embedding_padding, cut_embedding_padding, transpose_shape0 - - -def get_param_size_per_dp(total, dp): - """ split params to each dp """ - size = math.ceil(total / dp) - return [size] * (dp - 1) + [total - (dp - 1) * size] - - -def get_group_id(name, shape): - return 1 if name.endswith('bias') or len(shape) == 1 else 0 - - -def split_optimer_param(params, index): - """ - param { - key1: tensor - key2: tensor - } - """ - result = ({}, {}) - for key, value in params.items(): - result[0][key] = value[:index].clone() - result[1][key] = value[index:].clone() - return result - - -def cat_optimer_param(params): - """ - param { - key1: tensor - key2: tensor - } - """ - result = {} - for k in params[0].keys(): - result[k] = torch.cat([param[k] for param in params], dim=0) - return result - - -def merge_optimizer_by_pp_tp(optimizers, args): - """ merge optimizers by pp tp """ - num_layers = sum([opts[0].num_layers for opts in optimizers]) - result = optimizers[0][0].clone_without_param(num_layers, end_offset=optimizers[-1][0].end_offset) - - # state - if optimizers[0][0].get_state_num() > 0: - state_keys = optimizers[0][0].get_keys() - print("merge_optimizer by pp tp, state keys: ", state_keys) - for group_id in range(2): - for opts in optimizers: - param_ids = opts[0].get_param_ids_in_groups()[group_id] - for param_id in param_ids: - name, shape, parallel_dim, _ = opts[0].param_map[param_id] - if parallel_dim < 0: - param = opts[0].get_state(param_id) - else: - param = {} - for key in state_keys: - data = [] - for opt in opts: - value = opt.get_state(param_id, key).view(shape) - data.append(value) - data = torch.cat(data, dim=parallel_dim) - if "dense_h_to_4h" in name and args.get("transpose_mlp_dense", False): - tp = len(opts) - data = transpose_shape0(data, tp, 2) - if opts[0].use_distributed_optimizer: - param[key] = data.view(-1) - else: - param[key] = data - result.add_state_in_group(param, group_id) - - # shard_fp32_from_float16_groups - if optimizers[0][0].get_shard_fp32_num() > 0: - print("merge_optimizer by pp tp, shard_fp32_from_float16_groups ") - for opts in optimizers: - param_groups = opts[0].get_param_ids_in_groups() - for group_id, param_ids in enumerate(param_groups): - for index, param_id in enumerate(param_ids): - name, shape, parallel_dim, _ = opts[0].param_map[param_id] - if parallel_dim < 0: - param = opts[0].get_shard_fp32_in_group(group_id, index) - else: - data = [] - for opt in opts: - value = opt.get_shard_fp32_in_group(group_id, index).view(shape) - data.append(value) - data = torch.cat(data, dim=parallel_dim) - if "dense_h_to_4h" in name and args.get("transpose_mlp_dense", False): - tp = len(opts) - data = transpose_shape0(data, tp, 2) - if opts[0].use_distributed_optimizer: - param = data.view(-1) - else: - param = data - result.add_shard_fp32(param, group_id) - result.update_param_groups_by_shard() - - result.debug(f"merge by pp{len(optimizers)} tp{len(optimizers[0])}") - return result - - -def check_split_group(named_parameters_shape, params_per_dp): - """ check which params group will be split when zero1 enable - returns: - 0: split in group0 - 1: split in group1 - -1: no split needed - """ - result = [] - cnt = 0 - d = len(params_per_dp) - 1 - for name, shape, _ in named_parameters_shape: - param_size = numpy.prod(shape) - while True: - if cnt + param_size > params_per_dp[d]: - group_id = get_group_id(name, shape) - result.append(group_id) - param_size -= params_per_dp[d] - cnt - cnt = 0 - d -= 1 - elif cnt + param_size == params_per_dp[d]: - result.append(-1) - cnt = 0 - d -= 1 - break - else: - cnt += param_size - break - assert cnt == 0 and d == -1 - return list(reversed(result)) - - -def get_named_parameters_shape_by_stage(named_parameters_shape, stage): - """ get named_parameters_shape by stage """ - return [meta for meta in named_parameters_shape if meta[0].startswith(f"model{stage}")] - - -def get_param_shape_in_groups(named_parameters_shape, params_per_dp): - """ split named_parameters_shape by dp """ - result = [[[], []]] - cnt = 0 - d = len(params_per_dp) - 1 - for meta in named_parameters_shape: - name = meta[0] - shape = meta[1] - param_size = numpy.prod(shape) - group_id = get_group_id(name, shape) - while True: - if cnt + param_size > params_per_dp[d]: - result[-1][group_id].append(params_per_dp[d] - cnt) - param_size -= params_per_dp[d] - cnt - cnt = 0 - d -= 1 - result.append([[], []]) - elif cnt + param_size == params_per_dp[d]: - result[-1][group_id].append(param_size) - cnt = 0 - d -= 1 - result.append([[], []]) - break - else: - result[-1][group_id].append(param_size) - cnt += param_size - break - assert cnt == 0 and d == -1 - return list(reversed(result[:-1])) - - -def merge_optimizer_by_dp(optimizers, named_parameters_shape): - """ merging optimizers by dp """ - new_optimizer = next(opt for opt in optimizers if opt.get_param_groups_num() == 2).clone_without_param() - opt_params_size = sum([opt.get_param_size() for opt in optimizers]) - model_params = sum([numpy.prod(shape) for name, shape, dim in named_parameters_shape]) - if opt_params_size == 0: - return new_optimizer - else: - assert opt_params_size == model_params - dp = len(optimizers) - num_stages = len(set([meta[0].split('.')[0] for meta in named_parameters_shape])) - cnt = [[0, 0] for _ in range(dp)] - new_param_id = 0 - - for group_id in range(2): - for stage in range(num_stages): - if num_stages > 1: - _named_parameters_shape = get_named_parameters_shape_by_stage(named_parameters_shape, stage) - else: - _named_parameters_shape = named_parameters_shape - total_params_size = sum([numpy.prod(shape) for name, shape, dim in _named_parameters_shape]) - params_per_dp = get_param_size_per_dp(total_params_size, dp) - which_group_split = check_split_group(_named_parameters_shape, params_per_dp) - assert len(which_group_split) == dp - - shape_in_group = get_param_shape_in_groups(_named_parameters_shape, params_per_dp) - - for d in range(dp - 1, -1, -1): - param_groups = optimizers[d].get_param_ids_in_groups() - if len(param_groups) <= group_id: - continue - start = cnt[d][group_id] - end = start + len(shape_in_group[d][group_id]) - for index, param_id in enumerate(param_groups[group_id][start:end]): - # state - state = optimizers[d].get_state(param_id) - assert list(state.values())[0].shape == shape_in_group[d][group_id][index] - if new_optimizer.has_param(new_param_id): - last_state = new_optimizer.get_state(new_param_id) - state = cat_optimer_param([state, last_state]) - new_optimizer.update_state(state, group_id, new_param_id) - else: - new_optimizer.add_state_in_group(state, group_id) - # shard_fp32 - shard_fp32 = optimizers[d].get_shard_fp32_by_param_id(param_id) - last_shard_fp32 = new_optimizer.get_shard_fp32_by_param_id(new_param_id) - if last_shard_fp32 is None: - new_optimizer.add_shard_fp32(shard_fp32, group_id) - else: - shard_fp32 = torch.cat([shard_fp32, last_shard_fp32], dim=0) - new_optimizer.update_shard_fp32(shard_fp32, new_param_id) - new_param_id += 1 - # If the param has been split, it need to be merged with next dp rank - if group_id == which_group_split[d]: - new_param_id -= 1 - cnt[d][group_id] = end - new_optimizer.build_param_map(named_parameters_shape) - return new_optimizer - - -class MegatronOptimizer(): - """ Megatron optimizer """ - def __init__(self, num_layers, start_offset, end_offset, params_num_per_layer): - """ - num_laerners (int): number of layers in pp rank - start_offset (int): start offset of attension params - end_offset (int): end offset of attension params - params_num_per_layer (list): number of group params per layer - """ - self.num_layers = num_layers - self.start_offset = start_offset - self.end_offset = end_offset - self.params_num_per_layer = params_num_per_layer - self.state = {} - self.param_groups = [{}, {}] - self.grad_scaler = {} - self.shard_fp32 = [[], []] - self.param_scheduler = {} - self.use_distributed_optimizer = False - - @ staticmethod - def generate_optimizer(model, num_layers, pp_rank=None): - """ geneate megatron optimizer """ - start_offset = 0 - end_offset = 0 - params_num_per_layer = [0, 0] # [group0, group1] - for name, shape, _ in model.get_transformer_layers(pp_rank, 0): - if name.endswith('weight'): - if len(shape) == 1: - params_num_per_layer[1] += 1 - else: - params_num_per_layer[0] += 1 - elif name.endswith('bias'): - params_num_per_layer[1] += 1 - else: - pass - if model.has_word_embeddings(pp_rank): - start_offset += 1 - if model.has_position_embeddings(pp_rank): - start_offset += 1 - if model.has_block_position_embeddings(pp_rank): - start_offset += 1 - if model.has_final_layernorm("weight", pp_rank): - end_offset += 1 - if model.has_final_layernorm("bias", pp_rank): - end_offset += 1 - if model.has_word_embeddings_for_head(pp_rank): - end_offset += 1 - opt = MegatronOptimizer(num_layers, start_offset, end_offset, params_num_per_layer) - opt.use_distributed_optimizer = model.use_distributed_optimizer - return opt - - def clone_without_param(self, num_layers=None, start_offset=None, end_offset=None): - """ clone without params """ - result = MegatronOptimizer(self.num_layers, self.start_offset, self.end_offset, self.params_num_per_layer) - result.use_distributed_optimizer = self.use_distributed_optimizer - result.param_groups = deepcopy(self.param_groups) - for group in result.param_groups: - group["params"] = [] - result.shard_fp32 = [[] for group in self.shard_fp32] - result.grad_scaler = deepcopy(self.grad_scaler) - result.param_scheduler = deepcopy(self.param_scheduler) - if num_layers is not None: - result.num_layers = num_layers - if start_offset is not None: - result.start_offset = start_offset - if end_offset is not None: - result.end_offset = end_offset - return result - - def add_word_embedding_for_head(self): - """ input: [0,1,2,3] [4,5,6,7,8,9] """ - """ output: [0,1,2,3,4] [5,6,7,8,9,10] """ - len1 = len(self.param_groups[0]['params']) - len2 = len(self.param_groups[1]['params']) - self.param_groups[0]['params'] = list(range(len1 + 1)) - self.param_groups[1]['params'] = list(range(len1 + 1, len1 + len2 + 1)) - for i in range(len2 + len1, len1, -1): - self.state[i] = self.state[i - 1] - self.state[len1] = deepcopy(self.state[0]) - self.shard_fp32[0].append(self.shard_fp32[0][0].clone()) - - def remove_word_embedding_for_head(self): - """ input: [0,1,2,3,4] [5,6,7,8,9,10] """ - """ output: [0,1,2,3] [4,5,6,7,8,9] """ - len1 = len(self.param_groups[0]['params']) - len2 = len(self.param_groups[1]['params']) - self.param_groups[0]['params'] = list(range(len1 - 1)) - self.param_groups[1]['params'] = list(range(len1 - 1, len1 + len2 - 1)) - for i in range(len1 - 1, len1 + len2 - 1): - self.state[i] = self.state[i + 1] - del self.state[len1 + len2 - 1] - self.shard_fp32[0].pop() - - def get_state_in_model_order(self): - """ Optimizer parameters are grouped by shape and name, such as: - group 0: emb_w、qkv_w、dense_w、h_to_4h_w、4h_to_h_w - group 1: norm_w、norm_b、qkv_b、dense_b、norm_w、norm_b、h_to_4h_b、4h_to_h_b、final_norm_w、final_norm_b - - return list of (param_id, group_id, name, size) in the order of model structure - """ - - result = [None] * len(self.param_map) - for param_id, meta in self.param_map.items(): - name, shape, _, index = meta - group_id = get_group_id(name, shape) - result[index] = (param_id, group_id, name, numpy.prod(shape)) - assert None not in result - return result - - def chunk_by_dp(self, dp, num_stages): - """ chuck by dp """ - if self.empty(): - return [deepcopy(self) for i in range(dp)] - - result = [self.clone_without_param() for i in range(dp)] - - if self.get_state_num() > 0: - assert self.get_state_num() == self.get_param_num() - for stage in range(num_stages): - param_groups = [[], []] - cnt = 0 - d = dp - 1 - model_prefix = f"model{stage}" if num_stages > 1 else "model" - state = [meta for meta in self.get_state_in_model_order() if meta[2].startswith(model_prefix)] - total_params_size = sum([meta[3] for meta in state]) - params_size_per_dp = get_param_size_per_dp(total_params_size, dp) - for param_id, group_id, name, _ in state: - param = self.get_state(param_id) - - while True: - param_size = numpy.prod(list(param.values())[0].shape) - if cnt + param_size > params_size_per_dp[d]: - offset = params_size_per_dp[d] - cnt - param, _ = split_optimer_param(param, -offset) - param_groups[group_id].append(_) - result[d].add_param_groups(param_groups) - param_groups = [[], []] - cnt = 0 - d -= 1 - elif cnt + param_size == params_size_per_dp[d]: - param_groups[group_id].append(param) - result[d].add_param_groups(param_groups) - param_groups = [[], []] - cnt = 0 - d -= 1 - break - else: - param_groups[group_id].append(param) - cnt += param_size - break - assert d == -1 and cnt == 0, f"{d} {cnt}" - - if self.get_shard_fp32_num() > 0: - assert self.get_shard_fp32_num() == self.get_param_num() - assert self.get_shard_fp32_size() == self.get_param_size(), f"{self.get_shard_fp32_size()} {self.get_param_size()}" - for stage in range(num_stages): - cnt = 0 - d = dp - 1 - model_prefix = f"model{stage}" if num_stages > 1 else "model" - state = [meta for meta in self.get_state_in_model_order() if meta[2].startswith(model_prefix)] - total_params_size = sum([meta[3] for meta in state]) - params_size_per_dp = get_param_size_per_dp(total_params_size, dp) - for param_id, group_id, name, _ in state: - shard_fp32 = self.get_shard_fp32_by_param_id(param_id) - while True: - shard_fp32_size = numpy.prod(shard_fp32.shape) - if cnt + shard_fp32_size > params_size_per_dp[d]: - offset = params_size_per_dp[d] - cnt - result[d].add_shard_fp32(shard_fp32[-offset:], group_id) - shard_fp32 = shard_fp32[:-offset] - cnt = 0 - d -= 1 - elif cnt + shard_fp32_size == params_size_per_dp[d]: - result[d].add_shard_fp32(shard_fp32, group_id) - cnt = 0 - d -= 1 - break - else: - result[d].add_shard_fp32(shard_fp32, group_id) - cnt += shard_fp32_size - break - assert d == -1 and cnt == 0 - - for opt in result: - opt.update_param_groups_by_shard() - - return result - - def chunk_by_pp_tp(self, pp, tp, args): - """ chunk by pp tp """ - self.debug(f"chunk by pp{pp} tp{tp}") - num_layers_per_pp = self.num_layers // pp - opts = [] - for p in range(pp): - opts.append([]) - for t in range(tp): - start_offset = self.start_offset if p == 0 else 0 - end_offset = self.end_offset if p == pp-1 else 0 - opt = self.clone_without_param(num_layers_per_pp, start_offset, end_offset) - opts[p].append(opt) - - if self.get_state_num() > 0: - assert self.get_state_num() == self.get_param_num() - state_range = self.get_state_range(pp) - print(f"chunk optimizer by pp{pp} tp{tp}, state range: {state_range}") - for p in range(pp): - for group_id, param_list in enumerate(state_range[p]): - for param_id in param_list: - chunk_param = [{} for t in range(tp)] - for key, value in self.state[param_id].items(): - name, shape, parallel_dim, _ = self.param_map[param_id] - if "dense_h_to_4h" in name and args.get("transpose_mlp_dense", False): - value = transpose_shape0(value.view(shape), 2, tp) - if self.use_distributed_optimizer: - value = value.view(-1) - if parallel_dim < 0: - for t in range(tp): - chunk_param[t][key] = value - else: - chunk_state = value.view(shape).chunk(tp, dim=parallel_dim) - del value - for t in range(tp): - chunk_param[t][key] = chunk_state[t].clone() - if self.use_distributed_optimizer: - chunk_param[t][key] = chunk_param[t][key].reshape(-1) - for t in range(tp): - opts[p][t].add_state(chunk_param[t]) - for t in range(tp): - assert opts[p][t].get_state_num() == opts[p][t].get_param_num() - - if self.get_shard_fp32_num() > 0: - assert self.get_shard_fp32_num() == self.get_param_num() - param_id = 0 - for group_id, shard in enumerate(self.shard_fp32): - for index, data in enumerate(shard): - p = (index-self.start_offset) // (self.params_num_per_layer[0]*num_layers_per_pp) if group_id == 0 \ - else index // (self.params_num_per_layer[1] * num_layers_per_pp) - p = max(p, 0) - p = min(p, pp-1) - - name, shape, parallel_dim, _ = self.param_map[param_id] - if "dense_h_to_4h" in name and args.get("transpose_mlp_dense", False): - data = transpose_shape0(data.view(shape), 2, tp) - if self.use_distributed_optimizer: - data = data.view(-1) - if parallel_dim < 0: - for t in range(tp): - if len(opts[p][t].shard_fp32) <= group_id: - opts[p][t].shard_fp32.append([]) - opts[p][t].shard_fp32[group_id].append(data) - else: - chunk_shard_fp32 = data.view(shape).chunk(tp, dim=parallel_dim) - del data - for t in range(tp): - if len(opts[p][t].shard_fp32) <= group_id: - opts[p][t].shard_fp32.append([]) - data = chunk_shard_fp32[t].clone() - if self.use_distributed_optimizer: - data = data.reshape(-1) - opts[p][t].shard_fp32[group_id].append(data) - param_id += 1 - - for p in range(pp): - for t in range(tp): - opts[p][t].update_param_groups_by_shard() - - for p in range(pp): - for t in range(tp): - assert opts[p][t].get_param_size() == opts[p][t].get_shard_fp32_size() - - return opts - - def get_shard_fp32_by_param_id(self, param_id): - """ get shard fp32 by param id """ - shard_fp32 = sum(self.shard_fp32, []) - return shard_fp32[param_id] if len(shard_fp32) > param_id else None - - def get_shard_fp32_in_group(self, group_id, index): - """" get shard fp32 by group id and index""" - return self.shard_fp32[group_id][index] - - def update_shard_fp32(self, param, param_id): - """" update shard fp32 """ - group_id = 0 - for group in self.shard_fp32: - if param_id >= len(group): - param_id -= len(group) - group_id += 1 - else: - break - self.shard_fp32[group_id][param_id] = param - - def add_shard_fp32(self, param, group_id): - """ add optimizer shard fp32 """ - self.shard_fp32[group_id].append(param.clone()) - - def set_param_groups(self, param_groups): - """ set param_groups """ - self.state = {} - for idx, param_group in enumerate(param_groups): - offset = len(self.state) - self.param_groups[idx]["params"] = list(range(offset, offset + len(param_group))) - for i, param in enumerate(param_group): - self.state[offset+i] = param - - def get_param_groups(self): - """ get param_groups """ - return [[self.state[param_id] for param_id in param_group.get("params", [])] \ - for param_group in self.param_groups] - - - def add_param_groups(self, param_groups): - """ add param_group """ - assert len(param_groups) <= len(self.param_groups) - new_param_groups = [origin_param_group + param_groups[idx] \ - for idx, origin_param_group in enumerate(self.get_param_groups())] - self.set_param_groups(new_param_groups) - - def update_param_groups_by_shard(self): - """ update parms group by shard """ - offset = 0 - for group_id, group in enumerate(self.shard_fp32): - self.param_groups[group_id]["params"] = list(range(offset, offset + len(group))) - offset += len(group) - - def add_state(self, param): - """ add optimizer param """ - param_id, group_id = self.new_param() - self.param_groups[group_id]["params"].append(param_id) - self.state[param_id] = param - - def add_state_in_group(self, param, group_id): - """ add optimizer param with group id """ - param_id, _ = self.new_param() - self.param_groups[group_id]["params"].append(param_id) - self.state[param_id] = param - - def update_state(self, param, group_id, param_id): - """ update optimizer param """ - assert param_id in self.state - assert param_id in self.param_groups[group_id]["params"] - self.state[param_id] = param - - def get_state_num(self): - """ get optimizer state num """ - s = 0 - for param_id, param in self.state.items(): - if len(param) > 0: - s += 1 - return s - - def get_state_size(self): - """ sum all state' size """ - s = 0 - for param_id, param in self.state.items(): - for key, data in param.items(): - s += numpy.prod(data.shape) - break - return s - - def get_param_size(self): - """ get param size """ - if self.get_state_num() > 0: - return self.get_state_size() - elif self.get_shard_fp32_num() > 0: - return self.get_shard_fp32_size() - else: - return 0 - - def get_param_num(self): - """ count all params """ - return sum([len(group.get("params", [])) \ - for group in self.param_groups]) - - def get_param_groups_num(self): - """ get param groups num """ - return len(self.param_groups) - - def get_shard_fp32_num(self): - return len(sum(self.shard_fp32, [])) - - def get_shard_fp32_size(self): - """ sum all shard_fp32' size """ - s = 0 - for group in self.shard_fp32: - for data in group: - s += numpy.prod(data.shape) - return s - - def get_state(self, param_id, key = None): - """ get state """ - if key == None: - return self.state.get(param_id, {}) - else: - return self.state[param_id][key] - - def get_param_ids_in_groups(self): - """" get param ids in groups """ - return [param_group["params"] for param_group in self.param_groups] - - def get_keys(self): - """ return keys of optimizer state """ - assert not self.empty() - return list(self.state[0].keys()) - - def get_state_range(self, pp): - """ divide state among various pp rank, and return list of (start, end) - """ - state_range = [[[], []] for i in range(pp)] - step0 = self.num_layers // pp * self.params_num_per_layer[0] - step1 = self.num_layers // pp * self.params_num_per_layer[1] - param_groups = self.get_param_ids_in_groups() - state_range[0][0] += range(self.start_offset) - i = self.start_offset - j = 0 - for p in range(pp-1): - state_range[p][0] += param_groups[0][i: i + step0] - state_range[p][1] += param_groups[1][j: j + step1] - i += step0 - j += step1 - state_range[-1][0] += param_groups[0][i:] - state_range[-1][1] += param_groups[1][j:] - return state_range - - def need_reshape(self): - """ whether need reshape """ - if self.empty(): - return False - return self.use_distributed_optimizer ^ (self.get_shard_fp32_by_param_id(0).dim == 1) - - def build_param_map(self, named_parameters_shape): - param_map = {} - group0, group1 = [], [] - model_param_id = 0 - for name, shape, parallel_dim in named_parameters_shape: - group_id = get_group_id(name, shape) - (group0, group1)[group_id].append(( - name, - shape, - parallel_dim, - model_param_id)) - model_param_id += 1 - for meta in group0 + group1: - param_id = len(param_map) - param_map[param_id] = meta - - need_reshape = self.need_reshape() - for param_id, meta in param_map.items(): - state = self.get_state(param_id) - for k, param in state.items(): - assert numpy.prod(param.shape) == numpy.prod(meta[1]), f"{param_map}, {param.shape} {meta}, {param_id}" - if need_reshape: - new_shape = -1 if self.use_distributed_optimizer else meta[1] - self.state[param_id][k] = param.view(new_shape) - shard_fp32 = self.get_shard_fp32_by_param_id(param_id) - self.update_shard_fp32(shard_fp32.view(new_shape), param_id) - self.param_map = param_map - - def cut_embedding_padding(self, vocab_size): - """ cut embedding padding """ - ids, _ = self.get_param_ids_in_groups() - embedding_param_ids = [] - if self.start_offset > 0: - embedding_param_ids.append(0) - if self.end_offset > 0: - embedding_param_ids.append(len(ids)-1) - for i in embedding_param_ids: - name, shape, parallel_dim, _ = self.param_map[i] - for k, v in self.state[i].items(): - data = cut_embedding_padding( - v.view(shape), - vocab_size - ) - if self.use_distributed_optimizer: - data = data.view(-1) - self.state[i][k] = data - - if self.get_shard_fp32_num() > 0: - for i in embedding_param_ids: - name, shape, parallel_dim, _ = self.param_map[i] - data = cut_embedding_padding( - self.shard_fp32[0][i].view(shape), - vocab_size - ) - if self.use_distributed_optimizer: - data = data.view(-1) - self.shard_fp32[0][i] = data - - def add_embedding_padding(self, divisible_by, vocab_size, tp, hidden_size, padded_vocab_size=None): - """ add embedding padding """ - ids, _ = self.get_param_ids_in_groups() - embedding_param_ids = [] - if self.start_offset > 0: - embedding_param_ids.append(0) - if self.end_offset > 0: - embedding_param_ids.append(len(ids)-1) - for i in embedding_param_ids: - for k, v in self.state[i].items(): - data = add_embedding_padding( - v.view(-1, hidden_size), - divisible_by, - vocab_size, - tp, - padded_vocab_size - ) - if self.use_distributed_optimizer: - data = data.view(-1) - self.state[i][k] = data - if self.get_shard_fp32_num() > 0: - for i in embedding_param_ids: - data = add_embedding_padding( - self.shard_fp32[0][i].view(-1, hidden_size), - divisible_by, - vocab_size, - tp, - padded_vocab_size - ) - if self.use_distributed_optimizer: - data = data.view(-1) - self.shard_fp32[0][i] = data - - def has_param(self, param_id): - """ check if optimizer has this param """ - for group in self.param_groups: - if param_id in group["params"]: - return True - return False - - def new_param(self): - """ generate new (param_id, group_id) """ - boundary = self.num_layers * self.params_num_per_layer[0] + self.start_offset - param_id = self.get_param_num() - group_id = 0 if param_id < boundary else 1 - return param_id, group_id - - def empty(self): - return self.get_state_num() == 0 and self.get_shard_fp32_num() == 0 - - def to_dict(self): - """ to dict """ - fp32_key = "shard_fp32_from_float16_groups" if self.use_distributed_optimizer \ - else "fp32_from_fp16_params" - return { \ - "optimizer": { \ - "optimizer": { \ - "state": self.state, \ - "param_groups": self.param_groups \ - }, \ - "grad_scaler": self.grad_scaler, \ - fp32_key: self.shard_fp32 \ - }, \ - "opt_param_scheduler": self.param_scheduler \ - } - def interleave(self, stage, pp): - """ interleave """ - origin_params = list(range(self.get_param_num())) - origin_transformer_params = origin_params[self.start_offset : -self.end_offset] - new_transformer_params = transpose_shape0(torch.tensor(origin_transformer_params), stage, pp).tolist() - new_params = origin_params[:self.start_offset] + new_transformer_params + origin_params[-self.end_offset:] - - _map = {} # model_param_index => optim_param_id - for param_id, meta in self.param_map.items(): - model_param_index = meta[3] - _map[model_param_index] = param_id - - new_state = {} - for old_param_id in range(self.get_param_num()): - model_param_index = self.param_map[old_param_id][3] - new_param_id = _map[new_params[model_param_index]] - new_state[old_param_id] = self.get_state(new_param_id) - self.state = new_state - - old_shard_fp32 = sum(self.shard_fp32, []) - self.shard_fp32 = [[None] * len(group) for group in self.shard_fp32] - for old_param_id in range(self.get_shard_fp32_num()): - model_param_index = self.param_map[old_param_id][3] - new_param_id = _map[new_params[model_param_index]] - self.update_shard_fp32(old_shard_fp32[new_param_id], old_param_id) - - def load(self, state_dict, named_parameters_shape=None): - """ load optimizer from dict """ - self.state = state_dict["optimizer"]["optimizer"]["state"] - self.param_groups = state_dict["optimizer"]["optimizer"]["param_groups"] - self.grad_scaler = state_dict["optimizer"].get("grad_scaler") - for k in ["shard_fp32_from_float16_groups", "fp32_from_fp16_params"]: - if k in state_dict["optimizer"]: - self.shard_fp32 = state_dict["optimizer"][k] - break - self.param_scheduler = state_dict.get("opt_param_scheduler") - - if named_parameters_shape is not None and self.get_param_num() > 0: - self.build_param_map(named_parameters_shape) - - def debug(self, msg): - """ debug """ - print(f"\n【Optimizer】{msg}") - print(f"-> layers: {self.num_layers}, params offset: [{self.start_offset}: -{self.end_offset}], params_per_layer: {self.params_num_per_layer}") - print(f"-> param num: {self.get_param_num()}, param size: {self.get_param_size()}") - print(f"-> state num: {self.get_state_num()}, state size: {self.get_state_size()}") - print(f"-> shard_fp32 num: {self.get_shard_fp32_num()}, shard_fp32 size: {self.get_shard_fp32_size()}\n") diff --git a/tools/convert_checkpoint/model.py b/tools/convert_checkpoint/model.py deleted file mode 100644 index a95750b2..00000000 --- a/tools/convert_checkpoint/model.py +++ /dev/null @@ -1,264 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import argparse -import os -import sys -import torch - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.dirname(SCRIPT_DIR)) - -from convert_checkpoint.huggingface_checkpoint import HuggingFaceCheckpoint -from convert_checkpoint.megatron_checkpoint import MegatronCheckpoint -from convert_checkpoint.mcore_checkpoint import McoreCheckpoint -from convert_checkpoint.huggingface_config import HuggingFaceConfig -from convert_checkpoint.megatron_config import MegatronConfig -from convert_checkpoint.mcore_config import McoreConfig -from convert_checkpoint.common_config import CommonConfig -from convert_checkpoint.arguments import parse_args, set_args -from convert_checkpoint import utils - -BIG_MODEL_LIST = ['llama2-70b', 'qwen-72b', 'codellama-70b', 'codellama-34b'] - - -class Model(): - """ - Model - """ - def __init__(self): - self.config = CommonConfig() - self.delay_convert_optimizer = False - - def convert_checkpoint(self, platform, save_path=None, m_config=None, save_optim=True): - """ - Convert common checkpoint to the platform checkpoint. - - Args: - platform (str): name of platform - args (dict): arguments - """ - - if platform == 'megatron': - return self.ckpt.convert(MegatronCheckpoint, self.config) - - if platform == 'mcore': - return self.ckpt.convert(McoreCheckpoint, self.config, save_path, m_config, save_optim) - - if platform == 'huggingface': - return self.ckpt.convert(HuggingFaceCheckpoint, self.config) - - def convert_config(self, platform): - """ - Convert common config to the platform config. - - Args: - platform (str): name of platform - """ - if platform == 'megatron': - return self.config.convert(MegatronConfig) - - if platform == 'mcore': - return self.config.convert(McoreConfig) - - if platform == 'huggingface': - return self.config.convert(HuggingFaceConfig) - - def load(self, args): - """ - Load checkpoint and config. - """ - - if not hasattr(args, "common_config_path") or args.common_config_path is None: - assert hasattr(args, "model_type_custom"), "model_type_custom or common_config_path is required" - base_dir = os.path.dirname(os.path.abspath(__file__)) - args.common_config_path = os.path.join(base_dir, f"config/{args.model_type_custom}.json") - else: - model_type_custom = os.path.splitext(os.path.basename(args.common_config_path))[0] - setattr(args, "model_type_custom", model_type_custom) - - platform = args.load_platform - ckpt_path = args.load_ckpt_path - - # load common config - assert isinstance(self.config, CommonConfig) - config_path = args.common_config_path - self.config.load(config_path) - common_args = self.config.get_args() - num_layers = common_args['num_layers'] - platform_args = self.config.get_args(platform) - dtype = self.config.get_dtype() - name_map = self.config.get_name_map(platform) - - # load checkpoint - if platform == 'huggingface': - hf_ckpt = HuggingFaceCheckpoint(num_layers) - hf_ckpt.set_dtype(dtype) - hf_ckpt.load(ckpt_path, args.safetensors) - self.ckpt = hf_ckpt.convert_to_common(self.config) - return hf_ckpt - - # load checkpoint - if platform == 'megatron': - self.delay_convert_optimizer = args.model_type_custom in BIG_MODEL_LIST - m_config = MegatronConfig() - m_config.load(ckpt_path) - m_config.update({"tensor_parallel_dim": self.config.get("tensor_parallel_dim")}) - m_ckpt = MegatronCheckpoint(num_layers) - m_ckpt.set_dtype(dtype) - load_optim = not self.delay_convert_optimizer and not args.no_load_optim - m_ckpt.load(ckpt_path, m_config, name_map, load_optim) - self.ckpt = m_ckpt.convert_to_common(self.config) - return m_ckpt - - # load checkpoint - if platform == 'mcore': - self.delay_convert_optimizer = args.model_type_custom in BIG_MODEL_LIST - m_config = McoreConfig() - m_config.load(ckpt_path) - m_config.update({"tensor_parallel_dim": self.config.get("tensor_parallel_dim")}) - m_ckpt = McoreCheckpoint(num_layers, args.load_ckpt_path) - m_ckpt.set_dtype(dtype) - load_optim = not self.delay_convert_optimizer and not args.no_load_optim - m_ckpt.load(ckpt_path, m_config, name_map, load_optim) - self.ckpt = m_ckpt.convert_to_common(self.config) - return m_ckpt - - def update_args(self, args, group): - """ update config accoding to args """ - self.config.update_args(vars(args), group) - - -def main(): - """ main """ - args = parse_args() - - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - - if args.load_platform == "mcore" or args.save_platform == "mcore": - assert args.transformer_impl == "transformer_engine", \ - "Only support transformer_engine implemenation for mcore now!" - - args.no_load_optim = True - args.no_save_optim = True - print(f"<< Warning: not support mcore optimizer now, so no_load_optim and no_save_optim are set to True! >>") - - model = Model() - source_ckpt = model.load(args) - for group in ["common", "megatron", "huggingface"]: - _args = parse_args(group) - model.update_args(_args, group) - if group == "megatron": - model.update_args(_args, "mcore") - - target_config = model.convert_config(args.save_platform) - save_optim = not args.no_save_optim and not model.delay_convert_optimizer - target_ckpt = model.convert_checkpoint(args.save_platform, args.save_ckpt_path, target_config, save_optim) - save_option = dict() - save_option["save_optim"] = save_optim - if isinstance(target_ckpt, HuggingFaceCheckpoint): - save_option["save_safe"] = args.safetensors - if args.save_platform != "mcore" and args.sub_num_layers_for_save is None and utils.LOADED_LAYERS is None: - if args.save_ckpt_path is not None: - target_ckpt.save(args.save_ckpt_path, target_config, **save_option) - - if not args.no_save_optim and model.delay_convert_optimizer: - from convert_checkpoint.optim import change_tp, change_pp, change_dp - is_change_pp = (source_ckpt.pp != target_ckpt.pp) or (source_ckpt.num_stages != target_ckpt.num_stages) - is_change_tp = (source_ckpt.tp != target_ckpt.tp) - assert not (is_change_pp and is_change_tp), "cann't change tp and pp at the same time" - if is_change_tp: - change_tp(source_ckpt, target_ckpt, args.load_ckpt_path, args.save_ckpt_path, model.config) - elif is_change_pp: - change_pp(source_ckpt, target_ckpt, args.load_ckpt_path, args.save_ckpt_path, model.config) - else: - change_dp(source_ckpt, target_ckpt, args.load_ckpt_path, args.save_ckpt_path) - - -def verl_convert_mcore_to_hf_v3(v3_params, args): - for filename in os.listdir(args.save_ckpt_path): - if filename.endswith(".safetensors") or filename == "model.safetensors.index.json": - file_path = os.path.join(args.save_ckpt_path, filename) - try: - os.remove(file_path) - except Exception as e: - print(f"删除失败 {file_path}: {str(e)}") - p_keys = set(v3_params.keys()) - utils.LOADED_STATE_DICT = {} - for p in p_keys: - utils.LOADED_STATE_DICT[p] = v3_params[p] - del v3_params[p] - set_args(args) - main() - - -def test(): - tp = 1 - pp = 2 - ep = 4 - num_experts = 16 - custom_pipeline_layers=None - args = argparse.Namespace() - args.load_platform = "mcore" - args.save_platform = "huggingface" - args.load_ckpt_path = None - args.save_ckpt_path = "/mnt/cluster/deepseek-ai/DeepSeek_V3_Lite_hf" - args.common_config_path = "./convert_checkpoint/config/deepseek-v3-lite.json" - args.megatron_path = None - args.model_type_custom = None - args.torch_dtype = None - args.vocab_size = None - args.vpp_scheduler = None - args.num_virtual_stages_per_pipeline_rank = None - args.decoder_first_pipeline_num_layers = None - args.decoder_last_pipeline_num_layers = None - args.use_distributed_optimizer = False - args.tensor_model_parallel_size = tp - args.pipeline_model_parallel_size = pp - args.data_parallel_size = 1 - args.expert_parallel_size = ep - args.pad_vocab_size_to = None - args.num_layers_per_virtual_pipeline_stage = None - args.transformer_impl = 'transformer_engine' - args.checkpoint_format = None - args.max_workers = 1 - args.num_experts = num_experts - args.no_load_optim = True - args.no_save_optim = True - args.no_te = True - args.moe_grouped_gemm = True - args.resume_convert = False - args.cache_path = None - args.layer_for_test = None - args.num_experts_for_test = None - args.sub_num_layers_for_save = None - args.custom_pipeline_layers = custom_pipeline_layers - args.safetensors = True - args.save_sub_checkpoint_by_pp = True - args.convert_to_fp8 = False - args.pretrain_as_fp8 = False - args.quant_method = 'te' - args.amax_epsilon = 0.0 - print(f"{args=}") - - v3_params = {} - for p in range(pp): - v3_params[p] = {} - for e in range(ep): - v3_params[p][e] = torch.load(f'/mnt/cluster/deepseek-ai/DeepSeek_V3_tp1pp2ep4/release/mp_rank_00_{p:03d}_{e:03d}/model_optim_rng.pt') - verl_convert_mcore_to_hf_v3(v3_params, args) - -from convert_checkpoint.utils import make_hf_sub_checkpoints - -def test_merge_hf_ckpt(): - make_hf_sub_checkpoints('/mnt/cluster/deepseek-ai/DeepSeek_V3_Lite_hf') - -if __name__ == "__main__": - #test() - main() diff --git a/tools/convert_checkpoint/optim.py b/tools/convert_checkpoint/optim.py deleted file mode 100644 index c6723ea9..00000000 --- a/tools/convert_checkpoint/optim.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -import os, sys -import torch - -SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.dirname(SCRIPT_DIR)) - -from convert_checkpoint.megatron_optimizer import MegatronOptimizer, merge_optimizer_by_pp_tp, merge_optimizer_by_dp -from convert_checkpoint.megatron_checkpoint import MegatronCheckpoint -from convert_checkpoint.megatron_config import MegatronConfig -from convert_checkpoint.common_config import CommonConfig -from convert_checkpoint.arguments import parse_args - - -def load_optim_by_dp(ckpt, load_path, p, t): - "load optimizer by dp strategy" - named_parameters_shape = ckpt.get_named_parameters_shape(p, t) - opts = [] - if ckpt.use_distributed_optimizer: - for d in range(ckpt.dp): - if ckpt.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}" - checkpoint_dir = os.path.join(load_path, checkpoint_dir) - checkpoint_path = os.path.join(checkpoint_dir, "optim.pt") - _state_dict = torch.load(checkpoint_path, map_location="cpu") - opt = MegatronOptimizer.generate_optimizer(ckpt, ckpt.num_layers // ckpt.pp, p) - opt.load(_state_dict) - opts.append(opt) - opt.debug(f"tp/pp/dp rank: {t}/{p}/{d}, load from: {checkpoint_path}") - return merge_optimizer_by_dp(opts, named_parameters_shape) - else: - opt = MegatronOptimizer.generate_optimizer(ckpt, ckpt.num_layers // ckpt.pp, p) - if ckpt.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}" - checkpoint_dir = os.path.join(load_path, checkpoint_dir) - checkpoint_path = os.path.join(checkpoint_dir, "model_optim_rng.pt") - state_dict = torch.load(checkpoint_path, map_location="cpu") - opt.load(state_dict, named_parameters_shape) - opt.build_param_map(named_parameters_shape) - opt.debug(f"tp/pp rank: {t}/{p}, load from: {checkpoint_path}") - return opt - - -def save_optim_by_dp(optim, ckpt, save_path, p, t): - """ save optimizer by dp strategy """ - os.makedirs(save_path, exist_ok=True) - folder_name = f"iter_{ckpt.iteration:07d}" if ckpt.iteration > 0 else "release" - release_dir = os.path.join(save_path, folder_name) - os.makedirs(release_dir, exist_ok=True) - - if ckpt.use_distributed_optimizer: - chunk_optimers = optim.chunk_by_dp(ckpt.dp, ckpt.num_stages) - del optim - for d in range(ckpt.dp): - if ckpt.pp == 1: - checkpoint_dir = f"mp_rank_{t:02d}_{d:03d}" - else: - checkpoint_dir = f"mp_rank_{t:02d}_{p:03d}_{d:03d}" - - checkpoint_dir = os.path.join(release_dir, checkpoint_dir) - os.makedirs(checkpoint_dir, exist_ok=True) - checkpoint_path = os.path.join(checkpoint_dir, "optim.pt") - torch.save( - chunk_optimers[d].to_dict(), - checkpoint_path, - ) - chunk_optimers[d].debug(f"tp/pp/dp rank {t}/{p}/{d}, saved to: {checkpoint_path}") - else: - checkpoint_dir = ( - f"mp_rank_{t:02d}" - if ckpt.pp == 1 - else f"mp_rank_{t:02d}_{p:03d}" - ) - checkpoint_dir = os.path.join(release_dir, checkpoint_dir) - checkpoint_name = "model_optim_rng.pt" - checkpoint_path = os.path.join(checkpoint_dir, checkpoint_name) - state_dict = torch.load(checkpoint_path) - state_dict.update(optim.to_dict()) - torch.save(state_dict, checkpoint_path) - optim.debug(f"tp/pp rank {t}/{p}, saved to: {checkpoint_path}") - - -def change_pp(s_ckpt, t_ckpt, load_path, save_path, config): - """ change pp """ - margs = config.get_args("megatron") - for t in range(s_ckpt.tp): - opts = [[load_optim_by_dp(s_ckpt, load_path, p, t)] \ - for p in range(s_ckpt.pp)] - optim = merge_optimizer_by_pp_tp(opts, margs) - del opts - optim.build_param_map(s_ckpt.get_named_parameters_shape(t=t)) - if s_ckpt.num_stages > 1: - optim.interleave(s_ckpt.pp, s_ckpt.num_stages) - if s_ckpt.pp == 1 and not margs.get("untie_embeddings_and_output_weights", False): - optim.add_word_embedding_for_head() - if t_ckpt.pp == 1 and not margs.get("untie_embeddings_and_output_weights", False): - optim.remove_word_embedding_for_head() - if t_ckpt.num_stages > 1: - optim.interleave(t_ckpt.num_stages, t_ckpt.pp) - opts = optim.chunk_by_pp_tp(t_ckpt.pp, 1, margs) - del optim - for p, opt in enumerate(opts): - opt[0].build_param_map(t_ckpt.get_named_parameters_shape(p, t)) - save_optim_by_dp(opt[0], t_ckpt, save_path, p, t) - del opts - - -def change_tp(s_ckpt, t_ckpt, load_path, save_path, config): - """ change tp """ - cargs = config.get_args("common") - margs = config.get_args("megatron") - for p in range(s_ckpt.pp): - opts = [[ load_optim_by_dp(s_ckpt, load_path, p, t) \ - for t in range(s_ckpt.tp) ]] - optim = merge_optimizer_by_pp_tp(opts, margs) - del opts - optim.build_param_map(s_ckpt.get_named_parameters_shape(p=p)) - if (p == 0 or p == s_ckpt.pp - 1) and margs.get("add_embedding_padding", False): - divisible_by = margs["make_vocab_size_divisible_by"] - padded_vocab_size = margs.get("pad_vocab_size_to") - vocab_size = cargs["vocab_size"] - hidden_size = cargs["hidden_size"] - optim.cut_embedding_padding(vocab_size) - optim.add_embedding_padding(divisible_by, vocab_size, t_ckpt.tp, hidden_size, padded_vocab_size) - - optim.use_distributed_optimizer = t_ckpt.use_distributed_optimizer - optim.build_param_map(t_ckpt.get_named_parameters_shape(p=p)) - opts = optim.chunk_by_pp_tp(1, t_ckpt.tp, margs) - del optim - for t, opt in enumerate(opts[0]): - opt.build_param_map(t_ckpt.get_named_parameters_shape(p, t)) - save_optim_by_dp(opt, t_ckpt, save_path, p, t) - del opts - - -def change_dp(s_ckpt, t_ckpt, load_path, save_path): - """ change dp """ - for p in range(s_ckpt.pp): - for t in range(s_ckpt.tp): - opt = load_optim_by_dp(s_ckpt, load_path, p, t) - save_optim_by_dp(opt, t_ckpt, save_path, p, t) - del opt - - -if __name__ == "__main__": - args = parse_args() - - if args.megatron_path is not None: - sys.path.insert(0, args.megatron_path) - - args = parse_args() - ckpt_path = args.load_ckpt_path - save_path = args.save_ckpt_path - config_path = args.common_config_path - - config = CommonConfig() - config.load(config_path) - for group in ["common", "megatron", "huggingface"]: - args = parse_args(group) - config.update_args(vars(args), group) - dtype = config.get_dtype() - - cargs = config.get_args() - num_layers = cargs['num_layers'] - name_map = config.get_name_map("megatron") - - # source ckpt - s_config = MegatronConfig() - s_config.load(ckpt_path) - s_config.update({"tensor_parallel_dim": config.get("tensor_parallel_dim")}) - s_ckpt = MegatronCheckpoint(num_layers) - s_ckpt.load(ckpt_path, s_config, name_map, load_optimizer=False) - s_ckpt.set_dtype(dtype) - s_ckpt.state_dict = [] - - # target ckpt - assert s_ckpt.iteration > 0 - folder_name = f"iter_{s_ckpt.iteration:07d}" - release_dir = os.path.join(save_path, folder_name) - t_config = MegatronConfig() - t_config.load(release_dir) - t_config.update({"tensor_parallel_dim": config.get("tensor_parallel_dim")}) - t_ckpt = MegatronCheckpoint(num_layers) - t_ckpt.load(release_dir, t_config, name_map, load_optimizer=False) - t_ckpt.set_dtype(dtype) - t_ckpt.state_dict = [] - - is_change_pp = (s_ckpt.pp != t_ckpt.pp) or (s_ckpt.num_stages != t_ckpt.num_stages) - is_change_tp = (s_ckpt.tp != t_ckpt.tp) - assert not (is_change_pp and is_change_tp), "cann't change tp and pp at the same time" - if is_change_tp: - change_tp(s_ckpt, t_ckpt, ckpt_path, save_path, config) - elif is_change_pp : - change_pp(s_ckpt, t_ckpt, ckpt_path, save_path, config) - else: - change_dp(s_ckpt, t_ckpt, ckpt_path, save_path) diff --git a/tools/convert_checkpoint/utils.py b/tools/convert_checkpoint/utils.py deleted file mode 100644 index be1b2c11..00000000 --- a/tools/convert_checkpoint/utils.py +++ /dev/null @@ -1,416 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -################################################################################ -# -# Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved -# -################################################################################ - -"""General utilities.""" - -import os -import json -import torch -from bisect import bisect_left -from math import floor, ceil - - -LOADED_STATE_DICT = None -LOADED_LAYERS = None -LOADED_MIN_E = None - - -def ceil_div(x: int, y: int) -> int: - return (x + y - 1) // y - - -def get_element_from_dict_by_path(d, path): - """ - Get element from dictionary by path. If element is not present, recursively add empty dictionaries. - - Args: - d (dict): the dictionary to get the element from - path (list): the path to the element which is delimited by "." - """ - path = path.split(".") - for k in path: - if k not in d: - d[k] = {} - d = d[k] - return d - - -def check_path_in_dict(d, path): - """ - check path exists in dictionary - """ - path = path.split(".") - for k in path: - if k not in d: - return False - d = d[k] - return True - - -def vocab_size_with_padding(orig_vocab_size, make_vocab_size_divisible_by, tp): - """Pad vocab size so it is divisible by model parallel size and - still having GPU friendly size.""" - - after = orig_vocab_size - multiple = make_vocab_size_divisible_by * tp - while (after % multiple) != 0: - after += 1 - return after - - -def add_embedding_padding(weight, divisible_by, orig_vocab_size, tp, padded_vocab_size=None): - """ add embedding padding """ - if padded_vocab_size is None: - padded_vocab_size = vocab_size_with_padding(orig_vocab_size, divisible_by, tp) - padding_size = padded_vocab_size - orig_vocab_size - else: - padding_size = padded_vocab_size - weight.shape[0] - if padding_size < 0: - return weight[0:padded_vocab_size, :] - elif padding_size > 0: - return torch.cat((weight, weight[-1].unsqueeze(0).expand(padding_size, -1))) - else: - return weight - - -def cut_embedding_padding(weight, orig_vocab_size): - """ cut embedding padding """ - #TODO - return weight[0:orig_vocab_size, :] - - -def transpose_shape0(param, m, n): - """ transpose on shape 0 """ - _shape = param.size() - current_shape = (m, n, _shape[0] // (m * n)) + _shape[1:] - return param.view(*current_shape) \ - .transpose(0, 1).contiguous() \ - .view(*_shape) - -def uneven_vpp_partition(num_layers, pp, vp, num_layers_in_first_pipeline_stage, num_layers_in_last_pipeline_stage): - assert num_layers is not None and num_layers > 0, "num_layers must be provided." - assert pp is not None and pp > 1, "pipeline model parallel size must be greater than 1." - assert vp is not None and vp == 2, "virtual pipeline must be 2." - assert num_layers_in_first_pipeline_stage is not None or num_layers_in_last_pipeline_stage is not None, \ - "num_layers_in_first_pipeline_stage or num_layers_in_last_pipeline_stage must be provided." - # Number of layers to distribute over rest of pipeline stages - layers_to_distribute = num_layers - # Number of pipeline stages left for distributing transformer layers - pipeline_stages_left = pp - parts_count = [0 for _ in range(pp*vp)] - # If the uneven first (last) pipeline stage is enabled, remove the specified number - # of layers to calculate the number of layers on each middle pipeline stage. - if num_layers_in_first_pipeline_stage is not None: - layers_to_distribute -= num_layers_in_first_pipeline_stage - parts_count[0] = ceil(num_layers_in_first_pipeline_stage / vp) - parts_count[pp * vp - 1] = num_layers_in_first_pipeline_stage - parts_count[0] - pipeline_stages_left -= 1 - - if num_layers_in_last_pipeline_stage is not None: - layers_to_distribute -= num_layers_in_last_pipeline_stage - parts_count[pp-1] = ceil(num_layers_in_last_pipeline_stage / vp) - parts_count[pp] = num_layers_in_last_pipeline_stage - parts_count[pp-1] - pipeline_stages_left -= 1 - num_layers_per_pipeline_rank = layers_to_distribute // pipeline_stages_left - for i in range(1, pp-1): - parts_count[i] = num_layers_per_pipeline_rank // vp - parts_count[2 * pp - 1 - i] = num_layers_per_pipeline_rank - parts_count[i] - if num_layers_in_first_pipeline_stage is None: - parts_count[0] = num_layers_per_pipeline_rank // vp - parts_count[2 * pp - 1] = num_layers_per_pipeline_rank - parts_count[0] - if num_layers_in_last_pipeline_stage is None: - parts_count[pp-1] = num_layers_per_pipeline_rank // vp - parts_count[pp] = num_layers_per_pipeline_rank - parts_count[pp-1] - return parts_count - - -def custom_partition_imbalanced(num_layers, num_parts, custom_layers): - """ - custom partition imbalanced. - first and last stages contain less layers, - other stages contain more layers - """ - splits = [] - if custom_layers.find(',') != -1: - splits = [int(s) for s in custom_layers.split(',')] - if len(splits) != num_parts: - raise ValueError( - f'the argments of custom_pipeline_layers must be equal to pipeline size {num_parts}.' - ) - assert num_layers == sum(splits), f'the sum of custom_pipeline_layers must be equal to num_layers {num_layers}.' - # First check for the trivial edge case - if num_layers <= num_parts: - parts = partition_uniform(num_layers, num_parts) - else: - parts = [0] * (num_parts + 1) - parts_count = [num_layers // num_parts] * num_parts - for i in range(num_parts): - parts_count[i] = splits[i] - for i in range(1, len(parts_count) + 1): - parts[i] = parts[i - 1] + parts_count[i - 1] - - return parts_count, parts - - -def partition_balanced(num_layers, num_parts, eps=1e-3): - """ - partition balanced. - """ - # First check for the trivial edge case - if num_layers <= num_parts: - parts = partition_uniform(num_layers, num_parts) - else: - weights = [1] * num_layers - weights_ = prefix_sum_inc(weights) - - # Find the smallest bottleneck (weight of heaviest partition) - bottleneck = _rb_partition_balanced(weights_, num_parts, eps=eps) - - # Now compute that partitioning - parts, success = _lprobe(weights_, num_parts, bottleneck) - assert success - - parts_count = [0] * num_parts - - for i in range(1, len(parts)): - parts_count[i - 1] = parts[i] - parts[i - 1] - - return parts_count, parts - - -def partition_uniform(num_items, num_parts): - """ - partition uniform. - """ - parts = [0] * (num_parts + 1) - # First check for the trivial edge case - if num_items <= num_parts: - for p in range(num_parts + 1): - parts[p] = min(p, num_items) - return parts - - chunksize = floor(num_items / num_parts) - for p in range(num_parts): - parts[p] = min(chunksize * p, num_items) - parts[num_parts] = num_items - return parts - - -def prefix_sum_inc(weights): - """ Compute an inclusive prefix sum. - - Example: - >>> prefix_sum_inc([3,4,5]) - [3, 7, 12] - """ - weights_ = [w for w in weights] - for x in range(1, len(weights_)): - weights_[x] += weights_[x - 1] - return weights_ - - -def _rb_partition_balanced(weights, num_parts, eps): - total_weight = weights[-1] - lower = total_weight / num_parts # best case heaviest partition - upper = total_weight # worst case heaviest partition - - # Do a binary search for the best partitioning - while upper > lower + eps: - mid = lower + ((upper - lower) / 2) - parts, success = _lprobe(weights, num_parts, mid) - if success: - upper = mid - else: - lower = mid + eps - return upper - - -def _lprobe(weights, num_parts, bottleneck): - num_items = len(weights) - total_weight = weights[-1] - - # initialize partitioning - parts = [0] * (num_parts + 1) - for p in range(1, num_parts + 1): - parts[p] = num_items - - bsum = bottleneck # running sum of target weight for pth partition - chunksize = num_items // num_parts - step = chunksize - for p in range(1, num_parts): - # Jump to the next bucket - while (step < num_items) and (weights[step] < bsum): - step += chunksize - - # Find the end index of partition p - parts[p] = bisect_left(weights, bsum, lo=step - chunksize, hi=min(step, num_items)) - # Nothing more to partition, return early - if parts[p] == num_items: - # See if the current partition is overweight. - part_size = weights[-1] - weights[parts[p - 1]] - return parts, part_size < bottleneck - - # Next partition target - bsum = weights[parts[p] - 1] + bottleneck - - return parts, bsum >= total_weight - -def touch_file(done_dir, p_id, ep_id): - if ep_id is None: - fname = f'{p_id}.done' - else: - fname = f'{p_id}_{ep_id}.done' - done_file_name = os.path.join(done_dir, fname) - with open(done_file_name, 'w'): - os.utime(done_file_name, None) - -def check_all_done(done_dir, p, ep): - fnames = [] - if ep is None: - for p_id in range(p): - fname = f'{p_id}.done' - fnames.append(fname) - else: - for p_id in range(p): - for ep_id in range(ep): - fname = f'{p_id}_{ep_id}.done' - fnames.append(fname) - all_done = True - for fname in fnames: - done_file_name = os.path.join(done_dir, fname) - if not os.path.exists(done_file_name): - all_done = False - break - return all_done - -def get_done_keys(done_dir, p, ep): - done_keys = [] - for p_id in range(p): - for ep_id in range(ep): - fname = f'{p_id}_{ep_id}.done' - if os.path.exists(os.path.join(done_dir, fname)): - done_keys.append((p_id, ep_id)) - return done_keys - - -def make_hf_sub_checkpoints(base_path): - # 初始化全局计数器 - global_file_count = 0 - sum_sub_count = 0 - - path = f'{base_path}/sub_checkpoint/' - # 假设文件列表是已知的，这里用一个列表模拟 - temp_paths = [] - for sub_dir_name in os.listdir(path): - if sub_dir_name.isdigit(): - temp_paths.append(sub_dir_name) - sorted_path_list = sorted(temp_paths, key=int) - - for index in sorted_path_list: - if index.isdigit(): # 检查是否为数字目录 - subdir_path = os.path.join(path, index) - if os.path.isdir(subdir_path): - for filename in os.listdir(subdir_path): - if filename.startswith('model-') and filename.endswith('.safetensors'): - parts = filename.split('-of-') - if len(parts) == 2: - global_file_count += 1 - # 遍历所有子目录 - all_dict = {} - for index in sorted_path_list: - if index.isdigit(): # 检查是否为数字目录 - subdir_path = os.path.join(path, index) - if os.path.isdir(subdir_path): - # 初始化子目录计数器 - local_file_count = 0 - - # 遍历子目录中的所有文件 - one_dict = {} - print(f"{subdir_path=}") - for filename in os.listdir(subdir_path): - if filename.startswith('model-') and filename.endswith('.safetensors'): - # 解析文件名，提取 i 和 sub_count - parts = filename.split('-of-') - if len(parts) == 2: - file_base, file_count = parts - i_str = file_base.split('-')[-1] - sub_count_str = file_count.split('.')[0] - i = int(i_str) - sub_count = int(sub_count_str) - - # 更新全局计数器 - local_file_count += 1 - - # 计算新的文件名 - new_i = sum_sub_count + i - new_filename = f'model-{new_i:05d}-of-{global_file_count:05d}.safetensors' - - # 重命名文件 - old_filepath = os.path.join(subdir_path, filename) - new_filepath = os.path.join(subdir_path, new_filename) - one_dict[filename] = new_filename - all_dict[subdir_path] = one_dict - - # 更新累计子文件个数 - sum_sub_count += local_file_count - - - # 用于存储合并后的metadata和weight_map - merged_metadata = {"total_size": 0} - merged_weight_map = {} - - # 遍历文件列表，合并metadata和weight_map - for index in sorted_path_list: - if index.isdigit(): # 检查是否为数字目录 - subdir_path = os.path.join(path, index) - if os.path.isdir(subdir_path): - file_name = f"{subdir_path}/model.safetensors.index.json" - with open(file_name, 'r') as f: - file_content = json.load(f) - # 合并metadata - merged_metadata["total_size"] += file_content["metadata"]["total_size"] - # 合并weight_map，并使用one_dict进行替换 - subdir_path = os.path.join(path, index) - one_dict = all_dict[subdir_path] - for key, value in file_content["weight_map"].items(): -# print(f"{key=}, {value=}, {one_dict=}") -# print(f"{one_dict[value]=}") - if value in one_dict: - # 替换成one_dict中对应的值 - merged_weight_map[key] = one_dict[value] - else: - # 如果没有找到替换项，则保留原值（这里可以根据需求调整） - # 注意：这里保留原值可能没有意义，因为通常我们不会想要保留文件名作为权重名 - # 但为了保持示例的完整性，我保留了这一行 - # 在实际应用中，你可能想要抛出一个错误或者记录一个警告 - merged_weight_map[key] = value # 这通常不是期望的行为，仅用于示例 - - # 构建新的dict - new_dict = { - "metadata": merged_metadata, - "weight_map": merged_weight_map - } - - # 将新的dict写回到model.safetensors.index.json文件中 - with open(f'{base_path}/model.safetensors.index.json', 'w') as f: - json.dump(new_dict, f, indent=4) - for index in sorted_path_list: - if index.isdigit(): # 检查是否为数字目录 - subdir_path = os.path.join(path, index) - if os.path.isdir(subdir_path): - one_dict = all_dict[subdir_path] - for filename, new_filename in one_dict.items(): - old_filepath = os.path.join(subdir_path, filename) - new_filepath = os.path.join(base_path, new_filename) - os.rename(old_filepath, new_filepath) - print(f'Renamed: {old_filepath} -> {new_filepath}') - - print(f"合并和替换完成，新的model.safetensors.index.json文件已生成。" - f"{base_path}/model.safetensors.index.json") - import shutil - shutil.rmtree(f'{base_path}/sub_checkpoint') \ No newline at end of file diff --git a/tools/data_preprocess/convert_to_webdataset.py b/tools/data_preprocess/convert_to_webdataset.py deleted file mode 100644 index 9acc2242..00000000 --- a/tools/data_preprocess/convert_to_webdataset.py +++ /dev/null @@ -1,195 +0,0 @@ -""" Convert dataset into WebDataset (WDS) format """ -import argparse -import json -import os -import yaml -import webdataset as wds -from tqdm import tqdm -import random -from megatron.energon.epathlib import EPath -from megatron.energon.flavors import BaseWebdatasetFactory -from megatron.energon.flavors.webdataset import MAIN_FOLDER_NAME - -def sample_loader_template(media: str=None): - """Returns a template for a sample_loader.py file.""" - return "\n".join([ - "def sample_loader(sample: dict) -> dict:", - " messages=[]", - " system=None", - " for message in sample['json']['texts']:", - " assert message['role'] in ['system','user','assistant']", - " if message['role'] == 'system':", - " system=message['content']", - " continue", - " messages.append(dict(", - " role=message['role'],", - " content=message['content']", - " ))", - " video = []", - " image = []", - " if sample['json']['media'] == 'video':", - " for name in sample['json']['name']:", - " video.append(sample.get(name))", - " elif sample['json']['media'] == 'image':", - " for name in sample['json']['name']:", - " image.append(sample.get(name))", - " return dict(", - " __key__=sample['__key__'],", - " __restore_key__=sample['__restore_key__'],", - " video=video if len(video) > 0 else None,", - " image=image if len(image) > 0 else None," if media == 'mix' else "", - " system=system,", - " messages=messages,", - " )", - "def part_filter(part: str) -> bool:", - " return True", - ]) - -def construct_sample(args, vision, paths, index, entry): - """ construct webdataset sample """ - assert vision == 'image' or vision == 'video' - directory = args.image_dir if vision == 'image' else args.video_dir - - vision_data = {} - vision_name = [] - - for i, path in enumerate(paths): - full_path = os.path.join(directory, path) - if not os.path.exists(full_path): - raise FileNotFoundError(f"Media file not found: {full_path}") - try: - with open(full_path, "rb") as vision_file: - vision_data.update({str(i) + '_' + os.path.basename(path) : vision_file.read()}) - vision_name.append(str(i) + '_' + os.path.basename(path)) - except IOError as e: - raise IOError(f"Failed to read media file {full_path}: {e}") - - content = { - "texts": entry[args.columns_messages], - "media": vision, - "name": vision_name - } - sample = { - "__key__": vision + '_' + str(index), - **vision_data, - "json": json.dumps(content).encode("utf-8"), - } - return sample - -def convert_to_wds(args): - """ Convert dataset to wds format """ - assert args.media in ['video', 'image', 'mix'], f"Invalid media type: {args.media}" - - if args.media == "video": - assert args.video_dir is not None - if args.media == "image": - assert args.image_dir is not None - if args.media == "mix": - assert args.video_dir is not None or args.image_dir is not None, "At least one media directory required for mix mode" - - if not os.path.exists(args.output_dir): - os.mkdir(args.output_dir) - - with open(args.json_file, 'r') as f: - data = json.load(f) - random.shuffle(data) - - tar = os.path.join(args.output_dir, 'pretrain-%d.tar') - with wds.ShardWriter(tar, maxcount=args.maxcount, maxsize=args.maxsize) as shard_writer: - for index, entry in enumerate(tqdm(data)): - if args.media == 'image': - image_path = entry.get('image') - if image_path is None: - images = entry.get('images') - if images is None or len(images) == 0: - raise ValueError(f"No image path found in entry {index}") - image_path = images[0] - with open(os.path.join(args.image_dir, image_path), "rb") as img_file: - image_data = img_file.read() - sample = { - "__key__": entry.get('id', image_path).replace('.', '_'), - "jpg": image_data, - "json": json.dumps(entry[args.columns_messages]).encode("utf-8"), - } - else: - video_paths = [entry.get('video')] if entry.get('video') is not None else entry.get('videos') - image_paths = [entry.get('image')] if entry.get('image') is not None else entry.get('images') - - if video_paths is not None: - sample = construct_sample(args, 'video', video_paths, index, entry) - elif image_paths is not None: - sample = construct_sample(args, 'image', image_paths, index, entry) - else: # for pure text - content = { - "texts": entry[args.columns_messages], - "media": "text", - } - sample = { - "__key__": 'text_' + str(index), - "json": json.dumps(content).encode("utf-8"), - } - shard_writer.write(sample) - if args.media == "mix" or args.media == "video": - write_config(EPath(args.output_dir).absolute(), args.media) - - print(f"Dataset successfully converted to wds") - -def write_config(path: EPath, media: str=None): - """ Write config to path """ - (path / MAIN_FOLDER_NAME).mkdir() - all_tars = list(path.glob("**/*.tar")) + list(path.glob("**/*.tgz")) - all_tars = [str(p.relative_to(path)) for p in sorted(all_tars)] - class_type = "MultiMixQASample" if media == 'mix' else "MultiVidQASample" - dataset_definition = { - "sample_type": { - "__module__": "aiak_training_llm.data.multimodal", - "__class__": class_type, - }, - "part_filter": "sample_loader.py:part_filter", - "sample_loader": "sample_loader.py:sample_loader" - } - with (path / MAIN_FOLDER_NAME / "dataset.yaml").open("w") as f: - yaml.dump(dataset_definition, f, sort_keys=False) - with (path / MAIN_FOLDER_NAME / "sample_loader.py").open("w") as f: - f.write(sample_loader_template(media)) - - BaseWebdatasetFactory.prepare_dataset( - path, - all_tars, - split_parts_ratio=[("train", 1.0), ("val", 0), ("test", 0)], - tar_index_only=False, - workers=96, - ) - -def _add_arguments(parser: argparse.ArgumentParser): - """Add arguments""" - group = parser.add_argument_group(title='wds') - group.add_argument('--output_dir', type=str, required=True, help='Output directory') - group.add_argument('--json_file', type=str, required=True, help='Json file') - group.add_argument('--image_dir', type=str, required=False, help='Image directory') - group.add_argument('--video_dir', type=str, required=False, help='Video directory') - group.add_argument('--maxcount', type=int, default=10000, help='Number of samples per shard') - group.add_argument('--maxsize', type=int, default=3000000000, help='Maximum size of each shard') - group.add_argument('--media', type=str, choices=["mix", "image", "video"], default="mix", help='Media type') - group.add_argument('--columns_messages', type=str, default="messages", help='Column name for messages') - - return parser - - -def parse_args(): - """arguments""" - parser = argparse.ArgumentParser() - _add_arguments(parser) - args = parser.parse_args() - - return args - - -def main(): - """main function""" - args = parse_args() - convert_to_wds(args) - - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/tools/data_preprocess/offline_packing/S2_hashbacket.ipynb b/tools/data_preprocess/offline_packing/S2_hashbacket.ipynb deleted file mode 100644 index 16e63a4b..00000000 --- a/tools/data_preprocess/offline_packing/S2_hashbacket.ipynb +++ /dev/null @@ -1,312 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "66eb0e74-9015-4cff-90c8-be8fef59b242", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-08-14 16:37:27,219 - INFO - 开始处理文件，总行数: 5520733\n", - "构建哈希桶: 98%|███████████████████████████████████████▏| 5400000/5520733 [00:05<00:00, 662131.48行/s]" - ] - } - ], - "source": [ - "from hashbacket import *\n", - "from pprint import pprint\n", - "import os\n", - "\n", - "def update_info(update_stats):\n", - " print(\"\\n更新分析:\")\n", - " print(f\"删除的key数: {update_stats['changes']['keys_removed']}\")\n", - " print(f\"删除的元素数: {update_stats['changes']['items_removed']}\")\n", - " print(f\"剩余key数: {update_stats['after']['total_keys']}\") \n", - "\n", - "if __name__ == \"__main__\":\n", - " #ZXW_MODIFY\n", - " # input_file = \"token_info_v2_vqa_5500k_s16k.txt\"\n", - " input_file = \"token_info_v2_vqa_pretrain_5M_8k.txt\"\n", - " bins_boxs = []\n", - " if not os.path.exists(input_file):\n", - " print(f\"文件 {input_file} 不存在！\")\n", - " assert input_file, f\"文件 {input_file} 不存在！\"\n", - " else:\n", - " processor = HashBucketProcessor(input_file)\n", - " processor.build_buckets()\n", - " processor.summary()\n", - " # processor.print_example(8192)\n", - "\n", - " # capacity = 16384\n", - " capacity = 8192 #ZXW_MODIFY\n", - " processor.find_items(capacity) # 寻找 capacity//2 的key\n", - " processor.summary()\n", - " print(\"=====================\"*8) \n", - " initial_summary = processor.get_hash_buckets_summary()\n", - " print(\"-------------------- initial_summary ----------------------\")\n", - " pprint(initial_summary)\n", - " # 构建追踪器\n", - " tracker = PackingTracker(processor)\n", - " # 第 1 轮 装填背包（2的幂次）\n", - " bin_boxs_001 = tracker.track_packing('pack_with_deletion')\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - "\n", - " # 第2轮 8000 以上的\n", - " bin_boxs_002 = tracker.track_packing('pack_with_flexible_seeds',\n", - " box_capacity=8192,#16384, #ZXW_MODIFY\n", - " seed_strategy=\"custom_half\",\n", - " seed_params={\"half\": 8000}, \n", - " min_items=1,\n", - " min_ratio=0.96,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - " \n", - " # 第3轮 8000 以上的\n", - " bin_boxs_003 = tracker.track_packing('pack_with_flexible_seeds',\n", - " box_capacity=8192,#16384, #ZXW_MODIFY\n", - " seed_strategy=\"custom_half\",\n", - " seed_params={\"half\": 7000}, \n", - " min_items=1,\n", - " min_ratio=0.95,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - " \n", - " # 第 4 轮 装填背包(所有大于一半 capacity 的种子)\n", - " bin_boxs_004 = tracker.track_packing('pack_with_min_items_constraint_multithread',\n", - " box_capacity=8192, #16384, #ZXW_MODIFY\n", - " min_items=6, # 10 for 16384 ,根据数据的分布来做\n", - " min_ratio=0.92,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - " \n", - " # 第 5 轮 装填背包（自由装填 for 8192）\n", - " bin_boxs_005 = tracker.track_packing('pack_with_flexible_seeds',\n", - " box_capacity=8192,#16384, #ZXW_MODIFY\n", - " seed_strategy=\"custom_half\",\n", - " seed_params={\"half\": 2080}, \n", - " min_items=8,\n", - " min_ratio=0.90,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - "\n", - " # 第 6 轮 自由装填 for 8192\n", - " bin_boxs_006 = tracker.track_packing('pack_with_flexible_seeds',\n", - " box_capacity=8192,#16384, #ZXW_MODIFY\n", - " seed_strategy=\"custom_half\",\n", - " seed_params={\"half\": 1800}, \n", - " min_items=4,\n", - " min_ratio=0.92,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - " \n", - " # 第 7 轮 自由装填 for 8192\n", - " bin_boxs_007 = tracker.track_packing('pack_with_flexible_seeds',\n", - " box_capacity=8192,#16384, #ZXW_MODIFY\n", - " seed_strategy=\"custom_half\",\n", - " seed_params={\"half\": 1280}, \n", - " min_items=6,\n", - " min_ratio=0.92,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n", - " \n", - " # 第 8 轮 自由装填 for 8192\n", - " bin_boxs_007 = tracker.track_packing('pack_with_flexible_seeds',\n", - " box_capacity=8192,\n", - " seed_strategy=\"top_n\",\n", - " seed_params={\"n\": 20}, # 1 for 16384 \n", - " min_items=4,\n", - " min_ratio=0.90,\n", - " max_workers = os.cpu_count(),\n", - " )\n", - " update_stats = processor.update_hash_buckets(remove_empty=True, verbose=True)\n", - " update_info(update_stats)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "51d1deb0-fdd7-4b03-81ef-d3176e4f3a67", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "db535d62-93bd-4a46-9a6b-4205e7aeb181", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 36724 (\\N{CJK UNIFIED IDEOGRAPH-8F74}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 65288 (\\N{FULLWIDTH LEFT PARENTHESIS}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 31532 (\\N{CJK UNIFIED IDEOGRAPH-7B2C}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 20108 (\\N{CJK UNIFIED IDEOGRAPH-4E8C}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 21015 (\\N{CJK UNIFIED IDEOGRAPH-5217}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 65289 (\\N{FULLWIDTH RIGHT PARENTHESIS}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 25955 (\\N{CJK UNIFIED IDEOGRAPH-6563}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 28857 (\\N{CJK UNIFIED IDEOGRAPH-70B9}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 22270 (\\N{CJK UNIFIED IDEOGRAPH-56FE}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n", - "/usr/local/lib/python3.12/dist-packages/IPython/core/pylabtools.py:170: UserWarning: Glyph 19968 (\\N{CJK UNIFIED IDEOGRAPH-4E00}) missing from font(s) DejaVu Sans.\n", - " fig.canvas.print_figure(bytes_io, **kw)\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1sAAAIjCAYAAAD1OgEdAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbZ9JREFUeJzt3Xl8lOW9///3PRMSwpKELUAMRIILiwIJQYprrXylqF2sdTlf6lZbKy6tR2ul7amWtgraHmsXFD1d9PzUupyvWEsFS0WhLRQDBCK7JSlIEQJCEpaQ5b7v3x+cjEySgYlmMp978no+Hj4q19yZXBcvxnJlZq5xfN/3BQAAAADoUKFkTwAAAAAAUhGbLQAAAABIADZbAAAAAJAAbLYAAAAAIAHYbAEAAABAArDZAgAAAIAEYLMFAAAAAAnAZgsAAAAAEoDNFgAAAAAkAJstAAAAAEgANlsAABO+9KUvqXv37tqyZUur22bPni3HcTR//vwkzExavny5QqGQvv3tb7d5+0MPPSTHcfTHP/6xk2cGALDM8X3fT/YkAACoqqrSiBEjNG7cOC1evDgyXllZqdGjR+uSSy7R//zP/yRtftOnT9evf/1rlZWVafTo0ZHxbdu2adSoUbr00kv14osvtvm1r732mi6//HJ169atzdvr6+t15MgRvf766x16XTgcbucqAQAdiWe2AAAm5Obm6qGHHtKbb76pp59+OjJ+6623qlu3bvrZz36WxNkdfXatf//++trXvqZjf055xx13nHB+nufpyiuv1MGDB9v8p1+/fvJ9v8OvAwAkF5stAIAZX/nKV3TOOefom9/8pj744AM9//zzWrhwoX70ox/ppJNOOu7XXnbZZSosLGzztkmTJqmkpCTy60WLFuncc89VTk6OevXqpdNPP13f+c53jnv/2dnZ+tnPfqa//e1v+tWvfiVJmjdvnv7whz9o9uzZGjx4cDtXCwBIdWy2AABmOI6jJ554QjU1NZo+fbr+/d//XSUlJbrttttO+LVXX321KisrVVpaGjW+bds2/f3vf9c111wjSVq/fr0uu+wy1dfX6wc/+IH+8z//U5/97Gf1t7/97YTf48orr9Sll16qe++9VxUVFfrGN76hs88+W1/72tc+2oIBACktLdkTAADgWKNHj9Y3v/lNzZo1S+FwWH/84x8VCp34Z4Of+9znlJGRoRdeeEETJkyIjL/44otyHEdXXXWVpKPPajU0NGjBggXq379/u+c3Z84cjR49WhMmTNCBAwf02muvyXGcdt8PACD18cwWAMCc5k1QXl6ezjjjjLi+JisrS1OnTtWLL74Y9X6lF154QZ/4xCc0dOhQSVJOTo4k6fe//708z2v33AoKCnT//fdr3759uuuuu+KeHwCg62GzBQAw5b333tP999+vM844Q++9954efvjhuL/26quv1nvvvafly5dLkrZu3apVq1bp6quvjrrmnHPO0Ve+8hUNHDhQ11xzjV588cV2bbyanzk79n1gAAC0xGYLAGDK7bffLklasGCBrrzySj3wwAOqqKiI62s/85nPqEePHpEj2F988UWFQiFdeeWVkWsyMzO1dOlS/fnPf9a1116r8vJyXX311fo//+f/yHXdjl8QAKDLYrMFADBj3rx5evXVV/XDH/5Q+fn5evTRR5Wenh7XARmS1LNnT1122WV66aWX5HmeXnjhBZ133nnKy8uLui4UCumiiy7SI488og0bNuiBBx7Q4sWL9eabbyZiWQCALorNFgDAhAMHDujrX/+6ioqKdMcdd0g6+p6tH/7wh1q4cKFeeumluO7n6quv1s6dO/WrX/1Ka9eujXoJoSTt27ev1deMGzdO0tEPAwYAoKNwGiEAwIT/+I//0M6dO/Xyyy8rHA5Hxm+77TY9/fTTuvPOO/XpT39avXv3Pu79XHLJJerdu7e++c1vKhwO64orroi6/Qc/+IGWLl2qSy+9VAUFBaqqqtJjjz2m/Px8nXvuuQlZGwCga+KZLQBA0q1atUpz5szRrbfeGnVsuySFw2HNnTtXu3bt0n/8x3+c8L66d++uz372szpw4IAuvPBC5ebmRt3+2c9+VkOHDtVvfvMb3XbbbZozZ47OP/98LV68WNnZ2R26LgBA18YzWwCApBs/fryamppi3j5hwoR2HV7xzDPP6Jlnnmnztk996lP61Kc+1e45HuuTn/xk1PHyAAC0hWe2AAAAACABeGYLAIBO8OKLL2r+/Plt3lZbW5uw6wAAyeP4vA4CAAAAADocLyMEAAAAgARgswUAAAAACcBmCwAAAAASgAMy4uB5nnbu3KnevXvLcZxkTwcAAABAkvi+rwMHDigvL0+h0PGfu2KzFYedO3dqyJAhyZ4GAAAAACPee+895efnH/caNltx6N27t6Sjv6FZWVlJnk3naWpqUllZmYqKipSWxh8VS2hjF21sootdtLGLNnbRJrlqa2s1ZMiQyB7heKgTh+aXDmZlZXW5zVbPnj2VlZXFA9kY2thFG5voYhdt7KKNXbSxIZ63F/E5W3Gora1Vdna2ampqutRmy/d91dXVKTMzk/eqGUMbu2hjE13soo1dtLGLNsnVnr0BpxHiuNLT05M9BcRAG7toYxNd7KKNXbSxizbBwGYLMbmuq5UrV8p13WRPBS3Qxi7a2EQXu2hjF23sok1wsNkCAAAAgARgswUAAAAACcBmCwAAAAASgNMI49CVTyN0XVfhcJiTboyhjV20sYkudtHGLtrYRZvk4jRCdJiGhoZkTwEx0MYu2thEF7toYxdt7KJNMLDZQkyu66q8vJyTbgyijV20sYkudtHGLtrYRZvgYLMFAAAAAAmQ1M3W0qVL9ZnPfEZ5eXlyHEevvPJK5LbGxkbde++9OvPMM9WzZ0/l5eXpuuuu086dO6PuY9++fZo2bZqysrKUk5Ojm266SQcPHoy6pry8XOedd566d++uIUOG6OGHH+6M5QEAAADowpK62Tp06JDGjh2rOXPmtLrt8OHDWr16tb73ve9p9erVevnll7V582Z99rOfjbpu2rRpWr9+vRYtWqT58+dr6dKluvnmmyO319bW6uKLL1ZBQYFWrVqlH//4x/r+97+vJ598MuHrSwXhcDjZU0AMtLGLNjbRxS7a2EUbu2gTDGZOI3QcR/PmzdPnP//5mNeUlpbqrLPO0rZt2zR06FBt3LhRo0aNUmlpqUpKSiRJCxcu1CWXXKIdO3YoLy9Pjz/+uL773e9q165dSk9PlyTNmDFDr7zyijZt2tTm96mvr1d9fX3k17W1tRoyZIg++OCDyIkjoVBIoVBInufJ87zItc3jruvq2N/aWOPNp8g0NTVFzaH5AdTytbixxtPS0iIn0xz7exoOh1vNMdY4a2JNrIk1sSbWxJpYE2tiTazp+Guqra1Vv3794jqNMO24txpTU1Mjx3GUk5MjSVq+fLlycnIiGy1Jmjx5skKhkFasWKHLL79cy5cv1/nnnx/ZaEnSlClT9NBDD2n//v3q06dPq+8za9YszZw5s9V4WVmZevbsKUkaMGCAhg8frsrKSu3ZsydyTX5+vvLz87VlyxbV1NRExgsLC5Wbm6t169aprq4uMj5ixAjl5OSorKws6g/ZmDFjlJ6erpUrV0bNoaSkRA0NDSovL4+MhcNhTZgwQTU1NVEbyMzMTI0dO1Z79+5VRUVFZDw7O1sjR47Uzp07tWPHjsh4W2tqbGzUySefrCFDhqTMmqTgdyosLNSGDRuiXjIb9DWlUqfGxkbl5eWl1Jqk4HcaPHiwDhw4oH/9618ps6ZU6dTY2Khu3bql1JpSpVNBQYHq6+tVWVmZMmtKlU7Nj5tUWlNQOh06dEjxCswzW0eOHNE555yjESNG6Nlnn5UkPfjgg3r66ae1efPmqGtzc3M1c+ZMTZ8+XRdffLGGDRumJ554InL7hg0bNHr0aG3YsEEjR45s9b14ZuvouOu6Wr16tcaPH6/09PSUWNOJ5h6UNXmep9LSUhUXF0e+V9DXlCqdWj5uUmFNLecYxDW5rquysjIVFxcrFPrwFfRBXlOqdGpsbNTq1atVXFystLS0lFhTqnSK9bgJ8ppSpVNDQ8MJHzdBW1OQOqXcM1uNjY266qqr5Pu+Hn/88YR/v4yMDGVkZLQaT0tLU1pa9G9Zc6yWmqPEO97yfj/KuOM4bY7HmmM8447jRP69M9ZUtn2/Kvce0rD+PVU0tE9C1hTP3K138jwv8h+clvcV1DUdbzxoazr2cZMqa4pnPChras/1QVlTkDs1/+UmHA63+uFRW9cHYU2p2Kkj1mptTUHu1BGPG2trClKnWLe3+TVxX5kkzRutbdu2afHixVG7x0GDBqmqqirq+qamJu3bt0+DBg2KXLN79+6oa5p/3XwNbJi9YKPmLvnwqeJbLijUjKmtn3kEAAAAgsD052w1b7Teffdd/fnPf1a/fv2ibp80aZKqq6u1atWqyNjixYvleZ4mTpwYuWbp0qVqbGyMXLNo0SKdfvrpbb5fCx9yHEeZmZlyHCfh36ts+/6ojZYkzV1SobLt+xP+vYOoM9ugfWhjE13soo1dtLGLNsGR1PdsHTx4UP/4xz8kSUVFRXrkkUd04YUXqm/fvho8eLC++MUvavXq1Zo/f74GDhwY+bq+fftGDryYOnWqdu/erblz56qxsVE33nijSkpK9Nxzz0k6eqjG6aefrosvvlj33nuv1q1bpy9/+cv66U9/GnVE/PHU1tYqOzs7rtdl4qN5efUO3fXi2lbjj1w1Vl8ozk/CjAAAAIDW2rM3SOozWytXrlRRUZGKiookSXfddZeKiop033336V//+pdeffVV7dixQ+PGjdPgwYMj/yxbtixyH88++6xGjBihiy66SJdcconOPffcqM/Qys7O1p/+9CdVVlZq/Pjxuvvuu3XffffFvdHqyjzPU1VVVdQbDRNlWP+e7Rrv6jqzDdqHNjbRxS7a2EUbu2gTHEl9z9YnP/lJHe+JtXiedOvbt2/kWaxYxowZo7/85S/tnl9X53meKioq1Ldv3zbfbIjkoY1dtLGJLnbRxi7a2EWb4KAOTKjc2/bnFcQaBwAAAKxjswUTeBkhAAAAUg2bLcTkOI6ys7M56cYg2thFG5voYhdt7KKNXbQJjqSeRhgUnEaYeLFOI+ydEdY7Mz+dhBkBAAAArQXmNELY5nmeduzYkdTTCA/Uu/rx65sS/v2DpjPboH1oYxNd7KKNXbSxizbBwWYLMXXmA7loaB/lZXdv87bV2/hg45b4j6xdtLGJLnbRxi7a2EWb4GCzBTMuLz6pzfHigj6dPBMAAADg42OzBTPumTJCA3qnR43l9k7XPVNGJGlGAAAAwEeX1A81hm2hUEgDBgzo1A/Le/LaEs1Z/K521R7RBafnstGKIRltEB/a2EQXu2hjF23sok1wcBphHDiNsHPMXrBRc5dURH59ywWFmjF1ZBJnBAAAAETjNEJ0CM/ztHXr1k5582XZ9v1RGy1JmrukQmXbORyjLZ3ZBu1DG5voYhdt7KKNXbQJDjZbiMnzPO3Zs6dTHsiVew+1Of7829sT/r2DqDPboH1oYxNd7KKNXbSxizbBwWYLJsT6nK0XVu7Q5+f8tZNnAwAAAHx8bLZgQtHQPhrSp+3P2VrzXo1eKOUZLgAAAAQLmy3EFAqFlJ+f3ykn3ZRt36/39h+Jefva96oTPocg6cw2aB/a2EQXu2hjF23sok1wcPQ7Ymp+IHeGWO/ZajZ2SE6nzCMoOrMN2oc2NtHFLtrYRRu7aBMcbIcRk+u62rhxo1zXTfj3ivWerWYn2ox1NZ3ZBu1DG5voYhdt7KKNXbQJDjZbiMn3fdXU1KgzPoqtaGgf3XJBYczbOQY+Wme2QfvQxia62EUbu2hjF22Cg80WzJgxdaQuOK1/zNt5dgsAAABBwmYLZpRt368lW/bGvP1ELzUEAAAALGGzhZhCoZAKCws77aSb4z1z9YWiPBUN7dMp8wiCzm6D+NHGJrrYRRu7aGMXbYKD0wgRUygUUm5ubqd9v+M9c5Wb1fZncHVVnd0G8aONTXSxizZ20cYu2gQH22HE5Lqu1q5da+KkGw7IiGapDaLRxia62EUbu2hjF22Cg80WYvJ9X3V1dZ120s2JDsDggIwPdXYbxI82NtHFLtrYRRu7aBMcbLZgRqPrHfd2DsgAAABAkLDZghndwrH/OHJABgAAAIKGzRZiCofDGjFihMLhcKd8v7+8uyfmbUP69uiUOQRFZ7dB/GhjE13soo1dtLGLNsHBaYSIyXEc5eTkdMr3Ktu+X/PKdsa8/Wdv/EP1TZ5mTB3ZKfOxrjPboH1oYxNd7KKNXbSxizbBwTNbiKmpqUmlpaVqampK+Pd6a3PVCa/hRMIPdWYbtA9tbKKLXbSxizZ20SY42GzhuKwdKcqJhB+y1gYfoo1NdLGLNnbRxi7aBAObLZjwydPj+2C+472vCwAAALCEzRZMKBraR7dcUHjC6+aV7eSlhAAAAAgENluIKRwOa8yYMZ120s2MqSN1wWn9T3gdLyXs/DaIH21sootdtLGLNnbRJjjYbOG40tPTO+17lW3fryVb9p7wOj7c+KjObIP2oY1NdLGLNnbRxi7aBAObLcTkuq5WrlzZaW/AjOcZKz7c+KjOboP40cYmuthFG7toYxdtgoPNFsyI5xmrc08d0AkzAQAAAD4+Nlswo2honxO+Z6vR9TppNgAAAMDHw2YLpoRDznFv7xbmjywAAACCwfF930/2JKyrra1Vdna2ampqlJWVlezpdBrf9+W6rsLhsBzn+JugjlC2fb8uf2zZca+Zd+vZvGdLnd8G8aONTXSxizZ20cYu2iRXe/YGPE2A42poaOi073WiAzJOze3JRusYndkG7UMbm+hiF23soo1dtAkGNluIyXVdlZeXd9pJNyc6IOPdqkN8oPH/6uw2iB9tbKKLXbSxizZ20SY42GzBjKKhfXTLBYXHvebG35bqx69v6qQZAQAAAB8dmy2YMmX0oOPeXl3XqDlvbtWEBxZ10owAAACAj4bNFo4rHA536veL54ONJWnPgYYu/wxXZ7dB/GhjE13soo1dtLGLNsHAaYRx6KqnESZDPCcSNptU2Fe/u3lSgmcEAAAAfIjTCNEhfN9XdXW1rO7Hiwu67smE1tt0ZbSxiS520cYu2thFm+Bgs4WYXNfVpk2bOvWkm3hfRpjbO133TBmR4NnYlYw2iA9tbKKLXbSxizZ20SY40pI9AeBYJzr+fUCvdF01YUiX3mgBAAAgGHhmC6YUDe2jy4vyYt6+52CDXI+nzAEAAGAfmy3E5DiOMjMz5ThOp37fn15dpIG902PePndJRZf/cONktcGJ0cYmuthFG7toYxdtgoPTCOPAaYSdK54TCR+5aqy+UJzfSTMCAAAAjuI0QnQIz/NUVVUlz/M69fvGc0jG42/9oxNmYley2uDEaGMTXeyijV20sYs2wcFmCzF5nqeKiopOfyBv3XPwhNe8W3VIL5Ru74TZ2JSsNjgx2thEF7toYxdt7KJNcLDZgjn7DzXEdd3a96oTOxEAAADgY2CzBXPGDsnp0OsAAACAZGCzhZgcx1F2dnann3Rz9YShGjck+7jX5PbO0NUThnbSjOxJVhucGG1sootdtLGLNnbRJjg4jTAOnEaYHJ9+dIk27Wr7/Vu3XFCoGVNHdvKMAAAA0NVxGiE6hOd52rFjR1LefFm2fX/MjZbEZ20lsw2OjzY20cUu2thFG7toExxsthBTMh/I8Rz/Hs81qYr/yNpFG5voYhdt7KKNXbQJjqRutpYuXarPfOYzysvLk+M4euWVV6Ju931f9913nwYPHqzMzExNnjxZ7777btQ1+/bt07Rp05SVlaWcnBzddNNNOngw+hmR8vJynXfeeerevbuGDBmihx9+ONFLw8c0rH/PE14TzxHxAAAAQLIkdbN16NAhjR07VnPmzGnz9ocfflg///nPNXfuXK1YsUI9e/bUlClTdOTIkcg106ZN0/r167Vo0SLNnz9fS5cu1c033xy5vba2VhdffLEKCgq0atUq/fjHP9b3v/99PfnkkwlfHz66oqF9dMsFhce9Jt4j4gEAAIBkSEvmN586daqmTp3a5m2+7+vRRx/Vf/zHf+hzn/ucJOm///u/NXDgQL3yyiu65pprtHHjRi1cuFClpaUqKSmRJP3iF7/QJZdcop/85CfKy8vTs88+q4aGBv3mN79Renq6Ro8erTVr1uiRRx6J2pShtVAopAEDBigUSs6efMbUkRrWv6fu/X/vtHl7Vz76PdltEBttbKKLXbSxizZ20SY4krrZOp7Kykrt2rVLkydPjoxlZ2dr4sSJWr58ua655hotX75cOTk5kY2WJE2ePFmhUEgrVqzQ5ZdfruXLl+v8889Xenp65JopU6booYce0v79+9WnT59W37u+vl719fWRX9fW1kqSmpqa1NTUJOnoH/JQKCTP86JeL9s87rqujj3oMdZ4OByW4ziR+z12XJJc141rPC0tTb7vR407jqNwONxqjrHG21pTQUFB5PZkrOmqkiF6u2KPXl37viTJ96Um31HxkCxdUZQX+R7tWdPx5h6kTsOGDYu6LRXWlCqdjn3cpMqajp1jUNc0fPhweZ4X9X2DvqZU6VRQUCDP8+T7fsqs6UTjQVlTW4+boK8pFTrF87gJ2pqC1Knl7cdjdrO1a9cuSdLAgQOjxgcOHBi5bdeuXcrNzY26PS0tTX379o26ZtiwYa3uo/m2tjZbs2bN0syZM1uNl5WVqWfPo+8lGjBggIYPH67Kykrt2bMnck1+fr7y8/O1ZcsW1dTURMYLCwuVm5urdevWqa6uLjI+YsQI5eTkqKysLOoP2ZgxY5Senq6VK1dGzaGkpEQNDQ0qLy+PjIXDYU2YMEE1NTXatGlTZDwzM1Njx47V3r17VVFRERnPzs7WyJEjtXPnTu3YsSMy3taaDh8+rFNOOUVDhw5Nypp+9toa9T38L91w6tHxBoU16owzdWFB96j7ac+aUqHTsGHDtHLlyqj/oAR9TanU6fDhwxo6dKhOOeWUlFmTFOxOWVlZysjIULdu3bRz586UWFMqdTp8+LB69OiRUmtKhU7du3dX79691bNnT/3zn/9MiTWlUqfmx00qrSkonQ4div+QNjOfs+U4jubNm6fPf/7zkqRly5bpnHPO0c6dOzV48ODIdVdddZUcx9ELL7ygBx98UE8//bQ2b94cdV+5ubmaOXOmpk+frosvvljDhg3TE088Ebl9w4YNGj16tDZs2KCRI1t/VlNbz2wNGTJEH3zwQeQs/a7wUwHXdbV69WqNHz9e6enpnb6md3Ye0Bce+5vSjnmG3Pelc04doN/eMMHkTzpONN5RnTzPU2lpqYqLiyPfK+hrSpVOLR83qbCmlnMM4ppc11VZWZmKi4ujXnYT5DWlSqfGxkatXr1axcXFSktLS4k1pUqnWI+bIK8pVTo1NDSc8HETtDUFqVNtba369esX1+dsmX1ma9CgQZKk3bt3R222du/erXHjxkWuqaqqivq6pqYm7du3L/L1gwYN0u7du6Ouaf518zUtZWRkKCMjo9V4Wlqa0tKif8uaY7XUHCXe8Zb3+1HGHcdpczzWHOMZdxwn8u+dvaa3NlfJl6PGFqeaLtmyV3e/tFY/vbrouHOPZzyonTzPi/wHp+V9BXVNxxsP2pqOfdykypriGQ/KmtpzfVDWFOROzX+5CYfDrX541Nb1QVhTKnbqiLVaW1OQO3XE48bamoLUKdbtbTH7rrphw4Zp0KBBeuONNyJjtbW1WrFihSZNmiRJmjRpkqqrq7Vq1arINYsXL5bneZo4cWLkmqVLl6qxsTFyzaJFi3T66ae3+RJCBMO8sp1d+kONAQAAYF9SN1sHDx7UmjVrtGbNGklHD8VYs2aNtm/fLsdxdOedd+pHP/qRXn31Vb3zzju67rrrlJeXF3mp4ciRI/XpT39aX/3qV/X222/rb3/7m26//XZdc801ysvLkyT93//7f5Wenq6bbrpJ69ev1wsvvKCf/exnuuuuu5K06uAIhULKz89v8ycEnSEvJ/O4t3flDzVOdhvERhub6GIXbeyijV20CY6kvmfrrbfe0oUXXthq/Prrr9dTTz0l3/d1//3368knn1R1dbXOPfdcPfbYYzrttNMi1+7bt0+33367/vCHPygUCumKK67Qz3/+c/Xq1StyTXl5uW677TaVlpaqf//+uuOOO3TvvffGPc/a2lplZ2fH9bpMdJwrHv+bVm2rjnn7vFvPVtFQnp0EAABA52nP3sDMARmWddXNluu62rJli0477bSYr2lNlLLt+3X5Y8ti3v6Fojw90sZ7trqKZLbB8dHGJrrYRRu7aGMXbZKrPXsDnntETL7vq6amRsnYj7+1ueq4t+8/3Hjc21NdMtvg+GhjE13soo1dtLGLNsHBZguB9ObmPRyQAQAAANPYbMGkT56ee8JruvIBGQAAALCPzRZiCoVCKiwsTMpJN0VD++iWCwqPe82w/j07aTb2JLMNjo82NtHFLtrYRRu7aBMcHJARh656QIYFZdv3q3LvIf1+zb+0ZMveyPj0Cwp179SRSZwZAAAAuiIOyECHcF1Xa9euleu6SZtD0dA+2rL7QNRG6/KivC6/0bLQBm2jjU10sYs2dtHGLtoEB5stxOT7vurq6pJ60k3Z9v2au6Qiamxe2c4ufziGhTZoG21sootdtLGLNnbRJjjYbMG0WIdgcDgGAAAArGOzBdNiHYLRlQ/HAAAAQDCw2UJM4XBYI0aMSOonk7d1KuH0CwpVNLRPkmZkg4U2aBttbKKLXbSxizZ20SY4OI0wDpxGmHzNpxIO69+zy2+0AAAAkDycRogO0dTUpNLSUjU1NSV7Kioa2kdfKM5no/W/LLVBNNrYRBe7aGMXbeyiTXCw2cJxcaSoXbSxizY20cUu2thFG7toEwxstgAAAAAgAdhsAQAAAEACcEBGHLrqARnNH5iXmZkpx3GSPR0cgzZ20cYmuthFG7toYxdtkosDMtBh0tPTkz0FxEAbu2hjE13soo1dtLGLNsHAZgsxua6rlStX8gZMg2hjF21sootdtLGLNnbRJjjYbAEAAABAArDZAgAAAIAEYLMFAAAAAAnAaYRx6MqnEbquq3A4zEk3xtDGLtrYRBe7aGMXbeyiTXJxGiE6TENDQ7KngBhoYxdtbKKLXbSxizZ20SYY2GwhJtd1VV5ezkk3BtHGLtrYRBe7aGMXbeyiTXCw2QIAAACABGCzBQAAAAAJwGYLxxUOh5M9BcRAG7toYxNd7KKNXbSxizbBwGmEceiqpxECAAAAiMZphOgQvu+rurpa7MftoY1dtLGJLnbRxi7a2EWb4GCzhZhc19WmTZs46cYg2thFG5voYhdt7KKNXbQJDjZbAAAAAJAAbLYAAAAAIAHYbCEmx3GUmZkpx3GSPRW0QBu7aGMTXeyijV20sYs2wcFphHHgNEIAAAAAEqcRooN4nqeqqip5npfsqaAF2thFG5voYhdt7KKNXbQJDjZbiMnzPFVUVPBANog2dtHGJrrYRRu7aGMXbYKDzRYAAAAAJACbLQAAAABIADZbiMlxHGVnZ3PSjUG0sYs2NtHFLtrYRRu7aBMcnEYYB04jBAAAACBxGiE6iOd52rFjB2++NIg2dtHGJrrYRRu7aGMXbYKDzRZi4oFsF23soo1NdLGLNnbRxi7aBAebLQAAAABIADZbAAAAAJAAbLYQUygU0oABAxQK8cfEGtrYRRub6GIXbeyijV20CQ5OI4wDpxECAAAAkDiNEB3E8zxt3bqVN18aRBu7aGMTXeyijV20sYs2wcFmCzF5nqc9e/bwQDaINnbRxia62EUbu2hjF22Cg80WAAAAACQAmy0AAAAASAA2W4gpFAopPz+fk24Moo1dtLGJLnbRxi7a2EWb4OA0wjhwGiEAAAAAidMI0UFc19XGjRvlum6yp4IWaGMXbWyii120sYs2dtEmONhsISbf91VTUyOe/LSHNnbRxia62EUbu2hjF22Cg80WAAAAACQAmy0AAAAASAA2W4gpFAqpsLCQk24Moo1dtLGJLnbRxi7a2EWb4OA0wjhwGiEAAAAAidMI0UFc19XatWs56cYg2thFG5voYhdt7KKNXbQJDjZbiMn3fdXV1XHSjUG0sYs2NtHFLtrYRRu7aBMcpjdbruvqe9/7noYNG6bMzEwNHz5cP/zhD6P+YPm+r/vuu0+DBw9WZmamJk+erHfffTfqfvbt26dp06YpKytLOTk5uummm3Tw4MHOXg4AAACALsT0Zuuhhx7S448/rl/+8pfauHGjHnroIT388MP6xS9+Ebnm4Ycf1s9//nPNnTtXK1asUM+ePTVlyhQdOXIkcs20adO0fv16LVq0SPPnz9fSpUt18803J2NJAAAAALoI0wdkXHbZZRo4cKB+/etfR8auuOIKZWZm6plnnpHv+8rLy9Pdd9+tb37zm5KkmpoaDRw4UE899ZSuueYabdy4UaNGjVJpaalKSkokSQsXLtQll1yiHTt2KC8v74Tz6KoHZDR/YF52drYcx0n2dHAM2thFG5voYhdt7KKNXbRJrvbsDdI6aU4fydlnn60nn3xSW7Zs0Wmnnaa1a9fqr3/9qx555BFJUmVlpXbt2qXJkydHviY7O1sTJ07U8uXLdc0112j58uXKycmJbLQkafLkyQqFQlqxYoUuv/zyVt+3vr5e9fX1kV/X1tZKkpqamtTU1CTp6JGboVBInufJ87zItc3jrutGvdwx1ng4HJbjOJH7PXZcUqs3PsYaT0tLk+/7UeOO4ygcDreaY6zxttbUq1cv+b4vx3FSZk3Hm3uQ1pSVlRV1P6mwplTpdOzjJlXWdOwcg7qmnJwceZ7X5uMmqGtKlU69evWS67optaYTjQdlTW09boK+plTpdKLHTRDXdKK5W1lTy9uPx/Rma8aMGaqtrdWIESMUDofluq4eeOABTZs2TZK0a9cuSdLAgQOjvm7gwIGR23bt2qXc3Nyo29PS0tS3b9/INS3NmjVLM2fObDVeVlamnj17SpIGDBig4cOHq7KyUnv27Ilck5+fr/z8fG3ZskU1NTWR8cLCQuXm5mrdunWqq6uLjI8YMUI5OTkqKyuL+kM2ZswYpaena+XKlVFzKCkpUUNDg8rLyyNj4XBYEyZMUE1NjTZt2hQZz8zM1NixY7V3715VVFRExrOzszVy5Ejt3LlTO3bsiIy3XFPzT01GjhypgoKClFhTqnQqKCjQX/7yF3Xv3j3yE62grylVOjU/boYPH65TTz01JdaUCp169+6tw4cPKzc3V++//35KrClVOm3dujXyE/qcnJyUWFOqdMrIyFBTU5Py8/O1bdu2lFhTqnTauHFj5HHTo0ePlFhTkDodOnRI8TL9MsLnn39e99xzj3784x9r9OjRWrNmje6880498sgjuv7667Vs2TKdc8452rlzpwYPHhz5uquuukqO4+iFF17Qgw8+qKefflqbN2+Ouu/c3FzNnDlT06dPb/V923pma8iQIfrggw8iTxV2hZ8KuK6r1atXa/z48UpPT0+JNZ1o7kFZk+d5Ki0tVXFxceR7BX1NqdKp5eMmFdbUco5BXJPruiorK1NxcXHUh4AGeU2p0qmxsVGrV69WcXGx0tLSUmJNqdIp1uMmyGtKlU4NDQ0nfNwEbU1B6lRbW6t+/foF/2WE99xzj2bMmKFrrrlGknTmmWdq27ZtmjVrlq6//noNGjRIkrR79+6ozdbu3bs1btw4SdKgQYNUVVUVdb9NTU3at29f5OtbysjIUEZGRqvxtLQ0paVF/5Y1x2qpOUq84y3v96OMO47T5nisOcYz7jhO5N9TZU3HCuqaPM+L/Aen5X0FdU3HGw/amo593KTKmuIZD8qa2nN9UNYU5E7Nf7kJh8OtfnjU1vVBWFMqduqItVpbU5A7dcTjxtqagtQp1u1tMX0a4eHDh1v9hjXvZiVp2LBhGjRokN54443I7bW1tVqxYoUmTZokSZo0aZKqq6u1atWqyDWLFy+W53maOHFiJ6wCAAAAQFdk+mWEN9xwg/785z/riSee0OjRo1VWVqabb75ZX/7yl/XQQw9JOno8/OzZs/X0009r2LBh+t73vqfy8nJt2LBB3bt3lyRNnTpVu3fv1ty5c9XY2Kgbb7xRJSUleu655+KaR1c+jbCurk6ZmZmcdGMMbeyijU10sYs2dtHGLtokV8qcRviLX/xC3/ve93TrrbeqqqpKeXl5+trXvqb77rsvcs23vvUtHTp0SDfffLOqq6t17rnnauHChZGNliQ9++yzuv3223XRRRcpFArpiiuu0M9//vNkLClw0tPTkz0FxEAbu2hjE13soo1dtLGLNsFg+pktK7rqM1tNTU1auXKlSkpK2vXaVCQebeyijU10sYs2dtHGLtokV3v2BqbfswUAAAAAQcVmCwAAAAASgM0WAAAAACQA79mKQ1d9z1bzB8w1f5YD7KCNXbSxiS520cYu2thFm+TiPVvoMA0NDcmeAmKgjV20sYkudtHGLtrYRZtgYLOFmFzXVXl5uVzXTfZU0AJt7KKNTXSxizZ20cYu2gQHmy0AAAAASAA2WwAAAACQAGy2cFzhcDjZU0AMtLGLNjbRxS7a2EUbu2gTDJxGGIeuehohAAAAgGicRogO4fu+qqurxX7cHtrYRRub6GIXbeyijV20CQ42W4jJdV1t2rSJk24Moo1dtLGJLnbRxi7a2EWb4GCzBQAAAAAJwGYLAAAAABKAzRZichxHmZmZchwn2VNBC7SxizY20cUu2thFG7toExycRhgHTiMEAAAAIHEaITqI53mqqqqS53nJngpaoI1dtLGJLnbRxi7a2EWb4GCzhZg8z1NFRQUPZINoYxdtbKKLXbSxizZ20SY42GwBAAAAQAKw2QIAAACABGCzhZgcx1F2djYn3RhEG7toYxNd7KKNXbSxizbBwWmEceA0QgAAAAASpxGig3iepx07dvDmS4NoYxdtbKKLXbSxizZ20SY42GwhJh7IdtHGLtrYRBe7aGMXbeyiTXCw2QIAAACABGCzBQAAAAAJwGYLMYVCIQ0YMEChEH9MrKGNXbSxiS520cYu2thFm+DgNMI4cBohAAAAAInTCNFBPM/T1q1befOlQbSxizY20cUu2thFG7toExxsthCT53nas2cPD2SDaGMXbWyii120sYs2dtEmONhsAQAAAEACsNkCAAAAgARgs4WYQqGQ8vPzOenGINrYRRub6GIXbeyijV20CQ5OI4wDpxECAAAAkDiNEB3EdV1t3LhRrusmeypogTZ20cYmuthFG7toYxdtgoPNFmLyfV81NTXiyU97aGMXbWyii120sYs2dtEmONhsAQAAAEACsNkCAAAAgARgs4WYQqGQCgsLOenGINrYRRub6GIXbeyijV20CY52nUY4adIkOY4T17W+76tv37764x//+JEnZwWnEQIAAACQ2rc3SGvPHR85ckRlZWVxXz9hwoT23D2McV1X69at0xlnnKFwOJzs6eAYtLGLNjbRxS7a2EUbu2gTHO167jHeZ7U+6vWwxfd91dXVcdKNQbSxizY20cUu2thFG7toExy80BMAAAAAEoDNFgAAAAAkAJstxBQOhzVixAheC2wQbeyijU10sYs2dtHGLtoER7sOyDhw4IA+9alPnfD1oY7jyPd9XkcacI7jKCcnJ9nTQBtoYxdtbKKLXbSxizZ20SY42rXZWr9+fbs2UJz9H2xNTU0qKytTUVGR0tLa9UcFCUYbu2hjE13soo1dtLGLNsHRrjo/+9nPtH///rivz8/P16233truScEO13WTPQXEQBu7aGMTXeyijV20sYs2wdCuzdZ///d/a86cOXE/u3XPPfew2QIAAADQJbVrs5WWlqbzzz8/7ut5zxYAAACArooPNUZM4XBYY8aM4aQbg2hjF21sootdtLGLNnbRJjg4wQLHlZ6enuwpIAba2EUbm+hiF23soo1dtAkGNluIyXVdrVy5kjdgGkQbu2hjE13soo1dtLGLNsHRrvds1dfX67//+7/jupbP2QIAAADQlbVrs/Xd735XBw4ciPv673znO+2eEAAAAACkgnZttiZNmqTGxsa4r8/MzGz3hAAAAAAgFTh+O17rN2LECJ199tknfHmg4zjyfV/r16/X22+//bEnmWy1tbXKzs5WTU2NsrKykj2dTuP7vlzXVTgc5mRJY2hjF21sootdtLGLNnbRJrnaszdo1zNbmZmZ+s1vfhP39RMmTGjP3cOghoYGnqE0ijZ20cYmuthFG7toYxdtgoHP2UJMruuqvLyck24Moo1dtLGJLnbRxi7a2EWb4ODodwAAAABIAPObrX/961/60pe+pH79+ikzM1NnnnmmVq5cGbnd933dd999Gjx4sDIzMzV58mS9++67Ufexb98+TZs2TVlZWcrJydFNN92kgwcPdvZSAAAAAHQhCd1sfdzP2dq/f7/OOeccdevWTQsWLNCGDRv0n//5n+rTp0/kmocfflg///nPNXfuXK1YsUI9e/bUlClTdOTIkcg106ZN0/r167Vo0SLNnz9fS5cu1c033/yx5tZVhMPhZE8BMdDGLtrYRBe7aGMXbeyiTTC06zTCyy+/XLt27Yr7zkeOHNmuAzVamjFjhv72t7/pL3/5S5u3+76vvLw83X333frmN78pSaqpqdHAgQP11FNP6ZprrtHGjRs1atQolZaWqqSkRJK0cOFCXXLJJdqxY4fy8vJOOI+uehohAAAAgGgJO41w3rx5H2ti7fXqq69qypQpuvLKK7VkyRKddNJJuvXWW/XVr35VklRZWaldu3Zp8uTJka/Jzs7WxIkTtXz5cl1zzTVavny5cnJyIhstSZo8ebJCoZBWrFihyy+/vNX3ra+vV319feTXtbW1kqSmpiY1NTVJkkKhkEKhkDzPk+d5kWubx13XjXpmL9Z485Gdzfd77LikVm98jDWelpYWOQa0meM4CofDreYYa7zlmnzfV21trXJychQOh1NiTSeae1DW5DiO9u/fr969e0cOogn6mlKlU8vHTSqsqeUcg7gmSTp48KB69+7d5hyDuKZU6eS6rmpra5WVlaVQKJQSa0qVTlLbj5sgrylVOjU1NZ3wcRO0NQWpU8vbj6ddm60rrrhC77//ftzXjxo1Sr/61a/a8y2iVFRU6PHHH9ddd92l73znOyotLdXXv/51paen6/rrr488yzZw4MCorxs4cGDktl27dik3Nzfq9rS0NPXt2zfms3SzZs3SzJkzW42XlZWpZ8+ekqQBAwZo+PDhqqys1J49eyLX5OfnKz8/X1u2bFFNTU1kvLCwULm5uVq3bp3q6uoi4yNGjFBOTo7Kysqi/pCNGTNG6enpUe9Pk6SSkhI1NDSovLw8MhYOhzVhwgTV1NRo06ZNkfHMzEyNHTtWe/fuVUVFRWQ8OztbI0eO1M6dO7Vjx47IeMs1+b6v6upqjRo1SgUFBSmxplTpVFBQoDVr1igzMzOy2Qr6mlKlU/Pj5pRTTtGpp56aEmtKhU69e/fWgQMHNHjw4Kj/HwvymlKl09atW1VdXa2cnBzl5OSkxJpSpVNGRobq6+tVUFCgbdu2pcSaUqXTxo0bI4+bHj16pMSagtTp0KFDile7XkZYVFSksrKyuO/8rLPO+lgfapyenq6SkhItW7YsMvb1r39dpaWlWr58uZYtW6ZzzjlHO3fu1ODBgyPXXHXVVXIcRy+88IIefPBBPf3009q8eXPUfefm5mrmzJmaPn16q+/b1jNbQ4YM0QcffBB5qrAr/FTAdV2tXr1a48ePV3p6ekqs6URzD8qaPM9TaWmpiouLI98r6GtKlU4tHzepsKaWcwzimlzXVVlZmYqLixUKffh25SCvKVU6NTY2avXq1SouLlZaWlpKrClVOsV63AR5TanSqaGh4YSPm6CtKUidamtr1a9fv45/GWFnf27W4MGDNWrUqKixkSNH6v/9v/8nSRo0aJAkaffu3VGbrd27d2vcuHGRa6qqqqLuo6mpSfv27Yt8fUsZGRnKyMhoNZ6Wlqa0tOjfsuZYLTVHiXe85f1+lHHHcdocjzXHeMYdx4n8e6qs6VhBXZPneZH/4LS8r6Cu6XjjQVvTsY+bVFlTPONBWVN7rg/KmoLcqfkvN+FwuNUPj9q6PghrSsVOHbFWa2sKcqeOeNxYW1OQOsW6vS2mj34/55xzWj0jtWXLFhUUFEiShg0bpkGDBumNN96I3F5bW6sVK1Zo0qRJkqRJkyapurpaq1atilyzePFieZ6niRMndsIqgstxnKiXqcEO2thFG5voYhdt7KKNXbQJjna9jLC4uFirV6+O+84/7ssIS0tLdfbZZ2vmzJm66qqr9Pbbb+urX/2qnnzySU2bNk2S9NBDD2n27Nl6+umnNWzYMH3ve99TeXm5NmzYoO7du0uSpk6dqt27d2vu3LlqbGzUjTfeqJKSEj333HNxzYPTCAEAAABI7dsbmH5ma8KECZo3b55+97vf6YwzztAPf/hDPfroo5GNliR961vf0h133KGbb75ZEyZM0MGDB7Vw4cLIRkuSnn32WY0YMUIXXXSRLrnkEp177rl68sknk7GkQPE8T1VVVa1O9ELy0cYu2thEF7toYxdt7KJNcLTrPVuHDh3Sl7/85biu9X3/Y3+osSRddtlluuyyy2Le7jiOfvCDH+gHP/hBzGv69u0b97NY+JDneaqoqFDfvn3bfP0rkoc2dtHGJrrYRRu7aGMXbYKjXZutBQsWqLGxMe7rMzMz2z0hAAAAAEgF7dpsrVixQgcOHIj7+tzcXA0dOrTdkwIAAACAoGvX844PPPCAunfvHjka/UT/PPjgg4maNzqB4zjKzs7mpBuDaGMXbWyii120sYs2dtEmOBL6ocYTJkxQaWnpR5qYJZxGCAAAAEBK4GmE7d09s9sONs/ztGPHDk66MYg2dtHGJrrYRRu7aGMXbYKD40sQEw9ku2hjF21sootdtLGLNnbRJjjYbAEAAABAArTrNMLGxkYtXbo0rms76nO2AAAAACCI2rXZuvbaa7VgwYK4r7/hhhvaOx8YEgqFNGDAAD4szyDa2EUbm+hiF23soo1dtAmOdp1G2NjY2K5nq0KhkNLS2rWfM4nTCAEAAABI7dsbtGsnNHr0aOXn58d1re/7Onz4sFasWNGebwFDPM9TZWWlhg0bxk9OjKGNXbSxiS520cYu2thFm+Bo12arZ8+eWrx4cdzXT5gwod0Tgh2e52nPnj0qKCjggWwMbeyijU10sYs2dtHGLtoEB5+zBQAAAAAJwFYYAAAAABKAzRZiCoVCys/P5+lpg2hjF21sootdtLGLNnbRJjiCf1QgEqb5gQx7aGMXbWyii120sYs2dtEmONq12aqtrdXZZ58t3/dP+H4s3/fVv3//jzU5JJfrutqyZYtOO+00hcPhZE8Hx6CNXbSxiS520cYu2thFm+Bo12brgw8+0MyZMzVt2rREzQeG+L6vmpqadn22GjoHbeyijU10sYs2dtHGLtoER7te6PnAAw/olltu0ZVXXql9+/Ylak4AAAAAEHjt2mzdeuutKi8v1wcffKBRo0bpD3/4Q6LmBQAAAACB1u4DMoYNG6bFixfrl7/8pb7whS9o5MiRSkuLvpvVq1d32ASRPKFQSIWFhZx0YxBt7KKNTXSxizZ20cYu2gTHRzqNcNu2bXr55ZfVp08ffe5zn2u12UJqCIVCys3NTfY00Aba2EUbm+hiF23soo1dtAmOdu+S/uu//kt33323Jk+erPXr12vAgAGJmBcMcF1X69at0xlnnMFJN8bQxi7a2EQXu2hjF23sok1wtGuz9elPf1pvv/22fvnLX+q6665L1JxghO/7qqur46Qbg2hjF21sootdtLGLNnbRJjjatdlyXVfl5eV8iBoAAAAAnEC7NluLFi1K1DwAAAAAIKVwhAliCofDGjFiBK8FNog2dtHGJrrYRRu7aGMXbYLD8Xmx5wnV1tYqOztbNTU1ysrKSvZ0AAAAACRJe/YGPLOFmJqamlRaWqqmpqZkTwUt0MYu2thEF7toYxdt7KJNcLDZwnG5rpvsKSAG2thFG5voYhdt7KKNXbQJBjZbAAAAAJAAbLYAAAAAIAE4ICMOXfWAjOYPzMvMzJTjOMmeDo5BG7toYxNd7KKNXbSxizbJ1Z69Qbs+ZwtdT3p6erKn8LGUbd+vyr2HNKx/TxUN7ZPs6XSooLdJZbSxiS520cYu2thFm2DgZYSIyXVdrVy5MrBvwJy9YKMuf2yZ7npxrS5/bJlmL9iY7Cl1mKC3SWW0sYkudtHGLtrYRZvgYLOFlFS2fb/mLqmIGpu7pEJl2/cnaUYAAADoathsISVV7j3UrnEAAACgo7HZQkoa1r9nu8YBAACAjsZphHHoyqcRuq6rcDgcyJNuZi/YGPVSwukXFOreqSOTOKOOE/Q2qYw2NtHFLtrYRRu7aJNcnEaIDtPQ0KDMzMxkT+MjmTF1pKaMHpSypxEGuU2qo41NdLGLNnbRxi7aBAMvI0RMruuqvLw80CfdFA3toy8U56fcRisV2qQq2thEF7toYxdt7KJNcLDZAgAAAIAEYLMFAAAAAAnAZgvHFQ6Hkz0FxEAbu2hjE13soo1dtLGLNsHAaYRx6KqnEQIAAACI1p69Ac9sISbf91VdXS324/bQxi7a2EQXu2hjF23sok1wsNlCTK7ratOmTZx0YxBt7KKNTXSxizZ20cYu2gQHmy0AAAAASAA2WwAAAACQAGy2EJPjOMrMzJTjOMmeClqgjV20sYkudtHGLtrYRZvg4DTCOHAaIQAAAACJ0wjRQTzPU1VVlTzPS/ZU0AJt7KKNTXSxizZ20cYu2gQHmy3E5HmeKioqeCAbRBu7aGMTXeyijV20sYs2wcFmCwAAAAASgM0WAAAAACQAmy3E5DiOsrOzOenGINrYRRub6GIXbeyijV20CQ5OI4wDpxECAAAAkDiNEB3E8zzt2LGDN18aRBu7aGMTXeyijV20sYs2wcFmCzHxQLaLNnbRxia62EUbu2hjF22CIy3ZEwDao2z7flXuPaRh/XuqaGifZE8HAAAAiClQz2zNnj1bjuPozjvvjIwdOXJEt912m/r166devXrpiiuu0O7du6O+bvv27br00kvVo0cP5ebm6p577lFTU1Mnzx4f1+wFG3X5Y8t014trdfljyzR7wcZkTwkAAACIKTCbrdLSUj3xxBMaM2ZM1Pi///u/6w9/+INeeuklLVmyRDt37tQXvvCFyO2u6+rSSy9VQ0ODli1bpqefflpPPfWU7rvvvs5eQuCEQiENGDBAoVDy/5iUbd+vuUsqosbmLqlQ2fb9SZpRcllqg2i0sYkudtHGLtrYRZvgCEShgwcPatq0afqv//ov9enz4UvHampq9Otf/1qPPPKIPvWpT2n8+PH67W9/q2XLlunvf/+7JOlPf/qTNmzYoGeeeUbjxo3T1KlT9cMf/lBz5sxRQ0NDspYUCKFQSMOHDzfxQK7ce6hd46nOUhtEo41NdLGLNnbRxi7aBEcg3rN122236dJLL9XkyZP1ox/9KDK+atUqNTY2avLkyZGxESNGaOjQoVq+fLk+8YlPaPny5TrzzDM1cODAyDVTpkzR9OnTtX79ehUVFbX6fvX19aqvr4/8ura2VpLU1NQUeflhKBRSKBSS53lRb05sHnddV8eeqh9rPBwOy3GcVi9rDIfDko4+MxfPeFpamnzfjxp3HEfhcLjVHGONt1yT53natm2bhg0bprS0tKSuqaBvd3UL+fJ9qcl3FJKvcEgq6NtdTU1Nca/pRHMPSidJ2rp1qwoKCiK/DvqaUqVTy8dNKqyp5RyDuCbf97V9+3YVFBRE3UeQ15QqnZqamrRt2zYVFBQoHA6nxJpSpVOsx02Q15QqnRobG0/4uAnamoLUqT1vRzK/2Xr++ee1evVqlZaWtrpt165dSk9PV05OTtT4wIEDtWvXrsg1x260mm9vvq0ts2bN0syZM1uNl5WVqWfPnpKkAQMGaPjw4aqsrNSePXsi1+Tn5ys/P19btmxRTU1NZLywsFC5ublat26d6urqIuMjRoxQTk6OysrKov6QjRkzRunp6Vq5cmXUHEpKStTQ0KDy8vLIWDgc1oQJE1RTU6NNmzZFxjMzMzV27Fjt3btXFRUfvgQvOztbI0eO1M6dO7Vjx47IeMs1+b6v6upqpaenq6CgIOlruv8T3bX2/cN6qTKsU7N9XTc6U427/qGVu+JfU6p0Kigo0Hvvvae9e/dGPtAw6GtKlU7NjxtJOvXUU1NiTc2C3Kl37946cOCA0tLS9P7776fEmlKl09atW1VdXa29e/cqJycnJdaUKp0yMjJUX1+vHj16aNu2bSmxplTptHHjxsjjpkePHimxpiB1OnQo/ldWmf5Q4/fee08lJSVatGhR5L1an/zkJzVu3Dg9+uijeu6553TjjTdGPQslSWeddZYuvPBCPfTQQ7r55pu1bds2vf7665HbDx8+rJ49e+q1117T1KlTW33ftp7ZGjJkiD744IPIB5d1hZ8KuK6r1atXa/z48UpPTzexpvIdNfrnvjqd3K+Hxpz04YfIWflJx0dZ00fp5HmeSktLVVxcHPleQV9TqnRq+bhJhTW1nGMQ1+S6rsrKylRcXBz1spsgrylVOjU2Nmr16tUqLi5WWlpaSqwpVTrFetwEeU2p0qmhoeGEj5ugrSlInWpra9WvX7+4PtTY9DNbq1atUlVVlYqLiyNjrutq6dKl+uUvf6nXX39dDQ0Nqq6ujnp2a/fu3Ro0aJAkadCgQXr77bej7rf5tMLma1rKyMhQRkZGq/G0tDSlpUX/ljXHaqk5SrzjLe/3o4w7jtPmeKw5xjPuOE7k3y2sqfjkfio+uc0vl9T+tVpY04nm2Na453mR/+C0vK+grul440Fb07GPm1RZUzzjQVlTe64PypqC3Kn5LzfNL4U63tyDsqZU7NQRa7W2piB36ojHjbU1BalTrNvbYvpddRdddJHeeecdrVmzJvJPSUmJpk2bFvn3bt266Y033oh8zebNm7V9+3ZNmjRJkjRp0iS98847qqqqilyzaNEiZWVladSoUZ2+piAJhULKz89v8w8tkos2dtHGJrrYRRu7aGMXbYLD9MsI23Lsywglafr06Xrttdf01FNPKSsrS3fccYckadmyZZKOPhM2btw45eXl6eGHH9auXbt07bXX6itf+YoefPDBuL5nbW2tsrOz43qqEAAAAEDqas/eIPDb4Z/+9Ke67LLLdMUVV+j888/XoEGD9PLLL0duD4fDmj9/vsLhsCZNmqQvfelLuu666/SDH/wgibMOBtd1tXHjxlavrUXy0cYu2thEF7toYxdt7KJNcJh+z1Zb3nrrrahfd+/eXXPmzNGcOXNifk1BQYFee+21BM8s9fi+r5qaGgXsyc8ugTZ20cYmuthFG7toYxdtgiPwz2wBAAAAgEVstgAAAAAgAdhsIaZQKKTCwkJOujGINnbRxia62EUbu2hjF22CI3CnESYDpxECAAAAkLrYaYRIHNd1tXbtWk66MYg2dtHGJrrYRRu7aGMXbYKDzRZi8n1fdXV1nHRjEG3soo1NdLGLNnbRxi7aBAebLQAAAABIADZbAAAAAJAAbLYQUzgc1ogRIxQOh5M9FbRAG7toYxNd7KKNXbSxizbBwWmEceA0QgAAAAASpxGigzQ1Nam0tFRNTU3JngpaoI1dtLGJLnbRxi7a2EWb4GCzhePiSFG7aGMXbWyii120sYs2dtEmGNhsAQAAAEACsNkCAAAAgATggIw4dNUDMpo/MC8zM1OO4yR7OjgGbeyijU10sYs2dtHGLtokFwdkoMOkp6cnewqIgTZ20cYmuthFG7toYxdtgoHNFmJyXVcrV67kDZgG0cYu2thEF7toYxdt7KJNcLDZAgAAAIAEYLMFAAAAAAnAZgsAAAAAEoDTCOPQlU8jdF1X4XCYk26MoY1dtLGJLnbRxi7a2EWb5OI0QnSYhoaGZE8BMdDGLtrYRBe7aGMXbeyiTTCw2UJMruuqvLyck24Moo1dtLGJLnbRxi7a2EWb4GCzBQAAAAAJwGYLAAAAABKAzRaOKxwOJ3sKiIE2dtHGJrrYRRu7aGMXbYKB0wjj0FVPIwQAAAAQjdMI0SF831d1dbXYj9tDG7toYxNd7KKNXbSxizbBwWYLMbmuq02bNnHSjUG0sYs2NtHFLtrYRRu7aBMcbLYAAAAAIAHYbAEAAABAArDZQkyO4ygzM1OO4yR7KmiBNnbRxia62EUbu2hjF22Cg9MI48BphAAAAAAkTiNEB/E8T1VVVfI8L9lTQQu0sYs2NtHFLtrYRRu7aBMcbLYQk+d5qqio4IFsEG3soo1NdLGLNnbRxi7aBAebLQAAAABIADZbAAAAAJAAbLYQk+M4ys7O5qQbg2hjF21sootdtLGLNnbRJjg4jTAOnEYIAAAAQOI0QnQQz/O0Y8cO3nxpEG3soo1NdLGLNnbRxi7aBAebLcTEA9ku2thFG5voYhdt7KKNXbQJDjZbAAAAAJAAbLYAAAAAIAHYbCGmUCikAQMGKBTij4k1tLGLNjbRxS7a2EUbu2gTHJxGGAdOIwQAAAAgcRohOojnedq6dStvvjSINnbRxia62EUbu2hjF22Cg80WYvI8T3v27OGBbBBt7KKNTXSxizZ20cYu2gQHmy0AAAAASAA2WwAAAACQAGy2EFMoFFJ+fj4n3RhEG7toYxNd7KKNXbSxizbBwWmEceA0QgAAAAASpxGig7iuq40bN8p13WRPBS3Qxi7a2EQXu2hjF23sok1wsNlCTL7vq6amRjz5aQ9t7KKNTXSxizZ20cYu2gQHmy0AAAAASAA2WwAAAACQAGy2EFMoFFJhYSEn3RhEG7toYxNd7KKNXbSxizbBwWmEceA0QgAAAAASpxGig7iuq7Vr13LSjUG0sYs2NtHFLtrYRRu7aBMcbLYQk+/7qqur46Qbg2hjF21sootdtLGLNnbRJjhMb7ZmzZqlCRMmqHfv3srNzdXnP/95bd68OeqaI0eO6LbbblO/fv3Uq1cvXXHFFdq9e3fUNdu3b9ell16qHj16KDc3V/fcc4+ampo6cykAAAAAuhjTm60lS5botttu09///nctWrRIjY2Nuvjii3Xo0KHINf/+7/+uP/zhD3rppZe0ZMkS7dy5U1/4whcit7uuq0svvVQNDQ1atmyZnn76aT311FO67777krEkAAAAAF1EoA7I2LNnj3Jzc7VkyRKdf/75qqmp0YABA/Tcc8/pi1/8oiRp06ZNGjlypJYvX65PfOITWrBggS677DLt3LlTAwcOlCTNnTtX9957r/bs2aP09PQTft+uekBG8wfmZWdny3GcZE8Hx6CNXbSxiS520cYu2thFm+Rqz94grZPm1CFqamokSX379pUkrVq1So2NjZo8eXLkmhEjRmjo0KGRzdby5ct15plnRjZakjRlyhRNnz5d69evV1FRUavvU19fr/r6+siva2trJUlNTU2Rlx+GQiGFQiF5nifP8yLXNo+7rhv1OtpY4+FwWI7jtHpZYzgclqRWb3yMNZ6Wlibf96PGHcdROBxuNcdY422tqVevXvJ9X47jpMyajjf3IK0pKysr6n5SYU2p0unYx02qrOnYOQZ1TTk5OfI8r83HTVDXlCqdevXqJdd1U2pNJxoPypraetwEfU2p0ulEj5sgrulEc7eypva8HSkwmy3P83TnnXfqnHPO0RlnnCFJ2rVrl9LT05WTkxN17cCBA7Vr167INcdutJpvb76tLbNmzdLMmTNbjZeVlalnz56SpAEDBmj48OGqrKzUnj17Itfk5+crPz9fW7ZsiWwOJamwsFC5ublat26d6urqIuMjRoxQTk6OysrKov6QjRkzRunp6Vq5cmXUHEpKStTQ0KDy8vLIWDgc1oQJE1RTU6NNmzZFxjMzMzV27Fjt3btXFRUVkfHs7GyNHDlSO3fu1I4dOyLjLdfU/FOTkSNHqqCgICXWlCqdCgoK9Je//EXdu3eP/EQr6GtKlU7Nj5vhw4fr1FNPTYk1pUKn3r176/Dhw8rNzdX777+fEmtKlU5bt26N/IQ+JycnJdaUKp0yMjLU1NSk/Px8bdu2LSXWlCqdNm7cGHnc9OjRIyXWFKROx76l6UQC8zLC6dOna8GCBfrrX/+q/Px8SdJzzz2nG2+8MepZKEk666yzdOGFF+qhhx7SzTffrG3btun111+P3H748GH17NlTr732mqZOndrqe7X1zNaQIUP0wQcfRJ4q7Ao/FXBdV6tXr9b48eOVnp6eEms60dyDsibP81RaWqri4uLI9wr6mlKlU8vHTSqsqeUcg7gm13VVVlam4uLiqA8BDfKaUqVTY2OjVq9ereLiYqWlpaXEmlKlU6zHTZDXlCqdGhoaTvi4CdqagtSptrZW/fr1S52XEd5+++2aP3++li5dGtloSdKgQYPU0NCg6urqqGe3du/erUGDBkWuefvtt6Pur/m0wuZrWsrIyFBGRkar8bS0NKWlRf+WNcdqqTlKvOMt7/ejjDuO0+Z4rDnGM+44TuTfU2VNxwrqmjzPi/wHp+V9BXVNxxsP2pqOfdykypriGQ/KmtpzfVDWFOROzX+5CYfDrX541Nb1QVhTKnbqiLVaW1OQO3XE48bamoLUKdbtbTF9GqHv+7r99ts1b948LV68WMOGDYu6ffz48erWrZveeOONyNjmzZu1fft2TZo0SZI0adIkvfPOO6qqqopcs2jRImVlZWnUqFGdsxAAAAAAXY7plxHeeuuteu655/T73/9ep59+emQ8OztbmZmZko6+vPC1117TU089paysLN1xxx2SpGXLlkk6+nTfuHHjlJeXp4cffli7du3Stddeq6985St68MEH45pHVz6NsK6uTpmZmZx0Ywxt7KKNTXSxizZ20cYu2iRXe/YGpjdbsf7w/Pa3v9UNN9wg6eiHGt9999363e9+p/r6ek2ZMkWPPfZY1EsEt23bpunTp+utt95Sz549df3112v27NlxPwXYlTdbrutGnqqGHbSxizY20cUu2thFG7tok1wps9myoqtutpqamrRy5UqVlJS067WpSDza2EUbm+hiF23soo1dtEmu9uwNTL9nCwAAAACCis0WAAAAACQAmy0AAAAASADesxWHrvqeLd58aRdt7KKNTXSxizZ20cYu2iQX79lCh2loaEj2FD6Wsu379fLqHSrbvj/ZU+lwQW+TymhjE13soo1dtLGLNsHAZgsxua6r8vJyua6b7Kl8JLMXbNTljy3TXS+u1eWPLdPsBRuTPaUOE/Q2qYw2NtHFLtrYRRu7aBMcbLaQksq279fcJRVRY3OXVKTkM1wAAACwic0WUlLl3kPtGgcAAAA6GpstHFc4HE72FD6SYf17tms8iILapiugjU10sYs2dtHGLtoEA6cRxqGrnkYYdLMXbIx6KeH0Cwp179SRSZwRAAAAgq49e4O0TpoTAsj3fdXU1Cg7OzuQx4rOmDpSU0YPUuXeQxrWv6eKhvZJ9pQ6TNDbpDLa2EQXu2hjF23sok1w8DJCxOS6rjZt2hTok26KhvbRF4rzU2qjJaVGm1RFG5voYhdt7KKNXbQJDjZbAAAAAJAAvIwQKa1s+/6UfBkhAAAA7GOzhZgcx1FmZmZgXwvc8oCMWy4o1IwUOSAj6G1SGW1sootdtLGLNnbRJjg4jTAOnEYYPGXb9+vyx5a1Gp9369k8wwUAAICPrD17A96zhZg8z1NVVZU8z0v2VNot1T/UOMhtUh1tbKKLXbSxizZ20SY42GwhJs/zVFFREcgHcqp/qHGQ26Q62thEF7toYxdt7KJNcLDZQkoqGtpHt1xQGDU2/YJCXkIIAACATsMBGUhZqfyhxgAAALCPzRZichwn8J9MXjS0T0puslKhTaqijU10sYs2dtHGLtoEB6cRxoHTCAEAAABInEaIDuJ5nnbs2MGbLw2ijV20sYkudtHGLtrYRZvgYLOFmHgg20Ubu2hjE13soo1dtLGLNsHBZgsAAAAAEoADMpDSyrbv5zRCAAAAJAWbLcQUCoU0YMAAhULBfAJ09oKNmrukIvLrWy4o1IypI5M4o44T9DapjDY20cUu2thFG7toExycRhgHTiMMnrLt+3X5Y8tajc+79Wye4QIAAMBHxmmE6BCe52nr1q2BfPNl5d5D7RoPmiC3SXW0sYkudtHGLtrYRZvgYLOFmDzP0549ewL5QB7Wv2e7xoMmyG1SHW1sootdtLGLNnbRJjjYbCElFQ3to1suKIwam35BIS8hBAAAQKfhgAykrBlTR2rK6EGcRggAAICkYLOFmEKhkPLz8wN90k3R0D4puclKhTapijY20cUu2thFG7toExycRhgHTiMEAAAAIHEaITqI67rauHGjXNdN9lTQAm3soo1NdLGLNnbRxi7aBAebLcTk+75qamrEk5/20MYu2thEF7toYxdt7KJNcLDZAgAAAIAEYLMFAAAAAAnAZgsxhUIhFRYWctKNQbSxizY20cUu2thFG7toExycRhgHTiMEAAAAIHEaITqI67pau3YtJ90YRBu7aGMTXeyijV20sYs2wcFmCzH5vq+6ujpOujGINnbRxia62EUbu2hjF22Cg80WAAAAACQAmy0AAAAASAA2W4gpHA5rxIgRCofDyZ4KWqCNXbSxiS520cYu2thFm+DgNMI4cBohAAAAAInTCNFBmpqaVFpaqqampmRPBS3Qxi7a2EQXu2hjF23sok1wsNnCcXGkqF20sYs2NtHFLtrYRRu7aBMMacmeAJBIZdv3q3LvIQ3r31NFQ/skezoAAADoQthsIWXNXrBRc5dURH59ywWFmjF1ZBJnBAAAgK6EAzLi0FUPyGj+wLzMzEw5jpPs6bRL2fb9uvyxZa3G5916dko8wxXkNqmONjbRxS7a2EUbu2iTXByQgQ6Tnp6e7Cl8JG9trmrXeBAFtU1XQBub6GIXbeyijV20CQY2W4jJdV2tXLmSN2AaRBu7aGMTXeyijV20sYs2wcFmCynpk6fntmscAAAA6GhstpCSiob20S0XFEaNTb+gMCXerwUAAIBg4DRCpKwZU0dqyuhBHP0OAACApOA0wjh05dMIXddVOBzmpBtjaGMXbWyii120sYs2dtEmuTiNEB2moaEh2VNADLSxizY20cUu2thFG7toEwxsthCT67oqLy/npBuDaGMXbWyii120sYs2dtEmOHjPFgAAANDByrbvj/m+8ePdlojvh+TpUputOXPm6Mc//rF27dqlsWPH6he/+IXOOuusZE+r3XgwAQAA2HT3i2v02jvvq67Ri4xlZYSV37eHHEmu52vT7oOR2y44rb+e/vJElW3fr7c2V2n9v2pUtr1atUcalRYOKadHN/Xtka4zTspWcUEfdQuHVNC3e+Trf/z6Jr3w9nvae6gh6j7rGlw5jlRycl/tPVCvyr2HdOBIo2rqGtWnZ7o+eXquhg/opUbXU7dwKObfK18o3a6nl/1TVbVH1Duzm4b37ylJCoUc9e7eTe/tOxz5PsMH9NLWPQf17u4D8jxfNXWNqmt0dcHpuZo8cqDe2lwl6ehH8WzZfUBr36vW2CE5unrC0Mjfb4/9+tEnZSsvJ1PdwqETztOqLnNAxgsvvKDrrrtOc+fO1cSJE/Xoo4/qpZde0ubNm5Wbe/zPXrJ0QMbsBRs1d0lF5Ne3XFCoGVNHJuR7NTU1qaysTEVFRUpL61L7cvNoYxdtbKKLXbSxizbtd/p3X1O9m/i/WncL+br2FE//3z9CavQ67oAMR1K3sCPf9xVynE5ZiyRlpDmqb4r/eyXy77/xaM/eoMtstiZOnKgJEybol7/8pSTJ8zwNGTJEd9xxh2bMmHHcr7Wy2Srbvl+XP7as1fi8W88O1A4fAAAg1Vz8yFvaUnUo2dPoMpL599/27A26xI8pGhoatGrVKn3729+OjIVCIU2ePFnLly9vdX19fb3q6+sjv66trZV09Cc8TU1Nka8PhULyPE+e9+HTxM3jruvq2H1srPHmIzub7/fYcUlRb3ysrDogyZcjKe2Yo00qqw6oaGifyDGgzRzHUTgcbjXHWOMt1+T7vmpra5WTk6NwOJyQNR1vPC0trcPXdKK5B2VNjuNo//796t27d+TI16CvKVU6tXzcpMKaWs4xiGuSpIMHD6p3795tzjGIa0qVTq7rqra2VllZWQqFQimxplTpJLX9uAnymhLVae2OalXuPaiQJE+O0hxfx57I3uQd/Rtct1D0cxxHx6VuLY6sa/TU6u97R8cdOfLVLeQrr4e087Dk+Y6afEch+Qofc73v6+i44yt8zFw8X3J9R2HHV+iYcdc/el8t5+56nbOmtLbmfpw1vfj2P3VmXm9Jnf9nr+Xtx9MlNlt79+6V67oaOHBg1PjAgQO1adOmVtfPmjVLM2fObDVeVlamnj2Pvk51wIABGj58uCorK7Vnz57INfn5+crPz9eWLVtUU1MTGS8sLFRubq7WrVunurq6yPiIESOUk5OjsrKyqP8YjBkzRunp6Vq5cmVkrGddo7qFpF5p0heHffgHqfeh9yQNVU1NTdR6MjMzNXbsWO3du1cVFR++9DA7O1sjR47Uzp07tWPHjsh4yzX5vq/q6mqNGjVKBQUFCVmTJJWUlKihoUHl5eWRsXA4rAkTJnT4mjqjU2esqaCgQGvWrFFmZmZksxX0NaVKp+bHzSmnnKJTTz01JdaUCp169+6tAwcOaPDgwXr//fdTYk2p0mnr1q2qrq5WTk6OcnJyUmJNqdIpIyND9fX1Kigo0LZt21JiTYnqtKvmiG441dPSXY421zi6/GRPOekfznHBjpB2HJKmDfeiNiH/UxnSwSbphlOjfzj01LuhVn/fa/Skp94N66Se0qVDPOX3lHYckvbVO3qpMqxTs32dP+jDjcOOw9KC98Iq6ueruN+H45trHC3d5eicgb5Oz/5wfPUHjlbtdfR/8j3l9/hwLp21pqn5H45XN+iEaxqm3Vq5sr5dnZp93D97hw7F/wxml3gZ4c6dO3XSSSdp2bJlmjRpUmT8W9/6lpYsWaIVK1ZEXd/WM1tDhgzRBx98EHmqMFk/vfnJn7boiaUVkd3/V88dprsuPj0hP71xXVerV6/W+PHjlZ6entI/kQramjzPU2lpqYqLiyPfK+hrSpVOLR83qbCmlnMM4ppc11VZWZmKi4sVCn34t4IgrylVOjU2Nmr16tUqLi5WWlpaSqwpVTrFetwEeU2JfGbrmif/3mnPAmWm+frScE/PbA2pwe2az2zN+vwZumJ8vqTO/7NXW1urfv368TLCZv3791c4HNbu3bujxnfv3q1Bgwa1uj4jI0MZGRmtxtPS0lq9QbQ5VkvNUeIdj/XG05bjMy4ZpSlnDG7zNELHcdq8n1hzjGfccZzIvydqTccbT8SajhXUNXmeF/k/hpb3FdQ1HW88aGs69nGTKmuKZzwoa2rP9UFZU5A7Nf/lJhwOt/rhUVvXB2FNqdipI9ZqbU0d2Wn8yf1103nDI4eYNfnO0R1HC7EOszjm4MIIP+a4o0av+X+PbrSkoxsir43rPd+R18ZcXN9RW+dfxJp7Z6yppVhrGpufo6snntxqvLP+7LXnwJgusdlKT0/X+PHj9cYbb+jzn/+8pKN/WX3jjTd0++23J3dyH0HR0D6d8oZAx3GiXqYGO2hjF21sootdtLGLNu0zY+pITRk9SM+/vV1/3rBb1XWNCjmOuoUdNbqeXP/oMzu+/vfUv7SQemekyfN8Vdc1yvelcEgKOY66p4fVo1tY1XWN8jxf4ZCj9LSQQo6jcMiRfE+HvUb1yeymAVmZ8nxf2/cd1pEmT5ndwurfK0MHjzTqUIOr7t3Cyu2doUbXky+pe1pI9U2ePF/af6hejf+742p0PXmSws6HpxGGQo4amo4+JdUrPU35fTLVIz2sQw2u6hpcfXCwXocaXHm+5DhSRlpI3cJHN3cZ4ZDS00LKTA+raGgfbdhZo827DsqXlN4tpN7pYTW4njLSwjr/tAH60icK9OAfN+jdqoMKO46yMru1+j3OzcrQF4rzdfWEoZ0X9mPqEi8jlI4e/X799dfriSee0FlnnaVHH31UL774ojZt2tTqvVwtWTmNEAAAAEBytWdv0Pp5thR19dVX6yc/+Ynuu+8+jRs3TmvWrNHChQtPuNHqyjzPU1VVVasTvZB8tLGLNjbRxS7a2EUbu2gTHF1msyVJt99+u7Zt26b6+nqtWLFCEydOTPaUTPM8TxUVFTyQDaKNXbSxiS520cYu2thFm+DoUpstAAAAAOgsbLYAAAAAIAHYbCEmx3GUnZ3NKUQG0cYu2thEF7toYxdt7KJNcHSZ0wg/Dk4jBAAAACBxGiE6iOd52rFjB2++NIg2dtHGJrrYRRu7aGMXbYKDzRZi4oFsF23soo1NdLGLNnbRxi7aBAebLQAAAABIADZbAAAAAJAAbLYQUygU0oABAxQK8cfEGtrYRRub6GIXbeyijV20CQ5OI4wDpxECAAAAkDiNEB3E8zxt3bqVN18aRBu7aGMTXeyijV20sYs2wcFmCzF5nqc9e/bwQDaINnbRxia62EUbu2hjF22Cg80WAAAAACRAWrInEATNb2urra1N8kw6V1NTkw4dOqTa2lqlpfFHxRLa2EUbm+hiF23soo1dtEmu5j1BPEdfUCcOBw4ckCQNGTIkyTMBAAAAYMGBAweUnZ193Gs4jTAOnudp586d6t27txzHSfZ0Ok1tba2GDBmi9957j1MYjaGNXbSxiS520cYu2thFm+TyfV8HDhxQXl7eCY/f55mtOIRCIeXn5yd7GkmTlZXFA9ko2thFG5voYhdt7KKNXbRJnhM9o9WMAzIAAAAAIAHYbAEAAABAArDZQkwZGRm6//77lZGRkeypoAXa2EUbm+hiF23soo1dtAkODsgAAAAAgATgmS0AAAAASAA2WwAAAACQAGy2AAAAACAB2GwBAAAAQAKw2UpxS5cu1Wc+8xnl5eXJcRy98sorUbf7vq/77rtPgwcPVmZmpiZPnqx333036pp9+/Zp2rRpysrKUk5Ojm666SYdPHgw6pry8nKdd9556t69u4YMGaKHH3440UsLtFmzZmnChAnq3bu3cnNz9fnPf16bN2+OuubIkSO67bbb1K9fP/Xq1UtXXHGFdu/eHXXN9u3bdemll6pHjx7Kzc3VPffco6ampqhr3nrrLRUXFysjI0OnnHKKnnrqqUQvL9Aef/xxjRkzJvJBkZMmTdKCBQsit9PFjtmzZ8txHN15552RMfokx/e//305jhP1z4gRIyK30yV5/vWvf+lLX/qS+vXrp8zMTJ155plauXJl5Hb+HpAcJ598cqvHjOM4uu222yTxmEkpPlLaa6+95n/3u9/1X375ZV+SP2/evKjbZ8+e7WdnZ/uvvPKKv3btWv+zn/2sP2zYML+uri5yzac//Wl/7Nix/t///nf/L3/5i3/KKaf4//Zv/xa5vaamxh84cKA/bdo0f926df7vfvc7PzMz03/iiSc6a5mBM2XKFP+3v/2tv27dOn/NmjX+JZdc4g8dOtQ/ePBg5JpbbrnFHzJkiP/GG2/4K1eu9D/xiU/4Z599duT2pqYm/4wzzvAnT57sl5WV+a+99prfv39//9vf/nbkmoqKCr9Hjx7+XXfd5W/YsMH/xS9+4YfDYX/hwoWdut4gefXVV/0//vGP/pYtW/zNmzf73/nOd/xu3br569at832fLla8/fbb/sknn+yPGTPG/8Y3vhEZp09y3H///f7o0aP9999/P/LPnj17IrfTJTn27dvnFxQU+DfccIO/YsUKv6Kiwn/99df9f/zjH5Fr+HtAclRVVUU9XhYtWuRL8t98803f93nMpBI2W11Iy82W53n+oEGD/B//+MeRserqaj8jI8P/3e9+5/u+72/YsMGX5JeWlkauWbBgge84jv+vf/3L933ff+yxx/w+ffr49fX1kWvuvfde//TTT0/wilJHVVWVL8lfsmSJ7/tHO3Tr1s1/6aWXItds3LjRl+QvX77c9/2jG+lQKOTv2rUrcs3jjz/uZ2VlRVp861vf8kePHh31va6++mp/ypQpiV5SSunTp4//q1/9ii5GHDhwwD/11FP9RYsW+RdccEFks0Wf5Ln//vv9sWPHtnkbXZLn3nvv9c8999yYt/P3ADu+8Y1v+MOHD/c9z+Mxk2J4GWEXVllZqV27dmny5MmRsezsbE2cOFHLly+XJC1fvlw5OTkqKSmJXDN58mSFQiGtWLEics3555+v9PT0yDVTpkzR5s2btX///k5aTbDV1NRIkvr27StJWrVqlRobG6PajBgxQkOHDo1qc+aZZ2rgwIGRa6ZMmaLa2lqtX78+cs2x99F8TfN94Phc19Xzzz+vQ4cOadKkSXQx4rbbbtOll17a6veQPsn17rvvKi8vT4WFhZo2bZq2b98uiS7J9Oqrr6qkpERXXnmlcnNzVVRUpP/6r/+K3M7fA2xoaGjQM888oy9/+ctyHIfHTIphs9WF7dq1S5KiHqjNv26+bdeuXcrNzY26PS0tTX379o26pq37OPZ7IDbP83TnnXfqnHPO0RlnnCHp6O9benq6cnJyoq5t2eZEv++xrqmtrVVdXV0ilpMS3nnnHfXq1UsZGRm65ZZbNG/ePI0aNYouBjz//PNavXq1Zs2a1eo2+iTPxIkT9dRTT2nhwoV6/PHHVVlZqfPOO08HDhygSxJVVFTo8ccf16mnnqrXX39d06dP19e//nU9/fTTkvh7gBWvvPKKqqurdcMNN0jiv2WpJi3ZEwC6uttuu03r1q3TX//612RPBf/r9NNP15o1a1RTU6P/+Z//0fXXX68lS5Yke1pd3nvvvadvfOMbWrRokbp3757s6eAYU6dOjfz7mDFjNHHiRBUUFOjFF19UZmZmEmfWtXmep5KSEj344IOSpKKiIq1bt05z587V9ddfn+TZodmvf/1rTZ06VXl5ecmeChKAZ7a6sEGDBklSq9Ntdu/eHblt0KBBqqqqirq9qalJ+/bti7qmrfs49nugbbfffrvmz5+vN998U/n5+ZHxQYMGqaGhQdXV1VHXt2xzot/3WNdkZWXxF6DjSE9P1ymnnKLx48dr1qxZGjt2rH72s5/RJclWrVqlqqoqFRcXKy0tTWlpaVqyZIl+/vOfKy0tTQMHDqSPETk5OTrttNP0j3/8g8dNEg0ePFijRo2KGhs5cmTkJZ78PSD5tm3bpj//+c/6yle+EhnjMZNa2Gx1YcOGDdOgQYP0xhtvRMZqa2u1YsUKTZo0SZI0adIkVVdXa9WqVZFrFi9eLM/zNHHixMg1S5cuVWNjY+SaRYsW6fTTT1efPn06aTXB4vu+br/9ds2bN0+LFy/WsGHDom4fP368unXrFtVm8+bN2r59e1Sbd955J+r/BBctWqSsrKzI/7lOmjQp6j6ar2m+D8TH8zzV19fTJckuuugivfPOO1qzZk3kn5KSEk2bNi3y7/Sx4eDBg9q6dasGDx7M4yaJzjnnnFYfK7JlyxYVFBRI4u8BFvz2t79Vbm6uLr300sgYj5kUk+wTOpBYBw4c8MvKyvyysjJfkv/II4/4ZWVl/rZt23zfP3rka05Ojv/73//eLy8v9z/3uc+1eeRrUVGRv2LFCv+vf/2rf+qpp0Yd+VpdXe0PHDjQv/baa/1169b5zz//vN+jRw+OfD2O6dOn+9nZ2f5bb70VdfTr4cOHI9fccsst/tChQ/3Fixf7K1eu9CdNmuRPmjQpcnvzsa8XX3yxv2bNGn/hwoX+gAED2jz29Z577vE3btzoz5kzh2NfT2DGjBn+kiVL/MrKSr+8vNyfMWOG7ziO/6c//cn3fbpYc+xphL5Pn2S5++67/bfeesuvrKz0//a3v/mTJ0/2+/fv71dVVfm+T5dkefvtt/20tDT/gQce8N99913/2Wef9Xv06OE/88wzkWv4e0DyuK7rDx061L/33ntb3cZjJnWw2Upxb775pi+p1T/XX3+97/tHj3393ve+5w8cONDPyMjwL7roIn/z5s1R9/HBBx/4//Zv/+b36tXLz8rK8m+88Ub/wIEDUdesXbvWP/fcc/2MjAz/pJNO8mfPnt1ZSwyktppI8n/7299Grqmrq/NvvfVWv0+fPn6PHj38yy+/3H///fej7uef//ynP3XqVD8zM9Pv37+/f/fdd/uNjY1R17z55pv+uHHj/PT0dL+wsDDqe6C1L3/5y35BQYGfnp7uDxgwwL/ooosiGy3fp4s1LTdb9EmOq6++2h88eLCfnp7un3TSSf7VV18d9VlOdEmeP/zhD/4ZZ5zhZ2Rk+CNGjPCffPLJqNv5e0DyvP76676kVr/fvs9jJpU4vu/7SXlKDQAAAABSGO/ZAgAAAIAEYLMFAAAAAAnAZgsAAAAAEoDNFgAAAAAkAJstAAAAAEgANlsAAAAAkABstgAAAAAgAdhsAQAAAEACsNkCAAAAgARIS/YEAAD4KFzX1XnnnadBgwbp5ZdfjozX1NTojDPO0HXXXacNGzaosrKy1dcePnxYCxYs0N///nc98MADSk9Pj7q9qalJ1157re68806NHj1avXr1anUfGRkZWrFihe644w4tWbJEoVD0zy+PHDmiJ554QpL0ta99Td27d4+63fM8XXDBBfrFL36hiRMnqr6+vtX3OHjwoNavX6+MjIz4f2MAAGaw2QIABFI4HNZTTz2lcePG6dlnn9W0adMkSXfccYf69u2r+++/X+eff77WrFnT6mtvuOEGNTY26sCBA/rWt76lG264Ier2t956SwsXLpTv+8rPz9dbb73V6j4+8YlPSJL27NmjV199VSeffHLU7d///vdVV1cnSbrmmmv0/e9/P+r2f/7zn5oxY4YkyXGcNuf5yU9+Ur7vn/g3AwBgEpstAEBgnXbaaZo9e7buuOMOfepTn9Lbb7+t559/XqWlpa2erQIAoLOx2QIABNodd9yhefPm6dprr9U777yj++67T2PHjk32tAAAYLMFAAg2x3H0+OOPa+TIkTrzzDMjL80DACDZOI0QABB4v/nNb9SjRw9VVlZqx44dyZ4OAACS2GwBAAJu2bJl+ulPf6r58+frrLPO0k033cShEgAAE9hsAQAC6/Dhw7rhhhs0ffp0XXjhhfr1r3+tt99+W3Pnzk321AAAYLMFAAiub3/72/J9X7Nnz5YknXzyyfrJT36ib33rW/rnP/+Z3MkBALo8DsgAAATSkiVLNGfOHL311lvq0aNHZPxrX/uaXn75ZV5OCABIOjZbAIBAuuCCC9TU1NTmba+//rqkDz94GACAZOBlhAAAAACQADyzBQBIWSNHjlRJSUmbt2VmZio3N1cPPvigfvnLX7a6/YYbblAoFNLBgwfbvI/+/ftLkoYPH64vfvGLbX6PKVOmSJLmz5+v+fPnx7w9Jycn5jxDIX4uCgBB5fi8oB0AAAAAOhw/LgMAAACABGCzBQAAAAAJwGYLAAAAABKAzRYAAAAAJACbLQAAAABIADZbAAAAAJAAbLYAAAAAIAHYbAEAAABAAvz/IjEqeSvswDIAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "all_rems=[(k,len(processor.hash_buckets[k])) for k in processor.hash_buckets]\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# 假设 all_rems 是你已有的列表，例如：\n", - "# all_rems = [(1000, 1075), (1001, 569), ...]\n", - "\n", - "# 提取 x 和 y 数据\n", - "x_values = [item[0] for item in all_rems]\n", - "y_values = [item[1] for item in all_rems]\n", - "\n", - "# 创建图形\n", - "plt.figure(figsize=(10, 6)) # 设置图形大小（可选）\n", - "\n", - "# 绘制散点图（推荐用于观察分布）\n", - "plt.scatter(x_values, y_values, s=10) # s 控制点的大小\n", - "\n", - "# 或者绘制折线图（如果数据有序）\n", - "# plt.plot(x_values, y_values)\n", - "\n", - "# 添加标签和标题\n", - "plt.xlabel('X轴（第一列）')\n", - "plt.ylabel('Y轴（第二列）')\n", - "plt.title('X vs Y 散点图')\n", - "\n", - "# 显示网格（可选）\n", - "plt.grid(True, linestyle='--', alpha=0.7)\n", - "\n", - "# 显示图形\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "a9121a82-8d5e-44be-8a7b-5caae5ec0171", - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'all_rems' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[1], line 6\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mnumpy\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;66;03m# Assuming allrems is a list of lists where each sublist contains [x, y]\u001b[39;00m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;66;03m# Extract x and y values\u001b[39;00m\n\u001b[0;32m----> 6\u001b[0m x_values \u001b[38;5;241m=\u001b[39m [item[\u001b[38;5;241m0\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m \u001b[43mall_rems\u001b[49m]\n\u001b[1;32m 7\u001b[0m y_values \u001b[38;5;241m=\u001b[39m [item[\u001b[38;5;241m1\u001b[39m] \u001b[38;5;28;01mfor\u001b[39;00m item \u001b[38;5;129;01min\u001b[39;00m all_rems]\n\u001b[1;32m 9\u001b[0m \u001b[38;5;66;03m# Define the number of bins\u001b[39;00m\n", - "\u001b[0;31mNameError\u001b[0m: name 'all_rems' is not defined" - ] - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "# Assuming allrems is a list of lists where each sublist contains [x, y]\n", - "# Extract x and y values\n", - "x_values = [item[0] for item in all_rems]\n", - "y_values = [item[1] for item in all_rems]\n", - "\n", - "# Define the number of bins\n", - "num_bins = 10\n", - "\n", - "# Create bins based on x values\n", - "bins = np.linspace(min(x_values), max(x_values), num_bins + 1)\n", - "\n", - "# Calculate sum of y values for each bin\n", - "bin_indices = np.digitize(x_values, bins) - 1\n", - "bin_sums = np.zeros(num_bins)\n", - "for i in range(len(x_values)):\n", - " if 0 <= bin_indices[i] < num_bins:\n", - " bin_sums[bin_indices[i]] += y_values[i]\n", - "\n", - "# Create the histogram\n", - "plt.figure(figsize=(10, 6))\n", - "bars = plt.bar(bins[:-1], bin_sums, width=np.diff(bins), edgecolor='black', align='edge')\n", - "\n", - "# Add labels on top of each bar showing the sum\n", - "for bar, sum_val in zip(bars, bin_sums):\n", - " plt.text(bar.get_x() + bar.get_width()/2, bar.get_height(), \n", - " f'{sum_val:.1f}', ha='center', va='bottom')\n", - "\n", - "plt.xlabel('X Values')\n", - "plt.ylabel('Sum of Y Values')\n", - "plt.title('Histogram of Y Value Sums by X Range')\n", - "plt.grid(True, alpha=0.3)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d843ee3f-a8b4-4ab3-b027-71164002ad9c", - "metadata": {}, - "outputs": [], - "source": [ - "tracker.print_summary() " - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tools/data_preprocess/offline_packing/configs/s1_config_emova.yaml b/tools/data_preprocess/offline_packing/configs/s1_config_emova.yaml deleted file mode 100644 index e821e8a7..00000000 --- a/tools/data_preprocess/offline_packing/configs/s1_config_emova.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# 数据路径配置 -data: - # 数据样本目录 - directory: "/vlm/xiangan/datasets/aiak_caption_emova_300k" - # 存储配对文件名的临时文件 - output_base: "base_name_v2_emova1.txt" - # 最终输出文件（包含token长度信息） - output_token: "token_info_v2_emova1.txt" - -# 模型路径 -model: - checkpoint: "/vlm/pretrain_models/Qwen2.5-VL-7B-Instruct" - -sample: - # 训练数据的最大长度 - max_len: 8192 - # 决定解析方式 - task_type: pretrain - max_prompt: null - max_answer: null - -# 图像处理参数 -image: - min_pixels: 3136 # 4*28*28 - max_pixels: 5017600 # 3578*28*28(2805152,4096),6400*28*28(5017600,8192) - # 最大宽高比限制（超过此值的图片将被过滤）,这是隐含信息（qwen vl自己处理） - max_aspect_ratio: 200 - -# 并行处理参数 -processing: - # 每个进程处理的样本块大小 - chunk_size: 5000 - # 归并参数（排序），每N个stage0文件合并为1个stage1文件 - stage1_merge_chunk: 10 - n_workers: 96 - # 线程池最小线程数 - min_workers: 10 - # 线程池最大线程数 - max_workers: 32 - -# 日志与临时文件 -logging: - # 日志级别（DEBUG, INFO, WARNING, ERROR, CRITICAL） - level: "INFO" - # 日志文件路径 - file: "processing.log" - # 是否使用 /dev/shm 作为临时目录 - use_shm: false diff --git a/tools/data_preprocess/offline_packing/configs/s1_config_emova_3000tk.yaml b/tools/data_preprocess/offline_packing/configs/s1_config_emova_3000tk.yaml deleted file mode 100644 index bc07086a..00000000 --- a/tools/data_preprocess/offline_packing/configs/s1_config_emova_3000tk.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# 数据路径配置 -data: - # 数据样本目录 - directory: "/vlm/xiangan/datasets/aiak_caption_emova_300k" - # 存储配对文件名的临时文件 - output_base: "base_name_v2_emova_3000tk.txt" - # 最终输出文件（包含token长度信息） - output_token: "token_info_v2_emova_3000tk.txt" - -# 模型路径 -model: - checkpoint: "/vlm/pretrain_models/Qwen2.5-VL-7B-Instruct" - -sample: - # 训练数据的最大长度 - max_len: 8192 - # 决定解析方式 - task_type: pretrain - max_prompt: null - max_answer: null - -# 图像处理参数 -image: - min_pixels: 3136 # 4*28*28 - max_pixels: 2352000 # 3578*28*28(2805152,4096),6400*28*28(5017600,8192) - # 最大宽高比限制（超过此值的图片将被过滤）,这是隐含信息（qwen vl自己处理） - max_aspect_ratio: 200 - -# 并行处理参数 -processing: - # 每个进程处理的样本块大小 - chunk_size: 5000 - # 归并参数（排序），每N个stage0文件合并为1个stage1文件 - stage1_merge_chunk: 10 - n_workers: 96 - # 线程池最小线程数 - min_workers: 10 - # 线程池最大线程数 - max_workers: 32 - -# 日志与临时文件 -logging: - # 日志级别（DEBUG, INFO, WARNING, ERROR, CRITICAL） - level: "INFO" - # 日志文件路径 - file: "processing.log" - # 是否使用 /dev/shm 作为临时目录 - use_shm: false diff --git a/tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_16k.yaml b/tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_16k.yaml deleted file mode 100644 index 4108d09d..00000000 --- a/tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_16k.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# 数据路径配置 -data: - # 数据样本目录 - directory: "/data_3/aiak_pretrain_data_5M_vqa" - # 存储配对文件名的临时文件 - output_base: "base_name_v2_vqa_pretrain_5M_16k.txt" - # 最终输出文件（包含token长度信息） - output_token: "token_info_v2_vqa_pretrain_5M_16k.txt" - -# 模型路径 -model: - checkpoint: "/vlm/xiangan/pretrain_models/rice_vl/rice_vl_rice_300m_qwen2.5_7b_adapter_v1_fixed_tokenizer_huggingface" - -sample: - # 训练数据的最大长度 - max_len: 16384 - # 决定解析方式 - task_type: sft - max_prompt: null - max_answer: null - -# 图像处理参数 -image: - min_pixels: 3136 # 4*28*28 - max_pixels: 8294720 # 10580*28*28(8294720,16384) - # 最大宽高比限制（超过此值的图片将被过滤）,这是隐含信息（qwen vl自己处理） - max_aspect_ratio: 200 - -# 并行处理参数 -processing: - # 每个进程处理的样本块大小 - chunk_size: 5000 - # 归并参数（排序），每N个stage0文件合并为1个stage1文件 - stage1_merge_chunk: 20 - n_workers: 64 - # 线程池最小线程数 - min_workers: 10 - # 线程池最大线程数 - max_workers: 32 - # 超时设置（根据数据量定，1M数据按 45分钟(2700s)估算） - time_out: 20000 - -# 日志与临时文件 -logging: - # 日志级别（DEBUG, INFO, WARNING, ERROR, CRITICAL） - level: "INFO" - # 日志文件路径 - file: "./logs/s1_processing_vqa_pretrain_5M_16k.log" - # 是否使用 /dev/shm 作为临时目录 - use_shm: true diff --git a/tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_8k.yaml b/tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_8k.yaml deleted file mode 100644 index bfa9d9bc..00000000 --- a/tools/data_preprocess/offline_packing/configs/s1_config_vqa_pretrain_5M_8k.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# 数据路径配置 -data: - # 数据样本目录 - directory: "/data_3/aiak_pretrain_data_5M_vqa" - # 存储配对文件名的临时文件 - output_base: "base_name_v2_vqa_pretrain_5M_8k.txt" - # 最终输出文件（包含token长度信息） - output_token: "token_info_v2_vqa_pretrain_5M_8k.txt" - -# 模型路径 -model: - checkpoint: "/vlm/xiangan/pretrain_models/rice_vl/rice_vl_rice_300m_qwen2.5_7b_adapter_v1_fixed_tokenizer_huggingface" - -sample: - # 训练数据的最大长度 - max_len: 8192 - # 决定解析方式 - task_type: sft - max_prompt: null - max_answer: null - -# 图像处理参数 -image: - min_pixels: 3136 # 4*28*28 - max_pixels: 4014080 # 5120*28*28(4014080,8192) - # 最大宽高比限制（超过此值的图片将被过滤）,这是隐含信息（qwen vl自己处理） - max_aspect_ratio: 200 - -# 并行处理参数 -processing: - # 每个进程处理的样本块大小 - chunk_size: 5000 - # 归并参数（排序），每N个stage0文件合并为1个stage1文件 - stage1_merge_chunk: 20 - n_workers: 64 - # 线程池最小线程数 - min_workers: 10 - # 线程池最大线程数 - max_workers: 32 - # 超时设置（根据数据量定，1M数据按 45分钟(2700s)估算） - time_out: 20000 - -# 日志与临时文件 -logging: - # 日志级别（DEBUG, INFO, WARNING, ERROR, CRITICAL） - level: "INFO" - # 日志文件路径 - file: "./logs/s1_processing_vqa_pretrain_5M_8k.log" - # 是否使用 /dev/shm 作为临时目录 - use_shm: false diff --git a/tools/data_preprocess/offline_packing/convert_packedsample_to_wds.py b/tools/data_preprocess/offline_packing/convert_packedsample_to_wds.py deleted file mode 100644 index 64afb52a..00000000 --- a/tools/data_preprocess/offline_packing/convert_packedsample_to_wds.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -""" -把目录级多图 captioning 数据打包成 WebDataset -目录结构： -raw_packing_data/ -├── ps_00000000.img000.jpg -├── ps_00000000.img001.jpg -├── ps_00000000.json -... - -JSON 格式： -{ - "images": ["img000.jpg", "img001.jpg", ...], - "prompt": ["描述", "what about", ""], - "captions": ["stri", "str2", "str3"] -} -一条 json + 对应若干 jpg = 1 条 tar 记录 -""" - -######-----------------------------------------###### -######-----------------------------------------###### -######-----------------------------------------###### - -import argparse -import uuid -import json -import os -import yaml -import webdataset as wds -from tqdm import tqdm -from pathlib import Path -from megatron.energon.epathlib import EPath -from megatron.energon.flavors import BaseWebdatasetFactory -from megatron.energon.flavors.webdataset import MAIN_FOLDER_NAME -from megatron.energon.flavors.webdataset.prepare import WebdatasetPreparator -from megatron.energon.flavors.webdataset.structs import ShardInfo, WebdatasetInfo, WebdatasetSplits - - -def sample_loader_template(media: str=None): - """Returns a template for a sample_loader.py file.""" - return "\n".join([ - "def sample_loader(sample: dict) -> dict:", - " messages=[]", - " for message in sample['json']:", - " assert message['role'] in ['system','user','assistant']", - " messages.append(dict(", - " role=message['role'],", - " content=message['content']", - " ))", - " return dict(", - " __key__=sample['__key__'],", - " __restore_key__=sample['__restore_key__'],", - " video=sample.get('mp4'),", - " image=sample.get('jpg')," if media == 'mix' else "", - " messages=messages,", - " )", - "def part_filter(part: str) -> bool:", - " return True", - ]) - -### ZXW - -def sample_loader_template_caption(media=None): - """适配整条多图 captioning 的 loader""" - return "\n".join([ - "def sample_loader(sample: dict) -> dict:", - " data = sample['json']", - " images = [sample.get(f'img{i}.jpg') for i in range(len(data['images']))]", - " captions = data['captions'] ", - " prompts = data['prompts']", - " return dict(", - " __key__=sample['__key__'],", - " __restore_key__=sample['__restore_key__'],", - " captions=captions,", - " prompts=prompts,", - " images=images,", - " )", - "def part_filter(part: str) -> bool:", - " return True", - ]) -# 生成 1 条数据 -def stream_samples_caption(src_dir: str): - for json_path in Path(src_dir).glob("*.json"): - sample_id = json_path.stem # ps_00000000 - with json_path.open("r", encoding="utf-8") as f: - raw = json.load(f) - yield { - "id": sample_id, - "images": raw["images"], # [img000.jpg, ...] - "prompts": raw.get("prompts", []), # [str, ...] - "captions": raw["captions"] # [str, ...] - } - -def construct_sample_caption(args, entry): - """整条样本打包""" - sample = {"__key__": entry["id"]} - for idx, img_name in enumerate(entry["images"]): - img_path = os.path.join(args.image_dir, f"{entry["id"]}.{img_name}") - with open(img_path, "rb") as f: - sample[f"img{idx}.jpg"] = f.read() - - payload = { - "prompts": entry["prompts"], - "captions": entry["captions"], - "images": entry["images"] - } - sample["json"] = json.dumps(payload, ensure_ascii=False).encode("utf-8") - return sample - -### ZXW - -def construct_sample(args, vision, path, entry): - """ construct webdataset sample """ - # 断言vision的值只能是'image'或'video' - assert vision == 'image' or vision == 'video' - # 根据vision的值，确定directory的路径 - directory = args.image_dir if vision == 'image' else args.video_dir - - # 打开vision_file文件 - with open(os.path.join(directory, path), "rb") as vision_file: - # 读取vision_file文件的内容 - vision_data = vision_file.read() - # 构造sample字典 - sample = { - "__key__": entry.get('id', path).replace('.', '_'), - # 根据vision的值，确定sample的键 - "jpg" if vision == 'image' else 'mp4': vision_data, - # 将entry[args.columns_messages]转换为json格式，并编码为utf-8 - "json": json.dumps(entry[args.columns_messages]).encode("utf-8"), - } - # 返回sample字典 - return sample - - -def convert_to_wds(args): - """ Convert dataset to wds format """ - if not os.path.exists(args.output_dir): - os.mkdir(args.output_dir) - - tar = os.path.join(args.output_dir, 'pretrain-%06d.tar') - if args.mode == "caption_pack": - # 新模式 - with wds.ShardWriter(tar, maxcount=args.maxcount, maxsize=args.maxsize) as sink: - for entry in tqdm(stream_samples_caption(args.json_file)): - sample=construct_sample_caption(args, entry) - # print(sample.keys()) - sink.write(sample) - # break - # sink.write(construct_sample_caption(args.image_dir, entry)) - - write_config(EPath(args.output_dir).absolute(), args.media, - template_func=sample_loader_template_caption, - class_name="PackedCaptioningSample") - print(f"Dataset successfully converted to wds") - - - -def write_config(path: EPath, media=None, template_func=None, class_name=None): - (path / MAIN_FOLDER_NAME).mkdir(exist_ok=True) - all_tars = list(path.glob("**/*.tar")) + list(path.glob("**/*.tgz")) - all_tars = [str(p.relative_to(path)) for p in sorted(all_tars)] - - if class_name is None: - class_name = "MultiMixQASample" if media == 'mix' else "MultiVidQASample" - dataset_definition = { - "sample_type": { - "__module__": "aiak_training_llm.data.multimodal", - "__class__": class_name, - }, - "part_filter": "sample_loader.py:part_filter", - "sample_loader": "sample_loader.py:sample_loader" - } - - with (path / MAIN_FOLDER_NAME / "dataset.yaml").open("w") as f: - yaml.dump(dataset_definition, f, sort_keys=False) - - tpl = (template_func or sample_loader_template)(media) - with (path / MAIN_FOLDER_NAME / "sample_loader.py").open("w") as f: - f.write(tpl) - - BaseWebdatasetFactory.prepare_dataset( - path, - all_tars, - split_parts_ratio=[("train", 1.0), ("val", 0), ("test", 0)], - tar_index_only=False, - workers=32, - ) - - -def _add_arguments(parser: argparse.ArgumentParser): - """Add arguments""" - group = parser.add_argument_group(title='wds') - group.add_argument('--output_dir', type=str, required=True, help='Output directory') - group.add_argument('--json_file', type=str, required=True, - help='目录（多图 captioning）或单文件（旧格式）') - group.add_argument('--image_dir', type=str, required=False, help='Image directory') - group.add_argument('--video_dir', type=str, required=False, help='Video directory') - group.add_argument('--maxcount', type=int, default=10000, help='Number of samples per shard') - group.add_argument('--maxsize', type=int, default=3000000000, help='Maximum size of each shard') - group.add_argument('--media', type=str, choices=["mix", "image", "video"], default="image", help='Media type') - group.add_argument('--columns_messages', type=str, default="messages", help='Column name for messages') - # 新增模式选择 - group.add_argument('--mode', type=str, - choices=["chat", "caption_pack"], - default="chat", - help="chat=旧格式(单图对话); caption_pack=新格式(整条多图caption)") - return parser - - -def parse_args(): - """arguments""" - parser = argparse.ArgumentParser() - _add_arguments(parser) - args = parser.parse_args() - - return args - - -def main(): - """main function""" - args = parse_args() - convert_to_wds(args) - - -if __name__ == '__main__': - main() - - diff --git a/tools/data_preprocess/offline_packing/convert_pairs.py b/tools/data_preprocess/offline_packing/convert_pairs.py deleted file mode 100644 index 12f26046..00000000 --- a/tools/data_preprocess/offline_packing/convert_pairs.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -""" -convert_pairs.py -把 {name}.json 转成目标格式，字段顺序：content, role - -# 全量转换 -python convert_pairs.py /data_1/test_samples /data_1/converted_samples - -# 抽查前 100 条 -python convert_pairs.py /path/to/src /path/to/dst --samples 100 -python convert_pairs.py /data_1/aiak_caption_emova+llava_recap /data_3/aiak_pretrain_vqa_5500k --samples 10 # 可以选择不拷贝图片（line 70） - -# 指定进程数 -python convert_pairs.py /path/to/src /path/to/dst --workers 8 - -""" -import argparse -import json -import os -import shutil -from glob import iglob -from multiprocessing import Pool, cpu_count -from typing import List, Tuple - -from tqdm import tqdm - -logger = None # 若需要写到文件可自行初始化 - - -def build_pairs(src_dir: str) -> List[Tuple[str, str]]: - """收集成对的 json / jpg 绝对路径""" - pairs = [] - for json_path in iglob(os.path.join(src_dir, "*.json")): - base = os.path.splitext(os.path.basename(json_path))[0] - jpg_path = os.path.join(src_dir, f"{base}.jpg") - if os.path.isfile(jpg_path): - pairs.append((json_path, jpg_path)) - return pairs - - - -def convert_one(pair: Tuple[str, str], dst_dir: str) -> str: - """单条转换逻辑（含拷贝图片）""" - json_src, jpg_src = pair - base = os.path.splitext(os.path.basename(json_src))[0] - dst_json = os.path.join(dst_dir, f"{base}.json") - dst_jpg = os.path.join(dst_dir, f"{base}.jpg") - - # 读原 json - with open(json_src, "r", encoding="utf-8") as f: - data = json.load(f) - - # 1. 修正键名 - messages_raw = data.get("massages:", []) - # 2. 调整字段顺序 - messages_new = [ - {"content": m["content"], "role": m["role"]} - for m in messages_raw - ] - new_obj = { - "messages": messages_new, - "images": [os.path.basename(jpg_src)] - } - - # 写目标 json - with open(dst_json, "w", encoding="utf-8") as f: - json.dump(new_obj, f, ensure_ascii=False, indent=None) - - # 3. 拷贝图片 - shutil.copy2(jpg_src, dst_jpg) - - return base - - -def process_chunk(chunk: List[Tuple[str, str]], dst_dir: str, pos: int): - """供多进程 map 使用的 chunk 处理""" - ret = [] - for p in chunk: - ret.append(convert_one(p, dst_dir)) - return ret - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("src", help="含 json/jpg 的源目录") - parser.add_argument("dst", help="输出目录") - parser.add_argument("--workers", type=int, default=cpu_count(), - help="进程数，默认=CPU核心数") - parser.add_argument("--samples", type=int, default=None, - help="仅处理前 N 条") - args = parser.parse_args() - - os.makedirs(args.dst, exist_ok=True) - pairs = build_pairs(args.src) - if not pairs: - print("未找到任何 json/jpg 成对文件") - return - - if args.samples: - pairs = pairs[: args.samples] - - # 分块：一条一个任务粒度太大，按进程数切块 - chunk_size = max(1, len(pairs) // args.workers) - chunks = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)] - - total = 0 - - with Pool(processes=args.workers) as pool: - with tqdm(total=len(pairs), desc="convert") as bar: - for chunk in chunks: - future = pool.apply_async(process_chunk, (chunk, args.dst, total)) - done = future.get() - total += len(done) - if total % 2000 == 0: - print(f"[INFO] 已处理 {total} 条") - bar.update(len(done)) - - print(f"[DONE] 总计转换 {total} 条数据") - # 输出日志位置提示（如需要） - # print("详细日志见 output.log") - - -if __name__ == "__main__": - main() diff --git a/tools/data_preprocess/offline_packing/hashbacket.py b/tools/data_preprocess/offline_packing/hashbacket.py deleted file mode 100644 index 63790983..00000000 --- a/tools/data_preprocess/offline_packing/hashbacket.py +++ /dev/null @@ -1,2223 +0,0 @@ -import os -import sys -import time -import logging -import numpy as np -from pathlib import Path -from itertools import islice -from tqdm import tqdm -from collections import defaultdict -from typing import List, Tuple, Dict, Optional, Union, Set -from concurrent.futures import ThreadPoolExecutor, as_completed -import threading -import queue -import bisect - -class HashBucketProcessor: - """哈希桶处理器，用于处理大型数据文件并进行高效装箱""" - - DTYPE_SAMPLE_INFO = np.dtype([ - ("w", np.uint16), # 用于存储 ViT 部分的权重（可以是 ViT 部分的像素数或者 ViT 部分的处理能力） - ("l", np.uint16), # 用于存储 llm 部分的曲种（ LLM 输入部分的 tokens 数亩） - ("name", "U64") # sample‘s name - ]) - - def __init__(self, file_path: Union[str, Path], logger: Optional[logging.Logger] = None): - self.file_path = Path(file_path) - if not self.file_path.exists(): - raise FileNotFoundError(f"文件不存在: {file_path}") - - self.hash_buckets = defaultdict(lambda: np.array([], dtype=self.DTYPE_SAMPLE_INFO)) - self.total_lines = 0 - self.hb2_keys = [] # 可以除以哪些 2 的幂次 - self._logger = logger or self._setup_default_logger() - - @staticmethod - def _setup_default_logger() -> logging.Logger: - """设置默认日志器""" - logger = logging.getLogger(__name__) - if not logger.handlers: - handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.setLevel(logging.INFO) - return logger - - def estimate_memory_usage(self) -> int: - """估算当前哈希桶内存占用""" - total_size = sys.getsizeof(self.hash_buckets) - for key, arr in self.hash_buckets.items(): - total_size += sys.getsizeof(key) + arr.nbytes - return total_size - - def _count_file_lines(self) -> int: - """计算文件总行数，使用更高效的方法""" - try: - with self.file_path.open('rb') as f: - return sum(1 for _ in f) - except Exception as e: - self._logger.warning(f"快速计数失败，使用标准方法: {e}") - with self.file_path.open('r', encoding='utf-8') as f: - return sum(1 for _ in f) - - def _parse_line(self, line: str) -> Optional[Tuple[int, int, str]]: - """解析单行数据，返回 (w, l, name) 或 None""" - line = line.strip() - if ':' not in line: - return None - - try: - name, key_str = line.split(':', 1) - key = int(key_str) - if 0 <= key <= 65535: - return (0, key, name) - except (ValueError, IndexError): - pass - return None - - def _update_buckets(self, parsed_data: List[Tuple[int, int, str]]) -> None: - """更新哈希桶""" - data_array = np.array(parsed_data, dtype=self.DTYPE_SAMPLE_INFO) - unique_l_values = np.unique(data_array['l']) - - for l_val in unique_l_values: - mask = data_array['l'] == l_val - chunk = data_array[mask] - - if l_val in self.hash_buckets: - self.hash_buckets[l_val] = np.concatenate([self.hash_buckets[l_val], chunk]) - else: - self.hash_buckets[l_val] = chunk - - def build_buckets(self, chunk_size: int = 100000) -> None: - """构建哈希桶""" - self.total_lines = self._count_file_lines() - self._logger.info(f"开始处理文件，总行数: {self.total_lines}") - - with self.file_path.open('r', encoding='utf-8') as file: - with tqdm(total=self.total_lines, unit='行', desc='构建哈希桶') as pbar: - while True: - lines = list(islice(file, chunk_size)) - if not lines: - break - - pbar.update(len(lines)) - - # 并行解析数据 - parsed_data = [] - for line in lines: - parsed = self._parse_line(line) - if parsed: - parsed_data.append(parsed) - - if parsed_data: - self._update_buckets(parsed_data) - - @staticmethod - def factors_of_two(a: int, C: int) -> List[Tuple[int, int]]: - """返回所有满足 b * 2^n = a 且 b > C 的 (b, n) 对""" - if a < 0 or C < 0: - raise ValueError("a 必须为正整数，C 必须为非负整数") - res = [] - n = 0 - b = a - while b > C: - res.append((b, n)) - if b & 1: - break - b >>= 1 - n += 1 - return res - - def find_items(self, capacity: int) -> defaultdict[np.ndarray]: - """从哈希桶中查找符合条件的项目""" - - if not self.hash_buckets: - self._logger.warning("哈希桶为空，请先构建哈希桶") - return - - for key, value in self.hash_buckets.items(): - if not isinstance(value, np.ndarray) or value.dtype != self.DTYPE_SAMPLE_INFO: - raise TypeError(f"哈希桶数据格式错误，key={key}") - break - self.hb2_keys=[] - min_l_value = min(self.hash_buckets.keys()) - valid_b_values = [b for b, _ in self.factors_of_two(capacity, min_l_value - 1)] - - for b in valid_b_values: - if b in self.hash_buckets: - self.hb2_keys.append(b) - - self._logger.info(f"找到 {len(self.hb2_keys)} 个有效的桶键") - - def delete_by_index(self, result: defaultdict[np.ndarray], key: int, index: int) -> None: - """按索引删除元素""" - if key in result and 0 <= index < len(result[key]): - result[key] = np.delete(result[key], index) - - def get_statistics(self) -> Dict[str, Union[int, float]]: - """获取统计信息""" - total_items = sum(len(arr) for arr in self.hash_buckets.values()) - memory_gb = self.estimate_memory_usage() / (1024**3) - - return { - "bucket_count": len(self.hash_buckets), - "total_items": total_items, - "memory_usage_gb": memory_gb, - "hb2_keys_count": [len(self.hb2_keys),self.hb2_keys], - "file_lines": self.total_lines - } - - def __len__(self) -> int: - """返回总数据项数""" - return sum(len(arr) for arr in self.hash_buckets.values()) - - def __repr__(self) -> str: - return f"HashBucketProcessor(buckets={len(self.hash_buckets)}, items={len(self)})" - - def summary(self) -> None: - """打印摘要信息""" - stats = self.get_statistics() - print(f"=== 哈希桶处理摘要 ===") - print(f"哈希桶数量: {stats['bucket_count']}") - print(f"总数据项: {stats['total_items']}") - print(f"内存占用: {stats['memory_usage_gb']:.2f} GB") - print(f"有效桶键: {stats['hb2_keys_count']}") - print(f"处理行数: {stats['file_lines']}") - - def _cleanup_empty_keys(self, verbose: bool = False) -> int: - """ - 清理哈希桶中元素个数为0的key - - 参数: - verbose: 是否打印清理详情 - - 返回: - int: 删除的空key数量 - """ - # 1. 收集需要删除的空key - empty_keys = [] - for key in list(self.hash_buckets.keys()): - if len(self.hash_buckets[key]) == 0: - empty_keys.append(key) - - # 2. 删除空key - for key in empty_keys: - del self.hash_buckets[key] - - # 3. 记录日志 - if verbose or empty_keys: - self._logger.info(f"清理空key: 删除了 {len(empty_keys)} 个空key") - if verbose and empty_keys: - self._logger.debug(f"删除的key: {sorted(empty_keys)}") - - return len(empty_keys) - - def update_hash_buckets(self, remove_empty: bool = True, verbose: bool = False) -> dict: - """ - 更新哈希桶结构，包括清理空key和统计信息 - - 参数: - remove_empty: 是否删除空key - verbose: 是否打印详细信息 - - 返回: - dict: 更新后的统计信息 - """ - # 1. 基础统计 - stats = { - 'before': { - 'total_keys': len(self.hash_buckets), - 'total_items': sum(len(arr) for arr in self.hash_buckets.values()), - 'empty_keys': sum(1 for arr in self.hash_buckets.values() if len(arr) == 0) - } - } - - # 2. 可选：删除空key - removed_keys = 0 - if remove_empty: - removed_keys = self._cleanup_empty_keys(verbose) - - # 3. 更新后统计 - stats['after'] = { - 'total_keys': len(self.hash_buckets), - 'total_items': sum(len(arr) for arr in self.hash_buckets.values()), - 'empty_keys': sum(1 for arr in self.hash_buckets.values() if len(arr) == 0) - } - - # 4. 计算变化 - stats['changes'] = { - 'keys_removed': removed_keys, - 'items_removed': stats['before']['total_items'] - stats['after']['total_items'] - } - - # 5. 记录日志 - if verbose or stats['changes']['keys_removed'] > 0: - self._logger.info("哈希桶更新完成:") - self._logger.info(f" 📊 Key数量: {stats['before']['total_keys']} → {stats['after']['total_keys']}") - self._logger.info(f" 📦 元素总数: {stats['before']['total_items']} → {stats['after']['total_items']}") - self._logger.info(f" 🗑️ 删除空key: {stats['changes']['keys_removed']}") - - return stats - - def get_hash_buckets_summary(self) -> dict: - """ - 获取哈希桶的摘要信息 - - 返回: - dict: 包含详细统计信息的字典 - """ - # 基础统计 - total_keys = len(self.hash_buckets) - total_items = sum(len(arr) for arr in self.hash_buckets.values()) - empty_keys = sum(1 for arr in self.hash_buckets.values() if len(arr) == 0) - - # 按大小分类统计 - size_distribution = { - 'large': 0, # >= 8192 - 'medium': 0, # 2048-8192 - 'small': 0 # < 2048 - } - - items_by_size = { - 'large': 0, - 'medium': 0, - 'small': 0 - } - - for key, arr in self.hash_buckets.items(): - count = len(arr) - if key >= 8192: - size_distribution['large'] += 1 - items_by_size['large'] += count - elif key >= 2048: - size_distribution['medium'] += 1 - items_by_size['medium'] += count - else: - size_distribution['small'] += 1 - items_by_size['small'] += count - - # 返回完整摘要 - return { - 'basic': { - 'total_keys': total_keys, - 'total_items': total_items, - 'empty_keys': empty_keys, - 'non_empty_keys': total_keys - empty_keys - }, - 'size_distribution': size_distribution, - 'items_by_size': items_by_size, - 'memory_usage': self.estimate_memory_usage() - } - - def print_example(self, key: int) -> None: - """打印示例数据""" - if key in self.hash_buckets: - arr = self.hash_buckets[key] - print(f"Key {key} 的数据数量: {len(arr)}") - print("前3条数据:") - for item in arr[:3]: - print(f" w: {item['w']}, l: {item['l']}, name: {item['name']}") - else: - print(f"Key {key} 不存在。") - - def pack_with_deletion(self, box_capacity: int = 16384) -> List[np.ndarray]: - """按容量装箱，优先多样性，装箱后立即从原桶中删除已用元素 - （用于单独处理 (box_capacity/key)==2^n 的 key ） - 中间遇到不满箱时，更换 1次 装箱策略 - """ - from collections import deque - - boxes = [] - - # 为每个 key 维护一个 deque，方便 pop （仅考虑存在元素的桶，后面数据量非常大时可以考虑放入 while 循环） - key_queues = {k: deque(enumerate(self.hash_buckets[k])) - for k in self.hb2_keys - if k in self.hash_buckets and len(self.hash_buckets[k]) > 0} - - while any(key_queues.values()): - current_box_items = [] - current_sum = 0 - used_indices = defaultdict(list) # key -> list of indices to delete - - keys_to_try = deque(sorted(key_queues.keys())) - - while keys_to_try and current_sum < box_capacity: - key = keys_to_try.popleft() - queue = key_queues[key] - if not queue: - continue - - idx, item = queue[0] - l_val = key #item['l'] - if current_sum + l_val <= box_capacity: - queue.popleft() - current_box_items.append(item) - current_sum += l_val - used_indices[key].append(idx) - - # 如果该 key 还有剩余，放回队列尾部 - if queue: - keys_to_try.append(key) - - if current_box_items and current_sum==box_capacity: - # 满箱：输出并删除 - boxes.append(np.array(current_box_items, dtype=self.DTYPE_SAMPLE_INFO)) - - # 从 self.hash_buckets 中删除已用元素 - for key, indices in used_indices.items(): - indices = sorted(indices, reverse=True) - for idx in indices: - self.hash_buckets[key] = np.delete(self.hash_buckets[key], idx) - - # 更新 key_queues 中的 deque（未删除的又更新到 key_que） - # 更新 key_queues 中的 deque（用掉的元素），这样做在很大程度上可以避免完全陷入死循环（除非所有队列剩余元素数目完全相同） - key_queues[key] = deque(enumerate(self.hash_buckets[key])) - else: - # 加一个判断，如果各个队列元素数完全相同，则改变一次 packing 策略(这种情况跳不出循环♻️) ，再回到原始方法 - self._logger.info(f"当前箱子没有满: {current_sum}") - self._logger.info(f"当前箱子元素: {current_box_items}") - - left_elems = [len(self.hash_buckets[k]) for k in self.hb2_keys if k in self.hash_buckets and len(self.hash_buckets[k])>0] - # 拼包剩余的 key - left_keys = [k for k in self.hb2_keys if k in self.hash_buckets and len(self.hash_buckets[k])>0] - print(f"剩余的key及其元素数量：(keys, nums):({left_keys},{left_elems})") - if len(set(left_elems)) == 1: - self._logger.info(f"改变拼包策略，尝试跳出 循环♻️") - b_succeed=False - # todo ...... 不考虑多样性的拼包 - current_box2 = [] - current_sum2 = 0 - used_keys_num = defaultdict(int) # 记录这个桶用了几个元素 - for key2 in left_keys: # 取出 1个桶 - if b_succeed: # 只拼一个 - print(f"改变策略拼包成功:✅✅✅✅✅✅✅✅✅✅") - break - arr2 = self.hash_buckets[key2] - l_val2 = key2 - for item2 in arr2: - if current_sum2 + l_val2 <= box_capacity: - current_box2.append(item2) - current_sum2 += l_val2 - used_keys_num[key2] += 1 - - if current_sum2==box_capacity: - boxes.append(np.array(current_box2, dtype=self.DTYPE_SAMPLE_INFO)) - current_box2 = [] - current_sum2 = 0 - # 删除元素 - for kkey, knum in used_keys_num.items(): - for _ in range(knum): - # self.delete_by_index(self.hash_buckets, kkey,0) - self.hash_buckets[kkey] = np.delete(self.hash_buckets[kkey], 0) - key_queues[kkey] = deque(enumerate(self.hash_buckets[key])) - # 重新同步 key_queues 和 self.hash_buckets - # key_queues = {k: deque(enumerate(self.hash_buckets[k])) - # for k in self.hb2_keys if k in self.hash_buckets} - print(f"改变策略拼包成功:✅✅✅✅✅{boxes[-1]}") - used_keys_num = defaultdict(int) - b_succeed = True - break - else: - current_box2 = [] - current_sum2 = 0 - used_keys_num = defaultdict(int) - b_succeed = False - print(f"改变策略拼包失败:❌❌❌❌❌") - break - pass - else: - print(f"num of left_elems:{left_elems}") - - return boxes - - def pack_with_deletion_recursion(self, box_capacity: int = 16384) -> List[np.ndarray]: - """递归多样性优先装箱：只输出/删除满箱，所有不满箱混合重装，直到只剩一个不满箱。 - （用于单独处理 (box_capacity/key)==2^n 的 key ） - 递归实现 - """ - from collections import deque, defaultdict - def recursive_diversity_pack(key_queues): - boxes = [] - not_full_items = [] - print("----------- pack_with_deletion_recursion -----------") - while any(key_queues.values()): - current_box = [] - current_sum = 0 - used_indices = defaultdict(list) - keys_to_try = deque(sorted(key_queues.keys())) - - # 多样性优先：每轮从不同桶取 - while keys_to_try and current_sum < box_capacity: - key = keys_to_try.popleft() - queue = key_queues[key] - if not queue: - continue - idx, item = queue[0] - l_val = item['l'] - if current_sum + l_val <= box_capacity: - queue.popleft() - current_box.append((key, idx, item)) - current_sum += l_val - used_indices[key].append(idx) - if queue: - keys_to_try.append(key) - - if current_sum == box_capacity: - # 满箱，输出并记录要删除的索引 - boxes.append(np.array([item for _, _, item in current_box], dtype=self.DTYPE_SAMPLE_INFO)) - for key, indices in used_indices.items(): - # 删除已用元素 - indices = sorted(indices, reverse=True) - for idx in indices: - self.hash_buckets[key] = np.delete(self.hash_buckets[key], idx) - # 更新 key_queues - key_queues[key] = deque(enumerate(self.hash_buckets[key])) - elif current_box: - # 不满箱，暂存 - not_full_items.extend(current_box) - - return boxes, not_full_items - - # 初始化 key_queues - key_queues = {k: deque(enumerate(self.hash_buckets[k])) for k in self.hb2_keys if k in self.hash_buckets} - boxes, not_full_items = recursive_diversity_pack(key_queues) - - # 混合所有不满箱元素递归装箱 - while not_full_items: - # 混合所有剩余元素，重新分桶 - mixed = defaultdict(list) - for _, _, item in not_full_items: - mixed[item['l']].append(item) - key_queues = {k: deque(enumerate(np.array(v, dtype=self.DTYPE_SAMPLE_INFO))) for k, v in mixed.items()} - new_boxes, new_not_full_items = recursive_diversity_pack(key_queues) - boxes.extend(new_boxes) - if not new_boxes or not new_not_full_items: - break - not_full_items = new_not_full_items - return boxes, not_full_items - - def pack_large_seed_parallel_multithread(self, box_capacity: int = 16384, min_ratio: float = 0.95, - max_workers: int = None) -> List[np.ndarray]: - """ - 多线程版本（处理 pack_with_deletion 之后的元素）：大种子并行装箱，小元素作为共享资源，实时删除元素 - （对于一个箱子中的物品数量没有任何限制，速度会比较快一点） - 参数: - box_capacity: 箱子容量 - min_ratio: 最小装载率阈值 - max_workers: 最大线程数，None时自动设置为CPU核心数 - - 返回: - List[np.ndarray]: 成功装箱的箱子列表 - """ - if max_workers is None: - max_workers = min(os.cpu_count(), 8) # 限制最大线程数 - - half = box_capacity // 2 - # half = 4096 - large_keys = [k for k in self.hash_buckets.keys() if k >= half] - small_keys = [k for k in self.hash_buckets.keys() if k < half] - - if not large_keys: - self._logger.warning("没有找到大种子元素") - return [] - - # 1. 线程安全的共享资源管理器 - class SharedResourceManager: - def __init__(self, hash_buckets, small_keys, large_keys): - self.lock = threading.RLock() # 可重入锁 - self.hash_buckets = hash_buckets # 直接引用原始哈希桶 - self.small_keys = small_keys - self.large_keys = large_keys - - # 初始化可用的small keys - self.available_small_keys = sorted([ - k for k in small_keys - if k in hash_buckets and len(hash_buckets[k]) > 0 - ]) - - # 统计信息 - self.total_processed = 0 - self.successful_boxes = 0 - - def get_seed_item(self, seed_key: int) -> tuple: - """线程安全地获取种子元素""" - with self.lock: - if (seed_key in self.hash_buckets and - len(self.hash_buckets[seed_key]) > 0): - - item = self.hash_buckets[seed_key][0] - self.hash_buckets[seed_key] = self.hash_buckets[seed_key][1:] - return True, item - return False, None - - def get_item_by_key(self, target_key: int) -> tuple: - """线程安全地从指定key获取一个元素并删除""" - with self.lock: - if (target_key in self.hash_buckets and - len(self.hash_buckets[target_key]) > 0): - - item = self.hash_buckets[target_key][0] - self.hash_buckets[target_key] = self.hash_buckets[target_key][1:] - - # 如果这个key的桶空了，从available_small_keys中移除 - if (len(self.hash_buckets[target_key]) == 0 and - target_key in self.available_small_keys): - self.available_small_keys.remove(target_key) - - return True, item - return False, None - - def get_available_small_keys(self) -> List[int]: - """获取当前可用的小key列表""" - with self.lock: - return self.available_small_keys.copy() - - def update_stats(self, success: bool): - """更新统计信息""" - with self.lock: - self.total_processed += 1 - if success: - self.successful_boxes += 1 - - def get_stats(self) -> dict: - """获取统计信息""" - with self.lock: - small_items_count = sum( - len(self.hash_buckets[k]) for k in self.small_keys - if k in self.hash_buckets - ) - large_items_count = sum( - len(self.hash_buckets[k]) for k in self.large_keys - if k in self.hash_buckets - ) - - return { - 'small_items_remaining': small_items_count, - 'large_items_remaining': large_items_count, - 'available_small_keys': len(self.available_small_keys), - 'total_processed': self.total_processed, - 'successful_boxes': self.successful_boxes, - 'success_rate': (self.successful_boxes / max(1, self.total_processed)) - } - - # 2. 二分查找函数 - def search_for_fit_key(available_keys: List[int], remaining_capacity: int) -> int: - """在可用的key中二分查找最大能装入的key""" - if not available_keys: - return -1 - index = bisect.bisect(available_keys, remaining_capacity) - return -1 if index == 0 else (index - 1) - - # 3. 单个种子的装箱函数 - def pack_single_seed(seed_key: int, shared_manager: SharedResourceManager, - thread_id: int) -> tuple: - """为单个种子进行装箱""" - try: - # 获取种子元素 - success, seed_item = shared_manager.get_seed_item(seed_key) - if not success: - return False, None, thread_id, 0, "无可用种子" - - current_box = [seed_item] - remaining_capacity = box_capacity - seed_key - items_added = 1 - - # 贪心装箱：优先装入大的元素 - max_iterations = 1000 # 防止死循环 - iteration = 0 - - while remaining_capacity > 0 and iteration < max_iterations: - iteration += 1 - - # 获取当前可用的小key - available_keys = shared_manager.get_available_small_keys() - if not available_keys: - break - - # 二分查找最大可装入的key - best_key_index = search_for_fit_key(available_keys, remaining_capacity) - if best_key_index == -1: - break - - best_key = available_keys[best_key_index] - - # 尝试从该key获取元素 - success, item = shared_manager.get_item_by_key(best_key) - if not success: - continue # 这个key已经被其他线程用完，重试 - - current_box.append(item) - remaining_capacity -= best_key - items_added += 1 - - # 如果装满就停止 - if remaining_capacity == 0: - break - - # 检查装载率 - current_capacity = box_capacity - remaining_capacity - is_successful = current_capacity >= min_ratio * box_capacity - - result_box = current_box if is_successful else None - load_ratio = current_capacity / box_capacity - - return (is_successful, result_box, thread_id, current_capacity, - f"装载率:{load_ratio:.1%}, 物品数:{items_added}") - - except Exception as e: - return False, None, thread_id, 0, f"装箱异常: {str(e)}" - - # 4. 准备所有大种子任务 - seed_tasks = [] - total_large_items = 0 - - for key in large_keys: - if key in self.hash_buckets: - count = len(self.hash_buckets[key]) - total_large_items += count - # 为每个大元素创建一个装箱任务 - for _ in range(count): - seed_tasks.append(key) - - if not seed_tasks: - self._logger.warning("没有可用的大种子元素") - return [] - - # 5. 初始化共享资源管理器 - shared_manager = SharedResourceManager(self.hash_buckets, small_keys, large_keys) - initial_stats = shared_manager.get_stats() - - self._logger.info(f"开始多线程装箱:") - self._logger.info(f" 🌱 大种子任务: {len(seed_tasks)} 个") - self._logger.info(f" 🔧 线程数: {max_workers}") - self._logger.info(f" 📦 目标容量: {box_capacity}") - self._logger.info(f" 📊 最小装载率: {min_ratio:.1%}") - self._logger.info(f" 🗂️ 小元素: {initial_stats['small_items_remaining']} 个") - - # 6. 多线程执行 - output_boxes = [] - failed_reasons = defaultdict(int) - start_time = time.time() - - with ThreadPoolExecutor(max_workers=max_workers, - thread_name_prefix="PackWorker") as executor: - - # 提交所有任务 - future_to_task = {} - for i, seed_key in enumerate(seed_tasks): - future = executor.submit(pack_single_seed, seed_key, shared_manager, i) - future_to_task[future] = (seed_key, i) - - # 处理结果 - with tqdm(total=len(seed_tasks), unit='seed', - desc=f'多线程装箱', dynamic_ncols=True) as pbar: - - for future in as_completed(future_to_task): - seed_key, task_id = future_to_task[future] - - try: - success, box, thread_id, capacity, info = future.result(timeout=30) - - shared_manager.update_stats(success) - - if success and box is not None: - output_boxes.append(np.array(box, dtype=self.DTYPE_SAMPLE_INFO)) - else: - failed_reasons[info] += 1 - - pbar.update(1) - - # 每50个任务更新一次描述 - if pbar.n % 50 == 0: - current_stats = shared_manager.get_stats() - pbar.set_description( - f'装箱进度(成功:{current_stats["successful_boxes"]}, ' - f'成功率:{current_stats["success_rate"]:.1%}, ' - f'剩余小元素:{current_stats["small_items_remaining"]})' - ) - - except Exception as e: - self._logger.error(f"任务 {task_id} (种子key={seed_key}) 执行失败: {e}") - failed_reasons[f"执行异常: {str(e)}"] += 1 - pbar.update(1) - - end_time = time.time() - - # 7. 输出详细统计信息 - final_stats = shared_manager.get_stats() - - if output_boxes: - total_items = sum(len(box) for box in output_boxes) - avg_items_per_box = total_items / len(output_boxes) - total_capacity_used = len(output_boxes) * box_capacity - - self._logger.info(f"多线程装箱完成:") - self._logger.info(f" ⏱️ 总耗时: {end_time - start_time:.2f}秒") - self._logger.info(f" 📦 成功箱子: {len(output_boxes)}") - self._logger.info(f" 📊 总成功率: {final_stats['success_rate']:.2%}") - self._logger.info(f" 📈 平均每箱物品数: {avg_items_per_box:.1f}") - self._logger.info(f" 💾 总计使用物品: {total_items}") - self._logger.info(f" 🔗 剩余小元素: {final_stats['small_items_remaining']}") - self._logger.info(f" 🔑 剩余小key数: {final_stats['available_small_keys']}") - - if failed_reasons: - self._logger.info(f" ❌ 失败原因统计:") - for reason, count in failed_reasons.items(): - self._logger.info(f" {reason}: {count}次") - else: - self._logger.warning("没有成功装箱任何物品") - self._logger.info(f"失败原因: {dict(failed_reasons)}") - - return output_boxes - - def pack_with_min_items_constraint_multithread(self, box_capacity: int = 16384, - min_items: int = 10, min_ratio: float = 0.95, - max_workers: int = None) -> List[np.ndarray]: - """ - 多线程多约束装箱：容量约束 + 最小物品数量约束 - （对于每个箱子内的最少物品数增加限制,保证后续的 attn 时间尽量接近） - 参数: - box_capacity: 箱子容量 - min_items: 每箱最少物品数量 - min_ratio: 最小装载率阈值 - max_workers: 最大线程数 - - 返回: - List[np.ndarray]: 满足所有约束的箱子列表 - """ - if max_workers is None: - max_workers = min(os.cpu_count(), 6) # 约束问题计算量大，减少线程数 - - half = box_capacity // 2 - # half = 4096 - print(f"种子筛选条件参数 half:{half}") - large_keys = [k for k in self.hash_buckets.keys() if k >= half] - small_keys = [k for k in self.hash_buckets.keys() if k < half] - - if not large_keys: - self._logger.warning("没有找到大种子元素") - return [] - - # 1. 种子潜力评估器 - class SeedPotentialAnalyzer: - def __init__(self, hash_buckets, small_keys, box_capacity, min_items): - self.hash_buckets = hash_buckets - self.small_keys = sorted(small_keys) - self.box_capacity = box_capacity - self.min_items = min_items - - def calculate_potential(self, seed_key: int) -> float: - """计算种子的装箱成功潜力""" - remaining_capacity = self.box_capacity - seed_key - - if remaining_capacity <= 0: - return 0.0 - - # 统计小元素的分布 - available_small_items = sum( - len(self.hash_buckets[k]) for k in self.small_keys - if k in self.hash_buckets - ) - - if available_small_items == 0: - return 0.0 - - # 估算能装入的物品数量（保守估计） - min_small_key = min(self.small_keys) if self.small_keys else remaining_capacity - max_possible_items = remaining_capacity // min_small_key - - # 考虑实际可用性（不是所有小key都有元素） - practical_items = min(max_possible_items, available_small_items // 2) - total_items = practical_items + 1 # +1 for seed - - # 潜力评分 - count_score = min(total_items / self.min_items, 1.0) if self.min_items > 0 else 1.0 - capacity_score = seed_key / self.box_capacity - diversity_score = len([k for k in self.small_keys if k <= remaining_capacity]) / len(self.small_keys) if self.small_keys else 0 - - return count_score * 0.5 + capacity_score * 0.3 + diversity_score * 0.2 - - # 2. 增强的共享资源管理器 - class EnhancedSharedManager: - def __init__(self, hash_buckets, small_keys, large_keys): - self.lock = threading.RLock() - self.hash_buckets = hash_buckets - self.small_keys = sorted(small_keys) - self.large_keys = large_keys - - # 维护可用key的统计信息 - self.key_stats = {} - self._update_key_stats() - - # 性能统计 - self.stats = { - 'total_attempts': 0, - 'successful_boxes': 0, - 'failed_by_count': 0, - 'failed_by_ratio': 0, - 'failed_by_capacity': 0 - } - - def _update_key_stats(self): - """更新key的统计信息""" - self.key_stats = {} - for k in self.small_keys: - if k in self.hash_buckets and len(self.hash_buckets[k]) > 0: - self.key_stats[k] = len(self.hash_buckets[k]) - - def get_seed_item(self, seed_key: int) -> Tuple[bool, Optional[np.record]]: - """获取种子元素""" - with self.lock: - if (seed_key in self.hash_buckets and - len(self.hash_buckets[seed_key]) > 0): - item = self.hash_buckets[seed_key][0] - self.hash_buckets[seed_key] = self.hash_buckets[seed_key][1:] - return True, item - return False, None - - def get_item_by_key(self, target_key: int) -> Tuple[bool, Optional[np.record]]: - """获取指定key的元素""" - with self.lock: - if (target_key in self.hash_buckets and - len(self.hash_buckets[target_key]) > 0): - item = self.hash_buckets[target_key][0] - self.hash_buckets[target_key] = self.hash_buckets[target_key][1:] - - # 更新统计 - if target_key in self.key_stats: - self.key_stats[target_key] -= 1 - if self.key_stats[target_key] <= 0: - del self.key_stats[target_key] - - return True, item - return False, None - - def get_available_keys_with_counts(self) -> Dict[int, int]: - """获取可用key及其元素数量""" - with self.lock: - return self.key_stats.copy() - - def rollback_items(self, items_to_rollback: List[Tuple[int, np.record]]): - """回滚失败装箱的元素""" - with self.lock: - for key, item in reversed(items_to_rollback): # 逆序回滚 - self.hash_buckets[key] = np.insert(self.hash_buckets[key], 0, item) - # 更新统计 - if key in self.small_keys: - self.key_stats[key] = self.key_stats.get(key, 0) + 1 - - def update_stats(self, result_type: str): - """更新统计信息""" - with self.lock: - self.stats['total_attempts'] += 1 - if result_type in self.stats: - self.stats[result_type] += 1 - - def get_current_stats(self) -> Dict: - """获取当前统计""" - with self.lock: - total_small_items = sum( - len(self.hash_buckets[k]) for k in self.small_keys - if k in self.hash_buckets - ) - return { - **self.stats, - 'remaining_small_items': total_small_items, - 'available_key_types': len(self.key_stats) - } - - # 3. 智能装箱策略 - def is_feasible_quick_check(remaining_capacity: int, current_items: int, - available_keys: Dict[int, int], min_items: int) -> bool: - """快速可行性检查""" - if current_items >= min_items: - return True - - needed_items = min_items - current_items - if not available_keys: - return False - - # 贪心估算：优先使用小key - sorted_keys = sorted(available_keys.keys()) - possible_items = 0 - remaining_cap = remaining_capacity - - for key in sorted_keys: - if remaining_cap <= 0: - break - max_from_this_key = min(remaining_cap // key, available_keys[key]) - possible_items += max_from_this_key - remaining_cap -= max_from_this_key * key - - if possible_items >= needed_items: - return True - - return False - - def select_optimal_key(strategy: str, available_keys: Dict[int, int], - remaining_capacity: int, current_items: int, min_items: int) -> Optional[int]: - """根据策略选择最优key""" - suitable_keys = [k for k in available_keys.keys() - if k <= remaining_capacity and available_keys[k] > 0] - if not suitable_keys: - return None - - if strategy == "prioritize_count": - # 优先数量：选择最小的key - return min(suitable_keys) - elif strategy == "prioritize_capacity": - # 优先容量：选择最大的key - return max(suitable_keys) - elif strategy == "balanced": - # 平衡策略：选择中等大小，但考虑可用数量 - suitable_keys.sort() - # 倾向于选择有较多可用元素的key - key_scores = [(k, available_keys[k] * (remaining_capacity / k)) for k in suitable_keys] - key_scores.sort(key=lambda x: x[1], reverse=True) - return key_scores[0][0] - else: - return suitable_keys[0] - - # 4. 核心装箱函数 - def pack_single_seed_with_constraints(seed_key: int, shared_manager: EnhancedSharedManager, - thread_id: int) -> Tuple: - """带约束的单种子装箱""" - try: - # 获取种子 - success, seed_item = shared_manager.get_seed_item(seed_key) - if not success: - shared_manager.update_stats('failed_by_capacity') - return False, None, thread_id, 0, 0, "无可用种子" - - current_box = [seed_item] - used_items = [(seed_key, seed_item)] # 用于回滚 - remaining_capacity = box_capacity - seed_key - items_count = 1 - - max_iterations = min_items * 16 # 防止无限循环 - iteration = 0 - - # 装箱主循环 - while (remaining_capacity > 0 and - items_count < min_items * 8 and # 允许超过最小数量 - iteration < max_iterations): - - iteration += 1 - available_keys = shared_manager.get_available_keys_with_counts() - - # 快速可行性检查 - if (items_count < min_items and - not is_feasible_quick_check(remaining_capacity, items_count, - available_keys, min_items)): - # 无法达到最小物品数，提前退出 - shared_manager.rollback_items(used_items) - shared_manager.update_stats('failed_by_count') - return False, None, thread_id, 0, items_count, f"无法达到{min_items}个物品" - - # 动态策略选择 - if items_count < min_items * 0.8: - strategy = "prioritize_count" - elif items_count < min_items: - strategy = "balanced" - else: - strategy = "prioritize_capacity" - - # 选择下一个key - target_key = select_optimal_key(strategy, available_keys, - remaining_capacity, items_count, min_items) - if target_key is None: - break - - # 获取元素 - success, item = shared_manager.get_item_by_key(target_key) - if not success: - continue # 该key已被其他线程用完 - - current_box.append(item) - used_items.append((target_key, item)) - remaining_capacity -= target_key - items_count += 1 - - # 如果达到完美装载，可以提前结束 - if remaining_capacity == 0 and items_count >= min_items: - break - - # 检查所有约束 - current_capacity = box_capacity - remaining_capacity - load_ratio = current_capacity / box_capacity - - meets_count = items_count >= min_items - meets_ratio = load_ratio >= min_ratio - meets_capacity = remaining_capacity >= 0 - - success = meets_count and meets_ratio and meets_capacity - - if success: - shared_manager.update_stats('successful_boxes') - return True, current_box, thread_id, current_capacity, items_count, f"成功：{items_count}个物品，装载率{load_ratio:.1%}" - else: - # 装箱失败，回滚 - shared_manager.rollback_items(used_items) - if not meets_count: - shared_manager.update_stats('failed_by_count') - reason = f"物品数不足：{items_count}<{min_items}" - elif not meets_ratio: - shared_manager.update_stats('failed_by_ratio') - reason = f"装载率不足：{load_ratio:.1%}<{min_ratio:.1%}" - else: - shared_manager.update_stats('failed_by_capacity') - reason = "容量约束失败" - - return False, None, thread_id, current_capacity, items_count, reason - - except Exception as e: - shared_manager.update_stats('failed_by_capacity') - return False, None, thread_id, 0, 0, f"装箱异常: {str(e)}" - - # 5. 种子预处理和筛选 - analyzer = SeedPotentialAnalyzer(self.hash_buckets, small_keys, box_capacity, min_items) - - # 收集并评估所有种子 - seed_candidates = [] - for key in large_keys: - if key in self.hash_buckets: - potential = analyzer.calculate_potential(key) - count = len(self.hash_buckets[key]) - for _ in range(count): - seed_candidates.append((key, potential)) # 确保是元组 - if not seed_candidates: - self._logger.warning("没有可用的种子候选") - return [] - - # 按潜力排序，只处理高潜力种子 - seed_candidates.sort(key=lambda x: x[1], reverse=True) - potential_threshold = 0.2 # 只处理潜力>0.2的种子 - # high_potential_seeds = [seed for seed, potential in seed_candidates if potential > potential_threshold] - # 修复：正确处理筛选逻辑 - high_potential_candidates = [(seed, potential) for seed, potential in seed_candidates - if potential > potential_threshold] - - # 最后保底：至少保留前50%的种子 - if len(high_potential_candidates) < len(seed_candidates) * 0.5: - mid_point = len(seed_candidates) // 2 - high_potential_candidates = seed_candidates[:mid_point] - - # 修复：正确提取种子列表 - selected_seeds = [seed for seed, potential in high_potential_candidates] - - self._logger.info(f"种子筛选完成:") - self._logger.info(f" 📊 总种子数: {len(seed_candidates)}") - self._logger.info(f" 🎯 筛选后: {len(selected_seeds)}") - self._logger.info(f" 🚀 筛选率: {len(selected_seeds)/len(seed_candidates):.1%}") - self._logger.info(f" 🔧 线程数: {max_workers}") - self._logger.info(f" 📦 约束: 容量≥{min_ratio:.0%}, 物品≥{min_items}个") - - # 6. 初始化共享管理器 - shared_manager = EnhancedSharedManager(self.hash_buckets, small_keys, large_keys) - initial_stats = shared_manager.get_current_stats() - - self._logger.info(f"初始资源状态:") - self._logger.info(f" 🗂️ 小元素总数: {initial_stats['remaining_small_items']}") - self._logger.info(f" 🔑 可用小key种类: {initial_stats['available_key_types']}") - - # 7. 多线程执行装箱 - output_boxes = [] - detailed_results = [] - start_time = time.time() - - with ThreadPoolExecutor(max_workers=max_workers, - thread_name_prefix="ConstraintPack") as executor: - - # 提交所有任务 - future_to_seed = {} - for i, seed_key in enumerate(selected_seeds): - future = executor.submit(pack_single_seed_with_constraints, seed_key, shared_manager, i) - future_to_seed[future] = (seed_key, i) - - # 处理结果 - with tqdm(total=len(selected_seeds), unit='seed', - desc='多约束装箱', dynamic_ncols=True) as pbar: - - completed_tasks = 0 - for future in as_completed(future_to_seed): - seed_key, task_id = future_to_seed[future] - - try: - success, box, thread_id, capacity, item_count, info = future.result(timeout=60) - - if success and box is not None: - output_boxes.append(np.array(box, dtype=self.DTYPE_SAMPLE_INFO)) - - detailed_results.append({ - 'seed_key': seed_key, - 'success': success, - 'capacity': capacity, - 'item_count': item_count, - 'info': info, - 'thread_id': thread_id - }) - - completed_tasks += 1 - pbar.update(1) - - # 每100个任务更新一次进度描述 - if completed_tasks % 100 == 0: - current_stats = shared_manager.get_current_stats() - success_rate = current_stats['successful_boxes'] / max(1, current_stats['total_attempts']) - pbar.set_description( - f'多约束装箱(成功:{current_stats["successful_boxes"]}, ' - f'成功率:{success_rate:.1%}, ' - f'剩余:{current_stats["remaining_small_items"]})' - ) - - except Exception as e: - self._logger.error(f"任务 {task_id} (种子={seed_key}) 失败: {e}") - detailed_results.append({ - 'seed_key': seed_key, - 'success': False, - 'capacity': 0, - 'item_count': 0, - 'info': f"任务异常: {str(e)}", - 'thread_id': -1 - }) - pbar.update(1) - - end_time = time.time() - - # 8. 详细统计分析 - final_stats = shared_manager.get_current_stats() - - # 按失败原因分类 - failure_analysis = defaultdict(int) - success_details = [] - - for result in detailed_results: - if result['success']: - success_details.append(result) - else: - # 简化失败原因 - info = result['info'] - if '物品数不足' in info: - failure_analysis['物品数量不足'] += 1 - elif '装载率不足' in info: - failure_analysis['装载率不足'] += 1 - elif '无可用种子' in info: - failure_analysis['种子耗尽'] += 1 - elif '无法达到' in info: - failure_analysis['可行性检查失败'] += 1 - else: - failure_analysis['其他原因'] += 1 - - # 9. 输出详细报告 - if output_boxes: - # 成功装箱的统计 - total_items_packed = sum(len(box) for box in output_boxes) - avg_items_per_box = total_items_packed / len(output_boxes) - capacities = [result['capacity'] for result in success_details] - avg_capacity = sum(capacities) / len(capacities) if capacities else 0 - avg_load_ratio = avg_capacity / box_capacity - - item_counts = [result['item_count'] for result in success_details] - min_items_in_box = min(item_counts) if item_counts else 0 - max_items_in_box = max(item_counts) if item_counts else 0 - - self._logger.info(f"🎉 多约束装箱完成!") - self._logger.info(f"📊 执行统计:") - self._logger.info(f" ⏱️ 总耗时: {end_time - start_time:.2f}秒") - self._logger.info(f" 🎯 处理种子: {len(selected_seeds)}") - self._logger.info(f" 📦 成功箱子: {len(output_boxes)}") - self._logger.info(f" 📈 总成功率: {len(output_boxes)/len(selected_seeds):.2%}") - - self._logger.info(f"📦 装箱质量:") - self._logger.info(f" 📊 平均装载率: {avg_load_ratio:.1%}") - self._logger.info(f" 🔢 平均物品数: {avg_items_per_box:.1f}") - self._logger.info(f" 📉 物品数范围: {min_items_in_box}-{max_items_in_box}") - self._logger.info(f" 💾 总打包物品: {total_items_packed}") - - self._logger.info(f"🔗 剩余资源:") - self._logger.info(f" 🗂️ 小元素: {final_stats['remaining_small_items']}") - self._logger.info(f" 🔑 可用key类型: {final_stats['available_key_types']}") - - if failure_analysis: - self._logger.info(f"❌ 失败分析:") - for reason, count in failure_analysis.items(): - percentage = count / len(selected_seeds) * 100 - self._logger.info(f" {reason}: {count}次 ({percentage:.1f}%)") - else: - self._logger.warning("⚠️ 没有成功装箱任何物品!") - self._logger.info(f"失败原因分布: {dict(failure_analysis)}") - self._logger.info(f"建议:") - self._logger.info(f" 1. 降低 min_items (当前: {min_items})") - self._logger.info(f" 2. 降低 min_ratio (当前: {min_ratio})") - self._logger.info(f" 3. 检查数据分布是否合理") - - return output_boxes - - # ''' - # # 10. 将函数绑定到类 - # # HashBucketProcessor.pack_with_min_items_constraint_multithread = pack_with_min_items_constraint_multithread - - # # 11. 使用示例 - # def demo_constrained_packing(): - # """演示多约束装箱的使用""" - - # # 创建处理器 - # processor = HashBucketProcessor("data.txt") - # processor.build_buckets() - # processor.find_items(16384) - - # print("=== 原始装箱（无物品数量约束） ===") - # boxes_original = processor.pack_large_seed_parallel_multithread( - # box_capacity=16384, - # min_ratio=0.95, - # max_workers=4 - # ) - # print(f"原始方法成功箱子: {len(boxes_original)}") - - # print("\n=== 多约束装箱（至少10个物品） ===") - # boxes_constrained = processor.pack_with_min_items_constraint_multithread( - # box_capacity=16384, - # min_items=10, - # min_ratio=0.95, - # max_workers=4 - # ) - # print(f"约束方法成功箱子: {len(boxes_constrained)}") - - # # 对比分析 - # if boxes_constrained: - # avg_items_constrained = sum(len(box) for box in boxes_constrained) / len(boxes_constrained) - # print(f"约束方法平均每箱物品数: {avg_items_constrained:.1f}") - - # return boxes_original, boxes_constrained - - # # 如果想直接运行演示 - # if __name__ == "__main__": - # # demo_constrained_packing() - # pass - # ''' - def pack_with_flexible_seeds(self, box_capacity: int = 16384, - seed_strategy: str = "auto", - seed_params: dict = None, - min_items: int = 10, min_ratio: float = 0.95, - max_workers: int = None) -> List[np.ndarray]: - """ - 自定义种子选择策略 + 背包元素数限制 + 输出背包最小容量 - - 参数: - seed_strategy: 种子策略 - - "auto": 自动使用 box_capacity // 2 - - "custom_half": 使用自定义的half值 - - "specified_keys": 使用指定的key列表 - - "size_range": 使用大小范围筛选 - - "top_n": 使用最大的N个keys作为种子 - - "capacity_ratio": 指定占用 capacity 的百分比的 - seed_params: 策略参数字典 - - - """ - if max_workers is None: - max_workers = min(os.cpu_count(), 6) - - if seed_params is None: - seed_params = {} - - # 🎯 根据策略生成种子 - if seed_strategy == "auto": - half = box_capacity // 2 - large_keys = [k for k in self.hash_buckets.keys() if k >= half] - - elif seed_strategy == "custom_half": - custom_half = seed_params.get("half", box_capacity // 3) - # max_elems = seed_params.get("n_max", None) # 每个 key 中最多取出的种子数÷ - large_keys = [k for k in self.hash_buckets.keys() if k >= custom_half] - - elif seed_strategy == "specified_keys": - specified_keys = seed_params.get("keys", []) - large_keys = [k for k in specified_keys if k in self.hash_buckets] - - elif seed_strategy == "size_range": - min_size = seed_params.get("min_size", box_capacity // 3) - max_size = seed_params.get("max_size", box_capacity) - large_keys = [k for k in self.hash_buckets.keys() - if min_size <= k <= max_size] - - elif seed_strategy == "top_n": - n = seed_params.get("n", 5) - available_keys = sorted(self.hash_buckets.keys(), reverse=True) - large_keys = available_keys[:n] - - elif seed_strategy == "capacity_ratio": - min_ratio = seed_params.get("min_ratio", 0.3) # 至少30%容量 - max_ratio = seed_params.get("max_ratio", 1.0) # 最多100%容量 - min_size = int(box_capacity * min_ratio) - max_size = int(box_capacity * max_ratio) - large_keys = [k for k in self.hash_buckets.keys() - if min_size <= k <= max_size] - - # # 利用分位数 Quartiles 进行种子的选择(Q1,Q2,Q3) - # elif seed_strategy == "quartiles": - # q_n = seed_params.get("q_n", 3) # 至少30%容量 - # elems_max_num = seed_params.get("max_num", 20) # 每一个 key 里面取出作为种子的的最大元素个数 - # pass - - else: - raise ValueError(f"不支持的种子策略: {seed_strategy}") - - # 生成小元素列表 - small_keys = [k for k in self.hash_buckets.keys() if k not in large_keys] - - # 策略信息 - self._logger.info(f"种子策略: {seed_strategy}") - self._logger.info(f"策略参数: {seed_params}") - self._logger.info(f" 🌱 种子keys(max): {large_keys[-1]}") - self._logger.info(f" 🔧 填充keys数量: {len(small_keys)}") - - if not large_keys: - self._logger.warning(f"策略 {seed_strategy} 没有生成任何种子") - return [] - - - # 1. 种子潜力评估器 - class SeedPotentialAnalyzer: - def __init__(self, hash_buckets, small_keys, box_capacity, min_items): - self.hash_buckets = hash_buckets - self.small_keys = sorted(small_keys) - self.box_capacity = box_capacity - self.min_items = min_items - - def calculate_potential(self, seed_key: int) -> float: - """计算种子的装箱成功潜力""" - remaining_capacity = self.box_capacity - seed_key - - if remaining_capacity <= 0: - return 0.0 - - # 统计小元素的分布 - available_small_items = sum( - len(self.hash_buckets[k]) for k in self.small_keys - if k in self.hash_buckets - ) - - if available_small_items == 0: - return 0.0 - - # 估算能装入的物品数量（保守估计） - min_small_key = min(self.small_keys) if self.small_keys else remaining_capacity - max_possible_items = remaining_capacity // min_small_key - - # 考虑实际可用性（不是所有小key都有元素） - practical_items = min(max_possible_items, available_small_items // 2) - total_items = practical_items + 1 # +1 for seed - - # 潜力评分 - count_score = min(total_items / self.min_items, 1.0) if self.min_items > 0 else 1.0 - capacity_score = seed_key / self.box_capacity - diversity_score = len([k for k in self.small_keys if k <= remaining_capacity]) / len(self.small_keys) if self.small_keys else 0 - - return count_score * 0.5 + capacity_score * 0.3 + diversity_score * 0.2 - - # 2. 增强的共享资源管理器 - class EnhancedSharedManager: - def __init__(self, hash_buckets, small_keys, large_keys): - self.lock = threading.RLock() - self.hash_buckets = hash_buckets - self.small_keys = sorted(small_keys) - self.large_keys = large_keys - - # 维护可用key的统计信息 - self.key_stats = {} - self._update_key_stats() - - # 性能统计 - self.stats = { - 'total_attempts': 0, - 'successful_boxes': 0, - 'failed_by_count': 0, - 'failed_by_ratio': 0, - 'failed_by_capacity': 0 - } - - def _update_key_stats(self): - """更新key的统计信息""" - self.key_stats = {} - for k in self.small_keys: - if k in self.hash_buckets and len(self.hash_buckets[k]) > 0: - self.key_stats[k] = len(self.hash_buckets[k]) - - def get_seed_item(self, seed_key: int) -> Tuple[bool, Optional[np.record]]: - """获取种子元素""" - with self.lock: - if (seed_key in self.hash_buckets and - len(self.hash_buckets[seed_key]) > 0): - item = self.hash_buckets[seed_key][0] - self.hash_buckets[seed_key] = self.hash_buckets[seed_key][1:] - return True, item - return False, None - - def get_item_by_key(self, target_key: int) -> Tuple[bool, Optional[np.record]]: - """获取指定key的元素""" - with self.lock: - if (target_key in self.hash_buckets and - len(self.hash_buckets[target_key]) > 0): - item = self.hash_buckets[target_key][0] - self.hash_buckets[target_key] = self.hash_buckets[target_key][1:] - - # 更新统计 - if target_key in self.key_stats: - self.key_stats[target_key] -= 1 - if self.key_stats[target_key] <= 0: - del self.key_stats[target_key] - - return True, item - return False, None - - def get_available_keys_with_counts(self) -> Dict[int, int]: - """获取可用key及其元素数量""" - with self.lock: - return self.key_stats.copy() - - def rollback_items(self, items_to_rollback: List[Tuple[int, np.record]]): - """回滚失败装箱的元素""" - with self.lock: - for key, item in reversed(items_to_rollback): # 逆序回滚 - self.hash_buckets[key] = np.insert(self.hash_buckets[key], 0, item) - # 更新统计 - if key in self.small_keys: - self.key_stats[key] = self.key_stats.get(key, 0) + 1 - - def update_stats(self, result_type: str): - """更新统计信息""" - with self.lock: - self.stats['total_attempts'] += 1 - if result_type in self.stats: - self.stats[result_type] += 1 - - def get_current_stats(self) -> Dict: - """获取当前统计""" - with self.lock: - total_small_items = sum( - len(self.hash_buckets[k]) for k in self.small_keys - if k in self.hash_buckets - ) - return { - **self.stats, - 'remaining_small_items': total_small_items, - 'available_key_types': len(self.key_stats) - } - - # 3. 智能装箱策略 - def is_feasible_quick_check(remaining_capacity: int, current_items: int, - available_keys: Dict[int, int], min_items: int) -> bool: - """快速可行性检查""" - if current_items >= min_items: - return True - - needed_items = min_items - current_items - if not available_keys: - return False - - # 贪心估算：优先使用小key - sorted_keys = sorted(available_keys.keys()) - possible_items = 0 - remaining_cap = remaining_capacity - - for key in sorted_keys: - if remaining_cap <= 0: - break - max_from_this_key = min(remaining_cap // key, available_keys[key]) - possible_items += max_from_this_key - remaining_cap -= max_from_this_key * key - - if possible_items >= needed_items: - return True - - return False - - def select_optimal_key(strategy: str, available_keys: Dict[int, int], - remaining_capacity: int, current_items: int, min_items: int) -> Optional[int]: - """根据策略选择最优key""" - suitable_keys = [k for k in available_keys.keys() - if k <= remaining_capacity and available_keys[k] > 0] - if not suitable_keys: - return None - - if strategy == "prioritize_count": - # 优先数量：选择最小的key - return min(suitable_keys) - elif strategy == "prioritize_capacity": - # 优先容量：选择最大的key - return max(suitable_keys) - elif strategy == "balanced": - # 平衡策略：选择中等大小，但考虑可用数量 - suitable_keys.sort() - # 倾向于选择有较多可用元素的key - key_scores = [(k, available_keys[k] * (remaining_capacity / k)) for k in suitable_keys] - key_scores.sort(key=lambda x: x[1], reverse=True) - return key_scores[0][0] - else: - return suitable_keys[0] - - # 4. 核心装箱函数 - def pack_single_seed_with_constraints(seed_key: int, shared_manager: EnhancedSharedManager, - thread_id: int) -> Tuple: - """带约束的单种子装箱""" - try: - # 获取种子 - success, seed_item = shared_manager.get_seed_item(seed_key) - if not success: - shared_manager.update_stats('failed_by_capacity') - return False, None, thread_id, 0, 0, "无可用种子" - - current_box = [seed_item] - used_items = [(seed_key, seed_item)] # 用于回滚 - remaining_capacity = box_capacity - seed_key - items_count = 1 - - max_iterations = min_items * 16 # 防止无限循环 由 5--->15(12 for 16384) - iteration = 0 - - # 装箱主循环 - while (remaining_capacity > 0 and - items_count < min_items * 8 and # 允许超过最小数量（可能有非常小的值）(5 for 16384) - iteration < max_iterations): - - iteration += 1 - available_keys = shared_manager.get_available_keys_with_counts() - - # 快速可行性检查 - if (items_count < min_items and - not is_feasible_quick_check(remaining_capacity, items_count, - available_keys, min_items)): - # 无法达到最小物品数，提前退出 - shared_manager.rollback_items(used_items) - shared_manager.update_stats('failed_by_count') - return False, None, thread_id, 0, items_count, f"无法达到{min_items}个物品" - - # 动态策略选择 - if items_count < min_items * 0.8: - strategy = "prioritize_count" - elif items_count < min_items: - strategy = "balanced" - else: - strategy = "prioritize_capacity" - - # 选择下一个key - target_key = select_optimal_key(strategy, available_keys, - remaining_capacity, items_count, min_items) - if target_key is None: - break - - # 获取元素 - success, item = shared_manager.get_item_by_key(target_key) - if not success: - continue # 该key已被其他线程用完 - - current_box.append(item) - used_items.append((target_key, item)) - remaining_capacity -= target_key - items_count += 1 - - # 如果达到完美装载，可以提前结束 - if remaining_capacity == 0 and items_count >= min_items: - break - - # 检查所有约束 - current_capacity = box_capacity - remaining_capacity - load_ratio = current_capacity / box_capacity - - meets_count = items_count >= min_items - meets_ratio = load_ratio >= min_ratio - meets_capacity = remaining_capacity >= 0 - - success = meets_count and meets_ratio and meets_capacity - - if success: - shared_manager.update_stats('successful_boxes') - return True, current_box, thread_id, current_capacity, items_count, f"成功：{items_count}个物品，装载率{load_ratio:.1%}" - else: - # 装箱失败，回滚 - shared_manager.rollback_items(used_items) - if not meets_count: - shared_manager.update_stats('failed_by_count') - reason = f"物品数不足：{items_count}<{min_items}" - elif not meets_ratio: - shared_manager.update_stats('failed_by_ratio') - reason = f"装载率不足：{load_ratio:.1%}<{min_ratio:.1%}" - else: - shared_manager.update_stats('failed_by_capacity') - reason = "容量约束失败" - - return False, None, thread_id, current_capacity, items_count, reason - - except Exception as e: - shared_manager.update_stats('failed_by_capacity') - return False, None, thread_id, 0, 0, f"装箱异常: {str(e)}" - - # 5. 种子预处理和筛选 - analyzer = SeedPotentialAnalyzer(self.hash_buckets, small_keys, box_capacity, min_items) - - # 收集并评估所有种子 - seed_candidates = [] - for key in large_keys: - if key in self.hash_buckets: - potential = analyzer.calculate_potential(key) - count = len(self.hash_buckets[key]) - for _ in range(count): - seed_candidates.append((key, potential)) # 确保是元组 - if not seed_candidates: - self._logger.warning("没有可用的种子候选") - return [] - - # 按潜力排序，只处理高潜力种子 - seed_candidates.sort(key=lambda x: x[1], reverse=True) - potential_threshold = 0.2 # 只处理潜力>0.2的种子 - # high_potential_seeds = [seed for seed, potential in seed_candidates if potential > potential_threshold] - # 修复：正确处理筛选逻辑 - high_potential_candidates = [(seed, potential) for seed, potential in seed_candidates - if potential > potential_threshold] - - # 最后保底：至少保留前50%的种子 - if len(high_potential_candidates) < len(seed_candidates) * 0.5: - mid_point = len(seed_candidates) // 2 - high_potential_candidates = seed_candidates[:mid_point] - - # 修复：正确提取种子列表 - selected_seeds = [seed for seed, potential in high_potential_candidates] - - self._logger.info(f"种子筛选完成:") - self._logger.info(f" 📊 总种子数: {len(seed_candidates)}") - self._logger.info(f" 🎯 筛选后: {len(selected_seeds)}") - self._logger.info(f" 🚀 筛选率: {len(selected_seeds)/len(seed_candidates):.1%}") - self._logger.info(f" 🔧 线程数: {max_workers}") - self._logger.info(f" 📦 约束: 容量≥{min_ratio:.0%}, 物品≥{min_items}个") - - # 6. 初始化共享管理器 - shared_manager = EnhancedSharedManager(self.hash_buckets, small_keys, large_keys) - initial_stats = shared_manager.get_current_stats() - - self._logger.info(f"初始资源状态:") - self._logger.info(f" 🗂️ 小元素总数: {initial_stats['remaining_small_items']}") - self._logger.info(f" 🔑 可用小key种类: {initial_stats['available_key_types']}") - - # 7. 多线程执行装箱 - output_boxes = [] - detailed_results = [] - start_time = time.time() - - with ThreadPoolExecutor(max_workers=max_workers, - thread_name_prefix="ConstraintPack") as executor: - - # 提交所有任务 - future_to_seed = {} - for i, seed_key in enumerate(selected_seeds): - future = executor.submit(pack_single_seed_with_constraints, seed_key, shared_manager, i) - future_to_seed[future] = (seed_key, i) - - # 处理结果 - with tqdm(total=len(selected_seeds), unit='seed', - desc='多约束装箱', dynamic_ncols=True) as pbar: - - completed_tasks = 0 - for future in as_completed(future_to_seed): - seed_key, task_id = future_to_seed[future] - - try: - success, box, thread_id, capacity, item_count, info = future.result(timeout=60) - - if success and box is not None: - output_boxes.append(np.array(box, dtype=self.DTYPE_SAMPLE_INFO)) - - detailed_results.append({ - 'seed_key': seed_key, - 'success': success, - 'capacity': capacity, - 'item_count': item_count, - 'info': info, - 'thread_id': thread_id - }) - - completed_tasks += 1 - pbar.update(1) - - # 每100个任务更新一次进度描述 - if completed_tasks % 100 == 0: - current_stats = shared_manager.get_current_stats() - success_rate = current_stats['successful_boxes'] / max(1, current_stats['total_attempts']) - pbar.set_description( - f'多约束装箱(成功:{current_stats["successful_boxes"]}, ' - f'成功率:{success_rate:.1%}, ' - f'剩余:{current_stats["remaining_small_items"]})' - ) - - except Exception as e: - self._logger.error(f"任务 {task_id} (种子={seed_key}) 失败: {e}") - detailed_results.append({ - 'seed_key': seed_key, - 'success': False, - 'capacity': 0, - 'item_count': 0, - 'info': f"任务异常: {str(e)}", - 'thread_id': -1 - }) - pbar.update(1) - - end_time = time.time() - - # 8. 详细统计分析 - final_stats = shared_manager.get_current_stats() - - # 按失败原因分类 - failure_analysis = defaultdict(int) - success_details = [] - - for result in detailed_results: - if result['success']: - success_details.append(result) - else: - # 简化失败原因 - info = result['info'] - if '物品数不足' in info: - failure_analysis['物品数量不足'] += 1 - elif '装载率不足' in info: - failure_analysis['装载率不足'] += 1 - elif '无可用种子' in info: - failure_analysis['种子耗尽'] += 1 - elif '无法达到' in info: - failure_analysis['可行性检查失败'] += 1 - else: - failure_analysis['其他原因'] += 1 - - # 9. 输出详细报告 - if output_boxes: - # 成功装箱的统计 - total_items_packed = sum(len(box) for box in output_boxes) - avg_items_per_box = total_items_packed / len(output_boxes) - capacities = [result['capacity'] for result in success_details] - avg_capacity = sum(capacities) / len(capacities) if capacities else 0 - avg_load_ratio = avg_capacity / box_capacity - - item_counts = [result['item_count'] for result in success_details] - min_items_in_box = min(item_counts) if item_counts else 0 - max_items_in_box = max(item_counts) if item_counts else 0 - - self._logger.info(f"🎉 多约束装箱完成!") - self._logger.info(f"📊 执行统计:") - self._logger.info(f" ⏱️ 总耗时: {end_time - start_time:.2f}秒") - self._logger.info(f" 🎯 处理种子: {len(selected_seeds)}") - self._logger.info(f" 📦 成功箱子: {len(output_boxes)}") - self._logger.info(f" 📈 总成功率: {len(output_boxes)/len(selected_seeds):.2%}") - - self._logger.info(f"📦 装箱质量:") - self._logger.info(f" 📊 平均装载率: {avg_load_ratio:.1%}") - self._logger.info(f" 🔢 平均物品数: {avg_items_per_box:.1f}") - self._logger.info(f" 📉 物品数范围: {min_items_in_box}-{max_items_in_box}") - self._logger.info(f" 💾 总打包物品: {total_items_packed}") - - self._logger.info(f"🔗 剩余资源:") - self._logger.info(f" 🗂️ 小元素: {final_stats['remaining_small_items']}") - self._logger.info(f" 🔑 可用key类型: {final_stats['available_key_types']}") - - if failure_analysis: - self._logger.info(f"❌ 失败分析:") - for reason, count in failure_analysis.items(): - percentage = count / len(selected_seeds) * 100 - self._logger.info(f" {reason}: {count}次 ({percentage:.1f}%)") - else: - self._logger.warning("⚠️ 没有成功装箱任何物品!") - self._logger.info(f"失败原因分布: {dict(failure_analysis)}") - self._logger.info(f"建议:") - self._logger.info(f" 1. 降低 min_items (当前: {min_items})") - self._logger.info(f" 2. 降低 min_ratio (当前: {min_ratio})") - self._logger.info(f" 3. 检查数据分布是否合理") - - - - # 只输出 1 个用于状态跟踪，输出3个用于实际应用 - return output_boxes#, failure_analysis, final_stats - - def pack_simplest_strategy( - self, - keys: List[int], - m: int, - box_capacity: int = 16384, - min_ratio: float = 0.95, - max_workers: int = None, - ) -> List[np.ndarray]: - """ - 极简装箱策略： - 1. 从指定 keys 中随机选 m 个种子； - 2. 其余所有剩余元素作为装填池； - 3. 多线程装箱（成功删，失败回滚）； - 4. 剩余元素单线程兜底，最后一批强制输出并清空。 - """ - import random - import threading - from concurrent.futures import ThreadPoolExecutor, as_completed - - if max_workers is None: - max_workers = min(os.cpu_count(), 8) - - # ---------- 1. 构造种子池 & 装填池 ---------- - seed_pool = [] # [(key, item), ...] - fill_buckets = defaultdict(list) # key -> [item, ...] - - # 1.1 收集种子池 & 未选中的同 key 元素 - for k in keys: - if k not in self.hash_buckets or len(self.hash_buckets[k]) == 0: - continue - arr = self.hash_buckets[k] - # 随机选 m 个，若不足则全选 - chosen = random.sample(list(arr), min(m, len(arr))) - seed_pool.extend([(k, item) for item in chosen]) - # 未选中的进入装填池 - mask = np.ones(len(arr), dtype=bool) - idxs = [i for i, it in enumerate(arr) if it in chosen] - mask[idxs] = False - fill_buckets[k].extend(arr[mask]) - - # 1.2 非 keys 的所有元素归入装填池 - for k in self.hash_buckets: - if k not in keys: - fill_buckets[k].extend(self.hash_buckets[k]) - - if not seed_pool: - self._logger.warning("种子池为空，直接输出剩余元素为一箱") - # 强制输出一箱 - leftover = [] - for k, items in fill_buckets.items(): - leftover.extend(items) - if leftover: - box = np.array(leftover, dtype=self.DTYPE_SAMPLE_INFO) - # 清空 hash_buckets - for k in list(self.hash_buckets.keys()): - del self.hash_buckets[k] - return [box] - return [] - - # ---------- 2. 线程安全的资源管理器 ---------- - class SimpleManager: - def __init__(self, seed_items, fill_dict, dtype): - self.lock = threading.RLock() - # 种子队列 - self.seed_q = seed_items[:] # 拷贝 - # 装填池 - self.fill = defaultdict(deque) - for k, lst in fill_dict.items(): - self.fill[k] = deque(lst) - # 统计 - self.boxes = [] - self.attempts = 0 - self.success = 0 - self.dtype = dtype - - def pop_seed(self): - with self.lock: - if not self.seed_q: - return None - return self.seed_q.pop() - - def pop_fill(self, key): - with self.lock: - if not self.fill[key]: - return None - return self.fill[key].popleft() - - def add_box(self, box): - with self.lock: - self.boxes.append(np.array(box, dtype=self.dtype)) - self.success += 1 - - def rollback(self, rollback_items): - with self.lock: - for key, item in reversed(rollback_items): - self.fill[key].appendleft(item) - - def remaining_elements(self): - with self.lock: - return sum(len(q) for q in self.fill.values()) - - def all_items(self): - with self.lock: - items = [] - for k, q in self.fill.items(): - items.extend(q) - return items - - from collections import deque - # mgr = SimpleManager(seed_pool, fill_buckets) - mgr = SimpleManager(seed_pool, fill_buckets, self.DTYPE_SAMPLE_INFO) - - # ---------- 3. 多线程装箱 ---------- - def pack_once(args): - seed_key, seed_item, tid = args - box = [seed_item] - used = [(seed_key, seed_item)] - rem = box_capacity - seed_key - - # 贪心装填 - for k in sorted(mgr.fill.keys(), reverse=True): - while rem >= k and mgr.fill[k]: - it = mgr.pop_fill(k) - if it is None: - break - box.append(it) - used.append((k, it)) - rem -= k - if rem == 0: - break - - load = box_capacity - rem - if load >= min_ratio * box_capacity: - mgr.add_box(box) - return True, tid, load - else: - mgr.rollback(used) - return False, tid, load - - # 构造任务列表 - tasks = [(k, it, i) for i, (k, it) in enumerate(mgr.seed_q)] - mgr.seed_q.clear() # 清空，由任务列表取代 - - with ThreadPoolExecutor(max_workers=max_workers) as exe: - futs = [exe.submit(pack_once, t) for t in tasks] - for f in as_completed(futs): - ok, tid, load = f.result() - mgr.attempts += 1 - - # ---------- 4. 单线程兜底 ---------- - leftover_keys = list(mgr.fill.keys()) - random.shuffle(leftover_keys) - - while mgr.remaining_elements() > 0: - # 随机找一个种子：从剩余 key 中随机挑一个元素 - candidates = [(k, mgr.fill[k][0]) for k in leftover_keys if mgr.fill[k]] - if not candidates: - break - seed_key, seed_item = random.choice(candidates) - mgr.pop_fill(seed_key) # 取出作为种子 - - box = [seed_item] - used = [(seed_key, seed_item)] - rem = box_capacity - seed_key - - # 继续装填 - for k in sorted(mgr.fill.keys(), reverse=True): - while rem >= k and mgr.fill[k]: - it = mgr.pop_fill(k) - if it is None: - break - box.append(it) - used.append((k, it)) - rem -= k - if rem == 0: - break - - # 强制输出 - mgr.add_box(box) - - # ---------- 5. 同步回 self.hash_buckets ---------- - # 此时 mgr.fill 已全部清空，hash_buckets 直接置空 - for k in list(self.hash_buckets.keys()): - del self.hash_buckets[k] - - self._logger.info( - f"pack_simplest_strategy 完成：多线程任务 {mgr.attempts}，" - f"成功 {mgr.success}，兜底输出 {len(mgr.boxes) - mgr.success} 箱" - ) - return mgr.boxes - - - - - def check_hash_buckets_state(self): - """检查哈希桶的当前状态""" - total_items = sum(len(arr) for arr in self.hash_buckets.values()) - # total_keys = len(self.hash_buckets) - total_keys = len([key for key in self.hash_buckets if len(self.hash_buckets[key])>0]) # 没有删除 元素为0 的 key - - # 按key大小分类统计 - key_distribution = defaultdict(int) - for key in self.hash_buckets.keys(): - if key >= 8192: - key_distribution['large'] += len(self.hash_buckets[key]) - elif key >= 2048: - key_distribution['medium'] += len(self.hash_buckets[key]) - else: - key_distribution['small'] += len(self.hash_buckets[key]) - - print(f"当前哈希桶状态:") - print(f" 📦 总元素数: {total_items}") - print(f" 🔑 总key数: {total_keys}") - print(f" 📊 分布情况:") - for size, count in key_distribution.items(): - print(f" {size}: {count} 个元素") - - return { - 'total_items': total_items, - 'total_keys': total_keys, - 'key_distribution': dict(key_distribution) - } - -# ###---------------------------------------------------------------------------------------------- -# ###---------------------------------------------------------------------------------------------- -# ###---------------------------------------------------------------------------------------------- -# class PackingTracker: -# """装箱操作追踪器""" -# def __init__(self, processor): -# self.processor = processor -# self.history = [] - -# def track_packing(self, strategy_name: str, **kwargs): -# """记录一次装箱操作""" - -# # 记录操作前状态 -# before_state = self.processor.check_hash_buckets_state() - -# # 执行装箱 -# boxes = getattr(self.processor, strategy_name)(**kwargs) - -# # 记录操作后状态 -# after_state = self.processor.check_hash_buckets_state() - -# # 计算变化 -# change = { -# 'strategy': strategy_name, -# 'kwargs': kwargs, -# 'before': before_state, -# 'after': after_state, -# 'boxes_count': len(boxes), -# 'items_used': before_state['total_items'] - after_state['total_items'] -# } - -# self.history.append(change) -# return boxes - -# def print_summary(self): -# """打印装箱历史摘要""" -# print("\n=== 装箱操作历史 ===") -# for i, op in enumerate(self.history, 1): -# print(f"\n操作 {i}: {op['strategy']}") -# print(f"参数: {op['kwargs']}") -# print(f"装箱数: {op['boxes_count']}") -# print(f"使用元素: {op['items_used']}") -# print(f"成功率: {op['boxes_count'] / op['kwargs'].get('max_workers', 1):.1%}") - -class PackingTracker: - """装箱操作追踪器""" - def __init__(self, processor): - self.processor = processor - self.history = [] - - def track_packing(self, strategy_name: str, **kwargs): - """记录一次装箱操作""" - before_state = self.processor.check_hash_buckets_state() - # 支持返回详细统计（如 total_attempts），否则只返回箱子列表 - result = getattr(self.processor, strategy_name)(**kwargs) - if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], dict): - boxes = result[0] - stats = result[1] - total_attempts = stats.get('total_attempts', len(boxes)) - else: - boxes = result - total_attempts = len(boxes) - after_state = self.processor.check_hash_buckets_state() - change = { - 'strategy': strategy_name, - 'kwargs': kwargs, - 'before': before_state, - 'after': after_state, - 'boxes_count': len(boxes), - 'items_used': before_state['total_items'] - after_state['total_items'], - 'total_attempts': total_attempts - } - self.history.append(change) - return boxes - - def print_summary(self): - """打印装箱历史摘要""" - print("\n=== 装箱操作历史 ===") - for i, op in enumerate(self.history, 1): - print(f"\n操作 {i}: {op['strategy']}") - print(f"参数: {op['kwargs']}") - print(f"装箱数: {op['boxes_count']}") - print(f"使用元素: {op['items_used']}") - if op.get('total_attempts', 0): - rate = op['boxes_count'] / op['total_attempts'] - print(f"成功率: {rate:.1%} ({op['boxes_count']}/{op['total_attempts']})") - else: - print(f"成功率: N/A") - - -# # 使用示例 -# tracker = PackingTracker(processor) -# tracker.track_packing('pack_large_seed_parallel_multithread', -# box_capacity=16384, min_ratio=0.95) -# tracker.track_packing('pack_with_min_items_constraint_multithread', -# box_capacity=16384, min_items=10, min_ratio=0.90) -# tracker.print_summary() - - -def analyze_packing_history(tracker): - """分析装箱历史""" - print("\n=== 详细分析 ===") - - total_boxes = sum(op['boxes_count'] for op in tracker.history) - total_items = sum(op['items_used'] for op in tracker.history) - - print(f"总装箱数: {total_boxes}") - print(f"总使用元素: {total_items}") - - # 分析每个策略的效果 - strategy_stats = defaultdict(lambda: {'count': 0, 'items': 0, 'boxes': 0}) - for op in tracker.history: - strategy = op['strategy'] - strategy_stats[strategy]['count'] += 1 - strategy_stats[strategy]['items'] += op['items_used'] - strategy_stats[strategy]['boxes'] += op['boxes_count'] - - print("\n策略效果对比:") - for strategy, stats in strategy_stats.items(): - avg_boxes = stats['boxes'] / stats['count'] - avg_items = stats['items'] / stats['count'] - print(f"{strategy}:") - print(f" 平均装箱数: {avg_boxes:.1f}") - print(f" 平均使用元素: {avg_items:.0f}") - print(f" 平均成功率: {avg_boxes / 4:.1%}") # 假设使用4个线程 -#### --------------------------------------- #### -## CKPT 相关的 tools -import pickle - -def save_ckpt(tracker, file_path: str): - """ - 保存 tracker（包含 processor）到文件 - """ - with open(file_path, 'wb') as f: - pickle.dump(tracker, f) - print(f"已保存ckpt到 {file_path}") - -def load_ckpt(file_path: str): - """ - 加载 tracker（包含 processor）状态 - """ - with open(file_path, 'rb') as f: - tracker = pickle.load(f) - print(f"已加载ckpt: {file_path}") - return tracker - -def save_bin_boxes(bin_boxes, file_path: str): - """ - 保存单步装箱结果 - """ - with open(file_path, 'wb') as f: - pickle.dump(bin_boxes, f) - print(f"已保存装箱结果到 {file_path}") - -def load_bin_boxes(file_path: str): - """ - 加载单步装箱结果 - """ - with open(file_path, 'rb') as f: - bin_boxes = pickle.load(f) - print(f"已加载装箱结果: {file_path}") - return bin_boxes diff --git a/tools/data_preprocess/offline_packing/readme.ipynb b/tools/data_preprocess/offline_packing/readme.ipynb deleted file mode 100644 index b5f3eb7f..00000000 --- a/tools/data_preprocess/offline_packing/readme.ipynb +++ /dev/null @@ -1,128 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5e151774-fcfd-4fe1-b927-ecccfa5e0263", - "metadata": {}, - "source": [ - "### 格式说明" - ] - }, - { - "cell_type": "markdown", - "id": "3f38c9cf-cd9c-4746-9fe4-8a53b5cf64a2", - "metadata": {}, - "source": [ - "- VQA 的 json 格式 \n", - "```json\n", - "{\n", - " \"messages\": [\n", - " {\n", - " \"content\": \"nHow would you describe the image to someone who cannot see it?\",\n", - " \"role\": \"user\"\n", - " },\n", - " {\n", - " \"content\": \"The image shows an aged envelope with a slightly creased and stained surface. On the top left corner, handwritten in ink is \\\"Gone Away\\\" with some illegible cursive scribbling beneath it. Two postage stamps from Palestine are affixed in the top right corner; one is valued at 5 mils and the other at 10 mils, both canceled with a postmark. The 5 mil stamp features a landscape with a building and palm trees, and the 10 mil stamp depicts the Tower of David. Below the stamps is a circular postmark from London E.C., dated \\\"1:30 PM 13 OC 39\\\". Addressed to \\\"Messrs. Walter, Berger & Co., 21-23 Farringdon Rd., London, E.C.1, ENGLAND\\\", the typeset is slightly faded. There is a rectangular purple stamp marked \\\"PARTI\\\" indicating that the addressee has departed. Overlapping the typed address and purple stamp is a watermark or promotional stamp in red, reading \\\"MORE AT Philatelyclub.com\\\".\",\n", - " \"role\": \"assistant\"\n", - " }\n", - " ],\n", - " \"images\": [\n", - " \"allava-caption-part1allava-caption-part117625.jpg\"\n", - " ]\n", - "}\n", - "```\n", - "- S1" - ] - }, - { - "cell_type": "markdown", - "id": "d1631778-c48a-4080-b4b5-0eefe36311d1", - "metadata": {}, - "source": [ - "### S1: 统计训练过程中转换为的 token 数" - ] - }, - { - "cell_type": "markdown", - "id": "dcb64723-4aa0-4202-afe0-cf9d3d9883eb", - "metadata": {}, - "source": [ - "- 配置文件：参见 s1_config_vqa_pretrain_5M_8k.yaml 里面有详细说明\n", - "- 执行：\n", - " `python s1_get_tokenlens_v2-sft.py --config ./configs/s1_config_vqa_pretrain_5M_8k.yaml`\n", - "- 生成文件：token_info_v2_vqa_pretrain_5M_8k.txt，作为 S2 的输入" - ] - }, - { - "cell_type": "markdown", - "id": "e6a5ec62-cba5-4e51-bb3a-61b3b53adcdb", - "metadata": {}, - "source": [ - "### S2：按设定的序列长度做 packing " - ] - }, - { - "cell_type": "markdown", - "id": "bcd0a5bb-6e63-49bd-a397-857bc9e0de80", - "metadata": {}, - "source": [ - "#### S2-1 整个数据集做 packing，输出 sample 分组 \n", - "- `S2_hashbacket.ipynb`里面有一个示例流程，可以参考\n", - "- 具体使用哪些策略，按什么样的顺序，需要结合实际数据集来调整" - ] - }, - { - "cell_type": "markdown", - "id": "164e3599-2e98-40c4-87ea-c854393eb307", - "metadata": {}, - "source": [ - "#### S2-2 根据 S2-1 的结果处理数据，转换到打包前的装备状态\n", - "使用示例：`python -u s2_prepare_rawsamples-vqa_5500k-16k-fast.py 2>&1 | tee ./logs/s2_proc_vqa_5500k-16k-fast.log`\n", - "python 里面会有输出路径，S3 要使用这个路径" - ] - }, - { - "cell_type": "markdown", - "id": "dd98f556-4d28-418c-b654-ae82a78892c7", - "metadata": {}, - "source": [ - "### S3：打包成 energon 格式数据" - ] - }, - { - "cell_type": "markdown", - "id": "4ed0df86-a65d-4d0d-b529-86a9f267b0e5", - "metadata": {}, - "source": [ - "这个已经在脚本里写好了，如何调用参考 `token_info_v2_emova_3000tk.txt`,改一下路径即可" - ] - }, - { - "cell_type": "markdown", - "id": "d71bc19c-b33b-4240-8fd3-cf0aad0166e1", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tools/data_preprocess/offline_packing/s1_get_tokenlens_v2.py b/tools/data_preprocess/offline_packing/s1_get_tokenlens_v2.py deleted file mode 100644 index affe50e6..00000000 --- a/tools/data_preprocess/offline_packing/s1_get_tokenlens_v2.py +++ /dev/null @@ -1,717 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# 使用方式 -python s1_get_tokenlens_v2.py --config ./configs/s1_config_emova.yaml -python s1_get_tokenlens_v2.py --config ./configs/s1_config_emova_3000tk.yaml - -可配置核心参数： -1. 模型与数据路径： - - CKPT_DIR: 模型权重目录（需提前部署） - 示例："/vlm/pretrain_models/Qwen2.5-VL-7B-Instruct" - - DEFAULT_DIRECTORY: 数据集根目录（需包含同名JSON与图像文件） - 示例："/vlm/xiangan/datasets/aiak_caption_emova_300k" - -2. 计算资源控制： - - STAGE1_CHUNK: 归并基数（每10个stage0文件合并为1个stage1文件） - - chunk_size: 进程级数据块大小（每个进程处理5000个样本） - - 动态线程数：通过get_adaptive_workers自动调节（CPU>80%或内存>85%时减半） - -3. 输出与临时文件： - - OUTPUT_FILE: 配对文件名清单（base_name_v2_emova.txt） - - TOKEN_INFO_FILE: 最终Token长度结果文件（token_info_v2_emova.txt） - - 临时文件：stage0_*/stage1_*（自动清理，存储中间排序结果） - -4. 其它： - - MIN_PIXELS / MAX_PIXELS: 图像预处理的像素范围限制，默认值分别为4*28*28和3578*28*28 - - n_workers: 线程池大小，默认值为96（可通过get_adaptive_workers动态调整） - - CAP_TEMPLATE: 数据格式模版，可根据实际情况变更 - -环境依赖与部署： -1. 必备Python库（建议Python 3.8+）： - pip install psutil Pillow jinja2 transformers qwen_vl_utils orjson - -2. 模型准备： - 需提前下载Qwen2.5-VL模型权重并部署到CKPT_DIR，确保AutoProcessor可正确加载 - -3. 生成文件： - - 临时文件： - - base_name_v1.txt：存储所有配对文件的基础名称（不含扩展名） - - stage0_*：分块处理的临时文件，包含局部排序的(token长度, 文件名)数据 - - stage1_*：合并stage0文件的临时文件，包含中级排序结果 - - 最终文件： - - token_info_*.txt：全局按token长度升序排列的结果文件，格式为"文件名:token长度" - - 日志文件： - - processing.log：详细记录数据处理全过程的日志信息 - -使用工作流： -① 配置调整：根据硬件环境修改CKPT_DIR、DEFAULT_DIRECTORY等路径参数 -② 依赖安装：确保所有Python库已正确安装（特别注意transformers版本兼容性） -③ 数据校验：确认DEFAULT_DIRECTORY中存在同名JSON与图像文件对 -④ 启动执行：python s1_get_tokenlens_v1.py -⑤ 监控分析：通过processing.log追踪流程，重点关注： - - 初始配对文件数（验证数据完整性） - - 各进程/线程的Token计算耗时 - - 归并阶段的文件合并行数校验 -⑥ 结果验证：检查token_info_v2_emova.txt的行数与原始样本数是否匹配 - -""" - -import os -import json -import orjson -import threading -import logging -import psutil -import tempfile -import queue -import yaml -import argparse -from pathlib import Path -from concurrent.futures import ThreadPoolExecutor, as_completed -from heapq import merge -from PIL import Image -from jinja2 import Template -from transformers import AutoProcessor -from transformers import BitsAndBytesConfig -from qwen_vl_utils import fetch_image -from queue import Empty -import multiprocessing -from multiprocessing import Pool, Manager, Value - -# 声明全局的跨进程计数器（在主模块中定义，让子进程继承） -global_total_counter = None - -# ✅ 解析命令行参数 -parser = argparse.ArgumentParser(description="Token Length Processor") -parser.add_argument("--config", type=str, default="config.yaml", help="Path to config.yaml") -parser.add_argument("--log-level", type=str, default=None, - choices=["DEBUG", "INFO", "WARNING", "ERROR"], - help="Override log level from config") -args = parser.parse_args() - -# ✅ 加载配置文件 -CONFIG_PATH = Path(args.config) -if not CONFIG_PATH.exists(): - raise FileNotFoundError(f"配置文件不存在: {CONFIG_PATH}") -with open(CONFIG_PATH, 'r', encoding='utf-8') as f: - cfg = yaml.safe_load(f) - -# ✅ 从配置中读取参数，覆盖原有常量 -MAX_TOKEN_LEN = cfg['sample']['max_len'] -task_type = cfg['sample']['task_type'] - -DEFAULT_DIRECTORY = Path(cfg['data']['directory']) -OUTPUT_FILE = Path(cfg['data']['output_base']) -TOKEN_INFO_FILE = Path(cfg['data']['output_token']) -CKPT_DIR = cfg['model']['checkpoint'] -MIN_PIXELS = cfg['image']['min_pixels'] -MAX_PIXELS = cfg['image']['max_pixels'] -# 归并参数（仅两级：stage0 → stage1） -STAGE1_CHUNK = cfg['processing']['stage1_merge_chunk'] -chunk_size = cfg['processing']['chunk_size'] -n_workers = cfg['processing']['n_workers'] -MIN_WORKERS = cfg['processing']['min_workers'] -MAX_WORKERS = cfg['processing']['max_workers'] -use_shm = cfg['logging']['use_shm'] -log_level = cfg['logging']['level'] -log_file = cfg['logging']['file'] -if args.log_level: - log_level = args.log_level.upper() - -# 日志配置 - 详细记录数据流向和合并过程 -file_handler = logging.FileHandler( - log_file, - delay=True, - encoding='utf-8' -) -stream_handler = logging.StreamHandler() - -logging.basicConfig( - level=log_level, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[file_handler, stream_handler] -) -logger = logging.getLogger(__name__) - -EXTENSIONS = (".json", ".jpg") - - -temp_dir = '/dev/shm' if use_shm else None # None 表示使用系统默认临时目录 - -def count_lines(file_path): - """统计文件有效行数（非空且含分隔符）""" - if not os.path.exists(file_path) or os.path.getsize(file_path) == 0: - return 0 - try: - with open(file_path, 'r', encoding='utf-8') as f: - return sum(1 for line in f if line.strip() and ':' in line.strip()) - except Exception as e: - logger.error(f"❌ 统计文件 {file_path} 行数失败: {str(e)}") - return 0 - - -# def find_paired_files(directory): -# """查找目录中配对的json和jpg文件""" -# directory = Path(directory) -# json_files = set() -# image_files = set() - -# for file in directory.iterdir(): -# if file.is_file(): -# base_name = file.stem -# ext_lower = file.suffix.lower() -# if ext_lower == '.json': -# json_files.add(base_name) -# elif ext_lower in ('.jpg', '.jpeg'): -# image_files.add(base_name) - -# paired_names = json_files.intersection(image_files) -# logger.info(f"ℹ️ 找到 {len(paired_names)} 对匹配文件（总样本数）") -# return paired_names - -# def find_paired_files(directory): -# """查找目录中配对的json和jpg文件（高性能版）""" -# directory = Path(directory) -# json_files = set() -# image_files = set() - -# # 使用 os.scandir 遍历 -# with os.scandir(directory) as it: -# for entry in it: -# if entry.is_file(): -# name, ext = os.path.splitext(entry.name) -# ext_lower = ext.lower() -# if ext_lower == '.json': -# json_files.add(name) -# elif ext_lower in ('.jpg', '.jpeg'): -# image_files.add(name) - -# paired_names = json_files.intersection(image_files) -# logger.info(f"ℹ️ 找到 {len(paired_names)} 对匹配文件（总样本数）") -# return paired_names - -def find_paired_files(directory): - directory = Path(directory) - files = os.listdir(directory) - json_set = {f[:-5] for f in files if f.lower().endswith('.json')} - img_set = {f[:-4] for f in files if f.lower().endswith(('.jpg', '.jpeg'))} - paired = json_set & img_set - logger.info(f"找到 {len(paired)} 对匹配文件") - return paired - - -def write_base_names_to_file(base_names, output_file): - """将配对文件名写入文件""" - try: - content = "\n".join(sorted(base_names)) + "\n" - with open(output_file, 'w', encoding='utf-8') as f: - f.write(content) - logger.info(f"ℹ️ 已将 {len(base_names)} 个配对文件名写入 {output_file}") - except Exception as e: - logger.error(f"❌ 写入 {output_file} 失败: {str(e)}") - raise - - -def read_lines_in_chunks(file_path, chunk_size): - """按块读取文件内容""" - file_path = Path(file_path) - if not file_path.exists(): - raise FileNotFoundError(f"{file_path} 不存在") - - with open(file_path, 'r', encoding='utf-8') as f: - while True: - chunk = [line.strip() for _, line in zip(range(chunk_size), f) if line.strip()] - if not chunk: - break - logger.info(f"ℹ️ 读取数据块，包含 {len(chunk)} 个样本") - yield chunk - - -# 预编译模板 -if task_type=="pretrain": - CAP_TEMPLATE = Template("<|vision_start|><|image_pad|><|vision_end|>{{ captions[0].content }}<|im_end|>") -elif task_type=="sft": - chat_template = """{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}""" - CAP_TEMPLATE = Template(chat_template) - pass - -def process_sample(json_path, img_path, processor): - """处理单个样本，返回(token_len, 文件名)""" - try: - if not Path(json_path).exists(): - raise FileNotFoundError(f"❌ JSON文件不存在: {json_path}") - if not Path(img_path).exists(): - raise FileNotFoundError(f"❌ 图片文件不存在: {img_path}") - - # 读取并渲染JSON内容 - with open(json_path, 'r', encoding='utf-8') as f: - json_data = json.load(f) - # with open(json_path, 'rb') as f: - # json_data = orjson.loads(f.read()) - - txt_input = CAP_TEMPLATE.render(captions=json_data['captions']) - img_input = fetch_image({ - 'type': 'image', - 'image': img_path, - "min_pixels": MIN_PIXELS, - "max_pixels": MAX_PIXELS, - }) - - # 计算token长度 - base_name = Path(img_path).stem - inputs = processor( - text=[txt_input], - images=img_input, - videos=None, - padding=True, - return_tensors="pt", - ) - return (inputs["input_ids"].shape[1], base_name) - - except Exception as e: - return (None, f"❌ 处理失败 [{Path(img_path).stem}]: {str(e)}") - - -def get_adaptive_workers(min_workers=20, max_workers=96): - """根据系统负载调整线程数""" - try: - cpu_usage = psutil.cpu_percent(interval=0.5) - mem_usage = psutil.virtual_memory().percent - if cpu_usage > 80 or mem_usage > 85: - adjusted = max(min_workers, max_workers // 2) - logger.info(f"系统负载过高，线程数调整为 {adjusted} (CPU: {cpu_usage}%, 内存: {mem_usage}%)") - return adjusted - return max_workers - except Exception as e: - logger.warning(f"获取系统负载失败，使用默认线程数 {max_workers}: {str(e)}") - return max_workers - -gt_maxlen=0 -def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN): - """合并多个已排序文件，按token_len升序，过滤掉 > max_token 的数据，返回(输出路径, 数据条数)""" - if not input_files: - logger.warning("⚠️ 没有文件可合并") - return (None, 0) - - # 验证输入文件并统计总数据量 - valid_files = [] - total_lines = 0 - for f in input_files: - line_count = count_lines(f) - if line_count > 0: - valid_files.append(f) - total_lines += line_count - logger.debug(f"ℹ️ 待合并文件 {os.path.basename(f)} 包含 {line_count} 条数据") - else: - logger.warning(f"⚠️ 文件 {os.path.basename(f)} 为空或无效，跳过") - - if not valid_files: - return (None, 0) - - # 定义排序键（按token_len整数排序） - def sort_key(line): - # _, token_str = line.strip().split(':', 1) - token_str = line.strip().split(':')[-1] - return int(token_str) - - try: - with open(output_file, 'w', encoding='utf-8') as out_f: - # 创建所有文件的迭代器 - iterators = [] - file_handles = [] - for fpath in valid_files: - try: - fh = open(fpath, 'r', encoding='utf-8') - file_handles.append(fh) - iterators.append(((sort_key(line), line) for line in fh)) - except Exception as e: - logger.error(f"❌ 打开文件 {os.path.basename(fpath)} 失败: {str(e)}") - - # # 归并排序并写入 - # for _, line in merge(*iterators, key=lambda x: x[0]): - # out_f.write(line) - # 归并排序并写入，过滤掉 > max_token 的行(后续可添加其他条件) - filtered_max_len = 0 - for _, line in merge(*iterators, key=lambda x: x[0]): - _, token_str = line.strip().split(':', 1) - if int(token_str) <= max_token: # ← 只保留 ≤ 8192 - out_f.write(line) - else: - logger.warning(f"⚠️ token长度：{token_str} > {max_token}: 剔除!!") - filtered_max_len+=1 - gt_maxlen - - # 关闭所有文件句柄 - for fh in file_handles: - try: - fh.close() - except Exception as e: - logger.warning(f"⚠️ 关闭文件 {fh.name} 失败: {str(e)}") - - # 验证输出文件数据完整性 - output_lines = count_lines(output_file)+filtered_max_len - if output_lines != total_lines: # 过滤掉不满足条件的 - logger.error(f"❌ 合并数据丢失！输入 {total_lines} 条，输出 {output_lines} 条，已删除错误文件") - if os.path.exists(output_file): - os.remove(output_file) - return (None, 0) - else: - logger.info(f"✅ 📊 合并成功，输入 {total_lines} 条，输出 {output_lines-filtered_max_len} 条（token ≤ {max_token}）的数据") - - return (output_file, output_lines-filtered_max_len) - except Exception as e: - logger.error(f"❌ 合并文件失败: {str(e)}") - if os.path.exists(output_file): - try: - os.remove(output_file) - except Exception as e: - logger.warning(f"⚠️ 删除失败文件 {output_file} 失败: {str(e)}") - return (None, 0) - - -def stage1_merger(input_queue, chunk_size, stage1_files, stop_event): - """ - 修复版stage1合并线程 - - 确保所有stage0文件被合并，包括最后不足10个的文件 - - 解决线程超时和数据丢失问题 - """ - buffer = [] - batch_counter = 0 - logger.info(f"💡 stage1合并线程启动，每 {chunk_size} 个stage0文件合并一次") - - try: - # 循环条件：队列有文件 或 缓冲区有文件 或 未收到停止信号 - while (not input_queue.empty()) or buffer or (not stop_event.is_set()): - # 从队列取文件（带超时防止永久阻塞） - if not input_queue.empty(): - try: - file_path = input_queue.get(timeout=1) # 超时1秒，避免永久阻塞 - buffer.append(file_path) - input_queue.task_done() - logger.debug(f"ℹ️ stage1接收文件 {os.path.basename(file_path)}，当前缓冲区: {len(buffer)}/{chunk_size}") - - # 达到合并数量则执行合并 - if len(buffer) >= chunk_size: - batch_counter += 1 - merged_file = tempfile.NamedTemporaryFile( - mode='w', delete=False, - prefix=f"stage1_batch{batch_counter:03d}_", - encoding='utf-8' - ).name - - # 执行合并 - merged_path, line_count = merge_files_by_token(buffer, merged_file) - if merged_path and line_count > 0: - stage1_files.append(merged_path) - logger.info(f"📊 stage1批次 {batch_counter} 完成: {os.path.basename(merged_path)}，包含 {line_count} 条数据（合并了 {len(buffer)} 个文件）") - else: - logger.warning(f"⚠️ stage1批次 {batch_counter} 合并失败，跳过该批次") - - # 清空缓冲区 - buffer = [] - except Empty: - continue # 队列为空时继续循环 - except Exception as e: - logger.error(f"❌ stage1处理文件时错误: {str(e)}", exc_info=True) - else: - # 队列为空时，检查是否需要强制合并剩余文件 - if buffer and stop_event.is_set(): - # 收到停止信号且缓冲区有文件，强制合并 - batch_counter += 1 - merged_file = tempfile.NamedTemporaryFile( - mode='w', delete=False, - prefix=f"stage1_remaining_batch{batch_counter:03d}_", - encoding='utf-8', - dir=temp_dir - ).name - - merged_path, line_count = merge_files_by_token(buffer, merged_file) - if merged_path and line_count > 0: - stage1_files.append(merged_path) - logger.info(f"📊 stage1剩余文件合并完成: {os.path.basename(merged_path)}，包含 {line_count} 条数据（合并了 {len(buffer)} 个文件）") - else: - logger.warning(f"❌ stage1剩余文件合并失败，数据可能丢失") - buffer = [] - else: - # 短暂休眠，减少CPU占用 - threading.Event().wait(0.5) - - # 最终检查：确保缓冲区为空（防止遗漏） - if buffer: - logger.error(f"❌ stage1线程退出时缓冲区仍有 {len(buffer)} 个文件未处理！数据丢失") - - except Exception as e: - logger.error(f"❌ stage1线程异常退出: {str(e)}", exc_info=True) - finally: - logger.info(f"📊 stage1线程退出，共生成 {len(stage1_files)} 个文件") - -# 新增：每个进程的处理函数（负责处理一个大chunk） -def process_chunk(args): - """ - 单个进程的处理逻辑：处理一个大chunk，内部用多线程并行 - - Args: - args: 包含chunk数据、处理器配置、队列等参数的元组 - """ - # 从全局变量获取计数器，而非参数 - global global_total_counter - - chunk_idx, chunk, ckpt_dir, min_pixels, max_pixels, stage0_queue = args - processor = None - processed_count = 0 # 记录当前进程处理的有效样本数 - - - try: - # 每个进程单独初始化处理器（进程间不能共享processor实例） - # quant_config = BitsAndBytesConfig(load_in_4bit=True) - processor = AutoProcessor.from_pretrained( - ckpt_dir, - min_pixels=min_pixels, - max_pixels=max_pixels, - trust_remote_code=True, - use_fast=False - ) - - # 生成当前chunk的文件路径列表 - full_paths = [] - for fn in chunk: - full_paths.append(str(DEFAULT_DIRECTORY / f"{fn}.json")) - full_paths.append(str(DEFAULT_DIRECTORY / f"{fn}.jpg")) - - n_samples = len(chunk) - logger.info(f"👉 进程 {multiprocessing.current_process().name} 开始处理块 {chunk_idx}，包含 {n_samples} 个样本") - - # 进程内创建线程池（复用线程） - n_workers = get_adaptive_workers(min_workers=MIN_WORKERS, max_workers=MAX_WORKERS) # 单个进程的线程数可适当减少 - chunk_results = [] - with ThreadPoolExecutor( - max_workers=n_workers, - thread_name_prefix=f"proc-{multiprocessing.current_process().pid}-thread" - ) as executor: - tasks = [ - executor.submit( - process_sample, - full_paths[idx*2], - full_paths[idx*2+1], - processor - ) for idx in range(n_samples) - ] - - # 收集线程任务结果 - for future in as_completed(tasks): - try: - token_len, name = future.result() - if token_len is not None: - chunk_results.append((token_len, name)) - processed_count += 1 # 统计有效样本 - else: - logger.warning(name) - except Exception as e: - logger.error(f"❌ 进程内任务错误: {str(e)}") - - # 写入stage0文件并放入跨进程队列 - if chunk_results: - chunk_results_sorted = sorted(chunk_results, key=lambda x: x[0]) - with tempfile.NamedTemporaryFile( - mode='w+', delete=False, - prefix=f"stage0_chunk{chunk_idx:03d}_", - encoding='utf-8', - dir=temp_dir - ) as f: - stage0_file = f.name - for token_len, name in chunk_results_sorted: - f.write(f"{name}:{token_len}\n") - - line_count = count_lines(stage0_file) - stage0_queue.put(stage0_file) # 放入跨进程队列 - # logger.info(f"进程 {multiprocessing.current_process().name} 完成块 {chunk_idx}，生成 {line_count} 条数据") - logger.info(f"�� 进程 {multiprocessing.current_process().name} 完成块 {chunk_idx}，有效样本 {processed_count}/{n_samples}") - - # 【关键】跨进程累加总数据量（使用Value原子操作） - with global_total_counter.get_lock(): - global_total_counter.value += processed_count - - return stage0_file # 返回生成的文件路径，用于后续清理 - - except Exception as e: - logger.error(f"❌ 进程 {multiprocessing.current_process().name} 处理失败: {str(e)}") - finally: - if processor: - del processor - return None - - -### -def main(): - global global_total_counter # 引用全局变量 - processor = None # 模型处理器实例 - stage0_files = [] # 记录所有stage0文件（用于验证和清理） - stage1_files = [] # 记录所有stage1文件（用于最终合并） - - try: - - logger.info(f"💡 --------------开始数据处理流程--------------") - - # 1. 查找配对文件并写入临时文件（json和jpg文件名相同的样本） - base_names = find_paired_files(DEFAULT_DIRECTORY) # DEFAULT_DIRECTORY 是原始数据存放位置（jpg 和 json） - total_original = len(base_names) # 原始样本总数 - logger.info(f"👉 找到 {total_original} 对原始样本文件") - if total_original == 0: - logger.warning("⚠️ 无原始样本，退出程序") - return - # 将配对文件名写入文件，用于后续分块读取 - write_base_names_to_file(base_names, OUTPUT_FILE) - - # 2. 初始化跨进程队列（用于传递stage0文件路径给合并线程） - manager = Manager() # 进程间共享队列需要用Manager - stage0_queue = manager.Queue() - stop_event = manager.Event() # 跨进程停止信号 - - # 跨进程计数器，用于统计总处理样本数（初始值0） - global_total_counter = Value('i', 0) # 'i'表示整数类型 - - # 3 启动stage1合并线程（守护线程） - stage1_thread = threading.Thread( - target=stage1_merger, - args=(stage0_queue, STAGE1_CHUNK, stage1_files, stop_event), - daemon=True - ) - stage1_thread.start() - logger.info("💡 stage1合并线程已启动") - - # 4. 处理数据并生成stage0文件（每块数据单独处理并排序） - # n_workers = 96 #get_adaptive_workers() - - # 4.1 读取所有数据块（准备分给多个进程) - chunk_size = 5000 # 每个进程处理的大chunk尺寸（根据内存调整） - all_chunks = list(read_lines_in_chunks(OUTPUT_FILE, chunk_size)) - total_chunks = len(all_chunks) - n_processes = min(multiprocessing.cpu_count(), total_chunks) - logger.info(f"👉 划分为 {total_chunks} 个块，启动 {n_processes} 个进程处理") - - # 4.2 准备进程池参数（包含模型配置、队列等） - process_args = [ - ( - idx + 1, # chunk索引 - chunk, # chunk数据 - CKPT_DIR, # 模型路径 - MIN_PIXELS, - MAX_PIXELS, - stage0_queue, # 跨进程队列 - ) for idx, chunk in enumerate(all_chunks) - ] - - # 4.3 启动进程池（进程数建议设为CPU核心数的1~2倍） - with Pool(processes=n_processes) as process_pool: - # 并行处理所有大chunk - # stage0_files = process_pool.map(process_chunk, process_args) - result = process_pool.map_async(process_chunk, process_args) - try: - stage0_files = result.get(timeout=3600) # 1小时超时 - except multiprocessing.TimeoutError: - logger.error("❌ 部分进程处理超时，强制终止") - process_pool.terminate() - - # 过滤空结果 - stage0_files = [f for f in stage0_files if f is not None] - logger.info(f"✅ 所有进程处理完成，共生成 {len(stage0_files)} 个stage0文件") - # 统计数据 - total_processed = global_total_counter.value # 直接从全局变量获取 # 获取总处理样本数 - logger.info(f"👉 原始样本数: {total_original}, 有效处理样本数: {total_processed}") - - # 验证数据完整性 - if total_processed != total_original: - logger.warning(f"❌ 数据不完整！原始 {total_original} 个，有效处理 {total_processed} 个，差异 {total_original - total_processed} 个") - else: - logger.info("✅ 数据完整性验证通过，所有样本均被有效处理") - - # 5. 等待处理完成（确保所有文件被合并） - # 等待stage0队列所有文件被处理 - logger.info("🔄 等待stage0队列处理完成...") - stage0_queue.join() # 阻塞直到所有stage0文件被消费 - logger.info("💡 stage0队列所有文件已处理完毕") - - # 发送停止信号给stage1线程，强制处理剩余文件 - logger.info("💡 通知stage1线程停止并处理剩余文件...") - stop_event.set() - - # 延长超时时间至60秒，确保大文件合并完成 - timeout_counter = 0 - while stage1_thread.is_alive() and timeout_counter < 60: - logger.debug(f"🔄 等待stage1线程完成（{timeout_counter}/60秒）") - threading.Event().wait(1) # 等待1秒后重试 - timeout_counter += 1 - - if stage1_thread.is_alive(): - logger.warning("⚠️ stage1线程超时未退出，可能存在异常（但已尝试强制合并剩余文件）") - else: - logger.info("💡 stage1线程已正常退出") - - # 验证stage1文件数量是否匹配（每10个stage0合并1个，不足10个也算1个） - expected_stage1_count = (len(stage0_files) + STAGE1_CHUNK - 1) // STAGE1_CHUNK - if len(stage1_files) != expected_stage1_count: - logger.warning(f"⚠️ ℹ️ stage1文件数量异常！预期 {expected_stage1_count} 个，实际 {len(stage1_files)} 个") - else: - logger.info(f"✅ stage1文件数量验证通过: {len(stage1_files)} 个") - - # 6. 最终合并所有stage1文件到token_info_1.txt - if not stage1_files: - logger.warning("⚠️ 没有生成stage1文件，检查中间处理是否出错") - return - - # 统计stage1文件总数据量 - stage1_total = sum(count_lines(f) for f in stage1_files) - logger.info(f"ℹ️ 开始最终合并: {len(stage1_files)} 个stage1文件，总数据量: {stage1_total} 条") - - # 合并到最终文件 - final_path, final_lines = merge_files_by_token(stage1_files, TOKEN_INFO_FILE) - - if final_path and final_lines > 0: - logger.info(f"✅ 最终结果文件生成完成: {TOKEN_INFO_FILE}，包含 {final_lines} 条数据") - # 验证总数据量 - if final_lines != total_processed: - logger.error(f"❌ 数据量不一致！处理总数据 {total_processed} 条，最终文件 {final_lines} 条") - else: - logger.info("✅💡 数据量验证通过，所有数据已正确写入最终文件") - else: - logger.error("❌ 最终文件合并失败") - - # 最终合并后再次验证 - if os.path.exists(TOKEN_INFO_FILE): - final_count = count_lines(TOKEN_INFO_FILE) - logger.info(f"ℹ️ 最终结果文件包含 {final_count} 条数据") - if final_count != total_processed: - logger.error(f"❌ 最终文件数据不完整！处理 {total_processed} 条，最终文件 {final_count} 条") - else: - logger.info("✅ 最终文件数据完整性验证通过") - - except Exception as e: - logger.error(f"❌ 主流程错误: {str(e)}", exc_info=True) - finally: - # 清理资源 - if processor: - del processor - - # 确保停止信号被触发 - stop_event.set() - - # 等待最终文件写入完成 - threading.Event().wait(2) - - # 清理临时文件（保留最终文件） - all_temp_files = stage0_files + stage1_files - for fpath in all_temp_files: - if fpath != str(TOKEN_INFO_FILE) and os.path.exists(fpath): - try: - os.remove(fpath) - logger.debug(f"已清理临时文件: {os.path.basename(fpath)}") - except Exception as e: - logger.warning(f"清理临时文件失败 {os.path.basename(fpath)}: {str(e)}") - - logger.info("程序执行完毕") - - -if __name__ == "__main__": - main() - diff --git a/tools/data_preprocess/offline_packing/s1_get_tokenlens_v3-sft.py b/tools/data_preprocess/offline_packing/s1_get_tokenlens_v3-sft.py deleted file mode 100644 index 8dfb086a..00000000 --- a/tools/data_preprocess/offline_packing/s1_get_tokenlens_v3-sft.py +++ /dev/null @@ -1,733 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# 使用方式 -python s1_get_tokenlens_v2.py --config ./configs/s1_config_emova.yaml -python s1_get_tokenlens_v2.py --config ./configs/s1_config_emova_3000tk.yaml -python s1_get_tokenlens_v2-sft.py --config ./configs/s1_config_llava_vqa.yaml -python s1_get_tokenlens_v2-sft.py --config ./configs/s1_config_llava_vqa_600k.yaml -python s1_get_tokenlens_v2-sft.py --config ./configs/s1_config_vqa_20sps.yaml # 20条的示例 -python s1_get_tokenlens_v2-sft.py --config ./configs/s1_config_vqa_5500k_s16k.yaml -python s1_get_tokenlens_v2-sft.py --config ./configs/s1_config_vqa_pretrain_5M_8k.yaml - -可配置核心参数： -1. 模型与数据路径： - - CKPT_DIR: 模型权重目录（需提前部署） - 示例："/vlm/pretrain_models/Qwen2.5-VL-7B-Instruct" - - DEFAULT_DIRECTORY: 数据集根目录（需包含同名JSON与图像文件） - 示例："/vlm/xiangan/datasets/aiak_caption_emova_300k" - -2. 计算资源控制： - - STAGE1_CHUNK: 归并基数（每10个stage0文件合并为1个stage1文件） - - chunk_size: 进程级数据块大小（每个进程处理5000个样本） - - 动态线程数：通过get_adaptive_workers自动调节（CPU>80%或内存>85%时减半） - -3. 输出与临时文件： - - OUTPUT_FILE: 配对文件名清单（base_name_v2_emova.txt） - - TOKEN_INFO_FILE: 最终Token长度结果文件（token_info_v2_emova.txt） - - 临时文件：stage0_*/stage1_*（自动清理，存储中间排序结果） - -4. 其它： - - MIN_PIXELS / MAX_PIXELS: 图像预处理的像素范围限制，默认值分别为4*28*28和3578*28*28 - - n_workers: 线程池大小，默认值为96（可通过get_adaptive_workers动态调整） - - CAP_TEMPLATE: 数据格式模版，可根据实际情况变更 - -环境依赖与部署： -1. 必备Python库（建议Python 3.8+）： - pip install psutil Pillow jinja2 transformers qwen_vl_utils orjson - -2. 模型准备： - 需提前下载Qwen2.5-VL模型权重并部署到CKPT_DIR，确保AutoProcessor可正确加载 - -3. 生成文件： - - 临时文件： - - base_name_v1.txt：存储所有配对文件的基础名称（不含扩展名） - - stage0_*：分块处理的临时文件，包含局部排序的(token长度, 文件名)数据 - - stage1_*：合并stage0文件的临时文件，包含中级排序结果 - - 最终文件： - - token_info_*.txt：全局按token长度升序排列的结果文件，格式为"文件名:token长度" - - 日志文件： - - processing.log：详细记录数据处理全过程的日志信息 - -使用工作流： -① 配置调整：根据硬件环境修改CKPT_DIR、DEFAULT_DIRECTORY等路径参数 -② 依赖安装：确保所有Python库已正确安装（特别注意transformers版本兼容性） -③ 数据校验：确认DEFAULT_DIRECTORY中存在同名JSON与图像文件对 -④ 启动执行：python s1_get_tokenlens_v1.py -⑤ 监控分析：通过processing.log追踪流程，重点关注： - - 初始配对文件数（验证数据完整性） - - 各进程/线程的Token计算耗时 - - 归并阶段的文件合并行数校验 -⑥ 结果验证：检查token_info_v2_emova.txt的行数与原始样本数是否匹配 - -""" - -import os -import json -import orjson -import threading -import logging -import psutil -import tempfile -import queue -import yaml -import argparse -from pathlib import Path -from concurrent.futures import ThreadPoolExecutor, as_completed -from heapq import merge -from PIL import Image -from jinja2 import Template -from transformers import AutoProcessor -from transformers import BitsAndBytesConfig -from qwen_vl_utils import fetch_image -from queue import Empty -import multiprocessing -from multiprocessing import Pool, Manager, Value - -# 声明全局的跨进程计数器（在主模块中定义，让子进程继承） -global_total_counter = None - -# ✅ 解析命令行参数 -parser = argparse.ArgumentParser(description="Token Length Processor") -parser.add_argument("--config", type=str, default="config.yaml", help="Path to config.yaml") -parser.add_argument("--log-level", type=str, default=None, - choices=["DEBUG", "INFO", "WARNING", "ERROR"], - help="Override log level from config") -args = parser.parse_args() - -# ✅ 加载配置文件 -CONFIG_PATH = Path(args.config) -if not CONFIG_PATH.exists(): - raise FileNotFoundError(f"配置文件不存在: {CONFIG_PATH}") -with open(CONFIG_PATH, 'r', encoding='utf-8') as f: - cfg = yaml.safe_load(f) - -# ✅ 从配置中读取参数，覆盖原有常量 -MAX_TOKEN_LEN = cfg['sample']['max_len'] -task_type = cfg['sample']['task_type'] - -DEFAULT_DIRECTORY = Path(cfg['data']['directory']) -OUTPUT_FILE = Path(cfg['data']['output_base']) -TOKEN_INFO_FILE = Path(cfg['data']['output_token']) -CKPT_DIR = cfg['model']['checkpoint'] -MIN_PIXELS = cfg['image']['min_pixels'] -MAX_PIXELS = cfg['image']['max_pixels'] -TIME_OUT = cfg['processing']['time_out'] -# 归并参数（仅两级：stage0 → stage1） -STAGE1_CHUNK = cfg['processing']['stage1_merge_chunk'] -chunk_size = cfg['processing']['chunk_size'] -n_workers = cfg['processing']['n_workers'] -MIN_WORKERS = cfg['processing']['min_workers'] -MAX_WORKERS = cfg['processing']['max_workers'] -use_shm = cfg['logging']['use_shm'] -log_level = cfg['logging']['level'] -log_file = cfg['logging']['file'] -if args.log_level: - log_level = args.log_level.upper() - -# 日志配置 - 详细记录数据流向和合并过程 -file_handler = logging.FileHandler( - log_file, - delay=True, - encoding='utf-8' -) -stream_handler = logging.StreamHandler() - -logging.basicConfig( - level=log_level, - format="%(asctime)s - %(levelname)s - %(message)s", - handlers=[file_handler, stream_handler] -) -logger = logging.getLogger(__name__) - -EXTENSIONS = (".json", ".jpg") - - -temp_dir = '/dev/shm' if use_shm else None # None 表示使用系统默认临时目录 - -def count_lines(file_path): - """统计文件有效行数（非空且含分隔符）""" - if not os.path.exists(file_path) or os.path.getsize(file_path) == 0: - return 0 - try: - with open(file_path, 'r', encoding='utf-8') as f: - return sum(1 for line in f if line.strip() and ':' in line.strip()) - except Exception as e: - logger.error(f"❌ 统计文件 {file_path} 行数失败: {str(e)}") - return 0 - - -# def find_paired_files(directory): -# """查找目录中配对的json和jpg文件""" -# directory = Path(directory) -# json_files = set() -# image_files = set() - -# for file in directory.iterdir(): -# if file.is_file(): -# base_name = file.stem -# ext_lower = file.suffix.lower() -# if ext_lower == '.json': -# json_files.add(base_name) -# elif ext_lower in ('.jpg', '.jpeg'): -# image_files.add(base_name) - -# paired_names = json_files.intersection(image_files) -# logger.info(f"ℹ️ 找到 {len(paired_names)} 对匹配文件（总样本数）") -# return paired_names - -# def find_paired_files(directory): -# """查找目录中配对的json和jpg文件（高性能版）""" -# directory = Path(directory) -# json_files = set() -# image_files = set() - -# # 使用 os.scandir 遍历 -# with os.scandir(directory) as it: -# for entry in it: -# if entry.is_file(): -# name, ext = os.path.splitext(entry.name) -# ext_lower = ext.lower() -# if ext_lower == '.json': -# json_files.add(name) -# elif ext_lower in ('.jpg', '.jpeg'): -# image_files.add(name) - -# paired_names = json_files.intersection(image_files) -# logger.info(f"ℹ️ 找到 {len(paired_names)} 对匹配文件（总样本数）") -# return paired_names - -def find_paired_files(directory): - directory = Path(directory) - files = os.listdir(directory) - json_set = {f[:-5] for f in files if f.lower().endswith('.json')} - img_set = {f[:-4] for f in files if f.lower().endswith(('.jpg', '.jpeg'))} - paired = json_set & img_set - logger.info(f"找到 {len(paired)} 对匹配文件") - return paired - - -def write_base_names_to_file(base_names, output_file): - """将配对文件名写入文件""" - try: - content = "\n".join(sorted(base_names)) + "\n" - with open(output_file, 'w', encoding='utf-8') as f: - f.write(content) - logger.info(f"ℹ️ 已将 {len(base_names)} 个配对文件名写入 {output_file}") - except Exception as e: - logger.error(f"❌ 写入 {output_file} 失败: {str(e)}") - raise - - -def read_lines_in_chunks(file_path, chunk_size): - """按块读取文件内容""" - file_path = Path(file_path) - if not file_path.exists(): - raise FileNotFoundError(f"{file_path} 不存在") - - with open(file_path, 'r', encoding='utf-8') as f: - while True: - chunk = [line.strip() for _, line in zip(range(chunk_size), f) if line.strip()] - if not chunk: - break - logger.info(f"ℹ️ 读取数据块，包含 {len(chunk)} 个样本") - yield chunk - - -# 预编译模板 -if task_type=="pretrain": - CAP_TEMPLATE = Template("<|vision_start|><|image_pad|><|vision_end|>{{ captions[0].content }}<|im_end|>") -elif task_type=="sft": - chat_template = """{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{{ message['content'] | replace('', '<|vision_start|><|image_pad|><|vision_end|>') }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}""" - CAP_TEMPLATE = Template(chat_template) - pass - -def process_sample(json_path, img_path, processor): - """处理单个样本，返回(token_len, 文件名)""" - try: - if not Path(json_path).exists(): - raise FileNotFoundError(f"❌ JSON文件不存在: {json_path}") - if not Path(img_path).exists(): - raise FileNotFoundError(f"❌ 图片文件不存在: {img_path}") - - # 读取并渲染JSON内容 - with open(json_path, 'r', encoding='utf-8') as f: - json_data = json.load(f) - # with open(json_path, 'rb') as f: - # json_data = orjson.loads(f.read()) - if task_type=="pretrain": - txt_input = CAP_TEMPLATE.render(captions=json_data['captions']) - elif task_type=="sft": - txt_input = CAP_TEMPLATE.render(json_data) - img_input = fetch_image({ - 'type': 'image', - 'image': img_path, - "min_pixels": MIN_PIXELS, - "max_pixels": MAX_PIXELS, - }) - # print(img_input) - # 计算token长度 - base_name = Path(img_path).stem - inputs = processor( - text=[txt_input], - images=img_input, - videos=None, - padding=True, - return_tensors="pt", - ) - # print(inputs["input_ids"]) - # print(inputs["input_ids"].shape) - return (inputs["input_ids"].shape[1], base_name) - - except Exception as e: - return (None, f"❌ 处理失败 [{Path(img_path).stem}]: {str(e)}") - - -def get_adaptive_workers(min_workers=20, max_workers=96): - """根据系统负载调整线程数""" - try: - cpu_usage = psutil.cpu_percent(interval=0.5) - mem_usage = psutil.virtual_memory().percent - if cpu_usage > 80 or mem_usage > 85: - adjusted = max(min_workers, max_workers // 2) - logger.info(f"系统负载过高，线程数调整为 {adjusted} (CPU: {cpu_usage}%, 内存: {mem_usage}%)") - return adjusted - return max_workers - except Exception as e: - logger.warning(f"获取系统负载失败，使用默认线程数 {max_workers}: {str(e)}") - return max_workers - -gt_maxlen=0 -def merge_files_by_token(input_files, output_file, max_token=MAX_TOKEN_LEN): - """合并多个已排序文件，按token_len升序，过滤掉 > max_token 的数据，返回(输出路径, 数据条数)""" - if not input_files: - logger.warning("⚠️ 没有文件可合并") - return (None, 0) - - # 验证输入文件并统计总数据量 - valid_files = [] - total_lines = 0 - for f in input_files: - line_count = count_lines(f) - if line_count > 0: - valid_files.append(f) - total_lines += line_count - logger.debug(f"ℹ️ 待合并文件 {os.path.basename(f)} 包含 {line_count} 条数据") - else: - logger.warning(f"⚠️ 文件 {os.path.basename(f)} 为空或无效，跳过") - - if not valid_files: - return (None, 0) - - # 定义排序键（按token_len整数排序） - def sort_key(line): - # _, token_str = line.strip().split(':', 1) - token_str = line.strip().split(':')[-1] - return int(token_str) - - try: - with open(output_file, 'w', encoding='utf-8') as out_f: - # 创建所有文件的迭代器 - iterators = [] - file_handles = [] - for fpath in valid_files: - try: - fh = open(fpath, 'r', encoding='utf-8') - file_handles.append(fh) - iterators.append(((sort_key(line), line) for line in fh)) - except Exception as e: - logger.error(f"❌ 打开文件 {os.path.basename(fpath)} 失败: {str(e)}") - - # # 归并排序并写入 - # for _, line in merge(*iterators, key=lambda x: x[0]): - # out_f.write(line) - # 归并排序并写入，过滤掉 > max_token 的行(后续可添加其他条件) - filtered_max_len = 0 - for _, line in merge(*iterators, key=lambda x: x[0]): - _, token_str = line.strip().split(':', 1) - if int(token_str) <= max_token: # ← 只保留 ≤ 8192 - out_f.write(line) - else: - logger.warning(f"⚠️ token长度：{token_str} > {max_token}: 剔除!!") - filtered_max_len+=1 - gt_maxlen - - # 关闭所有文件句柄 - for fh in file_handles: - try: - fh.close() - except Exception as e: - logger.warning(f"⚠️ 关闭文件 {fh.name} 失败: {str(e)}") - - # 验证输出文件数据完整性 - output_lines = count_lines(output_file)+filtered_max_len - if output_lines != total_lines: # 过滤掉不满足条件的 - logger.error(f"❌ 合并数据丢失！输入 {total_lines} 条，输出 {output_lines} 条，已删除错误文件") - if os.path.exists(output_file): - os.remove(output_file) - return (None, 0) - else: - logger.info(f"✅ 📊 合并成功，输入 {total_lines} 条，输出 {output_lines-filtered_max_len} 条（token ≤ {max_token}）的数据") - - return (output_file, output_lines-filtered_max_len) - except Exception as e: - logger.error(f"❌ 合并文件失败: {str(e)}") - if os.path.exists(output_file): - try: - os.remove(output_file) - except Exception as e: - logger.warning(f"⚠️ 删除失败文件 {output_file} 失败: {str(e)}") - return (None, 0) - - -def stage1_merger(input_queue, chunk_size, stage1_files, stop_event): - """ - 修复版stage1合并线程 - - 确保所有stage0文件被合并，包括最后不足10个的文件 - - 解决线程超时和数据丢失问题 - """ - buffer = [] - batch_counter = 0 - logger.info(f"💡 stage1合并线程启动，每 {chunk_size} 个stage0文件合并一次") - - try: - # 循环条件：队列有文件 或 缓冲区有文件 或 未收到停止信号 - while (not input_queue.empty()) or buffer or (not stop_event.is_set()): - # 从队列取文件（带超时防止永久阻塞） - if not input_queue.empty(): - try: - file_path = input_queue.get(timeout=1) # 超时1秒，避免永久阻塞 - buffer.append(file_path) - input_queue.task_done() - logger.debug(f"ℹ️ stage1接收文件 {os.path.basename(file_path)}，当前缓冲区: {len(buffer)}/{chunk_size}") - - # 达到合并数量则执行合并 - if len(buffer) >= chunk_size: - batch_counter += 1 - merged_file = tempfile.NamedTemporaryFile( - mode='w', delete=False, - prefix=f"stage1_batch{batch_counter:03d}_", - encoding='utf-8', - dir=temp_dir - ).name - - # 执行合并 - merged_path, line_count = merge_files_by_token(buffer, merged_file) - if merged_path and line_count > 0: - stage1_files.append(merged_path) - logger.info(f"📊 stage1批次 {batch_counter} 完成: {os.path.basename(merged_path)}，包含 {line_count} 条数据（合并了 {len(buffer)} 个文件）") - else: - logger.warning(f"⚠️ stage1批次 {batch_counter} 合并失败，跳过该批次") - - # 清空缓冲区 - buffer = [] - except Empty: - continue # 队列为空时继续循环 - except Exception as e: - logger.error(f"❌ stage1处理文件时错误: {str(e)}", exc_info=True) - else: - # 队列为空时，检查是否需要强制合并剩余文件 - if buffer and stop_event.is_set(): - # 收到停止信号且缓冲区有文件，强制合并 - batch_counter += 1 - merged_file = tempfile.NamedTemporaryFile( - mode='w', delete=False, - prefix=f"stage1_remaining_batch{batch_counter:03d}_", - encoding='utf-8', - dir=temp_dir - ).name - - merged_path, line_count = merge_files_by_token(buffer, merged_file) - if merged_path and line_count > 0: - stage1_files.append(merged_path) - logger.info(f"📊 stage1剩余文件合并完成: {os.path.basename(merged_path)}，包含 {line_count} 条数据（合并了 {len(buffer)} 个文件）") - else: - logger.warning(f"❌ stage1剩余文件合并失败，数据可能丢失") - buffer = [] - else: - # 短暂休眠，减少CPU占用 - threading.Event().wait(0.5) - - # 最终检查：确保缓冲区为空（防止遗漏） - if buffer: - logger.error(f"❌ stage1线程退出时缓冲区仍有 {len(buffer)} 个文件未处理！数据丢失") - - except Exception as e: - logger.error(f"❌ stage1线程异常退出: {str(e)}", exc_info=True) - finally: - logger.info(f"📊 stage1线程退出，共生成 {len(stage1_files)} 个文件") - -# 新增：每个进程的处理函数（负责处理一个大chunk） -def process_chunk(args): - """ - 单个进程的处理逻辑：处理一个大chunk，内部用多线程并行 - - Args: - args: 包含chunk数据、处理器配置、队列等参数的元组 - """ - # 从全局变量获取计数器，而非参数 - global global_total_counter - - chunk_idx, chunk, ckpt_dir, min_pixels, max_pixels, stage0_queue = args - processor = None - processed_count = 0 # 记录当前进程处理的有效样本数 - - - try: - # 每个进程单独初始化处理器（进程间不能共享processor实例） - # quant_config = BitsAndBytesConfig(load_in_4bit=True) - processor = AutoProcessor.from_pretrained( - ckpt_dir, - min_pixels=min_pixels, - max_pixels=max_pixels, - trust_remote_code=True, - use_fast=False - ) - - # 生成当前chunk的文件路径列表 - full_paths = [] - for fn in chunk: - full_paths.append(str(DEFAULT_DIRECTORY / f"{fn}.json")) - full_paths.append(str(DEFAULT_DIRECTORY / f"{fn}.jpg")) - - n_samples = len(chunk) - logger.info(f"👉 进程 {multiprocessing.current_process().name} 开始处理块 {chunk_idx}，包含 {n_samples} 个样本") - - # 进程内创建线程池（复用线程） - n_workers = get_adaptive_workers(min_workers=MIN_WORKERS, max_workers=MAX_WORKERS) # 单个进程的线程数可适当减少 - chunk_results = [] - with ThreadPoolExecutor( - max_workers=n_workers, - thread_name_prefix=f"proc-{multiprocessing.current_process().pid}-thread" - ) as executor: - tasks = [ - executor.submit( - process_sample, - full_paths[idx*2], - full_paths[idx*2+1], - processor - ) for idx in range(n_samples) - ] - - # 收集线程任务结果 - for future in as_completed(tasks): - try: - token_len, name = future.result() - if token_len is not None: - chunk_results.append((token_len, name)) - processed_count += 1 # 统计有效样本 - else: - logger.warning(name) - except Exception as e: - logger.error(f"❌ 进程内任务错误: {str(e)}") - - # 写入stage0文件并放入跨进程队列 - if chunk_results: - chunk_results_sorted = sorted(chunk_results, key=lambda x: x[0]) - with tempfile.NamedTemporaryFile( - mode='w+', delete=False, - prefix=f"stage0_chunk{chunk_idx:03d}_", - encoding='utf-8', - dir=temp_dir - ) as f: - stage0_file = f.name - for token_len, name in chunk_results_sorted: - f.write(f"{name}:{token_len}\n") - - line_count = count_lines(stage0_file) - stage0_queue.put(stage0_file) # 放入跨进程队列 - # logger.info(f"进程 {multiprocessing.current_process().name} 完成块 {chunk_idx}，生成 {line_count} 条数据") - # logger.info(f"�� 进程 {multiprocessing.current_process().name} 完成块 {chunk_idx}，有效样本 {processed_count}/{n_samples}") - proc_status = "🟢" if processed_count==n_samples else "🟡" - logger.info(f"{proc_status} 进程 {multiprocessing.current_process().name} 完成块 {chunk_idx}，有效样本 {processed_count}/{n_samples}") - - # 【关键】跨进程累加总数据量（使用Value原子操作） - with global_total_counter.get_lock(): - global_total_counter.value += processed_count - - return stage0_file # 返回生成的文件路径，用于后续清理 - - except Exception as e: - logger.error(f"❌ 进程 {multiprocessing.current_process().name} 处理失败: {str(e)}") - finally: - if processor: - del processor - return None - - -### -def main(): - global global_total_counter # 引用全局变量 - processor = None # 模型处理器实例 - stage0_files = [] # 记录所有stage0文件（用于验证和清理） - stage1_files = [] # 记录所有stage1文件（用于最终合并） - - try: - - logger.info(f"💡 --------------开始数据处理流程--------------") - - # 1. 查找配对文件并写入临时文件（json和jpg文件名相同的样本） - base_names = find_paired_files(DEFAULT_DIRECTORY) # DEFAULT_DIRECTORY 是原始数据存放位置（jpg 和 json） - total_original = len(base_names) # 原始样本总数 - logger.info(f"👉 找到 {total_original} 对原始样本文件") - if total_original == 0: - logger.warning("⚠️ 无原始样本，退出程序") - return - # 将配对文件名写入文件，用于后续分块读取 - write_base_names_to_file(base_names, OUTPUT_FILE) - - # 2. 初始化跨进程队列（用于传递stage0文件路径给合并线程） - manager = Manager() # 进程间共享队列需要用Manager - stage0_queue = manager.Queue() - stop_event = manager.Event() # 跨进程停止信号 - - # 跨进程计数器，用于统计总处理样本数（初始值0） - global_total_counter = Value('i', 0) # 'i'表示整数类型 - - # 3 启动stage1合并线程（守护线程） - stage1_thread = threading.Thread( - target=stage1_merger, - args=(stage0_queue, STAGE1_CHUNK, stage1_files, stop_event), - daemon=True - ) - stage1_thread.start() - logger.info("💡 stage1合并线程已启动") - - # 4. 处理数据并生成stage0文件（每块数据单独处理并排序） - # n_workers = 96 #get_adaptive_workers() - - # 4.1 读取所有数据块（准备分给多个进程) - # chunk_size = chunk_size # 每个进程处理的大chunk尺寸（根据内存调整） - all_chunks = list(read_lines_in_chunks(OUTPUT_FILE, chunk_size)) - total_chunks = len(all_chunks) - n_processes = min(multiprocessing.cpu_count(), total_chunks) - logger.info(f"👉 划分为 {total_chunks} 个块，启动 {n_processes} 个进程处理") - - # 4.2 准备进程池参数（包含模型配置、队列等） - process_args = [ - ( - idx + 1, # chunk索引 - chunk, # chunk数据 - CKPT_DIR, # 模型路径 - MIN_PIXELS, - MAX_PIXELS, - stage0_queue, # 跨进程队列 - ) for idx, chunk in enumerate(all_chunks) - ] - - # 4.3 启动进程池（进程数建议设为CPU核心数的1~2倍） - with Pool(processes=n_processes) as process_pool: - # 并行处理所有大chunk - # stage0_files = process_pool.map(process_chunk, process_args) - result = process_pool.map_async(process_chunk, process_args) - try: - stage0_files = result.get(timeout=TIME_OUT) # 超时设置 - except multiprocessing.TimeoutError: - logger.error("❌ 部分进程处理超时，强制终止") - process_pool.terminate() - - # 过滤空结果 - stage0_files = [f for f in stage0_files if f is not None] - logger.info(f"✅ 所有进程处理完成，共生成 {len(stage0_files)} 个stage0文件") - # 统计数据 - total_processed = global_total_counter.value # 直接从全局变量获取 # 获取总处理样本数 - logger.info(f"👉 原始样本数: {total_original}, 有效处理样本数: {total_processed}") - - # 验证数据完整性 - if total_processed != total_original: - logger.warning(f"❌ 数据不完整！原始 {total_original} 个，有效处理 {total_processed} 个，差异 {total_original - total_processed} 个") - else: - logger.info("✅ 数据完整性验证通过，所有样本均被有效处理") - - # 5. 等待处理完成（确保所有文件被合并） - # 等待stage0队列所有文件被处理 - logger.info("🔄 等待stage0队列处理完成...") - stage0_queue.join() # 阻塞直到所有stage0文件被消费 - logger.info("💡 stage0队列所有文件已处理完毕") - - # 发送停止信号给stage1线程，强制处理剩余文件 - logger.info("💡 通知stage1线程停止并处理剩余文件...") - stop_event.set() - - # 延长超时时间至60秒，确保大文件合并完成 - timeout_counter = 0 - while stage1_thread.is_alive() and timeout_counter < 60: - logger.debug(f"🔄 等待stage1线程完成（{timeout_counter}/60秒）") - threading.Event().wait(1) # 等待1秒后重试 - timeout_counter += 1 - - if stage1_thread.is_alive(): - logger.warning("⚠️ stage1线程超时未退出，可能存在异常（但已尝试强制合并剩余文件）") - else: - logger.info("💡 stage1线程已正常退出") - - # 验证stage1文件数量是否匹配（每10个stage0合并1个，不足10个也算1个） - expected_stage1_count = (len(stage0_files) + STAGE1_CHUNK - 1) // STAGE1_CHUNK - if len(stage1_files) != expected_stage1_count: - logger.warning(f"⚠️ ℹ️ stage1文件数量异常！预期 {expected_stage1_count} 个，实际 {len(stage1_files)} 个") - else: - logger.info(f"✅ stage1文件数量验证通过: {len(stage1_files)} 个") - - # 6. 最终合并所有stage1文件到token_info_1.txt - if not stage1_files: - logger.warning("⚠️ 没有生成stage1文件，检查中间处理是否出错") - return - - # 统计stage1文件总数据量 - stage1_total = sum(count_lines(f) for f in stage1_files) - logger.info(f"ℹ️ 开始最终合并: {len(stage1_files)} 个stage1文件，总数据量: {stage1_total} 条") - - # 合并到最终文件 - final_path, final_lines = merge_files_by_token(stage1_files, TOKEN_INFO_FILE) - - if final_path and final_lines > 0: - logger.info(f"✅ 最终结果文件生成完成: {TOKEN_INFO_FILE}，包含 {final_lines} 条数据") - # 验证总数据量 - if final_lines != total_processed: - logger.error(f"❌ 数据量不一致！处理总数据 {total_processed} 条，最终文件 {final_lines} 条") - else: - logger.info("✅💡 数据量验证通过，所有数据已正确写入最终文件") - else: - logger.error("❌ 最终文件合并失败") - - # 最终合并后再次验证 - if os.path.exists(TOKEN_INFO_FILE): - final_count = count_lines(TOKEN_INFO_FILE) - logger.info(f"ℹ️ 最终结果文件包含 {final_count} 条数据") - if final_count != total_processed: - logger.error(f"❌ 最终文件数据不完整！处理 {total_processed} 条，最终文件 {final_count} 条") - else: - logger.info("✅ 最终文件数据完整性验证通过") - - except Exception as e: - logger.error(f"❌ 主流程错误: {str(e)}", exc_info=True) - finally: - # 清理资源 - if processor: - del processor - - # 确保停止信号被触发 - stop_event.set() - - if stage1_thread and stage1_thread.is_alive(): - stage1_thread.join(timeout=2) - - # 等待最终文件写入完成 - threading.Event().wait(2) - - # 清理临时文件（保留最终文件） - all_temp_files = stage0_files + stage1_files - for fpath in all_temp_files: - if fpath != str(TOKEN_INFO_FILE) and os.path.exists(fpath): - try: - os.remove(fpath) - logger.debug(f"已清理临时文件: {os.path.basename(fpath)}") - except Exception as e: - logger.warning(f"清理临时文件失败 {os.path.basename(fpath)}: {str(e)}") - - logger.info("程序执行完毕") - - -if __name__ == "__main__": - main() - diff --git a/tools/data_preprocess/offline_packing/s2_data_depict.ipynb b/tools/data_preprocess/offline_packing/s2_data_depict.ipynb deleted file mode 100644 index 98ed52d9..00000000 --- a/tools/data_preprocess/offline_packing/s2_data_depict.ipynb +++ /dev/null @@ -1,846 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6ccdd8b0-af82-44e1-becd-6e16fc0c79ac", - "metadata": {}, - "source": [ - "## Histgram" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ee294a6b-b771-4f2a-b024-a2f10930acb8", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, - "outputs": [], - "source": [ - "# 读取文件并提取数值\n", - "values = []\n", - "with open(\"/workspace/RiceVL/data_procs/token_info_v2_emova_3000tk.txt\", \"r\", encoding=\"utf-8\") as f:\n", - " for line in f:\n", - " # 分割每行的标识符和数值，取冒号后的数值部分并转换为整数\n", - " num = int(line.split(\":\")[-1].strip())\n", - " values.append(num)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f53e106-34e7-4a5f-8acb-bd5b783af811", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "from collections import Counter\n", - "\n", - "# 统计频次\n", - "count_result = Counter(values)\n", - "# 对x轴的数字进行排序，使折线图更连贯\n", - "unique_nums = sorted(count_result.keys())\n", - "frequencies = [count_result[num] for num in unique_nums]\n", - "\n", - "# 创建画布\n", - "plt.figure(figsize=(10, 6))\n", - "\n", - "# 绘制折线图，添加标记点使数据更清晰\n", - "# plt.plot(unique_nums, frequencies, marker='o', linestyle='-', color='b', markersize=8, linewidth=2)\n", - "plt.plot(unique_nums, frequencies, \n", - " marker='.', # 圆形标记点\n", - " linestyle='-.', # 不显示连接线\n", - " color='b', # 点的颜色\n", - " markersize=3, # 点的大小\n", - " markerfacecolor='red', # 点的填充色\n", - " markeredgecolor='red') # 点的边缘色\n", - "\n", - "# 添加标题和坐标轴标签\n", - "plt.title('Frequency of Each Number (Line Plot)', fontsize=15)\n", - "plt.xlabel('Number', fontsize=12)\n", - "plt.ylabel('Frequency', fontsize=12)\n", - "\n", - "# # 为每个点添加数值标签\n", - "# for x, y in zip(unique_nums, frequencies):\n", - "# plt.text(x, y + 0.1, str(y), ha='center', fontsize=10)\n", - "\n", - "# 添加网格线，便于查看数值\n", - "plt.grid(alpha=0.3)\n", - "\n", - "# 调整布局\n", - "plt.tight_layout()\n", - "\n", - "# 显示图像\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8bf9e51-adaa-4fef-a81d-b3244932aa9a", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "# 创建画布\n", - "plt.figure(figsize=(10, 6))\n", - "\n", - "# 绘制直方图\n", - "# bins：直方图的分箱数量（可根据数据范围调整）\n", - "# edgecolor：设置柱形边缘颜色，区分不同分箱\n", - "n, bins, patches = plt.hist(values, bins=5, edgecolor='black', color='lightgreen')\n", - "\n", - "# 添加标题和坐标轴标签\n", - "plt.title('Data Distribution Histogram', fontsize=15)\n", - "plt.xlabel('Value Range', fontsize=12)\n", - "plt.ylabel('Frequency', fontsize=12)\n", - "\n", - "# 在每个柱形上方标注频数\n", - "# for i in range(len(n)):\n", - "# plt.text(bins[i] + (bins[i+1] - bins[i])/2, n[i] + 0.1, \n", - "# f'{int(n[i])}', ha='center', fontsize=10)\n", - "\n", - "# 添加网格线，便于查看\n", - "plt.grid(axis='y', alpha=0.3)\n", - "\n", - "# 调整布局\n", - "plt.tight_layout()\n", - "\n", - "# 显示图像\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4243c1ef-bc5c-445b-9f17-500e3404c95c", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import matplotlib.pyplot as plt\n", - "plt.rcdefaults()\n", - "plt.rcParams[\"axes.unicode_minus\"] = False # 解决负号显示问题\n", - "\n", - "# 绘制直方图\n", - "plt.figure(figsize=(12, 6))\n", - "# bins参数控制区间数量，可根据数据范围调整（如数值范围60-400，建议设30-50）\n", - "n, bins, patches = plt.hist(values, bins=40, edgecolor=\"black\", alpha=0.7)\n", - "\n", - "# 添加标题和坐标轴标签\n", - "plt.title('Distribution of Numeric Values') \n", - "plt.xlabel('Value') \n", - "plt.ylabel('Frequency')\n", - "\n", - "# 显示网格线\n", - "plt.grid(axis=\"y\", linestyle=\"--\", alpha=0.5)\n", - "\n", - "# 显示图像\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bd17cb8a-3637-436e-9c58-3c48546cb2e6", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "# ---------------------- 1. 读取数据 ----------------------\n", - "values = []\n", - "with open(\"/workspace/RiceVL/data_procs/token_info_v2_emova_3000tk.txt\", 'r', encoding='utf-8') as f:\n", - " for line in f:\n", - " try:\n", - " num = int(line.split(':')[-1].strip())\n", - " values.append(num)\n", - " except:\n", - " continue # 避免异常数据影响，如有需可细化处理\n", - "\n", - "if not values:\n", - " raise ValueError(\"No valid numeric values found in tl.txt\")\n", - "\n", - "# ---------------------- 2. 计算关键点 ----------------------\n", - "min_val = min(values)\n", - "max_val = max(values)\n", - "# 等距划分三个中间点\n", - "a = min_val + (max_val - min_val) / 4\n", - "b = min_val + 2 * (max_val - min_val) / 4\n", - "c = min_val + 3 * (max_val - min_val) / 4\n", - "\n", - "# 区间划分（转为元组方便后续处理）\n", - "bins = [min_val, a, b, c, max_val]\n", - "labels = [\n", - " f\"[{min_val:.2f}, {a:.2f})\",\n", - " f\"[{a:.2f}, {b:.2f})\",\n", - " f\"[{b:.2f}, {c:.2f})\",\n", - " f\"[{c:.2f}, {max_val:.2f}]\"\n", - "]\n", - "\n", - "# 按区间分组\n", - "grouped_data = {label: [] for label in labels}\n", - "for num in values:\n", - " for i in range(len(bins) - 1):\n", - " if bins[i] <= num < bins[i + 1]:\n", - " grouped_data[labels[i]].append(num)\n", - " break\n", - " # 处理刚好等于 max_val 的情况（放入最后一组）\n", - " else:\n", - " if num == max_val:\n", - " grouped_data[labels[-1]].append(num)\n", - "\n", - "# ---------------------- 3. 绘制每个区间的直方图 ----------------------\n", - "plt.rcdefaults() # 重置为默认配置，避免字体等问题\n", - "fig, axes = plt.subplots(nrows=len(labels), ncols=1, figsize=(8, 5 * len(labels)))\n", - "\n", - "for i, (label, data) in enumerate(grouped_data.items()):\n", - " ax = axes[i] if len(labels) > 1 else axes\n", - " # 绘制直方图\n", - " ax.hist(data, bins='auto', edgecolor='black', alpha=0.7)\n", - " # 设置标题和标签\n", - " ax.set_title(f\"Histogram for {label} (Count: {len(data)})\")\n", - " ax.set_xlabel(\"Value\")\n", - " ax.set_ylabel(\"Frequency\")\n", - " # 添加数据量文本\n", - " ax.text(0.95, 0.95, f\"Count: {len(data)}\", \n", - " horizontalalignment='right', verticalalignment='top', \n", - " transform=ax.transAxes, fontsize=10, bbox=dict(facecolor='white', alpha=0.8))\n", - "\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "# 额外打印各区间数据量汇总\n", - "print(\"Interval Data Count Summary:\")\n", - "for label, data in grouped_data.items():\n", - " print(f\"{label}: {len(data)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "372783cd-ad8d-48d6-9a41-6b51b6fa9055", - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "\n", - "def plot_interval_histograms(file_path, split_points):\n", - " \"\"\"\n", - " 读取文件中的数值，按 [min, a), [a, b), [b, c), [c, max] 划分区间，绘制每个区间直方图并统计数据量\n", - " :param file_path: 包含数值的文件路径，每行格式建议为 \"xxx: 数值\"\n", - " :param split_points: 分割关键点列表，如 [a, b, c]，需满足 a < b < c\n", - " \"\"\"\n", - " # ---------------------- 1. 读取并提取数值 ----------------------\n", - " values = []\n", - " with open(file_path, 'r', encoding='utf-8') as f:\n", - " for line in f:\n", - " try:\n", - " # 假设每行是类似 \"xxx: 数值\" 格式，提取最后面的数值部分并转成整数\n", - " num = int(line.strip().split(':')[-1].strip())\n", - " values.append(num)\n", - " except (IndexError, ValueError):\n", - " # 遇到格式异常的行，跳过处理\n", - " continue\n", - " if not values:\n", - " raise ValueError(\"文件中未提取到有效数值，请检查文件内容格式\")\n", - "\n", - " # ---------------------- 2. 确定区间边界 ----------------------\n", - " min_val = min(values)\n", - " max_val = max(values)\n", - " # 结合传入的分割点，构建完整区间边界\n", - " bins = [min_val] + split_points + [max_val]\n", - " # 构建区间标签，方便识别\n", - " interval_labels = [\n", - " f\"[{min_val}, {split_points[0]})\",\n", - " f\"[{split_points[0]}, {split_points[1]})\",\n", - " f\"[{split_points[1]}, {split_points[2]})\",\n", - " f\"[{split_points[2]}, {max_val}]\"\n", - " ]\n", - "\n", - " # ---------------------- 3. 按区间分组数据 ----------------------\n", - " grouped_data = {label: [] for label in interval_labels}\n", - " for num in values:\n", - " for i in range(len(bins) - 1):\n", - " lower_bound = bins[i]\n", - " upper_bound = bins[i + 1]\n", - " # 处理左闭右开区间，最后一个区间是左闭右闭\n", - " if i < len(bins) - 2 and lower_bound <= num < upper_bound:\n", - " grouped_data[interval_labels[i]].append(num)\n", - " break\n", - " elif i == len(bins) - 2 and lower_bound <= num <= upper_bound:\n", - " grouped_data[interval_labels[i]].append(num)\n", - " break\n", - "\n", - " # ---------------------- 4. 绘制每个区间的直方图 ----------------------\n", - " plt.rcdefaults() # 重置 matplotlib 配置，避免字体等潜在问题\n", - " fig, axes = plt.subplots(nrows=len(interval_labels), ncols=1, figsize=(8, 5 * len(interval_labels)))\n", - " # 处理单轴和多轴情况，让代码更鲁棒\n", - " axes = np.array(axes).reshape(-1)\n", - "\n", - " for i, (label, data) in enumerate(grouped_data.items()):\n", - " ax = axes[i]\n", - " # 绘制直方图，用 auto 自动适配 bins\n", - " ax.hist(data, bins='auto', edgecolor='black', alpha=0.7)\n", - " # 设置标题，带上该区间的数据量\n", - " ax.set_title(f\"Histogram of {label} (Data Count: {len(data)})\")\n", - " ax.set_xlabel(\"Value\")\n", - " ax.set_ylabel(\"Frequency\")\n", - " # 在图中添加数据量文本标注，放在右上角，带半透明背景\n", - " ax.text(\n", - " 0.95, 0.95, \n", - " f\"Count: {len(data)}\", \n", - " horizontalalignment='right', \n", - " verticalalignment='top', \n", - " transform=ax.transAxes, \n", - " fontsize=10, \n", - " bbox=dict(facecolor='white', alpha=0.8)\n", - " )\n", - "\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n", - " # ---------------------- 5. 打印区间数据量汇总 ----------------------\n", - " print(\"各区间数据量统计：\")\n", - " for label, data_list in grouped_data.items():\n", - " print(f\"{label}: {len(data_list)} 条\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f3444d4f-00b8-4647-9f04-d945074a9127", - "metadata": {}, - "outputs": [], - "source": [ - "file_name = \"/workspace/RiceVL/data_procs/token_info_v2_emova_3000tk.txt\"\n", - "split_points = [1024, 2048, 4096] # 你可以根据实际需求调整这三个分割点\n", - "try:\n", - " plot_interval_histograms(file_name, split_points)\n", - "except Exception as e:\n", - " print(f\"执行过程中出现错误：{e}\")" - ] - }, - { - "cell_type": "markdown", - "id": "14c61ceb-ead9-4620-bd6b-5f9f005ca203", - "metadata": {}, - "source": [ - "## VQA 5500k数据集" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "0fdfe51f-7114-4b3a-9464-f023d2a8d5ba", - "metadata": {}, - "outputs": [], - "source": [ - "# 读取文件并提取数值\n", - "values = []\n", - "with open(\"/workspace/data4packing/RiceVL/data_procs/token_info_v2_vqa_pretrain_5M_8k.txt\", \"r\", encoding=\"utf-8\") as f:\n", - " for line in f:\n", - " # 分割每行的标识符和数值，取冒号后的数值部分并转换为整数\n", - " num = int(line.split(\":\")[-1].strip())\n", - " values.append(num)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "39c9be2c-13b9-44ab-87ff-390d27543781", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA90AAAJOCAYAAACqS2TfAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAvE9JREFUeJzs3XtcVHX+x/HXAMNlvCsCongBMbXRMAkvmd1MS7Ns3XYrW7XsYmnbvba2XG3b7X6/brV22bSt3S1/2cUys6y8opGNZiFoXlHxAuoIDMz5/TEyOAKKMJcz8H4+HjyYc853zvcz8x3Kz3xvFsMwDERERERERETE7yJCHYCIiIiIiIhIY6WkW0RERERERCRAlHSLiIiIiIiIBIiSbhEREREREZEAUdItIiIiIiIiEiBKukVEREREREQCREm3iIiIiIiISIAo6RYREREREREJECXdIiIiIiIiIgGipFtExA8sFssxf84666xQhygB9M4779C/f39sNhsWi4WuXbse9zldu3Y13edm4sSJWCwWvvrqK7/c76yzzvK+ltruuXTpUlP/jdS1Pc1g3759tGvXjksvvdTn/BtvvIHFYmHixIl1uk9lu23cuNH/QTbAV199Ve1vJCoqiqSkJC6++GIWLlxY7TnBei3bt28nLi6OG2+8MaD1iEh4igp1ACIijcmECRNqPN+zZ88gRyLBsmLFCq688kpiY2MZPnw4rVu3Jj4+vs7PHzt2LM2bN6/xWmP63PzlL3/h66+/DnUYjdrf/vY39u7dy1/+8pdQhxJQiYmJnH/++QCUlJSQk5PDhx9+yNy5c3nhhRe44YYb/F7nV199xdlnn82ECRN44403ql3v0KED1113HS+++CK33HILPXr08HsMIhK+lHSLiPhRTf8Yk8Zt7ty5uN1unnvuOa6++uoTfv7jjz8eNj2p9RUXF8eiRYtYsGAB5557bqjDaZS2b9/Oc889x+jRo7Hb7Q2611tvvYXT6aRjx45+is6/evbs6fPfWsMweOCBB5g+fTq33347Y8eOJSEhIehx3XXXXTz//PPcf//9vPvuu0GvX0TMS8PLRUREGmDLli0ApKamhjgS86rseWzsPbChNHPmTEpLSxk/fnyD79W5c2d69uyJ1Wr1Q2SBZ7FYuP/++0lLS+PQoUN8/vnnIYmjY8eOnH322XzwwQfs2LEjJDGIiDkp6RYRCaIj58x+9tlnnH322bRu3RqLxcK+ffu85ebNm8eoUaNo3749MTExpKamctttt7F79+4a77tnzx6mTp1KcnIysbGx9O7dm2eeeQbDMGqckzp9+nQsFkutPfOV841r8tNPPzFx4kRSUlKIiYkhMTGRyy67jDVr1lQrWzmXdPr06WzatIkrrriC9u3bExcXR2ZmJnPnzq31vfrpp5+YNGkSXbt2JSYmhoSEBE4//XQef/xxysvLAbDb7VgsFn7++eca77F582YiIyPp1q0bhmHUWteRdu/ezZ133kl6ejqxsbG0bduW888/v9o/5Ctf2+uvvw7A2Wef7Z1nGqgRDzk5Odx1113079/f57Nx4403sm3btlqft3nzZv74xz/So0cP4uLiaNu2LZmZmcyYMYPi4uIan7No0SLOOeccWrRoQcuWLRk1ahRr166tV9xjxowhIyOD7777js8++6xOz6nPZ7Ryzu/EiRPZuXMnkyZNIikpiWbNmjFkyBAWL17sLfvyyy/Tt29f4uLiSElJYfr06bjd7lrjKSsr4y9/+QtpaWnExsaSmprKtGnTKCkpqbF8eXk5L730EoMGDaJly5bExcWRkZHB008/7f381vR6DMPgueee45RTTsFms5GRkXHc98owDP75z3/SokULRo0addzyx1PbPOjK/5ZUVFTwyCOP0KNHD2JiYkhJSeHuu++mtLS0xvs5nU4eeugh+vXrR/PmzWnevDkDBw7kzTffbHCslSIiIjjllFMAz+e9LtauXcu4cePo0KED0dHRdOzYkfHjx1f778nEiRM5++yzAXjzzTd95pRPnz7dp+wVV1yBy+XSqCcR8aGkW0QkBGbPns0FF1zAwYMHueCCCzjttNO8CcSf/vQnLrjgAr744gtOOukkLrroIqKionjqqacYMGBAtR6UvXv3MmTIEF544QUMw+Diiy+mY8eO3HHHHfzxj3/0a9xz5syhX79+vPnmm8THx3PRRRfRrVs33nvvPbKysli0aFGNz9u4cSOnnXYay5cv59xzz6Vfv36sXLmSMWPG1Ngr9Z///Id+/foxc+ZMbDYbl1xyCf3792fz5s3ceeedHDhwAIDrr78egNdee63GemfOnInb7eaaa66p9UuEI23dupWsrCwef/xxysrKGDNmDP369eOLL75gxIgRPPXUU96y3bt3Z8KECaSlpQEwYsQIJkyYwIQJE+jevftx66qPhx9+2BvDkCFDGDlyJIZh8NJLL5GZmVlj4v3NN9/Qt29fnnvuOVwuF6NHj+b000+nqKiI6dOnk5+fX+05c+fO5ZxzzsHpdDJy5Eg6dOjAJ598wtChQykoKDjhuC0WCzNmzAColqQEwt69exk0aBALFizgrLPOok+fPnz33Xecd955rFmzhptvvpnbbruNlJQUhg0bRlFRETNmzOD++++v8X6GYTB27Fgee+wxevfuzahRo9izZw9//etfufDCC6moqPApf+jQIYYPH86NN97IL7/8wsCBAznvvPPYvn07t956K2PHjq01wZ88eTK33347CQkJXHTRRXUaQbF27Vo2bNjAwIEDiY2NPfE37ARdccUVPPjgg5x00kkMHz6c/fv38+ijjzJp0qRqZXfu3MmgQYO49957KSgo4Mwzz2To0KGsW7eOiRMnctNNN/ktrv379wMQExNz3LILFiwgMzOT2bNn06FDB++Q9H/9619kZmbyzTffeMsOGTKEESNGAJCWlub9O58wYUK1L0UqFwT8+OOP/fOiRKRxMEREpMEAoy7/SZ0wYYK37L///e9q19977z0DMOx2u5Gbm+s973a7jWnTphmA8fvf/97nOZMnTzYA4/zzzzcOHjzoPb9s2TKjefPmBmB06dLF5zl/+ctfDMB4/fXXa4yzS5cu1V7Phg0bjGbNmhnNmzc35s+f73Pt008/NaxWq5GSkmKUlpZ6z7/++uve13v77bcbFRUV3mtPPfWUARhnnHGGz71++eUXIzY21oiKijJmzZrlc83tdhufffaZUVJSYhiGYezbt8+w2WxG+/btfeo1DMOoqKgwOnfubERGRhpbt26t8XUe7cILLzQA44orrvC53zfffGPYbDYjMjLS+P77732eU9mmCxcurFMdlSrf4w0bNtT5OV9++aVRUFDgc66iosKYMWOGARhXXXWVz7Xdu3cb7du3NwDjscce83n/DcMwFi9ebOzYsaPaa4mIiDA++OAD7/ny8nJj7NixBmDcf//9dY73zDPPNADjm2++MQzDMPr3728Axscff+wts2TJEgMwzjzzTJ/n1uczunDhQu/n7corrzTKysqq3a93795GcnKysX79eu+1NWvWGNHR0YbNZjP279/vc8/K+3Xq1MnIy8vznt+5c6dht9sNwHjqqad8nnPjjTd6/1b37dvnPV9cXGyMHDnSAIyXXnqpxtcTHx9vOByOGl9zbV566SUDMO69994ar1f+HU6YMKFO96tst6M/m5XvRa9evYzt27d7z+fn5xutW7c2AJ/31TAM7+u9+eabvX+3hmEYBQUFRmZmpgEYn376aZ3iqmzfoz8rhmEYO3bsMFq2bGkAPv99qum1HDhwwEhMTDQA4/nnn/e5z5NPPult70OHDlWruy7vYXx8vBETE+PzfBFp2pR0i4j4QeU/Rmv7qfwHX2VSM2rUqBrvc8oppxiA8eOPP1a75na7jYyMDCMyMtLYtWuXYRiefzzGxcUZERER1f6xaxiGcffdd/st6b755psNwHjuuedqfM4f//hHAzDef/9977nKf+x369atWlLscrmMNm3aGFar1efaDTfcYADG5MmTa6znaFdffbUBGO+9957P+U8//dQAjIsuuqhO98nLyzMAo3nz5sbu3burXb/tttsMwLjmmmt8zjc06T7Wz9HJ3LF07NjRaNeunc+5Rx55xPuFTF1UvpZx48ZVu5adnV1rwlObo5PuuXPnGoCRmZnpLROIpLtly5bGnj17fK7t27fPsFgsBmC89tpr1e53ySWX1NiOlW3xyiuvVHtO5WcsLS3Ne27Hjh3eL6CcTme152zfvt2Ijo42+vbtW+Preeyxx2p8vcdS+Tfz5ptv1njd30n30V+6GYZhTJ06tVp7ff/99wZgnHbaadW+8DEMw1i1atUJ/Y3WlHQfOnTIWLp0qTFgwAADME466SSjvLz8mK9l5syZBmAMGjSoxnoqvxx6++23q9Vdl/fw9NNPNwBj1apVdXpdItL4afVyERE/qm3LsKO3hLrooouqldm5cyc//PAD6enpNa4+bLFYOP3008nJyWHlypWMGDGClStXcujQIbKysrzDnI90+eWX88gjj9Tz1fiqHAb+m9/8psbrZ5xxBs8++yzLly/nkksu8bl21llnER0d7XMuKiqKbt26sWrVKnbv3k2HDh0A+OKLL4CqoePHM3nyZGbOnMmrr77qsz/xq6++CsB1111Xp/t8++23AJx//vm0bdu22vU//OEPPPnkkz7DTv3hWFuG9e7du9q53bt38+GHH+JwONi3b593aLPL5WL37t3s2bPHG/+JvpeVhg8fXu1c5RZI27dvP6F7HenCCy/ktNNOY8WKFcydO5fRo0fX+17HkpmZSZs2bXzOtWrVirZt27J79+4aX1/lMO7aXt9ll11W7dz5559PmzZtyMvLY/v27XTo0IGvvvoKl8vF+eefT1xcXLXnJCUlkZ6ezo8//sihQ4eqlanpvw3Hs3PnToBqrzkQrFard37zkWr6fFT+N2PMmDFERFSf0Vg5x3v58uUnFMPXX39d43SR7t27M2fOHCIjI4/5/Mq/4XHjxtV4/corr2TlypV88803tZY5lsq/v127dp3wc0WkcVLSLSLiR3VdPKdz587VzlUuWpSbm3vc+ceFhYUA3jm8Xbp0qbGcP7eiqozveNsIVcZ2pE6dOtVYtkWLFgA+CzBVLoJU05cINTnttNM49dRT+eKLL9iwYQPdunVjx44dzJ07l06dOnn38z2eyveytves8vzWrVvrdL+6OpEtw9555x2uu+4675z2muzfv9/7j/4TfS8r1dReNbVVfcyYMYORI0cyffr0gCXdtX1Gmzdvzu7du2u8XvnFR02vr02bNt7Xf7QuXbqwd+9etm3bRocOHbx/J6+++qr3i5/a7Nmzp1osNf234XiKiooAao3Rn5KSkmpMamv6fFS+F3/+85/585//XOs9a1uMrjZH7tMdFRVFu3btGDhwIBdeeGGdVlwP9N96y5YtAXwWxxSRpk1Jt4hICNS02FHlwkpJSUneRXtqU1uS7S81LfJUea623vxKAwYMqHaupl4uf5o8eTLXXXcd//znP3nwwQd58803cblcXH311cft9aqruizEFki//vorEydOBODpp59m1KhRdOzY0dtTOnjwYJYsWVLnVdqPJZDtdcEFFzBw4ECWLl3KBx984B3hcKKOtdL48eIP5OurjCsjI8O7mnZtalrwqz4LobVq1QqoWkgskE7kvat8L4YMGXLCX/wcy9H7dPtbQ//WK78Ead26tR+iEZHGQEm3iIhJVPYuxsfH1/kflJUJy6+//lrj9drOVw71rqnHtKKiosYVqjt16kReXh5PPPEE7dq1q1N89ZGSkkJubi55eXl12i4JPKsp33HHHbz++utMnz6d1157jYiIiBpXU65NcnIyUPt7Vtee/kD55JNPKCsr44477uDmm2+udr2mVchTUlJYt24deXl59OnTJxhh1smMGTMYMWIEM2bM4KWXXqqxTH0+o4Gyd+9e9u/fX2NP8qZNm4Cqz0/l3/GQIUN47rnnghJfQkIC4Ok5N5PK92LMmDHcfvvtIY6mSqD/1vfu3QtA+/bt6/V8EWl8tGWYiIhJdOrUiZ49e7J27Vp++eWXOj2nf//+xMXFsXLlyhqTrn//+981Pq8yWa+pnoULF+JyuaqdP++88wD44IMP6hRbfQ0bNgyAV155pc7PadasGVdeeSXbtm3jrrvuIjc3lxEjRpzQUN0hQ4YAnj3SaxoW+vbbbwOeueuhUPkP+ZqGfi9atKjaVnJQv/cyGIYPH87gwYP54Ycf+N///ldjmfp8RgPpvffeq3bu888/Z8+ePaSmpnrjPfvss4mMjOSjjz4KWoyVPeq17VcfKsH6b8aJqvwbfuedd2q8XtPfeuWXQDXtsX60devWERMTQ69evRoaqog0Ekq6RURM5P7778ftdjN27FhycnKqXd+9e7fPPNHmzZvzhz/8gYqKCm666SYOHTrkvZadnc3zzz9fYz1Dhw4FPP+4rOzVAdiwYUOte3vffvvtxMXFcccdd/D+++9Xu15aWsp///tftmzZUpeXWqtbbrmF2NhYXn31Vd59912fa4ZhMH/+/Brn3U6ePBnAu4/1tddee0L1pqamMmrUKPbv38/NN9/skzAtWbKEl156icjISKZMmXKiL8kvKheqevvttzl48KD3/NatW72v/WjXXHMN8fHxfPrppzz99NPVhp4vXbrUuwhXsFXu2/3iiy/WeL0+n9FAmjFjhk8chYWF3HnnnQA+n4mOHTty9dVXs3HjRi6//PIavwxZv359rV821EdlcrhixQq/3dMfBgwYwHnnncd3333HlClTKC4urlbmhx9+YN68eUGN63e/+x2JiYl8++231b6QevbZZ8nOzqZjx46MHTvWe76yd/x4X2zk5eWxe/dusrKygrJnuoiEBw0vFxExkSuuuII1a9bw97//nf79+5ORkUFaWhqGYZCXl8fq1atp3ry5T0L50EMP8fXXX/PJJ5+QlpbG0KFD2bt3L19++SXXX389L7zwQrV60tLSGD9+PG+99RYZGRkMHToUp9PJ0qVLGTlyJE6ns9rQy+7du/POO+9wxRVXMHbsWLp3706vXr1o1qwZW7duZdWqVRw8eJDvv/++1oXT6qJHjx68/vrrjB8/nssuu4wHHniAvn37UlRUhMPhYPPmzezdu7fafNg+ffowePBgFi9eTFJSUr0W6frHP/7BGWecwVtvvcXXX3/NoEGD2LVrF1999RUVFRU88cQTdR7yXld33HFHrauX22w2b1J60UUXcfLJJ5OdnU337t05/fTTKSkpYeHChWRkZHhf+5Hatm3Lf/7zHy666CJuvfVWnn32WU477TQOHTrETz/9xPr16/n++++9w5ODadiwYZxxxhm1rgZfn89ooHTu3Jm+ffty8sknc+6552K1Wvnyyy/Zt28fZ599drUvAZ555hk2btzI//73P+bNm0dGRgadO3fm4MGDrF27lvXr13PxxRf7JHUN0atXL7p168ayZcsoKSmpNdn7+OOPGThwYK33acgc+9q8/fbbnH/++bz44ovMnj2bjIwMkpOTKSoqYvXq1WzevJmbb765zgse+kOzZs2YNWsWo0eP5vrrr+eVV16hR48erFu3ju+//57mzZvzzjvv+LyPXbt2pW/fvmRnZ5OVlcXJJ59MZGQkF110kc+K81999RUAo0aNCtrrEZEwENody0REGgcO7197PHXd0/nrr782Lr30UiM5OdmwWq1Gu3btjL59+xpTp041vv7662rlCwsLjRtuuMFISkoyYmJijJ49exqPP/644Xa7a9yn2zAMo7S01PjTn/5kpKSkGNHR0UZaWprx4IMPGuXl5TXugVxp/fr1xo033mikp6cbsbGxRosWLYyTTjrJuOyyy4z33nvPZ8/tyv2B//KXv9R4r9r2AzYMw/jhhx+MK6+80ujYsaNhtVqNhIQE4/TTTzeeeOIJw+Vy1Xi/++67zwCMe+65p8brdVFYWGjcfvvtRlpamhEdHW20bt3aGD58uPHZZ5/VWD6Q+3S3atXK5zl79uwxbrjhBqNr165GTEyMkZqaatx9993GwYMHj/le5ufnG5MnTza6du1qREdHG23btjX69+9vPPDAA0ZxcXGdX0ttn6XaHL1P99G+/PJL72utaf/vE/2MHm8v5WN9rmvbF7zyNZeUlBj33nuv9z3s0qWL8ec//7nGvbgNwzDKy8uNN9980zjnnHOMtm3bGlar1UhOTjYGDRpkzJgxw/j555/rHFtd/O1vf6txv3rDqPo7PN5P5WfnWPt019b+x/pbP3TokPHss88agwcPNlq1amVER0cbKSkpxplnnmk89thjxubNm+v0Gmvap/t4jvV34XA4jMsvv9xITEw0rFar0aFDB+PKK6801q1bV+O9cnNzjTFjxhjt2rUzIiIiany955xzjmG1Wo2CgoI6xygijZ/FMPywzKmIiJiWxWKhS5cuPkNjGyPDMOjVqxe//PIL69ev9+67LNIUFBQU0K1bN4YNG8bcuXNDHU6TtGXLFrp06cJvf/vbalNjRKRp05xuERFpFP773//y888/M3LkSCXc0uQkJSVx00038fHHH/Pjjz+GOpwm6bHHHiMiIoIHHngg1KGIiMmop1tEpJFr7D3d11xzDfv27eOjjz6ivLycZcuW0b9//1CHJRJ0+/btIy0tjbPPPpv//ve/oQ6nSdm+fTupqalcddVVtS4OKCJNl5JuEZFGrrEn3RaLhaioKNLT03nggQf47W9/G+qQRERERLyUdIuIiIiIiIgEiOZ0i4iIiIiIiASIkm4RERERERGRAIkKdQCNldvtZtu2bbRo0QKLxRLqcERERERERMSPDMNg//79JCcnExFRe3+2ku4A2bZtGykpKaEOQ0RERERERAJo8+bNdOrUqdbrSroDpEWLFoCnAVq2bBniaDzcbje7du2iffv2x/wmRoJD7WEeagtzUXuYh9rCXNQe5qL2MA+1hbk0pfYoLi4mJSXFm/vVRkl3gFQOKW/ZsqWpku6SkhJatmzZ6P8AwoHawzzUFuai9jAPtYW5qD3MRe1hHmoLc2mK7XG86cRN410QERERERERCQEl3SIiIiIiIiIBoqRbREREREREJECUdIuIiIiIiIgEiJJuERERERERkQBR0i0iIiIiIiISIEq6RURERERERAJESbeIiIiIiIhIgCjpFhEREREREQkQJd0iIiIiIiIiAaKkW0RERERERCRAlHSLiIiIiIiIBIiSbhEREREREZEAUdItIiIiIiIiEiCmSroXLVrE6NGjSU5OxmKxMGfOHO81l8vF3XffTZ8+fWjWrBnJycmMHz+ebdu2+dxjz549jBs3jpYtW9K6dWsmTZrEgQMHfMqsXr2aM844g9jYWFJSUnj00UerxfKf//yHnj17EhsbS58+ffjkk08C8ppFRERERESk8TJV0n3w4EFOOeUUXnjhhWrXnE4nq1at4v7772fVqlW8//77/Pzzz1x00UU+5caNG8eaNWuYP38+H330EYsWLeK6667zXi8uLmb48OF06dKFlStX8thjjzF9+nReeeUVb5nFixdz+eWXM2nSJL7//nvGjBnDmDFjcDgcgXvxIiIiIiIi0uhYDMMwQh1ETSwWCx988AFjxoyptcyKFSvIysri119/pXPnzvz000/07t2bFStWkJmZCcC8efMYOXIkW7ZsITk5mZdeeok///nPFBQUEB0dDcCf/vQn5syZw7p16wD4/e9/z8GDB/noo4+8dQ0cOJCMjAxefvnlOsVfXFxMq1atKCoqomXLlvV8F/zL7Xazc+dOEhISiIgw1fctTZLawzzUFuai9jAPtYW5qD3MRe1hHmoLc2lK7VHXnC+s34WioiIsFgutW7cGYMmSJbRu3dqbcAMMGzaMiIgIli1b5i0zdOhQb8INMGLECH7++Wf27t3rLTNs2DCfukaMGMGSJUsC/IpERERERESkMYkKdQD1VVJSwt13383ll1/u/VahoKCAhIQEn3JRUVG0bduWgoICb5lu3br5lElMTPRea9OmDQUFBd5zR5apvEdNSktLKS0t9R4XFxcDnm963G53PV+lf7ndbgzDME08IeV0gsMBdjvYbCEJQe1hHmoLc1F7mIfawlzUHuai9jAPtYW5NKX2qOtrDMuk2+Vy8bvf/Q7DMHjppZdCHQ4ADz30EDNmzKh2fteuXZSUlIQgourcbjdFRUUYhtHoh3ock9NJ/HnnEZWfT3lqKoXz54ck8VZ7mIfawlzUHuahtjAXtYe5qD3MQ21hLk2pPfbv31+ncmGXdFcm3L/++itffvmlz9j5pKQkdu7c6VO+vLycPXv2kJSU5C2zY8cOnzKVx8crU3m9Jvfccw+33Xab97i4uJiUlBTat29vqjndFouF9u3bN/o/gGNavpyI/HwAovLzSdi5E7Kygh6G2sM81BbmovYwD7WFuag9zEXtYR5qC3NpSu0RGxtbp3JhlXRXJty5ubksXLiQdu3a+VwfNGgQ+/btY+XKlfTv3x+AL7/8ErfbzYABA7xl/vznP+NyubBarQDMnz+fk046iTZt2njLLFiwgFtuucV77/nz5zNo0KBaY4uJiSEmJqba+YiICFN92CwWi+liCrbt7ftygDTSyYO0NCL69oUQvR9qD/NQW5iL2sM81BbmovYwF7WHeagtzKWptEddX5+p3oUDBw6Qk5NDTk4OABs2bCAnJ4dNmzbhcrn47W9/S3Z2NrNmzaKiooKCggIKCgooKysDoFevXpx//vlce+21LF++nO+++46pU6dy2WWXkZycDMAVV1xBdHQ0kyZNYs2aNbz77rs888wzPr3UN998M/PmzeOJJ55g3bp1TJ8+nezsbKZOnRr090T87+9/r3rcBKaaiIiIiIhICJkq6c7OzqZfv37069cPgNtuu41+/foxbdo0tm7dyocffsiWLVvIyMigQ4cO3p/Fixd77zFr1ix69uzJueeey8iRIxkyZIjPHtytWrXi888/Z8OGDfTv35/bb7+dadOm+ezlPXjwYGbPns0rr7zCKaecwn//+1/mzJmD3W4P3pshAbNlnsPTyw1EbMjzLKgmIiIiIiISAKYaXn7WWWdxrG3D67KleNu2bZk9e/Yxy/Tt25dvvvnmmGUuvfRSLr300uPWJ+FnU0s7uds8w8uNtDQs+jJFREREREQCxFQ93SLB0CzCSTMOAGA5PDVBREREREQkEJR0S9PidPKfX/qQzOHV6Tdvhi++CG1MIiIiIiLSaCnplqbF4SCxvMD33NdfhyYWERERERFp9JR0S9OSmkrF0efOPDMUkYiIiIiISBOgpFualvx8Io84LGufBMOGhSwcERERERFp3JR0S9OSmkpFhBUAF5Gsf3sZ2GwhDkpERERERBorJd3StKxdS6TbBYCVCqy560IckIiIiIiINGZKuqVJ6zTtKnA6Qx2GiIiIiIg0Ukq6pWnp2hXjiMO4PdsgOztk4YiIiIiISOOmpFualgULsIQ6BhERERERaTKUdEuTktd7NGVEeY8PtU2G3r1DGJGIiIiIiDRmSrqlSZm7wMYmUgBwEeEZXj5woOZ1i4iIiIhIQCjplial2QYH3dkAgBW352RenuZ1i4iIiIhIQCjpliYl8Vw76+lW/UJJSfCDERERERGRRk9JtzQp5eUQTVn1C7GxwQ9GREREREQavajjFxFpPIYnO2jOVt+TUVFaTE1ERERERAJCPd3SpHyyLpUyrL4ny8th7drQBCQiIiIiIo2akm5pUprtyCcaV6jDEBERERGRJkJJtzQphUl2fiEdwLtftzs1DTIzQxmWiIiIiIg0Ukq6pUkpi7KRQQ43ZS2jA9vJYhnOJavBZgt1aCIiIiIi0ghpITVpUioq4BA2NrTPYg+wh3gqYkIdlYiIiIiINFbq6ZYmpbzYyWksp3mEkzg8jyv2O0MdloiIiIiINFJKuqXpcDoZcXdfljOAv8218wOexy3PzACnEm8REREREfE/Jd3SdGRnk04eAGls8D6Oys8FhyOUkYmIiIiISCOlpFuaLOPwb3eUFVJTQxqLiIiIiIg0Tkq6peno3RsXkd5Dy+HfEeUuyM8PTUwiIiIiItKoKemWpiM/HysV3sOKwx//kuRuYLeHKioREREREWnElHRL02G3k2tJB+BQ2w64D3/8o3ds0UJqIiIiIiISEEq6pemw2TjH+IIZ3Idj+G1YKQcgosIFc+eGODgREREREWmMokIdgEiwGLsKyaM70bgo+3ckREVBeTlYrTB6dKjDExERERGRRkhJtzQZpf+dSywuAKKpwHj8GTbtiCZxUCqxNluIoxMRERERkcZISbc0GSVnnEsMnlXLDeDZ3BGMfGEUseRBWhqsXg1KvkVERERExI80p1uajm0F3m3CLMCaF74inTzPibw8yM4OVWQiIiIiItJIKemWJqOku51f8KxeTno62+gQ2oBERERERKTR0/ByaTJcVhsZ5NAvysF3i1N5qv3AqovdukFmZuiCExERERGRRkk93dK4OZ2wfDk4nZSXwyFs5ERnQX5+1dBygDfe0HxuERERERHxOyXd0mitX+1kc3wGDBgAGRlU7HcSh5Msy3JITaU4yTPUfLs1BXr3Dm2wIiIiIiLSKCnplkbrr793kHIo13OQmwvZ2fxIHxYeHAADBvDW+C/YRAodXJth8GBPr7iIiIiIiIgfKemWRuu7It+F0z59dx9p5HuO8/NJXzOHzmz2HOfmgsMRmkBFRERERKTRUtItjVZZlGfhtCyWQU4OyQd+8bneppnLJynHbg9BlCIiIiIi0pgp6ZZGy2LxLJy2giwABix+0uf6nk59fJJyLaQmIiIiIiL+pi3DpNHq2BF2bXJixwHZJXRie9XFDh1YGTukKilXvi0iIiIiIgGgnm5ptG4eV8g6erKcATBxImVYAaiwRMLSpSx3KNMWEREREZHAUtItjZPTybC7MqoWStuwgWhcAEQaFVBQQEJCCOMTEREREZEmQUm3NE7Z2bRzbq06Tkry9nS7I62Qmsqtt0JGDyfX91uu7cJERERERCQglHRLo3Rw2z7fE5Mne3u6IypckJ9P765OvjcyePn7AZCRocRbRERERET8Tkm3NEr7ftruc+x+4inySAUgP+rw9mAOh2d/btA+3SIiIiIiEhBKuqVRKh01FheR3uOI/UXEUMJQvua+UTlgs1HaMdU75NyI8gw5FxERERER8Scl3dIoxXaKZxL/8DnXiW1EU8KV13lWLd//Q753yLml3DPkXERERERExJ+UdEvjVFjIP7mm2ul5jCD61589B3Y7uaQB4OqS5hlyLiIiIiIi4kdKuqVR2vbkO4cHjvuKAs65sScUFhIfD5bKC0bwYhMRERERkaZDSbc0Hk4nLPds/7W7sPYsOgJg9mxwOOhOHgDWTXlaSE1ERERERPxOSbc0Dk6nZ9uvAZ7tv9aeNAbXsT7eZWVgt5Nn8QwvL0vR8HIREREREfE/Jd3SKOz71nf7r+Y7N7KDhNqf0KMHAJbD48sNS+1FRURERERE6ktJtzQKuzvY+YV0z0F6OuXl0ImC2p/QujU4HKS6PcPLYzS8XEREREREAkBJtzQKFRUwide4oNnXkJNDfttM78rk1URGQu/eYK9K1PfGp2t4uYiIiIiI+F1UqAMQaTCnk+isDL4hl4KDCRi7VmA4bcRxCIByjvqgV1R49uTOymIQixnNXE6aMJp7bLZQRC8iIiIiIo2Ykm4Jfw4HXV2e+dxJ7MRITeVOdzsS2Ql4PuQGR2wP1q2bp1fb6WQpA0knj+0vpcEDq0GJt4iIiIiI+JGGl0v4S02l4oiPssVd4U24K43lPZafezf83/955m7bbJCdTfrhLcM6OPMgOzuoYYuIiIiISOOnnm4Jf/n5ROL2Hh49nHy3pR2FRiIrRkyjx1AbrdWZLSIiIiIiQaKebgl/SUkYRxxGHnW5WSsriziT0Xf15Mv3CqsuZGZC2uHF1tLSPMciIiIiIiJ+pKRbwt+CBRy5zfbRW27H7vNsHdaZzQz/06ngdHou2GywdCnMnOn5rfncIiIiIiLiZ0q6Jezt6X+uT0/3kdwRkZCc7D1uvndz1X7cTif7Th4MV19NcfdTobCwlruIiIiIiIjUj5JuCXvZr+VU692uVBafBEuWQEqK50T6EftxOxy03ulZ9bxl0WY49YhecBERERERET9Q0i1hL/u5xbVei925FTZuhHXrYNkyyMmpGkZut1PUMqWq8OYjesFFRERERET8wFRJ96JFixg9ejTJyclYLBbmzJnjc90wDKZNm0aHDh2Ii4tj2LBh5Obm+pTZs2cP48aNo2XLlrRu3ZpJkyZx4MABnzKrV6/mjDPOIDY2lpSUFB599NFqsfznP/+hZ8+exMbG0qdPHz755BO/v17xjxe5sdbh5QCUlHgS7aws33nbNhtv3bKKTdTQCy4iIiIiIuIHpkq6Dx48yCmnnMILL7xQ4/VHH32UZ599lpdffplly5bRrFkzRowYQUlJibfMuHHjWLNmDfPnz+ejjz5i0aJFXHfddd7rxcXFDB8+nC5durBy5Uoee+wxpk+fziuvvOIts3jxYi6//HImTZrE999/z5gxYxgzZgwO9YKa0lY6M5r/q71AbGytl9xt4+nJOu4fflQvuIiIiIiIiB+Yap/uCy64gAsuuKDGa4Zh8PTTT3Pfffdx8cUXA/DWW2+RmJjInDlzuOyyy/jpp5+YN28eK1asIPPw9k/PPfccI0eO5PHHHyc5OZlZs2ZRVlbGzJkziY6O5uSTTyYnJ4cnn3zSm5w/88wznH/++dx5550A/PWvf2X+/Pk8//zzvPzyy0F4J+RExOHEAHbThnbs9blmREVh6d271udGR3t+u1wBDFBERERERJosU/V0H8uGDRsoKChg2LBh3nOtWrViwIABLFmyBIAlS5bQunVrb8INMGzYMCIiIli2bJm3zNChQ4muzLaAESNG8PPPP7N3715vmSPrqSxTWY+YiNPJaux8zMW0Y2+1YeaW8nLIz6/16bFuJzlk8PDCAZCRoYXURERERETEr0zV030sBQWevZYTExN9zicmJnqvFRQUkJCQ4HM9KiqKtm3b+pTp1q1btXtUXmvTpg0FBQXHrKcmpaWllJaWeo+Li4sBcLvduN3uOr/OQHK73RiGYZp4/GL1arqzwXt49CrmRnw8Ru/eUMtrbrttNT04vC5Abi7u1as9c7+DoFG2R5hSW5iL2sM81BbmovYwF7WHeagtzKUptUddX2PYJN1m99BDDzFjxoxq53ft2uUz5zyU3G43RUVFGIZBRETYDHI4toQEDtDNJ/E+UvENN3DowAE4ajG9SlvbdOYX0ulBLuWpqRQmJMDOnYGM2KtRtkeYUluYi9rDPNQW5qL2MBe1h3moLcylKbXH/v3761QubJLupKQkAHbs2EGHDh2853fs2EFGRoa3zM6jEqby8nL27NnjfX5SUhI7duzwKVN5fLwylddrcs8993Dbbbd5j4uLi0lJSaF9+/a0bNnyRF5qwLjdbiwWC+3bt29UfwCpOBjMt4zttY4xvzxKh4qtgGc+d4upU2kRH1/rc5snwCAWc0/vD7lt4YUkHKOsvzXW9ghHagtzUXuYh9rCXNQe5qL2MA+1hbk0pfaIPcaCzUcKm6S7W7duJCUlsWDBAm+SXVxczLJly7jhhhsAGDRoEPv27WPlypX0798fgC+//BK3282AAQO8Zf785z/jcrmwWq0AzJ8/n5NOOok2bdp4yyxYsIBbbrnFW//8+fMZNGhQrfHFxMQQExNT7XxERISpPmwWi8V0MflDMa1Z0/cKRm94FiqgpGV7Yn/MxnLUdIOjVex3soTB9FibC0PSg76CeWNtj3CktjAXtYd5qC3MRe1hLmoP81BbmEtTaY+6vj5TvQsHDhwgJyeHnJwcwLN4Wk5ODps2bcJisXDLLbfw4IMP8uGHH/Ljjz8yfvx4kpOTGTNmDAC9evXi/PPP59prr2X58uV89913TJ06lcsuu4zk5GQArrjiCqKjo5k0aRJr1qzh3Xff5ZlnnvHppb755puZN28eTzzxBOvWrWP69OlkZ2czderUYL8lcjxOz0JoyxnAjI9OpVNJHgCxxbvgGHPwK62e7fCZ0422hRMRERERET8yVU93dnY2Z599tve4MhGeMGECb7zxBnfddRcHDx7kuuuuY9++fQwZMoR58+b5dOvPmjWLqVOncu655xIREcHYsWN59tlnvddbtWrF559/zpQpU+jfvz/x8fFMmzbNZy/vwYMHM3v2bO677z7uvfde0tPTmTNnDna7PQjvgpwQR1XS3O7gZkhJgc2bIT0d6tBem1ravXO66/ocERERERGRurIYhnH0LkviB8XFxbRq1YqioiJTzeneuXMnCQkJjWeoR2EhZe2TicZFRaSVyPz1nh5uu71Ow8TvuANefMKJHQfLD9btOf7SKNsjTKktzEXtYR5qC3NRe5iL2sM81Bbm0pTao645X+N+F6TxW7uWaFwARFa4YONGz5ZfdUyeTz8dDmFjBXV/joiIiIiISF0p6ZYmLTPT87uGNfBEREREREQaTEm3hLfevSk7vDSBERkFvXuf0NNdnk5ySkv9HZiIiIiIiIiSbglzRd/nE005AJaKcsjPP6HnL1oEbSlkAq/j3lkYiBBFRERERKQJM9Xq5SInynVS1erj5anpRJ3g6uPlBYVsx7MQm9HJCtu2QXx8gKIVEREREZGmRj3dEtYOWWxc2HYx9yTNxLJ48QkvhnZ5i7nehdgsLhfMnRuIMEVEREREpIlS0i1hrbjAyUd7BvNQwdVEnjEYnM4Ten70b0ZTbjk8JzwqCkaPDkSYIiIiIiLSRCnplrBm/dlBD3I9B7m54HCc2PNb2Yjq1hkAS+fO2jZMRERERET8Skm3hLWD3ezkkeo5SE2FE5zTjcNRtfhafv4JJ+0iIiIiIiLHoqRbwppx0EkKmzwHmzad8PBy7HbKU9M99+qefuJJu4iIiIiIyDEo6Zaw1nzhXO+WYZSXn/hCaDYbCVtzyGIZWz7K0fByERERERHxKyXdEtY2ZYymDKvnwGqt10JoFTE2fojOwmVVwi0iIiIiIv6lfbolrO0vt7GJTnRnA3TqVK+e6qKiAAQmIiIiIiKCerolzMX8mO1JuAE2bKjfQmhOJyxffuLzwUVERERERI5DSbeEL6eTU569uuq4PquXO52QkQEDBnh+K/EWERERERE/UtIt4cvhoOOhvKrj0tJ63YPc+u/zLSIiIiIicixKuiV82e1sIqXqeOvWE0+aU1NxHV6IzR1p9fSWi4iIiIiI+ImSbglfNhs3n/otBSR6jtPrsc92fj5WXABEVLggP9/PQYqIiIiISFOmpFvCl9PJP9afQxI7KGrRERYvPvHVy1NTcVk8Pd0V6ukWERERERE/U9It4Ss7m4Riz5zuVvu3wtq1J36P/HyshqenO1I93SIiIiIi4mdKuqVpS02lvLKnO0I93SIiIiIi4l9KuiV8ZWbyqzUNAGeHNMjMPPF75OcTVdnT7VZPt4iIiIiI+JeSbglfNhsXd11NFstY9cbqE5/PDWC3s7N1OgCFbeqxEJuIiIiIiMgxKOmWsHbAbWMFWUQ0r0fCDWCz8fSEHLJYxpPjc+qXuIuIiIiIiNRCSbeEtchSJ6exnJgKZ73vYWlmw4Gddtsd4Kz/fURERERERI4WFeoAROrN6eTT7RmkkkvJlenwU069eqptOPmBvqS/lwcr02B1PYeqi4iIiIiIHEU93RK+HA5SK3IBiN2UCw5HvW7TeWc26Xi2HiMvD7Kz/RWhiIiIiIg0cUq6JXzZ7ayP8CyCVtql/oug7d7tz6BERERERESqKOmW8GWzMTjOswjalrk59R4SXp6RSS6erceMtHpuPSYiIiIiIlIDJd0S1vZXeFYvj2pZ/znY511s4xQ8W48dWqr53CIiIiIi4j9KuiWsWV2e1cutroatOh6Hk96soaxIq5eLiIiIiIj/aPVyCVvGQScrK/qSTh7l56XBmvr1UrepKGQ7yUTjwuhphe3bID4+ABGLiIiIiEhTo55uCVuuJVWrjkdtrP+q4wWvziUaFwCWchfMneu3GEVEREREpGlT0i1hy7m3xPdESUnNBY9jz6nnYhx+bACce25DwhIREREREfFS0i1hKy7WP/c5N3UjlsOPLQAbN/rnxiIiIiIi0uQp6ZawtXjVUVl3bP2ycGuFf3rMRUREREREjqakW8LW7m5V+2vjz/21lXSLiIiIiIifKOmWsDV0KPzyxxc59NTLsHRp/ffXPrqHfMoUcGrrMBERERERaThtGSbhyemkrGdfRpV6Vi/n+cdgdf22DKN3b8qJJIoKz/GWLeBwQFaW/+IVEREREZEmST3dEp4cDjpVJtwAeXmeRLk+8vOrEm6AqChITW1YfCIiIiIiIijplnBlt1fN5wbPnG67vd732t86peq4vBzy8xsWn4iIiIiICEq6JVzZbJzCaobxGdfTwDndNhsL//otrsrZFlarerpFRERERMQvlHRLWHuRqfyDyTB4cIMWPyv6YSNWyj0HLhesXeunCEVEREREpClT0i1h65LuDnqQ6znIza3/nG7gldf8FJSIiIiIiMgRlHRL2Io7zU6uJd1zkJ5e/zndwAa64jr851BhiYLevf0RooiIiIiINHFKuiVsvTbbRvqBHFi2DHJy6j+n2+lkMYOx4gYg0ijX8HIREREREfEL7dMt4c1ma/h+2g4Hndnqe66kpGH3FBERERERQT3dEsZ2rStk699eh8LCht3IbqcsoaPvudjYht1TREREREQEJd0SrgoLadUrmY73XU1FUnLDEm+bjfX/zWEznQAoSkiDzEw/BSoiIiIiIk2Zkm4JT3PnEo0LgMgKF8yd26DbVbSO5yR+JotlfPjA6vrPDxcRERERETmCkm4JT6NHU4YVgIoIK4we3aDbGYbndwwlxG/MbtCe3yIiIiIiIpWUdEt4io+nA9uYyEzef34bxMc36HaRpU5WY+cbzuSCh8+Evn2VeIuIiIiISIMp6ZawdQgbazmZipiGDwXv7cymOxuqTuTlgcPR4PuKiIiIiEjTpqRbwpJx0EkOGSxnACPvzWhwr7Sl9Kgtwjp2BLu9QfcUERERERFR0i1hyb3aQQ9yAWi5I7fhvdJHbxE2c6YWUxMRERERkQZT0i1hyZWSShlRABhRUZCa2rAbZmZCWprncceOcOqpDYxQRERERERESbeEKWPlKqIpB8BSXg6rVjXshjYb37+8lL0tUmDrVhg8WAupiYiIiIhIgynplrAUuy3f90R+fs0FT8D2b/Nps3+z5yDXD0PWRURERESkyVPSLWFpVtlvKav8+EZEwG9/2+B7ts2sGrKOP4asi4iIiIhIk6ekW8LSxg3AkUm3H/Q4WDVkHX8MWRcRERERkSZPSbeEpVPW/883QZ47t8H3jN7i/yHrIiIiIiLStCnplvDjdNLr40eqjq1WGD26wbc1fvNb3xXR/TBkXUREREREmjYl3RJ+HA66s6Hq+KOPID6+wbe1doinA9uZyEz2/7zdL/cUEREREZGmTUm3hB+7nV9IB2ATKX7bUzsqquqxq9wvtxQRERERkSYu6vhFREzGZuMcviCbLDqz2bOndk4O2GwNum3k3kK2k0w0LoyTrbB9m3q7RURERESkQcKqp7uiooL777+fbt26ERcXR1paGn/9618xDMNbxjAMpk2bRocOHYiLi2PYsGHk5ub63GfPnj2MGzeOli1b0rp1ayZNmsSBAwd8yqxevZozzjiD2NhYUlJSePTRR4PyGqUOnE5WNx9CEjs8x37aU9vy0VyicXkel7v8sjibiIiIiIg0bWGVdD/yyCO89NJLPP/88/z000888sgjPProozz33HPeMo8++ijPPvssL7/8MsuWLaNZs2aMGDGCkpISb5lx48axZs0a5s+fz0cffcSiRYu47rrrvNeLi4sZPnw4Xbp0YeXKlTz22GNMnz6dV155JaivV2rhcND2wOaq45QUsNsbft/RoynDCkB5hH8WZxMRERERkaYtrIaXL168mIsvvphRo0YB0LVrV9555x2WL18OeHq5n376ae677z4uvvhiAN566y0SExOZM2cOl112GT/99BPz5s1jxYoVZGZmAvDcc88xcuRIHn/8cZKTk5k1axZlZWXMnDmT6OhoTj75ZHJycnjyySd9knMJEbudPLqQxq/sb5lEi1WrGjy0HACbjc0kk8av7LO2QwPLRURERESkocIq6R48eDCvvPIKv/zyCz169OCHH37g22+/5cknnwRgw4YNFBQUMGzYMO9zWrVqxYABA1iyZAmXXXYZS5YsoXXr1t6EG2DYsGFERESwbNkyLrnkEpYsWcLQoUOJjo72lhkxYgSPPPIIe/fupU2bNtViKy0tpbS01HtcXFwMgNvtxu12+/29qA+3241hGKaJp9527iSVXwFoXlyA+8ABaNu24fddvpy0w/eNLy3A6NcP46ef/JPQ16DRtEcjoLYwF7WHeagtzEXtYS5qD/NQW5hLU2qPur7GsEq6//SnP1FcXEzPnj2JjIykoqKCv/3tb4wbNw6AgoICABITE32el5iY6L1WUFBAQkKCz/WoqCjatm3rU6Zbt27V7lF5raak+6GHHmLGjBnVzu/atctnaHsoud1uioqKMAyDiIiwmlngo/kzz9D88GMLcPCZZzhw990Nvq+1oIB2Rxxbtmxhzzff4OrXr8H3rkljaY/GQG1hLmoP81BbmIvaw1zUHuahtjCXptQe+/fvr1O5sEq633vvPWbNmsXs2bO9Q75vueUWkpOTmTBhQkhju+eee7jtttu8x8XFxaSkpNC+fXtatmwZwsiquN1uLBYL7du3D+8/gJtvxnj6aSyAAdhuvhnbUV+k1EtSks+hERVFm/79A7aCeaNpj0ZAbWEuag/zUFuYi9rDXNQe5qG2MJem1B6xsbF1KhdWSfedd97Jn/70Jy677DIA+vTpw6+//spDDz3EhAkTSDqcNO3YsYMOHTp4n7djxw4yMjIASEpKYufOnT73LS8vZ8+ePd7nJyUlsWPHDp8ylcdJRyVmlWJiYoiJial2PiIiwlQfNovFYrqYTpTRpSsp/Mq1vMoNK68loWtn/9w4K4sdUR1JLN8KgKW8HMvGjeCPhL4WjaE9Ggu1hbmoPcxDbWEuag9zUXuYh9rCXJpKe9T19YXVu+B0Oqu9sMjISO9Y+m7dupGUlMSCBQu814uLi1m2bBmDBg0CYNCgQezbt4+VK1d6y3z55Ze43W4GDBjgLbNo0SJcLpe3zPz58znppJNqHFouwVVRAVvpzHT+SpS/Em4Am42L4hfjItJzHBUFqan+u7+IiIiIiDQ5YZV0jx49mr/97W98/PHHbNy4kQ8++IAnn3ySSy65BPB8o3LLLbfw4IMP8uGHH/Ljjz8yfvx4kpOTGTNmDAC9evXi/PPP59prr2X58uV89913TJ06lcsuu4zk5GQArrjiCqKjo5k0aRJr1qzh3Xff5ZlnnvEZPi6hU15e9TjKz2M10ivWYaWiqqJVq/xbgYiIiIiINClhNbz8ueee4/777+fGG29k586dJCcnc/311zNt2jRvmbvuuouDBw9y3XXXsW/fPoYMGcK8efN8xtvPmjWLqVOncu655xIREcHYsWN59tlnvddbtWrF559/zpQpU+jfvz/x8fFMmzZN24WZRHk5xOHEjoOoMjvgv9XFR9vzYeERJ/Lz/XZvERERERFpeiyGYRihDqIxKi4uplWrVhQVFZlqIbWdO3eSkJAQ1vMr9m1zsrNjBj3IxeiejuWHHL9t67X2m0K6D+1ANOUYkVFYCrYHdCG1xtAejYHawlzUHuahtjAXtYe5qD3MQ21hLk2pPeqa8zXud0EapbJVDnqQC4BlfS44HH67d1yneFLJYwb3sfnrvIAl3CIiIiIi0jSE1fByEYD17lRaYyUaF1itfl3sLLGFky8ZRg9yKbvyXViT47dedBERERERaXrU0y1hp2VhvifhBnC5/Drv2pZf1YsevdG/vegiIiIiItL0KOmWsJNnpFKG1XPg555ukpKoXOTAOHwsIiIiIiJSX0q6JezMfjBwPd0sWIDl8EPL4WMREREREZH6UtItYWdzKzu/kO45SE8Hu91/Nx89GtfhpQ7cEVEwerT/7i0iIiIiIk2Okm4JO0OG28ggh8GRyyAnx78LndlsFMZ0AuBgu05aRE1ERERERBpESbeEnZNPhkPYiB2a5f+kODubDqUbAWixayNkZ/v3/iIiIiIi0qQo6Zaw8+WXnt8LF4Y2DhERERERkeNR0i1hZ3uek9NYThxO/988M5OC5mkAFLfsCL17+78OERERERFpMpR0S3hxOnn+uwyWM4B19ITCQv/e32bj4TFL2UQKLYu3wuDB4AxAci8iIiIiIk2Ckm4JLw4HPcgFoDOb4dRT/Z4UdyrN99wbIDcXHA6/3l9ERERERJoOJd0SXux2NpFSdbx5s9+T4vZn29nWPEBbkomIiIiISJOipFvCi81GP1axGc+2XqSl+T0pnnCDjeQ1X8B998EXX2jbMBERERERqbeoUAcgUj9G4G5dWAjdu4PLBY88Atu2QXx84OoTEREREZFGSz3dEl6cTr7nVFLY6jnOy/P78PKit+d6Em7w/J4716/3FxERERGRpkNJt4QXh6NqkTOAlBS/Dy+/6cNzffvRzz3Xr/cXEREREZGmQ0m3hBe7nY3RnkXOStqnwKpVfp9zffvlBViOPFFQ4Nf7i4iIiIhI06GkW8KLzcaDY3O4tu8y1s1ZF5C51gmDUinD6jmwWiE11e91iIiIiIhI06CF1CTsvDbbBmQF7P627flEc8Sc7vx8LaQmIiIiIiL1op5uCT9OJyxf7vkdANZ+dn7BM4TdnaZ9ukVEREREpP6UdEt4cTqhb18YMMDzOwCJd0wbGxnkkMUy9i7M0T7dIiIiIiJSb0q6JbxkZ3u2CQPP7+xsv1cRGQlRkZ7HpaV+v72IiIiIiDQhSrpFjuZ0sqrCznIGEH+2PWDD2EVEREREpPFT0i3hJTOT0s5pALhT0yAz0/91fPst3dkAQPSWDfDtt/6vQ0REREREmgStXi7hxWYj5qfV4HAQYbcHZr71hg3HPhYREREREakj9XSLHO2CCzAOPzQOH4uIiIiIiNSHkm4JK8ZBJ7tTMmDAANx9MwIz37qgAMvhh5bDxyIiIiIiIvWhpFvCSsUPDtrtyQUgIi8XHA7/V2Kv2qd7EymQmur/OkREREREpEloUNK9fft2f8UhUidlPaoSYndaOtjt/q/EZmMQi9lECp3ZDIMHawVzERERERGplwYl3SkpKQwfPpx//etfHDx40F8xSZC5DzjZ/enysEgsy6NtZJBDFstwrcgJzEJqQBr5noQbIDdAPeoiIiIiItLoNSjpfuCBB9i2bRsTJkwgMTGRK6+8knnz5uF2u/0VnwSa08mujn1pN3IAB7v3NX3i7XJBHE56s4aossDF+pv7q3rUSQ9Qj7qIiIiIiDR6DUq67733XhwOBytXrmTy5Ml89dVXjBw5kuTkZG699Vays7P9FacESnY2icV5ADTbngcmb7M9vxSynWTe4GoiU5KhsDAg9UQ09/Sozxi5DHJyAtajLiIiIiIijZtfFlLr168fjz/+OJs3b2b+/PmMGjWK119/nQEDBtC7d2/+/ve/s2nTJn9UJU2c5aO5ROPyHLhcMHduQOr5YYkTOw4W7grQXuAiIiIiItIk+HX1covFwhlnnMHIkSMZOHAghmGQm5vL9OnTSU1N5dJLL9Xia2aTmcmW2DQA9iemQWZmiAM6tp0DRlOG1XNgtcLo0f6vxOnkL3MyWM4AXlmRYfoh9yIiIiIiYl5+S7oXLlzINddcQ2JiIr/73e8oKCjg8ccfZ8uWLWzfvp2HH36YBQsW8Ic//MFfVYo/2GwMjFtNFst4+YbVpu/VPRgXTyrreTnhPli/HuLj/V+Jw0EPPNuS9UCLqImIiIiISP1FNeTJP/zwA7NmzeKdd95h27ZtJCUlcc011zB+/Hj69OnjU/aOO+4gNjaWO+64o0EBi/9t3WtjK1mUfQB3/iXU0Rybq8jJlwyjx85cGPZuYOZb2+24uqVj3ZBLaWIKMdqnW0RERERE6qlBSXe/fv2Ii4tjzJgxjB8/nvPOO4+IiNo7z08++WQGDRrUkColgDp2DHUExxeXV9UL7d3KKyvLv5XYbFiXL4ZTTyVm8+F9urWYmoiIiIiI1EODhpfPnDmTHTt2MGvWLEaMGHHMhBvg7LPPZuHChQ2pUgKgchr3DTeENo662JVoJxfPHHTS0gK3lVd+PmzWPt0iIiIiItIwDUq6J06cSPPmzf0Vi4RIdLmT01iO1WX+BcNKS4NTz3ZbKhWRRyzYpiHmIiIiIiJSDw1Kup999llGjBhR6/ULLriAl156qSFVSKA5ncxe61mpe+h1PQO277W/ZEQ5SMezrzh5eQHrgd72TT6RFUdsTZafH5B6RERERESkcWtQ0v3Pf/6T3r1713q9d+/evPLKKw2pQgLN4aBLmWeOdFzhZjj1VFNvkdXn4lRISfEcpKcHbHh5876puCzq6RYRERERkYZpUNKdl5dHr169ar3es2dP8vLyGlKFBJrdziZSqo43bzbv/GWnk7LMwbB5M+XxCfDFFwFb3Owkaz5WQz3dIiIiIiLSMA1KuqOjoykoKKj1+vbt24+7uJqEmM3GYL6lgETPcQB7jxvM4SD6V0+vfFThThgyJHC98qmpnh5uUE+3iIiIiIjUW4My4oEDB/LGG2+wf//+ateKiop4/fXXGThwYEOqkEBzeva9TmIHO2NTYPFi826NZbdTaAtSr3x+vqeHG9TTLSIiIiIi9dagfbr/8pe/cOaZZ5KRkcEtt9zCySefDIDD4eDpp59m+/btzJ492y+BSoA4qva9TijZ7Eku4+NDHFQtbDbi1q7i4Kmn0mzP5oD2ym+NSaU9VqJxqadbRERERETqrUFJ94ABA5g7dy7XX389N998MxaLBQDDMOjWrRsffvghgwYN8kugEiB2O4Vt04nfk8vO1ukkmHVo+WHNusTD5nWeHm67PWC98rHb8j0JN1T1dJv1ywgRERERETGtBiXdAOeddx7r16/n+++/9y6alpaWxqmnnupNwsXEbDaemZjDZ086GHypnafNOrT8sIenOdn2uYMRt9sZlRW4WCP62vmFdHqQi9E9HYvJv4wQERERERFzanDSDRAREUH//v3p37+/P24nQfbgkzYgi7abQh3JcTidjH8qg+QDueybkg6jcgLW0x3d2kYGOdhxsHCxnWYm/zJCRERERETMyS9J99q1a8nPz2fv3r0YhlHt+vjx4/1RjTR1DgfJBzzzz1vvyvUMMc/KCkhVMTFVj0tLoVlAahERERERkcauQUl3Xl4eV155JcuXL68x2QawWCxKuk3urrvg0UehY8dQR3Icdjtbm6XT8WAuRQnptArgkO+oMic5ZNCDXMqHpoMjx7yruouIiIiIiGk1KOm+/vrr+fHHH3n66ac544wzaNOmjb/ikiCKtzk5DQcxFXbAxImlzcZNQ3LY8pmDmx6w84dAJsFHrOoetSGwveoiIiIiItJ4NSjp/u6777j33nu56aab/BWPBJvTyaTnMriTXAo+SIcXc0zdo1te7vkdGRngiux28ixppBl5lKWkEa2F1EREREREpB4iGvLk+Ph4WrVq5a9YJASMHx203e3p0U0qPtyja1ZOJy8uzmA5A7jw/gxwOgNaXeXi+zVPnBARERERETm+BiXdkydP5u2336aiosJf8UiQVfTybI0FcKBtCqSmhjiiY3A46HTI8wVBy4IAf0HgcJDq9myBF7M5z9xfRoiIiIiIiGk1aHh5jx49qKio4JRTTuHqq68mJSWFyBrG/f7mN79pSDUSQBUxNgaxmO85lc57NsPgwZCTY8oh5ka3VMotVqyGC3eklYhAfkFgt5NLGunksS8+jdYaXi4iIiIiIvXQoKT797//vffxHXfcUWMZi8WinnATc7shjXw6s9lzIte8i4ZV5OZjNVwARFS4ID8f4uMDXu/ePdA64LWIiIiIiEhj1KCke+HChf6KQ0KkogIceIaY9yAX0tPBpL26RrdUyrASjYtyi5WoQPZ0Oxyk4xle3s2dZ9ovIkRERERExNwalHSfeeaZ/opDQqSiAg5hI4Mcpgx18NindlMOLQeI2pSPBU9Pd5QR4J7u1FSMKCuWchdYreae6y4iIiIiIqbVoIXUKpWWlrJkyRL+7//+j8LCQn/cUoLE7YY4nNhxcCjNvAk3gKWPnYpUz6JvAe+Rz8/3JNwArsMJvoiIiIiIyAlqcNL97LPP0qFDB4YMGcJvfvMbVq9eDUBhYSHx8fHMnDmzwUFK4FTsd5KDZxuu577NCPg2XA1isxH5Yw4sWxb4xd7sdozungTf3cnkq7qLiIiIiIhpNSjpfv3117nllls4//zz+ec//4lhVO1oHB8fzznnnMO///3vBgcpgWNZ4/DM5QYsuebep/vQIbj+VhuTZ2bhsga4R95m45YBi9lMJyK2bIaBA839hYSIiIiIiJhSg5LuJ554gosvvpjZs2czevToatf79+/PmjVrGlKFBFhZj6p9us28iBpAaSm88gr84x9wxPc7AfP9rLWksMVzkJcH2dmBr1RERERERBqVBiXd69ev54ILLqj1etu2bdm9e3dDqpAAq4jxLKKWxTLuGp5j6jndRybaFkvg62vfvMT3RElJzQVFRERERERq0aCku3Xr1sdcOG3t2rUkJSU1pAoJsMrVy1eQxZoN5k24wbPoW6UIvywBeGw33BrreyI2tuaCIiIiIiIitWhQ6jJy5EheeeUV9u3bV+3amjVrePXVV7nooosaUkU1W7du5corr6Rdu3bExcXRp08fso8Y9msYBtOmTaNDhw7ExcUxbNgwcnNzfe6xZ88exo0bR8uWLWndujWTJk3iwIEDPmVWr17NGWecQWxsLCkpKTz66KN+fR1mUVFR9fjIpNaMgp10l/XNJJc0z0FaGmRmBr5SERERERFpVBqUujz44INUVFRgt9u57777sFgsvPnmm1x55ZVkZmaSkJDAtGnT/BUre/fu5fTTT8dqtfLpp5+ydu1annjiCdq0aeMt8+ijj/Lss8/y8ssvs2zZMpo1a8aIESMoOWJo8Lhx41izZg3z58/no48+YtGiRVx33XXe68XFxQwfPpwuXbqwcuVKHnvsMaZPn84rr7zit9diFkd23oZD0h2Hk9NYjuVQEBY1s9k4hdVM6LkMVq829dB7ERERERExp6iGPDk5OZmVK1dy77338u6772IYBv/6179o0aIFl19+OQ8//DDx8fH+ipVHHnmElJQUXn/9de+5bt26eR8bhsHTTz/Nfffdx8UXXwzAW2+9RWJiInPmzOGyyy7jp59+Yt68eaxYsYLMwz2Xzz33HCNHjuTxxx8nOTmZWbNmUVZWxsyZM4mOjubkk08mJyeHJ5980ic5bwySk2HWLBg3LgyS7gOe7c16kAsZ6QHfNiwy0jP0fnVsFijfFhERERGRemhQ0g2QkJDAa6+9xmuvvcauXbtwu920b9+eiACM//3www8ZMWIEl156KV9//TUdO3bkxhtv5NprrwVgw4YNFBQUMGzYMO9zWrVqxYABA1iyZAmXXXYZS5YsoXXr1t6EG2DYsGFERESwbNkyLrnkEpYsWcLQoUOJjo72lhkxYgSPPPIIe/fu9elZr1RaWkppaan3uLi4GAC3243bJNms2+3GMIxa4onA7TZwu4OwLHg9uVev9m5vRm4u7tWrISsrYPV5FmsL3Pty7PaQYFJbmIvawzzUFuai9jAXtYd5qC3MpSm1R11fY4OT7iO1b9/en7erJj8/n5deeonbbruNe++9lxUrVvDHP/6R6OhoJkyYQEFBAQCJiYk+z0tMTPReKygoICEhwed6VFQUbdu29SlzZA/6kfcsKCioMel+6KGHmDFjRrXzu3bt8hnaHkput5uioiIMw/D5UmT//ligNSUlZezcuTd0AR7HZmtr2mMlGhdGVBQ7W7SAnTsDVl9xcTTQltLScnbu9P8q/LW1hwSf2sJc1B7mobYwF7WHuag9zENtYS5NqT32799fp3INSrofeOCB45axWCzcf//9DanGy+12k5mZyd///ncA+vXrh8Ph4OWXX2bChAl+qaO+7rnnHm677TbvcXFxMSkpKbRv356WLVuGMLIqbrcbi8XiMxLhxx9h8uQI4nBid/5IQvOTTTt3uWjvRqJxAWApLydh/37o1Stg9bVr55lD3rckMO9LTe0hoaG2MBe1h3moLcxF7WEuag/zUFuYS1Nqj9g67m7UoKR7+vTptV6zWCwYhuHXpLtDhw707t3b51yvXr343//+B+DdnmzHjh106NDBW2bHjh1kZGR4y+w8qne0vLycPXv2eJ+flJTEjh07fMpUHte2BVpMTAwxMTHVzkdERJjqw2axWHxiqqjwJJY5ZNAjJxdODfxc6fra36Uvv5DuGWKenk5E374BXcY8psLJD/Ql/dc8yEgLyGJqR7eHhI7awlzUHuahtjAXtYe5qD3MQ21hLk2lPer6+hr0LlTOVz7yp7y8nLy8PG699VYyMzOrJbgNcfrpp/Pzzz/7nPvll1/o0qUL4FlULSkpiQULFnivFxcXs2zZMgYNGgTAoEGD2LdvHytXrvSW+fLLL3G73QwYMMBbZtGiRbhcLm+Z+fPnc9JJJ9U4tDycnXwyvDY522euNA5HaIOqhRMbg1jMPYkzYfHigH8x0H1fNunkeQ7y8uCIrelERERERETqwu9fPURERNCtWzcef/xx0tPTuemmm/x271tvvZWlS5fy97//nfXr1zN79mxeeeUVpkyZAni+Ubnlllt48MEH+fDDD/nxxx8ZP348ycnJjBkzBvD0jJ9//vlce+21LF++nO+++46pU6dy2WWXkZycDMAVV1xBdHQ0kyZNYs2aNbz77rs888wzPsPHG4uYCicXzbm66kRaGtjtoQvoGMr2OVnCYB7acTUMHgzOwG4bVsPABRERERERkRMS0P7+oUOH8sknn/jtfqeddhoffPAB77zzDna7nb/+9a88/fTTjBs3zlvmrrvu4qabbuK6667jtNNO48CBA8ybN89nvP2sWbPo2bMn5557LiNHjmTIkCE+e3C3atWKzz//nA0bNtC/f39uv/12pk2b1ui2CwPA4aB5QV7V8cyZphxaDtBhtyOoPfLWQZlsjEwDwJ2aBkeseC8iIiIiIlIXFsMwArZH1LXXXsv//vc/9uzZE6gqTKu4uJhWrVpRVFRkqoXUdu7cSUJCgnf+Qb7DSUT/DLqW5bIpJp3Oe3JMm3TjdEJGhifhTg/S3PPCQpg7F0aPBj/uOQ81t4eEhtrCXNQe5qG2MBe1h7moPcxDbWEuTak96przNWghtbfeeqvG8/v27WPRokW8//77XHPNNQ2pQgJsww4bo8tysONg4oN2bjRrwg2eBPuLL+DVV+HaawOfcDudMHCgZz53WmAWUhMRERERkcatQUn3xIkTa70WHx/Pn/70J6ZNm9aQKiTAKirgEDZcGVnceEeoozmOwkLo3h1cLnjkEdi2ze+9zz6ysz0JN1QtpDZ0aODqExERERGRRqdBSfeGDRuqnbNYLLRp04YWLVo05NYSJOXllXtRO8BpN3VP7le3z+WsyhXlXS7PsO+rrgptUCIiIiIiIsfQoKS7cqsuCV/GwcN7dK/LxWVPx+rIMW3ivTZtNIOxEo0LrFbPPOtAyswkPyKNVHcepSlpxGghNREREREROUGNe2a7HJctv2pFcOsG8+7RDXDFH+PZtGQbux+fGfih5QA2G+d3WE0Wy3DM0nxuERERERE5cQ3q6Y6IiMBisZzQcywWC+Xl5Q2pVvxob0c7v5BOD3L5NSadLibdoxugdWtoPTAeBgZvSHnlTnOlpUGrUkREREREGpEGJd3Tpk1jzpw5rFmzhhEjRnDSSScBsG7dOj7//HPsdjtjxozxR5wSIKWRNjLIYWKmgxe/Nvec7qBzOvlwcwZdycU5MR1+ydH7IyIiIiIiJ6RBSXdycjI7d+7E4XB4E+5KP/30E+eccw7Jyclce+21DQpSAqe83LN6eV7bLDB5PvnRR7BkCZx1Fpx3XhAqdDjoWuYZem/benjofVZWECoWEREREZHGokFzuh977DGmTp1aLeEG6NWrF1OnTuXRRx9tSBUSYJUj/W04Yflyz97UJrXwYyfz/76cxV8EKcbUVFwWKwDuSCukpganXhERERERaTQalHRv2bIFq9Va63Wr1cqWLVsaUoUEWEWFZ8uwRz7PgAEDICPDnIm308nds/uynAFMfbVvcGLMz8dqeLYoi6hwQX5+4OsUEREREZFGpUFJt91u58UXX2Tr1q3Vrm3ZsoUXX3yRPn36NKQKCbDycsgk27uCObkmXcE8O5uE4jwA2u3Ng+zswNdpt7PVlg5AUWI6mHiRORERERERMacGzel+6qmnGDFiBD169OCSSy6he/fuAOTm5jJnzhwMw+Dtt9/2S6ASIE4n/+TqquO0NCWXlWw2bj4zh02fOrjhfjtXaRE1ERERERE5QQ1KuocMGcKyZcu4//77+eCDDzh06BAAcXFxjBgxghkzZqin2+S6HXSQTl7ViZkzzblCd2Ym2+O60eHQBorbd6NlZmZQqq2IseHATqvNDnBqdXcRERERETkxDUq6wTPE/IMPPsDtdrNr1y4A2rdvT0REg0auS5CMuN1O6WvpxGzKJS8ynbQgJbP14a6cDRHEz1ac4SSHDHo8kgvvp0NOjhJvERERERGpM79lLxEREcTGxhIfH6+EO5zYbOT9L4cslnF26xzzJpQOBx0PeXrkW+7IC9q8864HHOaf7y4iIiIiIqbV4Ow4Ozub888/H5vNRrt27fj6668BKCws5OKLL+arr75qaBUSYJZmNlaQxUHDpAk3gN3O5ljPomb7OwRvUbP52+38gqde0rWYmoiIiIiInJgGJd2LFy9myJAh5ObmcuWVV+J2u73X4uPjKSoq4h//+EeDg5TAefZZ6N/byWksJ6bChFuFVbLZGN/X0yP/1VM5QeuR79MHJvEaD53/tYaWi4iIiIjICWvQnO57772XXr16sXTpUvbv389rr73mc/3ss8/mzTffbFCAElhF2w/PWSaX9cXp4MwxbWJ50PD0yFuaBalCp5OHP8sggVw2f5UO5ASpYhERERERaSwa1NO9YsUKrrrqKmJiYrBYLNWud+zYkYKCgoZUIQF23eCqOcvdDXPPWS4v9/yOavDyf3XkcJCwz/PepJTkBmdvcBERERERaVQalHRbrVafIeVH27p1K82bN29IFRJgiefacXX1zFneRAqkpoY4otoFPem22ylqn1Z1fPXV4DTxEHwRERERETGdBiXdAwcO5L///W+N1w4ePMjrr7/OmWee2ZAqJNBsNnZ8sJhNpNCZzTB4sGkTyx49oG9faNkySBXabCy5ZmbVcV7wVk0XEREREZHGoUFJ94wZM8jOzmbUqFF8+umnAPzwww+89tpr9O/fn127dnH//ff7JVAJjAUL4L+P5nsSbjD1tlj//S/88ANkZQWvzv0nZZLL4d7utDStXi4iIiIiIiekQUn3gAED+OSTT1i/fj3jx48H4Pbbb+e6666joqKCTz75hL59+/olUAmM99+He98Jk22xnE5YvjyoPfHR0UGrSkREREREGqF6z441DIP9+/czePBgfv75Z3JycsjNzcXtdpOWlkb//v1rXFxNzKW8HA5h4/37c/jThQ5Pwm3G1cudTsjI8PTEp6cHbfuudtsdpJPnOagcXh7MrnYREREREQlr9e7pLisro23btjz77LMAZGRkcOmll/L73/+ezMxMJdxhwuU6/MBm8ySTZky4wZPs5npWEg/mEPjSdLuGl4uIiIiISL3VO+mOiYkhKSmJmJgYf8YjQRZR4uQ0luM+4OTAATCMUEdUC7udvAjPEPjSLsEbAh8dDRYOr9B/jJX6RUREREREatKgOd0TJ07krbfeoqyszF/xSDA5nTw4ty/LGcClf+tLQgsnFRWhDqoWNhvFny/m57tnUrFocdB65Nv/mk13NngONmzQXt0iIiIiInJCGrTjcZ8+fZgzZw4nn3wyEydOpGvXrsTFxVUr95vf/KYh1UigZGeTdMAzXzmdPDLJxu0eGuKgauF00u+GwZ6h5e8Hb053QkLAqxARERERkUasQUn35Zdf7n1c29ZgFouFCtN2n8qRPv8MrNZQR1GLmuZ0B2FBs7j+vSnDSjQujCgrlt69A16niIiIiIg0HiecdN97771cdtll9O3bl4ULFwYiJgmWzEy22dJIduZR1D6NVkMywaTr37lSUomIsBLpdmFYrVhSU4NSb+y2fCx4VpuzlLsgPx/i44NSt4iIiIiIhL8TTroffvhh7HY7ffv25cwzz2T37t0kJCQwf/58zjnnnEDEKIFis3HroKXELZjLiGmjudysK5cDpT/l09x9OPl1BS/5taSl4o6MIqKiHCMqKmjJvoiIiIiINA4NWkitkmHaJa/lmJxOnlgymDe4msybBnPtOCdOZ6iDqpkz1c4veFYvN9KDt3o5a9cSUVEOgKW8HNauDU69IiIiIiLSKPgl6ZbwZKzIppPTM086nVx+mO2gtDTEQdWiNNJGBjkMjlyGJUiLqImIiIiIiDSUku6myulk50VXew9zScOB3bRbUVfuShcdHeSKMzMpap8GQGnnNMjMDHIAIiIiIiISzuq1evnGjRtZtWoVAEVFRQDk5ubSunXrGsufeuqp9YtOAsfhILE4z3t4NTM5hM28Sfc+Jzlk0ONQLmQEb8swbDYuSV3NgV0O7n/Uzmj1sIuIiIiIyAmoV9J9//33V9si7MYbb6xWzjAMbRlmVnY7+zuk02J7LptIYS2erbDMmnRHrHXQg+BvGQYw+vc2fh2YReeeQalOREREREQakRNOul9//fVAxCHBZrPRYvViNrc/lc5sZikDmcRM3AcyIdF8vbnFne3kkkY6eZCWFryF1IBbbw1aVSIiIiIi0siccNI9YcKEQMQhIZD/RT6pbAYgnTwWcSau89LBkWO6hcrMusCbiIiIiIjIsWghtSZszH1V23BVsm44PHTbZKw/Ozy93AB5eUGNsWi7k50fLae4wKT7qYmIiIiIiGkp6W7CfszzbMN1Bl+Ti2eFblfXIO6BfQL2JB/xBUEw9+l2OnH26EvC6AFU2Pti2o3MRURERETElJR0C6XEMpClZLGMLR/lmG5oOcAhi+cLgmv7LgveyuUA2dl0OODpYW+zOw++/TY49YqIiIiISKOgpLsJi8OzDddyBrCEwTiwUxFjvoQbquZ0W4O9T/fRJk1Sb7eIiIiIiNSZku4mzE7VNlw9yMWOw7Rbhtlw4ojK4MXsAZCREbzENzOTfc07Vh1v2WLKOe8iIiIiImJOSrqbMAdV86R/IR0HdtMm3RenOUgtP2qf7mCw2Xjp+hw2keI5DuZ8chERERERCXtKupuwQ3jmSWexjHsvyOGcUTYzTuf2sNs9CS8EPfG1xMfTj1W8dvpMWLzYlHPeRURERETEnE54n25pXA5hYwVZfPMBxMSEOppjsNk8Ce/cuTB6dFAT35ZRTpYykPTv8mBgGqxercRbRERERETqRD3dQseOEB3qBcqO440XnWzqNBiuvhoGDw7qYmbJ27J99wjPzg5a3SIiIiIiEt6UdAsnnQQWS6ijOLayVQ46l4ZgTjewclXQqhIRERERkUZGSbdgsXgSb5sNvv8+1NHU7Oyb7BxIDs2c7oKOmeSR6jlITYXMzKDVLSIiIiIi4U1Jt7BgAZSUwKFDUFER6mhqln6Kjea5ObBsGeTkBHVO9UkngZvDQwHMPiRARERERERMRUm3APDNN7BxI/TpE+pIjsFmg6ysoC9idtPZDt853dqnW0RERERE6khJdxOW0NzJaSynb3cnnTtDly7mXcF82ceFfHft6/z0bWHQ697Sumo/c+3TLSIiIiIiJ0JbhjVVTidb22cQdSAXN+lQuBjy8z0Jpdm2wyospP/oZKIMFxUzrbBjG8THB636yBY2+pBDZoyDRTkmfH9ERERERMS01NPdVDkcRG3wrAYesT6X4u79YMAAyk7OCOp2XHUydy5RhguASLfLs1d3EHXqBLm/wJw5Qa1WREREREQaASXdTZXdTnFCmvewZdEWAKI3Bnc7rjoZPZpyixWAiggrjB4d1Oqjypx0HJVB2wsGQEaG+b6UEBERERER01LS3VTZbIzd+WK1086OJpyzHB/PVedvYyIz+ddjwR1aDni+hMgNzR7hIiIiIiIS3pR0N1VOJy9yo/dws7UbQ/maZf/IMeWc5T2WeN7kKmgb5IQbKE23s7OVZ1SAkZpmvi8lRERERETEtJR0N1WOI7bBAv7S+Q2+YSjl0eZLuAEshzwrrccZoRnaXVzkBsBd4Q5J/SIiIiIiEp6UdDdVdjuHUjzbYOVHpbOueSYAbjPmlE4nLy3JYDkDGPnnjKDPqY75MZvubAAg8tcNkJ0d1PpFRERERCR8Keluqmw2Pn8khyyWMTAmh7IoTw+3KZNuh4OUEs+c6hbbQzCnuqTk2MciIiIiIiK1UNLdhE26ycYKsth10EbE4U+CKZNuu51NMZ451Qc7aE61iIiIiIiEDyXdTdju3VWPTZ10A8bhuCyhDcNDPd0iIiIiIlJHSroFqEq6KypCG0dNjB8ddHF5Fn2zbc8L/vDy2Fjf4ylTtFe3iIiIiIjUiZJuASAy0vPbjD3dRrdUyrAC4I60QmpqcAPIzGRfi45Vx1u2aK9uERERERGpk7BOuh9++GEsFgu33HKL91xJSQlTpkyhXbt2NG/enLFjx7Jjxw6f523atIlRo0Zhs9lISEjgzjvvpLy83KfMV199xamnnkpMTAzdu3fnjTfeCMIrCq62FHItLzOMz2ldXshpLMdyyBw9uPs3FLJm8rPkTn4c99uzicYFQESFC/LzgxuMzcarN+awiRTPcXq65pWLiIiIiEidRIU6gPpasWIF//jHP+jbt6/P+VtvvZWPP/6Y//znP7Rq1YqpU6fym9/8hu+++w6AiooKRo0aRVJSEosXL2b79u2MHz8eq9XK3//+dwA2bNjAqFGjmDx5MrNmzWLBggVcc801dOjQgREjRgT9tQZEYSHb6UA0ni8bXMuisFLO/jvS4ZIcsIVwv+7CQmJSEzmZqm53d2QUERXlGOnpWEKQ8LrbxNOTddxzoYP737WH9v0REREREZGwEZY93QcOHGDcuHG8+uqrtGnTxnu+qKiIf/7znzz55JOcc8459O/fn9dff53FixezdOlSAD7//HPWrl3L22+/TUZGBhdccAF//etfeeGFFygrKwPg5Zdfplu3bjzxxBP06tWLqVOn8tvf/pannnoqJK83ECr+b6434QawGp7HLQpCsCXXUfa/OptofMe5R1SUs3XSfVgWLw5Jwmv1jG7H5Qp61SIiIiIiEsbCMumeMmUKo0aNYtiwYT7nV65cicvl8jnfs2dPOnfuzJIlSwBYsmQJffr0ITEx0VtmxIgRFBcXs2bNGm+Zo+89YsQI7z0ahQtHU0Zk9fMmGDr9zuzqa5QbQMd/PgiDB4dkETMbTnLI4IHPBkBGhhZSExERERGROgm74eX//ve/WbVqFStWrKh2raCggOjoaFq3bu1zPjExkYKCAm+ZIxPuyuuV145Vpri4mEOHDhEXF1et7tLSUkpLS73HxcXFALjdbtwmWZ3M7XZjGIYnpjZtGcUnzKdquLz7tdfg97/3rNYdwpifdZzBtfhuD+Z9nJuLe/VqyMoKakwJO1fTg1y/xnBke0hoqS3MRe1hHmoLc1F7mIvawzzUFubSlNqjrq8xrJLuzZs3c/PNNzN//nxij97GKcQeeughZsyYUe38rl27KDHJvs5ut5uioqLDfwQRfMcQckkjnTzKu3al8Oyz4cABz08ITWJmtf24y4kgCjeuLl3ZnZAAO3cGNaayHp3YEpNKp9J8z3vlhxiObI+IiLAcdNJoqC3MRe1hHmoLc1F7mIvawzzUFubSlNpj//79dSoXVkn3ypUr2blzJ6eeeqr3XEVFBYsWLeL555/ns88+o6ysjH379vn0du/YsYOkpCQAkpKSWL58uc99K1c3P7LM0Sue79ixg5YtW9bYyw1wzz33cNttt3mPi4uLSUlJoX379rRs2bL+L9qP3G43FouF9u3b43ZHAFVfBhTujuTMMxOZ8Vgcv/td6GLE6eQK3q52ehft6cAOoiwWEhISgj6v+3e/c2L5uwXyIDIy0i8xHNkejf0/SGantjAXtYd5qC3MRe1hLmoP81BbmEtTao+6dgSHVdJ97rnn8uOPP/qcu+qqq+jZsyd33303KSkpWK1WFixYwNixYwH4+eef2bRpE4MGDQJg0KBB/O1vf2Pnzp2exAmYP38+LVu2pHfv3t4yn3zyiU898+fP996jJjExMcTExFQ7HxERYaoPm8ViISIiAsOIwI6DdPIASNqfR6v9azh4MIuQhrt2LYnsrXa6A54vQSwbN2BZtQqGDg16XOR53itLXh6WtWv9MsS9sj3M9BlpqtQW5qL2MA+1hbmoPcxF7WEeagtzaSrtUdfXF1ZJd4sWLbAftchXs2bNaNeunff8pEmTuO2222jbti0tW7bkpptuYtCgQQwcOBCA4cOH07t3b/7whz/w6KOPUlBQwH333ceUKVO8SfPkyZN5/vnnueuuu7j66qv58ssvee+99/j444+D+4IDzIGdX0inB7mUdUnnpX/Z6dwzxEHZ7aynG93ZUHuZUAzXt9shLc2TeKelhXyxORERERERCQ+N7quHp556igsvvJCxY8cydOhQkpKSeP/9973XIyMj+eijj4iMjGTQoEFceeWVjB8/ngceeMBbplu3bnz88cfMnz+fU045hSeeeILXXnut8ezRfdghbAxiMa+fMZPo7MX0P8NG+/ahjqoOQjCf/6uvIP8Y3wOIiIiIiIjUJKx6umvy1Vdf+RzHxsbywgsv8MILL9T6nC5dulQbPn60s846i++//94fIZpSVBQUbXfSbMhgIr/JhcHpkJMTkj2wfWRnH7uXOyICDk8DCCZbvoNUt2d4OXl5nr3Mg7yCuoiIiIiIhJ9G19MtdWOxQMtNDiLzqrbBeutuBytXhjauo4eOVxx93e2G/PyghVOp9+/slKakeQ40vFxEREREROpISXdTZrdDejoA25ul8drzJSxb6AxpSKXFvkl35NEFQpTwNm8OddwRQERERERExEtJdxNVXg6TbrJxw6AcSj77GosFFnEmlz+SAc7QJd6FP273OTaOeOy2RMKXX4ZkCHxJtoP4fUcNLxcRERERETkOJd1NVEUFzJwJL79lo9waS9IBT0LZpjA3pAml8/yxlB1easCwWLAccS3CqIAFC0IS16aoVMqwAuCOskJqakjiEBERERGR8KKku4mKjIS//93zY82wU9DCM8x8T3x6SOcrW5rZ2EQKAEbbdj7X3BGRMHp0KMLi4Iq1ROMCIKLc5dm3W0RERERE5DjCfvVyqZ+oKLjnnsojG3+5OIfv33Zw2c12bgvhCuau7x30Orx6ecTuQp9rZbGtiA1RbImJIalWRERERETCnHq6BYDyaBsryKIsKrRbhj3xQdUwbiIjcR0xwDzWuQe+/TYkcSUP6101vDzSGpJty0REREREJPwo6W6i3G5YtcrzU1Hh2f668nwo7f8h3zuMm4oKXmOSb4HVq4MfFEB+VVwRFa6QbFsmIiIiIiLhR0l3E1VaCv37e34OHTJP0l2abucXPPPLSU+n3YCTfAvExAQ/KAC7nc0xnn269ydpn24REREREakbJd0CmCfpLo+2kUEOWSyDnBwunTsRd8ThpQeiouDyy0MWW+V7ZCkrCem2aiIiIiIiEj6UdDdRhuF7bJak++hc1tI+nogd2z37m23fDvHxoQnM4aDjIc+2as33bIWMDCXeIiIiIiJyXEq6BYulKumuqAhtLCV7nOSQwXIGVCW28fFw1VWhS7gBUlMxIo74c9m6FbKzQxePiIiIiIiEBSXdTdTRPd2RkZ7foe7pvuWMbHqQ6znIzQWHI7QBVcrPxxLqN0dERERERMKOkm7BYoHEFk4uSlpOu7gQDpl2Ohn0ytVVx2kmWrDMbvcsoFapa1fIzAxZOCIiIiIiEh6UdDdRPj3dTif3vJvB/xUM4La3MkI3V9nhIKUsr+r4xRfBFtp9w71sNlr8uBQ6dfIcVw4NEBEREREROQYl3ULEWodnKDeEdki33Y4z+Yje5BtvNNdiZfn5sGWL53FennmGvouIiIiIiGkp6W6ijuzpNk62Q3rV3tghG9Jts2F7/cWqY7MltnaTvE8iIiIiIhI2lHQLlmY2Xrkxhwk9l/HMVTmhG9LtdHp6tyuZaU438M1KG51yv+DpFvfBF1+YZ+i7iIiIiIiYVlSoA5DQOHr18k2FNt5al0XrgtDEA3h6tfOOmNM9c6apEtuSrYXk053o/S6M7o9g2bYttNuYiYiIiIiI6amnW7BY4A9/gI8/huuvD2Egdjub4zzDt/cnpZtudfD2S+cSjQsAi8sFc+eGOCIRERERETE7Jd0CwEknwciR0Lt3CIOw2RjXO4cslvHNczmm6uUGKBw0mjKsABhRVhg9OsQRiYiIiIiI2SnpbqKaN4f16z0/0dGhjqZKkcvGCrKwtjJXwg3gbhNPKuuZwX0Yues1tFxERERERI5Lc7qbqIgIzzpllVavhh+WOOljcZBxpT1kvczGQSen4aCZxQ6YK/GOLHWykHNIJw+GveN500zWGy8iIiIiIuainm4BYN77TgZMziDj+gGQkRGa/bGdTub8msFyBnDq1SGK4RjspdmehBs8C75lZ4c2IBERERERMT0l3U3UoUNwzz2en4oKSNzloAe5nou5uaHZH9vhILXcE0Ps5hDFcAyJLUt8T5SU1FxQRERERETkMCXdTVRpKTz8sOfH7YY9yXZ+wbNyOOnpodkf224n35IKgCsl1VR7dIuIiIiIiNSH5nQ3UTExcOutnscREeCOtZFBDndd4GD6f0M0p9vppJOxCYCobZs8w8vNPGdaPd0iIiIiInIc6uluouLi4MknPT+RkZ7E+xA21rfNClmiWz5nLtGUA2CpKDffPtixsb7HU6aYbt65iIiIiIiYi5JuATxJdxxOuu1aHrJE8tC5Jt8HOzMTOnasOt6yxXTzzkVERERExFyUdDdRFRWwebPnxzAgutxJDhn89fPQrV5eEmFjE508BymdzDe03GajaN5ijIhIz3FUFKSmhjYmERERERExNSXdTdS+fdC5s+fH7YaEnaFfvbziBwfd2QCAZcMGU/Yif/HqRizuCs9BeTmsXRvagERERERExNSUdAsAxZ1Dv3r5gYRU7/ByrFZT9iLv2BHqCEREREREJJwo6W6iDMP32NLMs3r5zQOXQU5OSIZ2dzPyicblOXC5ID8/6DEcz85mXal86wyArl1DF4yIiIiIiJiekm7BYvF0LB/CxppmoVu9PPIUO6SleQ7S0ky5T/do9/9hOfzYAvB//xfCaERERERExOyUdAvgSbrjcJK2O3Srl4eDU/sdNUTg6CEDIiIiIiIiR1DS3UQdmStaLGDDyQ+WDP6RE7rVy/P+zwF5eYcP8ky5kJrliitw4Vm93B0RCVdcEeKIRERERETEzJR0CwAXdnWQboR29fJfKsy/kBo2G9siOwNQltTZfNuaiYiIiIiIqSjpbqKO7unGbvesWg4hW728V7T5F1IjO5suFZ5tzWK3bYDs7BAHJCIiIiIiZqakWzxsNs+q5ctCt3p51wvNv5CaiIiIiIjIiVDSLQCsXQujLrVx9cuhW708LGRmsi/e88VAaUoaZGaGOCARERERETEzJd1N1NGLbhcXw8JPnOz5LHSrl+/9xvwLqWGz8cczV5PFMv75x9X6gkJERERERI5JSbcAkNbByfbEDOZsC93q5W/n2PmF0M4rr4v0U2ysi7TT9YBD26uJiIiIiMgxKelu4iwWz+/2Oxy02hHa1cud2MgghxkjQzevvC6alxTiqOjJyBmh+4JCRERERETCg5LuJqpNG1i8GL799vAJE6xeXrHfiR0Hu5Lspk24cTr53cOn0pnNnuMQfUEhIiIiIiLhISrUAUhoREfDoEFVx05sLHo4h+YbHQyZHIKk1+nk8of7ci95bJmdBs+ZdL60w0FH9+aqY4vFnPuJi4iIiIiIKainWwDYuRN+MxbuvTdEAWRn063Cs4hap5I88+5/bbdTGJNUdWwYniEDIiIiIiIiNVDS3UQdPAhPPglPP+05ji53kkMGi0o1T/mYbDZWpozxPbdgQUhCERERERER81PS3UQVF8Ptt8Mdd3iOY9c76EEIF1Lr3RuXxQpARaQVevcObv0nIOOSNN8TKSmhCURERERERExPSXcTFRcHV14J48YdPmGv2q7L6B6ChdTy87EaLgAiK1yQnx/c+k9A4l0TcREJgBERCRMnhjYgERERERExLS2k1kS1bg3/+lfVsbWVjUEsZjRzeenL0cQFexGz1FRcWLHiwh1pJcLMi5PZbFi7psDGjVg6p5hzwTcRERERETEF9XQLAFaXkyUM5g2uJvrswcGf052fjxVPT3eEyXu6yc6GjRs9jzduNO+ibyIiIiIiEnJKupsow4DSUs8PgPXnqjndkXkhmNNtt/NrjGd4u7NTaPYJr7eSkoDctmyfk5JFy5nzWiFF85drcTsRERERkTCk4eVN1NatnvW/rFYoK4PIUzxzunuQS3lqOlHBTnptNrrsyQGHA5s9BPuEn4jevamwRBFplHuOr78e1qzxa8zOQicFHTJILc9lJFaicUF6OuTkmPu9ERERERERH+rpFg+bjSxrDlkso+DTnNAkdjYbZGWZP6nMz69KuCEgQ8yzX84mtdwz8iD68LD7kKwqLyIiIiIiDaKku4kyjOrnog6Pe3C5ghtL2LHbqUjq4HvOj0PM3QecpNx/RfUL3bqF17B7ERERERFR0t3UWSyHHzidLC3NYDkD6DgqIyTzh8dd4uT2M5azbb3J5y7bbLieezlgt98/71u6sbXaefeh0oDVKSIiIiIigaGkWzwcDrq7Dw9n/jUEw5idTqb/XwZPfDuA+PMyTL9o2Mrc1r4nYmP9c2Onk8jrrqrxUkTBNg0vFxEREREJM0q6m6jK4eXenu7D+2QDuKOsEOx9sh0O0o3DSf9G889dXn6gN2WH1yE0oqKgd2//3NjhoPnebTVecmOBpCT/1CMiIiIiIkGhpFs8jtwnuzwE+2QfnUyaPLms+HEt0XgWU7OUl8Patf65sd1OETUvJBeBAaedZvpRACIiIiIiUkVJdxNVrafbboe0NM/jtLTgL9i1YMGxj02mXdQ+3xP79tVUrF5cRNZ+cedO+OILv9UlIiIiIiKBpaRbTGHvGaNxR3qGt2O1wujRoQ3oOC489agh4NtqHhJ+wrKziWf/scuY/AsJERERERGpoqS7ifP2dDsckJfneZyXF/Q51T9vspFf0clz0KmT6ffqjrxoJJW7rhkAI0f65b53Tdxx/EL/+Y+GmIuIiIiIhAkl3U1UtX267Xa22tIBKEpMD/7wcoeD7mzwPN6wwfQLqUVv2Ujl9xUWgHXrGn5Tp5PbN1zve+7eeylp1sb33Pbtpn9/RERERETEQ0m3eNhsZL+Ww+ybl7Flbk7Qe5p3tLeTSwjnlJ+gZpElvicmTGh47/O335LIXt9zvXrhvu2O6mWDvbq8iIiIiIjUi5LuJs47vBy4+HIbVzydxcmnBX9o98GDYMHtOXC7g17/ibLEHbUvd0FBgxc4K/x4SfWT33+PLb559fOLFzeoLhERERERCY6wSrofeughTjvtNFq0aEFCQgJjxozh559/9ilTUlLClClTaNeuHc2bN2fs2LHs2OE7T3bTpk2MGjUKm81GQkICd955J+Xl5T5lvvrqK0499VRiYmLo3r07b7zxRqBfXlAlJMBH7zn57K/LfXtonU5Yvjzoc4abrcv2HV6enR3U+k9YZiblrY8a9j1pUv3ft8JCWj87vfr5M8+EK66g4ujzX39dv3pERERERCSowirp/vrrr5kyZQpLly5l/vz5uFwuhg8fzsGDB71lbr31VubOnct//vMfvv76a7Zt28ZvfvMb7/WKigpGjRpFWVkZixcv5s033+SNN95g2rRp3jIbNmxg1KhRnH322eTk5HDLLbdwzTXX8NlnnwX19QaSDSej/pzBkNsHQEYGOJ0s/rCQgjY9YUDVuWApORS0qvzDZmP3tff6nissrP9c6//9j6ijz8XHw7BhEB/Pr3e/4Hutcns3ERERERExtbBKuufNm8fEiRM5+eSTOeWUU3jjjTfYtGkTK1euBKCoqIh//vOfPPnkk5xzzjn079+f119/ncWLF7N06VIAPv/8c9auXcvbb79NRkYGF1xwAX/961954YUXKCsrA+Dll1+mW7duPPHEE/Tq1YupU6fy29/+lqeeeipkr93vHA7IzfU8zs2F7Gw6XXwqSWWbq84FcbGuDe0yfed0Z2YGre76an/VaHzWo+vSpd5z0ctbta1+8vLLvXPrU5N9R2Jw002eJF9EREREREytWudaOCkqKgKgbVtPwrJy5UpcLhfDhg3zlunZsyedO3dmyZIlDBw4kCVLltCnTx8SExO9ZUaMGMENN9zAmjVr6NevH0uWLPG5R2WZW265pdZYSktLKS0t9R4XFxcD4Ha7cZtkjrLb7cYwDNxuN/tTekNiOq125GKkp2O43XRms7eskZKC0bt30OZX7yuL4xRW8+Dvf+SW106G2Fjzz+3+6kuOmBKP+9ZbTyjuI9vj4NqNtDr6+jnnVN2rc2ffb8jcbtyvvw63396AFyCVjmwLCT21h3moLcxF7WEuag/zUFuYS1Nqj7q+xrBNut1uN7fccgunn3469sO9iwUFBURHR9O6dWufsomJiRQUFHjLHJlwV16vvHasMsXFxRw6dIi4uLhq8Tz00EPMmDGj2vldu3ZRUlJS7XwouN1uioqKMAyDzZutnL0jh9NiV/PuvAQAotqkEb83j8K4jpR/+ikcOOD5CYLCwhYcohm/Jp7MziDW2xBWm412Rxzvbd0a186ddX5+ZXvgdMLfnvJJuivat2dX375w+H5Ww/CpC8Dypz+xY+RIaHf0FTlRR/5tRESE1QCgRkntYR5qC3NRe5iL2sM81Bbm0pTaY//+/XUqF7ZJ95QpU3A4HHz77behDgWAe+65h9tuu817XFxcTEpKCu3bt6dly5YhjKyK2+3GYrHQvn17DCOCIedZiI0dQEJXzyDp5+7M4b/3rmLYYIP7u3QJ6rZhtkO7mcB7dG42ioSEhKDV2yBduvgctmnVyrNCXR1522PDBqLc273nDcDy9dckdO1aVfi883DFNsNaUrV+gcXtJmHRIrj+qL295YQd+bfR2P/nEA7UHuahtjAXtYe5qD3MQ21hLk2pPWJjY49fiDBNuqdOncpHH33EokWL6NSpk/d8UlISZWVl7Nu3z6e3e8eOHSQlJXnLLF++3Od+laubH1nm6BXPd+zYQcuWLWvs5QaIiYkhJiam2vmIiAhTfdgsFgsRERF07BjB5597zwIQGxvBP7mGHgty4dR0yMkJTuJdWMiT73QkChcVj1iJuG2bZxExs7PbKSOS6MNri0f89reercNOIHaLxUJE374U0J4kdnnOAZalS6FXr6qCzZuzeuKjZLw8xef5EWlpYKLPVzir/Nsw099rU6b2MA+1hbmoPcxF7WEeagtzaSrtUdfXF1bvgmEYTJ06lQ8++IAvv/ySbt26+Vzv378/VquVBQsWeM/9/PPPbNq0iUGDBgEwaNAgfvzxR3YeMQx4/vz5tGzZkt69e3vLHHmPyjKV92iskgod9OCIxdWCtZDa3LlEGS4AIt0umDs3OPU2VH6+N+EGoKIC3nmnXrcqOfr7r8GDq5Xp1SvS59gAjM5d+PIjJ4WfBH+bNxEREREROb6wSrqnTJnC22+/zezZs2nRogUFBQUUFBRw6JBnv6lWrVoxadIkbrvtNhYuXMjKlSu56qqrGDRoEAMHDgRg+PDh9O7dmz/84Q/88MMPfPbZZ9x3331MmTLF21M9efJk8vPzueuuu1i3bh0vvvgi7733HrfeemvIXnsw7GuXShlWz4HVCqmpwal48GDvKuDG4eOwYLdT3uaoVcePWEyvrkq+zaYr231PLl5crVzMFWMxjli6zQIYvXpy0uh04kcNwN2jp1Y0FxERERExmbBKul966SWKioo466yz6NChg/fn3Xff9ZZ56qmnuPDCCxk7dixDhw4lKSmJ999/33s9MjKSjz76iMjISAYNGsSVV17J+PHjeeCBB7xlunXrxscff8z8+fM55ZRTeOKJJ3jttdcYMWJEUF9vIOXlQcuWcMTofLZ/sZZoPD3OuFywdm1wglm82JtKWg4fhwWbjchnnvE9d3iKwok4tGOf74nISBg9unrB+Hg2XjDZ51QE0JFtnsdbN8Opp6rHW0RERETERMJqTrdhGMctExsbywsvvMALL7xQa5kuXbrwySefHPM+Z511Ft9///0Jxxgu3G7Yvx8sR+x5NWwY8GkIghk92tOz7nJ5fteUcJqUZd9e3xN//CP85v/bu/Pwpqr8f+Dvm6TbpXSB0pYCLS0gFcreoYKoM1JB5KeijDIKyCIqMyDgOIqDGzPK4qjzBRcQZBSXQQYRFVRAFARRLLIUKGApFChbN0rXQNsk5/fHadLeJukCTZs279fz9GnuvefenOTkNv2c9d56jYUX6ae1O/75T6fjws9GDUY0ljq/2JkzcljAwIF1fn4iIiIiInKdZtXSTQ3HUf2F4YZ4pKGL3OjSBYiPb5zMhIQA588D770nfzeHSdSsqs1gjkuXgK+/rtclCsK7a3fU8L736lzi9BhQ0T0/sPqK30RERERE1FQYdHu4qi3dhgAVA/QHcVdYEnDwYKMuGTZ6NPDSS8Dp07WndSvV1oQHAHz+eb0ukde5P8pQMUmaXi+7iDt7uodHo6b+HgqA8gE3cGw3EREREZGbaFbdy8m14uKAQpMKoJG7Jufm4pMdEfBGOcQNXsCFZtTaHR+P0qC28Mm/WLmvnl27db8dqZwF3WyWY+lvvtlx4pAQKKdPoyDqOgTC8aRtXiX5EP36QUlNbdSKEyIiIiIisseWbg9l7V5etaW7QRmNwO66LWNl+fQz2wRuiqkc+OwzF2XKBVQVylvVxlg/8QSQmlrnS5SX1/M5IyPx1XXP1JhEOXsW2LOnnhcmIiIiIqKGxqCbtOoRLDuTsS8XWW1igYQEoG/fWq91StGut45q66+7O++LF+x3Dh6MS4cykP78+7V29c6OrP9Y+lGT2tSaRowfz5nMiYiIiIiaGINuD1e1pfvSOSPOhfatc7CsYTQCO3ag+JP18BlwPcJKz8j9aWlyNu0aFPYeguOoCLSjo4EhQ+r1GppcbKz9vrw8+PeORszLk2EJC68x8DYVGrEYM7AoZjHwyy916hLeasqDELV0U1AyMmp974mIiIiIyLUYdHsoh6uvpaSgQ0mafFyHYNnGaISpR2/gllvg/+DdCENlgCmAWteuNnmrSMBuPNnmPdnK3tzGIQ8ZgjwEa3YJAF6wAAB0FjOwfLnjcy9exF2PReAtzMSs9JmywqMulR0hIVA2bbJ7Trtts7lOL4GIiIiIiFyDQbeHq9pYqg6MQ3472c1ZdOkiZ1ari5QUGE6fcHx9AFizpubTd+QiGX3xet5k4IYbml+XaFVFwe1jNLvs2qDnz3f4uvw+/xwGVAmM09PrXtkxZAjQrZt83K0b0m8cZ5+H4cOb3/tJRERERNSCMOj2UI5aun18gKAA+bhe86vFxeEcQp0fv+BgzLOV0YjbnuyNTjgnt0+caJYTgEVPvrXmBCUlwLp1drvL86sFxDodEBNTtydVVSA5GUhKApKTEfPfefbLiRUVATt31u16RERERETU4Bh0U6WUFBn0AvJ3XVtcVRWrMM7hIQEAjz7q/Nw9e9AB1YLyK1fq9rzuJCys9jRLltjtOnxYr91hscjW7rpSVblEmapCiYrEryPn2qeZPJmt3URERERETYRBt4fTzMUVEwOLwQsAILy86t7iCuB/+KN9KysqWsx//bV+mfL1rV96dxAfj7LgdjWn2bVLO6Ga0YjrNr2jTRMRUfdu/Q70ntDPfue5c5xQjYiIiIioiTDo9lDt2wMrVgCLFlXZmZ4Onalivezy8jq3uJ47kItdGOy8S/r48UBGhuNj8fHIbFVlibCYmDotmeV2VBXeu350WPGgsXZt5eNPPkF7VJvV/P33r2kiOd+wIMcH6lGBQkREREREDYdBt4cKCgIefhgYV7VXeFwcTurlRGpXOtZ9IrWL89+CV7V9dsGng67VAABVxVPDU3ATtmPt49uBQ4ea3+zlVgUFtY6Fz826gis7dkOczoClerd7RQH697+2PMTHozC4vf3+L7+8tusSEREREdFVYdBNGrqKT4TFUvdz2h3cbrfPLvhs08bp+X2uM+Ku60/AP6FH8w24ASAuDiI6usYkbec+Ad9bElDeLx6G6geFqN94bkdUFT898I79/scf57huIiIiIqImwKDbQxUXA5s3A9u2VdmZkoKocjmRmnq+jhOpGY3Ab4drTzd7tuMu5rm5+Ntr7fHU0cm4fWJ77Zjn5kZVoaSkAH/9q9Mk1soI70s59gfrM3N5DXq3dTBb/OXLzXJWeCIiIiKi5o5Bt4c6fRq4/XZgTNXlpePicNpHrvtcEtGtbt3L9+xBe1QJIAMCnAeO775rv2/VKsBkko9NJrndnKkq8Pe/w1TRjl3rGO+q6jtzuRMdZoyGxVFH9/z8a742ERERERHVD4NuD+XrC/TrB/TqVWWnquLRnj9jIt7D9oU/162rd1aWdltV5brRixdrdgug2gByKe9csXZHcbFdmmZHVZHXOhIAkB8Yicv2Hckd69jxmmYutwkJge63ozBV35+UdHXXMxohknazezoRERER0VVg0O2hunQB9u0Dvv++yk6jEW/tuwErMRk3zLqhTkFW2S97tTsyM2Vr7ZQpKAkMt+1WANm8Xs3xw2VX9wLcWUoKQotki3VwQQZ23vB07ee0aQPs399wY9q7d8eFYRO1++bPr3/3faMRlzr3hXJDAspiYpt3938iIiIioibAoJsq7dmDbpBjutvknajTGOCckZO0XaijomRrraqi1ewZ2sQHD1Y+NhqBb79Fl68XadP4+19V1t1KXBzMMbKbvrlLN9zy2RPIR4DT5EJVgdRUICSkQbPR6ZZu9jsddfGvSUoKgnPSAADeWWdg6t5Tjs3fzZZvIiIiIqK6YNBN16Qk6aB29PAbb1S21lYPoMvlGuAiJxeXQroAw4ejLQoqj+v1wIMPujS/jUJVoT+UDCQlQX8wGd4RIQjav93p+G6xaFGDB9wAHK53Lp57rs6t1d98mIvD7/+CbLS17TPkZcMSFQUkJMDSqzcDbyIiIiKiWjDo9lC//Sa7mCckVNkZH4/sQLlOd75/B6BHj5ovkpuLrnPGaPedP1/5+IEHUFY1JH/+eSAjA6XX90bw5UzNaSYowOHDrgk+m4KqAgMHVlZA7N/vdA1v5Z//dE3wOmQILiBU+1wWC7ByZa2nZu3NwG0TwtDznZkIxUXNMesfDV36CWDnzgbKLBERERFRy8Sg20OVlcmh15ph1qqKdx76BRnohKDic8DgwTUHgxs2QFe9/fYPf9BcLx/BldtmM6783xL4XrRf0soA0bIDuDvvhNDrHR5Szp6t2/Js9aWq+Btet9td/OXmGsv17DEjlPh+8ELti7Vbxo5nazcRERERUQ0YdHso4aSv821d0hGJM3IjLa3mYHDwYPsu0z//XPl4zx6EIk9zuKh1uPNltFasqCHHzVxICJT0dCAsTG5HR0N07AgAEN3quDzbVXjyD/vt9vnv/A647jq7YPnn74zY/upu/GfYJ3bl5owuNxumx2cC337L4JuIiIiIyAEG3R5OqdbnedAjcbLfOSB/1xQMbtum7TKt1wN33lnj8wX/7x2n3awxf34tuW3mIiNl94KkJCAlBeLoUVz85huIffsabtbyavrOuMXxgXPngHnzKgNloxFht/XCLU8n4IXTU+r1HIb3VgDDh0PExjLwJiIiIiKqhkE3XbXjBW21O5Yt047Jjo+HJThYk8RwLNX5BZ10v25Rqo71VlWU9+vnsoAbAHTDEoFOnRwfnD9fVqzk5mLnk+vQBXKZM6eVIrVQzpyRgfyyZVxajIiIiIioAoNuD2XtXl69pdu4OwU4IZcNw4kTNXYv3/d5tXW3L13SbqsqLEuWXWNO6Zqoqpw17447HB/PzER556648Z3xdbqc06EBVvPnA1OnAu3bM/AmIiIiIgKDbqpm7W9xSEPdupfvSqoWsXt726UxRITV7YljYhwucUUNQFW1E9xV41VSUOfWbeXttyECAmtPaDIBy5fX8apERERERC0Xg24PV72lW1UBXV0iMKMRM6vPjD1qlH26Hj1gquk6HToAmzcDhw65tJu1x5s4EUJ3jbd7dDQwcSKUE8ch2rWrPf2zzwIZGdf2nEREREREzRyDbg/lbPbyP8amoIuoQ/fy775DZ1Rb+isz0z5dejoMzjIxZQpw7BgwbBgDblcLCYFy8iTg61v/c60VIykpspxCQqCcOgXMmlX7ua+9Vv/nIyIiIiJqQRh0k1ZMDODlJR97ecltR7Zv124HBTnuih4XB3MHJxN53Xcfg+3GFBkJJCfXPi7b2xt4/HE5FjwpyXHFiKrKSdPCah4+IN58E1i/nrOaExEREZHHYtDtoZxNpIb0dKC8XD4uL5fbDlzq2EO7Y948xwG0qkL5+GP7/VFRwJAh9cs0Xbvu3aGsWVNzmnnzgDfeALp3r5xp3RFVla3fNQTeCgDcfTfQuzcDbyIiIiLySAy6SeM3QxzOqt3kRrduTidSK0tJ0+6oYeyubmA8LNVbuz/8kK3cTWXkSFnp4YhOB0ycWPdrhYTIipm5c2tOd+IEsGdP3a9LRERERNRCMOj2cNVbukuEiuuMybgzNAlITnYaGC/J+5O2m/KkSc6fRFWhS95XuV50t26cqbwpqSpw5Igcpz13LnDPPcCcOcDixUBWlnat9bpe76mn5ERrqGFZsbFj2dpNRERERB7H6RxX1LJFRMgllVu31u739QX8YETnksOAMcZx0G004oktd2iXmcrKkt2RnQkJkWOEU1Jk6zlbuZuWqspx2sOGNdz1UlLkz6drgddetU9z9izw3XfAXXc1zHMSERERETUDDLo9VHg48Pe/2+9vdTkXFxAB75JyiAgvKOfP27d87tmDoJIL9ifXRlXlGGFqmSrKV4mJAV5/zfEU+aNGAadOyUndiIiIiIg8ALuXk0aHfRvgDTmRmlJeDmzYYJemKD1LuyM8nN3FqVJICLBpk+NjQgADBrCbORERERF5DAbdHqqkBNi3z34ZbsM9d6IMcskwYfAC7rxTc1wI4PBbW7Uncekvqm7IEJiCnYwNz82V3cyJiIiIiDwAg24PdfQoMGSAEXMSd2taHZV2IejsdR4T8R7O/WrftfyDf+diwN53tBe76abGyDI1J6oKw8r/OD0s7r67xhnviYiIiIhaCgbdHsrbZESKoS/WZyUAfftqAu8SvxB8gEm43Mq+pfLMSysr2sGJapGYCBEe7vCQAgDXXw+sX1+3ruZGI7B7N7ulExEREVGzw6DbQ/XWpSDGVLHWdlqapp95kLcRv8NulF6yD3B6GJPsL7Z3r6uySc2ZqkI5dAiW0DDHx41G4O67gV69HAbTosSI0i07IN5eAgQHAwkJclmy3FwXZ5yIiIiIqOEw6PZUcXFyvWxA/o6Lk4+NRvyQG4fdSEDXUXHaYCg5GfeWr7W/1l/+4vr8UvMUEgLdyXSc7TfCeZr0dODrr7Ut2UYjMtvFwWfYLVCmTwPKyuT+7Gz5eWXgTURERETNBINuT6WqQHIykJQkf1snQtu5E9E4CQDwvXAS2Lmz8pw5c7RrcwPAww9z+Seqmaqi49ypNSYR998vW7LbtweSk3HunyvQ/vJJx4nz82XXdAbeRERERNQMMOj2UAcPAjfcquL+1wZqZx4/WS3QsW4bjSjbtdv+Qi+84LpMUsuRmCi7hjthq8wpLITo1w8dXplZ8/Vyc2GJ7cHAm4iIiIjcHoNuD1VYWNnIrTF6NMyKAQBgUfTAiIpuwTt3wjv/ojbt0KFs5aa6UVU5b8C8ebUmtetN4YTuYg7Qpw8nVyMiIiIit8ag20MJIX/rqn8CVBX6qI7ymDADt94KGI0w7dxlf5HXXnNtJqllUVVg1iyIkHYNd83z57VDIIiIiIiI3AyDbg9lscjfSvVmxZQU4NSpyu0TJ4A9e3D5pWoB9tixcqkxovpQVShHjwAVS4mJepzqNO2f/sRu5kRERETkthh0eyinLd1xcRAxXSq3o6KAjAy0RnHluQDwt7+5OovUUoWEyMqcpCQoX35Zt3N++w3KRx85PnbpEszdr2c3cyIiIiJySwy6PZS1pdtR9/L/jN2Kcshx3Th/Hli/XpNEAYAff3R1FqklU1Vg4EA5wVpUVM1p9+8HuncH7r0XVwJDHSbR5+UiO7IXMkc9AmRkuCDDRERERERXx9DUGaCm4bR7OYC+ed/DCya5UV4O9O0L8emn2gmuRH06BhM5oarAkSNyXPbevcArrwAFBfKYogD79lUOY1BV+K75CBg+3OGlQi+mA1+mQ3y5Aleu7w1Dx/ZAwiCY4gfBr18s8MUXEACUUaMqh1DEx2tn7zcaZX7i4rT7iYiIiIiuEoNuD+W0ezmAfs/fCbHMAMVkAgwGiLHjcPnfb0C9mCUTGAzAgw82XmapZVNVYNgw+TNzpgzAT54ERo+WXdGrGjJELj1WfWm7KhQAfkcPAkcPAls2wwtySIRS8SNmzqysQOrcGTh8GPD1BYxGKEOGyK7vrVsD48YBzzzDGfqJiIiI6Jqwe7mHqqmlW99ahQiPkBsREVAOJFcG3ACwapV9METUEKwB+GOPOf6MWZce27wZws+vzpdVnDzGqVPAJ58A334Ln+f/AeXECbm/qAhYuhQiKkoum5eaehUvhoiIiIiILd0eq6aWbtMPO2E4WzEuNiMDZ97+HJ2qJsjMdHX2iJyrCMyV/fuB2NhrvpyYMgU6AMEOjikAsGkTRGwslLffBu6/H0hPZ/dzIiIiIqoztnR7qJpaukt/03bdjfh2pTZBbRNfETWG7t2B06eBCROu6TIObgHHaaZNg7ldKJCQAEt4e+Q//iwwdy7w5JOcvI2IiIiInGLQ7aFqaulu9ccRtjWRBQB99QRBQS7LF1G9REYCK1cCOTnAq68CI0e69On0FXeGrqgQQW/NB/7xD+Df/4aIioJl1D3Aa69xzXAiIiIi0mDQ7aEsFsAPRsQZd9uvb3zqlK31z64VMDRUzvhM5E5CQuTa8V99JVu/Z89GaVgHbZqAAJc9vQJA9+UXwFNPQbRrh4wHn4Z44w0G4ERERETEoNtTdWlvxKmgvnjvcIJckqlq4H3livMT772XY1nJvUVGAgsXwif9GLB5M/DOO7Il/MIFuT13LjB0qMueXgEQ+cmrUGbOhGjXDgfe3QXTzw4qtxwxGoHddUxLRNRA0j5LxrGIm3G5d3+c63s7zC8vYKUhEVED4kRqHqqHJQXIT5MbaWlyRuiBA2s/MSzMtRkjaijWmdCrsi5NBshx2P37AxcvQvj5QQgB3ZUrcvjEH/4gu6p/9RXwxRdXnQUFQO9HB0MBUGrwgXHEfQju3wVo00Yuu2edod1olEulTZ4MnDuHy2GdgLtGwe+6SGDiRK4WQEQuU/ZrMrr+sZ/s2XYB6AAABzYDz88Bfv4ZGDSoaTNIRNQCMOj2VHFxQLduMuDu1k1uW/n6Oj+vbVvX542oMURGysA7JQWiRw9kZ2cjNDsbut69K3tzPPAA0KOH7LJ+laxDNHxMpfDZ8DGwQW5bZs6EOWEwxPFj8LqYqxnK4Zd1Bnj3TQCAeOopKB99xF4mRNTgTOkZKLr5Djj7ZheDB0NZtw64555GzRcRUUvDoNtTqSqQnCxbuKsvfxQfjyvwgi/KtecoigxCiFoKVZU9PCyWysdVZxdUVeDIEWDPHhmgr18v5zX47TcZBJ87B2zcCLF/f51mQa9KB0CX9HOt6RQAGD8e4rHHoKxcKVvgGXwT0bVKToa+Xz+nATcg//6Ie++F8s03wIgRjZUzIqIWh0G3J7MGGQ72695ZCkydot0/dSq7uZLnUVXg5pvl43Hj7I/PmwclNxdYvhx47jlACAjUbSmy+lCMRuD++2EGoB81CujTp7Kbuqo6rkAjInIkIwOWfv3qNLGPAgB33AG88gowfTr/xhARXQVFCOviUdSQCgsLERgYiIKCAgS4cNbk+rBYLLILbWgodI7WCqvKaAS6dAEyMyv3nT4tu+RSg6hXeZBLNVhZ5OYCGzYAsbFyXHhpacNl0gkBwKzzgsFSDpPeC4aE3wFjxwJ6vWyZysxs+mA8IwNYsgQYMEDOCxEfX2N+eG+4D5aFe2mQ8jAacbltBPyuFNT/3DZtgNRUVsBX4P3hPlgW7sWTyqOuMR9buskxVQVOnAC++05OpPKXvzDgJqpNSAgwaZJ8nJcHrFsHvPUWsG8fUF5e87lXSQFgsMhrG8zl8n79WXZbt7W4BwTIWdG7d9eebDRWtpAD9q3lRqPsWg/UGig7YsrMxbFxz+L675drW/6DgoAPPgASE9lqRtTITB994jzg/r//A1q3lj3bTCb743l5EJ07QzlyhP8TEBHVA4Nuck5Vgbvukj9EVD+qKrujjxunDW6NRhmI/+MflWn/3/8Dvv4asHY8CgwECq6iFaoaW6BbWAgRGwtz4jAYvPTy+idPwrx7D/TCDOu/1gYA5QBE60B464CywsvwFmXyoF4PvPGGnE29DoGy+HkXlBsHo4ejg/n5wN13y3wcPMh/3okaifhmI/TVh45ZVZ2p/IEHgMcfB957zy6ZUlICERUFZetW2aOHiIhq1bLb+4mI3IF1/gRVla3hc+fKtcPfe0/+3rAByM6u3D5/HkhKkkM6XnyxQbKgADB89y2wcSOwejWQlAS9MAOQwba1BtYLgHdRAVBQUBlwA4DZDEybBrRqBYwfL7uMO5OcDNw4GPraMlVQABEVBbz8MrBjB9cnJ3KhzMUfACPvcDzfxNat2qXBVBV4802I0FCH11IAiFtvhbnbdbICkWt6ExHViGO6XaTZj+kml2N5uA+3L4vcXGDVKuDsWeDVV5s6NwBk1/Xy7tfDGxZg6VKgVy/beHbT4BthwFV8tRgMwOLFsMTE4NLlywi+7Tbo/P2dp6/ag4Dd1F3C7e8ND2MpLsalVasQ/NlnsKSfQuHFYgSUXEKpzgel/kFo0zEICAlBcWA4Sv2C0fYvD8D8zjKYNn8L76wLjgPuTz4B/vQnx09oNAIPPywr6mogAJgDAmGYOQNo1w6IipLDSK5iWEpzwvvDfbAs3IsnlUddYz4G3bV4++238eqrryIzMxN9+vTBm2++iYGOZvyuhkE31Ybl4T6aVVnk5gIrV8ply0wm4MoV4NQpGXyWlDRJlqxfIgrQoDO322Zqv/FG4P77Zev/Rx9BFBXDfOE89KmpUADZPV7vBfirMPj5AeHh8p/9v/1NdtOPiQHS0+senBuNwM6dKP9+OwxbNqLszHnohRmG8DB5vLgY6NdPdo8PCJA9AAoKri34r+/4eWuFQ3g48PHHwM6dct4Ab2/Zq2LQIGDIkGsOeJrVvdGcWcszJkYuUwjIz8GBA8DkyTBlZQNmC5TC/Np7kFRR6/04eTLwn//UfJHcXLlU4lX8u2i7N1upMHt5QycsKFW84FVaCp8OoXKyyZISOUFbcbEM2K33blaWHIpy4QIwerTsJeRmFW28P9wHy8K9eFJ5MOhuAP/73//w0EMP4Z133kFCQgIWLVqETz/9FKmpqQh10uXKikE31Ybl4T5aRFlU/af955/lJG6HDwOXLslJEZua9Z/2nJyrOv1qgnnrOdbfJshAXkAPExR4G/TQwQxhARQfL5SZvaArM8ILpqsKbGzTTil6QFFgVvTQCQu8vOU2DAYZOOTkAFeuwGQWMAtADwEFFttzynzqoVMAg07m2QDAZAGEosAsdPASZdDX8r6YAMDXD9DpYAgMABISZEWBn5+c2b57d+Dtt+VnR68HNm+WxwYPlp+d4mJYcnNhKi2Dzs8XhitXZBm2bw+cOSOHGggBdO4s17pv21a2cP70E+DjA1x3nawISE4G8vJguVKK8stlAATKAZQB0APwAeDrpZPnmM3yOgEB8rPr5SUDMyEAnU62nhYUyP2AzOvIkTJwGz1aBmI7dwInT8rXeOqUrJiy8vWVQR2gDd6cVXpUva+qVtxUT280yp4eQ4dWVoItWSLzrtPJISNlZYCPDywWAYuiAy5fBgBcgQ9UczF01crTVPH+NPTygzZ6vVzdoC4zkVuXRfzHP+TrcLHqn2sTACMUtIKAHtZgXg9YAIsALFCgM+ihE2ZAp8Di5QO92QR9QGv5mW/XTn5G+vYF9u6V53p7Ax07AhcvAjfdJI/l5cnPzW+/VX6GalgBwlJcjEs//ojgm26quVcOuVyL+B5vQTypPBh0N4CEhAT87ne/w1tvvQVAfoA6deqExx9/HM8880yN5zLoptqwPNxHiy+L1FRgxgwZSOn18p/moCC5/nhRkZww6aefZHAeFgb4+8sAccUKGTSFhsrjV/N10aqVrAAYMkRu79kjA5KZMx3PjkyNxhXryTclE+TrsVZeOHt9VdOZABRCh2BYNMGuTq/HFbOAT0VlSNWKlULoEFitksQaHDeb93T6dDlfRH2X/srNlUNJqi4n2sJULUNtucsSVwEY9ECZWcCv4nNjPa6HnBfDUnG+DgI6WADoYdBXVp5ZhKyH89LJbbOQz2OBlvXbSNHrUK7zgbelFAZFAHo9TKUmCJhhMBhg1ushyi1Q9DrAZAYUQOj0gNkMBcL2fEIAOj1gUBRY9HqYyiyAQQ9vPQCTCSYhAIt83ULIykAoBijeesAs08qxRSZAkRVhBj9ZsWcqMwEGA/ReeigGA1BQALNOB9E6AEpYmJyjxMcbQmcAjEYIf3+ZrvSKrNDw95ffNxfzYOjVU07St3o1TKXl0IWFQmcskd8ZBQUQxcVAdAwUn4qePUVFsJSXw1haCrVVK+jatgUOHIApKwu4XApDVCdZCbh/v6zc8/MDoqPlRJ5hYbKCr6wMGDYM+PZbmZ+uXSvT6/XyJzgYyMmBKTcXupISCL0BSlwcdEGBQFgYxIkTEPv2AcUl0LVtIysPhw6FZf16oE1FZeK5s8DCf8lKvP97HUpJiXwNnaOBwymyovT554HsbCi/7JI9vCwWWXGkr/irYzDIfOflyV4i587JeWHCwoCICLk/rKJnVlaWfFxeDqSlAcOHyx4kpaWy4ikqSt7P4eHyPGuvEkDe72vXyrJJSZGrs1h7oJw+LfOzbp28xq+/yklRX34Z2LsXluXLcemeexAcHQ3d/v3y3IIC+Txr1sj/Nzp1kuf88IPM/6BBwCOPyOfesAG4807tYzddqpBB9zUqKyuDqqpYu3YtRo0aZds/YcIE5Ofn48svv9SkLy0tRWmVNXkLCwvRqVMnXLp0ya2C7pycHLRr165lBhbNDMvDfbAs6sBolF+uixZB2b+/ToGFCAiASEtz/EVpNAJffw1l8mQonECN6KrVJ9AXAMR3313brOPWVv60NNlT4vhxKCUlzaOygcgN1HbPNmXlnfDywoTEc9i4ETiHCHhX9uGqU76cpane88yZ8oqqKm+YUAY9dIoCgzBBeHlBnD3rloF3YWEhgoODuU731crNzYXZbEaYtaaoQlhYGH777Te79AsWLMA/qi4BVCEnJwdXqnZta0IWiwUFBQUQQjCwcAMsD/fBsqijYcPkj9EIry++gP/KlSj384NvcjL0ZWWVLYWKgoK5c1E+bpysoc/Odny9W24BDh2C14EDwMmTaLViBQzHj0NfXl6vfziqjiknaukEgNL27WGJigLatEHhn/8Mv59+gtcvv0CXkwOlsBDKlSuAyQSd0QhL69awhIfD1Ls3ip94QnapdnZP1lVsrPyxtkQZjfBatgxt/vWvyqEWXl6AyQS9ELw3iaqo7X5oyvtFKS9HnzProIO3JuAG6pYvZ2mUWo5becECa58Pb5htX/BKeTkKV63CZWeTPjahoqKiOqVjS7cT58+fR4cOHfDzzz9jUJVlNJ5++mls374dSUlJmvRs6ab6Ynm4D5ZFA6jvRGA1sBQXI3/dOgRt3Ajdjh1y7KvRKLvCBQfL7n4hIXINdL1edocD5ARzBw7IbvTnz8uubIoiA3+dTnZNNJsBnQ6KxWKrdQe03UmFrmJ8saoCHTrI7rRt28pufmazfO6DByuvV1Qkn8fJ9Z3V+gtrV0Ggcsw3IK9T9atZCLvrCQDCx0emU1WgdWt5vskEFBZCKShw6T9uV9sSIyDHpaPqfWaxQKkIzJpN9+yrUP21OfzsGQzaz4GXlxxr/PvfA61awdKqFXISE9EuNtY9/1Y5mujMOvljerrsppqRAfTvD3zzjUzTpQtw6JDsJnvqVOW9q9NV9om23hOKIu95s7nyHql2fwDNrKs/eRx3b+nOSzmLsjIgvG8HKObGbekWOp38DjWZ5BAJnSIfs6W75QoJCYFer0dWVpZmf1ZWFsLDw+3S+/j4wMfHx26/Tqdzqy9GRVHcLk+ejOXhPlgW18jfXwYGDXQt0+23Q/fQQ/Urj6efrnvaiuBAiYuT2xWTbymjR0NpiC916/Wts1FfuSInH1uzBhg/Hsq990KpT8VE1eulp0OJi6v5fOtYvIgIOXbvs8+AH3+UwYvFIq9nNstgXan4F6hrVxnIpKfL8YN+frB0746iwEC0vnABuj595KRUa9ZAmTEDWL9env/TT/IaxcXyup07y4qSnBwZOMXGAjfcAISFQXnwQcfvb24usGEDlNhYYMEC2RLbqZOc/KptWxloWd/DiAi5X6+X4whLSiqfX6+XgaqXl0xvDdysrAEcoA3enFV6WCsIrAGws/SKIitqyspkem9v4PrrZVBpMgGPPgrl5ptloNm2LZCXB8VaWbRqlXwqZ+9NVRYLlOxs9/1b5e8vy7qq0FDH92Ytc+PUS9X7uWJSO6VfP2DZMjkpWna2HMd79qwcf1peLssqL09+DkpK5D6dTn5urJ+jK1dk2VYN8quwKAosigKdENA5asMSQt5vVSvYroa1ItA6qaBeX1nJZzBUTCpXUbloNstzqldOVGWtvKiaNweVfbBYtNevnlank++r9R6xpq0Y0w2dTo5jto7p9vaWx4zGyrlDrlQb052XB/SsHNNtm1ekpHJMN4qL5eSG1tUanIzpRlaWfM86NeyYbuTmyvwYDLKCKVCO6caJE8C+fZUz8leM6cb69ZUTRJ49C+VfFWO6X39dph04UOYnJQXw84NSMaYbuxp/TLcyejTaWv8OZV7QjOlW6jCmW3EyplupGNOt1DKmW6kyplupMqZbufPOhvludoG6/i1mS3cNEhISMHDgQLz55psAZGtYZGQkpk+fzonU6JqxPNwHy8K9sDzcB8vCvbA83AvLw32wLNyLJ5VHXWM+tnTX4K9//SsmTJiA+Ph4DBw4EIsWLUJJSQkmTZrU1FkjIiIiIiKiZoBBdw3GjBmDnJwcvPDCC8jMzETfvn2xadMmu8nViIiIiIiIiBxh0F2L6dOnY/r06U2dDSIiIiIiImqGWnYneyIiIiIiIqImxKCbiIiIiIiIyEUYdBMRERERERG5CINuIiIiIiIiIhdh0E1ERERERETkIgy6iYiIiIiIiFyEQTcRERERERGRizDoJiIiIiIiInIRBt1ERERERERELsKgm4iIiIiIiMhFGHQTERERERERuQiDbiIiIiIiIiIXYdBNRERERERE5CIMuomIiIiIiIhchEE3ERERERERkYsYmjoDLZUQAgBQWFjYxDmpZLFYUFRUBF9fX+h0rG9paiwP98GycC8sD/fBsnAvLA/3wvJwHywL9+JJ5WGN9ayxnzMMul2kqKgIANCpU6cmzgkRERERERG5SlFREQIDA50eV0RtYTldFYvFgvPnz6N169ZQFKWpswNA1sR06tQJZ86cQUBAQFNnx+OxPNwHy8K9sDzcB8vCvbA83AvLw32wLNyLJ5WHEAJFRUWIiIiosVWfLd0uotPp0LFjx6bOhkMBAQEt/gZoTlge7oNl4V5YHu6DZeFeWB7uheXhPlgW7sVTyqOmFm6rlt3JnoiIiIiIiKgJMegmIiIiIiIichEG3R7Ex8cHL774Inx8fJo6KwSWhzthWbgXlof7YFm4F5aHe2F5uA+WhXthedjjRGpERERERERELsKWbiIiIiIiIiIXYdBNRERERERE5CIMuomIiIiIiIhchEG3B3n77bfRuXNn+Pr6IiEhAbt3727qLDV7O3bswJ133omIiAgoioIvvvhCc1wIgRdeeAHt27eHn58fEhMTkZaWpkmTl5eHsWPHIiAgAEFBQXj44YdRXFysSXPw4EHcdNNN8PX1RadOnfCvf/3L1S+t2VmwYAF+97vfoXXr1ggNDcWoUaOQmpqqSXPlyhVMmzYNbdu2hb+/P0aPHo2srCxNmoyMDIwcORKqqiI0NBRPPfUUTCaTJs0PP/yA/v37w8fHB127dsXKlStd/fKalaVLl6J379629TkHDRqEjRs32o6zHJrWwoULoSgKZs2aZdvHMmk8c+fOhaIomp/Y2FjbcZZF4zp37hzGjRuHtm3bws/PD7169cKePXtsx/k93ng6d+5sd28oioJp06YB4L3RmMxmM55//nlER0fDz88PXbp0wUsvvYSqU4Hx3qgnQR5h9erVwtvbW7z33nvi8OHD4pFHHhFBQUEiKyurqbPWrH3zzTfi2WefFevWrRMAxOeff645vnDhQhEYGCi++OILceDAAXHXXXeJ6OhocfnyZVua22+/XfTp00f88ssv4scffxRdu3YVDzzwgO14QUGBCAsLE2PHjhUpKSnik08+EX5+fmLZsmWN9TKbheHDh4v3339fpKSkiOTkZHHHHXeIyMhIUVxcbEszdepU0alTJ/H999+LPXv2iBtuuEEMHjzYdtxkMom4uDiRmJgo9u/fL7755hsREhIi/v73v9vSpKenC1VVxV//+ldx5MgR8eabbwq9Xi82bdrUqK/Xna1fv158/fXX4tixYyI1NVXMmTNHeHl5iZSUFCEEy6Ep7d69W3Tu3Fn07t1bzJw507afZdJ4XnzxRdGzZ09x4cIF209OTo7tOMui8eTl5YmoqCgxceJEkZSUJNLT08XmzZvF8ePHbWn4Pd54srOzNffFli1bBACxbds2IQTvjcY0b9480bZtW/HVV1+JkydPik8//VT4+/uLxYsX29Lw3qgfBt0eYuDAgWLatGm2bbPZLCIiIsSCBQuaMFctS/Wg22KxiPDwcPHqq6/a9uXn5wsfHx/xySefCCGEOHLkiAAgfv31V1uajRs3CkVRxLlz54QQQixZskQEBweL0tJSW5rZs2eL7t27u/gVNW/Z2dkCgNi+fbsQQr73Xl5e4tNPP7WlOXr0qAAgdu3aJYSQlSg6nU5kZmba0ixdulQEBATY3v+nn35a9OzZU/NcY8aMEcOHD3f1S2rWgoODxYoVK1gOTaioqEh069ZNbNmyRdxyyy22oJtl0rhefPFF0adPH4fHWBaNa/bs2WLIkCFOj/N7vGnNnDlTdOnSRVgsFt4bjWzkyJFi8uTJmn333nuvGDt2rBCC98bVYPdyD1BWVoa9e/ciMTHRtk+n0yExMRG7du1qwpy1bCdPnkRmZqbmfQ8MDERCQoLtfd+1axeCgoIQHx9vS5OYmAidToekpCRbmptvvhne3t62NMOHD0dqaiouXbrUSK+m+SkoKAAAtGnTBgCwd+9elJeXa8ojNjYWkZGRmvLo1asXwsLCbGmGDx+OwsJCHD582Jam6jWsaXgvOWY2m7F69WqUlJRg0KBBLIcmNG3aNIwcOdLufWOZNL60tDREREQgJiYGY8eORUZGBgCWRWNbv3494uPjcd999yE0NBT9+vXDu+++azvO7/GmU1ZWho8//hiTJ0+Goii8NxrZ4MGD8f333+PYsWMAgAMHDmDnzp0YMWIEAN4bV4NBtwfIzc2F2WzW/BECgLCwMGRmZjZRrlo+63tb0/uemZmJ0NBQzXGDwYA2bdpo0ji6RtXnIC2LxYJZs2bhxhtvRFxcHAD5Xnl7eyMoKEiTtnp51PZeO0tTWFiIy5cvu+LlNEuHDh2Cv78/fHx8MHXqVHz++efo0aMHy6GJrF69Gvv27cOCBQvsjrFMGldCQgJWrlyJTZs2YenSpTh58iRuuukmFBUVsSwaWXp6OpYuXYpu3bph8+bN+POf/4wZM2bggw8+AMDv8ab0xRdfID8/HxMnTgTAv1ON7ZlnnsGf/vQnxMbGwsvLC/369cOsWbMwduxYALw3roahqTNARNTQpk2bhpSUFOzcubOps+KxunfvjuTkZBQUFGDt2rWYMGECtm/f3tTZ8khnzpzBzJkzsWXLFvj6+jZ1djyetaUIAHr37o2EhARERUVhzZo18PPza8KceR6LxYL4+HjMnz8fANCvXz+kpKTgnXfewYQJE5o4d57tP//5D0aMGIGIiIimzopHWrNmDf773/9i1apV6NmzJ5KTkzFr1ixERETw3rhKbOn2ACEhIdDr9XYzPGZlZSE8PLyJctXyWd/bmt738PBwZGdna46bTCbk5eVp0ji6RtXnoErTp0/HV199hW3btqFjx462/eHh4SgrK0N+fr4mffXyqO29dpYmICCA/zBX4e3tja5du2LAgAFYsGAB+vTpg8WLF7McmsDevXuRnZ2N/v37w2AwwGAwYPv27XjjjTdgMBgQFhbGMmlCQUFBuO6663D8+HHeH42sffv26NGjh2bf9ddfb+vuz+/xpnH69Gl89913mDJlim0f743G9dRTT9lau3v16oXx48fjiSeesPWW4r1Rfwy6PYC3tzcGDBiA77//3rbPYrHg+++/x6BBg5owZy1bdHQ0wsPDNe97YWEhkpKSbO/7oEGDkJ+fj71799rSbN26FRaLBQkJCbY0O3bsQHl5uS3Nli1b0L17dwQHBzfSq3F/QghMnz4dn3/+ObZu3Yro6GjN8QEDBsDLy0tTHqmpqcjIyNCUx6FDhzRfElu2bEFAQIDtH7NBgwZprmFNw3upZhaLBaWlpSyHJjB06FAcOnQIycnJtp/4+HiMHTvW9phl0nSKi4tx4sQJtG/fnvdHI7vxxhvtlpY8duwYoqKiAPB7vKm8//77CA0NxciRI237eG80LqPRCJ1OGybq9XpYLBYAvDeuSlPP5EaNY/Xq1cLHx0esXLlSHDlyRDz66KMiKChIM8Mj1V9RUZHYv3+/2L9/vwAg/v3vf4v9+/eL06dPCyHkcgpBQUHiyy+/FAcPHhR33323w+UU+vXrJ5KSksTOnTtFt27dNMsp5Ofni7CwMDF+/HiRkpIiVq9eLVRVbZHLKVyLP//5zyIwMFD88MMPmiVHjEajLc3UqVNFZGSk2Lp1q9izZ48YNGiQGDRokO24dbmRYcOGieTkZLFp0ybRrl07h8uNPPXUU+Lo0aPi7bff5nIj1TzzzDNi+/bt4uTJk+LgwYPimWeeEYqiiG+//VYIwXJwB1VnLxeCZdKYnnzySfHDDz+IkydPip9++kkkJiaKkJAQkZ2dLYRgWTSm3bt3C4PBIObNmyfS0tLEf//7X6Gqqvj4449tafg93rjMZrOIjIwUs2fPtjvGe6PxTJgwQXTo0MG2ZNi6detESEiIePrpp21peG/UD4NuD/Lmm2+KyMhI4e3tLQYOHCh++eWXps5Ss7dt2zYBwO5nwoQJQgi5pMLzzz8vwsLChI+Pjxg6dKhITU3VXOPixYvigQceEP7+/iIgIEBMmjRJFBUVadIcOHBADBkyRPj4+IgOHTqIhQsXNtZLbDYclQMA8f7779vSXL58WfzlL38RwcHBQlVVcc8994gLFy5ornPq1CkxYsQI4efnJ0JCQsSTTz4pysvLNWm2bdsm+vbtK7y9vUVMTIzmOUiIyZMni6ioKOHt7S3atWsnhg4dagu4hWA5uIPqQTfLpPGMGTNGtG/fXnh7e4sOHTqIMWPGaNaFZlk0rg0bNoi4uDjh4+MjYmNjxfLlyzXH+T3euDZv3iwA2L3HQvDeaEyFhYVi5syZIjIyUvj6+oqYmBjx7LPPapb24r1RP4oQQjRJEzsRERERERFRC8cx3UREREREREQuwqCbiIiIiIiIyEUYdBMRERERERG5CINuIiIiIiIiIhdh0E1ERERERETkIgy6iYiIiIiIiFyEQTcRERERERGRizDoJiIiIiIiInIRBt1ERETkcj/88AMURcHatWubOitERESNikE3ERFRC7Jy5UooigJfX1+cO3fO7vjvf/97xMXFNUHOiIiIPBODbiIiohaotLQUCxcubOpsEBEReTwG3URERC1Q37598e677+L8+fNNnZVGVVJS0tRZICIi0mDQTURE1ALNmTMHZrO5xtbuU6dOQVEUrFy50u6YoiiYO3eubXvu3LlQFAXHjh3DuHHjEBgYiHbt2uH555+HEAJnzpzB3XffjYCAAISHh+P11193+Jxmsxlz5sxBeHg4WrVqhbvuugtnzpyxS5eUlITbb78dgYGBUFUVt9xyC3766SdNGmuejhw5ggcffBDBwcEYMmRI3d4gIiKiRsKgm4iIqAWKjo7GQw891OCt3WPGjIHFYsHChQuRkJCAl19+GYsWLcJtt92GDh064JVXXkHXrl3xt7/9DTt27LA7f968efj6668xe/ZszJgxA1u2bEFiYiIuX75sS7N161bcfPPNKCwsxIsvvoj58+cjPz8ft956K3bv3m13zfvuuw9GoxHz58/HI4880mCvlYiIqCEYmjoDRERE5BrPPvssPvzwQ7zyyitYvHhxg1xz4MCBWLZsGQDg0UcfRefOnfHkk09iwYIFmD17NgDggQceQEREBN577z3cfPPNmvPz8vJw9OhRtG7dGgDQv39/3H///Xj33XcxY8YMCCEwdepU/OEPf8DGjRuhKAoA4LHHHkPPnj3x3HPP4dtvv9Vcs0+fPli1alWDvD4iIqKGxpZuIiKiFiomJgbjx4/H8uXLceHChQa55pQpU2yP9Xo94uPjIYTAww8/bNsfFBSE7t27Iz093e78hx56yBZwA8Af//hHtG/fHt988w0AIDk5GWlpaXjwwQdx8eJF5ObmIjc3FyUlJRg6dCh27NgBi8WiuebUqVMb5LURERG5AoNuIiKiFuy5556DyWRqsJnMIyMjNduBgYHw9fVFSEiI3f5Lly7Znd+tWzfNtqIo6Nq1K06dOgUASEtLAwBMmDAB7dq10/ysWLECpaWlKCgo0FwjOjr6Wl8WERGRy7B7ORERUQsWExODcePGYfny5XjmmWc0x6xdt6szm81Or6fX6+u0DwCEEPXIqWRtxX711VfRt29fh2n8/f01235+fvV+HiIiosbCoJuIiKiFe+655/Dxxx/jlVde0ewPDg4GAOTn52v2nz592mV5sbZkWwkhcPz4cfTu3RsA0KVLFwBAQEAAEhMTXZYPIiKixsLu5URERC1cly5dMG7cOCxbtgyZmZm2/QEBAQgJCbGbZXzJkiUuy8uHH36IoqIi2/batWtx4cIFjBgxAgAwYMAAdOnSBa+99hqKi4vtzs/JyXFZ3oiIiFyBLd1EREQe4Nlnn8VHH32E1NRU9OzZ07Z/ypQpWLhwIaZMmYL4+Hjs2LEDx44dc1k+2rRpgyFDhmDSpEnIysrCokWL0LVrV9tSXzqdDitWrMCIESPQs2dPTJo0CR06dMC5c+ewbds2BAQEYMOGDS7LHxERUUNj0E1EROQBunbtinHjxuGDDz7Q7H/hhReQk5ODtWvXYs2aNRgxYgQ2btyI0NBQl+Rjzpw5OHjwIBYsWICioiIMHToUS5YsgaqqtjS///3vsWvXLrz00kt46623UFxcjPDwcCQkJOCxxx5zSb6IiIhcRRFXM8sJEREREREREdWKY7qJiIiIiIiIXIRBNxEREREREZGLMOgmIiIiIiIichEG3UREREREREQuwqCbiIiIiIiIyEUYdBMRERERERG5CINuIiIiIiIiIhdh0E1ERERERETkIgy6iYiIiIiIiFyEQTcRERERERGRizDoJiIiIiIiInIRBt1ERERERERELsKgm4iIiIiIiMhF/j/WYXFsSz0WSwAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "from collections import Counter\n", - "\n", - "# 统计频次\n", - "count_result = Counter(values)\n", - "# 对x轴的数字进行排序，使折线图更连贯\n", - "unique_nums = sorted(count_result.keys())\n", - "frequencies = [count_result[num] for num in unique_nums]\n", - "\n", - "# 创建画布\n", - "plt.figure(figsize=(10, 6))\n", - "\n", - "# 绘制折线图，添加标记点使数据更清晰\n", - "# plt.plot(unique_nums, frequencies, marker='o', linestyle='-', color='b', markersize=8, linewidth=2)\n", - "plt.plot(unique_nums, frequencies, \n", - " marker='.', # 圆形标记点\n", - " linestyle='-.', # 不显示连接线\n", - " color='b', # 点的颜色\n", - " markersize=3, # 点的大小\n", - " markerfacecolor='red', # 点的填充色\n", - " markeredgecolor='red') # 点的边缘色\n", - "\n", - "# 添加标题和坐标轴标签\n", - "plt.title('Frequency of Each Number (Line Plot)', fontsize=15)\n", - "plt.xlabel('Number', fontsize=12)\n", - "plt.ylabel('Frequency', fontsize=12)\n", - "\n", - "# # 为每个点添加数值标签\n", - "# for x, y in zip(unique_nums, frequencies):\n", - "# plt.text(x, y + 0.1, str(y), ha='center', fontsize=10)\n", - "\n", - "# 添加网格线，便于查看数值\n", - "plt.grid(alpha=0.3)\n", - "\n", - "# 调整布局\n", - "plt.tight_layout()\n", - "\n", - "# 显示图像" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "8878790f-2c44-40c9-9d66-7901dcf6537c", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAA94AAAJOCAYAAABBfN/cAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAZaxJREFUeJzt3Xd0VFX/9uF7QkgjCaTTEnqHUEKLgHQBEcQughKwoIICigWVZgPFrogiSFEQAQXLI00gIB1CLyK9dyQJLYHMfv/gZX6OSaRMDjMJn2utrOXsveec7z4ng3PnNJsxxggAAAAAAFjCy90FAAAAAACQlxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBII+w2WxOP/nz51d4eLiqVaumhIQE/fDDD7p48aK7y7xuTZo0cZpfvnz5FBISorJly+qee+7R8OHDlZycfMX37969+8YV/S+JiYmy2WxKSEhwah80aJBsNpvGjh3rlrr+qWTJkrLZbO4uI5PL22jQoEH/Oe5y/f/ez546LwDAzYHgDQB5TJcuXdSlSxd17NhRDRo00MWLFzV+/Hjde++9qlSpklasWJEj6xk7duxVBaGc1qpVK3Xp0kUPP/ywmjVrptDQUP3yyy/q2bOnoqOjLQuvu3fvls1mU5MmTSxZ/o2QF+bgLtn90QQAgKvh7e4CAAA5K6vguWPHDr3yyiuaPHmymjZtqsWLF6tGjRo3vLac8PLLL2cKjsnJyfrwww/15ptvqmvXrrpw4YIef/xxpzHjx4/X2bNnVaxYsRtYrbO6detqy5YtKliwoNtquJK5c+fqwoUL7i4jx+XVeQEAcgeOeAPATaBMmTL6/vvv9eijj+rs2bPq1q2bu0vKUQULFtSgQYMcf3R49tlndeTIEacxMTExqlixovLnz++GCi8JCAhQxYoVVaRIEbfVcCVlypRRxYoV3V1Gjsur8wIA5A4EbwC4ibz//vsqUKCA1qxZo0WLFjn1/e9//1O3bt1UqVIlBQcHq0CBAqpevbrefvttpaWlOY1t0qSJunbtKkkaPHiw07XXl8OvMUbfffedHnzwQZUvX14FChRQUFCQ6tatq88//1x2uz3H59e5c2c1bNhQ58+f11dffZWp5qyu/d2zZ4+eeuoplS9fXgEBAQoNDVWVKlXUvXt3bd26VdKl64tLlSolSVqwYIHTfP956rHNZlPJkiWVnp6u119/XRUrVpSvr686dOgg6epOV16+fLlatWqlQoUKKTg4WC1bttSyZcsyjbvSqf7/nu/VzuG/roVeunSp7rzzTkVERMjX11clS5bU008/rYMHD/5nfXv37tVDDz2kiIgI+fv7q3bt2vrll1+y3QZWyG5eGzduVOfOnVW6dGn5+fkpIiJCNWrUUO/evXXo0CFJUkJCgpo2bSpJGjdunNO2+/f2v5ZtJEkXLlzQO++8owoVKsjPz08xMTF67rnndPr06Sx/Z/95uUBKSoqee+45lSpVSvnz51fv3r0lSadOndKnn36qVq1aqUSJEvL19VVYWJhat26tOXPmZFnHP9f1/fffq06dOgoICFCxYsX04osvKj09XdKls2c6duyoyMhIBQQEqGnTplq/fv217AoAuClxqjkA3EQKFiyoNm3aaOrUqZo/f74aNmzo6Hv00Ud17tw5Va1aVbGxsUpOTtaKFSv06quvau7cuZo9e7by5csnSWrdurUuXryoxYsXq3r16k6nrZctW1aSlJaWpoceekhhYWGqXLmyatWqpRMnTmjJkiXq0aOHVqxYYcn12A8++KAWLVqk+fPn67XXXvvPsfv27VOtWrV08uRJlStXTrfffrsyMjK0Z88effXVV4qPj1eFChVUo0YN3XPPPfrhhx8UFRWl1q1bO5bxz20oSXa7XR06dNDChQvVuHFjxcbGKiws7KpqX7Jkibp3766yZcuqTZs22r59u37//XctXLhQv/zyi2677bZr3yD/37XMISvffvutEhISlJGRoQYNGig6OlqrV6/WiBEj9OOPPyoxMTHLI8q7d+9WnTp1FBQUpObNm2vv3r1aunSpOnTooBkzZrg0J1clJSU5/lATGxurO++8U2fPntXOnTv18ccfq0OHDipSpIgaNmyow4cPa9asWSpTpozT9vrn7/61biNjjB544AFNmzZNBQoU0G233ab8+fNrzJgxWrRokby9s/+adu7cOTVu3Fh79uxR48aNVatWLYWEhEiSli1bpmeffVYlS5ZUhQoVFB8fr71792r27NmaPXu2Ro0ale1ZLx9//LE+++wzNWnSRK1bt9Yff/yhYcOG6ciRI3rttdd0yy23KDw8XM2aNdPmzZuVmJiopk2bavPmzYqKinJxjwBAHmYAAHmCJHM1/6y/+eabRpLp2LGjU/v06dPN2bNnndpSUlLMHXfcYSSZcePGOfWNGTPGSDIDBw7Mcj0XLlww06ZNM+np6U7tR48eNbVr1zaSzIIFC65iZpc0btzYSDLz58//z3GLFi0ykkyRIkWyfP+uXbscbQMGDDCSTM+ePTMtZ8+ePWb79u2O17t27TKSTOPGjbNd9+V9ULZsWbN///5M/fPnzzeSTJcuXZzaBw4c6Hjvq6++aux2u6Pv888/d8znn/vnSts/q/lezRxKlCiR6fdo7969xt/f3+TLl8/89NNPjvaMjAzTu3dvI8nUrl3b6T2X65Nknn/+eZORkeHo+/DDD40k06hRo2zr+LfL2yi7+f67/n/OO7t5PfLII0aSee+99zItZ8uWLebgwYOO19ntu8uuZxt98803RpIpVaqU2bdvn6P9+PHjpkaNGo7tl9U+lGTi4+PN33//namWnTt3mqVLl2ZqX716tSlUqJAJDg42qampTn2Xf18CAwPNypUrHe2HDh0yUVFRxmazmUqVKpmXX37Z8ftpt9vNww8/bCSZAQMGZLldAACXcKo5ANxkwsPDJUl///23U/udd94pf39/p7agoCB9+OGHkqSffvrpmtbj7e2tDh06ZLqmOiIiQkOGDLmuZV6N7OaXlWPHjkmSWrRokakvJiZGZcqUua4ahgwZcl03cStRooTjsVmXPfXUU6pXr54OHTqkH3744brqcdWoUaN07tw53X///Wrfvr2j3cvLS0OHDlXRokW1atUqLV68ONN7S5UqpbffflteXv/3laNnz54KCQnRsmXLHKcwX61/X9rw7589e/Zc9bL+a/9f67X417ONvvjiC0nS66+/ruLFizvaw8LCNGzYsCuu85NPPlGhQoUytZcqVUr169fP1F6zZk316NFDKSkpmj9/fpbL7N27t2rXru14XbhwYT300EMyxigtLU2vv/664/fTZrOpb9++ki5dvgAAyB6nmgPATcYYI0lZXu+6bds2/fbbb9q+fbvOnDkju93uGL9t27brWt/atWs1e/Zs7dmzR2fPnpUxRqmpqS4t87/81/z+LS4uTpL0yiuvKF++fGrRooX8/PxcWr/NZlO7du2u67333HNPlqcXd+zYUcuXL9cff/yhzp07u1Tf9fjjjz8kSZ06dcrU5+vrq/vuu08ff/yx/vjjDzVo0MCpv0mTJvLx8XFq8/b2VqlSpbR69WqdOHHimgLuvy9t+LepU6fqzJkzV7WsuLg4zZgxQz169NCbb76phg0b/ufp3f/lWrfRhQsXtHLlStlsNt17772Z3tOiRQuFhobq5MmTWa6vSJEiTgH53zIyMjR37lwtWbJEhw4dctyn4fJnLrvPXlan/pcuXVrSpX357z+kXe67fD08ACBrBG8AuMkcP35ckhQaGupoM8aob9+++vDDDx3B9d8uh+WrlZ6eroSEBH333XfZjrnWZV6NrOaXnYSEBM2ePVuTJ09Wu3bt5Ofnpzp16qh169bq1q2bChcufM3rj4yMlK+v7zW/T7p0xDsrJUuWlKRsb9BltcvrvVzHv11uP3DgQKa+fx7J/aegoCBJynTjvivp0KHDfz47PjEx8aqD9wsvvKBFixY5rlMODAxUfHy82rZtq4SEhGt67Nu1bqMTJ04oPT1dERER2f6xJyYmJtvgHRMTk20t+/fv1x133KF169ZlOya7z15WZ2oEBgZese9a9yMA3Gzy7KnmCxcuVLt27VS0aFHZbDZNnz79mpdhjNF7772n8uXLy9fXV8WKFdNbb72V88UCwA20Zs0aSVLlypUdbd9//70++OADFS9eXFOnTtWBAweUnp7uOL1UUraBPDsffPCBvvvuO1WrVk0zZszQkSNHHMu8fLfwa13m1chqftnJly+fvv/+e61evVoDBw5UnTp1tHz5cr366qsqX768lixZcs3rd/WIeU6x4q7x2fmvswv+eYq5pwkODta8efP0xx9/6MUXX1TlypU1b9489e7dWxUqVMjRMzKu5gyMa/Ffv2ePPfaY1q1bp3vuuUfLly/XqVOnlJGRIWOMvvzyS0nZf/b+a3958r4EAE+XZ/8FPXPmjKpXr67hw4df9zJ69eqlUaNG6b333tOff/6pn3/+WXXr1s3BKgHgxkpOTtasWbMkyfF4JEmaNm2aJGnEiBG65557VLRoUccppTt37ryudV1e5nfffafWrVsrMjLS5WVeje+//16S8/yupGbNmho0aJAWLlyoY8eOqU+fPkpNTXU8nulGye765MvtRYsWdbRdPn379OnTWb5n3759OVbX5fVmV9/lx11dz3Xt7maz2dSwYUO98847Wr58uQ4ePKiOHTvqyJEjevXVV696Ode6jcLCwpQ/f34dP35c58+fz/I917MPz5w5ozlz5igqKkrff/+96tatq4IFCzpCs5WfPQBA9vJs8G7Tpo3efPNN3XXXXVn2p6WlqW/fvipWrJgKFCigevXqKTEx0dG/ZcsWjRgxQj/99JPat2+vUqVKKS4uTi1btrxBMwCAnPf888/rzJkzqlOnjuLj4x3tl29EltVpwZMnT85yWZeD38WLF7Psv55luuqbb77R4sWLFRAQoMcee+y6lhEcHKwhQ4bIZrNp48aNjvYrzTcn/Pjjj8rIyMjUPmnSJEnOj/26fF30X3/9lWn8X3/9pb1792Zqv945NGrUSJKyvGwgPT1dU6ZMcRqXm0VGRjpOZb+W/X+t2yh//vyqW7eujDH68ccfM71n3rx5OnHixDXXn5ycLLvdriJFijge/3fZhQsXHH8QAwDcWHk2eF9Jz549tXTpUk2aNEnr16/Xfffdp9atWztOK/vll19UunRp/frrrypVqpRKliypxx57LNtrrQDAk+3cuVMPPPCARo8erQIFCmj06NFO/eXLl5ckjRw50ukU1MvP8M3K5SN8l08b/7fLy7x85+bLpk6dqvHjx1/fRLKRnJyswYMHq2vXrpKkzz77TBEREVd83zfffOMUri6bMWOGjDGKjo52tIWHhyt//vzasWNHluE4J+zevVuDBw92ahs5cqSWLl2qqKgo3XPPPY72OnXqKCAgQDNmzFBSUpKj/fjx43rssceyPNX8eufw6KOPyt/fX5MmTdL//vc/R7vdbtcrr7yiAwcOKC4uLtON1TzdF198oV27dmVq/+233yTJaf9f6ff9erbRk08+KUkaMGCA0/XxJ0+e1AsvvHBdc4qMjFTBggW1ceNGpzuoZ2Rk6KWXXsryDzUAAOvdlDdX27t3r8aMGaO9e/c6/kfat29fzZw5U2PGjNHbb7+tnTt3as+ePZoyZYrGjx+vjIwM9enTR/fee6/mzZvn5hkAQPYSEhIkXfrCn5KSor/++kt//vmnjDEqV66cJk6cqGrVqjm959lnn9XYsWP1+eefKzExUbGxsTpw4IAWLVqk559/Xu+9916m9dSvX1+RkZGaOnWqmjRpotKlS8vLy0vdunXTLbfcohdffFEzZ87Uyy+/rClTpqh8+fLatm2bVq1apb59+2a5zKsxdOhQjR07VtKl06z379+vNWvWKD09XcHBwfrss8/08MMPX9WyfvjhBz3yyCMqU6aMqlWrJn9/f+3atUvLly+Xl5eX3nzzTcdYHx8ftW7dWr/88ouqV6+uWrVqycfHRw0aNHAEflc9/vjjGjp0qH788UfFxsZq+/btWrlypfLnz6+xY8cqICDAMTYwMFB9+/bV66+/roYNG6px48ay2Wxavny5KlWqpPj4eC1dutRp+dc7h5iYGH355ZdKSEhQu3bt1KBBA0VHR2v16tXaunWroqKi9O233+bINriRvvjiCz311FOqXLmyKlWqJG9vb/35559at26d/Pz8NGDAAMfYkiVLKjY2VqtWrVLdunVVpUoV5cuXT+3bt1f79u2vaxt16tRJP/74o6ZNm6aKFSuqefPmypcvn+bPn68yZcqofv36WrZsWaa7wv8Xb29vvfjii3r11VfVuHFjNWvWTKGhoVq+fLmOHDmiHj16uHQZHgDgOt3oB4e7gyQzbdo0x+tff/3VSDIFChRw+vH29jb333+/McaYxx9/3EgyW7dudbwvKSnJSDJ//vnnjZ4CAFyRJKcfb29vExoaaqpWrWq6dOlifvzxR3Px4sVs379lyxbTrl07ExkZaQICAkzNmjXNyJEjHcsuUaJEpvesXLnStGzZ0hQsWNDYbDYjyYwZM8bRv3TpUtOsWTMTEhJigoKCzC233GJ++OEHs2vXLiPJNG7c+Krn17hxY6f5eXl5mYIFC5rSpUubu+++2wwfPtwkJydf8f27du1ytC1YsMD06NHD1KhRw4SFhRk/Pz9TunRp8+CDD5qVK1dmWsaRI0fMww8/bAoXLmzy5ctnJJkuXbo4+rPbTpfNnz8/03uMMWbgwIGObbdkyRLTvHlzExQUZAIDA03z5s3N4sWLs1ye3W43w4YNM2XLljX58+c3xYsXN88//7w5c+ZMlvO9mjmUKFHCZPf1YPHixaZdu3YmLCzM5M+f38TExJinnnrK7N+/P9PYMWPGGElm4MCBWS4ru/qyc3kbZbe8f9f/7+VmNa+ff/7ZdOvWzVSpUsUUKlTIBAQEmPLly5vHHnssy//Xb9u2zXTo0MGEhYUZLy+vLOu5lm1kjDHp6elmyJAhply5csbHx8cUK1bMPPPMMyYlJcWULVvW2Gw2c/bsWcf4q/3sjBs3ztSsWdMEBASYsLAwc+edd5p169Zlu1/+a39caV9e6fceAGCMzRgLbinrYWw2m6ZNm6YOHTpIunTjnU6dOmnTpk2Zrn8KDAxU4cKFNXDgQL399tu6cOGCo+/cuXMKCAjQ7NmzudYbAABYZv/+/SpVqpTKli2rLVu2uLscAICLbspTzWvWrKmMjAwdPXo02xvBNGjQQBcvXtSOHTtUpkwZSf93A5vsnrMKAABwLdavX69KlSo57vgvSUeOHFFCQoIuXryozp07u7E6AEBOybNHvE+fPq3t27dLuhS0P/jgAzVt2lShoaGKiYlR586dtXjxYr3//vuqWbOmjh07prlz5yo2NlZt27aV3W5XnTp1FBgYqI8++kh2u109evRQcHCwZs+e7ebZAQCAvKB169ZasWKFatSooaioKB06dEhJSUk6ffq06tSpo4ULF3rMs+EBANcvzwbvxMTELJ/h2qVLF40dO1YXLlzQm2++qfHjx+vAgQMKDw9X/fr1NXjwYMdNhw4ePKhnnnlGs2fPVoECBdSmTRu9//77Cg0NvdHTAQAAedCkSZM0ZswYbdiwQSdPnlS+fPlUvnx53XvvverTp4/TDfUAALlXng3eAAAAAAB4gpv2Od4AAAAAANwIBG8AAAAAACyUp+5qbrfbdfDgQQUFBclms7m7HAAAAABAHmWMUWpqqooWLSovr/8+pp2ngvfBgwcVHR3t7jIAAAAAADeJffv2qXjx4v85Jk8F76CgIEmXJh4cHOzmagAAAAAAeVVKSoqio6MdOfS/5Kngffn08uDgYII3AAAAAMByV3OZMzdXAwAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEb+S4QYMGyWazOf1UrFjR0X/+/Hn16NFDYWFhCgwM1D333KMjR444LWPv3r1q27atAgICFBkZqRdeeEEXL150GjNhwgRVr15dAQEBKlKkiLp166YTJ044+r/66is1atRIISEhCgkJUYsWLbRixQqnZSQkJGSqtXXr1k5jVq9erZYtW6pQoUIKCwvTE088odOnT+fU5gIAAACQxxG8YYkqVaro0KFDjp9FixY5+vr06aNffvlFU6ZM0YIFC3Tw4EHdfffdjv6MjAy1bdtW6enpWrJkicaNG6exY8dqwIABjjGLFy/WI488okcffVSbNm3SlClTtGLFCj3++OOOMYmJierYsaPmz5+vpUuXKjo6WrfddpsOHDjgVGvr1q2dav3uu+8cfQcPHlSLFi1UtmxZLV++XDNnztSmTZuUkJBgwVYDAAAAkBd5u7sA5E3e3t4qXLhwpvbk5GSNHj1aEydOVLNmzSRJY8aMUaVKlbRs2TLVr19fs2fP1ubNm/X7778rKipKNWrU0BtvvKGXXnpJgwYNko+Pj5YuXaqSJUvq2WeflSSVKlVK3bt31zvvvONY14QJE5zWPWrUKP3www+aO3euHnnkEUe7r69vlrVK0q+//qr8+fNr+PDh8vK69HeqL774QrGxsdq+fbvKli3r2oYCAAAAkOdxxBuW2LZtm4oWLarSpUurU6dO2rt3ryQpKSlJFy5cUIsWLRxjK1asqJiYGC1dulSStHTpUlWrVk1RUVGOMa1atVJKSoo2bdokSYqPj9e+ffv022+/yRijI0eOaOrUqbr99tuzrens2bO6cOGCQkNDndoTExMVGRmpChUq6KmnnnI6XT0tLU0+Pj6O0C1J/v7+kuR0FB8AAAAAskPwRo6rV6+exo4dq5kzZ2rEiBHatWuXGjVqpNTUVB0+fFg+Pj4qVKiQ03uioqJ0+PBhSdLhw4edQvfl/st9ktSgQQNNmDBBDzzwgHx8fFS4cGEVLFhQw4cPz7aul156SUWLFnUK/a1bt9b48eM1d+5cvfPOO1qwYIHatGmjjIwMSVKzZs10+PBhDRs2TOnp6fr777/18ssvS5IOHTrk2oYCAAAAcFMgeCPHtWnTRvfdd59iY2PVqlUr/fbbbzp16pQmT56cY+vYvHmzevXqpQEDBigpKUkzZ87U7t279eSTT2Y5fujQoZo0aZKmTZsmPz8/R/uDDz6o9u3bq1q1aurQoYN+/fVXrVy5UomJiZIuXas+btw4vf/++woICFDhwoVVqlQpRUVFOR0FBwAAAIDseFRyuNLdsJE7FSpUSOXLl9f27dtVuHBhpaen69SpU05jjhw54rjOunDhwpnucn759eUxQ4YMUYMGDfTCCy84Av7nn3+ur7/+OtOR6Pfee09Dhw7V7NmzFRsb+5+1li5dWuHh4dq+fbuj7aGHHtLhw4d14MABnThxQoMGDdKxY8dUunTp69oeAAAAAG4uHhW8pf++GzZyp9OnT2vHjh0qUqSI4uLilD9/fs2dO9fRv3XrVu3du1fx8fGSLl2/vWHDBh09etQxZs6cOQoODlblypUlXbpe+99HnPPlyydJMsY42t5991298cYbmjlzpmrXrn3FWvfv368TJ06oSJEimfqioqIUGBio77//Xn5+fmrZsuU1bAUAAAAANyuPu6t5dnfDRu7Rt29ftWvXTiVKlNDBgwc1cOBA5cuXTx07dlTBggX16KOP6rnnnlNoaKiCg4P1zDPPKD4+XvXr15ck3XbbbapcubIefvhhvfvuuzp8+LBee+019ejRQ76+vpKkdu3a6fHHH9eIESPUqlUrHTp0SL1791bdunVVtGhRSdI777yjAQMGaOLEiSpZsqTj+vDAwEAFBgbq9OnTGjx4sO655x4VLlxYO3bs0IsvvqiyZcuqVatWjvl89tlnuuWWWxQYGKg5c+bohRde0NChQzNdpw4AAAAAWbGZfx4edLNBgwZp2LBhKliwoPz8/BQfH68hQ4YoJibmqt6fkpKiggULKjk5WcHBwRZX65q9e/fq+PHj7i7DEv369dPq1auVnJyskJAQ1ahRQ08//bSio6MlXbpT+IcffqhZs2YpPT1d8fHxevnllxUeHu5YxqFDhzRkyBCtWrVK/v7+uuOOO/TMM8/I2/v//lY0adIk/fDDDzpw4ICCgoJUp04dPfvss4qMjJQk3XHHHVneAO2JJ55Q9+7ddf78eT3//PPaunWrUlNTFRERofr16+upp55SWFiYY/yAAQO0aNEinT17ViVLltTDDz+stm3bWrX5PEZ4ePhVf/YAAACAm8215E+PCt4zZszQ6dOnVaFCBR06dEiDBw/WgQMHtHHjRgUFBWUan5aWprS0NMfrlJQURUdH6++///bo4L1v3z5VqVpF586ec3cpQLb8A/y1aeMmxx9MAAAAAPyflJQUhYSEXFXw9qhTzdu0aeP479jYWNWrV08lSpTQ5MmT9eijj2YaP2TIEA0ePDhT+7Fjx3T+/HlLa3XF/v37VblSZTV5uokKFSvk7nKATE4dOKXEzxO1f/9+x+n9AAAAAP5PamrqVY/1qOD9b/+8G3ZW+vXrp+eee87x+vIR74iICI8+4n3gwAElJSXp1hK3qkD1Au4uB8jkpPdJJSUlyc/Pz3HqPgAAAID/88/HFF+JRwfvy3fDfvjhh7Ps9/X1zfJonJeXl0c/Y9lms8lut0s2XfoBPI1NstvtstlsHv1ZAgAAANzlWr4ne9Q36r59+2rBggXavXu3lixZorvuustxN2wAAAAAAHIjjzrivX//fnXs2FEnTpxQRESEGjZsqGXLlikiIsLdpQEAAAAAcF08KnhPmjTJ3SUAAAAAAJCjPOpUcwAAAAAA8hqCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABby2OA9dOhQ2Ww29e7d292lAAAAAABw3TwyeK9cuVJffvmlYmNj3V0KAAAAAAAu8bjgffr0aXXq1ElfffWVQkJC3F0OAAAAAAAu8bjg3aNHD7Vt21YtWrRwdykAAAAAALjM290F/NOkSZO0evVqrVy58qrGp6WlKS0tzfE6JSVFkmS322W32y2pMScYY+Tl5SUZXfoBPI2RvLy8ZIzx6M8SAAAA4C7X8j3ZY4L3vn371KtXL82ZM0d+fn5X9Z4hQ4Zo8ODBmdqPHTum8+fP53SJOeb8+fOKi4tT6MVQ+Z/0d3c5QCahF0MVFxen8+fP6+jRo+4uBwAAAPA4qampVz3WZozxiGOu06dP11133aV8+fI52jIyMmSz2eTl5aW0tDSnPinrI97R0dH6+++/FRwcfMNqv1Zr1qxR3bp11ef3PoquHu3ucoBM9q3bpw9bfKgVK1aoZs2a7i4HAAAA8DgpKSkKCQlRcnLyFfOnxxzxbt68uTZs2ODU1rVrV1WsWFEvvfRSptAtSb6+vvL19c3U7uXldelUbg9ls9kunZZg06UfwNPYLp06c/kPXwAAAACcXcv3ZI8J3kFBQapatapTW4ECBRQWFpapHQAAAACA3IJDWQAAAAAAWMhjjnhnJTEx0d0lAAAAAADgEo54AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGXgvehQ4dyqg4AAAAAAPIkl4J3dHS0brvtNn3zzTc6c+aMy8WMGDFCsbGxCg4OVnBwsOLj4zVjxgyXlwsAAAAAgLu4FLxff/11HTx4UF26dFFUVJQ6d+6smTNnym63X9fyihcvrqFDhyopKUmrVq1Ss2bNdOedd2rTpk2ulAkAAAAAgNu4FLxfeeUVbdy4UUlJSXryySeVmJio22+/XUWLFlWfPn20atWqa1peu3btdPvtt6tcuXIqX7683nrrLQUGBmrZsmWulAkAAAAAgNt458RCatasqZo1a2rYsGGaN2+eJk6cqDFjxuiTTz5RhQoV1LlzZ3Xu3FkxMTFXvcyMjAxNmTJFZ86cUXx8fJZj0tLSlJaW5nidkpIiSbLb7dd91P1GMMbIy8tLMrr0A3gaI3l5eckY49GfJQAAAMBdruV7co4E78tsNpsaNWqkU6dO6cCBA5o9e7a2bdumQYMGacCAAbrrrrv0ySefqEiRItkuY8OGDYqPj9f58+cVGBioadOmqXLlylmOHTJkiAYPHpyp/dixYzp//nyOzSunnT9/XnFxcQq9GCr/k/7uLgfIJPRiqOLi4nT+/HkdPXrU3eUAAAAAHic1NfWqx9qMMTlyzHX+/PmaMGGCfvjhB6WkpKhatWp65JFH1KlTJ3l7e2vMmDF6++23VatWLf3+++/ZLic9PV179+5VcnKypk6dqlGjRmnBggVZhu+sjnhHR0fr77//VnBwcE5MyxJr1qxR3bp11ef3PoquHu3ucoBM9q3bpw9bfKgVK1aoZs2a7i4HAAAA8DgpKSkKCQlRcnLyFfOnS0e8161bpwkTJui7777TwYMHVbhwYT322GN65JFHVK1aNaexffv2lZ+fn/r27fufy/Tx8VHZsmUlSXFxcVq5cqU+/vhjffnll5nG+vr6ytfXN1O7l5fXpVO5PZTNZrt0WoJNl34AT2O7dOqMzWbz6M8SAAAA4C7X8j3ZpeBds2ZN+fv7q0OHDnrkkUfUsmXL/1x5lSpVsr1eOzt2u93pqDYAAAAAALmJS8H766+/1r333qvAwMCrGt+0aVM1bdo02/5+/fqpTZs2iomJUWpqqiZOnKjExETNmjXLlTIBAAAAAHAbl4J3QkJCDpVxydGjR/XII4/o0KFDKliwoGJjYzVr1iy1bNkyR9cDAAAAAMCN4lLw/uSTT/S///0v2yPSbdq0Ufv27fXUU09d1fJGjx7tSjkAAAAAAHgcl+6aNHr06Gwf9SVJlStX1siRI11ZBQAAAAAAuZpLwXvHjh2qVKlStv0VK1bUjh07XFkFAAAAAAC5mkvB28fHR4cPH862/9ChQzyKCAAAAABwU3MpFdevX19jx45Vampqpr7k5GSNGTNG9evXd2UVAAAAAADkai7dXG3gwIFq3LixatSood69e6tKlSqSpI0bN+qjjz7SoUOHNHHixBwpFAAAAACA3Mil4F2vXj398ssv6t69u3r16iWbzSZJMsaoVKlS+vnnnxUfH58jhQIAAAAAkBu5FLwlqWXLltq+fbvWrFnjuJFamTJlVKtWLUcQBwAAAADgZuVy8JYkLy8vxcXFKS4uLicWBwAAAABAnpEjwXvz5s3auXOn/v77bxljMvU/8sgjObEaAAAAAAByHZeC944dO9S5c2etWLEiy8AtSTabjeANAAAAALhpuRS8u3fvrg0bNuijjz5So0aNFBISklN1AQAAAACQJ7gUvBcvXqxXXnlFzzzzTE7VAwAAAABAnuLlypvDw8NVsGDBnKoFAAAAAIA8x6Xg/eSTT+rbb79VRkZGTtUDAAAAAECe4tKp5uXLl1dGRoaqV6+ubt26KTo6Wvny5cs07u6773ZlNQAAAAAA5FouBe8HHnjA8d99+/bNcozNZuOIOAAAAADgpuVS8J4/f35O1QEAAAAAQJ7kUvBu3LhxTtUBAAAAAECe5FLwviwtLU2rV6/W0aNH1aBBA4WHh+fEYgEAAAAAyPVcuqu5JH3yyScqUqSIGjZsqLvvvlvr16+XJB0/flzh4eH6+uuvXS4SAAAAAIDcyqXgPWbMGPXu3VutW7fW6NGjZYxx9IWHh6tZs2aaNGmSy0UCAAAAAJBbuRS833//fd15552aOHGi2rVrl6k/Li5OmzZtcmUVAAAAAADkai4F7+3bt6tNmzbZ9oeGhurEiROurAIAAAAAgFzNpeBdqFAhHT9+PNv+zZs3q3Dhwq6sAgAAAACAXM2l4H377bdr5MiROnXqVKa+TZs26auvvlL79u1dWQUAAAAAALmaS8H7zTffVEZGhqpWrarXXntNNptN48aNU+fOnVW7dm1FRkZqwIABOVUrAAAAAAC5jkvBu2jRokpKSlLr1q31/fffyxijb775Rr/88os6duyoZcuW8UxvAAAAAMBNzdvVBURGRmrUqFEaNWqUjh07JrvdroiICHl5ufyIcAAAAAAAcj2Xg/c/RURE5OTiAAAAAADI9VwK3q+//voVx9hsNvXv39+V1QAAAAAAkGu5FLwHDRqUbZ/NZpMxhuANAAAAALipuXQhtt1uz/Rz8eJF7dixQ3369FHt2rV19OjRnKoVAAAAAIBcJ8fvgObl5aVSpUrpvffeU7ly5fTMM8/k9CoAAAAAAMg1LL31+K233qrffvvNylUAAAAAAODRLA3eq1at4rFiAAAAAICbmks3Vxs/fnyW7adOndLChQv1448/6rHHHnNlFQAAAAAA5GouBe+EhIRs+8LDw/Xyyy9rwIABrqwCAAAAAIBczaXgvWvXrkxtNptNISEhCgoKcmXRAAAAAADkCS4F7xIlSuRUHQAAAAAA5Enc+QwAAAAAAAu5dMTby8tLNpvtmt5js9l08eJFV1YLAAAAAECu4VLwHjBggKZPn65NmzapVatWqlChgiTpzz//1OzZs1W1alV16NAhJ+oEAAAAACBXcil4Fy1aVEePHtXGjRsdofuyLVu2qFmzZipatKgef/xxl4oEAAAAACC3cuka72HDhqlnz56ZQrckVapUST179tS7777ryioAAAAAAMjVXAre+/fvV/78+bPtz58/v/bv3+/KKgAAAAAAyNVcCt5Vq1bV559/rgMHDmTq279/vz7//HNVq1bNlVUAAAAAAJCruXSN94cffqhWrVqpfPnyuuuuu1S2bFlJ0rZt2zR9+nQZY/Ttt9/mSKEAAAAAAORGLgXvhg0bavny5erfv7+mTZumc+fOSZL8/f3VqlUrDR48mCPeAAAAAICbmkvBW7p0uvm0adNkt9t17NgxSVJERIS8vFw6ix0AAAAAgDzB5eB9mZeXl/z8/BQYGEjoBgAAAADg/3M5Ia9atUqtW7dWQECAwsLCtGDBAknS8ePHdeeddyoxMdHVVQAAAAAAkGu5FLyXLFmihg0batu2bercubPsdrujLzw8XMnJyfryyy9dLhIAAAAAgNzKpeD9yiuvqFKlStq8ebPefvvtTP1NmzbV8uXLXVkFAAAAAAC5mkvBe+XKleratat8fX1ls9ky9RcrVkyHDx92ZRUAAAAAAORqLgXv/PnzO51e/m8HDhxQYGCgK6sAAAAAACBXcyl4169fX1OnTs2y78yZMxozZowaN27syioAAAAAAMjVXAregwcP1qpVq9S2bVvNmDFDkrRu3TqNGjVKcXFxOnbsmPr3758jhQIAAAAAkBu59BzvevXq6bffftNTTz2lRx55RJL0/PPPS5LKlCmj3377TbGxsa5XCQAAAABALnXdwdsYo9TUVN1yyy3aunWr1q5dq23btslut6tMmTKKi4vL8oZrAAAAAADcTK47eKenpys0NFRvv/22XnzxRdWoUUM1atTIwdIAAAAAAMj9rvsab19fXxUuXFi+vr45WQ8AAAAAAHmKSzdXS0hI0Pjx45Wenp5T9QAAAAAAkKe4dHO1atWqafr06apSpYoSEhJUsmRJ+fv7Zxp39913u7IaAAAAAAByLZeCd8eOHR3/nd1jw2w2mzIyMlxZDQAAAAAAudY1B+9XXnlFDz74oGJjYzV//nwragIAAAAAIM+45uA9dOhQVa1aVbGxsWrcuLFOnDihyMhIzZkzR82aNbOiRgAAAAAAci2Xbq52mTEmJxYDAAAAAECekyPBGwAAAAAAZI3gDQAAAACAha7rrua7d+/W6tWrJUnJycmSpG3btqlQoUJZjq9Vq9b1VQcAAAAAQC53XcG7f//+mR4f9vTTT2caZ4zhcWIAAAAAgJvaNQfvMWPGWFEHAAAAAAB50jUH7y5dulhRBwAAAAAAeRI3VwMAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCBG8AAAAAACxE8AYAAAAAwEIEbwAAAAAALETwBgAAAADAQgRvAAAAAAAsRPAGAAAAAMBCHhW8hwwZojp16igoKEiRkZHq0KGDtm7d6u6yAAAAAAC4bh4VvBcsWKAePXpo2bJlmjNnji5cuKDbbrtNZ86ccXdpAAAAAABcF293F/BPM2fOdHo9duxYRUZGKikpSbfeequbqgIAAAAA4Pp5VPD+t+TkZElSaGholv1paWlKS0tzvE5JSZEk2e122e126wu8TsYYeXl5SUaXfgBPYyQvLy8ZYzz6swQAAAC4y7V8T/bY4G2329W7d281aNBAVatWzXLMkCFDNHjw4Eztx44d0/nz560u8bqdP39ecXFxCr0YKv+T/u4uB8gk9GKo4uLidP78eR09etTd5QAAAAAeJzU19arHemzw7tGjhzZu3KhFixZlO6Zfv3567rnnHK9TUlIUHR2tiIgIBQcH34gyr8uBAwcunT7vfasKhBZwdzlAJif3nVRSUpL8/PwUGRnp7nIAAAAAj+Pn53fVYz0yePfs2VO//vqrFi5cqOLFi2c7ztfXV76+vpnavby8Lp3K7aFsNtul0xJsuvQDeBrbpbNObDabR3+WAAAAAHe5lu/JHhW8jTF65plnNG3aNCUmJqpUqVLuLgkAAAAAAJd4VPDu0aOHJk6cqJ9++klBQUE6fPiwJKlgwYLy9+daaAAAAABA7uNR55COGDFCycnJatKkiYoUKeL4+f77791dGgAAAAAA18Wjjngbw7O1AAAAAAB5i0cd8QYAAAAAIK8heAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAW8qjgvXDhQrVr105FixaVzWbT9OnT3V0SAAAAAAAu8ajgfebMGVWvXl3Dhw93dykAAAAAAOQIb3cX8E9t2rRRmzZt3F0GAAAAAAA5xqOOeAMAAAAAkNd41BHva5WWlqa0tDTH65SUFEmS3W6X3W53V1lXZIyRl5eXZHTpB/A0RvLy8pIxxqM/SwAAAIC7XMv35FwdvIcMGaLBgwdnaj927JjOnz/vhoquzvnz5xUXF6fQi6HyP+nv7nKATEIvhiouLk7nz5/X0aNH3V0OAAAA4HFSU1OvemyuDt79+vXTc88953idkpKi6OhoRUREKDg42I2V/bcDBw4oKSlJt3rfqgKhBdxdDpDJyX0nlZSUJD8/P0VGRrq7HAAAAMDj+Pn5XfXYXB28fX195evrm6ndy8vr0qncHspms106LcGmSz+Ap7FdOnXGZrN59GcJAAAAcJdr+Z7sUcH79OnT2r59u+P1rl27tHbtWoWGhiomJsaNlQEAAAAAcH08KnivWrVKTZs2dby+fBp5ly5dNHbsWDdVBQAAAADA9fOo4N2kSRMZw22+AQAAAAB5BxdvAgAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCNwAAAAAAFiJ4AwAAAABgIYI3AAAAAAAWIngDAAAAAGAhgjcAAAAAABYieAMAAAAAYCGCN4Cb3ogRIxQbG6vg4GAFBwcrPj5eM2bMcPTv2LFDd911lyIiIhQcHKz7779fR44ccVrGX3/9pTvvvFPh4eEKDg5Ww4YNNX/+fKcxK1euVPPmzVWoUCGFhISoVatWWrdunaP//PnzSkhIULVq1eTt7a0OHTpkqnXRokVq0KCBwsLC5O/vr4oVK+rDDz/M2Q0CAACAHEXwBnDTK168uIYOHaqkpCStWrVKzZo105133qlNmzbpzJkzuu2222Sz2TRv3jwtXrxY6enpateunex2u2MZd9xxhy5evKh58+YpKSlJ1atX1x133KHDhw9Lkk6fPq3WrVsrJiZGy5cv16JFixQUFKRWrVrpwoULkqSMjAz5+/vr2WefVYsWLbKstUCBAurZs6cWLlyoLVu26LXXXtNrr72mkSNHWr+hAAAAcF1sxhjj7iJySkpKigoWLKjk5GQFBwe7u5xsrV69WnFxcXp+/vOKrh7t7nKATPat26f3m76vpKQk1apVy93luEVoaKiGDRum6OhotWnTRn///bfj35Xk5GSFhIRo9uzZatGihY4fP66IiAgtXLhQjRo1kiSlpqYqODhYc+bMUYsWLbRq1SrVqVNHe/fuVXT0pc/9hg0bFBsbq23btqls2bJO609ISNCpU6c0ffr0K9Z69913q0CBAvrmm29ydiMAAAAgW9eSPzniDQD/kJGRoUmTJunMmTOKj49XWlqabDabfH19HWP8/Pzk5eWlRYsWSZLCwsJUoUIFjR8/XmfOnNHFixf15ZdfKjIyUnFxcZKkChUqKCwsTKNHj1Z6errOnTun0aNHq1KlSipZsuR117tmzRotWbJEjRs3dmneAAAAsA7BGwB06ehzYGCgfH199eSTT2ratGmqXLmy6tevrwIFCuill17S2bNndebMGfXt21cZGRk6dOiQJMlms+n333/XmjVrFBQUJD8/P33wwQeaOXOmQkJCJElBQUFKTEzUt99+K39/fwUGBmrmzJmaMWOGvL29r7ne4sWLy9fXV7Vr11aPHj302GOP5ej2AAAAQM4heAOALh2RXrt2rZYvX66nnnpKXbp00ebNmxUREaEpU6bol19+UWBgoAoWLKhTp06pVq1a8vK69E+oMUY9evRQZGSk/vjjD61YsUIdOnRQu3btHOH83LlzevTRR9WgQQMtW7ZMixcvVtWqVdW2bVudO3fumuv9448/tGrVKn3xxRf66KOP9N133+Xo9gAAAEDOufbDLACQB/n4+Dius46Li9PKlSv18ccf68svv9Rtt92mHTt26Pjx4/L29lahQoVUuHBhlS5dWpI0b948/frrr07XgX/++eeaM2eOxo0bp5dfflkTJ07U7t27tXTpUkdgnzhxokJCQvTTTz/pwQcfvKZ6S5UqJUmqVq2ajhw5okGDBqljx445tTkAAACQgwjeAJAFu92utLQ0p7bw8HBJl4L20aNH1b59e0nS2bNnJckRqC/z8vJy3Pn87Nmz8vLyks1mc+q32WxOd0fPqVoBAADgOQjeAG56/fr1U5s2bRQTE6PU1FRNnDhRiYmJmjVrliRpzJgxqlSpkiIiIrR06VL16tVLffr0UYUKFSRJ8fHxCgkJUZcuXTRgwAD5+/vrq6++0q5du9S2bVtJUsuWLfXCCy+oR48eeuaZZ2S32zV06FB5e3uradOmjlo2b96s9PR0nTx5UqmpqVq7dq0kqUaNGpKk4cOHKyYmRhUrVpQkLVy4UO+9956effbZG7S1AAAAcK0I3gCytWXLFneXcENs2bJF48aN0/HjxxUYGKhy5crps88+U1hYmFavXq0FCxbohRdeUHJysooWLaouXbrooYce0urVqx3L+OijjzR8+HA1btxYFy9eVOnSpfX+++8rIyPDMe6DDz7QyJEj9c0338jLy0sVKlTQxx9/rEOHDjmuBb/jjjsc/y1JNWvWlCQlJSVJkvbs2aMPP/xQBw4cUL58+VS8eHH16NFDHTp0cKrnZhAeHq6YmBh3lwEAAHBFPMfbDXiONzzdptmbNOqhUTL2PPPPA/Ig/wB//bnlT8I3AABwi2vJnxzxBpDJueRzMnajzl92VlT5KHeXA2Ry5K8j+rb7tzp+/DjBGwAAeDyCN4BsRZWP4qwMAAAAwEU8xxsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAACQo0qWLCmbzZbpp0ePHtq9e3eWfTabTVOmTMm0rBMnTqh48eKy2Ww6deqUU19aWppeffVVlShRQr6+vipZsqS+/vprR/9XX32lRo0aKSQkRCEhIWrRooVWrFhh9fQBIBNvdxcAAACAvGXlypXKyMhwvN64caNatmyp++67T9HR0Tp06JDT+JEjR2rYsGFq06ZNpmU9+uijio2N1YEDBzL13X///Tpy5IhGjx6tsmXL6tChQ7Lb7Y7+xMREdezYUbfccov8/Pz0zjvv6LbbbtOmTZtUrFixHJwxAPw3gjcAAAByVEREhNProUOHqkyZMmrcuLFsNpsKFy7s1D9t2jTdf//9CgwMdGofMWKETp06pQEDBmjGjBlOfTNnztSCBQu0c+dOhYaGSrp0pP2fJkyY4PR61KhR+uGHHzR37lw98sgjrkwRAK4Jp5oDAADAMunp6fr222/VrVs32Wy2TP1JSUlau3atHn30Uaf2zZs36/XXX9f48ePl5ZX5K+vPP/+s2rVr691331WxYsVUvnx59e3bV+fOncu2lrNnz+rChQuOoA4ANwrBGwCAXGbEiBGKjY1VcHCwgoODFR8f73Q0sEmTJpmunX3yySedlvHss88qLi5Ovr6+qlGjxn+ub/v27QoKClKhQoWc2n/88UfVrl1bhQoVUoECBVSjRg198803OTVN5BHTp0/XqVOnlJCQkGX/6NGjValSJd1yyy2OtrS0NHXs2FHDhg1TTExMlu/buXOnFi1apI0bN2ratGn66KOPNHXqVD399NPZ1vLSSy+paNGiatGihUtzAoBrxanmAADkMsWLF9fQoUNVrlw5GWM0btw43XnnnVqzZo2qVKkiSXr88cf1+uuvO94TEBCQaTndunXT8uXLtX79+mzXdeHCBXXs2FGNGjXSkiVLnPpCQ0P16quvqmLFivLx8dGvv/6qrl27KjIyUq1atcqh2SK3Gz16tNq0aaOiRYtm6jt37pwmTpyo/v37O7X369dPlSpVUufOnbNdrt1ul81m04QJE1SwYEFJ0gcffKB7771Xn3/+ufz9/Z3GDx06VJMmTVJiYqL8/PxyYGYAcPUI3gAA5DLt2rVzev3WW29pxIgRWrZsmSN4BwQEZLqO9p8++eQTSdKxY8f+M3i/9tprqlixopo3b54peDdp0sTpda9evTRu3DgtWrSI4A1J0p49e/T777/rxx9/zLJ/6tSpOnv2bKbrrefNm6cNGzZo6tSpkiRjjCQpPDxcr776qgYPHqwiRYqoWLFijtAtSZUqVZIxRvv371e5cuUc7e+9956GDh2q33//XbGxsTk9TQC4Ik41BwAgF8vIyNCkSZN05swZxcfHO9onTJig8PBwVa1aVf369dPZs2evednz5s3TlClTNHz48CuONcZo7ty52rp1q2699dZrXhfypjFjxigyMlJt27bNsn/06NFq3759ppux/fDDD1q3bp3Wrl2rtWvXatSoUZKkP/74Qz169JAkNWjQQAcPHtTp06cd7/vrr7/k5eWl4sWLO9reffddvfHGG5o5c6Zq166d01MEgKtC8AYAIBfasGGDAgMD5evrqyeffFLTpk1T5cqVJUkPPfSQvv32W82fP1/9+vXTN99885+n7GblxIkTSkhI0NixYxUcHJztuOTkZAUGBsrHx0dt27bVp59+qpYtW7o0N+QNdrtdY8aMUZcuXeTtnfkky+3bt2vhwoV67LHHMvWVKVNGVatWdfyUKlVK0qUj2pGRkZIu/Z6HhYWpa9eu2rx5sxYuXKgXXnhB3bp1c5xm/s4776h///76+uuvVbJkSR0+fFiHDx92CuuAJC1cuFDt2rVT0aJFZbPZNH36dKd+Y4wGDBigIkWKyN/fXy1atNC2bducxmT1/PqhQ4dmWs57772n8uXLy9fXV8WKFdNbb71l9fTgATjVHACAXKhChQpau3atkpOTNXXqVHXp0kULFixQ5cqV9cQTTzjGVatWTUWKFFHz5s21Y8cOlSlT5qqW//jjj+uhhx664tHroKAgrV27VqdPn9bcuXP13HPPqXTp0plOQ7/Z7N27V8ePH3d3GW61dOlS7d27V/Xq1dPq1asz9X/22WeKjIxUeHh4lv3/9Ndff0mS1q1bp6CgIEf7Rx99pGHDhqlWrVoqVKiQWrRooa5duzqW9/HHHys9PV333nuv0/KeeOIJde/e3dUp5mrh4eHZ3rjuZnTmzBlVr15d3bp10913352p/91339Unn3yicePGqVSpUurfv79atWqlzZs3O90z4PXXX9fjjz/ueP3P31fp0iU5s2fP1nvvvadq1arp5MmTOnnypHUTg8cgeAMAkAv5+PiobNmykqS4uDitXLlSH3/8sb788stMY+vVqyfp0hHGqw3e8+bN088//6z33ntP0qWjNHa7Xd7e3ho5cqS6desmSfLy8nLUUaNGDW3ZskVDhgy5qYP33r17VbFSRZ07m/1jrW4mWYWYf6pTp85VL+u/fq+OHDmiCRMmZHp2d1ZGjhypkSNHXvV68yL/AH/9ueVPwvf/16ZNG7Vp0ybLPmOMPvroI7322mu68847JUnjx49XVFSUpk+frgcffNAxNigoKNv7a2zZskUjRozQxo0bVaFCBUlynM2BvI/gDQBAHmC325WWlpZl39q1ayVJRYoUuerlLV26VBkZGY7XP/30k9555x0tWbJExYoVu646bhbHjx/XubPn1PnLzooqH+XucoBMjvx1RN92/1bHjx8neF+FXbt26fDhw06PoStYsKDq1aunpUuXOgXvoUOH6o033lBMTIweeugh9enTx3GpxS+//KLSpUvr119/VevWrWWMUYsWLfTuu+/ybPmbAMEbAIBcpl+/fmrTpo1iYmKUmpqqiRMnKjExUbNmzdKOHTs0ceJE3X777QoLC9P69evVp08f3XrrrU53c96+fbtOnz6tw4cP69y5c45wXrlyZfn4+KhSpUpO61y1apW8vLxUtWpVR9uQIUNUu3ZtlSlTRmlpafrtt9/0zTffaMSIETdkO3i6qPJRiq4e7e4yALjo8OHDkqSoKOc/pEVFRTn6JOnZZ59VrVq1FBoaqiVLlqhfv346dOiQPvjgA0mXnj2/Z88eTZkyRePHj1dGRob69Omje++9V/PmzbtxE4JbeGTwHj58uIYNG6bDhw+revXq+vTTT1W3bl13lwUA8DBbtmxxdwlusWXLFo0bN07Hjx9XYGCgypUrp88++0xhYWHaunWrpk+frvfff1/nzp1TVFSUmjZtqkcffdTpOtonnnhCSUlJjtc1a9aUdOmITFbPW969e7cyMjKclrFjxw6NGDFCR48ela+vr0qWLKk33nhDtWrVuuI1u3nZzfp7CdzsnnvuOcd/x8bGysfHR927d9eQIUPk6+vrOCNo/PjxKl++vKRLd/aPi4vT1q1bHaefI2/yuOD9/fff67nnntMXX3yhevXq6aOPPlKrVq20detWx10sAQA3t5QjKbJ52a75Tt150d9//60VK1ZoxYoVWfbv27dP48eP1/jx469qef9+Rvi/xcXFZdmelpam9evXa/369erXr99VrQsAcoPL12wfOXLE6ZKdI0eOqEaNGtm+r169erp48aJ2796tChUqqEiRIvL29naEbkmOs4v27t1L8M7jPC54f/DBB3r88cfVtWtXSdIXX3yh//3vf/r666/18ssvu7k6AIAnOJd8TsZuuIYWHmnz75s1460Z7i4DQA4pVaqUChcurLlz5zqCdkpKipYvX66nnnoq2/etXbtWXl5ejoOHDRo00MWLF52eMHH5jv0lSpSwdhJwO48K3unp6UpKSnL6S7mXl5datGihpUuXurEyAIAn4hpaeKIjfx1xdwkArtHp06e1fft2x+tdu3Zp7dq1Cg0NVUxMjHr37q0333xT5cqVczxOrGjRourQoYOkSzekXL58uZo2baqgoCAtXbpUffr0UefOnRUSEiJJatGihWrVqqVu3brpo48+kt1uV48ePdSyZUuno+DImzwqeB8/flwZGRlZ3rjgzz//zDQ+LS3N6c6pycnJkqRTp07JbrdbW6wLUlNTZbPZtH/dfqWdvrnv/ArPdGTbEX5H4dH4HYUn4/cTnu7YjmOy2WxKSkpSamqqu8vxCGvWrFGfPn0cry9fr92qVSv169dP9erVU/v27ZWQkKDTp0+rWrVqev3117V8+XJJl45cjxw5Uq+99pouXLigIkWKqEOHDrrvvvu0YMECx3JfeeUVffLJJ2rQoIH8/PxUr149Pf30005jcEnhwoUz5UJPk5KSIunSI+euxGauZtQNcvDgQRUrVkxLlixRfHy8o/3FF1/UggULHL/Ylw0aNEiDBw++0WUCAAAAACDp0v1Uihcv/p9jPOqId3h4uPLly6cjR5xP0Tpy5EiWD6Lv16+f090D7Xa7Tp48qbCwMNlsNsvrvRYpKSmKjo7Wvn37FBwc7O5y8A/sG8/G/vFc7BvPxv7xXOwbz8b+8VzsG891s+4bY4xSU1OzfBrIv3lU8Pbx8VFcXJzmzp3ruF7Cbrdr7ty56tmzZ6bxvr6+8vX1dWorVKjQDaj0+gUHB99Uv4y5CfvGs7F/PBf7xrOxfzwX+8azsX88F/vGc92M+6ZgwYJXNc6jgrd06XqKLl26qHbt2qpbt64++ugjnTlzxnGXcwAAAAAAchOPC94PPPCAjh07pgEDBujw4cOqUaOGZs6c6fEX1gMAAAAAkBWPC96S1LNnzyxPLc/NfH19NXDgwEynxsP92Deejf3judg3no3947nYN56N/eO52Deei31zZR51V3MAAAAAAPIaL3cXAAAAAABAXkbwBgAAAADAQgRvAAAAAAAsRPC+AYYPH66SJUvKz89P9erV04oVK9xdUp6zcOFCtWvXTkWLFpXNZtP06dOd+o0xGjBggIoUKSJ/f3+1aNFC27Ztcxpz8uRJderUScHBwSpUqJAeffRRnT592mnM+vXr1ahRI/n5+Sk6Olrvvvuu1VPL9YYMGaI6deooKChIkZGR6tChg7Zu3eo05vz58+rRo4fCwsIUGBioe+65R0eOHHEas3fvXrVt21YBAQGKjIzUCy+8oIsXLzqNSUxMVK1ateTr66uyZctq7NixVk8v1xsxYoRiY2Mdz92Mj4/XjBkzHP3sG88xdOhQ2Ww29e7d29HG/nGfQYMGyWazOf1UrFjR0c++ca8DBw6oc+fOCgsLk7+/v6pVq6ZVq1Y5+vle4D4lS5bM9Nmx2Wzq0aOHJD477pSRkaH+/furVKlS8vf3V5kyZfTGG2/on7cE47PjAgNLTZo0yfj4+Jivv/7abNq0yTz++OOmUKFC5siRI+4uLU/57bffzKuvvmp+/PFHI8lMmzbNqX/o0KGmYMGCZvr06WbdunWmffv2plSpUubcuXOOMa1btzbVq1c3y5YtM3/88YcpW7as6dixo6M/OTnZREVFmU6dOpmNGzea7777zvj7+5svv/zyRk0zV2rVqpUZM2aM2bhxo1m7dq25/fbbTUxMjDl9+rRjzJNPPmmio6PN3LlzzapVq0z9+vXNLbfc4ui/ePGiqVq1qmnRooVZs2aN+e2330x4eLjp16+fY8zOnTtNQECAee6558zmzZvNp59+avLly2dmzpx5Q+eb2/z888/mf//7n/nrr7/M1q1bzSuvvGLy589vNm7caIxh33iKFStWmJIlS5rY2FjTq1cvRzv7x30GDhxoqlSpYg4dOuT4OXbsmKOffeM+J0+eNCVKlDAJCQlm+fLlZufOnWbWrFlm+/btjjF8L3Cfo0ePOn1u5syZYySZ+fPnG2P47LjTW2+9ZcLCwsyvv/5qdu3aZaZMmWICAwPNxx9/7BjDZ+f6EbwtVrduXdOjRw/H64yMDFO0aFEzZMgQN1aVt/07eNvtdlO4cGEzbNgwR9upU6eMr6+v+e6774wxxmzevNlIMitXrnSMmTFjhrHZbObAgQPGGGM+//xzExISYtLS0hxjXnrpJVOhQgWLZ5S3HD161EgyCxYsMMZc2hf58+c3U6ZMcYzZsmWLkWSWLl1qjLn0hxUvLy9z+PBhx5gRI0aY4OBgx/548cUXTZUqVZzW9cADD5hWrVpZPaU8JyQkxIwaNYp94yFSU1NNuXLlzJw5c0zjxo0dwZv9414DBw401atXz7KPfeNeL730kmnYsGG2/Xwv8Cy9evUyZcqUMXa7nc+Om7Vt29Z069bNqe3uu+82nTp1Msbw2XEVp5pbKD09XUlJSWrRooWjzcvLSy1atNDSpUvdWNnNZdeuXTp8+LDTfihYsKDq1avn2A9Lly5VoUKFVLt2bceYFi1ayMvLS8uXL3eMufXWW+Xj4+MY06pVK23dulV///33DZpN7pecnCxJCg0NlSQlJSXpwoULTvunYsWKiomJcdo/1apVU1RUlGNMq1atlJKSok2bNjnG/HMZl8fwWbt6GRkZmjRpks6cOaP4+Hj2jYfo0aOH2rZtm2kbsn/cb9u2bSpatKhKly6tTp06ae/evZLYN+72888/q3bt2rrvvvsUGRmpmjVr6quvvnL0873Ac6Snp+vbb79Vt27dZLPZ+Oy42S233KK5c+fqr7/+kiStW7dOixYtUps2bSTx2XEVwdtCx48fV0ZGhtM/DJIUFRWlw4cPu6mqm8/lbf1f++Hw4cOKjIx06vf29lZoaKjTmKyW8c914L/Z7Xb17t1bDRo0UNWqVSVd2nY+Pj4qVKiQ09h/758rbfvsxqSkpOjcuXNWTCfP2LBhgwIDA+Xr66snn3xS06ZNU+XKldk3HmDSpElavXq1hgwZkqmP/eNe9erV09ixYzVz5kyNGDFCu3btUqNGjZSamsq+cbOdO3dqxIgRKleunGbNmqWnnnpKzz77rMaNGyeJ7wWeZPr06Tp16pQSEhIk8e+au7388st68MEHVbFiReXPn181a9ZU79691alTJ0l8dlzl7e4CANw8evTooY0bN2rRokXuLgX/UKFCBa1du1bJycmaOnWqunTpogULFri7rJvevn371KtXL82ZM0d+fn7uLgf/cvkIkCTFxsaqXr16KlGihCZPnix/f383Vga73a7atWvr7bffliTVrFlTGzdu1BdffKEuXbq4uTr80+jRo9WmTRsVLVrU3aVA0uTJkzVhwgRNnDhRVapU0dq1a9W7d28VLVqUz04O4Ii3hcLDw5UvX75Md2I8cuSIChcu7Kaqbj6Xt/V/7YfChQvr6NGjTv0XL17UyZMnncZktYx/rgPZ69mzp3799VfNnz9fxYsXd7QXLlxY6enpOnXqlNP4f++fK2377MYEBwfzJfgKfHx8VLZsWcXFxWnIkCGqXr26Pv74Y/aNmyUlJeno0aOqVauWvL295e3trQULFuiTTz6Rt7e3oqKi2D8epFChQipfvry2b9/OZ8fNihQposqVKzu1VapUyXEpAN8LPMOePXv0+++/67HHHnO08dlxrxdeeMFx1LtatWp6+OGH1adPH8dZV3x2XEPwtpCPj4/i4uI0d+5cR5vdbtfcuXMVHx/vxspuLqVKlVLhwoWd9kNKSoqWL1/u2A/x8fE6deqUkpKSHGPmzZsnu92uevXqOcYsXLhQFy5ccIyZM2eOKlSooJCQkBs0m9zHGKOePXtq2rRpmjdvnkqVKuXUHxcXp/z58zvtn61bt2rv3r1O+2fDhg1O/5DPmTNHwcHBji9X8fHxTsu4PIbP2rWz2+1KS0tj37hZ8+bNtWHDBq1du9bxU7t2bXXq1Mnx3+wfz3H69Gnt2LFDRYoU4bPjZg0aNMj02Mq//vpLJUqUkMT3Ak8xZswYRUZGqm3bto42PjvudfbsWXl5OcfDfPnyyW63S+Kz4zJ3390tr5s0aZLx9fU1Y8eONZs3bzZPPPGEKVSokNOdGOG61NRUs2bNGrNmzRojyXzwwQdmzZo1Zs+ePcaYS48+KFSokPnpp5/M+vXrzZ133pnlow9q1qxpli9fbhYtWmTKlSvn9OiDU6dOmaioKPPwww+bjRs3mkmTJpmAgIA8/+gDVz311FOmYMGCJjEx0enxIWfPnnWMefLJJ01MTIyZN2+eWbVqlYmPjzfx8fGO/suPDrntttvM2rVrzcyZM01ERESWjw554YUXzJYtW8zw4cN5dMhVePnll82CBQvMrl27zPr1683LL79sbDabmT17tjGGfeNp/nlXc2PYP+70/PPPm8TERLNr1y6zePFi06JFCxMeHm6OHj1qjGHfuNOKFSuMt7e3eeutt8y2bdvMhAkTTEBAgPn2228dY/he4F4ZGRkmJibGvPTSS5n6+Oy4T5cuXUyxYsUcjxP78ccfTXh4uHnxxRcdY/jsXD+C9w3w6aefmpiYGOPj42Pq1q1rli1b5u6S8pz58+cbSZl+unTpYoy59PiD/v37m6ioKOPr62uaN29utm7d6rSMEydOmI4dO5rAwEATHBxsunbtalJTU53GrFu3zjRs2ND4+vqaYsWKmaFDh96oKeZaWe0XSWbMmDGOMefOnTNPP/20CQkJMQEBAeauu+4yhw4dclrO7t27TZs2bYy/v78JDw83zz//vLlw4YLTmPnz55saNWoYHx8fU7p0aad1IGvdunUzJUqUMD4+PiYiIsI0b97cEbqNYd94mn8Hb/aP+zzwwAOmSJEixsfHxxQrVsw88MADTs+JZt+41y+//GKqVq1qfH19TcWKFc3IkSOd+vle4F6zZs0ykjJtc2P47LhTSkqK6dWrl4mJiTF+fn6mdOnS5tVXX3V67BefnetnM8YYtxxqBwAAAADgJsA13gAAAAAAWIjgDQAAAACAhQjeAAAAAABYiOANAAAAAICFCN4AAAAAAFiI4A0AAAAAgIUI3gAAAAAAWIjgDQAAAACAhQjeAAB4iN27d8tms2ns2LHuLgUAAOQggjcAANehffv2CggIUGpqarZjOnXqJB8fH504ceIGVnZ9Bg0aJJvN5vjJnz+/SpYsqWeffVanTp1yd3kAAORqBG8AAK5Dp06ddO7cOU2bNi3L/rNnz+qnn35S69atFRYWdoOru34jRozQN998o88++0x169bVp59+qjvuuMPdZQEAkKsRvAEAuA7t27dXUFCQJk6cmGX/Tz/9pDNnzqhTp043uDLX3HvvvercubO6d++uyZMn64EHHtDixYu1YsUKd5cGAECuRfAGAOA6+Pv76+6779bcuXN19OjRTP0TJ05UUFCQ2rdvr5MnT6pv376qVq2aAgMDFRwcrDZt2mjdunVXXE+TJk3UpEmTTO0JCQkqWbKkU5vdbtdHH32kKlWqyM/PT1FRUerevbv+/vvv652mGjVqJEnasWOHo+1q55OYmCibzabJkyfrrbfeUvHixeXn56fmzZtr+/btmdY1fPhwlS5dWv7+/qpbt67++OOPLOeflpamgQMHqmzZsvL19VV0dLRefPFFpaWlXfc8AQCwkre7CwAAILfq1KmTxo0bp8mTJ6tnz56O9pMnT2rWrFnq2LGj/P39tWnTJk2fPl333XefSpUqpSNHjujLL79U48aNtXnzZhUtWjRH6unevbvGjh2rrl276tlnn9WuXbv02Wefac2aNVq8eLHy589/zcvcvXu3JCkkJMTRtnPnzmuaz9ChQ+Xl5aW+ffsqOTlZ7777rjp16qTly5c7xowYMUI9e/ZUo0aN1KdPH+3evVsdOnRQSEiIihcv7hhnt9vVvn17LVq0SE888YQqVaqkDRs26MMPP9Rff/2l6dOnX/McAQCwGsEbAIDr1KxZMxUpUkQTJ050Ct5TpkzRhQsXHKeZV6tWTX/99Ze8vP7vRLOHH35YFStW1OjRo9W/f3+Xa1m0aJFGjRqlCRMm6KGHHnK0N23aVK1bt9aUKVOc2rNz8uRJSdKZM2c0b948DR8+XBEREbr11lsdY651PufPn9fatWvl4+Mj6VKI79WrlzZu3KiqVasqPT1d/fv3V506dTRv3jx5e1/6ehIbG6uEhASn4D1x4kT9/vvvWrBggRo2bOhor1q1qp588kktWbJEt9xyy7VsOgAALMep5gAAXKd8+fLpwQcf1NKlSx1HhqVL4TAqKkrNmzeXJPn6+jpCakZGhk6cOKHAwEBVqFBBq1evzpFapkyZooIFC6ply5Y6fvy44ycuLk6BgYGaP3/+VS2nQoUKioiIUMmSJdWtWzeVLVtWM2bMUEBAgGPMtc6na9eujtAt/d/p6zt37pQkrVq1SidOnNDjjz/uCN3SpTMK/nmk/fI8K1WqpIoVKzrNs1mzZpJ01fMEAOBGIngDAOCCy0e1L99kbf/+/frjjz/04IMPKl++fJIunR794Ycfqly5cvL19VV4eLgiIiK0fv16JScn50gd27ZtU3JysiIjIxUREeH0c/r06SyvQ8/KDz/8oDlz5mjixImqX7++jh49Kn9/f6cx1zqfmJgYp9eXw/Tla8/37NkjSSpbtqzTOG9v70zXsW/btk2bNm3KNMfy5ctL0lXPEwCAG4lTzQEAcEFcXJwqVqyo7777Tq+88oq+++47GWOc7mb+9ttvq3///urWrZveeOMNhYaGysvLS71795bdbv/P5dtsNhljMrVnZGQ4vbbb7YqMjNSECROyXE5ERMRVzefWW29VeHi4JKldu3aqVq2aOnXqpKSkJMdR7mudz+U/QPxbVvO6ErvdrmrVqumDDz7Isj86OvqalwkAgNUI3gAAuKhTp07q37+/1q9fr4kTJ6pcuXKqU6eOo3/q1Klq2rSpRo8e7fS+U6dOOUJudkJCQhynZP/T5aPEl5UpU0a///67GjRokOkI9fUKDAzUwIED1bVrV02ePFkPPvigJNfmk5USJUpIkrZv366mTZs62i9evKjdu3crNjbW0VamTBmtW7dOzZs3l81mu55pAQBww3GqOQAALrp8dHvAgAFau3Ztpmd358uXL9PR3SlTpujAgQNXXHaZMmX0559/6tixY462devWafHixU7j7r//fmVkZOiNN97ItIyLFy/q1KlTVzsdJ506dVLx4sX1zjvvONpcmU9WateurbCwMH311Ve6ePGio33ChAmZHoV2//3368CBA/rqq68yLefcuXM6c+bMddUAAICVOOINAICLSpUqpVtuuUU//fSTJGUK3nfccYdef/11de3aVbfccos2bNigCRMmqHTp0ldcdrdu3fTBBx+oVatWevTRR3X06FF98cUXqlKlilJSUhzjGjdurO7du2vIkCFau3atbrvtNuXPn1/btm3TlClT9PHHH+vee++95rnlz59fvXr10gsvvKCZM2eqdevWLs0nKz4+Pho0aJCeeeYZNWvWTPfff792796tsWPHqkyZMk5Hth9++GFNnjxZTz75pObPn68GDRooIyNDf/75pyZPnqxZs2apdu3a11UHAABW4Yg3AAA54HLYrlu3bqabhL3yyit6/vnnNWvWLPXq1UurV6/W//73v6u6HrlSpUoaP368kpOT9dxzz+nnn3/WN998o1q1amUa+8UXX2jkyJE6evSoXnnlFfXr10/z5s1T586d1aBBg+ue2xNPPKGCBQtq6NChLs8nOz179tQnn3yivXv3qm/fvvrjjz/0888/q1ChQvLz83OM8/Ly0vTp0zV06FBt2LBBffv21eDBg7Vy5Ur16tXLcZM1AAA8ic1cz51NAAAALGa32xUREaG77747y1PLAQDILTjiDQAA3O78+fOZrhsfP368Tp48qSZNmrinKAAAcghHvAEAgNslJiaqT58+uu+++xQWFqbVq1dr9OjRqlSpkpKSkuTj4+PuEgEAuG7cXA0AALhdyZIlFR0drU8++UQnT55UaGioHnnkEQ0dOpTQDQDI9TjiDQAAAACAhbjGGwAAAAAACxG8AQAAAACwEMEbAAAAAAALEbwBAAAAALAQwRsAAAAAAAsRvAEAAAAAsBDBGwAAAAAACxG8AQAAAACwEMEbAAAAAAAL/T+zQTcerxTdzQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "\n", - "# 创建画布\n", - "plt.figure(figsize=(10, 6))\n", - "\n", - "# 绘制直方图\n", - "# bins：直方图的分箱数量（可根据数据范围调整）\n", - "# edgecolor：设置柱形边缘颜色，区分不同分箱\n", - "n, bins, patches = plt.hist(values, bins=5, edgecolor='black', color='lightgreen')\n", - "\n", - "# 添加标题和坐标轴标签\n", - "plt.title('Data Distribution Histogram', fontsize=15)\n", - "plt.xlabel('Value Range', fontsize=12)\n", - "plt.ylabel('Frequency', fontsize=12)\n", - "\n", - "# 在每个柱形上方标注频数\n", - "for i in range(len(n)):\n", - " plt.text(bins[i] + (bins[i+1] - bins[i])/2, n[i] + 0.1, \n", - " f'{int(n[i])}', ha='center', fontsize=10)\n", - "\n", - "# 添加网格线，便于查看\n", - "plt.grid(axis='y', alpha=0.3)\n", - "\n", - "# 调整布局\n", - "plt.tight_layout()\n", - "\n", - "# 显示图像\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "45b5a51a-8972-486d-8e7e-1411224e0f68", - "metadata": { - "collapsed": true, - "jupyter": { - "outputs_hidden": true, - "source_hidden": true - }, - "scrolled": true - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABKUAAAJOCAYAAABm7rQwAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAa+JJREFUeJzt3XtcVHX+x/H3mcEZBAREEFQUQbGbpqY/XdculrZmZllbWZqaWZuVpdnVLprd7KZZrWVbqVmZZpvWlmlmml3c2iyztrwhaKIgSIKgcpk5vz9cTo6AAs6cUXg9Hw8ej+Yz3znf72fOGbS355wxTNM0BQAAAAAAANjIEewFAAAAAAAAoP4hlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAoB546KGHZBiGLXP16tVLvXr1sh6vXLlShmHo3XfftWX+a6+9Vq1bt7ZlrtoqLCzU9ddfr4SEBBmGobFjxwZ7SUFj57FZU+XH7sqVK4O9FAAA6iRCKQAATjCzZ8+WYRjWT2hoqJo3b66+ffvq+eef1969e/0yz44dO/TQQw9p7dq1ftmePx3Pa6uOxx9/XLNnz9ZNN92kN954Q0OHDq1ybOvWrWUYhm699dYKz9kd+B3PTj/9dLVq1UqmaVY5pmfPnoqPj1dZWZmNKwMAAFUhlAIA4AT18MMP64033tBLL71kBRZjx45Vhw4dtG7dOp+xDzzwgPbv31+j7e/YsUOTJk2qcfDzySef6JNPPqnRa2rqSGt75ZVXtGHDhoDOf6w+++wz/elPf9LEiRN1zTXXqEuXLkd9zSuvvKIdO3bYsDp71ebYrMyQIUP022+/6Ysvvqj0+YyMDK1evVqDBg1SSEjIMc8HAACOHaEUAAAnqH79+umaa67RiBEjNH78eC1dulSffvqpdu3apYsvvtjnf/RDQkIUGhoa0PXs27dPkuRyueRyuQI615E0aNBAbrc7aPNXx65duxQdHV3t8aeddpo8Ho+eeOKJwC3KZkVFRZL8d2wOHjxYhmFo7ty5lT7/9ttvyzRNDRky5JjnAgAA/kEoBQBAHXLeeefpwQcf1NatW/Xmm29a9cru27Ns2TKdeeaZio6OVkREhE466STdd999kg5eFvZ///d/kqQRI0ZYlwrOnj1b0sH7RrVv315r1qzR2WefrbCwMOu1h99TqpzH49F9992nhIQEhYeH6+KLL9Zvv/3mM6Z169a69tprK7z20G0ebW2V3VOqqKhId9xxh1q2bCm3262TTjpJzzzzTIVLvQzD0OjRo7Vo0SK1b99ebrdbp512mpYsWVL5G36YXbt2aeTIkYqPj1doaKg6duyo119/3Xq+/HK79PR0ffTRR9baMzIyjrjd1q1ba9iwYdU6W6qqe2pVdgyU97tgwQKdeuqpatiwoXr06KGffvpJkvTyyy+rbdu2Cg0NVa9evSpd5zfffKMLLrhAUVFRCgsL0znnnKOvvvqq0rl/+eUXDR48WI0bN9aZZ55Z5bok6c0331S3bt0UFhamxo0b6+yzzz7iGXgtW7bU2WefrXfffVelpaUVnp87d67atGmj7t27a+vWrbr55pt10kknqWHDhmrSpImuuOKKo+4HqXrHaLni4mJNnDhRbdu2ldvtVsuWLXX33XeruLjYZ9yRPosAANRlhFIAANQx5fcnOtL/wP/3v//VRRddpOLiYj388MOaMmWKLr74YitMOOWUU/Twww9Lkv72t7/pjTfe0BtvvKGzzz7b2sbu3bvVr18/derUSdOmTdO55557xHU99thj+uijj3TPPffotttu07Jly9SnT58aX7pVnbUdyjRNXXzxxXr22Wd1wQUXaOrUqTrppJN01113ady4cRXGf/nll7r55pt11VVX6amnntKBAwf017/+Vbt37z7iuvbv369evXrpjTfe0JAhQ/T0008rKipK1157rZ577jlr7W+88YZiY2PVqVMna+1xcXFH7fv+++9XWVmZ38+W+uKLL3THHXdo+PDheuihh/Trr7/qoosu0vTp0/X888/r5ptv1l133aXVq1fruuuu83ntZ599prPPPlsFBQWaOHGiHn/8ce3Zs0fnnXeevv322wpzXXHFFdq3b58ef/xx3XDDDVWuadKkSRo6dKgaNGighx9+WJMmTVLLli312WefHbGXIUOGaPfu3Vq6dKlP/aefftLPP/9snSX1n//8R19//bWuuuoqPf/88xo1apSWL1+uXr16WWf8HSuv16uLL75YzzzzjAYMGKAXXnhBAwcO1LPPPqtBgwZZ4472WQQAoE4zAQDACWXWrFmmJPM///lPlWOioqLMzp07W48nTpxoHvrH/rPPPmtKMnNycqrcxn/+8x9Tkjlr1qwKz51zzjmmJHPGjBmVPnfOOedYj1esWGFKMlu0aGEWFBRY9XfeeceUZD733HNWLSkpyRw+fPhRt3mktQ0fPtxMSkqyHi9atMiUZD766KM+4y6//HLTMAxz8+bNVk2S6XK5fGo//vijKcl84YUXKsx1qGnTppmSzDfffNOqlZSUmD169DAjIiJ8ek9KSjL79+9/xO1VNnbEiBFmaGiouWPHDtM0/3hvFyxYUGX/5Q4/Bsr7dbvdZnp6ulV7+eWXTUlmQkKCz5rHjx9vSrLGer1eMzU11ezbt6/p9Xqtcfv27TOTk5PN888/v8LcV1999VHXtWnTJtPhcJiXXnqp6fF4fMYeOk9l8vLyTLfbXWGee++915RkbtiwwVrj4VavXm1KMufMmWPVyt/fFStWWLXqHqNvvPGG6XA4zC+++MJn3IwZM0xJ5ldffWWaZvU+iwAA1FWcKQUAQB0UERFxxG/hK7+f0fvvvy+v11urOdxut0aMGFHt8cOGDVOjRo2sx5dffrmaNWumxYsX12r+6lq8eLGcTqduu+02n/odd9wh0zT18ccf+9T79OmjNm3aWI9PP/10RUZGasuWLUedJyEhQVdffbVVa9CggW677TYVFhbq888/P+ZeHnjgAb+fLdW7d2+fy/26d+8uSfrrX//qs7/K6+Xvw9q1a7Vp0yYNHjxYu3fvVm5urnJzc1VUVKTevXtr1apVFY6tUaNGHXU9ixYtktfr1YQJE+Rw+P5VtbLL/A7VuHFjXXjhhfrggw+se1aZpql58+apa9euateunSSpYcOG1mtKS0u1e/dutW3bVtHR0fr++++PusbqWLBggU455RSdfPLJ1nuTm5ur8847T5K0YsUKSf75LAIAcKKq16HUqlWrNGDAADVv3lyGYWjRokU13oZpmnrmmWfUrl07ud1utWjRQo899pj/FwsAQA0UFhb6BAqHGzRokHr27Knrr79e8fHxuuqqq/TOO+/U6H+KW7RoUaMbmqempvo8NgxDbdu2rdZ9fI7F1q1b1bx58wrvxymnnGI9f6hWrVpV2Ebjxo31+++/H3We1NTUCkFKVfPURkpKioYOHap//OMf2rlz5zFvT6rYb1RUlKSD92iqrF7+PmzatEmSNHz4cMXFxfn8vPrqqyouLlZ+fr7PNpKTk4+6nrS0NDkcDp166qm16mfIkCEqKirS+++/L0n6+uuvlZGR4XOD8/3792vChAnWPcZiY2MVFxenPXv2VFhzbW3atEn//e9/K7w35cHYrl27JPnnswgAwImqXn8fblFRkTp27KjrrrtOl112Wa22MWbMGH3yySd65pln1KFDB+Xl5SkvL8/PKwUAoPq2b9+u/Px8tW3btsoxDRs21KpVq7RixQp99NFHWrJkiebPn6/zzjtPn3zyiZxO51HnOfRsE3+p6kwYj8dTrTX5Q1XzmIfdFD1Y7r//fr3xxht68sknNXDgwArPH+k9rExV/R7tfSgPTZ5++ml16tSp0rERERE+jwNxzBzuoosuUlRUlObOnavBgwdr7ty5cjqduuqqq6wxt956q2bNmqWxY8eqR48eioqKkmEYuuqqq44aBlX3GPV6verQoYOmTp1a6fjy0M8fn0UAAE5U9TqU6tevn/r161fl88XFxbr//vv19ttva8+ePWrfvr2efPJJ65tVfv31V7300kv6+eefddJJJ0mq3r8AAgAQSG+88YYkqW/fvkcc53A41Lt3b/Xu3VtTp07V448/rvvvv18rVqxQnz59jnqpVE2Vn1lTzjRNbd68WaeffrpVa9y4sfbs2VPhtVu3blVKSor1uCZrS0pK0qeffqq9e/f6nC21fv1663l/SEpK0rp16+T1en3OlvL3PG3atNE111yjl19+2bqk7lBHeg/9qfwSx8jISPXp08ev2/V6vfrll1+qDLuOxO126/LLL9ecOXOUnZ2tBQsW6LzzzlNCQoI15t1339Xw4cM1ZcoUq3bgwIFK37fDVfcYbdOmjX788Uf17t37qMfr0T6LAADUVfX68r2jGT16tFavXq158+Zp3bp1uuKKK3TBBRdYf6n+17/+pZSUFH344YdKTk5W69atdf3113OmFAAgaD777DM98sgjSk5O9rlc6XCV/VlVHgCUf119eHi4JFXrf9SrY86cOT73uXr33Xe1c+dOn38gatOmjf7973+rpKTEqn344Yf67bfffLZVk7VdeOGF8ng8+vvf/+5Tf/bZZ2UYxhH/gaomLrzwQmVlZWn+/PlWraysTC+88IIiIiJ0zjnn+GUe6eC9pUpLS/XUU09VeK5NmzbKz8/XunXrrNrOnTu1cOFCv80vSV26dFGbNm30zDPPqLCwsMLzOTk5tdruwIED5XA49PDDD1c4a6m6Z6sNGTJEpaWluvHGG5WTk1Phs+B0Oits64UXXqjybLJDVfcYvfLKK5WZmalXXnmlwjb2799v3fOqOp9FAADqqnp9ptSRbNu2TbNmzdK2bdvUvHlzSdKdd96pJUuWaNasWXr88ce1ZcsWbd26VQsWLNCcOXPk8Xh0++236/LLLz/qVxYDAHCsPv74Y61fv15lZWXKzs7WZ599pmXLlikpKUkffPCBQkNDq3ztww8/rFWrVql///5KSkrSrl279OKLLyoxMVFnnnmmpIP/8x0dHa0ZM2aoUaNGCg8PV/fu3Wt9VnBMTIzOPPNMjRgxQtnZ2Zo2bZratm2rG264wRpz/fXX691339UFF1ygK6+8UmlpaXrzzTd9bjxe07UNGDBA5557ru6//35lZGSoY8eO+uSTT/T+++9r7NixFbZdW3/729/08ssv69prr9WaNWvUunVrvfvuu/rqq680bdq0I97jq6bKz5Z6/fXXKzx31VVX6Z577tGll16q2267Tfv27dNLL72kdu3a+e0m3tLBs3teffVV9evXT6eddppGjBihFi1aKDMzUytWrFBkZKT+9a9/1Xi7bdu21f33369HHnlEZ511li677DK53W795z//UfPmzTV58uSjbuOcc85RYmKi3n//fTVs2LDCbRouuugivfHGG4qKitKpp56q1atX69NPP1WTJk2Ouu3qHqNDhw7VO++8o1GjRmnFihXq2bOnPB6P1q9fr3feeUdLly5V165dq/VZBACgriKUqsJPP/0kj8dj3YyyXHFxsfUXFq/Xq+LiYs2ZM8ca99prr6lLly7asGGDdUkfAACBMGHCBEmSy+VSTEyMOnTooGnTpmnEiBFHDUAuvvhiZWRkaObMmcrNzVVsbKzOOeccTZo0ybqhdYMGDfT6669r/PjxGjVqlMrKyjRr1qxah1L33Xef1q1bp8mTJ2vv3r3q3bu3XnzxRYWFhVlj+vbtqylTpmjq1KkaO3asunbtqg8//FB33HGHz7ZqsjaHw6EPPvhAEyZM0Pz58zVr1iy1bt1aTz/9dIXtHouGDRtq5cqVuvfee/X666+roKBAJ510kmbNmqVrr73Wb/OUe+CBB/Tmm29WOLunSZMmWrhwocaNG6e7775bycnJmjx5sjZt2uTXUEqSevXqpdWrV+uRRx7R3//+dxUWFiohIUHdu3fXjTfeWOvtPvzww0pOTtYLL7yg+++/X2FhYTr99NM1dOjQar3e4XDo6quv1tNPP60BAwZU+Dw899xzcjqdeuutt3TgwAH17NlTn3766VEveZWqf4w6HA4tWrRIzz77rObMmaOFCxcqLCxMKSkpGjNmjPV3x+p8FgEAqKsM83i5a2eQGYahhQsXWjcMnT9/voYMGaL//ve/FW4wGRERoYSEBE2cOFGPP/64SktLref279+vsLAwffLJJzr//PPtbAEAAAAAAOCEwZlSVejcubM8Ho927dqls846q9IxPXv2VFlZmdLS0qxTtjdu3CjJfzczBQAAAAAAqIvq9ZlShYWF2rx5s6SDIdTUqVN17rnnKiYmRq1atdI111yjr776SlOmTFHnzp2Vk5Oj5cuX6/TTT1f//v3l9Xr1f//3f4qIiNC0adPk9Xp1yy23KDIyUp988kmQuwMAAAAAADh+1etQauXKlTr33HMr1IcPH67Zs2ertLRUjz76qObMmaPMzEzFxsbqT3/6kyZNmqQOHTpIknbs2KFbb71Vn3zyicLDw9WvXz9NmTJFMTExdrcDAAAAAABwwqjXoRQAAAAAAACCwxHsBQAAAAAAAKD+IZQCAAAAAACA7erdt+95vV7t2LFDjRo1kmEYwV4OAAAAAABAnWKapvbu3avmzZvL4aj6fKh6F0rt2LFDLVu2DPYyAAAAAAAA6rTffvtNiYmJVT5f70KpRo0aSTr4xkRGRgZ5NQAAAAAAAHVLQUGBWrZsaWUwVal3oVT5JXuRkZGEUgAAAAAAAAFytNsmcaNzAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgu5BgLwAol5OTo4KCgoDPExkZqbi4uIDPAwAAAAAAqkYoheNCTk6OrhlxvfL27gv4XDGNwvTmrFcJpgAAAAAACCJCKRwXCgoKlLd3n+J6/FXhMfEBm6coL1s5q/+pgoICQikAAAAAAIKIUArHlfCYeEU2TQzoHDkB3ToAAAAAAKgObnQOAAAAAAAA2xFKAQAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsRygFAAAAAAAA2xFKAQAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsRygFAAAAAAAA2xFKAQAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsRygFAAAAAAAA2xFKAQAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsRygFAAAAAAAA2xFKAQAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsRygFAAAAAAAA2xFKAQAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsF9RQatWqVRowYICaN28uwzC0aNGio76muLhY999/v5KSkuR2u9W6dWvNnDkz8IsFAAAAAACA34QEc/KioiJ17NhR1113nS677LJqvebKK69Udna2XnvtNbVt21Y7d+6U1+sN8EoBAAAAAADgT0ENpfr166d+/fpVe/ySJUv0+eefa8uWLYqJiZEktW7dOkCrAwAAAAAAQKCcUPeU+uCDD9S1a1c99dRTatGihdq1a6c777xT+/fvD/bSAAAAAAAAUANBPVOqprZs2aIvv/xSoaGhWrhwoXJzc3XzzTdr9+7dmjVrVqWvKS4uVnFxsfW4oKBAkuTxeOTxeCRJhmHI4XDI6/XKNE1rbFV1h8MhwzCqrJdv99C6pAqXGVZVdzqdMk3Tp16+lqrq1V378dyT0+mQw5AMmTJlyJAp45BtmFKN6gdn8q07DhnAfqIneqIneqIneqIneqIneqIneqInevJ/T9V1QoVSXq9XhmHorbfeUlRUlCRp6tSpuvzyy/Xiiy+qYcOGFV4zefJkTZo0qUI9LS1NERERkqSoqCg1a9ZM2dnZys/Pt8bExsYqNjZWmZmZKioqsuoJCQmKjo5WRkaGSkpKrHpiYqIiIiKUlpbms6OSk5MVEhKiTZs2+awhNTVVZWVlSk9Pt2oOh0Pt2rVTUVGRtm/fbtVdLpdSUlKUn5+vrKwsqx4eHq6WLVsqLy9Pubm5Vv1E6yknJ0c9u3VVdIyhMtd+bS8JU0xIiWJD/ggU93hcyi4NVdMGxYp2/rHG3DK3dpe51cK1X+GOMqueVRqqfI9LSe59chsHP5jFMYZ+jz547LCf6Ime6Ime6Ime6Ime6Ime6Ime6Ime/N9TeHi4qsMwD42zgsgwDC1cuFADBw6scszw4cP11VdfafPmzVbt119/1amnnqqNGzcqNTW1wmsqO1Oq/E2LjIy05q6LyeSJ1NPmzZs19MbRSuo3So3iWgTsTKmCnExt+fBFzZs5o8L9yNhP9ERP9ERP9ERP9ERP9ERP9ERP9ERPx95TYWGhoqKilJ+fb2UvlTmhzpTq2bOnFixYoMLCQussp40bN8rhcCgxMbHS17jdbrnd7gp1p9Mpp9PpUyvfKYeraf3w7dambhhGjer+WnuwenI4HPJ4vPKasiIkU4YqS0yPpe49ZAD7iZ7oiZ6OVKcneqInejpSnZ7oiZ7o6Uh1eqIneqqe2r/SDwoLC7V27VqtXbtWkpSenq61a9dq27ZtkqTx48dr2LBh1vjBgwerSZMmGjFihH755RetWrVKd911l6677rpKL90DAAAAAADA8SmoodR3332nzp07q3PnzpKkcePGqXPnzpowYYIkaefOnVZAJUkRERFatmyZ9uzZo65du2rIkCEaMGCAnn/++aCsHwAAAAAAALUT1Mv3evXq5XPd4eFmz55doXbyySdr2bJlAVwVAAAAAAAAAi2oZ0oBAAAAAACgfiKUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGC7kGAvALBbaUmJtm7dGtA5IiMjFRcXF9A5AAAAAAA4kRFKoV4pLsxXRvoWjb3vIbnd7oDNE9MoTG/OepVgCgAAAACAKhBKoV4pLd4vrxGi2D9dpibNkwIyR1FetnJW/1MFBQWEUgAAAAAAVIFQCvVSWOM4RTZNDNj2cwK2ZQAAAAAA6gZudA4AAAAAAADbEUoBAAAAAADAdkENpVatWqUBAwaoefPmMgxDixYtqvZrv/rqK4WEhKhTp04BWx8AAAAAAAACI6ihVFFRkTp27Kjp06fX6HV79uzRsGHD1Lt37wCtDAAAAAAAAIEU1Bud9+vXT/369avx60aNGqXBgwfL6XTW6OwqAAAAAAAAHB9OuHtKzZo1S1u2bNHEiRODvRQAAAAAAADUUlDPlKqpTZs26d5779UXX3yhkJDqLb24uFjFxcXW44KCAkmSx+ORx+ORJBmGIYfDIa/XK9M0rbFV1R0OhwzDqLJevt1D65Lk9XqrVXc6nTJN06devpaq6tVd+/Hck9PpkMOQDJkyZciQKeOQbZhSjeoHZ/KtO4yD80qSQ6ZUYXzFpLbquiHJ9KmXb7H8vaztMXY876farp2e6Ime6Ime6Ime6Ime6Ime6Ime6kdP1XXChFIej0eDBw/WpEmT1K5du2q/bvLkyZo0aVKFelpamiIiIiRJUVFRatasmbKzs5Wfn2+NiY2NVWxsrDIzM1VUVGTVExISFB0drYyMDJWUlFj1xMRERUREKC0tzWdHJScnKyQkRJs2bfJZQ2pqqsrKypSenm7VHA6H2rVrp6KiIm3fvt2qu1wupaSkKD8/X1lZWVY9PDxcLVu2VF5ennJzc636idZTTk6OenbrqugYQ2Wu/dpeEqaYkBLFhvwRKO7xuJRdGqqmDYoV7fxjjbllbu0uc6uFa7/CHWVWPas0VPkel5Lc++Q2Dn4wExIbaUNsE0lSSmihnIcEU+nF4SozHUoN3evT06YDjRRieJXs/uP98sjQ5gONFO7wKNG1z6oXm06tkxQfF6ucnBzrF0Jd2U918dijJ3qiJ3qiJ3qiJ3qiJ3qiJ3qiJ//2FB4eruowzEPjrCAyDEMLFy7UwIEDK31+z549aty4sZxOp1UrT+OcTqc++eQTnXfeeRVeV9mZUuVvWmRkpDV3XUwmT6SeNm/erKE3jlZSv1FqFNciYGdK7Vi/Rl/OeVpn3fiImiWl+qzRX2dK5e/K1NbFL+mtV6YrJSXFp9cTfT/VxWOPnuiJnuiJnuiJnuiJnuiJnuiJnvzbU2FhoaKiopSfn29lL5U5Yc6UioyM1E8//eRTe/HFF/XZZ5/p3XffVXJycqWvc7vdcrvdFepOp9Mn4JL+2CmHq2n98O3Wpm4YRo3q/lp7sHpyOBzyeLzymrIiJFPGYRfY6ZjrXlPWh8Wryk8p9FZarapuVFo3TVMOhyNgxxjHHj3REz3Vpk5P9ERP9HSkOj3REz3R05Hq9ERPNe2pOoIaShUWFmrz5s3W4/T0dK1du1YxMTFq1aqVxo8fr8zMTM2ZM0cOh0Pt27f3eX3Tpk0VGhpaoQ4AAAAAAIDjW1BDqe+++07nnnuu9XjcuHGSpOHDh2v27NnauXOntm3bFqzlAQAAAAAAIECCGkr16tXL57rDw82ePfuIr3/ooYf00EMP+XdRAAAAAAAACLjaX/gHAAAAAAAA1BKhFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsF1QQ6lVq1ZpwIABat68uQzD0KJFi444/r333tP555+vuLg4RUZGqkePHlq6dKk9iwUAAAAAAIDfBDWUKioqUseOHTV9+vRqjV+1apXOP/98LV68WGvWrNG5556rAQMG6IcffgjwSgEAAAAAAOBPIcGcvF+/furXr1+1x0+bNs3n8eOPP673339f//rXv9S5c2c/rw4AAAAAAACBckLfU8rr9Wrv3r2KiYkJ9lIAAAAAAABQA0E9U+pYPfPMMyosLNSVV15Z5Zji4mIVFxdbjwsKCiRJHo9HHo9HkmQYhhwOh7xer0zTtMZWVXc4HDIMo8p6+XYPrUsHQ7Tq1J1Op0zT9KmXr6WqenXXfjz35HQ65DAkQ6ZMGTJkyjhkG6ZUo/rBmXzrDuPgvJLkkClVGF8xqa26bkgyferlWyx/L2t7jB3P+6m2a6cneqIneqIneqIneqIneqIneqKn+tFTdZ2wodTcuXM1adIkvf/++2ratGmV4yZPnqxJkyZVqKelpSkiIkKSFBUVpWbNmik7O1v5+fnWmNjYWMXGxiozM1NFRUVWPSEhQdHR0crIyFBJSYlVT0xMVEREhNLS0nx2VHJyskJCQrRp0yafNaSmpqqsrEzp6elWzeFwqF27dioqKtL27dutusvlUkpKivLz85WVlWXVw8PD1bJlS+Xl5Sk3N9eqn2g95eTkqGe3roqOMVTm2q/tJWGKCSlRbMgfgeIej0vZpaFq2qBY0c4/1phb5tbuMrdauPYr3FFm1bNKQ5XvcSnJvU9u4+AHMyGxkTbENpEkpYQWynlIMJVeHK4y06HU0L0+PW060EghhlfJ7j/eL48MbT7QSOEOjxJd+6x6senUOknxcbHKycmxfiHUlf1UF489eqIneqIneqIneqIneqIneqInevJvT+Hh4aoOwzw0zgoiwzC0cOFCDRw48Khj582bp+uuu04LFixQ//79jzi2sjOlyt+0yMhIa+66mEyeSD1t3rxZQ28craR+o9QorkXAzpTasX6NvpzztM668RE1S0r1WaO/zpTK35WprYtf0luvTFdKSopPryf6fqqLxx490RM90RM90RM90RM90RM90RM9+benwsJCRUVFKT8/38peKnPCnSn19ttv67rrrtO8efOOGkhJktvtltvtrlB3Op1yOp0+tfKdcria1g/fbm3qhmHUqO6vtQerJ4fDIY/HK68pK0IyZRx2gZ2Oue41ZX1YvKr8lEJvpdWq6kalddM05XA4AnaMcezREz3RU23q9ERP9ERPR6rTEz3REz0dqU5P9FTTnqojqKFUYWGhNm/ebD1OT0/X2rVrFRMTo1atWmn8+PHKzMzUnDlzJB28ZG/48OF67rnn1L17d+u0sYYNGyoqKiooPQAAAAAAAKDmah9n+cF3332nzp07q3PnzpKkcePGqXPnzpowYYIkaefOndq2bZs1/h//+IfKysp0yy23qFmzZtbPmDFjgrJ+AAAAAAAA1E5Qz5Tq1auXz3WHh5s9e7bP45UrVwZ2QQAAAAAAALBFUM+UAgAAAAAAQP1EKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxXq1Bqy5Yt/l4HAAAAAAAA6pFahVJt27bVueeeqzfffFMHDhyo9eSrVq3SgAED1Lx5cxmGoUWLFh31NStXrtQZZ5wht9uttm3bavbs2bWeHwAAAAAAAMFRq1Dq+++/1+mnn65x48YpISFBN954o7799tsab6eoqEgdO3bU9OnTqzU+PT1d/fv317nnnqu1a9dq7Nixuv7667V06dIazw0AAAAAAIDgqVUo1alTJz333HPasWOHZs6cqZ07d+rMM89U+/btNXXqVOXk5FRrO/369dOjjz6qSy+9tFrjZ8yYoeTkZE2ZMkWnnHKKRo8ercsvv1zPPvtsbdoAAAAAAABAkIQc04tDQnTZZZepf//+evHFFzV+/Hjdeeeduu+++3TllVfqySefVLNmzfy1Vq1evVp9+vTxqfXt21djx46t8jXFxcUqLi62HhcUFEiSPB6PPB6PJMkwDDkcDnm9XpmmaY2tqu5wOGQYRpX18u0eWpckr9dbrbrT6ZRpmj718rVUVa/u2o/nnpxOhxyGZMiUKUOGTBmHbMOUalQ/OJNv3WEcnFeSHDKlCuMrJrVV1w1Jpk+9fIvl72Vtj7HjeT/Vdu30RE/0RE/0RE/0RE/0RE/0RE/0VD96qq5jCqW+++47zZw5U/PmzVN4eLjuvPNOjRw5Utu3b9ekSZN0ySWX1OqyvqpkZWUpPj7epxYfH6+CggLt379fDRs2rPCayZMna9KkSRXqaWlpioiIkCRFRUWpWbNmys7OVn5+vjUmNjZWsbGxyszMVFFRkVVPSEhQdHS0MjIyVFJSYtUTExMVERGhtLQ0nx2VnJyskJAQbdq0yWcNqampKisrU3p6ulVzOBxq166dioqKtH37dqvucrmUkpKi/Px8ZWVlWfXw8HC1bNlSeXl5ys3NteonWk85OTnq2a2romMMlbn2a3tJmGJCShQb8keguMfjUnZpqJo2KFa084815pa5tbvMrRau/Qp3lFn1rNJQ5XtcSnLvk9s4+MFMSGykDbFNJEkpoYVyHhJMpReHq8x0KDV0r09Pmw40UojhVbL7j/fLI0ObDzRSuMOjRNc+q15sOrVOUnxcrHJycqxfCHVlP9XFY4+e6Ime6Ime6Ime6Ime6Ime6Ime/NtTeHi4qsMwD42zqmnq1KmaNWuWNmzYoAsvvFDXX3+9LrzwQitpk6Tt27erdevWKisrO8KWDlmIYWjhwoUaOHBglWPatWunESNGaPz48VZt8eLF6t+/v/bt21dpKFXZmVLlb1pkZKQ1d11MJk+knjZv3qyhN45WUr9RahTXImBnSu1Yv0ZfznlaZ934iJolpfqs0V9nSuXvytTWxS/prVemKyUlxafXE30/1cVjj57oiZ7oiZ7oiZ7oiZ7oiZ7oiZ7821NhYaGioqKUn59vZS+VqdWZUi+99JKuu+46XXvttVVente0aVO99tprtdl8lRISEpSdne1Ty87OVmRkZKWBlCS53W653e4KdafTKafT6VMr3ymHq2n98O3Wpm4YRo3q/lp7sHpyOBzyeLzymrIiJFPGYRfY6ZjrXlPWh8Wryk8p9FZarapuVFo3TVMOhyNgxxjHHj3REz3Vpk5P9ERP9HSkOj3REz3R05Hq9ERPNe2pOmoVSh1+2ldlXC6Xhg8fXpvNV6lHjx5avHixT23ZsmXq0aOHX+cBAAAAAABAYNUqzpo1a5YWLFhQob5gwQK9/vrr1d5OYWGh1q5dq7Vr10qS0tPTtXbtWm3btk2SNH78eA0bNswaP2rUKG3ZskV333231q9frxdffFHvvPOObr/99tq0AQAAAAAAgCCpVSg1efJkxcbGVqg3bdpUjz/+eLW3891336lz587q3LmzJGncuHHq3LmzJkyYIEnauXOnFVBJB2/Q9dFHH2nZsmXq2LGjpkyZoldffVV9+/atTRsAAAAAAAAIklpdvrdt2zYlJydXqCclJfmESEfTq1cvn5thHW727NmVvuaHH36o9hwAAAAAAAA4/tTqTKmmTZtq3bp1Feo//vijmjRpcsyLAgAAAAAAQN1Wq1Dq6quv1m233aYVK1bI4/HI4/Hos88+05gxY3TVVVf5e40AAAAAAACoY2p1+d4jjzyijIwM9e7dWyEhBzfh9Xo1bNiwGt1TCgAAAAAAAPVTrUIpl8ul+fPn65FHHtGPP/6ohg0bqkOHDkpKSvL3+gAAAAAAAFAH1SqUKteuXTu1a9fOX2sBAAAAAABAPVGrUMrj8Wj27Nlavny5du3aJa/X6/P8Z5995pfFAQAAAAAAoG6qVSg1ZswYzZ49W/3791f79u1lGIa/1wUAAAAAAIA6rFah1Lx58/TOO+/owgsv9Pd6AAAAAAAAUA84avMil8ultm3b+nstAAAAAAAAqCdqFUrdcccdeu6552Sapr/XAwAAAAAAgHqgVpfvffnll1qxYoU+/vhjnXbaaWrQoIHP8++9955fFgcAAAAAAIC6qVahVHR0tC699FJ/rwUAAAAAAAD1RK1CqVmzZvl7HQAAAAAAAKhHanVPKUkqKyvTp59+qpdffll79+6VJO3YsUOFhYV+WxwAAAAAAADqplqdKbV161ZdcMEF2rZtm4qLi3X++eerUaNGevLJJ1VcXKwZM2b4e50AAAAAAACoQ2p1ptSYMWPUtWtX/f7772rYsKFVv/TSS7V8+XK/LQ4AAAAAAAB1U63OlPriiy/09ddfy+Vy+dRbt26tzMxMvywMAAAAAAAAdVetzpTyer3yeDwV6tu3b1ejRo2OeVEAAAAAAACo22oVSv3lL3/RtGnTrMeGYaiwsFATJ07UhRde6K+1AQAAAAAAoI6q1eV7U6ZMUd++fXXqqafqwIEDGjx4sDZt2qTY2Fi9/fbb/l4jAAAAAAAA6phahVKJiYn68ccfNW/ePK1bt06FhYUaOXKkhgwZ4nPjcwAAAAAAAKAytQqlJCkkJETXXHONP9cCAAAAAACAeqJWodScOXOO+PywYcNqtRgAAAAAAADUD7UKpcaMGePzuLS0VPv27ZPL5VJYWBihFAAAAAAAAI6oVt++9/vvv/v8FBYWasOGDTrzzDO50TkAAAAAAACOqlahVGVSU1P1xBNPVDiLCgAAAAAAADic30Ip6eDNz3fs2OHPTQIAAAAAAKAOqtU9pT744AOfx6ZpaufOnfr73/+unj17+mVhAAAAAAAAqLtqFUoNHDjQ57FhGIqLi9N5552nKVOm+GNdAAAAAAAAqMNqFUp5vV5/rwMAAAAAAAD1iF/vKQUAAAAAAABUR63OlBo3bly1x06dOrU2UwAAAAAAAKAOq1Uo9cMPP+iHH35QaWmpTjrpJEnSxo0b5XQ6dcYZZ1jjDMPwzyoBAAAAAABQp9QqlBowYIAaNWqk119/XY0bN5Yk/f777xoxYoTOOuss3XHHHX5dJAAAAAAAAOqWWt1TasqUKZo8ebIVSElS48aN9eijj/LtewAAAAAAADiqWoVSBQUFysnJqVDPycnR3r17j3lRAAAAAAAAqNtqFUpdeumlGjFihN577z1t375d27dv1z//+U+NHDlSl112mb/XCAAAAAAAgDqmVveUmjFjhu68804NHjxYpaWlBzcUEqKRI0fq6aef9usCAQAAAAAAUPfUKpQKCwvTiy++qKefflppaWmSpDZt2ig8PNyviwMAAAAAAEDdVKvL98rt3LlTO3fuVGpqqsLDw2Wapr/WBQAAAAAAgDqsVqHU7t271bt3b7Vr104XXnihdu7cKUkaOXKk7rjjDr8uEAAAAAAAAHVPrUKp22+/XQ0aNNC2bdsUFhZm1QcNGqQlS5b4bXEAAAAAAACom2p1T6lPPvlES5cuVWJiok89NTVVW7du9cvCAAAAAAAAUHfV6kypoqIinzOkyuXl5cntdh/zogAAAAAAAFC31SqUOuusszRnzhzrsWEY8nq9euqpp3Tuuef6bXEAAAAAAACom2oVSj311FP6xz/+oX79+qmkpER333232rdvr1WrVunJJ5+s8famT5+u1q1bKzQ0VN27d9e33357xPHTpk3TSSedpIYNG6ply5a6/fbbdeDAgdq0AgAAAAAAgCCoVSjVvn17bdy4UWeeeaYuueQSFRUV6bLLLtMPP/ygNm3a1Ghb8+fP17hx4zRx4kR9//336tixo/r27atdu3ZVOn7u3Lm69957NXHiRP3666967bXXNH/+fN133321aQUAAAAAAABBUOMbnZeWluqCCy7QjBkzdP/99x/zAqZOnaobbrhBI0aMkCTNmDFDH330kWbOnKl77723wvivv/5aPXv21ODBgyVJrVu31tVXX61vvvnmmNcCAAAAAAAAe9Q4lGrQoIHWrVvnl8lLSkq0Zs0ajR8/3qo5HA716dNHq1evrvQ1f/7zn/Xmm2/q22+/Vbdu3bRlyxYtXrxYQ4cOrXR8cXGxiouLrccFBQWSJI/HI4/HI+ngPbEcDoe8Xq9M07TGVlV3OBzWfbQqq5dv99C6JHm93mrVnU6nTNP0qZevpap6ddd+PPfkdDrkMCRDpkwZMmTKOGQbplSj+sGZfOsO4+C8kuSQKVUYX/H0warrhiTTp16+xfL3srbH2PG8n2q7dnqiJ3qiJ3qiJ3qiJ3qiJ3qiJ3qqHz1VV41DKUm65ppr9Nprr+mJJ56ozcstubm58ng8io+P96nHx8dr/fr1lb5m8ODBys3N1ZlnninTNFVWVqZRo0ZVefne5MmTNWnSpAr1tLQ0RURESJKioqLUrFkzZWdnKz8/3xoTGxur2NhYZWZmqqioyKonJCQoOjpaGRkZKikpseqJiYmKiIhQWlqaz45KTk5WSEiINm3a5LOG1NRUlZWVKT093ao5HA61a9dORUVF2r59u1V3uVxKSUlRfn6+srKyrHp4eLhatmypvLw85ebmWvUTraecnBz17NZV0TGGylz7tb0kTDEhJYoN+SNQ3ONxKbs0VE0bFCva+ccac8vc2l3mVgvXfoU7yqx6Vmmo8j0uJbn3yW0c/GAmJDbShtgmkqSU0EI5Dwmm0ovDVWY6lBq616enTQcaKcTwKtn9x/vlkaHNBxop3OFRomufVS82nVonKT4uVjk5OdYvhLqyn+risUdP9ERP9ERP9ERP9ERP9ERP9ERP/u0pPDxc1WGYh8ZZ1XTrrbdqzpw5Sk1NVZcuXSpMNnXq1GptZ8eOHWrRooW+/vpr9ejRw6rffffd+vzzzyu9JG/lypW66qqr9Oijj6p79+7avHmzxowZoxtuuEEPPvhghfGVnSlV/qZFRkZKqrvJ5InU0+bNmzX0xtFK6jdKjeJaBOxMqR3r1+jLOU/rrBsfUbOkVJ81+utMqfxdmdq6+CW99cp0paSk+PR6ou+nunjs0RM90RM90RM90RM90RM90RM90ZN/eyosLFRUVJTy8/Ot7KUyNTpTasuWLWrdurV+/vlnnXHGGZKkjRs3+oypyWlasbGxcjqdys7O9qlnZ2crISGh0tc8+OCDGjp0qK6//npJUocOHVRUVKS//e1vuv/++603tpzb7Zbb7a6wHafTKafT6VM7/LW1rR++3drUDcOoUd1faw9WTw6HQx6PV15TVoRkyjjsAjsdc91ryvqweFX5seqttFpV3ai0bpqmHA5HwI4xjj16oid6qk2dnuiJnujpSHV6oid6oqcj1emJnmraU3XUKJRKTU3Vzp07tWLFCknSoEGD9Pzzz1e4/K66XC6XunTpouXLl2vgwIGSDqZ0y5cv1+jRoyt9zb59+yo0XP6m1OKkLwAAAAAAAARBjUKpw0Ofjz/+2OfaxtoYN26chg8frq5du6pbt26aNm2aioqKrG/jGzZsmFq0aKHJkydLkgYMGKCpU6eqc+fO1uV7Dz74oAYMGFBl8gcAAAAAAIDjS61udF7OH2cmDRo0SDk5OZowYYKysrLUqVMnLVmyxDr7atu2bT5nRj3wwAMyDEMPPPCAMjMzFRcXpwEDBuixxx475rUAAAAAAADAHjUKpQzDqHDPqJrcQ6oqo0ePrvJyvZUrV/o8DgkJ0cSJEzVx4sRjnhcAAAAAAADBUePL96699lrrxuEHDhzQqFGjKnz73nvvvee/FQIAAAAAAKDOqVEoNXz4cJ/H11xzjV8XAwAAAAAAgPqhRqHUrFmzArUOAAAAAAAA1COOow8BAAAAAAAA/ItQCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALYjlAIAAAAAAIDtCKUAAAAAAABgO0IpAAAAAAAA2I5QCgAAAAAAALY7LkKp6dOnq3Xr1goNDVX37t317bffHnH8nj17dMstt6hZs2Zyu91q166dFi9ebNNqAQAAAAAAcKxCgr2A+fPna9y4cZoxY4a6d++uadOmqW/fvtqwYYOaNm1aYXxJSYnOP/98NW3aVO+++65atGihrVu3Kjo62v7FAwAAAAAAoFaCHkpNnTpVN9xwg0aMGCFJmjFjhj766CPNnDlT9957b4XxM2fOVF5enr7++ms1aNBAktS6dWs7lwwAAAAAAIBjFNRQqqSkRGvWrNH48eOtmsPhUJ8+fbR69epKX/PBBx+oR48euuWWW/T+++8rLi5OgwcP1j333COn01lhfHFxsYqLi63HBQUFkiSPxyOPxyNJMgxDDodDXq9XpmlaY6uqOxwOGYZRZb18u4fWJcnr9Var7nQ6ZZqmT718LVXVq7v247knp9MhhyEZMmXKkCFTxiHbMKUa1Q/O5Ft3GAfnlSSHTKnC+IrXtFZdNySZPvXyLZa/l7U9xo7n/VTbtdMTPdETPdETPdETPdETPdETPdFT/eipuoIaSuXm5srj8Sg+Pt6nHh8fr/Xr11f6mi1btuizzz7TkCFDtHjxYm3evFk333yzSktLNXHixArjJ0+erEmTJlWop6WlKSIiQpIUFRWlZs2aKTs7W/n5+daY2NhYxcbGKjMzU0VFRVY9ISFB0dHRysjIUElJiVVPTExURESE0tLSfHZUcnKyQkJCtGnTJp81pKamqqysTOnp6VbN4XCoXbt2Kioq0vbt2626y+VSSkqK8vPzlZWVZdXDw8PVsmVL5eXlKTc316qfaD3l5OSoZ7euio4xVObar+0lYYoJKVFsyB+B4h6PS9mloWraoFjRzj/WmFvm1u4yt1q49ivcUWbVs0pDle9xKcm9T27j4AczIbGRNsQ2kSSlhBbKeUgwlV4crjLTodTQvT49bTrQSCGGV8nuP94vjwxtPtBI4Q6PEl37rHqx6dQ6SfFxscrJybF+IdSV/VQXjz16oid6oid6oid6oid6oid6oid68m9P4eHhqg7DPDTOstmOHTvUokULff311+rRo4dVv/vuu/X555/rm2++qfCadu3a6cCBA0pPT7fOjJo6daqefvpp7dy5s8L4ys6UKn/TIiMjJdXdZPJE6mnz5s0aeuNoJfUbpUZxLQJ2ptSO9Wv05ZynddaNj6hZUqrPGv11plT+rkxtXfyS3nplulJSUnx6PdH3U1089uiJnuiJnuiJnuiJnuiJnuiJnujJvz0VFhYqKipK+fn5VvZSmaCeKRUbGyun06ns7GyfenZ2thISEip9TbNmzdSgQQOfS/VOOeUUZWVlqaSkRC6Xy2e82+2W2+2usB2n01nhcr/ynXK4mtYru4ywpnXDMGpU99fag9WTw+GQx+OV15QVIZkyDrvATsdc95qyPixeVX5KobfSalV1o9K6aZpyOBwBO8Y49uiJnuipNnV6oid6oqcj1emJnuiJno5Upyd6qmlP1VH7V/qBy+VSly5dtHz5cqvm9Xq1fPlynzOnDtWzZ09t3rzZJ53buHGjmjVrViGQAgAAAAAAwPEpqKGUJI0bN06vvPKKXn/9df3666+66aabVFRUZH0b37Bhw3xuhH7TTTcpLy9PY8aM0caNG/XRRx/p8ccf1y233BKsFgAAAAAAAFBDQb18T5IGDRqknJwcTZgwQVlZWerUqZOWLFli3fx827ZtPqeCtWzZUkuXLtXtt9+u008/XS1atNCYMWN0zz33BKsFAAAAAAAA1FDQQylJGj16tEaPHl3pcytXrqxQ69Gjh/79738HeFUAAAAAAAAIlKBfvgcAAAAAAID6h1AKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADY7rgIpaZPn67WrVsrNDRU3bt317ffflut182bN0+GYWjgwIGBXSAAAAAAAAD8Kuih1Pz58zVu3DhNnDhR33//vTp27Ki+fftq165dR3xdRkaG7rzzTp111lk2rRQAAAAAAAD+EvRQaurUqbrhhhs0YsQInXrqqZoxY4bCwsI0c+bMKl/j8Xg0ZMgQTZo0SSkpKTauFgAAAAAAAP4QEszJS0pKtGbNGo0fP96qORwO9enTR6tXr67ydQ8//LCaNm2qkSNH6osvvjjiHMXFxSouLrYeFxQUSDoYbHk8HkmSYRhyOBzyer0yTdMaW1Xd4XDIMIwq6+XbPbQuSV6vt1p1p9Mp0zR96uVrqape3bUfzz05nQ45DMmQKVOGDJkyDtmGKdWofnAm37rDODivJDlkShXGV0xqq64bkkyfevkWy9/L2h5jx/N+qu3a6Yme6Ime6Ime6Ime6Ime6Ime6Kl+9FRdQQ2lcnNz5fF4FB8f71OPj4/X+vXrK33Nl19+qddee01r166t1hyTJ0/WpEmTKtTT0tIUEREhSYqKilKzZs2UnZ2t/Px8a0xsbKxiY2OVmZmpoqIiq56QkKDo6GhlZGSopKTEqicmJioiIkJpaWk+Oyo5OVkhISHatGmTzxpSU1NVVlam9PR0q+ZwONSuXTsVFRVp+/btVt3lciklJUX5+fnKysqy6uHh4WrZsqXy8vKUm5tr1U+0nnJyctSzW1dFxxgqc+3X9pIwxYSUKDbkj0Bxj8el7NJQNW1QrGjnH2vMLXNrd5lbLVz7Fe4os+pZpaHK97iU5N4nt3Hwg5mQ2EgbYptIklJCC+U8JJhKLw5XmelQauhen542HWikEMOrZPcf75dHhjYfaKRwh0eJrn1Wvdh0ap2k+LhY5eTkWL8Q6sp+qovHHj3REz3REz3REz3REz3REz3REz35t6fw8HBVh2EeGmfZbMeOHWrRooW+/vpr9ejRw6rffffd+vzzz/XNN9/4jN+7d69OP/10vfjii+rXr58k6dprr9WePXu0aNGiSueo7Eyp8jctMjJSUt1NJk+knjZv3qyhN45WUr9RahTXImBnSu1Yv0ZfznlaZ934iJolpfqs0V9nSuXvytTWxS/prVemW5eX1pX9VBePPXqiJ3qiJ3qiJ3qiJ3qiJ3qiJ3ryb0+FhYWKiopSfn6+lb1UJqhnSsXGxsrpdCo7O9unnp2drYSEhArj09LSlJGRoQEDBli18jckJCREGzZsUJs2bXxe43a75Xa7K2zL6XTK6XT61Mp3yuFqWj98u7WpG4ZRo7q/1h6snhwOhzwer7ymrAjJlHHYBXY65rrXlPVh8aryUwq9lVarqhuV1k3TlMPhCNgxxrFHT/RET7Wp0xM90RM9HalOT/RET/R0pDo90VNNe6qOoIZSLpdLXbp00fLlyzVw4EBJB0Om5cuXa/To0RXGn3zyyfrpp598ag888ID27t2r5557Ti1btrRj2cBxIScnx7pHWiBFRkYqLi4u4PMAAAAAAOqXoIZSkjRu3DgNHz5cXbt2Vbdu3TRt2jQVFRVpxIgRkqRhw4apRYsWmjx5skJDQ9W+fXuf10dHR0tShTpQl+Xk5OiaEdcrb+++ow8+RjGNwvTmrFcJpgAAAAAAfhX0UGrQoEHKycnRhAkTlJWVpU6dOmnJkiXWzc+3bdt2TKeCAXVRQUGB8vbuU1yPvyo8Jv7oL6ilorxs5az+pwoKCgilAAAAAAB+FfRQSpJGjx5d6eV6krRy5cojvnb27Nn+XxBwggiPiVdk08SAzpET0K0DAAAAAOqr4yKUwvEv0Pcv2rp1q8pKywK2fQAAAAAAcHwhlMJR2XH/ogP792l75k61Ki0N2BwAAAAAAOD4QSiFo7Lj/kW70n7W1t9mylNGKAUAAAAAQH1AKIVqC+T9iwp3ZwVkuwAAAAAA4PjE19oBAAAAAADAdoRSAAAAAAAAsB2hFAAAAAAAAGxHKAUAAAAAAADbEUoBAAAAAADAdnz73gkuJydHBQUFAZ1j69atKistC+gcAAAAAACgfiGUOoHl5OTomhHXK2/vvoDOc2D/Pm3P3KlWpaUBnQcAAAAAANQfhFInsIKCAuXt3ae4Hn9VeEx8wObZlfaztv42U54yQikAAAAAAOAfhFJ1QHhMvCKbJgZs+4W7swK2bQAAAAAAUD8RSgEBUFpSoq1btwZs+9znCwAAAABwoiOUAvysuDBfGelbNPa+h+R2uwMyB/f5AgAAAACc6AilAD8rLd4vrxGi2D9dpibNkwIyB/f5AgAAAACc6AilgAAJaxwXsHt9cZ8vAAAAAMCJzhHsBQAAAAAAAKD+IZQCAAAAAACA7QilAAAAAAAAYDtCKQAAAAAAANiOUAoAAAAAAAC2I5QCAAAAAACA7QilAAAAAAAAYDtCKQAAAAAAANiOUAoAAAAAAAC2I5QCAAAAAACA7QilAAAAAAAAYDtCKQAAAAAAANiOUAoAAAAAAAC2I5QCAAAAAACA7QilAAAAAAAAYDtCKQAAAAAAANiOUAoAAAAAAAC2I5QCAAAAAACA7QilAAAAAAAAYDtCKQAAAAAAANiOUAoAAAAAAAC2Cwn2AgAc30pLSrR169aAzhEZGam4uLiAzgEAAAAAOL4QSgGoUnFhvjLSt2jsfQ/J7XYHbJ6YRmF6c9arBFMAAAAAUI8QSgGoUmnxfnmNEMX+6TI1aZ4UkDmK8rKVs/qfKigoIJQCAAAAgHqEUArAUYU1jlNk08SAbT8nYFsGAAAAAByvuNE5AAAAAAAAbEcoBQAAAAAAANsRSgEAAAAAAMB2x0UoNX36dLVu3VqhoaHq3r27vv322yrHvvLKKzrrrLPUuHFjNW7cWH369DnieAAAAAAAABx/gh5KzZ8/X+PGjdPEiRP1/fffq2PHjurbt6927dpV6fiVK1fq6quv1ooVK7R69Wq1bNlSf/nLX5SZmWnzygEAAAAAAFBbQQ+lpk6dqhtuuEEjRozQqaeeqhkzZigsLEwzZ86sdPxbb72lm2++WZ06ddLJJ5+sV199VV6vV8uXL7d55QAAAAAAAKitoIZSJSUlWrNmjfr06WPVHA6H+vTpo9WrV1drG/v27VNpaaliYmICtUwAAAAAAAD4WUgwJ8/NzZXH41F8fLxPPT4+XuvXr6/WNu655x41b97cJ9g6VHFxsYqLi63HBQUFkiSPxyOPxyNJMgxDDodDXq9XpmlaY6uqOxwOGYZRZb18u4fWJcnr9Var7nQ6ZZqmT718LYfWvV6vtQ1DpoxDtmFKMmVUu+7931Yqr0shIU45DMkh06d+eKpZdd2QZPrUy9cimXIYf8xhyKzR2mvSk8M4+F4eXKMpVRh/7D1J/9tfh7xfNV370Xo69P1yyPyj7ueeHMbB47F8VZXtv2Ptqfy/vV6vz2fH7s/TofXq/i443n9H0BM90RM90RM90RM90RM90RM9BaOn6gpqKHWsnnjiCc2bN08rV65UaGhopWMmT56sSZMmVainpaUpIiJCkhQVFaVmzZopOztb+fn51pjY2FjFxsYqMzNTRUVFVj0hIUHR0dHKyMhQSUmJVU9MTFRERITS0tJ8dlRycrJCQkK0adMmnzWkpqaqrKxM6enpVs3hcKhdu3YqKirS9u3brbrL5VJKSory8/OVlZUlScrLy9NpJ6Vqr6SYkBLFhvwRvu3xuJRdGqqmDYoV7fxjjbllbu0uc6uFa7/CHWVWPas0VPkel5Lc++Q2/jiIt5eESZL69T5XbeJdigjdK0lKLw5XmelQ6v8el9t0oJFCDK+S3X+8Xx4Z2nygkcIdHiW69ln1YtOpjOJwRTlL1TyxkWIvOF9J8S4Zrv3aXhIWkJ4SEhtpQ2wTSVJKaKGch4Q4/uopU1KrxBbqdMj7VeQN8WtPp8SHWe9XROhebS8JU5E3xO89JSQ2kvvMHiqRFOUsVUKDA9Z4f/WU5pY2lpQoLS1NOTk5Vj06Olput1s5OTk+n6cmTZrI4XD4jJWkuLg4eb1e7d6926o5HA7FxcWpuLhYe/bskdvtVnh4eKWfJ0kKDw9Xy5YtlZeXp9zcXKt+ov6OoCd6oid6oid6oid6oid6oid6CkZP4eHhqg7DPDTOsllJSYnCwsL07rvvauDAgVZ9+PDh2rNnj95///0qX/vMM8/o0Ucf1aeffqquXbtWOa6yM6XK37TIyEhJJ24yuWXLFl3zt9Fq1W+Uopq2CNiZUjt+XaNv5k7RmTdMUnyr1EPG++9Mqaz1a7T6rYNzNG2VGrAzpXasX6Mv5zyts258RM2SUqu59pr1lPnrGn39xtM6Z9Qj1vvl7zOlstZ/Z71f8a1SA3am1I71a/T1m1PU84ZJik9KDciZUru2/KLP//GgklPayB3qtuoez8FVOp2+qzyWeuOIMM1+ZYbi4uL41wt6oid6oid6oid6oid6oid6oqcA9VRYWKioqCjl5+db2UtlgnqmlMvlUpcuXbR8+XIrlPJ6D960fPTo0VW+7qmnntJjjz2mpUuXHjGQkiS32y23212h7nQ6D7ks6aDynXK4mtYP325t6oZhHLVevvOlg/+zX1m66K96WZlHXrM8tPiDt5KxVdeNquvmH3OURxaB6MlryvqwHN7Lkddes55M06z0/fJXT4e+X4fO4e+evKYO+aVWRa/H2FNJ8X55jRA17jZQTZonVbHSY1eUl62c1f9UYWGhdclwVZ8zf/0uCPbviGNZIz3RU1V1eqKn2tTpiZ7oiZ6OVKcneqKnutlTdQT98r1x48Zp+PDh6tq1q7p166Zp06apqKhII0aMkCQNGzZMLVq00OTJkyVJTz75pCZMmKC5c+eqdevW1qljERER1uV4AE48YY3jFNk0MaBz5Bx9CAAAAADAJkEPpQYNGqScnBxNmDBBWVlZ6tSpk5YsWWKdybBt2zaf1O2ll15SSUmJLr/8cp/tTJw4UQ899JCdSwcAAAAAAEAtBT2UkqTRo0dXebneypUrfR5nZGQEfkEAAAAAAAAIqNpf+AcAAAAAAADUEqEUAAAAAAAAbEcoBQAAAAAAANsRSgEAAAAAAMB2hFIAAAAAAACwHaEUAAAAAAAAbEcoBQAAAAAAANsRSgEAAAAAAMB2hFIAAAAAAACwHaEUAAAAAAAAbEcoBQAAAAAAANsRSgEAAAAAAMB2hFIAAAAAAACwHaEUAAAAAAAAbBcS7AUAgF1KS0q0devWgM4RGRmpuLi4gM4BAAAAAHUBoRSAeqG4MF8Z6Vs09r6H5Ha7AzZPTKMwvTnrVYIpAAAAADgKQikA9UJp8X55jRDF/ukyNWmeFJA5ivKylbP6nyooKCCUAgAAAICjIJQCUK+ENY5TZNPEgG0/J2BbBgAAAIC6hRudAwAAAAAAwHaEUgAAAAAAALAdoRQAAAAAAABsRygFAAAAAAAA2xFKAQAAAAAAwHZ8+x4A+FFpSYm2bt0a0DkiIyMVFxcX0DkAAAAAINAIpQDAT4oL85WRvkVj73tIbrc7YPPENArTm7NeJZgCAAAAcEIjlAIAPykt3i+vEaLYP12mJs2TAjJHUV62clb/UwUFBYRSAAAAAE5ohFIA4GdhjeMU2TQxYNvPCdiWAQAAAMA+3OgcAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA77ikFACeY0pISbd26NeDzREZGcjN1AAAAAAFDKAUAJ5DiwnxlpG/R2PsektvtDuhcMY3C9OasVwmmAAAAAAQEoRQAnEBKi/fLa4Qo9k+XqUnzpIDNU5SXrZzV/1RBQQGhFAAAAICAIJQCgBNQWOM4RTZNDOgcOQHdOgAAAID6jhudAwAAAAAAwHaEUgAAAAAAALAdl+8BACplx7f88Q1/AAAAQP1FKAUAqMCub/njG/4A4OhycnJUUFAQ0Dn4RwIAQDAQSgEAKrDjW/74hj8AOLqcnBxdM+J65e3dF9B5+EcCAEAwEEoBAKoU6G/54xv+AODICgoKlLd3n+J6/FXhMfEBmYN/JAAABAuhFAAAAHCcC4+J5x8JAAB1DqEUACBo7LiZusS9UgAAAIDjEaEUACAo7LqZusS9UgAAAIDjEaEUACAo7LiZunTwXik7Pn9bP/30k5KSAjcPZ2MBAAAANUMoBQAIqkDfTN2uM7I4GwsAAAComeMilJo+fbqefvppZWVlqWPHjnrhhRfUrVu3KscvWLBADz74oDIyMpSamqonn3xSF154oY0rBgCcKOw4I8uus7EkzsgCjic5OTkqKCgI6Bxbt25VWWlZQOeQuMcfACA4gh5KzZ8/X+PGjdOMGTPUvXt3TZs2TX379tWGDRvUtGnTCuO//vprXX311Zo8ebIuuugizZ07VwMHDtT333+v9u3bB6EDAMCJIJBnZHF/LKD+ycnJ0TUjrlfe3n0BnefA/n3anrlTrUpLAzYHv8MAAMES9FBq6tSpuuGGGzRixAhJ0owZM/TRRx9p5syZuvfeeyuMf+6553TBBRforrvukiQ98sgjWrZsmf7+979rxowZtq4dAACp7t0fq6SkRC6XK2Dbt2sOu+ax68wPO87K4SyW6isoKFDe3n2K6/FXhcfEB2yeXWk/a+tvM+UpC1woVdd+h3EcA8CJI6ihVElJidasWaPx48dbNYfDoT59+mj16tWVvmb16tUaN26cT61v375atGhRIJcKAMBR1YX7Y5WWlChz21YlJiUrpEFg/ppgxxx2zhPhcurJxx5WkyZNAjbH7t27dc8DD6mwOHDBhGRPL1LdCD7LL6sLj4kP6Oe+cHdWwLZ9uLrwO0ziOK4pQjwAwRTUUCo3N1cej0fx8b7/uhQfH6/169dX+pqsrKxKx2dlVf4HdnFxsYqLi63H+fn5kqTff/9dHo9HkmQYhhwOh7xer0zTtMZWVXc4HDIMo8p6+XYPrUuS1+utVt3pdMo0TZ96+VoOrRcUFMj0erVnZ4bKDuyTYfyxDdOUTEmGVGndof89eZS615QKdm2X02GoKPs3/e74oy5JjkPG1rhuSt7/TVeUm/nHHMYfdX/3VJSbKZmmCrJ+k8txDGs/Qk8Fu7ZLpunzfh3aqz96KszJ9Nknfll7JfWi3Ew5DEMFWb+pgeG7Rn/1tHfXdplerwqzffeJv3uqzjF2rD2Vz7E36zeFGJV/no61p6LcTJleb8VjuIq117anw4+xI/2OqG1Ph+6T3UYl4/3U096cyo8xf/ZU7d+Tx9jTnszNMpwN5G7TTVExTX238b/tmIdv+3/bUTXre3Zm6MCWDDlbd1F0bNOjji+vG5LP+3ikel5mukoztsmV0lWRMXFHXPux9LRnZ4bKtv0mV0pXNWocd9Txtelpb85O/bjiXY24eawaNmzo+zvba8o0zYN/3lej7vUc/PPd4fT9A8rr8erAgX3K3pWrk869QuHRMdV+D2rS097cnfrx8/d0/a13yOVyyev1VnvtNemprKRU27dtVfPEVnKF+v7P9pHegwp18+DfoQzDkHHoB82Uig8cUNaOTCW2ai1nA6fPeH/1VFpSrB1Z2Yr5bZM8xfv8/nej8v13pD+L/fX3vYL//Vns83eXmq69Gj2V/w5r2La7ImPijnxMVvN3weH1vTkHj+Prbh4rl9tdvWOppvX/HWM7M7erZVJyhWOssmOyNseet6xMO3dsV/PEJDkbOP2z9krqjVxOPXjfvYqJifH5f5tyhmEEpV4TwVojPdXM8bb247Wn6OhoRUdHH/d5xKH1yrKUwsJCSTr6e2wGUWZmpinJ/Prrr33qd911l9mtW7dKX9OgQQNz7ty5PrXp06ebTZs2rXT8xIkTTR38s4offvjhhx9++OGHH3744Ycffvjhhx+bfn777bcj5kJBPVMqNjZWTqdT2dnZPvXs7GwlJCRU+pqEhIQajR8/frzP5X5er1d5eXlq0qSJjAr/1HJ8KygoUMuWLfXbb78pMjIy2MuBDdjn9Qv7u/5hn9c/7PP6h31ev7C/6x/2ef3DPq8e0zS1d+9eNW/e/IjjghpKuVwudenSRcuXL9fAgQMlHQyNli9frtGjR1f6mh49emj58uUaO3asVVu2bJl69OhR6Xi3213hmvXo6Gh/LD9oIiMjOfjrGfZ5/cL+rn/Y5/UP+7z+YZ/XL+zv+od9Xv+wz48uKirqqGOC/u1748aN0/Dhw9W1a1d169ZN06ZNU1FRkfVtfMOGDVOLFi00efJkSdKYMWN0zjnnaMqUKerfv7/mzZun7777Tv/4xz+C2QYAAAAAAABqIOih1KBBg5STk6MJEyYoKytLnTp10pIlS6ybmW/bts26AZck/fnPf9bcuXP1wAMP6L777lNqaqoWLVqk9u3bB6sFAAAAAAAA1FDQQylJGj16dJWX661cubJC7YorrtAVV1wR4FUdf9xutyZOnBjQr9DF8YV9Xr+wv+sf9nn9wz6vf9jn9Qv7u/5hn9c/7HP/MkzzGL8DEQAAAAAAAKghx9GHAAAAAAAAAP5FKAUAAAAAAADbEUoBAAAAAADAdoRSJ4jp06erdevWCg0NVffu3fXtt98Ge0moplWrVmnAgAFq3ry5DMPQokWLfJ43TVMTJkxQs2bN1LBhQ/Xp00ebNm3yGZOXl6chQ4YoMjJS0dHRGjlypAoLC33GrFu3TmeddZZCQ0PVsmVLPfXUU4FuDZWYPHmy/u///k+NGjVS06ZNNXDgQG3YsMFnzIEDB3TLLbeoSZMmioiI0F//+ldlZ2f7jNm2bZv69++vsLAwNW3aVHfddZfKysp8xqxcuVJnnHGG3G632rZtq9mzZwe6PVTipZde0umnn67IyEhFRkaqR48e+vjjj63n2d912xNPPCHDMDR27Firxj6vWx566CEZhuHzc/LJJ1vPs7/rpszMTF1zzTVq0qSJGjZsqA4dOui7776znufvb3VL69atK3zODcPQLbfcIonPeV3j8Xj04IMPKjk5WQ0bNlSbNm30yCOP6NDbbfMZt5GJ4968efNMl8tlzpw50/zvf/9r3nDDDWZ0dLSZnZ0d7KWhGhYvXmzef//95nvvvWdKMhcuXOjz/BNPPGFGRUWZixYtMn/88Ufz4osvNpOTk839+/dbYy644AKzY8eO5r///W/ziy++MNu2bWteffXV1vP5+flmfHy8OWTIEPPnn3823377bbNhw4bmyy+/bFeb+J++ffuas2bNMn/++Wdz7dq15oUXXmi2atXKLCwstMaMGjXKbNmypbl8+XLzu+++M//0pz+Zf/7zn63ny8rKzPbt25t9+vQxf/jhB3Px4sVmbGysOX78eGvMli1bzLCwMHPcuHHmL7/8Yr7wwgum0+k0lyxZYmu/MM0PPvjA/Oijj8yNGzeaGzZsMO+77z6zQYMG5s8//2yaJvu7Lvv222/N1q1bm6effro5ZswYq84+r1smTpxonnbaaebOnTutn5ycHOt59nfdk5eXZyYlJZnXXnut+c0335hbtmwxly5dam7evNkaw9/f6pZdu3b5fMaXLVtmSjJXrFhhmiaf87rmscceM5s0aWJ++OGHZnp6urlgwQIzIiLCfO6556wxfMbtQyh1AujWrZt5yy23WI89Ho/ZvHlzc/LkyUFcFWrj8FDK6/WaCQkJ5tNPP23V9uzZY7rdbvPtt982TdM0f/nlF1OS+Z///Mca8/HHH5uGYZiZmZmmaZrmiy++aDZu3NgsLi62xtxzzz3mSSedFOCOcDS7du0yJZmff/65aZoH92+DBg3MBQsWWGN+/fVXU5K5evVq0zQPBpkOh8PMysqyxrz00ktmZGSktY/vvvtu87TTTvOZa9CgQWbfvn0D3RKqoXHjxuarr77K/q7D9u7da6ampprLli0zzznnHCuUYp/XPRMnTjQ7duxY6XPs77rpnnvuMc8888wqn+fvb3XfmDFjzDZt2pher5fPeR3Uv39/87rrrvOpXXbZZeaQIUNM0+Qzbjcu3zvOlZSUaM2aNerTp49Vczgc6tOnj1avXh3ElcEf0tPTlZWV5bN/o6Ki1L17d2v/rl69WtHR0eratas1pk+fPnI4HPrmm2+sMWeffbZcLpc1pm/fvtqwYYN+//13m7pBZfLz8yVJMTExkqQ1a9aotLTUZ5+ffPLJatWqlc8+79Chg+Lj460xffv2VUFBgf773/9aYw7dRvkYfi8El8fj0bx581RUVKQePXqwv+uwW265Rf3796+wX9jnddOmTZvUvHlzpaSkaMiQIdq2bZsk9ndd9cEHH6hr16664oor1LRpU3Xu3FmvvPKK9Tx/f6vbSkpK9Oabb+q6666TYRh8zuugP//5z1q+fLk2btwoSfrxxx/15Zdfql+/fpL4jNuNUOo4l5ubK4/H4/MLTpLi4+OVlZUVpFXBX8r34ZH2b1ZWlpo2berzfEhIiGJiYnzGVLaNQ+eA/bxer8aOHauePXuqffv2kg7uD5fLpejoaJ+xh+/zo+3PqsYUFBRo//79gWgHR/DTTz8pIiJCbrdbo0aN0sKFC3Xqqaeyv+uoefPm6fvvv9fkyZMrPMc+r3u6d++u2bNna8mSJXrppZeUnp6us846S3v37mV/11FbtmzRSy+9pNTUVC1dulQ33XSTbrvtNr3++uuS+PtbXbdo0SLt2bNH1157rSR+r9dF9957r6666iqdfPLJatCggTp37qyxY8dqyJAhkviM2y0k2AsAgLrqlltu0c8//6wvv/wy2EtBgJ100klau3at8vPz9e6772r48OH6/PPPg70sBMBvv/2mMWPGaNmyZQoNDQ32cmCD8n85l6TTTz9d3bt3V1JSkt555x01bNgwiCtDoHi9XnXt2lWPP/64JKlz5876+eefNWPGDA0fPjzIq0Ogvfbaa+rXr5+aN28e7KUgQN555x299dZbmjt3rk477TStXbtWY8eOVfPmzfmMBwFnSh3nYmNj5XQ6K3y7Q3Z2thISEoK0KvhL+T480v5NSEjQrl27fJ4vKytTXl6ez5jKtnHoHLDX6NGj9eGHH2rFihVKTEy06gkJCSopKdGePXt8xh++z4+2P6saExkZyf8kBYHL5VLbtm3VpUsXTZ48WR07dtRzzz3H/q6D1qxZo127dumMM85QSEiIQkJC9Pnnn+v5559XSEiI4uPj2ed1XHR0tNq1a6fNmzfzGa+jmjVrplNPPdWndsopp1iXbfL3t7pr69at+vTTT3X99ddbNT7ndc9dd91lnS3VoUMHDR06VLfffrt1BjSfcXsRSh3nXC6XunTpouXLl1s1r9er5cuXq0ePHkFcGfwhOTlZCQkJPvu3oKBA33zzjbV/e/TooT179mjNmjXWmM8++0xer1fdu3e3xqxatUqlpaXWmGXLlumkk05S48aNbeoG0sGvjx09erQWLlyozz77TMnJyT7Pd+nSRQ0aNPDZ5xs2bNC2bdt89vlPP/3k8wfdsmXLFBkZaf0luUePHj7bKB/D74Xjg9frVXFxMfu7Durdu7d++uknrV271vrp2rWrhgwZYv03+7xuKywsVFpampo1a8ZnvI7q2bOnNmzY4FPbuHGjkpKSJPH3t7ps1qxZatq0qfr372/V+JzXPfv27ZPD4RuFOJ1Oeb1eSXzGbRfsO63j6ObNm2e63W5z9uzZ5i+//GL+7W9/M6Ojo32+3QHHr71795o//PCD+cMPP5iSzKlTp5o//PCDuXXrVtM0D37daHR0tPn++++b69atMy+55JJKv260c+fO5jfffGN++eWXZmpqqs/Xje7Zs8eMj483hw4dav7888/mvHnzzLCwML5uNAhuuukmMyoqyly5cqXPVwvv27fPGjNq1CizVatW5meffWZ+9913Zo8ePcwePXpYz5d/rfBf/vIXc+3ateaSJUvMuLi4Sr9W+K677jJ//fVXc/r06XytcJDce++95ueff26mp6eb69atM++9917TMAzzk08+MU2T/V0fHPrte6bJPq9r7rjjDnPlypVmenq6+dVXX5l9+vQxY2NjzV27dpmmyf6ui7799lszJCTEfOyxx8xNmzaZb731lhkWFma++eab1hj+/lb3eDwes1WrVuY999xT4Tk+53XL8OHDzRYtWpgffvihmZ6ebr733ntmbGyseffdd1tj+Izbh1DqBPHCCy+YrVq1Ml0ul9mtWzfz3//+d7CXhGpasWKFKanCz/Dhw03TPPiVow8++KAZHx9vut1us3fv3uaGDRt8trF7927z6quvNiMiIszIyEhzxIgR5t69e33G/Pjjj+aZZ55put1us0WLFuYTTzxhV4s4RGX7WpI5a9Ysa8z+/fvNm2++2WzcuLEZFhZmXnrppebOnTt9tpORkWH269fPbNiwoRkbG2vecccdZmlpqc+YFStWmJ06dTJdLpeZkpLiMwfsc91115lJSUmmy+Uy4+LizN69e1uBlGmyv+uDw0Mp9nndMmjQILNZs2amy+UyW7RoYQ4aNMjcvHmz9Tz7u27617/+ZbZv3950u93mySefbP7jH//weZ6/v9U9S5cuNSVV2I+myee8rikoKDDHjBljtmrVygwNDTVTUlLM+++/3ywuLrbG8Bm3j2GaphmUU7QAAAAAAABQb3FPKQAAAAAAANiOUAoAAAAAAAC2I5QCAAAAAACA7QilAAAAAAAAYDtCKQAAAAAAANiOUAoAAAAAAAC2I5QCAAAAAACA7QilAAAAAAAAYDtCKQAAgONYr169NHbs2GAvAwAAwO8IpQAAAAJkwIABuuCCCyp97osvvpBhGFq3bp3NqwIAADg+EEoBAAAEyMiRI7Vs2TJt3769wnOzZs1S165ddfrppwdhZQAAAMFHKAUAABAgF110keLi4jR79myfemFhoRYsWKCBAwfq6quvVosWLRQWFqYOHTro7bffPuI2DcPQokWLfGrR0dE+c/z222+68sorFR0drZiYGF1yySXKyMjwT1MAAAB+QigFAAAQICEhIRo2bJhmz54t0zSt+oIFC+TxeHTNNdeoS5cu+uijj/Tzzz/rb3/7m4YOHapvv/221nOWlpaqb9++atSokb744gt99dVXioiI0AUXXKCSkhJ/tAUAAOAXhFIAAAABdN111yktLU2ff/65VZs1a5b++te/KikpSXfeeac6deqklJQU3Xrrrbrgggv0zjvv1Hq++fPny+v16tVXX1WHDh10yimnaNasWdq2bZtWrlzph44AAAD8g1AKAAAggE4++WT9+c9/1syZMyVJmzdv1hdffKGRI0fK4/HokUceUYcOHRQTE6OIiAgtXbpU27Ztq/V8P/74ozZv3qxGjRopIiJCERERiomJ0YEDB5SWluavtgAAAI5ZSLAXAAAAUNeNHDlSt956q6ZPn65Zs2apTZs2Ouecc/Tkk0/queee07Rp09ShQweFh4dr7NixR7zMzjAMn0sBpYOX7JUrLCxUly5d9NZbb1V4bVxcnP+aAgAAOEaEUgAAAAF25ZVXasyYMZo7d67mzJmjm266SYZh6KuvvtIll1yia665RpLk9Xq1ceNGnXrqqVVuKy4uTjt37rQeb9q0Sfv27bMen3HGGZo/f76aNm2qyMjIwDUFAABwjLh8DwAAIMAiIiI0aNAgjR8/Xjt37tS1114rSUpNTdWyZcv09ddf69dff9WNN96o7OzsI27rvPPO09///nf98MMP+u677zRq1Cg1aNDAen7IkCGKjY3VJZdcoi+++ELp6elauXKlbrvtNm3fvj2QbQIAANQIoRQAAIANRo4cqd9//119+/ZV8+bNJUkPPPCAzjjjDPXt21e9evVSQkKCBg4ceMTtTJkyRS1bttRZZ52lwYMH684771RYWJj1fFhYmFatWqVWrVrpsssu0ymnnKKRI0fqwIEDnDkFAACOK4Z5+E0JAAAAAAAAgADjTCkAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGA7QikAAAAAAADYjlAKAAAAAAAAtiOUAgAAAAAAgO0IpQAAAAAAAGC7/wcfzIk/nAl1DwAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import matplotlib.pyplot as plt\n", - "plt.rcdefaults()\n", - "plt.rcParams[\"axes.unicode_minus\"] = False # 解决负号显示问题\n", - "\n", - "# 绘制直方图\n", - "plt.figure(figsize=(12, 6))\n", - "# bins参数控制区间数量，可根据数据范围调整（如数值范围60-400，建议设30-50）\n", - "n, bins, patches = plt.hist(values, bins=40, edgecolor=\"black\", alpha=0.7)\n", - "\n", - "# 添加标题和坐标轴标签\n", - "plt.title('Distribution of Numeric Values') \n", - "plt.xlabel('Value') \n", - "plt.ylabel('Frequency')\n", - "\n", - "# 显示网格线\n", - "plt.grid(axis=\"y\", linestyle=\"--\", alpha=0.5)\n", - "\n", - "# 显示图像\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "425cc15a-19f7-46b0-9289-4f79fe6246bc", - "metadata": {}, - "outputs": [], - "source": [ - "sum(frequencies)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "b09a92fd-995d-4e33-be32-f28cd54cd8a6", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxUAAAfGCAYAAABh+JOYAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XlcVNX7B/DPDPuOCIobiLvmlpqG5ZYmLl+/mmvmSpot4F6aZrm0aJlbpdmvFC0tzfavmWW4lZrmjoobssk6DsIAArPc8/uD5sbIIjDADPB5v168au49c+8z1zswz5xznqMQQggQERERERGVk9LSARARERERUfXGpIKIiIiIiMzCpIKIiIiIiMzCpIKIiIiIiMzCpIKIiIiIiMzCpIKIiIiIiMzCpIKIiIiIiMzCpIKIiIiIiMzCpIKIiIiIiMzCpILISjVt2hRTp061dBjVkl6vx4IFC9CkSRMolUqMGDGiUs+nUCjkn/fff79Sz0XVw6OPPooFCxaU6Tnx8fFwdHTEsWPHKikqqik2b94MPz8/5OXlWToUIhmTCqIqsG3bNigUCpw+fbrI/X379kX79u3NPs++ffuwbNkys49T3W3duhWrV6/G6NGjsX37dsydO7fSz/nUU0/hiy++wNChQ+VtiYmJmDhxIlq3bg03Nzd4enqie/fu2L59O4QQRR5n9+7dCAwMhIuLCzw9PdGzZ08cPHiwVDEcP34cjz/+OJydneHr64tZs2YhKyurULu8vDwsXLgQDRs2hJOTE3r06IEDBw6U74UDuHr1KhYsWIDOnTvDzc0NDRo0wNChQ4u93xMSEjB27Fh4enrC3d0dw4cPx61bt0zaGN8zxf3s3LlTbvv9998jKCgIDRs2hIODAxo3bozRo0fj0qVLpYq/pPM8+eSTcruYmJhi2+3atcvkmAsXLsTGjRuRnJxc2suIFStWoEePHnjssccK7Tt8+DBGjhwJX19f2Nvbo169ehg2bBi+++67Uh+/Mt27dw/Lli3D4cOHzTpOSf/uBa+lWq3G6tWr0bt3b/j4+MDT0xOPPvoodu/eXeiYf//9N0JDQ/HQQw/BxcUFfn5+GDt2LK5fv16o7aeffoo+ffqgfv36cHBwQEBAAIKDgxETE2PSLicnB9OmTUP79u3h4eEBV1dXdOrUCRs2bIBOpzNpGx4ejmeffRatWrWCs7MzmjVrhunTpyMpKanIa1Ca9/HUqVOh1WrxySeflPbSElU6W0sHQERFu3btGpTKsuX9+/btw8aNG2t9YnHw4EE0atQI69atq7JzduzYERMnTjTZdufOHdy+fRujR4+Gn58fdDodDhw4gKlTp+LatWt45513TNovW7YMK1aswOjRozF16lTodDpcunQJCQkJDzz/+fPn0b9/f7Rt2xZr167F7du38f777+PGjRv45ZdfTNpOnToV33zzDebMmYOWLVti27ZtGDJkCA4dOoTHH3+8zK/9s88+w5YtWzBq1Ci89NJLyMjIwCeffIJHH30U+/fvx4ABA+S2WVlZ6NevHzIyMrB48WLY2dlh3bp16NOnD86fP4+6desCAHr37o0vvvii0LnWrVuHCxcuoH///vK2iIgI1KlTB7Nnz4a3tzeSk5OxdetWdO/eHSdOnECnTp1KjL+o85w+fRobNmzAwIEDC+0bP348hgwZYrItMDDQ5PHw4cPh7u6OTZs2YcWKFSWeHwBUKhW2b9+O7du3F9q3dOlSrFixAi1btsTzzz8Pf39/qNVq7Nu3D6NGjcLOnTvxzDPPPPAclenevXtYvnw5gPwvScy1YsUKBAQEmGzz9PSU///EiRN47bXXMGTIECxZsgS2trb49ttv8fTTT+PKlStyLADw7rvv4tixYxgzZgw6duyI5ORkfPTRR+jSpQv++usvky90zp07h4CAAPz3v/9FnTp1EB0djU8//RR79+7FhQsX0LBhQwD5ScXly5cxZMgQNG3aFEqlEsePH8fcuXNx8uRJfPnll/IxFy5ciLS0NIwZMwYtW7bErVu38NFHH2Hv3r04f/48fH195balfR87OjpiypQpWLt2LWbOnAmFQmH2NScymyCiShcWFiYAiL///rvI/X369BEPPfSQ2ecJCQkRlnxbZ2VlWezcBfXr169CrqeRwWAQOTk5xe4HIJYuXVrq4/3nP/8RLi4uQq/Xy9tOnDghFAqFWLt2bbliHDx4sGjQoIHIyMiQt3366acCgPj111/lbSdPnhQAxOrVq+VtOTk5onnz5iIwMLBc5z59+rTIzMw02Xbnzh3h4+MjHnvsMZPt7777rgAgTp06JW+LjIwUNjY2YtGiRSWe5969e8LNzU08+eSTD4wpOTlZ2Nraiueff74Mr+Rf06ZNEwqFQsTHx8vboqOjC127koSGhgp/f38hSdID265du1Y4OTkVuo579uwRAMTo0aOFVqst9Lz9+/eL//3vf6WKpzKpVKoyvw+K8qDflUa3bt0SMTExJtskSRJPPPGEcHBwMPlddOzYMZGXl2fS9vr168LBwUFMmDDhgTGdPn1aABArV658YNvQ0FABQCQlJcnbjhw5IgwGg0m7I0eOCADitddeM9le2vdxwbjCw8MfGBdRVeDwJyIrdf+cCp1Oh+XLl6Nly5ZwdHRE3bp18fjjj8vDVqZOnYqNGzcCMB3OYZSdnY358+ejSZMmcHBwQOvWrfH+++8XGoaTk5ODWbNmwdvbG25ubvjvf/+LhIQEKBQKkx6QZcuWQaFQ4MqVK3jmmWdQp04d+VvuixcvYurUqWjWrBkcHR3h6+uLZ599Fmq12uRcxmNcv34dEydOhIeHB3x8fPD6669DCIH4+Hj5G19fX1+sWbOmxGtmHJ5y6NAhXL58Wb4GxiEZpb0GCoUCoaGh2LlzJx566CE4ODhg//79D/5HK6WmTZvi3r170Gq18rb169fD19cXs2fPhhCiyGFLxdFoNDhw4AAmTpwId3d3efvkyZPh6uqKr7/+Wt72zTffwMbGBjNmzJC3OTo6Ytq0aThx4gTi4+PL/Hq6du0KV1dXk21169ZFr169EBkZabL9m2++wSOPPIJHHnlE3tamTRv079/fJM6i/O9//0NmZiYmTJjwwJjq1asHZ2dnpKenl/6F/CMvLw/ffvst+vTpg8aNGxfZJjs72+TfryhPPvkkYmNjcf78+Qee84cffkCPHj0KXcfXX38dXl5e2Lp1K+zs7Ao9LygoCP/5z3/kx6mpqZg2bRrq168PR0dHdOrUqVDvx+HDh03eF0bG98+2bdvkbVOnToWrqysSEhIwYsQIuLq6wsfHBy+//DIMBoP8PB8fHwDA8uXL5fed8feFTqfD1atXix3uU5zMzEz5HPcLCAiAv7+/yTaFQoERI0YgLy/PZDhdz549YW9vb9K2ZcuWeOihhwrdn0Vp2rQpAJTqXiqqbe/evQv1Ovfu3RteXl4m5y/L+xjIf995eXnhxx9/fGBcRFWBSQVRFcrIyMCdO3cK/dw/Brcoy5Ytw/Lly9GvXz989NFHeO211+Dn54ezZ88CAJ5//nl5/PcXX3wh/wCAEAL//e9/sW7dOgwaNAhr165F69at8corr2DevHkm55k6dSo+/PBDDBkyBO+++y6cnJxM5gncb8yYMbh37x7eeecdPPfccwCAAwcO4NatWwgODsaHH36Ip59+Grt27cKQIUOKnEswbtw4SJKEVatWoUePHnjrrbewfv16PPnkk2jUqBHeffddtGjRAi+//DKOHj1abCw+Pj744osv0KZNGzRu3Fi+Bm3bti3TNQDyh1DNnTsX48aNw4YNG+QPC+WRk5ODO3fuICYmBtu3b0dYWBgCAwPh5OQktwkPD8cjjzyCDz74AD4+PvLchI8++uiBx4+IiIBer0e3bt1Mttvb26Nz5844d+6cvO3cuXNo1aqVyYcWAOjevTsAlOoDcGklJyfD29tbfixJEi5evFgoTuP5o6KikJmZWezxdu7cCScnJ4wcObLI/enp6VCpVIiIiMD06dOh0WhMhkmV1r59+5Cenl5s8rJ8+XK4urrC0dERjzzyCH777bci23Xt2hUAHjjxWqfT4e+//0aXLl1Mtt+4cQNXr17FiBEj4Obm9sC4c3Jy0LdvX3zxxReYMGECVq9eDQ8PD0ydOhUbNmx44POLYzAYEBQUhLp16+L9999Hnz59sGbNGvzf//0fgPz33ccffwzg37lFX3zxhfzvlJCQgLZt22LRokWlPme/fv3g7u4OZ2dn/Pe//8WNGzdK9TzjvIuC911RhBBISUkptp1arUZqaipOnz6N4OBgACjyXtJqtbhz5w7i4+Px/fff4/3334e/vz9atGhR4vmzsrKQlZVlcv6yvI+NunTpwon9ZD0s10lCVHsYu/RL+rl/uI6/v7+YMmWK/LhTp05i6NChJZ6nuOFPP/zwgwAg3nrrLZPto0ePFgqFQty8eVMIIcSZM2cEADFnzhyTdlOnTi00tGHp0qUCgBg/fnyh8927d6/Qtq+++koAEEePHi10jBkzZsjb9Hq9aNy4sVAoFGLVqlXy9rt37wonJyeTa1KcooaTlfYaCJE/nEmpVIrLly8/8FzG9iUN+1i5cqXJv3X//v1FXFycvD8tLU0AEHXr1hWurq5i9erVYvfu3WLQoEECgNi8eXOJ5zcOkSl4bY3GjBkjfH195ccPPfSQeOKJJwq1u3z5cqnOVVpHjx4VCoVCvP766/I24xCZFStWFGq/ceNGAUBcvXq1yOOp1Wphb28vxo4dW+w5W7duLV9jV1dXsWTJkkLDTkpj1KhRwsHBQdy9e9dke2xsrBg4cKD4+OOPxU8//STWr18v/Pz8hFKpFHv37i3yWPb29uLFF18s8Xw3b94UAMSHH35osv3HH38UAMS6detKFff69esFALFjxw55m1arFYGBgcLV1VVoNBohhBCHDh0SAMShQ4dMnm8c3hUWFiZvmzJlSpH/Zg8//LDo2rWr/Lik4U/G45bmvbt7924xdepUsX37dvH999+LJUuWCGdnZ+Ht7W3ynimKWq0W9erVE7169Xrgeb744gsBQGzZsqXI/Q4ODvK9VLduXfHBBx8U2c74e834061bN3Hx4sUHnv/NN98sNHSpLO9joxkzZggnJ6cHno+oKnCiNlEV2rhxI1q1alVo+/z584vt5jfy9PTE5cuXcePGDbRs2bJM5923bx9sbGwwa9asQuf95ptv8MsvvyA0NFQe4vPSSy+ZtJs5c6bJkIiCXnjhhULbCn4Dn5ubi6ysLDz66KMAgLNnz6JXr14m7adPny7/v42NDbp164bbt29j2rRp8nZPT0+0bt26UJWg0irtNTDq06cP2rVrV65z3W/8+PHo1q0bVCoV9u7di5SUFOTk5Mj7jUOd1Go1du3ahXHjxgEARo8ejQ4dOuCtt97C888/X+zxjcdycHAotM/R0dHkXDk5OcW2K3gsc6SmpuKZZ55BQECASVnVB8VZ0vm/+eYbaLXaEoc+hYWFQaPR4NatWwgLC0NOTg4MBkOZCh5oNBr8/PPPGDJkiMnEYADw8/PDr7/+arJt0qRJaNeuHebPn19kj16dOnVw586dEs9pHBZYp06dQrEAKFUvBZB/j/v6+mL8+PHyNjs7O8yaNQvjx4/HkSNHTIZKlcX97/NevXoVOcG9KE2bNi222tn9xo4di7Fjx8qPR4wYgaCgIPTu3Rtvv/02Nm/eXOTzJEnChAkTkJ6ejg8//LDEc1y9ehUhISEIDAzElClTimzzyy+/IDc3F5GRkdixYweys7OLbNevXz8cOHAA6enpCA8Px4ULF4pta3T06FEsX74cY8eOxRNPPCFvL8v72KhOnTrIycnBvXv34OzsXOJ5iSobkwqiKtS9e/cih36U5oPHihUrMHz4cLRq1Qrt27fHoEGDMGnSJHTs2PGB542NjUXDhg0LfThp27atvN/4X6VSWajqSkld+fe3BYC0tDQsX74cu3btQmpqqsm+jIyMQu39/PxMHnt4eMDR0bHQ0AQPD49C8zJKq7TXwKio11Ve/v7+8vjv8ePHY8aMGRgwYACuXbsGJycnOQmzs7PD6NGj5ecplUqMGzcOS5cuRVxcXKHrZGR8flE163Nzc02SPCcnp2LbFTxWeWVnZ+M///kPMjMz8eeff5rMEXhQnCWdf+fOnfDy8sLgwYOLPXfBCkxPP/20/G9blrVDvv32W+Tm5pZq3gYAeHl5ITg4GKtWrcLt27cLzcEQQpS6Ms/9H7yNQ9RKGhJWUGxsLFq2bFkoiSruHi8tR0dHec6EUZ06dXD37t1yHa+sHn/8cfTo0QO///57sW1mzpyJ/fv34/PPPy+x2ldycjKGDh0KDw8PeX5RUfr16wcAGDx4MIYPH4727dvD1dXV5IsHAKhfvz7q168PIP9LgHfeeQdPPvkkbty4YVLVyejq1at46qmn0L59e3z22Wcm+8ryPjYy3jOs/kTWgHMqiKqJ3r17IyoqClu3bpX/IHXp0qXQH6aqVtQfurFjx+LTTz/FCy+8gO+++w6//fab3AsiSVKh9kX9YS/uj31pv/E0l7kfrksyevRoxMfHy/NDvLy85Mn397/uevXqAUCJH+AaNGgAAEVOhE1KSpLLYBrbFtcOgEnbstJqtRg5ciQuXryIH3/8sdDaK15eXnBwcCjz+ePi4vDHH39gzJgxRU5WLkqdOnXwxBNPmKxnURo7d+6Eh4dHmb7Rb9KkCYD8ZPp+6enpDxzfbyyje/+/cZs2bQDkj7WvSMV9AC2ut7S492JVatKkSZHXF8if47Jp0yasWrUKkyZNKvYYGRkZGDx4MNLT07F///5S3+vNmzfHww8/XKp7afTo0cjKyipy8nR8fDwGDhwIDw8P7Nu3r9AXHGV5HxvdvXsXzs7Olfr7iqi0mFQQVSPGb0W/+uorxMfHo2PHjiYVmYr7sODv74/ExMRC33hevXpV3m/8ryRJiI6ONml38+bNUsd49+5dhIeH49VXX8Xy5cvx1FNP4cknn0SzZs1KfYzKUNprUBWMwxiMvTZKpRKdO3eGSqUqVFEoMTERAAp9U1xQ+/btYWtrW2ixOa1Wi/Pnz6Nz587yts6dO+P69evy0BqjkydPyvvLQ5IkTJ48GeHh4fjyyy/Rp0+fQm2USiU6dOhQ5KJ4J0+eRLNmzYoc6vPVV19BCFHq3gOjnJycInvGipOUlIRDhw5h1KhRRQ5BKY5xSN79/0YJCQnQarVyT0Fx/Pz84OTkVOh916pVK7Ru3Ro//vhjqaqB+fv748aNG4US9/vvceMwq/urGZW3JwOo/G/Kb926VeR7wLguz5w5c7Bw4cJin5+bm4thw4bh+vXr2Lt3b5mHNpb2Xrr/vW2kVqsxcOBA5OXl4ddff5UTiILK8j42io6OfuD9RVRVmFQQVRP3D/txdXVFixYtTLrKXVxcABT+sDBkyBAYDIZClYTWrVsHhUIhDykJCgoCAGzatMmk3YPGKBdk/Fbz/h6F9evXl/oYlaG016AiqVSqIrdv2bIFCoXCpNrPuHHjYDAYTMp/5ubmYufOnWjXrp3Jt5RXr15FXFyc/NjDwwMDBgzAjh07TJKmL774AllZWRgzZoy8bfTo0TAYDHLlHiB/uEVYWBh69Oghf+teVjNnzsTu3buxadOmYqszGc//999/m3xwunbtGg4ePGgSZ0Fffvkl/Pz8il2Y7/4hdkB+mdPw8PBCww2joqIQFRVV5HF27dolj80vSlH/ngkJCdi6dSs6duxY6IPimTNnAOSXNC2JnZ0dunXrVmSytXz5cqjVakyfPh16vb7Q/t9++w179+4FkH+PJycnm6wqrdfr8eGHH8LV1VVO9Pz9/WFjY1Ooktr97/uyMI7nL6rsallKyhZ1jfft24czZ85g0KBBJtt3796NWbNmYcKECVi7dm2xxzQYDBg3bhxOnDiBPXv2FFqo0Eiv1xfZI3jq1ClERESY3Et37twpstfU2HNcsG12djaGDBmChIQE7Nu3r9g5cWV5HxudPXv2gfcXUVXhnAqiaqJdu3bo27evXJv89OnT+Oabb0zG+BpLWM6aNQtBQUGwsbHB008/jWHDhqFfv3547bXXEBMTg06dOuG3337Djz/+iDlz5qB58+by80eNGoX169dDrVbj0UcfxZEjR3D9+nUApfs20t3dHb1798Z7770HnU6HRo0a4bfffiv0LWxVK+01qEhvv/02jh07hkGDBsHPzw9paWn49ttv8ffff2PmzJkmc1Wef/55fPbZZwgJCcH169fh5+eHL774ArGxsfjf//5ncty2bduiT58+JusMvP322+jZsyf69OmDGTNm4Pbt21izZg0GDhxo8mGsR48eGDNmDBYtWoTU1FS0aNEC27dvR0xMDLZs2WJyHmMZ40OHDpW4SvL69euxadMmBAYGwtnZGTt27DDZ/9RTT8kJ70svvYRPP/0UQ4cOxcsvvww7OzusXbsW9evXx/z58wsd+9KlS7h48SJeffXVYu+/Dh06oH///ujcuTPq1KmDGzduYMuWLdDpdFi1apVJW2NZ0JiYmELH2blzJxo2bFjsa12wYAGioqLQv39/NGzYEDExMfjkk0+QnZ1dZMnWAwcOwM/PDw8//HCRxyto+PDheO2116DRaEzK/Y4bNw4RERF4++23ce7cOYwfP15eUXv//v1yzxAAzJgxA5988gmmTp2KM2fOoGnTpvjmm29w7NgxrF+/Xu4F8vDwwJgxY/Dhhx9CoVCgefPm2Lt3b5HJWWk5OTmhXbt22L17N1q1agUvLy+0b98e7du3l0vKTpkypdiCD0Y9e/bEww8/jG7dusHDwwNnz57F1q1b0aRJEyxevFhud+rUKUyePBl169ZF//79Cw1N6tmzp9w7On/+fPz0008YNmwY0tLSCt2fEydOBJBfMKFJkyYYN24cHnroIbi4uCAiIgJhYWHw8PDA66+/Lj9nx44d2Lx5M0aMGIFmzZohMzMTv/76Kw4cOIBhw4aZTMCeMGECTp06hWeffRaRkZEma1O4urpixIgR8uPSvo+B/KQ1LS0Nw4cPL/GaElUZyxWeIqo9yrOi9v0lZd966y3RvXt34enpKZycnESbNm3E22+/bbLKrl6vFzNnzhQ+Pj5CoVCYlJfNzMwUc+fOFQ0bNhR2dnaiZcuWYvXq1YVW+83OzhYhISHCy8tLuLq6ihEjRohr164JACYlXo3lYFUqVaHXc/v2bfHUU08JT09P4eHhIcaMGSMSExOLLUt7/zGmTJkiXFxcSnWdilJcu9JeAwAiJCTkgecp2L6oUpq//fab+M9//iOfz83NTTz22GMiLCysyFWWU1JSxJQpU4SXl5dwcHAQPXr0EPv37y/yfH369Cm0/Y8//hA9e/YUjo6OwsfHR4SEhMhlRAvKyckRL7/8svD19RUODg7ikUceKfI88+fPFwqFQkRGRpb4+o1lR4v7iY6ONmkfHx8vRo8eLdzd3YWrq6v4z3/+I27cuFHksV999VUBoMQynUuXLhXdunUTderUEba2tqJhw4bi6aefLvI5/v7+wt/fv9D2q1evCgBi3rx5xZ7nyy+/FL179xY+Pj7C1tZWeHt7i6eeekqcOXOmUFuDwSAaNGgglixZUuzxCkpJSRG2trbiiy++KHJ/eHi4GD58uKhXr56wtbUVPj4+YtiwYeLHH38sdJzg4GDh7e0t7O3tRYcOHUxKxBqpVCoxatQo4ezsLOrUqSOef/55cenSpSJLyhb1XjS+dws6fvy46Nq1q7C3tzd5T5SlpOxrr70mOnfuLDw8PISdnZ3w8/MTL774okhOTjZp96Ay3QVfQ58+fUpsa5SXlydmz54tOnbsKNzd3YWdnZ3w9/cX06ZNK3QP//3332LMmDHCz89PODg4CBcXF9GlSxexdu1aodPpTNr6+/sXe+6i7sXSvo8XLlwo/Pz8SrViO1FVUAhRRbMeiajaOn/+PB5++GHs2LGjzOPaawOFQoFXXnkFCxYsgIuLS42ZNNm9e3f4+/tjz549lg6l2vnhhx/wzDPPICoqqsjx80WZNm0arl+/jj/++KOSo6PqLi8vD02bNsWrr76K2bNnWzocIgCcU0FE9ymqFvr69euhVCrRu3dvC0RUPaxevRo+Pj7YuHGjpUOpEBqNBhcuXMCKFSssHUq19O677yI0NLTUCQUALF26FH///TdXSKYHCgsLg52dXZHrBBFZCnsqiMjE8uXLcebMGfTr1w+2trb45Zdf8Msvv8jjtamwgvXzW7VqVex6EkRERDUVkwoiMnHgwAEsX74cV65cQVZWFvz8/DBp0iS89tprsLVlbQciIiIqjEkFERERERGZhXMqiIiIiIjILEwqiIiIiIjILBwgXUEkSUJiYiLc3NxKtUAYEREREVFlEkIgMzMTDRs2hFJZuX0JTCoqSGJiIpo0aWLpMIiIiIiITMTHx6Nx48aVeg4mFRXEzc0NQP4/mru7u4WjISIiIqLaTqPRoEmTJvLn1MrEpKKCGIc8ubu7M6kgIiIiIqtRFUPzOVGbiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMwqSCiIiIiIjMYmvpAIiIiIhqu+TkZKSnp1s6DKpknp6e8PX1tXQYlYJJBREREZEFJScnY/To0cjNzbV0KFTJHB0d8c0339TIxIJJBREREZEFpaenIzc3F2+++SYCAgIsHQ5VkujoaLz++utIT09nUkFERERElSMgIABt2rSxdBhE5cKJ2kRU7alUKqhUKkuHQUREVGsxqSCiak2lUmFi8HRMDJ7OxIKIiMhCmFQQUbWm0WiQlnkPaZn3oNFoLB0OERFRrcSkgoiIiMiKJScnY+bMmWjWrBkcHBzQpEkTDBs2DOHh4VUei0KhwA8//FDm5x0+fBgKhaLQT3Jystxm5cqVeOSRR+Dm5oZ69ephxIgRuHbtmrw/LS0NM2fOROvWreHk5AQ/Pz/MmjULGRkZchu1Wo1BgwahYcOG8rUKDQ01+dLpzz//xGOPPYa6devCyckJbdq0wbp160zifVAsAJCbm4uQkBDUrVsXrq6uGDVqFFJSUsp8bWoKJhVEREREViomJgZdu3bFwYMHsXr1akRERGD//v3o168fQkJCLB1emV27dg1JSUnyT7169eR9R44cQUhICP766y8cOHAAOp0OAwcORHZ2NgAgMTERiYmJeP/993Hp0iVs27YN+/fvx7Rp0+RjKJVKDB8+HD/99BOuX7+Obdu24ffff8cLL7wgt3FxcUFoaCiOHj2KyMhILFmyBEuWLMH//d//lToWAJg7dy7+97//Yc+ePThy5AgSExMxcuTIyrx81k1QhcjIyBAAREZGhqVDIao1UlNTRXh4uOgc2Fd06z1A3Lx509IhERGVWWRkpOjatauIjIwstG/w4MGiUaNGIisrq9C+u3fvyv8fGxsr/vvf/woXFxfh5uYmxowZI5KTk+X9U6ZMEcOHDzd5/uzZs0WfPn3kx3369BEzZ84Ur7zyiqhTp46oX7++WLp0qbzf399fAJB//P39S/0aDx06JACYxPwgqampAoA4cuRIsW2+/vprYW9vL3Q6XbFtNmzYIBo3blziuZ566ikxceLEUseSnp4u7OzsxJ49e+Q2kZGRAoA4ceJEkcco6d+5slTl51P2VBBRtWScoD371dcRExsLvU5v6ZCIiCpUWloa9u/fj5CQELi4uBTa7+npCQCQJAnDhw9HWloajhw5ggMHDuDWrVsYN25cmc+5fft2uLi44OTJk3jvvfewYsUKHDhwAADw999/AwDCwsKQlJQkP46JiYFCocDhw4cfePzOnTujQYMGePLJJ3Hs2LES2xqHNXl5eZXYxt3dHba2Ra+SkJiYiO+++w59+vQp9hjnzp3D8ePHS2xzfyxnzpyBTqfDgAED5DZt2rSBn58fTpw4UfyLqsGYVBBRtWScoO3U7BEYJIG8vByo1WpLh0VEVGFu3rwJIcQD164IDw9HREQEvvzyS3Tt2hU9evTA559/jiNHjsgf/EurY8eOWLp0KVq2bInJkyejW7du8twNHx8fAPnJjK+vr/zYzs4OrVu3hrOzc7HHbdCgATZv3oxvv/0W3377LZo0aYK+ffvi7NmzRbaXJAlz5szBY489hvbt2xfZ5s6dO3jzzTcxY8aMQvvGjx8PZ2dnNGrUCO7u7vjss88KtWncuDEcHBzQrVs3hISEYPr06aWOJTk5Gfb29nJiZ1S/fn2TeSK1CZMKIqrWHF3dIUkG3I6Px/xFS1hWlohqDCFEqdpFRkaiSZMmaNKkibytXbt28PT0RGRkZJnO2bFjR5PHDRo0QGpqaonPadSoEa5evYru3bsX26Z169Z4/vnn0bVrV/Ts2RNbt25Fz549C02QNgoJCcGlS5ewa9euIvdrNBoMHToU7dq1w7JlywrtX7duHc6ePYsff/wRUVFRmDdvXqE2f/zxB06fPo3Nmzdj/fr1+Oqrr8oVC+XjitpEVP1JEiQbW2Rk50Kj0cjfnhERVWctW7aEQqHA1atXzT6WUqkslKTodLpC7ezs7EweKxQKSJJk9vmL0r17d/z555+FtoeGhmLv3r04evQoGjduXGh/ZmYmBg0aBDc3N3z//feFYgYAX19f+Pr6ok2bNvDy8kKvXr3w+uuvo0GDBnKbgIAAAECHDh2QkpKCZcuWYfz48aWKxdfXF1qtFunp6Sa9FSkpKfD19S3ztagJ2FNBRNWSWq3mPAoiqtG8vLwQFBSEjRs3mlQdMkpPTwcAtG3bFvHx8YiPj5f3XblyBenp6WjXrh2A/KFLSUlJJs8/f/58mWOys7ODwWAo8/OKcv78eZMP+UIIhIaG4vvvv8fBgwflD/0FaTQaDBw4EPb29vjpp5/g6Oj4wPMYk6K8vLwS2xTc/6BYunbtCjs7O5OyvteuXUNcXBwCAwMfGFNNxJ4KIqp2VCoV5i9cjJjYWLRp1dPS4RARVZqNGzfiscceQ/fu3bFixQp07NgRer0eBw4cwMcff4zIyEgMGDAAHTp0wIQJE7B+/Xro9Xq89NJL6NOnD7p16wYAeOKJJ7B69Wp8/vnnCAwMxI4dO3Dp0iU8/PDDZYqnadOmCA8Px2OPPQYHBwfUqVMHCQkJ6N+/Pz7//PNih0CtX78eAQEBeOihh5Cbm4vPPvsMBw8exG+//Sa3CQkJwZdffokff/wRbm5u8twEDw8PODk5yQnFvXv3sGPHDmg0Gnn9CR8fH9jY2GDfvn1ISUnBI488AldXV1y+fBmvvPIKHnvsMTRt2lS+pn5+fvJclaNHj+L999/HrFmzSh2Lh4cHpk2bhnnz5sHLywvu7u6YOXMmAgMD8eijj5bpmtYUTCqIqNrRaDRIz86BQRKQpH+/MdPrtFCr1WjevLkFoyMiqjjNmjXD2bNn8fbbb2P+/PlISkqCj48Punbtio8//hhA/hClH3/8ETNnzkTv3r2hVCoxaNAgfPjhh/JxgoKC8Prrr2PBggXIzc3Fs88+i8mTJyMiIqJM8axZswbz5s3Dp59+ikaNGiEmJgY6nQ7Xrl3DvXv3in2eVqvF/PnzkZCQAGdnZ3Ts2BG///47+vXrJ7cxvp6+ffuaPDcsLAxTp07F2bNncfLkSQBAixYtTNpER0ejadOmcHJywqeffoq5c+ciLy8PTZo0wciRI/Hqq6/KbSVJwqJFixAdHQ1bW1s0b94c7777Lp5//vlSxwLkz9tQKpUYNWoU8vLyEBQUhE2bNpXuQtZAClHaWUBUIo1GAw8PD7m0GRFVnqioKIwYPxmx8Ylo++Q4XP5lB4QCsIFApw4P4btdOzmvgoiqjatXr2LixInYsWPHAys9UfVliX/nqvx8yjkVRFQjCEmCpPx3sjYRERFVHSYVRERERERkFiYVRERERERkFiYVRERERERkFiYVRFTtqNVqGPQVUyediIiIzMekgoiqFeMaFbcTEgqtDgv8W1aWiIiIqg6TCiKqVoxrVEiSgIBpUiEkA27Hx2P+oiVQqVQWipCIiKj24eJ3RFRjCEmCZGsnl5XlWhVEVJ1ER0dbOgSqRDX935dJBREREZEFeXp6wtHREa+//rqlQ6FK5ujoCE9PT0uHUSmYVBARERFZkK+vL7755hukp6dbOhSqZJ6envD19bV0GJWCSQURERGRhfn6+tbYD5tUO3CiNhERERERmYVJBRFVK1yjgoiIyPowqSCiauNBa1QQERGRZTCpIKJqo6Q1KoiIiMhymFQQUY2j12kRGxvLBfCIiIiqCJMKIqpRjKtqz1m8DBODpzOxICIiqgJMKoioRhGSBElpC+c2vZCWeQ8ajcbSIREREdV4TCqIqEZydPO0dAhERES1BpMKIqo2WE6WiIjIOjGpIKJqgeVkiYiIrBeTCiKqFlhOloiIyHoxqSAiIiIiIrMwqSAiIiIiIrMwqSCiGkun1UKtVls6DCIiohqPSQUR1UjanCzERN/C/EVLuAAeERFRJWNSQURVTqVSISoqqlI/7Bu0eZAUtsjIzuUCeERERJXM1tIBEFHtolKpMDF4OtIy78HLzRk7wj6Dj4+PpcMiIiIiM7CngoiqlEajQVrmPTi37oW0zHsP7EVQqVQcvkRERGTl2FNBRBbh5F4H9x7Qxtiroc3Lw8wXZ3A1bSIiIivFpIKIrJZGo0GKOh0x0bewaMVKJCUlczVtIiIiK8ThT0RUZVQqlUmJV51Wi5s3byIyMrLYIU4Ggx6SwhZOzR7hatpERERWij0VRFQljEOZMu7eRa5OQHsvE7eibmDc5GDY2NiiY/v2+Hrn9mInbTu4uldxxERERFRaTCqIqEoYhzJF3bgBpb0jXNtlQxJK6CUJSoUCak0WNBoNfHx8ODGbiIiomuHwJyKqMgaDHkJhA4MkQZIKT7pWq9U4deoUxk6YgonB07kaNhERUTXBngoisgq597Ixa/5CZOdpcTshCU39/ZGVlWX2cfU6LdRqNZo3b14BURIREVFR2FNBRBYnJAMSE24jMioG9n4PwyAJGIroySgrSTLgdnw85i9awiFVRERElYhJBRFZnJAkSP8Mi7J1dgWQ38OQnp5u3oElCZLSFhnZuQ9cZI+IiIjKj0kFEVkdYw/DO6vXcsE7IiKiaoBzKojI+kgSJBtbZObkWToSIiIiKgUmFURULiqVCnfu3IG3t3exa0sQERFR7cDhT0RUZpGRkRgx5mn0HjAYYydMeeAkaJVKhZs3b3IoExERUQ3FngoiKhOVSoUZL81EROR1CIUN1JosREdHQ6PRwN3dvVCvhUqlwthnJuHCxQjohAJCCCgsFDsRERFVDiYVRFQmGo0G6dk5kCQB2AjodVrMX7QEuXoBLzdn7Aj7zCSx0Gg0uJORCa2kgCQkAGBSQUREVMNw+BMRmcVgMCAjOxfOrXshLfMeS7cSERHVQlaTVKxatQoKhQJz5syRt+Xm5iIkJAR169aFq6srRo0ahZSUFJPnxcXFYejQoXB2dka9evXwyiuvQK/Xm7Q5fPgwunTpAgcHB7Ro0QLbtm0rdP6NGzeiadOmcHR0RI8ePXDq1KnKeJlENZaTex3otPmrV6tUKvknNjaWcymIiIhqOKsY/vT333/jk08+QceOHU22z507Fz///DP27NkDDw8PhIaGYuTIkTh27BiA/G9Ihw4dCl9fXxw/fhxJSUmYPHky7Ozs8M477wAAoqOjMXToULzwwgvYuXMnwsPDMX36dDRo0ABBQUEAgN27d2PevHnYvHkzevTogfXr1yMoKAjXrl1DvXr1qvZiEFmxkpIE7b1MxETfwqz5C+Dk5AwICVAocSc9A7cTEiCEsEDEREREVBUs3lORlZWFCRMm4NNPP0WdOnXk7RkZGdiyZQvWrl2LJ554Al27dkVYWBiOHz+Ov/76CwDw22+/4cqVK9ixYwc6d+6MwYMH480338TGjRuh1WoBAJs3b0ZAQADWrFmDtm3bIjQ0FKNHj8a6devkc61duxbPPfccgoOD0a5dO2zevBnOzs7YunVr1V4MIiumUqkwMXg6Zr/6epFJgl6bC0lhizt305Galo7kO2lITUuHU7NHIEkCAkwqiIiIaiqLJxUhISEYOnQoBgwYYLL9zJkz0Ol0JtvbtGkDPz8/nDhxAgBw4sQJdOjQAfXr15fbBAUFQaPR4PLly3Kb+48dFBQkH0Or1eLMmTMmbZRKJQYMGCC3IaL8CddpmfdKTBIkyYDkpCTExd/G7bg43E5Mgo2jiwWiNaXX5Q/LIiIiosph0eFPu3btwtmzZ/H3338X2pecnAx7e3t4enqabK9fvz6Sk5PlNgUTCuN+476S2mg0GuTk5ODu3bswGAxFtrl69Wqxsefl5SEv79/Vfjk5lWoLR1f34ndKEiSFDWDQAwolhCRBkiw7n0JIBtyOj8f8RUvw3a6dXKiPiIioElispyI+Ph6zZ8/Gzp074ejoaKkwym3lypXw8PCQf5o0aWLpkIioCEKSICltkZGdy+SfiIioklgsqThz5gxSU1PRpUsX2NrawtbWFkeOHMEHH3wAW1tb1K9fH1qtFunp6SbPS0lJga+vLwDA19e3UDUo4+MHtXF3d4eTkxO8vb1hY2NTZBvjMYqyaNEiZGRkyD/x8fHlug5ERERERNWdxZKK/v37IyIiAufPn5d/unXrhgkTJsj/b2dnh/DwcPk5165dQ1xcHAIDAwEAgYGBiIiIQGpqqtzmwIEDcHd3R7t27eQ2BY9hbGM8hr29Pbp27WrSRpIkhIeHy22K4uDgAHd3d5MfIiIiIqLayGJzKtzc3NC+fXuTbS4uLqhbt668fdq0aZg3bx68vLzg7u6OmTNnIjAwEI8++igAYODAgWjXrh0mTZqE9957D8nJyViyZAlCQkLg4OAAAHjhhRfw0UcfYcGCBXj22Wdx8OBBfP311/j555/l886bNw9TpkxBt27d0L17d6xfvx7Z2dkIDg6uoqtBZP3UajX0Ov2DGxIREVGtYxXrVBRn3bp1UCqVGDVqFPLy8hAUFIRNmzbJ+21sbLB37168+OKLCAwMhIuLC6ZMmYIVK1bIbQICAvDzzz9j7ty52LBhAxo3bozPPvtMXqMCAMaNGweVSoU33ngDycnJ6Ny5M/bv319o8jZRbaVSqTB/4WLExMaiTauelg6HiIiIrIxVJRWHDx82eezo6IiNGzdi48aNxT7H398f+/btK/G4ffv2xblz50psExoaitDQ0FLHSlSbaDQapGfnwCAJi1dzIiIiIutj8XUqiIiIiIioemNSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUS1gl6nhVqttnQYRERENRKTCiJ6ILVaDYO++lZ9EpIBt+PjMX/REqhUKkuHQ0REVOMwqSCiEhnXqLidkAAhhKXDKRchSZCUtsjIzoVGo7F0OERERDUOkwoiKpFxjQpJEhConkkFERERVS4mFUREREREZBYmFUREREREZBYmFUREREREZBYmFUREREREZBYmFURUoupeTpaIiIgqH5MKIipWTSgnS0RERJWPSQURFYvlZImIiKg0mFQQEREREZFZmFQQEREREZFZmFQQUa2h12kRGxsLlUpl6VCIiIhqFCYVRFQrCMmA2/HxmLN4GSYGT2diQUREVIGYVBBRrSAkCZLSFs5teiEt8x40Go2lQyIiIqoxmFQQUa3i6OZp6RCIiIhqHCYVRERERERkFiYVRERERERkFiYVRFQstVoNg95g6TCIiIjIyjGpIKIiqVQqzF+4GLcTEiAEV9MmIiKi4jGpIKIiaTQapGfnQJIEBJhUEBERUfGYVBBRraPTaqFWqy0dBhERUY3BpIKIahVtThZiom9h/qIlXACPiIiogjCpIKJaxaDNg6SwRUZ2LhfAIyIiqiBMKoiIiIiIyCxMKoiIiIiIyCxMKoiIiIiIyCxMKoioEJVKhdjYWC58R0RERKVia+kAiMi6qFQqTAyejsTUO1z4joiIiEqFPRVEZEKj0SAt8x6cmj3Che+IiIioVJhUEFGRHF3dLR0CERERVRNMKoiIiIiIyCycU0FEtZZarYZWq4W3tzd8fHwsHQ4REVG1xZ4KIqqV9DotZs1fgN4DBmPshClQqVSWDomIiKjaYlJBRLWSwWDA3cxs5EkKqDVZ0Gg0lg6JiIio2mJSQUQm1Go19Dq9pcMgIiKiaoRJBRHJVCoV5i9cjJjYWOhq0cJ3ep0WsbGxHAJFRERUTkwqiEim0WiQnp0DgyQgSbUjqZAkA27Hx2PO4mWYGDydiQUREVE5MKkgotpNkiApbeHcphfSMu9xbgUREVE5MKkgIgLg6OZp6RCIiIiqLSYVRERERERkFiYVRERERERkFiYVRERERERkFiYVRERERERkFiYVRCRTq9Uw1KL1Ke6n02qhVqstHQYREVG1w6SCiAD8u/Dd7YQECCEsHU6V0+ZkISb6FuYvWsK1KoiIiMqISQURAfh34TtJEhCofUmFQZsHSWGLjOxcrlVBRERURkwqiIiIiIjILEwqiIiIiIjILEwqiAgAJ2kTERFR+TGpIKJaP0mbiIiIzMOkgohq/STtgvQ6lpUlIiIqKyYVRET/kCQDbsfHs6wsERFRGTGpICIykiRISpaVJSIiKismFURE9+EQKCIiorJhUkFErPxUgOAQKCIiojJjUkFUy7HykynBIVBERERlxqSCqJZj5SciIiIyF5MKIqIicF4FERFR6TGpICK6D+dVEBERlQ2TCiKi+3BeBRERUdnYWjoAIqLiGPQ6KBT87oOIiMjaMakgIqskJAOSk5KgsLEDlPxVRUREZM34l5qIrJKQJAiFDSBJUCpZlYqIiMiacVwBERERERGZhUkFEVEx9DotYmNjWQGKiIjoAZhUEJFZDHodDHq9pcOocMaysnMWL8PE4OlMLIiIiEpg0aTi448/RseOHeHu7g53d3cEBgbil19+kffn5uYiJCQEdevWhaurK0aNGoWUlBSTY8TFxWHo0KFwdnZGvXr18Morr0B/3wecw4cPo0uXLnBwcECLFi2wbdu2QrFs3LgRTZs2haOjI3r06IFTp05VymsmsjZqtRoGvaFczzVOpr6dmARdOY9hrYxlZZ3b9EJa5j2WliUiIiqBRZOKxo0bY9WqVThz5gxOnz6NJ554AsOHD8fly5cBAHPnzsX//vc/7NmzB0eOHEFiYiJGjhwpP99gMGDo0KHQarU4fvw4tm/fjm3btuGNN96Q20RHR2Po0KHo168fzp8/jzlz5mD69On49ddf5Ta7d+/GvHnzsHTpUpw9exadOnVCUFAQUlNTq+5iEFmASqXC/IWLcTshAUKUfTK0kCRIChsYJAmSVLOSCiNHN09Lh0BERGT1LJpUDBs2DEOGDEHLli3RqlUrvP3223B1dcVff/2FjIwMbNmyBWvXrsUTTzyBrl27IiwsDMePH8dff/0FAPjtt99w5coV7NixA507d8bgwYPx5ptvYuPGjdBqtQCAzZs3IyAgAGvWrEHbtm0RGhqK0aNHY926dXIca9euxXPPPYfg4GC0a9cOmzdvhrOzM7Zu3WqR60JUVTQaDdKzcyBJAgKssERERETlYzVzKgwGA3bt2oXs7GwEBgbizJkz0Ol0GDBggNymTZs28PPzw4kTJwAAJ06cQIcOHVC/fn25TVBQEDQajdzbceLECZNjGNsYj6HVanHmzBmTNkqlEgMGDJDbEBERERFR8Sy+TkVERAQCAwORm5sLV1dXfP/992jXrh3Onz8Pe3t7eHp6mrSvX78+kpOTAQDJyckmCYVxv3FfSW00Gg1ycnJw9+5dGAyGIttcvXq12Ljz8vKQl5cnP+Z4ayIiIiKqrSzeU9G6dWucP38eJ0+exIsvvogpU6bgypUrlg7rgVauXAkPDw/5p0mTJpYOiYiIiIjIIiyeVNjb26NFixbo2rUrVq5ciU6dOmHDhg3w9fWFVqtFenq6SfuUlBT4+voCAHx9fQtVgzI+flAbd3d3ODk5wdvbGzY2NkW2MR6jKIsWLUJGRob8Ex8fX67XT0TWT6fVQq1WWzoMIiIiq2XxpOJ+kiQhLy8PXbt2hZ2dHcLDw+V9165dQ1xcHAIDAwEAgYGBiIiIMKnSdODAAbi7u6Ndu3Zym4LHMLYxHsPe3h5du3Y1aSNJEsLDw+U2RXFwcJBL4Rp/iKjm0eZkISb6FuYvWsK1KoiIiIph0TkVixYtwuDBg+Hn54fMzEx8+eWXOHz4MH799Vd4eHhg2rRpmDdvHry8vODu7o6ZM2ciMDAQjz76KABg4MCBaNeuHSZNmoT33nsPycnJWLJkCUJCQuDg4AAAeOGFF/DRRx9hwYIFePbZZ3Hw4EF8/fXX+Pnnn+U45s2bhylTpqBbt27o3r071q9fj+zsbAQHB1vkuhCR9TBo8yApbJGRnQuNRgMfHx9Lh0RERGR1LJpUpKamYvLkyUhKSoKHhwc6duyIX3/9FU8++SQAYN26dVAqlRg1ahTy8vIQFBSETZs2yc+3sbHB3r178eKLLyIwMBAuLi6YMmUKVqxYIbcJCAjAzz//jLlz52LDhg1o3LgxPvvsMwQFBcltxo0bB5VKhTfeeAPJycno3Lkz9u/fX2jyNlFNolKpEBsbW+6F74iIiIiMLJpUbNmypcT9jo6O2LhxIzZu3FhsG39/f+zbt6/E4/Tt2xfnzp0rsU1oaChCQ0NLbENUU6hUKkwMno7E1DvlXviOiIiIyMjq5lQQUeXTaDRIy7wHp2aPcOE7IiIiMhuTCqJazNGVBQaIiIjIfEwqiIhKQa9jWVkiIqLilCupuHXrVkXHQURktSTJgNvx8SwrS0REVIxyJRUtWrRAv379sGPHDuTm5lZ0TERE1kWSICn/LStLREREpsqVVJw9exYdO3bEvHnz4Ovri+effx6nTp2q6NiIiIiIiKgaKFdS0blzZ2zYsAGJiYnYunUrkpKS8Pjjj6N9+/ZYu3YthwcQEREREdUiZk3UtrW1xciRI7Fnzx68++67uHnzJl5++WU0adJEXtSOiIiIiIhqNrOSitOnT+Oll15CgwYNsHbtWrz88suIiorCgQMHkJiYiOHDh1dUnEREREREZKXKtaL22rVrERYWhmvXrmHIkCH4/PPPMWTIECiV+TlKQEAAtm3bhqZNm1ZkrEREREREZIXKlVR8/PHHePbZZzF16lQ0aNCgyDb16tXDli1bzAqOiIiIiIisX7mSihs3bjywjb29PaZMmVKewxNRJVOr1dDr9JYOo9oxLoDXvHlzS4dCRERkVco1pyIsLAx79uwptH3Pnj3Yvn272UERUeVRqVSYv3AxYmJjodMbLB1OtSG4AB4REVGxypVUrFy5Et7e3oW216tXD++8847ZQRFR5dFoNEjPzoFBEpAkJhWlJbgAHhERUbHKlVTExcUhICCg0HZ/f3/ExcWZHRQREREREVUf5Uoq6tWrh4sXLxbafuHCBdStW9fsoKhiqVQqDtcgIiIiokpTrqRi/PjxmDVrFg4dOgSDwQCDwYCDBw9i9uzZePrppys6RjKDSqXCxODpmBg8nYkFEREREVWKclV/evPNNxETE4P+/fvD1jb/EJIkYfLkyZxTYWU0Gg3SMu/J/+/j42PhiIiIiIiopilXUmFvb4/du3fjzTffxIULF+Dk5IQOHTrA39+/ouMjIiIiIiIrV66kwqhVq1Zo1apVRcVCRERERETVULmSCoPBgG3btiE8PBypqamQJMlk/8GDByskOCIia8MF8IiIiAorV1Ixe/ZsbNu2DUOHDkX79u2hUCgqOi4iqiRqtRoGLnoHg14HhaJstSoKLoD33a6dnKNERET0j3IlFbt27cLXX3+NIUOGVHQ8RFSJjKtp305IgFDYWDoci5EkA5KTkqCwsYMQAqX9WkRIEiRbO3kBPCYVRERE+co9UbtFixYVHQsRVTLjatqSJAAbYelwLEeSIClsgH+GbrKvlYiIyDzlWqdi/vz52LBhA4SoxR9KiIiIiIgIQDl7Kv78808cOnQIv/zyCx566CHY2dmZ7P/uu+8qJDgiIiIiIrJ+5UoqPD098dRTT1V0LEREREREVA2VK6kICwur6DiIiIiIiKiaKtecCgDQ6/X4/fff8cknnyAzMxMAkJiYiKysrAoLjoiIiIiIrF+5eipiY2MxaNAgxMXFIS8vD08++STc3Nzw7rvvIi8vD5s3b67oOImIiIiIyEqVq6di9uzZ6NatG+7evQsnJyd5+1NPPYXw8PAKC46IiIiIiKxfuXoq/vjjDxw/fhz29vYm25s2bYqEhIQKCYyIyFrpdVrExsbC3d2dC+ARERGhnD0VkiTBYDAU2n779m24ubmZHRQRkbUSkgG34+MxZ/EyTAyeDpVKZemQiIiILK5cScXAgQOxfv16+bFCoUBWVhaWLl2KIUOGVFRsRERWR0gSJKUtnNv0QlrmPWg0GkuHREREZHHlGv60Zs0aBAUFoV27dsjNzcUzzzyDGzduwNvbG1999VVFx0hEFUStVsOgL9zLSGXn6OaJbEsHQUREZCXKlVQ0btwYFy5cwK5du3Dx4kVkZWVh2rRpmDBhgsnEbSKyHiqVCvMXLsbthAQIIaCwdEBERERUY5QrqQAAW1tbTJw4sSJjIaJKpNFokJ6dA0kSEGBSQURERBWnXEnF559/XuL+yZMnlysYIiIiIiKqfsqVVMyePdvksU6nw71792Bvbw9nZ2cmFVZIp2UJTCIiIiKqHOWq/nT37l2Tn6ysLFy7dg2PP/44J2pbIZ1Wi5joWyyBSURERESVolxJRVFatmyJVatWFerFIMszGPSQFCyBSVTRdFot1Gq1pcMgIiKyuApLKoD8yduJiYkVeUgyU8ESoo5unpYNhqgG0eZkISb6FuYvWsLePyIiqvXKNafip59+MnkshEBSUhI++ugjPPbYYxUSGJnPpISowsbS4RDVKAZtHiSFLTKyc6HRaDhXiYiIarVyJRUjRowweaxQKODj44MnnngCa9asqYi4qAIULCEKG2HpcIiIiIiohipXUiFJUkXHQURERERE1VSFzqkgIiIiIqLap1w9FfPmzSt127Vr15bnFEREREREVE2UK6k4d+4czp07B51Oh9atWwMArl+/DhsbG3Tp0kVup1AoKiZKIiIiIiKyWuVKKoYNGwY3Nzds374dderUAZC/IF5wcDB69eqF+fPnV2iQRERERERkvco1p2LNmjVYuXKlnFAAQJ06dfDWW2+x+hMRERERUS1TrqRCo9EUudiTSqVCZmam2UEREVUXeh1X1SYiIipXUvHUU08hODgY3333HW7fvo3bt2/j22+/xbRp0zBy5MiKjpGIyCpJkgG34+O5qjYREdV65ZpTsXnzZrz88st45plnoNPp8g9ka4tp06Zh9erVFRogEZHVkiRINlxVm4iIqFxJhbOzMzZt2oTVq1cjKioKANC8eXO4uLhUaHBERERERGT9zFr8LikpCUlJSWjZsiVcXFwghKiouIiIiIiIqJooV1KhVqvRv39/tGrVCkOGDEFSUhIAYNq0aSwnS0RERERUy5QrqZg7dy7s7OwQFxcHZ2dnefu4ceOwf//+CguOiIiIiIisX7nmVPz222/49ddf0bhxY5PtLVu2RGxsbIUERkQVQ6VSQaPRsOwpERERVZpyJRXZ2dkmPRRGaWlpcHBwMDsoIqoYKpUKE4OnIy3zHhyVAga9wdIhERERUQ1UruFPvXr1wueffy4/VigUkCQJ7733Hvr161dhwRGReTQaDdIy78G5dS+kZ+fAIDGpICIioopXrp6K9957D/3798fp06eh1WqxYMECXL58GWlpaTh27FhFx0hEZnJyr2PpEIiIiKgGK1dPRfv27XH9+nU8/vjjGD58OLKzszFy5EicO3cOzZs3r+gYiYiIiIjIipW5p0Kn02HQoEHYvHkzXnvttcqIiYiIiIiIqpEy91TY2dnh4sWLlRELEVG1pNdpWV2LiIhqtXINf5o4cSK2bNlS0bEQEVU7QjLgdnw85i9aApVKZelwiIiILKJcE7X1ej22bt2K33//HV27doWLi4vJ/rVr11ZIcERE1k5IEiRbO2Rk50Kj0cDHx8fSIREREVW5MiUVt27dQtOmTXHp0iV06dIFAHD9+nWTNgqFouKiIyIiIiIiq1empKJly5ZISkrCoUOHAADjxo3DBx98gPr161dKcEREREREZP3KNKdCCGHy+JdffkF2dnaFBkQVR61WcwVlIiIiIqp05ZqobXR/kkHWQ6VSYf7CxbidkMB/JyIiIiKqVGVKKhQKRaE5E5xDYZ00Gg3Ss3MgSQICTCqIiIiIqPKUaU6FEAJTp06Fg4MDACA3NxcvvPBCoepP3333XcVFSEREREREVq1MScWUKVNMHk+cOLFCgyEiqq6MC+A1b97c0qEQERFVuTINfwoLCyvVT2mtXLkSjzzyCNzc3FCvXj2MGDEC165dM2mTm5uLkJAQ1K1bF66urhg1ahRSUlJM2sTFxWHo0KFwdnZGvXr18Morr0Cv15u0OXz4MLp06QIHBwe0aNEC27ZtKxTPxo0b0bRpUzg6OqJHjx44depU6S8OEdVaXACPiIhqO7MmapvryJEjCAkJwV9//YUDBw5Ap9Nh4MCBJhWl5s6di//973/Ys2cPjhw5gsTERIwcOVLebzAYMHToUGi1Whw/fhzbt2/Htm3b8MYbb8htoqOjMXToUPTr1w/nz5/HnDlzMH36dPz6669ym927d2PevHlYunQpzp49i06dOiEoKAipqalVczGIqNoSkgRJaSsvgEdERFTblGtF7Yqyf/9+k8fbtm1DvXr1cObMGfTu3RsZGRnYsmULvvzySzzxxBMA8ntL2rZti7/++guPPvoofvvtN1y5cgW///476tevj86dO+PNN9/EwoULsWzZMtjb22Pz5s0ICAjAmjVrAABt27bFn3/+iXXr1iEoKAhA/irgzz33HIKDgwEAmzdvxs8//4ytW7fi1VdfrcKrQlRx1Go19Dr9gxsSERERmcGiPRX3y8jIAAB4eXkBAM6cOQOdTocBAwbIbdq0aQM/Pz+cOHECAHDixAl06NDBZAG+oKAgaDQaXL58WW5T8BjGNsZjaLVanDlzxqSNUqnEgAED5Db3y8vLg0ajMfkhsibGssIxsbHQ6ZlYEBERUeWxmqRCkiTMmTMHjz32GNq3bw8ASE5Ohr29PTw9PU3a1q9fH8nJyXKb+1f0Nj5+UBuNRoOcnBzcuXMHBoOhyDbGY9xv5cqV8PDwkH+aNGlSvhdOVEmMZYUNkoDBwKSCiIiIKo/VJBUhISG4dOkSdu3aZelQSmXRokXIyMiQf+Lj4y0dEhERERGRRVh0ToVRaGgo9u7di6NHj6Jx48bydl9fX2i1WqSnp5v0VqSkpMDX11duc3+VJmN1qIJt7q8YlZKSAnd3dzg5OcHGxgY2NjZFtjEe434ODg7yeh1ERERERLWZRXsqhBAIDQ3F999/j4MHDyIgIMBkf9euXWFnZ4fw8HB527Vr1xAXF4fAwEAAQGBgICIiIkyqNB04cADu7u5o166d3KbgMYxtjMewt7dH165dTdpIkoTw8HC5DRERERERFc2iPRUhISH48ssv8eOPP8LNzU2ev+Dh4QEnJyd4eHhg2rRpmDdvHry8vODu7o6ZM2ciMDAQjz76KABg4MCBaNeuHSZNmoT33nsPycnJWLJkCUJCQuSehBdeeAEfffQRFixYgGeffRYHDx7E119/jZ9//lmOZd68eZgyZQq6deuG7t27Y/369cjOzparQRERPYhep0VsbCzc3d3h4+Nj6XCIiIiqjEWTio8//hgA0LdvX5PtYWFhmDp1KgBg3bp1UCqVGDVqFPLy8hAUFIRNmzbJbW1sbLB37168+OKLCAwMhIuLC6ZMmYIVK1bIbQICAvDzzz9j7ty52LBhAxo3bozPPvtMLicLAOPGjYNKpcIbb7yB5ORkdO7cGfv37y80eZuIqCjGBfDmLF6GBt51sCPsMyYWRERUa1g0qRBCPLCNo6MjNm7ciI0bNxbbxt/fH/v27SvxOH379sW5c+dKbBMaGorQ0NAHxkREdD8hSZBs7eDcphfSok9Bo9EwqSAiolrDaqo/ERHVBI5unpYOgYiIqMoxqaih1Go1DHqDpcMgIiIiolqASUUNZFxJ+XZCQqmGmBERERERmYNJRQ1kXElZkgQEmFQQERERUeViUkFERERERGZhUlHL6LRaqNVqS4dBVYDzaoiIiKiqMKmoRbQ5WYiJvoX5i5ZApVJZOhyqRJxXY1lM3omIqLZhUlGLGLR5kBS2yMjOhUajsXQ4VIk4r8ZymLwTEVFtxKSCiKgCMXknIqLaiEkFERERERGZhUkFERERERGZxdbSAVDF4fhtIiIiIrIE9lTUECqVChODp2Ni8HRWnaEi6XU6SAaWmCUiIqKKx56KGkKj0SAt8x4AICsry8LRkLXR5mThdnw8oLRhiVkiIiKqcEwqiGoBgzYPQmEDSZIsHQoRERHVQBz+REREREREZmFSQUREREREZmFSQVQDqdVqGPSclE1ERERVg0kFUQ2jUqkwf+Fi3E5I4KRsIiIiqhJMKohqGI1Gg/TsHEiSgACTCktSq9VcP4aIiGoFJhVERJVAr9Ni/qIlmBg8nYkFERHVeEwqiIgqgcFgQEZ2LtIy70Gj0Vg6HCIiokrFpIKIiIiIiMzCpIKIiIiIiMzCpIKoGlCpVByXT0RERFbL1tIBEFHJVCoVJgZPhzYvD5s+WAdvb29oNBq4u7vDx8fH0uERERERMakgsnYajQYp6nTERN9C8IwX4eTkjKw8HbzcnLEj7DMmFkRERGRxHP5Uw+i0WiQmJnI15RpErVZDm5cHSWGLO3fTkZqWDufWvYqtKsTVtImIiKiqsaeiBtFptYiJvoU339+ApKRkrqZcAxRcHVsSQHJSEmwcnOH7uNsD2/Pfn4iIiKoKeypqEINBD0lhC6dmj3A15Rqi4OrYkmSApLCBQZJgMOih02qhVquLbc9/fyIiIqoqTCpqIAdXd0uHQBVApVIhNja2yKFM2pwsxETfwvxFS1gVysoVlfwRERHVNEwqiKyQseLT7FdfL3Iok0GbP8ciIztXnldRUhJClqHXaZn8ERFRrcA5FURWKDo6GqlpmvyhbNGxDxzKZExCElPvcD6FFTEYDCbJHyt1ERFRTcWeCiIrY5xsHRMbCxtHl1I9R6PRIC3zHufTEBERkUUwqSCyMsbJ1oZ/JmeXRK8zHa/vyPk0REREZAFMKoiqKUky4HZ8POYvWoKbN29Cr9NbOiQiIiKqpZhUEFVXkgRJaQt1ugavLXsTMbGx0HGStlW6v0eJiIiopmFSQVTNafNykZF1r1TDpajqFexRYgUoIiKqqZhUEFVjQjIgOSkJySmprPhkrf7pUSpY/peIiKimYUlZompMSBKEwgaQJCiVTCqIiIjIMthTQUREREREZmFSQUREREREZmFSQWRl1Go1DKziVOPodVrExsZysjYREdVITCqIrIhxNe3bCQmceF2DiH8qQM1ZvAwTg6czsSAiohqHSQWRFTGupi1JAgJMKmoK8U8FKOc2vZCWeY9VoIiIqMZhUkFEVEUc3TwtHQIREVGlYFJBRERERERmYVJRC+l1WqjVakuHQUREREQ1BJOKWkb6Z8Lo/EVLOFmUiIiIiCoEk4ra5p8JoxnZuZwsSkREREQVgkkFEVEV0mk5/JCIiGoeJhVERJXAoNdBMpguYqjNyUJM9C0OPyQiohqHSQURUQWTJAOSk5KQnJJqsoihQZsHScHhh0REVPPYWjoAIqIaR5IgKWwASbJ0JERERFWCPRVERFWMZZ2JiKimYVJBRFSFWNaZiIhqIiYVRERViWWdiYioBmJSQUREREREZmFSQUREREREZmFSQWQlVCoVJ+/WIpysTURENQlLyhJZAZVKhYnB03EvOxsGveHBT6BqTRSYrP3drp3w8fGxdEhERERmYU8FkRXQaDRIy7yH9OwcGCQmFTWd4GRtIiKqYZhUEBERERGRWZhUEBFZCOdVEBFRTcGkgojIAgQXwSMiohqESQURkQVwXgUREdUkTCqIiIiIiMgsTCqIiIiIiMgsTCqILEylUiE2NhZ6nd7SoRARERGVCxe/I7Ig46J3ial3cDshCQ3q17N0SERERERlxp6KWoqlLK2DcdE7p2aPwCAJLnxXC+l1WsTGxrICFBERVWtMKmohlrK0Po6u7pYOgSzA+F6cs3gZJgZP5/uRiIiqLYsmFUePHsWwYcPQsGFDKBQK/PDDDyb7hRB444030KBBAzg5OWHAgAG4ceOGSZu0tDRMmDAB7u7u8PT0xLRp05CVlWXS5uLFi+jVqxccHR3RpEkTvPfee4Vi2bNnD9q0aQNHR0d06NAB+/btq/DXay1YypLIOhjfi85teiFFnY7o6GhLh0RERFQuFk0qsrOz0alTJ2zcuLHI/e+99x4++OADbN68GSdPnoSLiwuCgoKQm5srt5kwYQIuX76MAwcOYO/evTh69ChmzJgh79doNBg4cCD8/f1x5swZrF69GsuWLcP//d//yW2OHz+O8ePHY9q0aTh37hxGjBiBESNG4NKlS5X34omI/qG0tUVM9C32HhIRUbVl0YnagwcPxuDBg4vcJ4TA+vXrsWTJEgwfPhwA8Pnnn6N+/fr44Ycf8PTTTyMyMhL79+/H33//jW7dugEAPvzwQwwZMgTvv/8+GjZsiJ07d0Kr1WLr1q2wt7fHQw89hPPnz2Pt2rVy8rFhwwYMGjQIr7zyCgDgzTffxIEDB/DRRx9h8+bNVXAliKg2M2jzICn+7T308fGxdEhERERlYrVzKqKjo5GcnIwBAwbI2zw8PNCjRw+cOHECAHDixAl4enrKCQUADBgwAEqlEidPnpTb9O7dG/b29nKboKAgXLt2DXfv3pXbFDyPsY3xPEREREREVDyrLSmbnJwMAKhfv77J9vr168v7kpOTUa+eaQlOW1tbeHl5mbQJCAgodAzjvjp16iA5ObnE8xQlLy8PeXl58mPOTSAiIiKi2spqeyqs3cqVK+Hh4SH/NGnSxNIhEVE1x1LPRERUXVltUuHr6wsASElJMdmekpIi7/P19UVqaqrJfr1ej7S0NJM2RR2j4DmKa2PcX5RFixYhIyND/omPjy/rSySCWq02WUnboNdBMnCtitpIYqlnIiKqxqw2qQgICICvry/Cw8PlbRqNBidPnkRgYCAAIDAwEOnp6Thz5ozc5uDBg5AkCT169JDbHD16FDqdTm5z4MABtG7dGnXq1JHbFDyPsY3xPEVxcHCAu7u7yQ9RWahUKsxfuBgxsbHQ6Q2QJAOSk5KQnJIKIYSlw6OqxlLPRERUjVk0qcjKysL58+dx/vx5APmTs8+fP4+4uDgoFArMmTMHb731Fn766SdERERg8uTJaNiwIUaMGAEAaNu2LQYNGoTnnnsOp06dwrFjxxAaGoqnn34aDRs2BAA888wzsLe3x7Rp03D58mXs3r0bGzZswLx58+Q4Zs+ejf3792PNmjW4evUqli1bhtOnTyM0NLSqLwnVIhqNBunZOTBIApJkyP9QqbCBQZIgwKSCqDgqlUr+iYqKYs8OEZEVsOhE7dOnT6Nfv37yY+MH/SlTpmDbtm1YsGABsrOzMWPGDKSnp+Pxxx/H/v374ejoKD9n586dCA0NRf/+/aFUKjFq1Ch88MEH8n4PDw/89ttvCAkJQdeuXeHt7Y033njDZC2Lnj174ssvv8SSJUuwePFitGzZEj/88APat29fBVeBiOhfxnkVzZs3t3QoVkmlUmFi8HRkZWZCobRBngR4uTljR9hnLMVLRGRBFk0q+vbtW+IwD4VCgRUrVmDFihXFtvHy8sKXX35Z4nk6duyIP/74o8Q2Y8aMwZgxY0oOmIioEokC8yq+27WTH5Lvo1KpcPbsWSSm3EFM9C0o7R3RftAkpEWdkIeM3blzB97e3rx2RERVzGpLyhIR1TZCkiDZ2nERvCKoVCqMfWYSLlyMgE4oIEEJIUmwc3GDFsDNmzexYuV7uH4jCu0faoevd27n9SMiqkJWO1GbiIjIKDo6GinqdGglRaF5RzqtFq8texMRkdeRawBS1GmIiIjgXAsioirEpKIWY018IuvE96YpY6W02wkJRQ6ZNRj0yMzJgyQJSJIet+PjMWfxMox9ZhIiIyMtEDERUe3DpKKWEqyJT2SVjO/NWfMX4NSpU7X6/alSqRAZGYmIiAioM7IgSeKBldHEP6V57Zp2wbmIywie8SITCyKiKsCkopYSrIlvcWq1GgY9F7ojU0KSYIACV67dwPRZL2Ni8PRamVgY51D0HjAYoS8X30tRHFt7R+hF/nVkYkFEVPmYVBBZwIOGc1DtZkz6ndv0QlrmvVqX+BurPKWo05EnKWDftEupeikKKZCgzQidXSuTMyKiqsLqT0QWYFz4TpIEYMOkgorm6OaJbEsHUcXur/IEpS0cXN3LfTxjRS11ugbR0dGsCEVEVEnYU0FEZMV02to1abukKk/lxTlkRESVj0kFEZGV0uZkISb6Vq34MKxSqXDq1CnMmvdKhQ8LNA4nU6drWGqWiKiScPgTEZGVMmjzIClsa/zQHZVKhYnB05GYegexsXEwGCqmh6IgY2/FnMXL0MC7DnaEfVZjrycRkSWwp4KIyIpJBUrM1tQKRtHR0UhN08Cp2SPlm5BdCgUnv6eo03H27Fn2WBARVSAmFURE1qyGVzAyVkKLiY2FjaNLpZ9PaWuLW1E3MH7qdIydMKXGXU8iIkthUlHL6XVaxMbG8g9rFeMaFVQWNXldmejoaKgzsmCQBCSp8t8TBm0eJKGEViih1mQhOjqav/+IiCoAk4parOAY49q6wJYlcI0KKi+9rmZVgrL0eyH3XjZmzV/I339ERBWASUUtVtsX2LKUgmtUVMbYcaqZamJZVEu+F4RkQGLCbURGxSAx5Q6rQhERmYlJRQ1hznAaRzfPig2GiCpcTRwCZclhgEKSIClsoNfrEBsbgzmLl2HsM5Nw7NgxJhdEROXApKIGsPQQAiKisrKW31vGZM2uaRecuXARw0Y/jRFjni5UaUulUpn8FNxGRERcp6JGKDiEADZMKohqA5VKhTt37sDb27tarrdw/9AnhYXjsbV3hCSUyDMIRFy5iuAZLyLs/z5G27ZtERkZiZdmzweEBCiUgJDw6stzsWrNBtjb23HNCyIisKeCiKjaUavVGPvMJPQeMJhlUQHodTpIhooZRiXuK+EbGRmJGS/NxPnLV5GQrEJiigrnIi5j1sIlOH/5KhJT7uDs2bOIjIys9f8ORFS7saeCiKiaycrKwp2MTORJCqSo02r0atsPos3Jwu34eEBpU2HDqIQkQbK1Q0Z2LpKSkpCenQOdXo/ExNtQ2NgBSls4NXsEuqS9iI6OwrjJwbCxsUWbVi2x7r2VCAgIkI9VW/9diKj2YVJBVMW4RgVVFKlARaj/+2gD7O3t4e7uXqs+yBq0eRAKG0iSVLkn+mdiNyQJSqWAg6t7/jYooTUACiEh4spVTJ/1Muq6OcvDpDZ9sA5t27at3NiIiKwAkwqiKmQtk1OphpAkSDa2SFGpETzjJRiUtvByc37gGH/jMB1LJh81Lbk29m44t+mF5AsHIEkCKSnJmBE6G9/t2lmrEj0iqp2YVFCNUnDyKpA/GdSavrnlGhVU0YzrLaTccUb7QZOQEnmkxOFQKpUKE4OnA4BFJhirVCpER0dj1rxXamRyrbS1lYdjQWkLdbqmVg9PI6Lag0kFVWvGJMLe3h7JyclY8NobuH4jCq1aNoe9vQOy8nRwtbexiiEIKpUKsbGxNerbWbI8IUkQ/wzLEUoFYqJvYdb8BXLlovtpNBqkZd6DTqtFREQEGjRoAABVUkXKmNAkpt5BbGwcDAapxiXXBYdjKaDH7fh4zJq/AB+seQ8BAQFMLoioxmJSQQAAnVYLtVqN5s2bWzqUUlGpVLh+/bqcRPg2bISEuBjohAJQ2uJOugYOjk5waRmIcwe+tPgQhIIfpmrit7NkHQzaPOjFv5WL/u+jDQBQaK6FTqtFTPQthL6yGCkJCYBSgY7t2+Prndsr9T1iTGicmj0CKTq2xiUU9xOSBINSiSvXbshzLazhCw4iosrAkrIEbU4WYqJvYf6iJdWiJKJKpcLYZyZh2MixuHjlGnINgE2TTtBKChgk028+7RwcICksvwqxyYcpDn2iyvTPYm4pKjUmBk9H7wGDMXrSdIx9ZpK8oJvBoIeksIW9fxfkGQTyJAXUmqxKf4+o1WrodXo4urpX6nmsScHF9c5FXEbwjBcLLaxHRFQTMKkgGLR5VvHBu7Sio6ORok43SSIcHvAhRa/T4ubNmxZPmmrThymyHOM8i+u3YpFrAJR+neQPtGfOnJGH4BV83+h1+b2VlUGlUiEyMhLzFy5GTGwsdLVwCKCtvaPci8TEgohqIiYVVK1ERkaWeYKnJBkQFxuD8VOnc6EwqhXEP+VPjUm38QPtpcireC5kNuJv3zZ5/4gCpWkjIyMRFRVVYe8T49C/GS/NhDojCwZJQJJqX1IBAChiYb2KvNZERJbEORVUbahUKsx4aSau3LhVtgmexlryQikP8eBkSap1/nkfSAKAyF9rwchYDlWdrsGM0NnI1YtSlaYtTsECCjdv3kRqmgZ6vRaG2ppMFFDUtbaWYhJEROZgTwVVCyqVChEREVBnZJk1J6Eyh3iUxDiWnMiaGQwGZGTnwrl1L6So03H27NlC36KrVKpiv1k3DnMa+8wk9B4wGCOenozxk59FdEwMq57dx3itbf0414KIagb2VJDVq6jKScYhHsbyjm5ublVSRtM4ZCsmNh5tWvWs1HMRVQQbW1vcirqB8VOno23rVnjv7RXw8vJCZmYmFi5ZBnt7O+wI+0xu7+PjI79P72VnQ52RhTxJkV9AISYWUEjspSiGnYODScUuLpRHRNUVkwqyetHR0UhN05hdhrJgecepL86CKjkZ7R9qV6llNAsO2cqvXc8PVmQevU6LxMTESv3mX6/NhSSUyDMIXLh0GUNHjoGNjS3q+zZEUkoqGjdsgIMHD2Lzlu2AkPDu2ytw9+5dpKZpkJuTDb0+v1fuQQUU6B//rIzOhfKIqDpjUkFWTaVSyRVjKuJbfuN4Znv/LshL/KXS51gUXEEbNiwjS+Yx9ra9+f4GJCUlV/p6J0KSIKCE1gAoFQrYN+0CXcJeREdHYUboHAilHRRCwtQXZyH5djz0sIHQawEbW0BZ/f+8VEUCZ1Rwsjx7K4ioOuKcCpJZar5BSYwfyiu6Ygy/QaXqyLjmgaXWO3FwdZcnfOuEAnqDHgalDez9u0Ar/fNYoSy0Xkx1VDCBq4oFK43/tup0Dc6ePcuqUERU7TCpIAD5ZVeN35JZ0x8ytVpducM8KiCRKmniamXHT7WTtSXF1hZPRbBEAicKlL8uuGBhSb9jrElxcRon8BfcV11eExGVXvXvn6aK8c+YXuMCeNbQ9W6yJoXCpsKPL5kx3MBYMrPgxNX1q1fB29sbQH4PS3JyMuYvXCx/y6mo8FdARJWtKhMmebiZUOYvWHj4W0wMngYnJxe4uJS/xG9lMP4ONBa7iIyMxEuz5wNCwqYP1sHb2xsajQZarRYvzZqLS1euolXL5njv7RWws7PDwiXLACHhrWWvw8vLCwBgb28Pd3d3q3mNRFQ2TCrIKt2/JoXCthI61co5OVKlUmHsM5Nw6cpV+NT3RVJKKnx9vBE84yXYKBVQKG2QlatF3K2b0AmF/C0nkwoiKi3jgoXXbkTBxsEZzZo2lb/wMX7Db6kP35GRkZj+Yiiu34hCq5bN8caiBVj65juIvBkDSAY5EcqTABt9HtIy7yFHL+HCpcsYNvpp+DZohITEJEh6LYaOHAMFFFDa2MCvaXPU83IvVFmMiKoHDn8iq1RwgnN5hh0Y9DoY9A9eF6JgmdnS1IhXqVSIjo7GnYxM5En/TFzV6xEfH4vL16Nw8dIVRFy7CZtGHaCVFDVibDkRWUiBldHz8nJw8+ZNnDp1CmMnTMHYZybh2LFjVTaESKVSISoqCpGRkZjx0kxERF5Hjs6AiCtXMWvhEly5cQt6vR56ANduRCHi2k0oGnXE5Ws3kJScDGEwyIuQ2jftAoPBAAMU0BqAPINAnqSAQ6ueSMu8h7Nnz2LshCmYGDxdXnX8/uFTRGR92FNBVsmcuQhCMiA5KQk2Ds5wb1/yMQqWmX1QjfiCdfiNsckTVxU2gEEPKJQQkgRbZ9dyxU5EdL+Ccy0Kfss/bPTTaNu6FT77+COzV+O+fziT8TEApKWlYcnyt5GVp4OjUsiLkEpCAmzt8uedJPyU/wWKJPKHq0oSFLZ2/5TSlkzOVdyQMkc3T6RrtXht2ZuIikuUe4Bz9RIS4mLRqmVzfPbxR/D29pZXbOdwKSLrwaSCrI6xjGx5K64ISZL/qJWmYpSxzKw6XYOIiAh06NABAOQ/qED+WN+bN28iNU0DvV7LhbyoRsrv4bOrtGMrFOwcNyrLtS4418K+aRcY4n+CBAUkg0DElasInvEiPljzHgICAuTnlOWDdlHDmVasfA9Xr16HJKT838O2Dnho0CSkXwov9PuvIuedGAx65Gi1cg9wooMzWvQcipxb0Yi4chUTg6fB1sYOt2Ji0dg/AJ4uDnhr2eto1aoVkwsiC2NSQVZFpVIhIiJC/iasqoYOGYdBzVm8DG4OttDrDbgZdQuSkKC0sUHDRk2QEBcDycYeDX3rV0lMRFWpLD185T22wsauRqxfYS5zrnXBD/AFe1qnz3oZbg62UCht4OLijPWrVwHI/0JEq9XKPRAFqVQqXL9+HfMXLsaVG7cgCcjDmZKSkmEw/Dt8UylJsHNxM/OVl4GxB9jY8ytJMCjy55gY7yOlXyecOfi1SY9NwQniAIp83URUOfjbnayGcXhRYuqdKqkLX5Cxt8KuaRdc/H03oLSRkxqlQgGbJp2gjYkFFBJ7KahGKmsPX3mPrVRyjlFFXuv7f3cp7R3h16ghgme8iKhbMfBt2AjJiQly5SVjpaXMzEzMXbAIkZFXTQpKFBrOZEXuv49s7R3lld+NvRhOTi7IytUiPvoWoFSgTauWeO/tFezJIKoCTCrIakRHRyM1TZP/By061iJ/0GztHcs0BpiIyBoYf3cZJAlanRZanRZ5Uv4XIjlx8bhw6bJJpaX6vg1xOyEBBgMgUL1/3xl7bIyVslr0HIq8m1EQkpArTnXq0AFf79wuP4cJBlHF4wBXMmGpVbWN8yhiYmNh4+hS5ecnIqqJCq6CXrDSkn3TLhZZlb2yiAKVsoyFMoTxdQsl1JosREdHY2LwdLlyVmRkpFxdipWliMzHngqSCTMWgzNXdHQ01BlZMEiiwodeEBGRqerWG1ERsrKykKJOx62oG0Wuj2FcwJS9GETlw54KkglJgqT8d1XtqmJutSciIqLSMBj0kISy0PoYCcmpCJ7xEsY+M6lUaxYRUWHsqSCLK7jQHWysP6lgaUwiqq30Oh0kQ+X1Jlf28YuitLVFbEwMYGMHpZAQPONFrFn1Nnx9fbkOBlEZMKkgizNnobuqxtKYRFRbaXOycDs+HlDaVEqvcmUfvzgGbV5+gQ6DHpJSiUuRVzFs9NPwD2jBdTCIyoBft5JFVbehTwUnA9aUCY5EVLMY9JXzbb/xw3dl/f6r7OOXRsHJ3Uq/Tjhz4SKGjX4aI8Y8jWPHjnFCN1EJmFSQRRUc+sQP6URE5jH2pianpFaLL2qqil6nRWJiYpl6xQuug2EsTTtizNM4deoUkwuiInD8BlUplUqFO3fuAMhf6dUS5WuJiGoqLjRYmLGy4Zvvb0BSUnKZky0hSRD4d5G96bNeRl03Z7y17HV4eXmxYhTRP5hUUJWJjIzE9BdDcfXqdUCpgF/T5nB1sKk28ymIiKj6Ma46bu5K4QVXLz9z8GsMHTkGNja26Ni+Pb7eud3qEguVSgWNRgOtVgsATH6o0jGpoCqhUqkw46WZiIi8DoNBgsLWFg6teiL9ymEYuC4FERFVsopam8M4LEovSVAqFPLCegU/wBtV9Qd542iAtLQ0LFn+Nu5mZiM++hagVKBNq5b47OOP0LZt2yqLh2oXJhVUiHFV7ebNm5t9LOM3JTdv3oQ6I0ueO6EA4OjmiWzzwyUiIrIYvU6L+YuWIPNeHuKjb0ESEgBAaWODNq1a4r23V8DX19ck4ajIZMP4dzY5ORkLXnsDV69ez/+yztYBzXsORd7NKAgpf+jW/eVyjdiDQRWBSQWZqMhVtVUqFSYGT0eKOh1xt25CJxScOEhERDWKwWBARnYuHJt2Qd7NKEj//J1TCEme4N2osR8S4+MgCanEZAMofcKhUqlw/fp1uUfC+Hc2/8s7QClJsHV2BZA/dMtQRLlcKJSwt7fDjrDPmFiQ2ZhUkAnjmFHjqtrm/JKJjo5GapoG9v4PQ3szSv72piBLLHRERERU0RzvG15lnOCtFUrYNOmEvJhYSEIUm2wA//ZurHtvJdzc3ODt7Q0AuHPnjpxsGJOJBa+9gcjIq5Bs7NG859Bi/84WFY9Dq564c+UwFEobKBUKnD17Fn5+frC3t+eCf1RuTCqowhl/4c1fuBgxsfFo06pnke0stdARERFRVSo4n6O4ZAPI792IuHIVU1+cBVVyMlq1zB+GfP1GFFq1bI43Fi3AipXvITLy6j+9EoBC8W+PRGkZhx/rdVrcjo/DuMnBsLGxhV/T5qjn5Y71q1fJbTnBm0qLSQUVSa/TIjY2tszfWKhUKox9ZhIuXIyATigApS2kYiZiy6uYSsV/s0JERGTNDHodDHq7cj33/snjxtEC9v5dkJf4C+6kawAAOfr8ZGPWwiVISkqGwQAI5P/tVJgTu8FgMuncoVVPJFw6iInB0xETEydP8C7Yc8IEg4rDpIIKMc6rmLN4GRp41ynTWEuNRoM7GZnQSor8saOsk05ERDWUcbFBGwdnuLevuKG8hSpVSRIkG1uzy+I+iNLWFrExMYDSBpIkAKXCpOek/UPtsGnDGgBF92AYFwVk4lE7MamgQozflDi36YW06FNlmluhVqu57gQREdUKBRcbLK5XviJVVFlcoOg5jYVGEEjCpOckITkFE4OnIy4+Aa1aNpd7MADIZWyLm/htrFLFORs1F5MKKlZZS76qVCrMX7gYtxMSOEeCiIjITAa9DgqFssKPW9Y5jQ6u7pAkAxITbkNhYweBf3swUhISIAkp/zi2Dgho2hQaTf6wrTt37gD4N+HIytPB1d5GXo3cyN7evtAifQWTEOOxitpnTFDYS2J5TCqowmg0GqRn58hrURAREVH5SP8MrVLY2EEIYdbcifuVa06jJEH6p1cGAGDswYi7LU80V0oSDJIBarUa018IwcWIyyYJR6u+I3Hm968wdOQYKP55RUobGzRs1ASJ8f/O4Vj08lys+WATsvJ0sIcBer0Bt2Ji0f6hdnhr6WuFEhQAWLL8bUBIePftFSaVswDzEg3jgoLG492/yOH9w8Bqc3LDpIJKpNOWbiE8lUqF2NhYDn0iIiKqCPd9iK/IpKKiFDccKysrC3cyMpFnECYJh8LWTp4Yjn++fFQqFHIVLCEJXLh0GROCn4PCzhGt+o7Exd++BJQ2gNIWKeo0zH55IaLiEtGyz1NyggIBSApbKIQkz/8I8G8CewdH2NnmJx4F1wQpWKrX2EtScBuQ33tiXFDw+o0o+XhZuVp5kcP71xxJTk6Wk5tNH6yrdauXM6mgYmlzshATfeuBC+EZF7lLTL3DoU9ERES1mF6nRWJiYpm+ZDQmJ8Zyu5L4Nwkx9qgolQIGgwGZOQYYJHFfggIAeij/6T3Jub0XV65dh8LWATDoTNYEgVKB5k39oVAocSsmFr4NGyE5MUEu33v16nVAqUDDRk2QEBeTv3AvFLhy7TpsHJzR4p9VyotacyQhLgZ62EAhJJPVy41JS03vvWBSQcUyaPMgKWwfuBCecZE7p2aPQIqO5dAnIiKiWshYPfLN9zcgKSnZIl8yOri6/9vLY9ADUEAUWBNESCI/4bCxA5S2+dtvJ8rle/MMAkB+e21M7L8LCv7Ta1RwTZD71xzJb6+HosDq5Y0a+yE5MQHtH2qHr3dur9GJRcXP/qFaQ6VS4dixY5g17xXExMbCxtHF0iERERGRhQhJgqT8p/RtJcyvNOgLV6wqrYK9IZLCBgZJgoAodghXWStt3b/AoVRwgUNJAbUmS57AXlOxp4JKRa1Wm3TfRUZGYvqLofKqniUtckdERES1R0WWvjUSBSauQ1nxH18rq9JWZVwLa8Wkgh5Ir9Ni1vwFiLoVg1Ytm+ONRQuw9M13cOXGLXlVTy5yR0RERJWl4JogFf2ZozIrbdUmTCrogbR5uVBrtcjRS4i4chWzFi5BUlIyDAaJ8yeIiIioeqsGlbaqA86poBIZs/fklFQIg6FSx0oSERERUfXEngoqWRHZe20aH0hERERED8aeCiIiIiIiMguTivts3LgRTZs2haOjI3r06IFTp05ZOiQiIiIiIqvGpKKA3bt3Y968eVi6dCnOnj2LTp06ISgoCKmpqZYOjYiIiIjIajGpKGDt2rV47rnnEBwcjHbt2mHz5s1wdnbG1q1bLR0aEREREZHVYlLxD61WizNnzmDAgAHyNqVSiQEDBuDEiRMWjIyIiIiIyLqx+tM/7ty5A4PBgPr165tsr1+/Pq5evVqofV5eHvLy8uTHGRkZAGCRJdgzMzNhMOghJAlQKHDv7p38RWKEJLcRklQjt1tjTNV9uzXGVN23W2NM1X27NcZU3bdbY0zVfbs1xlTdt1tjTKXZbtDrkZmZWeWfE43nE6LylwFQiKo4SzWQmJiIRo0a4fjx4wgMDJS3L1iwAEeOHMHJkydN2i9btgzLly+v6jCJiIiIiMokPj4ejRs3rtRzsKfiH97e3rCxsUFKSorJ9pSUFPj6+hZqv2jRIsybN09+LEkS0tLSULduXSgU5q/FqNFo0KRJE8THx8PdnetCPAivV9nwepUNr1fp8VqVDa9X2fB6lQ2vV9nUxOslhEBmZiYaNmxY6ediUvEPe3t7dO3aFeHh4RgxYgSA/EQhPDwcoaGhhdo7ODjAwcHBZJunp2eFx+Xu7l5jbuyqwOtVNrxeZcPrVXq8VmXD61U2vF5lw+tVNjXtenl4eFTJeZhUFDBv3jxMmTIF3bp1Q/fu3bF+/XpkZ2cjODjY0qEREREREVktJhUFjBs3DiqVCm+88QaSk5PRuXNn7N+/v9DkbSIiIiIi+heTivuEhoYWOdypqjk4OGDp0qWFhlhR0Xi9yobXq2x4vUqP16pseL3KhterbHi9yobXyzys/kRERERERGbh4ndERERERGQWJhVERERERGQWJhVERERERGQWJhVERERERGQWJhVWauPGjWjatCkcHR3Ro0cPnDp1ytIhVbmVK1fikUcegZubG+rVq4cRI0bg2rVrJm369u0LhUJh8vPCCy+YtImLi8PQoUPh7OyMevXq4ZVXXoFer6/Kl1Illi1bVuhatGnTRt6fm5uLkJAQ1K1bF66urhg1alShFeRry7UCgKZNmxa6XgqFAiEhIQBq97119OhRDBs2DA0bNoRCocAPP/xgsl8IgTfeeAMNGjSAk5MTBgwYgBs3bpi0SUtLw4QJE+Du7g5PT09MmzYNWVlZJm0uXryIXr16wdHREU2aNMF7771X2S+tUpR0vXQ6HRYuXIgOHTrAxcUFDRs2xOTJk5GYmGhyjKLux1WrVpm0qQ3XCwCmTp1a6FoMGjTIpA3vr38V9XtMoVBg9erVcpvacn+V5nNDRf0tPHz4MLp06QIHBwe0aNEC27Ztq+yXZ/0EWZ1du3YJe3t7sXXrVnH58mXx3HPPCU9PT5GSkmLp0KpUUFCQCAsLE5cuXRLnz58XQ4YMEX5+fiIrK0tu06dPH/Hcc8+JpKQk+ScjI0Per9frRfv27cWAAQPEuXPnxL59+4S3t7dYtGiRJV5SpVq6dKl46KGHTK6FSqWS97/wwguiSZMmIjw8XJw+fVo8+uijomfPnvL+2nSthBAiNTXV5FodOHBAABCHDh0SQtTue2vfvn3itddeE999950AIL7//nuT/atWrRIeHh7ihx9+EBcuXBD//e9/RUBAgMjJyZHbDBo0SHTq1En89ddf4o8//hAtWrQQ48ePl/dnZGSI+vXriwkTJohLly6Jr776Sjg5OYlPPvmkql5mhSnpeqWnp4sBAwaI3bt3i6tXr4oTJ06I7t27i65du5ocw9/fX6xYscLkfiv4u662XC8hhJgyZYoYNGiQybVIS0szacP7618Fr1NSUpLYunWrUCgUIioqSm5TW+6v0nxuqIi/hbdu3RLOzs5i3rx54sqVK+LDDz8UNjY2Yv/+/VX6eq0Nkwor1L17dxESEiI/NhgMomHDhmLlypUWjMryUlNTBQBx5MgReVufPn3E7Nmzi33Ovn37hFKpFMnJyfK2jz/+WLi7u4u8vLzKDLfKLV26VHTq1KnIfenp6cLOzk7s2bNH3hYZGSkAiBMnTgghate1Ksrs2bNF8+bNhSRJQgjeW0b3f4iRJEn4+vqK1atXy9vS09OFg4OD+Oqrr4QQQly5ckUAEH///bfc5pdffhEKhUIkJCQIIYTYtGmTqFOnjsm1WrhwoWjdunUlv6LKVdSHvvudOnVKABCxsbHyNn9/f7Fu3bpin1ObrteUKVPE8OHDi30O76/vS2wzfPhw8cQTT5hsq6331/2fGyrqb+GCBQvEQw89ZHKucePGiaCgoMp+SVaNw5+sjFarxZkzZzBgwAB5m1KpxIABA3DixAkLRmZ5GRkZAAAvLy+T7Tt37oS3tzfat2+PRYsW4d69e/K+EydOoEOHDiarogcFBUGj0eDy5ctVE3gVunHjBho2bIhmzZphwoQJiIuLAwCcOXMGOp3O5L5q06YN/Pz85Puqtl2rgrRaLXbs2IFnn30WCoVC3s57q7Do6GgkJyeb3EseHh7o0aOHyb3k6emJbt26yW0GDBgApVKJkydPym169+4Ne3t7uU1QUBCuXbuGu3fvVtGrsYyMjAwoFAp4enqabF+1ahXq1q2Lhx9+GKtXrzYZblHbrtfhw4dRr149tG7dGi+++CLUarW8j/dX8VJSUvDzzz9j2rRphfbVxvvr/s8NFfW38MSJEybHMLap7Z/TuKK2lblz5w4MBoPJzQwA9evXx9WrVy0UleVJkoQ5c+bgscceQ/v27eXtzzzzDPz9/dGwYUNcvHgRCxcuxLVr1/Ddd98BAJKTk4u8lsZ9NUmPHj2wbds2tG7dGklJSVi+fDl69eqFS5cuITk5Gfb29oU+xNSvX1++DrXpWt3vhx9+QHp6OqZOnSpv471VNONrK+q1F7yX6tWrZ7Lf1tYWXl5eJm0CAgIKHcO4r06dOpUSv6Xl5uZi4cKFGD9+PNzd3eXts2bNQpcuXeDl5YXjx49j0aJFSEpKwtq1awHUrus1aNAgjBw5EgEBAYiKisLixYsxePBgnDhxAjY2Nry/SrB9+3a4ublh5MiRJttr4/1V1OeGivpbWFwbjUaDnJwcODk5VcZLsnpMKqhaCAkJwaVLl/Dnn3+abJ8xY4b8/x06dECDBg3Qv39/REVFoXnz5lUdpkUNHjxY/v+OHTuiR48e8Pf3x9dff11rf8GV1pYtWzB48GA0bNhQ3sZ7iyqaTqfD2LFjIYTAxx9/bLJv3rx58v937NgR9vb2eP7557Fy5Uo4ODhUdagW9fTTT8v/36FDB3Ts2BHNmzfH4cOH0b9/fwtGZv22bt2KCRMmwNHR0WR7bby/ivvcQJWHw5+sjLe3N2xsbApVIkhJSYGvr6+ForKs0NBQ7N27F4cOHULjxo1LbNujRw8AwM2bNwEAvr6+RV5L476azNPTE61atcLNmzfh6+sLrVaL9PR0kzYF76vaeq1iY2Px+++/Y/r06SW2472Vz/jaSvod5evri9TUVJP9er0eaWlptfZ+MyYUsbGxOHDggEkvRVF69OgBvV6PmJgYALXvehXUrFkzeHt7m7z3eH8V9scff+DatWsP/F0G1Pz7q7jPDRX1t7C4Nu7u7rX6SzwmFVbG3t4eXbt2RXh4uLxNkiSEh4cjMDDQgpFVPSEEQkND8f333+PgwYOFumaLcv78eQBAgwYNAACBgYGIiIgw+QNk/IPerl27SonbWmRlZSEqKgoNGjRA165dYWdnZ3JfXbt2DXFxcfJ9VVuvVVhYGOrVq4ehQ4eW2I73Vr6AgAD4+vqa3EsajQYnT540uZfS09Nx5swZuc3BgwchSZKcnAUGBuLo0aPQ6XRymwMHDqB169bVcqhFSYwJxY0bN/D777+jbt26D3zO+fPnoVQq5WE+tel63e/27dtQq9Um7z3eX4Vt2bIFXbt2RadOnR7YtqbeXw/63FBRfwsDAwNNjmFsU9s+pxVi4YniVIRdu3YJBwcHsW3bNnHlyhUxY8YM4enpaVKJoDZ48cUXhYeHhzh8+LBJGbx79+4JIYS4efOmWLFihTh9+rSIjo4WP/74o2jWrJno3bu3fAxjabiBAweK8+fPi/379wsfH58aUfbzfvPnzxeHDx8W0dHR4tixY2LAgAHC29tbpKamCiHyy+j5+fmJgwcPitOnT4vAwEARGBgoP782XSsjg8Eg/Pz8xMKFC0221/Z7KzMzU5w7d06cO3dOABBr164V586dk6sVrVq1Snh6eooff/xRXLx4UQwfPrzIkrIPP/ywOHnypPjzzz9Fy5YtTUp+pqeni/r164tJkyaJS5cuiV27dglnZ+dqV8JSiJKvl1arFf/9739F48aNxfnz501+lxkryRw/flysW7dOnD9/XkRFRYkdO3YIHx8fMXnyZPkcteV6ZWZmipdfflmcOHFCREdHi99//1106dJFtGzZUuTm5srH4P11zqR6WEZGhnB2dhYff/xxoefXpvvrQZ8bhKiYv4XGkrKvvPKKiIyMFBs3bmRJWcGSslbrww8/FH5+fsLe3l50795d/PXXX5YOqcoBKPInLCxMCCFEXFyc6N27t/Dy8hIODg6iRYsW4pVXXjFZS0AIIWJiYsTgwYOFk5OT8Pb2FvPnzxc6nc4Cr6hyjRs3TjRo0EDY29uLRo0aiXHjxombN2/K+3NycsRLL70k6tSpI5ydncVTTz0lkpKSTI5RW66V0a+//ioAiGvXrplsr+331qFDh4p8702ZMkUIkV9W9vXXXxf169cXDg4Oon///oWuoVqtFuPHjxeurq7C3d1dBAcHi8zMTJM2Fy5cEI8//rhwcHAQjRo1EqtWraqql1ihSrpe0dHRxf4uM66JcubMGdGjRw/h4eEhHB0dRdu2bcU777xj8iFaiNpxve7duycGDhwofHx8hJ2dnfD39xfPPfdcoS/VeH/9+34UQohPPvlEODk5ifT09ELPr03314M+NwhRcX8LDx06JDp37izs7e1Fs2bNTM5RWymEEKKSOkGIiIiIiKgW4JwKIiIiIiIyC5MKIiIiIiIyC5MKIiIiIiIyC5MKIiIiIiIyC5MKIiIiIiIyC5MKIiIiIiIyC5MKIiIiIiIyC5MKIiKyGn379sWcOXMsHQYREZURkwoiIqoQw4YNw6BBg4rc98cff0ChUODixYtVHBUREVUFJhVERFQhpk2bhgMHDuD27duF9oWFhaFbt27o2LGjBSIjIqLKxqSCiIgqxH/+8x/4+Phg27ZtJtuzsrKwZ88ejBgxAuPHj0ejRo3g7OyMDh064KuvvirxmAqFAj/88IPJNk9PT5NzxMfHY+zYsfD09ISXlxeGDx+OmJiYinlRRERUKkwqiIioQtja2mLy5MnYtm0bhBDy9j179sBgMGDixIno2rUrfv75Z1y6dAkzZszApEmTcOrUqXKfU6fTISgoCG5ubvjjjz9w7NgxuLq6YtCgQdBqtRXxsoiIqBSYVBARUYV59tlnERUVhSNHjsjbwsLCMGrUKPj7++Pll19G586d0axZM8ycORODBg3C119/Xe7z7d69G5Ik4bPPPkOHDh3Qtm1bhIWFIS4uDocPH66AV0RERKXBpIKIiCpMmzZt0LNnT2zduhUAcPPmTfzxxx+YNm0aDAYD3nzzTXTo0AFeXl5wdXXFr7/+iri4uHKf78KFC7h58ybc3Nzg6uoKV1dXeHl5ITc3F1FRURX1soiI6AFsLR0AERHVLNOmTcPMmTOxceNGhIWFoXnz5ujTpw/effddbNiwAevXr0eHDh3g4uKCOXPmlDhMSaFQmAylAvKHPBllZWWha9eu2LlzZ6Hn+vj4VNyLIiKiEjGpICKiCjV27FjMnj0bX375JT7//HO8+OKLUCgUOHbsGIYPH46JEycCACRJwvXr19GuXbtij+Xj44OkpCT58Y0bN3Dv3j35cZcuXbB7927Uq1cP7u7ulfeiiIioRBz+REREFcrV1RXjxo3DokWLkJSUhKlTpwIAWrZsiQMHDuD48eOIjIzE888/j5SUlBKP9cQTT+Cjjz7CuXPncPr0abzwwguws7OT90+YMAHe3t4YPnw4/vjjD0RHR+Pw4cOYNWtWkaVtiYiocjCpICKiCjdt2jTcvXsXQUFBaNiwIQBgyZIl6NKlC4KCgtC3b1/4+vpixIgRJR5nzZo1aNKkCXr16oVnnnkGL7/8MpydneX9zs7OOHr0KPz8/DBy5Ei0bdsW06ZNQ25uLnsuiIiqkELcP1iViIiIiIioDNhTQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQUREREREZmFSQWQFmjZtiqlTp1o6jGpJr9djwYIFaNKkCZRKJUaMGFGp51MoFPLP+++/X6nnotrr1VdfRY8ePcr0HEmS0L59e7z99tuVFBVZM51OhyZNmmDTpk2WDoVqKSYVRBVs27ZtUCgUOH36dJH7+/bti/bt25t9nn379mHZsmVmH6e627p1K1avXo3Ro0dj+/btmDt3bqWf86mnnsIXX3yBoUOHytuuXr2KBQsWoHPnznBzc0ODBg0wdOjQYu+DhIQEjB07Fp6ennB3d8fw4cNx69YtkzbGe6m4n507d8ptv//+ewQFBaFhw4ZwcHBA48aNMXr0aFy6dKlUr6mk8zz55JNyu5iYmGLb7dq1qyyXsVh//vmnfMw7d+6Y7Lt27Rrmzp2Lnj17wtHREQqFAjExMUUeZ/fu3Zg4cSJatmwJhUKBvn37FtkuKysLS5cuxaBBg+Dl5QWFQoFt27aVOt6S/p2Sk5MLtf/pp5/QpUsXODo6ws/PD0uXLoVerzdpM2fOHFy4cAE//fRTqeP46quvEB8fj9DQ0EL7oqKi8Pzzz6NZs2ZwdHSEu7s7HnvsMWzYsAE5OTmlPkdl2rRpU5mue1Eq430IFP/+WLVqlUm70t6fhw8fLvE9VzAxDA8Px7PPPotWrVrB2dkZzZo1w/Tp05GUlGRyTDs7O8ybNw9vv/02cnNzy3H1iMxja+kAiCj/D5FSWbYcf9++fdi4cWOtTywOHjyIRo0aYd26dVV2zo4dO2LixIkm2z777DNs2bIFo0aNwksvvYSMjAx88sknePTRR7F//34MGDBAbpuVlYV+/fohIyMDixcvhp2dHdatW4c+ffrg/PnzqFu3LgCgd+/e+OKLLwqdf926dbhw4QL69+8vb4uIiECdOnUwe/ZseHt7Izk5GVu3bkX37t1x4sQJdOrUqcTXVNR5Tp8+jQ0bNmDgwIGF9o0fPx5Dhgwx2RYYGFjiOUpDkiTMnDkTLi4uyM7OLrT/xIkT+OCDD9CuXTu0bdsW58+fL/ZYH3/8Mc6cOYNHHnkEarW62HZ37tzBihUr4Ofnh06dOuHw4cPliv3/2bvzuKiq/3/gr2GXZViURRQQl1RcU1Mpc8kFk1zJJTX3TMO91CxzL7fccq1UtE+SpWVpmoq4J6WiuOeCLJqAJMIAyn5+f/ib++UyMywOA4O8no/HPHTOOXPuuXfuXOY99ywLFiyAt7e3LM3BwUH2/I8//kCfPn3QsWNHrF27FleuXMGiRYvw8OFDbNy4USrn5uaG3r1748svv0SvXr2Ktf3ly5dj0KBBsLe3l6Xv378f/fv3h6WlJYYNG4bGjRsjKysLp0+fxvTp03Ht2jV88803z7XPpWnDhg2oVq2aXndtDfE5VOvatSuGDRsmS3v55Zdlz4t7fjZs2FDrZ+5///sfDh8+LPvMzZw5E0lJSejfvz/q1auHu3fvYt26dfj9998REREBNzc3qezIkSPx8ccfIzg4GKNGjSr2cSMqFYKISlVQUJAAIM6dO6c1v0OHDqJRo0Z6bycwMFCU50c4LS2t3LadX6dOnUrleKrl5uaKp0+f6swHIObOnauRfv78eZGamipL+++//4Szs7N47bXXZOlLly4VAMTZs2eltBs3bghTU1Mxa9asQtv35MkTYWdnJ7p27VrkvsTHxwszMzPx/vvvF1lWm9GjRwuFQiHu3bsnpUVFRQkAYvny5c9VZ1E2btwoqlatKiZPniwAiMTERFn+o0ePhEqlEkIIsXz5cgFAREVFaa0rNjZW5ObmCiGEaNSokejQoYPWchkZGSIuLk4IIcS5c+cEABEUFFTsNhf1mc/Px8dHNGvWTGRnZ0tpn376qVAoFOLGjRuysrt37xYKhUJERkYWWe+FCxcEAHHkyBFZ+t27d4Wtra1o0KCBePDggcbrbt++LVavXl1k/WWhsPeouAz1OQQgAgMDi9x+Sc5PberWrSvq1asnSztx4oR0HudPAyA+/fRTjTreeust8frrrxd7m0Slhd2fiIxAwTEV2dnZmD9/PurVqwcrKytUrVoV7dq1Q0hICABgxIgRWL9+PQD5bXm19PR0fPjhh/Dw8IClpSXq16+PL7/8EkII2XafPn2KSZMmoVq1arCzs0OvXr3w77//QqFQyO6AzJs3DwqFAtevX8fgwYPh6OiIdu3aAQAuX76MESNGSN0q3NzcMGrUKI1fhtV13Lp1C0OHDoW9vT2cnZ3x2WefQQiBe/fuoXfv3lAqlXBzc8OKFSsKPWbqbjjHjh3DtWvXpGOg/pW5uMdAoVBgwoQJ2LFjBxo1agRLS0scPHiw6DetgJYtW8LW1laWVrVqVbz++uu4ceOGLH337t145ZVX8Morr0hpDRo0QOfOnfHTTz8Vup19+/YhNTUVQ4YMKbJNLi4usLa2RnJycvF35P/LzMzEzz//jA4dOqBmzZpay6SnpyMrK6vEdeuSlJSE2bNnY8GCBRq/8Ks5OTnBzs6uWPWpx9kUxdLSUvZrrz5SU1ORm5urNe/69eu4fv06xo4dCzOz/+so8MEHH0AIgd27d8vKq39V/+2334rc7q+//goLCwu0b99elr5s2TKkpaVhy5YtqF69usbr6tati8mTJ0vPc3JysHDhQtSpUweWlpaoVasWPvnkE2RmZspeV/AaoVbwWqbuGvbnn39i2rRpcHZ2ho2NDfr27YvExETZ665du4YTJ05In+X83dUiIyMRGRlZ5HEw9Ofw6dOnhXYtKsn5WdDZs2dx584djc92+/btNc7j9u3bw8nJSWOfgGd3VE6fPo2kpKTnagfR82JQQWQgKSkp+O+//zQe2dnZRb523rx5mD9/Pjp16oR169bh008/haenJy5cuAAAeP/996V+7v/73/+kBwAIIdCrVy+sWrUK3bt3x8qVK1G/fn1Mnz4d06ZNk21nxIgRWLt2LXr06IGlS5eiSpUqsnECBfXv3x9PnjzBF198gffeew8AEBISgrt372LkyJFYu3YtBg0ahJ07d6JHjx4aX+ABYODAgcjLy8OSJUvQpk0bLFq0CKtXr0bXrl1Ro0YNLF26FHXr1sVHH32EkydP6myLs7Mz/ve//6FBgwaoWbOmdAwaNmxYomMAPOtCNXXqVAwcOBBr1qxBrVq1Cn+DSiA+Ph7VqlWTnufl5eHy5cto1aqVRtnWrVsjMjISqampOuvbsWMHqlSpgn79+mnNT05ORmJiIq5cuYIxY8ZApVLJukkV14EDB5CcnKwzeJk/fz5sbW1hZWWFV155BYcPHy7xNgr67LPP4Obmhvfff1/vuspDp06doFQqYW1tjV69euH27duy/IsXLwKAxnvv7u6OmjVrSvlq9vb2qFOnDv78888it33mzBk0btwY5ubmsvR9+/ahdu3aePXVV4u1D2PGjMGcOXPQokULqSvQ4sWLMWjQoGK9XpeJEyfi0qVLmDt3LsaPH499+/bJxn6sXr0aNWvWRIMGDaTP8qeffirld+7c+bnOY7XS+Bxu27YNNjY2qFKlCnx8fBAcHPzc7dFGPUaqOD8YpKWlIS0tTbZPai1btoQQAmfOnCnV9hEVqfxukhC9mNRdIQp7FOyu4+XlJYYPHy49b9asmfD39y90O7q6P/36668CgFi0aJEs/e233xYKhULcuXNHCCFEeHi4ACCmTJkiKzdixAiNLj5z584VAMQ777yjsb0nT55opP3www8CgDh58qRGHWPHjpXScnJyRM2aNYVCoRBLliyR0h8/fiyqVKkiOya6aOtOVtxjIMSzbg0mJibi2rVrRW5LXV5b9ydtTp48KRQKhfjss8+ktMTERAFALFiwQKP8+vXrBQDxzz//aK3v0aNHwsLCQgwYMEDnNuvXry+dZ7a2tmL27NkaXSeKIyAgQFhaWorHjx/L0mNiYkS3bt3Exo0bxd69e8Xq1auFp6enMDExEb///nuJt6N26dIlYWpqKg4dOiSE+L/zpWD3p/xK0r2kuF1rnqf7048//ihGjBghtm/fLvbs2SNmz54trK2tRbVq1URsbKxGe/Onqb3yyiuibdu2GundunUTDRs2LLINNWvWFAEBAbK0lJQUAUD07t27WPsREREhAIgxY8bI0j/66CMBQBw9elRK0/U5KHgtU18Pu3TpIvLy8qT0qVOnClNTU5GcnCylFfYeeXl5CS8vr2LtR0Gl8Tl89dVXxerVq8Vvv/0mNm7cKBo3biwAiA0bNujcbknOz5ycHOHq6ipat25drH1auHChACBCQ0M18h48eCAAiKVLlxarLqLSwjsVRAayfv16hISEaDyaNm1a5GsdHBxw7do1jV86i+PAgQMwNTXFpEmTZOkffvghhBD4448/AEDq4vPBBx/Iyk2cOFFn3ePGjdNIq1KlivT/jIwM/Pfff2jbti0ASHdW8hszZoz0f1NTU7Rq1QpCCIwePVpKd3BwQP369bXOwlIcxT0Gah06dICPj89zbUuXhw8fYvDgwfD29saMGTOkdPVMO5aWlhqvsbKykpUpaPfu3cjKyir0l8ygoCAcPHgQGzZsQMOGDfH06VOd3XF0UalU2L9/P3r06KHRDcnT0xOHDh3CuHHj0LNnT0yePBkXL16Es7MzPvzwwxJtJ79JkybhzTff1Doo3NgNGDAAQUFBGDZsGPr06YOFCxfi0KFDePTokWwWn6Lee23vu6Ojo8YMWNo8evQIjo6OsjSVSgUAxe6Oc+DAAQDQuJunfl/3799frHq0GTt2rKyL5uuvv47c3FzExMQU6/XR0dE6Z/kqTGl9Dv/8809MnjwZvXr1wrhx4xAeHo7GjRvjk08+KZXZs0JDQ5GQkFCsuxQnT57E/PnzMWDAALzxxhsa+erzoDjnDVFp4uxPRAbSunVrrbfWi/MlYcGCBejduzdeeuklNG7cGN27d8e7775brIAkJiYG7u7uGl8kGjZsKOWr/zUxMdGYraZu3bo66y5YFnjWD37+/PnYuXMnHj58KMtLSUnRKO/p6Sl7bm9vDysrK43b+Pb29oXO2FOY4h4DNW37pY/09HS89dZbSE1NxenTp2V9vNVBWME+6gCkvtr5A7X8duzYAScnJ7z55ps6t51/BqZBgwZJ+1ySNTV+/vlnZGRkFOsLDvCsH/nIkSOxZMkS3L9/X+cYDF1+/PFHnDlzptjT31YE7dq1Q5s2bXDkyBEpraj3Xtv7LoSQfRkvjCjQ3VCpVAJAod3p8lNfEwpeA9zc3ODg4FDsAECbgp979Rffx48fP3edRTHU5xAALCwsMGHCBCnAUI8xe147duyAqakpBg4cWGi5f/75B3379kXjxo2xefNmrWXU50Fxzxui0sI7FURGqH379oiMjMTWrVulPx4tWrTQ+UekrGj7IztgwAB8++23GDduHH755RccPnxYuguSl5enUd7U1LRYaYDmlyRDKezLQ0llZWWhX79+uHz5Mn777TeNNUmcnJxgaWmpMcc8ACnN3d1dIy82NhanTp1C//79NfrN6+Lo6Ig33nhDtp5FcezYsQP29vZ46623iv0aDw8PAHiuwaHTp09H//79YWFhIf0irR5cfu/ePTx48KDEdRoDDw8P2fFQD5TW9d5re98fP36std98QVWrVtX4gq5UKuHu7l7iYE2fL6O67oqV9WfcUJ/D/PQ55/N7+vQp9uzZgy5dusDV1VVnuXv37qFbt26wt7fHgQMHdN6BUp8HxTlviEoTgwoiI6X+9Ve9oFXTpk1ls63o+sPv5eWFBw8eaPw6+c8//0j56n/z8vIQFRUlK3fnzp1it/Hx48cIDQ3Fxx9/jPnz56Nv377o2rUrateuXew6DKG4x6C05eXlYdiwYQgNDUVwcDA6dOigUcbExARNmjTRuhjX33//jdq1a2v9svDDDz9ACFHsuwdqT58+1XrHSJe4uDgcO3YMAQEBWruG6KLuqubs7Fyi9gHPviwFBwfD29tbeqxZswYA0KJFC431MCqKu3fvyo5H8+bNAUDjvX/w4AHu378v5ecXFRUl3W0qTIMGDTQ+ywDw1ltvITIyEmFhYUXWob4mFOx2mZCQgOTkZNnnxtHRUWNWsaysLK1f0ourtH5ZN+TnMD99zvn89u7dW+SMbo8ePUK3bt2QmZmJQ4cOaZ3JS019HhTnvCEqTQwqiIxQwW4/tra2qFu3ruxWvY2NDQBo/GHv0aMHcnNzsW7dOln6qlWroFAopK4zfn5+AJ4tOJXf2rVri91O9a+PBX9tXL16dbHrMITiHoPSNnHiRPz444/YsGGDztmZAODtt9/GuXPnZF9obt68iaNHj6J///5aXxMcHAxPT0+d3SwKdj0DnvVDDw0N1eiGV9j0nDt37kReXp7OLzj5pwFV+/fff7F161Y0bdq00C87uuzZs0fjoe4G8t1335XpwoZFiYuLwz///CObxU3bMTlw4ADCw8PRvXt3Ka1Ro0Zo0KABvvnmG9kv+hs3boRCocDbb78tqyMlJQWRkZHFmrnJ19cXV69e1ejOM2PGDNjY2GDMmDFISEjQeF1kZKQUwKmDt4Kf35UrVwKAbGa4OnXqaMzOVnC/SsrGxkbn9MfFnVIWKP3Pobb3NzU1FatXr0a1atXQsmXLYrVLl+DgYFhbW6Nv375a89PT09GjRw/8+++/OHDgAOrVq1dofeHh4VAoFKWyGCVRSXBMBZER8vHxQceOHdGyZUs4OTnh/Pnz2L17t2wKRvUfskmTJsHPzw+mpqYYNGgQevbsiU6dOuHTTz9FdHQ0mjVrhsOHD+O3337DlClTUKdOHen1AQEBWL16NR49eoS2bdvixIkTuHXrFoDi/WqoVCrRvn17LFu2DNnZ2ahRowYOHz6s9RfTslTcY1CaVq9ejQ0bNsDX1xfW1tb4/vvvZfl9+/aVAsEPPvgA3377Lfz9/fHRRx/B3NwcK1euhKurq9bBzlevXsXly5fx8ccf63xfmjRpgs6dO6N58+ZwdHTE7du3sWXLFmRnZ2PJkiWysuqpObUNfN2xYwfc3d1lawTkN2PGDERGRqJz585wd3dHdHQ0vv76a6Snp0tfTtW2bduGkSNHIigoqNBVkvv06aORpl6J+M0335R140hJSZECX/VUq+vWrYODgwMcHBxkn5GTJ09KX3wTExORnp6ORYsWAXjWxTD/mg7r1q1DcnKy1NVq3759uH//PoBnX1LVq1TPmjUL27dvR1RUlDT18KuvvoqXX34ZrVq1gr29PS5cuICtW7fCw8MDn3zyiWy/li9fjl69eqFbt24YNGgQrl69inXr1mHMmDEavywfOXIEQgj07t1b57FT6927NxYuXIgTJ07IBrvXqVMHwcHBGDhwIBo2bChbUfvMmTPYtWuX9N40a9YMw4cPxzfffIPk5GR06NABZ8+exfbt29GnTx906tRJqnfMmDEYN24cAgIC0LVrV1y6dAmHDh3Sq8tNy5YtsXHjRixatAh169aFi4uLNBC5sHM2P0N8DtevX49ff/0VPXv2hKenJ+Li4rB161bExsbif//7HywsLKSyJTk/gWddp/744w8EBARorK+hNmTIEJw9exajRo3CjRs3ZGtT2Nraanx+QkJC8Nprr2msCE5kcOU06xTRC+t5VtQuOA3jokWLROvWrYWDg4OoUqWKaNCggfj8889FVlaWVCYnJ0dMnDhRODs7C4VCIZteNjU1VUydOlW4u7sLc3NzUa9ePbF8+XLZlI5CCJGeni4CAwOFk5OTsLW1FX369BE3b94UAGRTvBY2vef9+/dF3759hYODg7C3txf9+/eXpjTUNi1twTqGDx8ubGxsinWctNFVrrjHAMVcKTd/eW1TaQ4fPrzQaYQLTit579498fbbbwulUilsbW3FW2+9JW7fvq11mx9//LEAIC5fvqyzXXPnzhWtWrUSjo6OwszMTLi7u4tBgwZpfY2u6Tn/+ecfAUBMmzZN53aCg4NF+/bthbOzszAzMxPVqlUTffv2FeHh4Rpl165dKwCIgwcP6qyvsP3Rdr6oV/TW9ii4T+o6tD0KvodeXl7Feu/U73P+tE8//VQ0b95c2NvbC3Nzc+Hp6SnGjx8v4uPjte7bnj17RPPmzYWlpaWoWbOmmD17tuyzrTZw4EDRrl27Yh+zpk2bitGjR2vNu3XrlnjvvfdErVq1hIWFhbCzsxOvvfaaWLt2rcjIyJDKZWdni/nz5wtvb29hbm4uPDw8xKxZs2RlhHi28vzMmTNFtWrVhLW1tfDz8xN37tzROaVswevhsWPHBABx7NgxKS0+Pl74+/sLOzs7AUA2vWxxp5Q1xOfw8OHDomvXrsLNzU2Ym5sLBwcH0a1bN63TuZbk/BRCiE2bNgkAYu/evTr3qbBzs2CdycnJwsLCQmzevLnIY0VU2hRClNFISCKqECIiIvDyyy/j+++/L3H//cpAoVBg+vTpUreS0hzk/aIZMGAAoqOjcfbs2fJuSoUTHx8Pb29v7Ny5s1h3KoBnC2EGBgYiNjZW54rk9GJbvXo1li1bhsjISF6bqMxxTAVRJaZtfvXVq1fDxMRE1jWE5JYvXw5nZ2esX7++vJtitIQQOH78uNTdiEpm9erVaNKkSbEDCuBZNxlPT0+el5VUdnY2Vq5cidmzZzOgoHLBOxVEldj8+fMRHh6OTp06wczMDH/88Qf++OMPjB07Fl9//XV5N88o5V934KWXXtKYf5+IiKgyYlBBVImFhIRg/vz5uH79OtLS0uDp6Yl3330Xn376KczMOI8DERERFQ+DCiIiIiIi0gvHVBARERERkV4YVBARERERkV4YVBARERERkV44ErOU5OXl4cGDB7CzsyvWSsRERERERIYkhEBqairc3d1hYmLYewkMKkrJgwcP4OHhUd7NICIiIiKSuXfvHmrWrGnQbTCoKCV2dnYAnr1pSqWynFtDRERERJWdSqWCh4eH9D3VkBhUlBJ1lyelUsmggoiIiIiMRll0zedAbSIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0guDCiIiIiIi0otZeTegMLm5uZg3bx6+//57xMfHw93dHSNGjMDs2bOhUCgAAEIIzJ07F99++y2Sk5Px2muvYePGjahXr55UT1JSEiZOnIh9+/bBxMQEAQEBWLNmDWxtbaUyly9fRmBgIM6dOwdnZ2dMnDgRM2bMKPN9JiIiIiooPj4eycnJ5d0M0pODgwPc3NzKuxkGYdRBxdKlS7Fx40Zs374djRo1wvnz5zFy5EjY29tj0qRJAIBly5bhq6++wvbt2+Ht7Y3PPvsMfn5+uH79OqysrAAAQ4YMQVxcHEJCQpCdnY2RI0di7NixCA4OBgCoVCp069YNXbp0waZNm3DlyhWMGjUKDg4OGDt2bLntPxEREVF8fDzefvttZGRklHdTSE9WVlbYvXv3CxlYGHVQcebMGfTu3Rv+/v4AgFq1auGHH37A2bNnATy7S7F69WrMnj0bvXv3BgB89913cHV1xa+//opBgwbhxo0bOHjwIM6dO4dWrVoBANauXYsePXrgyy+/hLu7O3bs2IGsrCxs3boVFhYWaNSoESIiIrBy5UoGFURERFSukpOTkZGRgYULF8Lb27u8m0PPKSoqCp999hmSk5MZVJS1V199Fd988w1u3bqFl156CZcuXcLp06excuVKAM/enPj4eHTp0kV6jb29Pdq0aYOwsDAMGjQIYWFhcHBwkAIKAOjSpQtMTEzw999/o2/fvggLC0P79u1hYWEhlfHz88PSpUvx+PFjODo6lt1OExEREWnh7e2NBg0alHcziLQy6qDi448/hkqlQoMGDWBqaorc3Fx8/vnnGDJkCIBntwMBwNXVVfY6V1dXKS8+Ph4uLi6yfDMzMzg5OcnKFIz81XXGx8drDSoyMzORmZkpPVepVPrsqt4SExO1tkGpVMLZ2bkcWkRERERElYVRBxU//fQTduzYgeDgYKlL0pQpU+Du7o7hw4eXa9sWL16M+fPnl2sb1BITEzF05BgkpT7RyHOys8b3QZsZWBARERGRwRh1UDF9+nR8/PHHGDRoEACgSZMmiImJweLFizF8+HCpP1pCQgKqV68uvS4hIQHNmzcHALi5ueHhw4eyenNycpCUlCS93s3NDQkJCbIy6ue6+rzNmjUL06ZNk56rVCp4eHjosbfPT6VSISn1CZx9A2Dj9H93bdKTEpAY9jNUKhWDCiIiIiIyGKNep+LJkycwMZE30dTUFHl5eQCe9S10c3NDaGiolK9SqfD333/D19cXAODr64vk5GSEh4dLZY4ePYq8vDy0adNGKnPy5ElkZ2dLZUJCQlC/fn2d4yksLS2hVCplj/Jm4+QKpUtN6ZE/wCAiIqIXV3x8PCZOnIjatWvD0tISHh4e6Nmzp+w7UllRKBT49ddfS/y6X375BV27doWzszOUSiV8fX1x6NAhjXLr169HrVq1YGVlhTZt2kgT+Kh17NgRCoVC9hg3bpyszKRJk9CyZUtYWlpKP0TnN2/ePI06FAoFbGxspDLffvstXn/9dTg6OsLR0RFdunTRaEtlYtRBRc+ePfH5559j//79iI6Oxp49e7By5Ur07dsXwLOTdsqUKVi0aBH27t2LK1euYNiwYXB3d0efPn0AAA0bNkT37t3x3nvv4ezZs/jzzz8xYcIEDBo0CO7u7gCAwYMHw8LCAqNHj8a1a9fw448/Ys2aNbI7EURERETGKDo6Gi1btsTRo0exfPlyXLlyBQcPHkSnTp0QGBhY3s0rtpMnT6Jr1644cOAAwsPD0alTJ/Ts2RMXL16Uyvz444+YNm0a5s6diwsXLqBZs2bw8/PT6JXy3nvvIS4uTnosW7ZMY3ujRo3CwIEDtbblo48+kr0+Li4OPj4+6N+/v1Tm+PHjeOedd3Ds2DGEhYXBw8MD3bp1w7///ltKR6SCEUZMpVKJyZMnC09PT2FlZSVq164tPv30U5GZmSmVycvLE5999plwdXUVlpaWonPnzuLmzZuyeh49eiTeeecdYWtrK5RKpRg5cqRITU2Vlbl06ZJo166dsLS0FDVq1BBLliwpUVtTUlIEAJGSkvL8O/yc7ty5I1q17yLeXvqLGBV0Vnq8vfQX0ap9F3Hnzp0ybxMRERGVjhs3boiWLVuKGzduaM1/8803RY0aNURaWppG3uPHj6X/x8TEiF69egkbGxthZ2cn+vfvL+Lj46X84cOHi969e8teP3nyZNGhQwfpeYcOHcTEiRPF9OnThaOjo3B1dRVz586V8r28vAQA6eHl5fU8uyzx8fER8+fPl563bt1aBAYGSs9zc3OFu7u7WLx4sayNkydPLlb9c+fOFc2aNSuyXEREhAAgTp48qbNMTk6OsLOzE9u3b9eaX9T7aAhl+f3UqO9U2NnZYfXq1YiJicHTp08RGRmJRYsWyaZ+VSgUWLBgAeLj45GRkYEjR47gpZdektXj5OSE4OBgpKamIiUlBVu3bpWtpg0ATZs2xalTp5CRkYH79+9j5syZZbKPRERERM8rKSkJBw8eRGBgoKxrjpqDgwMAIC8vD71790ZSUhJOnDiBkJAQ3L17V+cv9YXZvn07bGxs8Pfff2PZsmVYsGABQkJCAADnzp0DAAQFBSEuLk56Hh0dDYVCgePHjxd7O3l5eUhNTYWTkxMAICsrC+Hh4bKlBExMTNClSxeEhYXJXrtjxw5Uq1YNjRs3xqxZs/DkieZkNiWxefNmvPTSS3j99dd1lnny5Amys7Ol9lY2Rj1QmwxH1xS0AKehJSIiqiju3LkDIUSR61eEhobiypUriIqKkiaW+e6779CoUSOcO3cOr7zySrG32bRpU8ydOxcAUK9ePaxbtw6hoaHSeAjgWTCTf7Ibc3Nz1K9fH9bW1sXezpdffom0tDQMGDAAAPDff/8hNzdX61IC//zzj/R88ODB8PLygru7Oy5fvoyZM2fi5s2b+OWXX4q97fwyMjKwY8cOfPzxx4WWmzlzJtzd3WVBT2XCoKISKmwKWoDT0BIREVUUQohilbtx4wY8PDxkM1X6+PjAwcEBN27cKHFQkV/16tU1xjQUVKNGDdkX/6IEBwdj/vz5+O233zTWGyvK2LFjpf83adIE1atXR+fOnREZGYk6deqUqC4A2LNnD1JTUwtdzmDJkiXYuXMnjh8/DisrqxJv40XAoKIS0jUFLcBpaImIiCqSevXqQaFQlOgLuy4mJiYaQUr+mTHVzM3NZc8VCoU0M2dp2LlzJ8aMGYNdu3bJfvWvVq0aTE1NtS4DoGsJAADSbJ937tx5rqBi8+bNeOuttzTukKh9+eWXWLJkCY4cOaIRcFUmRj2mggyr4BS0nIaWiIioYnFycoKfnx/Wr1+P9PR0jfzk5GQAz2bDvHfvHu7duyflXb9+HcnJyfDx8QEAODs7Iy4uTvb6iIiIErfJ3Nwcubm5JX4dAPzwww8YOXIkfvjhB/j7+8vyLCws0LJlS9k0uXl5eQgNDZWWEtBGvQ/51zQrrqioKBw7dgyjR4/Wmr9s2TIsXLgQBw8eRKtWrUpc/4uEQQURERFRBbZ+/Xrk5uaidevW+Pnnn3H79m3cuHEDX331lfRlu0uXLmjSpAmGDBmCCxcu4OzZsxg2bBg6dOggfRl+4403cP78eXz33Xe4ffs25s6di6tXr5a4PbVq1UJoaCji4+Px+PFjAMC///6LBg0aFLqOQ3BwMIYNG4YVK1agTZs2iI+PR3x8PFJSUqQy06ZNw7fffovt27fjxo0bGD9+PNLT0zFy5EgAQGRkJBYuXIjw8HBER0dj7969GDZsGNq3by+7i3Dnzh1EREQgPj4eT58+RUREBCIiIpCVlSVr09atW1G9enW8+eabGu1dunQpPvvsM2zduhW1atWS2puWllbiY/YiYFBBREREVIHVrl0bFy5cQKdOnfDhhx+icePG6Nq1K0JDQ7Fx40YAz7oo/fbbb3B0dET79u3RpUsX1K5dGz/++KNUj5+fHz777DPMmDEDr7zyClJTUzFs2LASt2fFihUICQmBh4cHXn75ZQDPulHdvHmz0FmYvvnmG+Tk5CAwMBDVq1eXHpMnT5bKDBw4EF9++SXmzJmD5s2bIyIiAgcPHpS6JllYWODIkSPo1q0bGjRogA8//BABAQHYt2+fbFtjxozByy+/jK+//hq3bt3Cyy+/jJdffhkPHjyQyuTl5WHbtm0YMWIETE1NNdq7ceNGZGVl4e2335a198svvyzxMXsRcEwFERERUQVXvXp1rFu3DuvWrdNZxtPTE7/99luh9cyfPx/z58/Xma9tStiCq2f37NkTPXv2lKXVqlWryEHlxZ1udsKECZgwYYLWPA8PD5w4caLIOoqzLRMTE1l3sYKio6OLrKMy4Z0KIiIiIiLSC4MKIiIiIiLSC4MKIiIiIiLSC4MKIiIiIiLSC4MKIiIiIiLSC4MKIiIiIiLSC4MKIiIiIiLSC9epICIiIqoAoqKiyrsJpIcX/f1jUEFERERkxBwcHGBlZYXPPvusvJtCerKysoKDg0N5N8MgGFQQERERGTE3Nzfs3r0bycnJ5d0U0pODgwPc3NzKuxkGwaCCiIiIyMi5ubm9sF9G6cXAgdpERERERKQXBhVERERERKQXBhVERERERKQXBhVERERERKQXDtR+wWVnZSEmJkaWFhMTg5zsnHJqERERERG9aBhUvMAy01IQHXUXUz6ZB0tLSyk94+kT3P83Dp7Z2eXYOiIiIiJ6UTCoeIFlZz5FnsIM1dr2Q1V3Lyn9YeRVxNzbitwcBhVEREREpD8GFZWAtaMzlC41pedpj+LLsTVERERE9KLhQG0iIiIiItILgwoiIiIiItILgwoiIiIiItILgwoiIiIiItILB2qTBm1rWwCAUqmEs7NzObSIiIiIiIwZgwqS0bW2BQA42Vnj+6DNDCyIiIiISIZBBcnoWtsiPSkBiWE/Q6VSMaggIiIiIhkGFaRVwbUtACCxnNpCRERERMaNA7WJiIiIiEgvDCqIiIiIiEgvDCqIiIiIiEgvDCqIiIiIiEgvDCqIiIiIiEgvRh9U1KpVCwqFQuMRGBgIAMjIyEBgYCCqVq0KW1tbBAQEICEhQVZHbGws/P39YW1tDRcXF0yfPh05OTmyMsePH0eLFi1gaWmJunXrYtu2bWW1i0REREREFZrRBxXnzp1DXFyc9AgJCQEA9O/fHwAwdepU7Nu3D7t27cKJEyfw4MED9OvXT3p9bm4u/P39kZWVhTNnzmD79u3Ytm0b5syZI5WJioqCv78/OnXqhIiICEyZMgVjxozBoUOHynZniYiIiIgqIKNfp6LgQmtLlixBnTp10KFDB6SkpGDLli0IDg7GG2+8AQAICgpCw4YN8ddff6Ft27Y4fPgwrl+/jiNHjsDV1RXNmzfHwoULMXPmTMybNw8WFhbYtGkTvL29sWLFCgBAw4YNcfr0aaxatQp+fn5lvs9ERERERBWJ0d+pyC8rKwvff/89Ro0aBYVCgfDwcGRnZ6NLly5SmQYNGsDT0xNhYWEAgLCwMDRp0gSurq5SGT8/P6hUKly7dk0qk78OdRl1HdpkZmZCpVLJHkRERERElVGFCip+/fVXJCcnY8SIEQCA+Ph4WFhYwMHBQVbO1dUV8fHxUpn8AYU6X51XWBmVSoWnT59qbcvixYthb28vPTw8PPTdPSIiIiKiCqlCBRVbtmzBm2++CXd39/JuCmbNmoWUlBTpce/evfJuEhERERFRuTD6MRVqMTExOHLkCH755Rcpzc3NDVlZWUhOTpbdrUhISICbm5tU5uzZs7K61LND5S9TcMaohIQEKJVKVKlSRWt7LC0tYWlpqfd+ERERERFVdBXmTkVQUBBcXFzg7+8vpbVs2RLm5uYIDQ2V0m7evInY2Fj4+voCAHx9fXHlyhU8fPhQKhMSEgKlUgkfHx+pTP461GXUdRARERERkW4VIqjIy8tDUFAQhg8fDjOz/7u5Ym9vj9GjR2PatGk4duwYwsPDMXLkSPj6+qJt27YAgG7dusHHxwfvvvsuLl26hEOHDmH27NkIDAyU7jSMGzcOd+/exYwZM/DPP/9gw4YN+OmnnzB16tRy2V8iIiIiooqkQnR/OnLkCGJjYzFq1CiNvFWrVsHExAQBAQHIzMyEn58fNmzYIOWbmpri999/x/jx4+Hr6wsbGxsMHz4cCxYskMp4e3tj//79mDp1KtasWYOaNWti8+bNnE6WiIiIiKgYKkRQ0a1bNwghtOZZWVlh/fr1WL9+vc7Xe3l54cCBA4Vuo2PHjrh48aJe7SQiIiIiqowqRPcnIiIiIiIyXgwqiIiIiIhILxWi+xMZh+ysLMTExGikK5VKODs7l0OLiIiIiMgYMKigYslMS0F01F1M+WSexvocTnbW+D5oMwMLIiIiokqKQQUVS3bmU+QpzFCtbT9UdfeS0tOTEpAY9jNUKhWDCiIiIqJKikEFlYi1ozOULjVlaYnl1BYiIiIiMg4MKkhvHGtBREREVLkxqCC9cKwFERERETGoIL1wrAURERERMaigUsGxFkRERESVFxe/IyIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivRh9UPHvv/9i6NChqFq1KqpUqYImTZrg/PnzUr4QAnPmzEH16tVRpUoVdOnSBbdv35bVkZSUhCFDhkCpVMLBwQGjR49GWlqarMzly5fx+uuvw8rKCh4eHli2bFmZ7B8RERERUUVn1EHF48eP8dprr8Hc3Bx//PEHrl+/jhUrVsDR0VEqs2zZMnz11VfYtGkT/v77b9jY2MDPzw8ZGRlSmSFDhuDatWsICQnB77//jpMnT2Ls2LFSvkqlQrdu3eDl5YXw8HAsX74c8+bNwzfffFOm+/uiyc7KQkxMDCIjI2WPxMTE8m4aEREREZUis/JuQGGWLl0KDw8PBAUFSWne3t7S/4UQWL16NWbPno3evXsDAL777ju4urri119/xaBBg3Djxg0cPHgQ586dQ6tWrQAAa9euRY8ePfDll1/C3d0dO3bsQFZWFrZu3QoLCws0atQIERERWLlypSz4oOLLTEtBdNRdTPlkHiwtLWV5TnbW+D5oM5ydncupdURERERUmoz6TsXevXvRqlUr9O/fHy4uLnj55Zfx7bffSvlRUVGIj49Hly5dpDR7e3u0adMGYWFhAICwsDA4ODhIAQUAdOnSBSYmJvj777+lMu3bt4eFhYVUxs/PDzdv3sTjx48NvZsvpOzMp8hTmKFa236o5f+B9HD2DUBS6hOoVKrybiIRERERlRKjDiru3r2LjRs3ol69ejh06BDGjx+PSZMmYfv27QCA+Ph4AICrq6vsda6urlJefHw8XFxcZPlmZmZwcnKSldFWR/5tFJSZmQmVSiV7kCZrR2coXWpKDxsn16JfREREREQVilF3f8rLy0OrVq3wxRdfAABefvllXL16FZs2bcLw4cPLtW2LFy/G/Pnzy7UNRERERETGwKjvVFSvXh0+Pj6ytIYNGyI2NhYA4ObmBgBISEiQlUlISJDy3Nzc8PDhQ1l+Tk4OkpKSZGW01ZF/GwXNmjULKSkp0uPevXvPs4tERERERBWeUQcVr732Gm7evClLu3XrFry8vAA8G7Tt5uaG0NBQKV+lUuHvv/+Gr68vAMDX1xfJyckIDw+Xyhw9ehR5eXlo06aNVObkyZPIzs6WyoSEhKB+/fqymabys7S0hFKplD2IiIiIiCojow4qpk6dir/++gtffPEF7ty5g+DgYHzzzTcIDAwEACgUCkyZMgWLFi3C3r17ceXKFQwbNgzu7u7o06cPgGd3Nrp374733nsPZ8+exZ9//okJEyZg0KBBcHd3BwAMHjwYFhYWGD16NK5du4Yff/wRa9aswbRp08pr14mIiIiIKgyjHlPxyiuvYM+ePZg1axYWLFgAb29vrF69GkOGDJHKzJgxA+np6Rg7diySk5PRrl07HDx4EFZWVlKZHTt2YMKECejcuTNMTEwQEBCAr776Ssq3t7fH4cOHERgYiJYtW6JatWqYM2cOp5MlIiIiIioGow4qAOCtt97CW2+9pTNfoVBgwYIFWLBggc4yTk5OCA4OLnQ7TZs2xalTp567nURERERElZVRd38iIiIiIiLjx6CCiIiIiIj0wqCCiIiIiIj0wqCCiIiIiIj0wqCCiIiIiIj0YvSzP1HlkZiYCJVKpTVPqVTC2dm5jFtERERERMXBoIKMQmJiIoaOHIOk1Cda853srPF90GYGFkRERERGiEEFGQWVSoWk1Cdw9g2AjZOrLC89KQGJYT9DpVIxqCAiIiIyQgwqyKjYOLlC6VJTIz2xHNpCRERERMXDgdpERERERKQXBhVERERERKQXBhVERERERKQXjqmgMpedlYWYmBhZWkxMDHKyc8qpRURERESkDwYVVKYy01IQHXUXUz6ZB0tLSyk94+kT3P83Dp7Z2eXSLq6RQURERPT8GFRQmcrOfIo8hRmqte2Hqu5eUvrDyKuIubcVuTllH1RwjQwiIiIi/TCooHJh7egsmzo27VF8ubWFa2QQERER6YdBBdH/xzUyiIiIiJ4PZ38iIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9MKggIiIiIiK9cEVtqhCys7IQExOjka5UKuHs7FwOLSIiIiIiNQYVZPQy01IQHXUXUz6ZB0tLS1mek501vg/azMCCiIiIqBwxqCCjl535FHkKM1Rr2w9V3b2k9PSkBCSG/QyVSsWggoiIiKgcMaigCsPa0RlKl5qytAfsFkVERERU7hhUUIXFblFERERExoFBBVVY7BZFREREZBwYVFCFp61bVGI5tYWIiIioMuI6FUREREREpBcGFUREREREpBd2f6IXkq7F8rKysmBhYSFLi4mJQU52Tonr4gxTRERERM8wqKAXjq5ZobKzsvBvbAxqennDzPz/Tv2Mp09w/984eGZnF7sugDNMEREREakxqKAXjq5ZoR5GXsXd6K1wbN1bIz3m3lbk5mgGFZxhioiIiKhoDCrohVVwVqi0R/GFppekLoAzTBERERGpGfVA7Xnz5kGhUMgeDRo0kPIzMjIQGBiIqlWrwtbWFgEBAUhISJDVERsbC39/f1hbW8PFxQXTp09HTo68//zx48fRokULWFpaom7duti2bVtZ7B4RERER0QvBqIMKAGjUqBHi4uKkx+nTp6W8qVOnYt++fdi1axdOnDiBBw8eoF+/flJ+bm4u/P39kZWVhTNnzmD79u3Ytm0b5syZI5WJioqCv78/OnXqhIiICEyZMgVjxozBoUOHynQ/iYiIiIgqKqPv/mRmZgY3NzeN9JSUFGzZsgXBwcF44403AABBQUFo2LAh/vrrL7Rt2xaHDx/G9evXceTIEbi6uqJ58+ZYuHAhZs6ciXnz5sHCwgKbNm2Ct7c3VqxYAQBo2LAhTp8+jVWrVsHPz69M95WIiIiIqCIy+jsVt2/fhru7O2rXro0hQ4YgNjYWABAeHo7s7Gx06dJFKtugQQN4enoiLCwMABAWFoYmTZrA1dVVKuPn5weVSoVr165JZfLXoS6jrkOXzMxMqFQq2YOIiIiIqDIy6qCiTZs22LZtGw4ePIiNGzciKioKr7/+OlJTUxEfHw8LCws4ODjIXuPq6or4+GcDb+Pj42UBhTpfnVdYGZVKhadPn+ps2+LFi2Fvby89PDw89N1dIiIiIqIKyai7P7355pvS/5s2bYo2bdrAy8sLP/30E6pUqVKOLQNmzZqFadOmSc9VKhUDCyIiIiKqlIz6TkVBDg4OeOmll3Dnzh24ubkhKysLycnJsjIJCQnSGAw3NzeN2aDUz4sqo1QqCw1cLC0toVQqZQ8iIiIiosqoQgUVaWlpiIyMRPXq1dGyZUuYm5sjNDRUyr958yZiY2Ph6+sLAPD19cWVK1fw8OFDqUxISAiUSiV8fHykMvnrUJdR10FERERERIUz6qDio48+wokTJxAdHY0zZ86gb9++MDU1xTvvvAN7e3uMHj0a06ZNw7FjxxAeHo6RI0fC19cXbdu2BQB069YNPj4+ePfdd3Hp0iUcOnQIs2fPRmBgICwtLQEA48aNw927dzFjxgz8888/2LBhA3766SdMnTq1PHediIiIiKjCMOoxFffv38c777yDR48ewdnZGe3atcNff/0FZ2dnAMCqVatgYmKCgIAAZGZmws/PDxs2bJBeb2pqit9//x3jx4+Hr68vbGxsMHz4cCxYsEAq4+3tjf3792Pq1KlYs2YNatasic2bN3M6WSIiIiKiYjLqoGLnzp2F5ltZWWH9+vVYv369zjJeXl44cOBAofV07NgRFy9efK42EhERERFVdgbr/nT37l1DVU1EREREREbEYEFF3bp10alTJ3z//ffIyMgw1GaIiIiIiKicGSyouHDhApo2bYpp06bBzc0N77//Ps6ePWuozRERERERUTkxWFDRvHlzrFmzBg8ePMDWrVsRFxeHdu3aoXHjxli5ciUSExMNtWkiIiIiIipDBp9S1szMDP369cOuXbuwdOlS3LlzBx999BE8PDwwbNgwxMXFGboJRERERERkQAYPKs6fP48PPvgA1atXx8qVK/HRRx8hMjISISEhePDgAXr37m3oJhARERERkQEZbErZlStXIigoCDdv3kSPHj3w3XffoUePHjAxeRbHeHt7Y9u2bahVq5ahmkBERERERGXAYEHFxo0bMWrUKIwYMQLVq1fXWsbFxQVbtmwxVBOIiIiIiKgMGCyouH37dpFlLCwsMHz4cEM1gYiIiIiIyoDBxlQEBQVh165dGum7du3C9u3bDbVZIiIiIiIqYwYLKhYvXoxq1apppLu4uOCLL74w1GaJykx2VhZiYmIQGRkpe3C6ZCIiIqpsDNb9KTY2Ft7e3hrpXl5eiI2NNdRmicpEZloKoqPuYson82BpaSnLc7KzxvdBm+Hs7FxOrSMiIiIqWwYLKlxcXHD58mWN2Z0uXbqEqlWrGmqzRGUiO/Mp8hRmqNa2H6q6e0np6UkJSAz7GSqVikEFERERVRoGCyreeecdTJo0CXZ2dmjfvj0A4MSJE5g8eTIGDRpkqM0SlSlrR2coXWrK0tj5iYiIiCobgwUVCxcuRHR0NDp37gwzs2ebycvLw7BhwzimgoiIiIjoBWKwoMLCwgI//vgjFi5ciEuXLqFKlSpo0qQJvLy8in4xERERERFVGAYLKtReeuklvPTSS4beDBERERERlRODBRW5ubnYtm0bQkND8fDhQ+Tl5cnyjx49aqhNExERERFRGTJYUDF58mRs27YN/v7+aNy4MRQKhaE2RURERERE5chgQcXOnTvx008/oUePHobaBBERERERGQGDDtSuW7euoaonMlrqlbYLUiqVXLuCiIiIXkgGCyo+/PBDrFmzBuvWrWPXJ6o0uNI2ERERVUYGCypOnz6NY8eO4Y8//kCjRo1gbm4uy//ll18MtWmicsOVtomIiKgyMlhQ4eDggL59+xqqeiKjxpW2iYiIqDIxWFARFBRkqKqJiIiIiMiImBiy8pycHBw5cgRff/01UlNTAQAPHjxAWlqaITdLRERERERlyGB3KmJiYtC9e3fExsYiMzMTXbt2hZ2dHZYuXYrMzExs2rTJUJsmIiIiIqIyZLA7FZMnT0arVq3w+PFjVKlSRUrv27cvQkNDDbVZIiIiIiIqYwa7U3Hq1CmcOXMGFhYWsvRatWrh33//NdRmiYiIiIiojBnsTkVeXh5yc3M10u/fvw87OztDbZaIiIiIiMqYwYKKbt26YfXq1dJzhUKBtLQ0zJ07Fz169DDUZomIiIiIqIwZrPvTihUr4OfnBx8fH2RkZGDw4MG4ffs2qlWrhh9++MFQmyUiIiIiojJmsKCiZs2auHTpEnbu3InLly8jLS0No0ePxpAhQ2QDt4mIiIiIqGIzWFABAGZmZhg6dKghN0FEREREROXMYEHFd999V2j+sGHDDLVpIiIiIiIqQwYLKiZPnix7np2djSdPnsDCwgLW1tYMKoiIiIiIXhAGm/3p8ePHskdaWhpu3ryJdu3acaA2EREREdELxGBBhTb16tXDkiVLNO5iEBERERFRxVWmQQXwbPD2gwcPnuu1S5YsgUKhwJQpU6S0jIwMBAYGomrVqrC1tUVAQAASEhJkr4uNjYW/vz+sra3h4uKC6dOnIycnR1bm+PHjaNGiBSwtLVG3bl1s27btudpIRERERFTZGGxMxd69e2XPhRCIi4vDunXr8Nprr5W4vnPnzuHrr79G06ZNZelTp07F/v37sWvXLtjb22PChAno168f/vzzTwBAbm4u/P394ebmhjNnziAuLg7Dhg2Dubk5vvjiCwBAVFQU/P39MW7cOOzYsQOhoaEYM2YMqlevDj8/v+c8AkRERERElYPBgoo+ffrInisUCjg7O+ONN97AihUrSlRXWloahgwZgm+//RaLFi2S0lNSUrBlyxYEBwfjjTfeAAAEBQWhYcOG+Ouvv9C2bVscPnwY169fx5EjR+Dq6ormzZtj4cKFmDlzJubNmwcLCwts2rQJ3t7eUrsaNmyI06dPY9WqVQwqiIiIiIiKYLDuT3l5ebJHbm4u4uPjERwcjOrVq5eorsDAQPj7+6NLly6y9PDwcGRnZ8vSGzRoAE9PT4SFhQEAwsLC0KRJE7i6ukpl/Pz8oFKpcO3aNalMwbr9/PykOoiIiIiISDeDLn5XGnbu3IkLFy7g3LlzGnnx8fGwsLCAg4ODLN3V1RXx8fFSmfwBhTpfnVdYGZVKhadPn2pdATwzMxOZmZnSc5VKVfKdIyIiIiJ6ARgsqJg2bVqxy65cuVJr+r179zB58mSEhITAysqqtJpWKhYvXoz58+eXdzOIiIiIiMqdwYKKixcv4uLFi8jOzkb9+vUBALdu3YKpqSlatGghlVMoFDrrCA8Px8OHD2Xlc3NzcfLkSaxbtw6HDh1CVlYWkpOTZXcrEhIS4ObmBgBwc3PD2bNnZfWqZ4fKX6bgjFEJCQlQKpVa71IAwKxZs2SBk0qlgoeHh859ISIiIiJ6URksqOjZsyfs7Oywfft2ODo6Ani2IN7IkSPx+uuv48MPPyyyjs6dO+PKlSuytJEjR6JBgwaYOXMmPDw8YG5ujtDQUAQEBAAAbt68idjYWPj6+gIAfH198fnnn+Phw4dwcXEBAISEhECpVMLHx0cqc+DAAdl2QkJCpDq0sbS0hKWlZTGPBhERERHRi8tgQcWKFStw+PBhKaAAAEdHRyxatAjdunUrVlBhZ2eHxo0by9JsbGxQtWpVKX306NGYNm0anJycoFQqMXHiRPj6+qJt27YAgG7dusHHxwfvvvsuli1bhvj4eMyePRuBgYFSUDBu3DisW7cOM2bMwKhRo3D06FH89NNP2L9/f2kdDiIiIiKiF5bBggqVSoXExESN9MTERKSmppbadlatWgUTExMEBAQgMzMTfn5+2LBhg5RvamqK33//HePHj4evry9sbGwwfPhwLFiwQCrj7e2N/fv3Y+rUqVizZg1q1qyJzZs3czpZIiIiIqJiMFhQ0bdvX4wcORIrVqxA69atAQB///03pk+fjn79+j13vcePH5c9t7Kywvr167F+/Xqdr/Hy8tLo3lRQx44dcfHixeduFxERERFRZWWwoGLTpk346KOPMHjwYGRnZz/bmJkZRo8ejeXLlxtqs0QVTmJiotYpiZVKJZydncuhRUREREQlY7CgwtraGhs2bMDy5csRGRkJAKhTpw5sbGwMtUmiCicxMRFDR45BUuoTjTwnO2t8H7SZgQUREREZPYMvfhcXF4e4uDi0b98eVapUgRCi0GlkiV5U2VlZiImJkaXFxMTgYZIK1dsPhI3T/y3AmJ6UgMSwn6FSqRhUEBERkdEzWFDx6NEjDBgwAMeOHYNCocDt27dRu3ZtjB49Go6OjlixYoWhNk1kdDLTUhAddRdTPpknm4o44+kT3P83Dp52TlC61JS9RnOaAyIiIiLjZGKoiqdOnQpzc3PExsbC2tpaSh84cCAOHjxoqM0SGaXszKfIU5ihWtt+qOX/gfRwerk7cvMEcnOyy7uJRERERM/NYHcqDh8+jEOHDqFmTfmvr/Xq1dPoAkJUWVg7OsvuSKQ9ii/H1hARERGVDoPdqUhPT5fdoVBLSkriStRERERERC8QgwUVr7/+Or777jvpuUKhQF5eHpYtW4ZOnToZarNERERERFTGDNb9admyZejcuTPOnz+PrKwszJgxA9euXUNSUhL+/PNPQ22WiIiIiIjKmMHuVDRu3Bi3bt1Cu3bt0Lt3b6Snp6Nfv364ePEi6tSpY6jNEhERERFRGTPInYrs7Gx0794dmzZtwqeffmqITRARERERkZEwyJ0Kc3NzXL582RBVExERERGRkTFY96ehQ4diy5YthqqeiIiIiIiMhMEGaufk5GDr1q04cuQIWrZsCRsbG1n+ypUrDbVpIiIiIiIqQ6UeVNy9exe1atXC1atX0aJFCwDArVu3ZGUUCkVpb5aIiIiIiMpJqQcV9erVQ1xcHI4dOwYAGDhwIL766iu4urqW9qaIiIiIiMgIlPqYCiGE7Pkff/yB9PT00t4MEREREREZCYMN1FYrGGQQEREREdGLpdS7PykUCo0xExxDQVRy2VlZiImJ0ZqnVCrh7Oxcxi0iIiIi0q7UgwohBEaMGAFLS0sAQEZGBsaNG6cx+9Mvv/xS2psmemFkpqUgOuoupnwyT/os5edkZ43vgzYzsCAiIiKjUOpBxfDhw2XPhw4dWtqbIHrhZWc+RZ7CDNXa9kNVdy9ZXnpSAhLDfoZKpWJQQUREREah1IOKoKCg0q6SqNKydnSG0qWmRvoDHV2jsrKyYGFhoZHO7lJERERkSAZb/I6IDENX16jsrCz8GxuDml7eMDOXf7TZXYqIiIgMiUEFUQWjq2vUw8iruBu9FY6te8vS2V2KiIiIDI1BBVEFVbBrVNqjeK3pAJBYpi0jIiKiysbg61QQEREREdGLjUEFERERERHphd2fiIiMQGJiIlQqlUY6Z+4iIqKKgEEFEVE5S0xMxNCRY5CU+kQjjzN3ERFRRcCggoionKlUKiSlPoGzbwBsnFyldM7cRUREFQWDCiIiI2Hj5MqZu4iIqELiQG0iIiIiItILgwoiIiIiItILgwoiIiIiItILx1QQERmx7KwsxMTEaKRzqlkiIjImDCqIiIxUZloKoqPuYson82BpaSnL41SzRERkTBhUEBEZqezMp8hTmKFa236o6u4lpXOqWSIiMjYMKoiIjJy1ozOnmiUiIqPGgdpERERERKQXow4qNm7ciKZNm0KpVEKpVMLX1xd//PGHlJ+RkYHAwEBUrVoVtra2CAgIQEJCgqyO2NhY+Pv7w9raGi4uLpg+fTpycnJkZY4fP44WLVrA0tISdevWxbZt28pi94iIiIiIXghGHVTUrFkTS5YsQXh4OM6fP4833ngDvXv3xrVr1wAAU6dOxb59+7Br1y6cOHECDx48QL9+/aTX5+bmwt/fH1lZWThz5gy2b9+Obdu2Yc6cOVKZqKgo+Pv7o1OnToiIiMCUKVMwZswYHDp0qMz3l4iIiIioIjLqMRU9e/aUPf/888+xceNG/PXXX6hZsya2bNmC4OBgvPHGGwCAoKAgNGzYEH/99Rfatm2Lw4cP4/r16zhy5AhcXV3RvHlzLFy4EDNnzsS8efNgYWGBTZs2wdvbGytWrAAANGzYEKdPn8aqVavg5+dX5vtMRERERFTRGPWdivxyc3Oxc+dOpKenw9fXF+Hh4cjOzkaXLl2kMg0aNICnpyfCwsIAAGFhYWjSpAlcXV2lMn5+flCpVNLdjrCwMFkd6jLqOoiIiIiIqHBGfacCAK5cuQJfX19kZGTA1tYWe/bsgY+PDyIiImBhYQEHBwdZeVdXV8THxwMA4uPjZQGFOl+dV1gZlUqFp0+fokqVKlrblZmZiczMTOm5SqXSaz+JiIiIiCoqo79TUb9+fURERODvv//G+PHjMXz4cFy/fr28m4XFixfD3t5eenh4eJR3k4iIiIiIyoXR36mwsLBA3bp1AQAtW7bEuXPnsGbNGgwcOBBZWVlITk6W3a1ISEiAm5sbAMDNzQ1nz56V1aeeHSp/mYIzRiUkJECpVOq8SwEAs2bNwrRp06TnKpWKgQUZreysLMTExGikK5VKLp5GREREejP6oKKgvLw8ZGZmomXLljA3N0doaCgCAgIAADdv3kRsbCx8fX0BAL6+vvj888/x8OFDuLi4AABCQkKgVCrh4+MjlTlw4IBsGyEhIVIdulhaWsLS0rK0d4+o1GWmpSA66i6mfDJP45x1srPG90GbGVgQERGRXow6qJg1axbefPNNeHp6IjU1FcHBwTh+/DgOHToEe3t7jB49GtOmTYOTkxOUSiUmTpwIX19ftG3bFgDQrVs3+Pj44N1338WyZcsQHx+P2bNnIzAwUPpyNW7cOKxbtw4zZszAqFGjcPToUfz000/Yv39/ee46UanJznyKPIUZqrXth6ruXlJ6elICEsN+hkqlYlBBREREejHqoOLhw4cYNmwY4uLiYG9vj6ZNm+LQoUPo2rUrAGDVqlUwMTFBQEAAMjMz4efnhw0bNkivNzU1xe+//47x48fD19cXNjY2GD58OBYsWCCV8fb2xv79+zF16lSsWbMGNWvWxObNmzmdLL1wrB2doXSpKUtLLKe2EBER0YvFqIOKLVu2FJpvZWWF9evXY/369TrLeHl5aXRvKqhjx464ePHic7WRiIiIiKiyM/rZn4iIiIiIyLgxqCAiIiIiIr0wqCAiIiIiIr0wqCAiIiIiIr0wqCAiIiIiIr0Y9exPRGRYXGmbiIiISgODCqJKqqxW2k5MTIRKpdJIZ+BCRET04mBQQVRJlcVK24mJiRg6cgySUp9o5JVm4EJERETli0EFUSVnyJW2VSoVklKfwNk3ADZOrlJ6aQYuREREVP4YVBCRwdk4uRoscCEiIqLyx6CCiIwKx2AQERFVPAwqiMhoVIYxGNqCppiYGORk55RTi4iIiPTHoIKIyoW26WxjYmLwMEmF6u0HvpBjMHQFTRlPn+D+v3HwzM4up5YRERHph0EFEZU5XdPZSl+u7ZxeyDEYugauP4y8iph7W5Gbw6CCiIgqJgYVRKTB0Ivi6ZrOtrJ8uS44cD3tUXyJ6+DChUREZEwYVBCRzPMuivc8YwUKTmf7PF+uK6OyWriQiIiouBhUEJHM8yyKV95jBSrbjFFlsXAhERFRSTCoICKtSrIoXnmOFagMM0bpYsiFC4mIiEqCQQURlZrSGCtQUly1m4iIqPwxqCCiFwJX7SYiIio/DCqIqELQNdsRF44jIiIqfwwqiMjoFTbbEReOIyIiKn8MKojI6Oma7QioPGtbEBERGTMGFURUYWib7YhrWxAREZU/k/JuABERERERVWy8U0FExcbB0kRERKQNgwoiKhYOliYiIiJdGFQQUbFwsDQRERHpwqCCiEqEg6WJiIioIA7UJiIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivTCoICIiIiIivXBKWSJ6YelaAVypVMLZ2bkcWkRERPRiYlBBRC+kwlYAd7KzxvdBmxlYEBERlRIGFUT0QtK1Anh6UgISw36GSqViUEFERFRKGFQQ0QtN2wrgieXUFiIioheVUQ/UXrx4MV555RXY2dnBxcUFffr0wc2bN2VlMjIyEBgYiKpVq8LW1hYBAQFISEiQlYmNjYW/vz+sra3h4uKC6dOnIycnR1bm+PHjaNGiBSwtLVG3bl1s27bN0LtHRERERPRCMOqg4sSJEwgMDMRff/2FkJAQZGdno1u3bkhPT5fKTJ06Ffv27cOuXbtw4sQJPHjwAP369ZPyc3Nz4e/vj6ysLJw5cwbbt2/Htm3bMGfOHKlMVFQU/P390alTJ0RERGDKlCkYM2YMDh06VKb7S0RERERUERl196eDBw/Knm/btg0uLi4IDw9H+/btkZKSgi1btiA4OBhvvPEGACAoKAgNGzbEX3/9hbZt2+Lw4cO4fv06jhw5AldXVzRv3hwLFy7EzJkzMW/ePFhYWGDTpk3w9vbGihUrAAANGzbE6dOnsWrVKvj5+ZX5fhMRERERVSRGfaeioJSUFACAk5MTACA8PBzZ2dno0qWLVKZBgwbw9PREWFgYACAsLAxNmjSBq6urVMbPzw8qlQrXrl2TyuSvQ11GXYc2mZmZUKlUsgcRERERUWVUYYKKvLw8TJkyBa+99hoaN24MAIiPj4eFhQUcHBxkZV1dXREfHy+VyR9QqPPVeYWVUalUePr0qdb2LF68GPb29tLDw8ND730kIiIiIqqIjLr7U36BgYG4evUqTp8+Xd5NAQDMmjUL06ZNk56rVCoGFkRU7nQt+Adw0T8iIjKcChFUTJgwAb///jtOnjyJmjX/b2pINzc3ZGVlITk5WXa3IiEhAW5ublKZs2fPyupTzw6Vv0zBGaMSEhKgVCpRpUoVrW2ytLTUWFCLiKg8FbbgH8BF/4iIyHCMOqgQQmDixInYs2cPjh8/Dm9vb1l+y5YtYW5ujtDQUAQEBAAAbt68idjYWPj6+gIAfH198fnnn+Phw4dwcXEBAISEhECpVMLHx0cqc+DAAVndISEhUh1ERBWBrgX/AC76R0REhmXUQUVgYCCCg4Px22+/wc7OThoDYW9vjypVqsDe3h6jR4/GtGnT4OTkBKVSiYkTJ8LX1xdt27YFAHTr1g0+Pj549913sWzZMsTHx2P27NkIDAyUfskbN24c1q1bhxkzZmDUqFE4evQofvrpJ+zfv7/c9p2I6HlpW/AP4KJ/RERkOEY9UHvjxo1ISUlBx44dUb16denx448/SmVWrVqFt956CwEBAWjfvj3c3Nzwyy+/SPmmpqb4/fffYWpqCl9fXwwdOhTDhg3DggULpDLe3t7Yv38/QkJC0KxZM6xYsQKbN2/mdLJERERERMVg1HcqhBBFlrGyssL69euxfv16nWW8vLw0ujcV1LFjR1y8eLHEbSQiIiIiquyM+k4FEREREREZPwYVRERERESkFwYVRERERESkFwYVRERERESkFwYVRERERESkFwYVRERERESkFwYVRERERESkF6Nep4KIyBCys7IQExOjka5UKuHs7Fwq20hMTIRKpZKlxcTEICc7p1TqJyIiMiYMKoioUslMS0F01F1M+WQeLC0tZXlOdtb4Pmiz3oFFYmIiho4cg6TUJ7L0jKdPcP/fOHhmZ+tVPxERkbFhUEFElUp25lPkKcxQrW0/VHX3ktLTkxKQGPYzVCqV3kGFSqVCUuoTOPsGwMbJVUp/GHkVMfe2IjenfIKKsrhDQ0RElRODCiKqlKwdnaF0qSlLSyzlbdg4ucq2kfYovpS3UHxlcYeGiIgqLwYVRESVQFncoSEiosqLQQURUSVSFndoiIio8uGUskREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBcGFUREREREpBez8m4AEZGxyM7KQkxMjEa6UqmEs7NzObSIiIioYmBQQUQEIDMtBdFRdzHlk3mwtLSU5TnZWeP7oM0MLIiIiHRgUEFEBCA78ynyFGao1rYfqrp7SenpSQlIDPsZKpWKQQUREZEODCqIiPKxdnSG0qWmLC2xnNpCRERUUXCgNhERERER6YVBBRERERER6YVBBRERERER6YVBBRERERER6YUDtYmIisD1K4iIiArHoIKIqBCVYf0KBk1ERKQvBhVERIV40devqAxBExERGR6DCiKiYnhR16940YMmIiIqGwwqiIjohQ2aiIiobDCoICJ6TrrGIsTExCAnO6ccWkRERFQ+GFQQET2HwsYiZDx9gvv/xsEzO7ucWlc6jHEAd2JiIlQqlUY6B5UTEZUvow8qTp48ieXLlyM8PBxxcXHYs2cP+vTpI+ULITB37lx8++23SE5OxmuvvYaNGzeiXr16UpmkpCRMnDgR+/btg4mJCQICArBmzRrY2tpKZS5fvozAwECcO3cOzs7OmDhxImbMmFGWu0pEFYiusQgA8DDyKmLubUVuTsUNKoxxAHdiYiKGjhyDpNQnGnkcVE5EVL6MPqhIT09Hs2bNMGrUKPTr108jf9myZfjqq6+wfft2eHt747PPPoOfnx+uX78OKysrAMCQIUMQFxeHkJAQZGdnY+TIkRg7diyCg4MBACqVCt26dUOXLl2wadMmXLlyBaNGjYKDgwPGjh1bpvtLRBWLtrEIaY/iy6k1pae8B3BruyMRExODh0kqVG8/EDZOrmXeJiIi0s3og4o333wTb775ptY8IQRWr16N2bNno3fv3gCA7777Dq6urvj1118xaNAg3LhxAwcPHsS5c+fQqlUrAMDatWvRo0cPfPnll3B3d8eOHTuQlZWFrVu3wsLCAo0aNUJERARWrlzJoIKIKrXyGMCt646E1K3MzomDyomIjIxJeTdAH1FRUYiPj0eXLl2kNHt7e7Rp0wZhYWEAgLCwMDg4OEgBBQB06dIFJiYm+Pvvv6Uy7du3h4WFhVTGz88PN2/exOPHj8tob4iICHh29zgp9QmcfQNQy/8D6eH0cnfk5okK3a2MiOhFZfR3KgoTH/+si4Grq6ss3dXVVcqLj4+Hi4uLLN/MzAxOTk6yMt7e3hp1qPMcHR01tp2ZmYnMzEzpubaBg0RE9PxsnFxldyRehG5lREQvqgp9p6I8LV68GPb29tLDw8OjvJtERERERFQuKnRQ4ebmBgBISEiQpSckJEh5bm5uePjwoSw/JycHSUlJsjLa6si/jYJmzZqFlJQU6XHv3j39d4iIiIiIqAKq0EGFt7c33NzcEBoaKqWpVCr8/fff8PX1BQD4+voiOTkZ4eHhUpmjR48iLy8Pbdq0kcqcPHkS2fnmlA8JCUH9+vW1dn0CAEtLSyiVStmDiIjIGCQmJiIyMlLjkZjIIe1EZBhGP6YiLS0Nd+7ckZ5HRUUhIiICTk5O8PT0xJQpU7Bo0SLUq1dPmlLW3d1dWsuiYcOG6N69O9577z1s2rQJ2dnZmDBhAgYNGgR3d3cAwODBgzF//nyMHj0aM2fOxNWrV7FmzRqsWrWqPHaZiIjouRW2noethSmWfr4AVatW1cjjAoJEpA+jDyrOnz+PTp06Sc+nTZsGABg+fDi2bduGGTNmID09HWPHjkVycjLatWuHgwcPSmtUAMCOHTswYcIEdO7cWVr87quvvpLy7e3tcfjwYQQGBqJly5aoVq0a5syZw+lkiYgMTNd6FDnZOSWqxxhX/y4LJVnPI+n+HYT/9BXGTPpIY0FDgAsIEpF+jD6o6NixI4QQOvMVCgUWLFiABQsW6Czj5OQkLXSnS9OmTXHq1KnnbicREZVMketRZBdv6lhjXP27LJR0PY+0R/E6V4HnAoJEpC+jDyqIiOjFlH89ivy/qD+MvIqYe1uLvR5Fea/+XV6e9/hpW9AQ4AKCFZm2O1bAi3+njowLgwoiIipXpbUeRXms/m0MuJ5H5VbYGJoX+U4dGR8GFUREZHClNXaCiOR03bF60e/UkfFhUEFERCVS0kHRpTV2goh0K3jHCqgcd+rIeDCoICKiYnueQdGlNXaitLEfOhFR6WFQQURExabPoOjy6Puv667Ko0ePMHP2PKRlagY07Icux+CLiIqDQQUREZWYtkHRD3R8gS+vsROF3VVRd71qNWgqHFz/bz8qcz90bQEYgy/jUtKxSZV1/RYqHwwqiIhIb8X5Al/WYyd03VUB/q/rlaXSif3Qofv9e9GDL113YQDj++Jd0rFJlXX9Fio/DCqIiEhvxfkCX15jJ7TdVeG0q3K63r/Cgi9dd6aM7cu4LoVNxQoY3xfvko5Nqqzrt1D5YVBBRESlhl/gDaOspuQt+P7peu9ehF/BdX1JB5598X5w4gdcuXIFXl7yILm8g6aSjk2qrOu3UNljUEFERJSPsfVDN8YpeV+kX8G1TcX6IgRNRGWNQQUREdH/Z4xfJo11Sl6g4vwKXuIBzuUYNOka58HFIsnYMaggIiL6/4z5F/jymJL3RaDPnZ6yDpoKG+fBxSLJ2DGoICIiKqAkU+aWdx97Kpwx3+kpqLBxHsbYXqL8GFQQEREVoay6RZXVgGxDMrYxKWoV6U6PtnEextxeIoBBBRERUZHKoluUMQ7ILiljHJNCRGWDQQUREVExGbJbVEXqpqOLMY9JKS267sRkZWXBwsJCI72879BoY6x3k6hiY1BBRET0nAr7Zd7WwhRLP1+AqlWraryusC9vFambji7lNSbF0N3HdL3f2VlZ+Dc2BjW9vGFmLv9qpesOTXl1dePdJDIUBhVERETPSdcv80n37yD8p68wZtJHGl/cAO0BR0UbO1EShgi+CiqL7mOFrTx+N3orHFv3LtYdmvLs6lbY3SRjXfCPKgYGFURERHrSthK1ti9ugO6AoyKNnSip5w2+SvLLeVl2H9O18nhx79DExMTgYZIK1dsPLLeubgXbyjsYpC8GFURERAai7UumroCjIo2deF4lCb6edxyGMXUf0/VFXQog7ZyMpq2VYTwMGRaDCiIionKg69fuykhb8AUY5+rcJVFYdyljDSAryirpZHwYVBAREZFRKslMS8Y8JoUBZNnSNggeqFgzdFVEDCqIiIjI6JR0pqUXeUwKFZ+uQfClNUMXwCBEFwYVREREZHRKOtOSMXcporJT2IB9XTN0aZv16tGjR5g5ex7SMjXPJw5c145BBRERERmt4s60xC5FFVtp3xXQNWC/uLNeqe98tRo0FQ6u/1eeA9d1Y1BBRERERDrpGtsClE5XIF1dloDC7wqUxgKCRQ2mt1Q6FXshR6Byj9tgUEFEREREWhW2fgVQOl2BdHVZKmxBPl3dk553bE1x73wVdjyeZ9zGi4RBBRERERFppeuXfKDwrkDP052pYJelwr7A6+qeZOixNYUdj5KurP6iYVBBRERERIXStZaItq5ApTXIuagv8Nq6J5XV2BpdC1vqyqsMa30wqCAiIiKiEnueQc7aujMVNQ6isC/wZDwYVBARERFRiZV0kHNRQQjXGKnYGFQQERER0XMr7iDnooIQrjFSsTGoICIiIqIywzVGXkwm5d0AIiIiIiKq2BhUEBERERGRXhhUEBERERGRXhhUEBERERGRXhhUEBERERGRXhhUFLB+/XrUqlULVlZWaNOmDc6ePVveTSIiIiIiMmoMKvL58ccfMW3aNMydOxcXLlxAs2bN4Ofnh4cPH5Z304iIiIiIjBaDinxWrlyJ9957DyNHjoSPjw82bdoEa2trbN26tbybRkRERERktLj43f+XlZWF8PBwzJo1S0ozMTFBly5dEBYWplE+MzMTmZmZ0vOUlBQAgEqlMnxjC0hNTUVuTg6S46KRnfFESlc9vA+RlwdV/D2YKVBk+vO8xtDplXXbbJPxb5ttMv5tG2ObeDyMv008Hsbfpop2PNIfP0RuTg5SU1PL/HuientCCINvSyHKYisVwIMHD1CjRg2cOXMGvr6+UvqMGTNw4sQJ/P3337Ly8+bNw/z588u6mUREREREJXLv3j3UrFmz6IJ64J2K5zRr1ixMmzZNep6Xl4ekpCRUrVoVCoVC62tUKhU8PDxw7949KJXKsmpqhcRjVTI8XiXD41UyPF4lw+NVfDxWJcPjVTI8Xs/uUKSmpsLd3d3g22JQ8f9Vq1YNpqamSEhIkKUnJCTAzc1No7ylpSUsLS1laQ4ODsXallKprLQnd0nxWJUMj1fJ8HiVDI9XyfB4FR+PVcnweJVMZT9e9vb2ZbIdDtT+/ywsLNCyZUuEhoZKaXl5eQgNDZV1hyIiIiIiIjneqchn2rRpGD58OFq1aoXWrVtj9erVSE9Px8iRI8u7aURERERERotBRT4DBw5EYmIi5syZg/j4eDRv3hwHDx6Eq6trqdRvaWmJuXPnanSbIk08ViXD41UyPF4lw+NVMjxexcdjVTI8XiXD41W2OPsTERERERHphWMqiIiIiIhILwwqiIiIiIhILwwqiIiIiIhILwwqiIiIiIhILwwqSmDx4sV45ZVXYGdnBxcXF/Tp0wc3b96UlcnIyEBgYCCqVq0KW1tbBAQEaCyoFxsbC39/f1hbW8PFxQXTp09HTk6OrMzx48fRokULWFpaom7duti2bZuhd6/UFXW8kpKSMHHiRNSvXx9VqlSBp6cnJk2ahJSUFFk9CoVC47Fz505ZmcpwvACgY8eOGsdi3LhxsjI8v56Jjo7Weu4oFArs2rVLKlcZzq+NGzeiadOm0gJQvr6++OOPP6R8XrfkCjtevG5pKur84nVLrrDjxetW4ZYsWQKFQoEpU6ZIabx+GRFBxebn5yeCgoLE1atXRUREhOjRo4fw9PQUaWlpUplx48YJDw8PERoaKs6fPy/atm0rXn31VSk/JydHNG7cWHTp0kVcvHhRHDhwQFSrVk3MmjVLKnP37l1hbW0tpk2bJq5fvy7Wrl0rTE1NxcGDB8t0f/VV1PG6cuWK6Nevn9i7d6+4c+eOCA0NFfXq1RMBAQGyegCIoKAgERcXJz2ePn0q5VeW4yWEEB06dBDvvfee7FikpKRI+Ty//u945eTkyI5TXFycmD9/vrC1tRWpqalSPZXh/Nq7d6/Yv3+/uHXrlrh586b45JNPhLm5ubh69aoQgtetggo7XrxuaSrq/OJ1S66w48Xrlm5nz54VtWrVEk2bNhWTJ0+W0nn9Mh4MKvTw8OFDAUCcOHFCCCFEcnKyMDc3F7t27ZLK3LhxQwAQYWFhQgghDhw4IExMTER8fLxUZuPGjUKpVIrMzEwhhBAzZswQjRo1km1r4MCBws/Pz9C7ZFAFj5c2P/30k7CwsBDZ2dlSGgCxZ88ena+pTMerQ4cOsotpQTy/Cj+/mjdvLkaNGiVLq6znl6Ojo9i8eTOvW8WkPl7a8LqlKf/x4nWraIWdX7xuCZGamirq1asnQkJCZOcTr1/Ghd2f9KC+3e3k5AQACA8PR3Z2Nrp06SKVadCgATw9PREWFgYACAsLQ5MmTWQL6vn5+UGlUuHatWtSmfx1qMuo66ioCh4vXWWUSiXMzOTrMgYGBqJatWpo3bo1tm7dCpFveZXKdrx27NiBatWqoXHjxpg1axaePHki5fH80n1+hYeHIyIiAqNHj9bIq0znV25uLnbu3In09HT4+vryulWEgsdLG163/o+u48XrlnZFnV+8bj0TGBgIf39/jX3i9cu4cEXt55SXl4cpU6bgtddeQ+PGjQEA8fHxsLCwgIODg6ysq6sr4uPjpTIFV+hWPy+qjEqlwtOnT1GlShVD7JJBaTteBf33339YuHAhxo4dK0tfsGAB3njjDVhbW+Pw4cP44IMPkJaWhkmTJgGoXMdr8ODB8PLygru7Oy5fvoyZM2fi5s2b+OWXXwDw/Crs/NqyZQsaNmyIV199VZZeWc6vK1euwNfXFxkZGbC1tcWePXvg4+ODiIgIXre00HW8CuJ165nCjhevW5qKe35V9usWAOzcuRMXLlzAuXPnNPL4vcu4MKh4ToGBgbh69SpOnz5d3k2pEIo6XiqVCv7+/vDx8cG8efNkeZ999pn0/5dffhnp6elYvny5dPF8Eek6Xvm/uDRp0gTVq1dH586dERkZiTp16pR1M41GUefX06dPERwcLDuX1CrL+VW/fn1EREQgJSUFu3fvxvDhw3HixInybpbR0nW88n/x43Xr/xR2vHjd0lSc84vXLeDevXuYPHkyQkJCYGVlVd7NoSKw+9NzmDBhAn7//XccO3YMNWvWlNLd3NyQlZWF5ORkWfmEhAS4ublJZQrOSqB+XlQZpVJZIaNlXcdLLTU1Fd27d4ednR327NkDc3PzQutr06YN7t+/j8zMTACV73jl16ZNGwDAnTt3APD80nW8du/ejSdPnmDYsGFF1veinl8WFhaoW7cuWrZsicWLF6NZs2ZYs2YNr1s66DpearxuyRV1vPLjdat4x4vXrWfdmx4+fIgWLVrAzMwMZmZmOHHiBL766iuYmZnB1dWV1y8jwqCiBIQQmDBhAvbs2YOjR4/C29tblt+yZUuYm5sjNDRUSrt58yZiY2OlvpK+vr64cuUKHj58KJUJCQmBUqmUfqHw9fWV1aEuo6s/r7Eq6ngBz37p69atGywsLLB3795i/RIREREBR0dHWFpaAqhcx6ugiIgIAED16tUB8PzSZcuWLejVqxecnZ2LrPdFPb8KysvLQ2ZmJq9bxaQ+XgCvW8WR/3gVVJmvW7poO168bgGdO3fGlStXEBERIT1atWqFIUOGSP/n9cuIlN8Y8Ypn/Pjxwt7eXhw/flw2jduTJ0+kMuPGjROenp7i6NGj4vz588LX11f4+vpK+eqpzbp16yYiIiLEwYMHhbOzs9apzaZPny5u3Lgh1q9fXyGnNivqeKWkpIg2bdqIJk2aiDt37sjK5OTkCCGeTb337bffiitXrojbt2+LDRs2CGtrazFnzhxpO5XleN25c0csWLBAnD9/XkRFRYnffvtN1K5dW7Rv316qg+eX/PMohBC3b98WCoVC/PHHHxp1VJbz6+OPPxYnTpwQUVFR4vLly+Ljjz8WCoVCHD58WAjB61ZBhR0vXrc0FXa8eN3SVNTnUQhetwpTcDYxXr+MB4OKEgCg9REUFCSVefr0qfjggw+Eo6OjsLa2Fn379hVxcXGyeqKjo8Wbb74pqlSpIqpVqyY+/PBD2VSEQghx7Ngx0bx5c2FhYSFq164t20ZFUdTxOnbsmM4yUVFRQggh/vjjD9G8eXNha2srbGxsRLNmzcSmTZtEbm6ubFuV4XjFxsaK9u3bCycnJ2FpaSnq1q0rpk+fLpvvXQieXwX3ZdasWcLDw0PjnBGi8pxfo0aNEl5eXsLCwkI4OzuLzp07y77A8LolV9jx4nVLU2HHi9ctTUV9HoXgdaswBYMKXr+Mh0KIfHOQERERERERlRDHVBARERERkV4YVBARERERkV4YVBARERERkV4YVBARERERkV4YVBARERERkV4YVBARERERkV4YVBARERERkV4YVBARERERkV4YVBARkdHo2LEjpkyZUt7NICKiEmJQQUREpaJnz57o3r271rxTp05BoVDg8uXLZdwqIiIqCwwqiIioVIwePRohISG4f/++Rl5QUBBatWqFpk2blkPLiIjI0BhUEBFRqXjrrbfg7OyMbdu2ydLT0tKwa9cu9OnTB++88w5q1KgBa2trNGnSBD/88EOhdSoUCvz666+yNAcHB9k27t27hwEDBsDBwQFOTk7o3bs3oqOjS2eniIioWBhUEBFRqTAzM8OwYcOwbds2CCGk9F27diE3NxdDhw5Fy5YtsX//fly9ehVjx47Fu+++i7Nnzz73NrOzs+Hn5wc7OzucOnUKf/75J2xtbdG9e3dkZWWVxm4REVExMKggIqJSM2rUKERGRuLEiRNSWlBQEAICAuDl5YWPPvoIzZs3R+3atTFx4kR0794dP/3003Nv78cff0ReXh42b96MJk2aoGHDhggKCkJsbCyOHz9eCntERETFwaCCiIhKTYMGDfDqq69i69atAIA7d+7g1KlTGD16NHJzc7Fw4UI0adIETk5OsLW1xaFDhxAbG/vc27t06RLu3LkDOzs72NrawtbWFk5OTsjIyEBkZGRp7RYRERXBrLwbQEREL5bRo0dj4sSJWL9+PYKCglCnTh106NABS5cuxZo1a7B69Wo0adIENjY2mDJlSqHdlBQKhawrFfCsy5NaWloaWrZsiR07dmi81tnZufR2ioiICsWggoiIStWAAQMwefJkBAcH47vvvsP48eOhUCjw559/onfv3hg6dCgAIC8vD7du3YKPj4/OupydnREXFyc9v337Np48eSI9b9GiBX788Ue4uLhAqVQabqeIiKhQ7P5ERESlytbWFgMHDsSsWbMQFxeHESNGAADq1auHkJAQnDlzBjdu3MD777+PhISEQut64403sG7dOly8eBHnz5/HuHHjYG5uLuUPGTIE1apVQ+/evXHq1ClERUXh+PHjmDRpktapbYmIyDAYVBARUakbPXo0Hj9+DD8/P7i7uwMAZs+ejRYtWsDPzw8dO3aEm5sb+vTpU2g9K1asgIeHB15//XUMHjwYH330EaytraV8a2trnDx5Ep6enujXrx8aNmyI0aNHIyMjg3cuiIjKkEIU7KxKRERERERUArxTQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQUREREREemFQQWSkatWqhREjRpR3MyqknJwczJgxAx4eHjAxMUGfPn0Muj2FQiE9vvzyS4Nuiyqvjz/+GG3atCnRa/Ly8tC4cWN8/vnnBmoVlYdBgwZhwIAB5d0MIhkGFURlYNu2bVAoFDh//rzW/I4dO6Jx48Z6b+fAgQOYN2+e3vVUdFu3bsXy5cvx9ttvY/v27Zg6darBt9m3b1/873//g7+/v84yO3bsgEKhgK2trUbe2bNn8cEHH6Bly5YwNzeHQqHQWc/GjRvRv39/eHp6QqFQ6Aw+4+Li8PHHH6NTp06ws7ODQqHA8ePHi71P8+bNkwVM6oeVlZXW8lu2bEHDhg1hZWWFevXqYe3atcXeli6pqamYMWMGvL29YWlpiRo1auDtt9/GkydPpDLPu5/JyclwcXGBQqHA7t27NfLDw8PRvXt3KJVK2NnZoVu3boiIiChWu3/55RcMHDgQtWvXhrW1NerXr48PP/wQycnJGmVr1aql9TiPGzdOVm7KlCm4dOkS9u7dW6w2AMAPP/yAe/fuYcKECRp5kZGReP/991G7dm1YWVlBqVTitddew5o1a/D06dNib8OQNmzYgG3btuldT1xcHMaOHQtvb29UqVIFderUwbRp0/Do0SONsuvWrUPDhg2l823atGlIT0+XlYmOjtb6nikUCuzcuVMql5eXh23btqFXr17w8PCAjY0NGjdujEWLFiEjI0Nj2wkJCRg5ciRcXFxQpUoVtGjRArt27dIoN3PmTPz888+4dOmS3seGqLSYlXcDiEi7mzdvwsSkZHH/gQMHsH79+kofWBw9ehQ1atTAqlWrymybTZs2xdChQ3Xmp6WlYcaMGbCxsdGaf+DAAWzevBlNmzZF7dq1cevWLZ11LV26FKmpqWjdujXi4uJ0lrt58yaWLl2KevXqoUmTJggLCyv+DuWzceNGWSBkamqqUebrr7/GuHHjEBAQgGnTpuHUqVOYNGkSnjx5gpkzZz7XdlNSUtChQwfcv38fY8eORd26dZGYmIhTp04hMzMT1tbWeu3nnDlzZMFJfhcuXEC7du3g4eGBuXPnIi8vDxs2bECHDh1w9uxZ1K9fv9C6x44dC3d3dwwdOhSenp64cuUK1q1bhwMHDuDChQuoUqWKrHzz5s3x4YcfytJeeukl2XM3Nzf07t0bX375JXr16lWsfVy+fDkGDRoEe3t7Wfr+/fvRv39/WFpaYtiwYWjcuDGysrJw+vRpTJ8+HdeuXcM333xTrG0Y0oYNG1CtWjW97tqmpaXB19cX6enp+OCDD+Dh4YFLly5h3bp1OHbsGMLDw6Vr7cyZM7Fs2TK8/fbbmDx5Mq5fv461a9fi2rVrOHTokEbd77zzDnr06CFL8/X1lf7/5MkTjBw5Em3btsW4cePg4uKCsLAwzJ07F6GhoTh69Kj0A4JKpUK7du2QkJCAyZMnw83NDT/99BMGDBiAHTt2YPDgwVK9L7/8Mlq1aoUVK1bgu+++e+5jQ1SqBBEZXFBQkAAgzp07pzW/Q4cOolGjRnpvJzAwUJTnxzotLa3ctp1fp06dSuV4quXm5oqnT5/qzAcg5s6dW2gdM2fOFPXr1xdDhgwRNjY2Gvnx8fHiyZMnQoii38fo6GiRl5cnhBDCxsZGDB8+XGs5lUolHj16JIQQYteuXQKAOHbsWKHtzG/u3LkCgEhMTCy03JMnT0TVqlWFv7+/LF29r0lJScXeZn7jx48XDg4O4u7du4WWe579vHLlijAzMxMLFiwQAMSuXbtk+T169BCOjo7iv//+k9IePHggbG1tRb9+/Ypsu7btb9++XQAQ3377rSzdy8tL49jpsnv3bqFQKERkZGSRZS9cuCAAiCNHjsjS7969K2xtbUWDBg3EgwcPNF53+/ZtsXr16mK1x9AaNWokOnTooFcdO3bsEADE77//LkufM2eOACAuXLgghHj2/pqZmYl3331XVm7t2rUCgNi7d6+UFhUVJQCI5cuXF7rtzMxM8eeff2qkz58/XwAQISEhUtqyZcsEABEaGiql5ebmildeeUW4ubmJzMxMWR1ffvmlsLGxEampqUUcAaKywe5PREaq4JiK7OxszJ8/H/Xq1YOVlRWqVq2Kdu3aISQkBAAwYsQIrF+/HoC8j79aeno6PvzwQ3h4eMDS0hL169fHl19+CSGEbLtPnz7FpEmTUK1aNdjZ2aFXr174999/oVAoZHdA1F1jrl+/jsGDB8PR0RHt2rUDAFy+fBkjRoyQulW4ublh1KhRGl0N1HXcunULQ4cOhb29PZydnfHZZ59BCIF79+6hd+/eUCqVcHNzw4oVKwo9ZuouCceOHcO1a9ekY6DuClPcY6BQKDBhwgTs2LEDjRo1gqWlJQ4ePFj0m6bD7du3sWrVKqxcuRJmZtpvELu6umr8eq2Ll5dXod2j1Ozs7ODk5FSitmojhIBKpdI4TmrHjh3Do0eP8MEHH8jSAwMDkZ6ejv3795d4m8nJyQgKCpK6rGRlZSEzM1Nr2efZz8mTJ6Nv3754/fXXteafOnUKXbp0QdWqVaW06tWro0OHDvj999+RlpZWaP0dO3bUSOvbty8A4MaNG1pfk5WVpdHNpqAuXboAAH777bdCywHAr7/+CgsLC7Rv316WvmzZMqSlpWHLli2oXr26xuvq1q2LyZMnS89zcnKwcOFC1KlTB5aWlqhVqxY++eQTjfej4DVCreC1TN0d9M8//8S0adPg7OwMGxsb9O3bF4mJibLXXbt2DSdOnJA+y/mPa2RkJCIjI4s8DiqVCsCzz1h+6n1Xf+7CwsKQk5ODQYMGycqpn+fv1pRfeno6srKytOZZWFjg1Vdf1UjXdi6cOnUKzs7OeOONN6Q0ExMTDBgwAPHx8Thx4oSsjq5duyI9PV36G0BU3hhUEJWhlJQU/PfffxqP7OzsIl87b948zJ8/H506dcK6devw6aefwtPTExcuXAAAvP/+++jatSsA4H//+5/0AJ59KezVqxdWrVqF7t27Y+XKlahfvz6mT5+OadOmybYzYsQIrF27Fj169MDSpUtRpUqVQscJ9O/fH0+ePMEXX3yB9957DwAQEhKCu3fvYuTIkVi7di0GDRqEnTt3okePHlq/mA4cOBB5eXlYsmQJ2rRpg0WLFmH16tXo2rUratSogaVLl6Ju3br46KOPcPLkSZ1tcXZ2xv/+9z80aNAANWvWlI5Bw4YNS3QMgGddqKZOnYqBAwdizZo1qFWrVuFvUCGmTJmCTp06aXSTqChq164Ne3t72NnZYejQoUhISJDlX7x4EQDQqlUrWXrLli1hYmIi5ZfE6dOnkZGRgbp16+Ltt9+GtbU1qlSpgtdee63Y4xp02bVrF86cOYNly5bpLJOZmak1yLO2tkZWVhauXr1a4u3Gx8cDAKpVq6aRd/ToUVhbW8PW1ha1atXCmjVrtNZhb2+POnXq4M8//yxye2fOnEHjxo1hbm4uS9+3bx9q166t9cuuNmPGjMGcOXPQokULrFq1Ch06dMDixYs1vnyX1MSJE3Hp0iXMnTsX48ePx759+2RjP1avXo2aNWuiQYMG0mf5008/lfI7d+6Mzp07F7md9u3bw8TEBJMnT8Zff/2F+/fv48CBA/j888/Rp08fNGjQAACkIKng+67uZhceHq5R9/z582FrawsrKyu88sorOHz4cLH2Xdu5UNg5p237Pj4+qFKlSrHOBaIyUX43SYgqD3X3p8IeBbvreHl5ybq1NGvWrMguErq6zfz6668CgFi0aJEs/e233xYKhULcuXNHCCFEeHi4ACCmTJkiKzdixAiNLj7qrjHvvPOOxvbU3Xjy++GHHwQAcfLkSY06xo4dK6Xl5OSImjVrCoVCIZYsWSKlP378WFSpUkVnV5/8tHUnK+4xEOJZdyYTExNx7dq1IrelLq+r+9Pvv/8uzMzMpLqGDx+utftTfiXpxlZY96f8nqf70+rVq8WECRPEjh07xO7du8XkyZOFmZmZqFevnkhJSZG119TUVGsdzs7OYtCgQcXeptrKlSsFAFG1alXRunVrsWPHDrFhwwbh6uoqHB0dtXbbEaLo/Xzy5Inw9PQUs2bNEkI866YELd2fmjRpIl566SWRk5MjpWVmZgpPT08BQOzevbvE+zR69Ghhamoqbt26JUvv2bOnWLp0qfj111/Fli1bxOuvvy4AiBkzZmitp1u3bqJhw4ZFbq9mzZoiICBAlpaSkiIAiN69exerzREREQKAGDNmjCz9o48+EgDE0aNHpTRdn4OC1zL19bBLly5SNz4hhJg6daowNTUVycnJUlph3Z+8vLyEl5dXsfZj8+bNwsHBQXbNHT58uMjOzpbKqK9/CxculL324MGDAoCwtbWV0mJiYkS3bt3Exo0bxd69e8Xq1auFp6enMDEx0ehmpU2XLl2EUqkUjx8/ltImTpwoTExMRHR0tKzsoEGDBAAxYcIEjXpeeukl8eabbxbrGBAZGu9UEJWh9evXIyQkROPRtGnTIl/r4OCAa9eu4fbt2yXe7oEDB2BqaopJkybJ0j/88EMIIfDHH38AgNTFp2A3lokTJ+qsu+AMNYD8l76MjAz8999/aNu2LQBId1byGzNmjPR/U1NTtGrVCkIIjB49Wkp3cHBA/fr1cffuXZ1tKUxxj4Fahw4d4OPj81zbUsvKysLUqVMxbtw4vesqD5MnT8batWsxePBgBAQEYPXq1di+fTtu376NDRs2SOWePn0KCwsLrXVYWVk910xC6tST+E8AAMYiSURBVO5FCoUCoaGhGDx4MMaPH49ff/0Vjx8/lrr6ldSSJUuQnZ2NTz75pNByH3zwAW7duoXRo0fj+vXruHr1KoYNGyYNjC/pPgUHB2PLli348MMPUa9ePVne3r17MWPGDPTu3RujRo3CiRMn4Ofnh5UrV+L+/fsadTk6OuK///4rcpuPHj2Co6OjLE3dFcjOzq5Y7T5w4AAAaNzNUw8qf56ubWpjx46VdeN7/fXXkZubi5iYmGK9Pjo6GtHR0cUqW6NGDbRu3RqrV6/Gnj17MG3aNOzYsQMff/yxVKZFixZo06YNli5diqCgIERHR+OPP/7A+++/D3Nzc9l77unpiUOHDmHcuHHo2bMnJk+ejIsXL8LZ2VljwH1BX3zxBY4cOYIlS5bAwcFBSh8zZgxMTU0xYMAAnDlzBpGRkVi8eDH27NkDQPs5V9xzgagsMKggKkOtW7dGly5dNB4F//Brs2DBAiQnJ+Oll15CkyZNMH36dFy+fLlY242JiYG7u7vGF4mGDRtK+ep/TUxM4O3tLStXt25dnXUXLAsASUlJmDx5sjROwNnZWSqXkpKiUd7T01P23N7eHlZWVhrdROzt7fH48WOdbSlMcY+Bmrb9KqlVq1bhv//+w/z58/Wuy1gMHjwYbm5uOHLkiJRWpUoVnX3KMzIyij1WJD/1a3r27Cmbeapt27bw9vbGmTNnSlxndHQ0li9fjs8//1zrtL75jRs3Dp988gmCg4PRqFEjNGnSBJGRkZgxYwYAFPn6/E6dOoXRo0fDz8+vWOtFKBQKTJ06FTk5OVqnxhVCFGtMjbpsfkqlEsCzqXqLQ31NKHgNcHNzg4ODQ7EDAG0Kfu7V18Hn/Yzr8ueff+Ktt97C559/jsmTJ6NPnz5YsWIFZs+ejZUrV+L69etS2Z9//hnNmjXDqFGj4O3tjZ49e2LAgAF4+eWXi3zPnZycMHLkSNy8eVNrMAgAP/74I2bPno3Ro0dj/PjxsrymTZsiODgYkZGReO2111C3bl189dVXWL16NQDt51xJzgUiQ2NQQVRBtG/fHpGRkdi6dSsaN26MzZs3o0WLFti8eXO5tkvbF8YBAwbg22+/xbhx4/DLL7/g8OHD0l2QvLw8jfLapijVlgZofkkylOf5IpxfSkoKFi1ahPfeew8qlUr6VTUtLQ1CCERHR+Phw4el1Nqy5eHhgaSkJOl59erVkZubq7E/WVlZePToEdzd3Uu8DfVrCg6uBQAXF5fn+uI5Z84c1KhRAx07dpTeD3Xf9sTERERHR8vOz88//xwJCQk4deoULl++jHPnzkn5Bad71eXSpUvo1asXGjdujN27d+scqF+Qh4cHAMiOs9rjx4+1jssoqGrVqhrHSalUwt3dvcRjQvT54pqbm6s1vaw+419//TVcXV01xvz06tULQghZgFqjRg2cPn0at27dwsmTJ3H//n0sW7YM9+7dK9Z7Xtj7FhISgmHDhsHf3x+bNm3S+vq3334bDx48wNmzZxEWFoaYmBjUrl0bgPZzrrjnAlFZYFBBVIGofwlTL2jVtGlT2Wwruv7we3l54cGDBxq/Tv7zzz9SvvrfvLw8REVFycrduXOn2G18/PgxQkND8fHHH2P+/Pno27cvunbtKv1hLC/FPQal5fHjx0hLS8OyZcvg7e0tPX7++Wc8efIE3t7eGDt2bKlusyyoAyJnZ2cprXnz5gCgsbjj+fPnkZeXJ+WXRMuWLQEA//77r0begwcPZNsvrtjYWNy5cwe1a9eW3o933nkHwLPuTt7e3lL3IDX1rGZNmjQBABw5ckQaPFyUyMhIdO/eHS4uLjhw4ECJ7m6ou/lp28+oqCjpDlthGjRooPFZBoC33noLkZGRxVrPQ31NKNjtMiEhAcnJybLPjaOjo8bifllZWYWupVKU0vgVPiEhQWtgo54gIycnRyOvXr16eP311+Hm5obr168jLi5OmnmrMLret7///ht9+/ZFq1at8NNPPxUaXFpYWOCVV15B27ZtYWFhId0VLLj9nJwc3Lt3r1jnAlFZYFBBVEEUnI7V1tYWdevWlU3rqF5YreAf9h49eiA3Nxfr1q2Tpa9atQoKhQJvvvkmAMDPzw8AZP3lAZRoZWT1r48Ff21U38IvL8U9BqXFxcUFe/bs0Xh06tQJVlZW2LNnD2bNmlWq29RHbGysFGCp5Z/eU23jxo1ITExE9+7dpbQ33ngDTk5O2Lhxo0ZZa2vrQmcP06V+/fpo1qwZfvvtN1mf8cOHD+PevXvSTGclsWjRIo33Y+HChQCAGTNmYM+ePToXJwSedV05d+4cpkyZIluYUtuxi4+PR7du3WBiYoJDhw7pDIKSkpI0vvBmZ2djyZIlsLCwQKdOnWR5KSkpiIyMLNbMTb6+vrh69arG1K/qRRjHjBmjMZMX8CwYUs8+pZ6xrODnd+XKlQAge2/r1KmjMTvbN998o/NORXHY2NhoXYVc3c7iTCn70ksvISEhQaMr2Q8//ADg2UJyuuTl5WHGjBmwtraWjR/T9tn4999/sXXrVjRt2lQ2Ve+NGzfg7++PWrVq4ffffy/RXdDbt29j06ZNeOuttzTuVFy/fh0ZGRnFnsWLyNC4ojZRBeHj44OOHTuiZcuWcHJywvnz57F7927ZFIzqX3cnTZoEPz8/mJqaYtCgQejZsyc6deqETz/9FNHR0WjWrBkOHz6M3377DVOmTEGdOnWk16sH5D569Aht27bFiRMnpNWdi/OroVKpRPv27bFs2TJkZ2ejRo0aOHz4sNZfTMtScY9BabG2tkafPn000n/99VecPXtWIy8mJkaaAlj9i/+iRYsAPPu1+N1335XK7tu3D5cuXQLw7Avo5cuXpbK9evWSDfxXp1+7dg3As+mGT58+DQCYPXu2VG7YsGE4ceKELBj08vLCwIED0aRJE1hZWeH06dPYuXMnmjdvjvfff18qV6VKFSxcuBCBgYHo378//Pz8cOrUKXz//ff4/PPPZWtIHD9+HJ06dcLcuXOLXPl91apV6Nq1K9q1a4f3338fKSkpWLlyJV566SWN/ujF2U/1Oir5qQfKvvLKK7L35OTJk1iwYAG6deuGqlWr4q+//kJQUBC6d+8uW8NB17Hr3r077t69ixkzZuD06dNSW4BnXbrUQdHevXuxaNEivP322/D29kZSUhKCg4Nx9epVfPHFF3Bzc5Nt68iRIxBCoHfv3oUeOwDo3bs3Fi5ciBMnTqBbt25Sep06dRAcHIyBAweiYcOGshW1z5w5g127dknrSjRr1gzDhw/HN998g+TkZGlF8e3bt6NPnz6yoGfMmDHSqupdu3bFpUuXcOjQIb2657Rs2RIbN27EokWLULduXbi4uEjrOKinky1qsPaECRMQFBSEnj17YuLEifDy8sKJEyfwww8/oGvXrmjTpo1UdvLkycjIyEDz5s2RnZ2N4OBgaX/zjwGZMWMGIiMj0blzZ7i7uyM6Ohpff/010tPTZdMBp6amws/PD48fP8b06dM1BrbXqVNHtgK3j48P+vfvD09PT0RFRWHjxo1wcnLS2l0qJCQE1tbWzxVgExlEOcw4RVTpPM+K2gWnYVy0aJFo3bq1cHBwEFWqVBENGjQQn3/+ucjKypLK5OTkiIkTJwpnZ2ehUChk05KmpqaKqVOnCnd3d2Fubi7q1asnli9fLpvSUQgh0tPTRWBgoHBychK2traiT58+4ubNmwKAbIrXwlZbvn//vujbt69wcHAQ9vb2on///uLBgwc6p6UtWIeuaVeLu/K4rnLFPQYARGBgYJHbyV++qBW11XTtm3pqU22PglNqDh8+XGfZoKAgjbbpeuTXoUMHjbQxY8YIHx8fYWdnJ8zNzUXdunXFzJkzhUql0rpv33zzjahfv76wsLAQderUEatWrdI4tvv27RMAxKZNm4p1vEJCQkTbtm2FlZWVcHJyEu+++66Ii4vTKFfc/SxI15Syd+7cEd26dRPVqlUTlpaWokGDBmLx4sUaqxoLof3YFdae/O/n+fPnRc+ePUWNGjWEhYWFsLW1Fe3atRM//fST1vYOHDhQtGvXrtB9yq9p06Zi9OjRWvNu3bol3nvvPVGrVi1hYWEh7OzsxGuvvSbWrl0rMjIypHLZ2dli/vz5wtvbW5ibmwsPDw8xa9YsWRkhnq3+PHPmTFGtWjVhbW0t/Pz8xJ07d3ROKVvweqh+L/JPBxwfHy/8/f2FnZ2dxrEryZSy//zzj3j77beFh4eHMDc3F15eXuKjjz4S6enpsnJBQUGiWbNmwsbGRtjZ2YnOnTvLps1VCw4OFu3btxfOzs7CzMxMVKtWTfTt21eEh4fLyqlX3tb1KDgd9KBBg4SHh4ewsLAQ7u7uYty4cSIhIUHrPrVp00YMHTq0WPtPVBYUQpTRqEciqrAiIiLw8ssv4/vvv8eQIUPKuzlGR6FQYPr06VK3En0Heb/IZsyYgR9++AF37tyBpaVleTenQomPj8f/Y+8+w6Oq1r+P/wbSA5MQSKGE0HsHhVgQBAnlKAJHQUGKWFBQEQX+HEWaCoIoFkTPUcACgiiKBxEIQbAQQZEoTYQAoSUQjWGSQELKfl7wzJxMephM6vdzXXPB7L1mr7V3JpO591rrXo0bN9aaNWuK1FMhXe2xmThxok6dOmWXvhQVW1RUlLp06aJffvnlmuYsAc7AnAoAdvLKhb5kyRJVq1ZNPXv2LIMWVQyLFi2Sv7//Na+fUFV88803mjlzJgHFNViyZInat29f5IBCkkaOHKmGDRvyvqxkFixYoH/+858EFChX6KkAYGfOnDnau3evevfuLRcXF3399df6+uuv9dBDD+mdd94p6+aVS9nXbGjRokWu/PsAAFR2BBUA7ISHh2vOnDk6dOiQkpOT1bBhQ91333165plnipxjHwAAVC0EFQAAAAAcwpwKAAAAAA4hqAAAAADgEAZIl5CsrCydO3dONWvWLNICYQAAAIAzGYahpKQk1atXT9WqObcvgaCihJw7d07BwcFl3QwAAADAzunTp9WgQQOn1kFQUUJq1qwp6eoPzWw2l3FrAAAAUNVZLBYFBwfbvqc6E0FFCbEOeTKbzQQVAAAAKDdKY2g+E7UBAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOMSlrBsAAAAAx8XFxSkxMbGsm4EC+Pr6KigoqKyb4RQEFQAAABVcXFyc/vnPfyo1NbWsm4ICeHh46NNPP62UgQVBBQAAQAWXmJio1NRUzZs3T40bNy7r5iAPJ06c0MyZM5WYmEhQAQAAgPKrcePGatWqVVk3A1UQE7UBAAAAOISeCgAAcoiPj5fFYpEkmc1m+fv7l3GLAKB8I6gAACCb+Ph4jRr3gBKSLkmS/Gp66aMV7xJYAEABGP4EAEA2FotFCUmX5B86TP6hw5SQdMnWawEAyBtBBQAAefD2C5S3X2BZNwMoEXFxcXrsscfUpEkTubu7Kzg4WLfffrsiIiJKvS0mk0lffPHFNb121apV6tixo7y8vFS3bl3df//9+uuvv2z709PTNXfuXDVt2lQeHh7q2LGjNm/ebHeM2bNny2Qy2T2yT25PSEjQY489ppYtW8rT01MNGzbU448/rosXL9odJyIiQjfccINq1qypoKAgTZ8+XRkZGdd0XpUBQQUAAEAldvLkSXXt2lXbt2/XokWLtH//fm3evFm9e/fWxIkTy7p5RfbDDz9o9OjRGj9+vA4ePKh169Zpz549evDBB21lnn32Wb3zzjt64403dOjQIU2YMEFDhgzRvn377I7Vtm1bxcbG2h7ff/+9bd+5c+d07tw5vfzyyzpw4IBWrlypzZs3a/z48bYyv/76qwYOHKj+/ftr3759Wrt2rb788kv93//9n/MvRDnFnAoAAIBK7NFHH5XJZNKePXvk7e1t2962bVvdf//9tuenTp3SY489poiICFWrVk39+/fXG2+8ocDAqz12Y8eOVWJiol0vw+TJkxUVFaUdO3ZIknr16qUOHTrIw8ND7777rtzc3DRhwgTNnj1bktSoUSNJ0pAhQyRJISEhOnnyZJHOIzIyUo0aNdLjjz8u6Wr63IcfflgvvfSSrcyHH36oZ555RgMHDpQkPfLII9q2bZsWL16sjz76yFbOxcUl37Ui2rVrp88++8z2vGnTpnrhhRc0atQoZWRkyMXFRWvXrlWHDh303HPPSZKaNWumhQsX6u6779asWbNUs2bNIp1TZUJPBQAAQCWVkJCgzZs3a+LEiXYBhZWvr68kKSsrS4MHD1ZCQoJ27typ8PBwHT9+XMOHDy92ne+//768vb21e/duLVy4UHPnzlV4eLgk6aeffpIkrVixQrGxsbbnJ0+elMlksgUneQkNDdXp06e1adMmGYah8+fP69NPP7UFEJKUlpYmDw8Pu9d5enra9URI0tGjR1WvXj01adJEI0eO1KlTpwo8p4sXL8psNsvFxaXAelJTU7V3794Cj1VZEVQAAABUUseOHZNhGIUuiBcREaH9+/dr9erV6tq1q7p3764PPvhAO3futH3xL6oOHTpo1qxZat68uUaPHq1u3brZ5m5Ys6j5+voqKCjI9tzV1VUtW7aUl5dXvse98cYbtWrVKg0fPlxubm4KCgqSj4+Pli5daisTFhamV155RUePHlVWVpbCw8O1fv16xcbG2sp0797dNqRp2bJlOnHihG6++WYlJSXlWe+ff/6pefPm6aGHHrKrZ9euXfr444+VmZmps2fPau7cuZJkV1dVQlABAABQSRmGUaRyhw8fVnBwsIKDg23b2rRpI19fXx0+fLhYdXbo0MHued26dXXhwoUCX1O/fn39/vvvuv766/Mtc+jQIT3xxBN67rnntHfvXm3evFknT57UhAkTbGVee+01NW/eXK1atZKbm5smTZqkcePGqVq1/33lHTBggO666y516NBBYWFh2rRpkxITE/XJJ5/kqtNisWjQoEFq06aNbQiXJPXr10+LFi3ShAkT5O7urhYtWth6TLLXVZVUzbMGAACoApo3by6TyaTff//d4WNVq1YtV5CSnp6eq5yrq6vdc5PJpKysLIfrnz9/vm688UZNnTrVFhC89dZbWr58ua13wN/fX1988YVSUlIUExOj33//XTVq1FCTJk3yPa6vr69atGihY8eO2W1PSkpS//79VbNmTX3++ee5zmvKlClKTEzUqVOn9Oeff2rw4MGSVGBdlRlBBQAAQCXl5+ensLAwLV26VCkpKbn2JyYmSpJat26t06dP6/Tp07Z9hw4dUmJiotq0aSPp6hf2nEN7oqKiit0mV1dXZWZmFvt1ly5dytULUL16dUm5e2Q8PDxUv359ZWRk6LPPPrN94c9LcnKyoqOjVbduXds2i8Wifv36yc3NTV9++WWu+RNWJpNJ9erVk6enpz7++GMFBwerS5cuxT63yqBcBxXLli1Thw4dZDabZTabFRoaqq+//tq2v1evXrnyDGfvApOuZjIYNGiQvLy8FBAQoKlTp+bKIbxjxw516dJF7u7uatasmVauXFkapwcAAOB0S5cuVWZmpq6//np99tlnOnr0qA4fPqzXX39doaGhkqS+ffuqffv2GjlypH755Rft2bNHo0eP1i233KJu3bpJkm699Vb9/PPP+uCDD3T06FHNmjVLBw4cKHZ7GjVqpIiICMXFxenvv/+WJJ09e1atWrXSnj178n3d7bffrvXr12vZsmU6fvy4fvjhBz3++OO6/vrrVa9ePUnS7t27tX79eh0/flzfffed+vfvr6ysLE2bNs12nKefflo7d+7UyZMntWvXLg0ZMkTVq1fXPffcI+l/AUVKSoree+89WSwWxcXFKS4uzi4YsqbnPXjwoObNm6cFCxbo9ddftwU6VU25DioaNGigBQsWaO/evfr555916623avDgwTp48KCtzIMPPmiXZ3jhwoW2fZmZmRo0aJCuXLmiXbt26f3339fKlStt6b8k6cSJExo0aJB69+6tqKgoTZ48WQ888IC2bNlSqucKAADgDE2aNNEvv/yi3r1766mnnlK7du102223KSIiQsuWLZN09Y77hg0bVKtWLfXs2VN9+/ZVkyZNtHbtWttxwsLCNHPmTE2bNk3XXXedkpKSNHr06GK3Z/HixQoPD1dwcLA6d+4s6eowqiNHjujSpUv5vm7s2LF65ZVX9Oabb6pdu3a666671LJlS61fv95WJjU1Vc8++6zatGmjIUOGqH79+vr+++9tWa4k6cyZM7rnnnvUsmVL3X333apdu7Z+/PFH26TxX375Rbt379b+/fvVrFkz1a1b1/bI3pPz9ddf6+abb1a3bt301VdfacOGDbrzzjuLfT0qC5NR1Bk85YSfn58WLVqk8ePHq1evXurUqZOWLFmSZ9mvv/5a//jHP3Tu3DlbjuW3335b06dPV3x8vNzc3DR9+nR99dVXdpH2iBEjlJiYmGsFxoJYLBb5+PjYUo4BACqm6Ohojbh/ghoNelSSdPKrt7Rm+dtq2rRpGbcMyN/vv/+uUaNG6aOPPio00xPKRln8jErz+2m57qnILjMzU2vWrFFKSoqtq066ulx7nTp11K5dO82YMcMuwo2MjFT79u1tAYV0Ncq2WCy23o7IyEj17dvXrq6wsDBFRkYW2J60tDRZLBa7BwAAAFAVlfsVtffv36/Q0FClpqaqRo0a+vzzz20Thu69916FhISoXr16+u233zR9+nQdOXLE1g0WFxdnF1BIsj2Pi4srsIzFYtHly5fl6emZZ7vmz5+vOXPmlOi5AgAAABVRuQ8qWrZsqaioKF28eFGffvqpxowZo507d6pNmzZ2i5C0b99edevWVZ8+fRQdHe30buoZM2ZoypQptucWi8UutzMAAABQVZT74U9ubm5q1qyZunbtqvnz56tjx4567bXX8izbvXt3SbLlGQ4KCtL58+ftylifBwUFFVjGbDbn20shSe7u7rasVNYHAAAAUBWV+6Aip6ysLKWlpeW5z5or2ZpnODQ0VPv377dbxTE8PFxms9k2hCo0NNS2dHz2MtnnbQAAAADIX7ke/jRjxgwNGDBADRs2VFJSklavXq0dO3Zoy5Ytio6O1urVqzVw4EDVrl1bv/32m5588kn17NnTtjx8v3791KZNG913331auHCh4uLi9Oyzz2rixIlyd3eXJE2YMEFvvvmmpk2bpvvvv1/bt2/XJ598oq+++qosTx0A4GTx8fG2JBtms9mWThIAUHzlOqi4cOGCRo8erdjYWPn4+KhDhw7asmWLbrvtNp0+fVrbtm3TkiVLlJKSouDgYA0bNkzPPvus7fXVq1fXxo0b9cgjjyg0NFTe3t4aM2aM5s6dayvTuHFjffXVV3ryySf12muvqUGDBnr33XcVFhZWFqcMACgF8fHxGjXuASUkXc0Y6FfTSx+teJfAAhXeiRMnyroJyEdl/9mU66Divffey3dfcHCwdu7cWegxQkJCtGnTpgLL9OrVS/v27St2+wAAFZPFYlFC0iX5hw6TJMVHfiaLxUJQgQrL19dXHh4emjlzZlk3BQXw8PCwW4ivMinXQQUAAM7k7Xc1pXh8GbcDcFRQUJA+/fRTJSYmlnVTUABfX19bsqDKhqACAACgEggKCqq0X1hR/hFUAABQgPQrVxQTE2N7zqRuAMiNoAIAgHykJV/UyRPHNflfs21ZA5nUDQC5EVQAAJCP9LTLyjK5qE6PoapdL0QpCeeZ1A0AeSCoAACgEF61/GUOaCCJSd0AkBeCCgBAlZd93kRMTIwy0jPKuEUAULEQVAAAqrSc8yZSL1/SmbOxapieXtZNA4AKg6ACAFCl5Zw3cSH6gGJOL1dmBkEFABRVtbJuAAAA5YF13oSXb52ybgoAVDgEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEEFQAAAAAcQlABAAAAwCEuZd0AAABKQ3x8vCwWiyQpJiZGGekZZdwiAKg8CCoAAJVefHy8Ro17QAlJlyRJqZcv6czZWDVMTy/jlgFA5UBQAQCo9CwWixKSLsk/dJi8/QJ1IfqAYk4vV2YGQQUAlATmVAAAqgxvv0CZAxrIy7dOWTcFACoVggoAAAAADinXQcWyZcvUoUMHmc1mmc1mhYaG6uuvv7btT01N1cSJE1W7dm3VqFFDw4YN0/nz5+2OcerUKQ0aNEheXl4KCAjQ1KlTlZFhPzlvx44d6tKli9zd3dWsWTOtXLmyNE4PAAAAqBTKdVDRoEEDLViwQHv37tXPP/+sW2+9VYMHD9bBgwclSU8++aT++9//at26ddq5c6fOnTunoUOH2l6fmZmpQYMG6cqVK9q1a5fef/99rVy5Us8995ytzIkTJzRo0CD17t1bUVFRmjx5sh544AFt2bKl1M8XAAAAqIjK9UTt22+/3e75Cy+8oGXLlunHH39UgwYN9N5772n16tW69dZbJUkrVqxQ69at9eOPP6pHjx7aunWrDh06pG3btikwMFCdOnXSvHnzNH36dM2ePVtubm56++231bhxYy1evFiS1Lp1a33//fd69dVXFRYWVurnDAAAAFQ05bqnIrvMzEytWbNGKSkpCg0N1d69e5Wenq6+ffvayrRq1UoNGzZUZGSkJCkyMlLt27dXYGCgrUxYWJgsFouttyMyMtLuGNYy1mMAAAAAKFi57qmQpP379ys0NFSpqamqUaOGPv/8c7Vp00ZRUVFyc3OTr6+vXfnAwEDFxcVJkuLi4uwCCut+676CylgsFl2+fFmenp55tistLU1paWm259YFlQAAAICqptz3VLRs2VJRUVHavXu3HnnkEY0ZM0aHDh0q62Zp/vz58vHxsT2Cg4PLukkAAABAmSj3QYWbm5uaNWumrl27av78+erYsaNee+01BQUF6cqVK0pMTLQrf/78eQUFBUmSgoKCcmWDsj4vrIzZbM63l0KSZsyYoYsXL9oep0+fdvRUAQAAgAqp3AcVOWVlZSktLU1du3aVq6urIiIibPuOHDmiU6dOKTQ0VJIUGhqq/fv368KFC7Yy4eHhMpvNatOmja1M9mNYy1iPkR93d3dbqlvrAwAAAKiKyvWcihkzZmjAgAFq2LChkpKStHr1au3YsUNbtmyRj4+Pxo8frylTpsjPz09ms1mPPfaYQkND1aNHD0lSv3791KZNG913331auHCh4uLi9Oyzz2rixIlyd3eXJE2YMEFvvvmmpk2bpvvvv1/bt2/XJ598oq+++qosTx0AAACoMMp1UHHhwgWNHj1asbGx8vHxUYcOHbRlyxbddtttkqRXX31V1apV07Bhw5SWlqawsDC99dZbttdXr15dGzdu1COPPKLQ0FB5e3trzJgxmjt3rq1M48aN9dVXX+nJJ5/Ua6+9pgYNGujdd98lnSwAAABQROU6qHjvvfcK3O/h4aGlS5dq6dKl+ZYJCQnRpk2bCjxOr169tG/fvmtqIwAAAFDVVbg5FQAAAADKF4IKAAAAAA4p18OfAAAob9KvXFFMTIztudlslr+/fxm2CADKHkEFAABFlJZ8USdPHNfkf822ZRH0q+mlj1a8S2ABoEojqAAAoIjS0y4ry+SiOj2Gqna9EKUknFd85GeyWCwEFQCqNIIKAACKyauWv8wBDSRJ8WXcFgAoD5ioDQAAAMAhBBUAAAAAHEJQAQAAAMAhBBUAAAAAHEJQAQAAAMAhBBUAAAAAHEJQAQAAAMAhBBUAAAAAHMLidwAAOCD9yhXFxMRIksxmMytrA6iSCCoAALhGackXdfLEcU3+12y5u7vLr6aXPlrxLoEFgCqH4U8AAFyj9LTLyjK5qE6PofIPHaaEpEuyWCxl3SwAKHX0VAAA4CCvWv7y9PRSfFk3BADKCD0VAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIQQVAAAAABxCUAEAAADAIeU6qJg/f76uu+461axZUwEBAbrzzjt15MgRuzK9evWSyWSye0yYMMGuzKlTpzRo0CB5eXkpICBAU6dOVUZGhl2ZHTt2qEuXLnJ3d1ezZs20cuVKZ58eAAAAUCmU66Bi586dmjhxon788UeFh4crPT1d/fr1U0pKil25Bx98ULGxsbbHwoULbfsyMzM1aNAgXblyRbt27dL777+vlStX6rnnnrOVOXHihAYNGqTevXsrKipKkydP1gMPPKAtW7aU2rkCAAAAFZVLWTegIJs3b7Z7vnLlSgUEBGjv3r3q2bOnbbuXl5eCgoLyPMbWrVt16NAhbdu2TYGBgerUqZPmzZun6dOna/bs2XJzc9Pbb7+txo0ba/HixZKk1q1b6/vvv9err76qsLAw550gAAAAUAmU656KnC5evChJ8vPzs9u+atUq1alTR+3atdOMGTN06dIl277IyEi1b99egYGBtm1hYWGyWCw6ePCgrUzfvn3tjhkWFqbIyMh825KWliaLxWL3AAAAAKqict1TkV1WVpYmT56sG2+8Ue3atbNtv/feexUSEqJ69erpt99+0/Tp03XkyBGtX79ekhQXF2cXUEiyPY+LiyuwjMVi0eXLl+Xp6ZmrPfPnz9ecOXNK9BwBAACAiqjCBBUTJ07UgQMH9P3339ttf+ihh2z/b9++verWras+ffooOjpaTZs2dVp7ZsyYoSlTptieWywWBQcHO60+AAAAoLyqEMOfJk2apI0bN+qbb75RgwYNCizbvXt3SdKxY8ckSUFBQTp//rxdGetz6zyM/MqYzeY8eykkyd3dXWaz2e4BAAAAVEXlOqgwDEOTJk3S559/ru3bt6tx48aFviYqKkqSVLduXUlSaGio9u/frwsXLtjKhIeHy2w2q02bNrYyERERdscJDw9XaGhoCZ0JAAAAUHmV66Bi4sSJ+uijj7R69WrVrFlTcXFxiouL0+XLlyVJ0dHRmjdvnvbu3auTJ0/qyy+/1OjRo9WzZ0916NBBktSvXz+1adNG9913n3799Vdt2bJFzz77rCZOnCh3d3dJ0oQJE3T8+HFNmzZNv//+u9566y198sknevLJJ8vs3AEAAICKolwHFcuWLdPFixfVq1cv1a1b1/ZYu3atJMnNzU3btm1Tv3791KpVKz311FMaNmyY/vvf/9qOUb16dW3cuFHVq1dXaGioRo0apdGjR2vu3Lm2Mo0bN9ZXX32l8PBwdezYUYsXL9a7775LOlkAAACgCMr1RG3DMArcHxwcrJ07dxZ6nJCQEG3atKnAMr169dK+ffuK1T4AQPkWHx8vi8WimJgYZaRnOL2+9CtXFBMTY3tuNpvl7+/v9HoBoKyV66ACAIBrFR8fr1HjHlBC0iWlXr6kM2dj1TA93Wn1pSVf1MkTxzX5X7Ntw2v9anrpoxXvElgAqPQIKgAAlZLFYlFC0iX5hw5Tyt/xijm9XJkZzgsq0tMuK8vkojo9hqp2vRClJJxXfORnslgsBBUAKj2CCgBApebtF1jocNqS5FXLX+aAq+nP40utVgAoW+V6ojYAAACA8o+gAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOMSlrBsAAEBJiY+Pl8VikSTFxMQoIz2jjFsEAFWD04KK48ePq0mTJs46PAAAduLj4zVq3ANKSLokSUq9fElnzsaqYXp6GbcMACo/pw1/atasmXr37q2PPvpIqampzqoGAABJksViUULSJfmHDlOjQY/Kr3N/ZWYZyswgqAAAZ3NaUPHLL7+oQ4cOmjJlioKCgvTwww9rz549zqoOAABJkrdfoMwBDeTlW6esmwIAVYbTgopOnTrptdde07lz57R8+XLFxsbqpptuUrt27fTKK68oPj7eWVUDAAAAKEVOz/7k4uKioUOHat26dXrppZd07NgxPf300woODtbo0aMVGxvr7CYAAAAAcCKnBxU///yzHn30UdWtW1evvPKKnn76aUVHRys8PFznzp3T4MGDnd0EAAAAAE7ktOxPr7zyilasWKEjR45o4MCB+uCDDzRw4EBVq3Y1jmncuLFWrlypRo0aOasJAAAAAEqB04KKZcuW6f7779fYsWNVt27dPMsEBATovffec1YTAAAAAJQCpwUVR48eLbSMm5ubxowZ46wmAAAAACgFTptTsWLFCq1bty7X9nXr1un99993VrUAAAAASpnTgor58+erTp3cOcIDAgL04osvOqtaAAAAAKXMaUHFqVOn1Lhx41zbQ0JCdOrUKWdVCwAAAKCUOS2oCAgI0G+//ZZr+6+//qratWs7q1oAAAAApcxpQcU999yjxx9/XN98840yMzOVmZmp7du364knntCIESOcVS0AAACAUua07E/z5s3TyZMn1adPH7m4XK0mKytLo0ePZk4FAAAAUIk4Lahwc3PT2rVrNW/ePP3666/y9PRU+/btFRIS4qwqAQAAAJQBpwUVVi1atFCLFi2cXQ0AAACAMuK0oCIzM1MrV65URESELly4oKysLLv927dvd1bVAAAAAEqR0yZqP/HEE3riiSeUmZmpdu3aqWPHjnaPopg/f76uu+461axZUwEBAbrzzjt15MgRuzKpqamaOHGiateurRo1amjYsGE6f/68XZlTp05p0KBB8vLyUkBAgKZOnaqMjAy7Mjt27FCXLl3k7u6uZs2aaeXKlQ6dPwAA6VeuKCYmRtHR0YqPjy/r5gCA0zitp2LNmjX65JNPNHDgwGs+xs6dOzVx4kRdd911ysjI0L/+9S/169dPhw4dkre3tyTpySef1FdffaV169bJx8dHkyZN0tChQ/XDDz9IutpjMmjQIAUFBWnXrl2KjY3V6NGj5erqapswfuLECQ0aNEgTJkzQqlWrFBERoQceeEB169ZVWFiY4xcDAFDlpCVf1MkTxzX5X7Pl7u4uv5pe+mjFu/L39y/rpgFAiXPqRO1mzZo5dIzNmzfbPV+5cqUCAgK0d+9e9ezZUxcvXtR7772n1atX69Zbb5UkrVixQq1bt9aPP/6oHj16aOvWrTp06JC2bdumwMBAderUSfPmzdP06dM1e/Zsubm56e2331bjxo21ePFiSVLr1q31/fff69VXXyWoAABck/S0y8oyuahOj6Hy8PBQfORnslgsBBUAKiWnDX966qmn9Nprr8kwjBI75sWLFyVJfn5+kqS9e/cqPT1dffv2tZVp1aqVGjZsqMjISElSZGSk2rdvr8DAQFuZsLAwWSwWHTx40FYm+zGsZazHAADgWnnV8pe3X2DhBQGgAnNaT8X333+vb775Rl9//bXatm0rV1dXu/3r168v1vGysrI0efJk3XjjjWrXrp0kKS4uTm5ubvL19bUrGxgYqLi4OFuZ7AGFdb91X0FlLBaLLl++LE9Pz1ztSUtLU1pamu25xWIp1vkAAAAAlYXTggpfX18NGTKkxI43ceJEHThwQN9//32JHdMR8+fP15w5c8q6GQAAAECZc1pQsWLFihI71qRJk7Rx40Z9++23atCggW17UFCQrly5osTERLveivPnzysoKMhWZs+ePXbHs2aHyl4mZ8ao8+fPy2w259lLIUkzZszQlClTbM8tFouCg4Ov/SQBAACACsppcyokKSMjQ9u2bdM777yjpKQkSdK5c+eUnJxcpNcbhqFJkybp888/1/bt29W4cWO7/V27dpWrq6siIiJs244cOaJTp04pNDRUkhQaGqr9+/frwoULtjLh4eEym81q06aNrUz2Y1jLWI+RF3d3d5nNZrsHAAAAUBU5raciJiZG/fv316lTp5SWlqbbbrtNNWvW1EsvvaS0tDS9/fbbhR5j4sSJWr16tTZs2KCaNWva5kD4+PjI09NTPj4+Gj9+vKZMmSI/Pz+ZzWY99thjCg0NVY8ePSRJ/fr1U5s2bXTfffdp4cKFiouL07PPPquJEyfK3d1dkjRhwgS9+eabmjZtmu6//35t375dn3zyib766itnXR4AAACg0nDq4nfdunXT33//bTeEaMiQIbl6BfKzbNkyXbx4Ub169VLdunVtj7Vr19rKvPrqq/rHP/6hYcOGqWfPngoKCrKbBF69enVt3LhR1atXV2hoqEaNGqXRo0dr7ty5tjKNGzfWV199pfDwcHXs2FGLFy/Wu+++SzpZAAAAoAic1lPx3XffadeuXXJzc7Pb3qhRI509e7ZIxyhKOloPDw8tXbpUS5cuzbdMSEiINm3aVOBxevXqpX379hWpXQAAAAD+x2k9FVlZWcrMzMy1/cyZM6pZs6azqgUAAABQypwWVPTr109LliyxPTeZTEpOTtasWbM0cOBAZ1ULAAAAoJQ5bfjT4sWLFRYWpjZt2ig1NVX33nuvjh49qjp16ujjjz92VrUAAAAASpnTgooGDRro119/1Zo1a/Tbb78pOTlZ48eP18iRI/Nd+wEAAABAxeO0oEKSXFxcNGrUKGdWAQAAAKCMOS2o+OCDDwrcP3r0aGdVDQAAAKAUOS2oeOKJJ+yep6en69KlS3Jzc5OXlxdBBQAAAFBJOC37099//233SE5O1pEjR3TTTTcxURsAAACoRJwWVOSlefPmWrBgQa5eDAAAAAAVV6kGFdLVydvnzp0r7WoBAAAAOInT5lR8+eWXds8Nw1BsbKzefPNN3Xjjjc6qFgAAAEApc1pQceedd9o9N5lM8vf316233qrFixc7q1oAAAAApcxpQUVWVpazDg0AAACgHCn1ORUAAAAAKhen9VRMmTKlyGVfeeUVZzUDAAAAgJM5LajYt2+f9u3bp/T0dLVs2VKS9Mcff6h69erq0qWLrZzJZHJWEwAAAACUAqcFFbfffrtq1qyp999/X7Vq1ZJ0dUG8cePG6eabb9ZTTz3lrKoBAAAAlCKnzalYvHix5s+fbwsoJKlWrVp6/vnnyf4EAAAAVCJOCyosFovi4+NzbY+Pj1dSUpKzqgUAAABQypwWVAwZMkTjxo3T+vXrdebMGZ05c0afffaZxo8fr6FDhzqrWgAAAAClzGlzKt5++209/fTTuvfee5Wenn61MhcXjR8/XosWLXJWtQAAAABKmdOCCi8vL7311ltatGiRoqOjJUlNmzaVt7e3s6oEAAAAUAacvvhdbGysYmNj1bx5c3l7e8swDGdXCQAAAKAUOS2o+Ouvv9SnTx+1aNFCAwcOVGxsrCRp/PjxpJMFAAAAKhGnBRVPPvmkXF1dderUKXl5edm2Dx8+XJs3b3ZWtQCAKiY+Pl7R0dGKiYlRRnpGWTcHAKokp82p2Lp1q7Zs2aIGDRrYbW/evLliYmKcVS0AoAqJj4/XqHEPKCHpklIvX9KZs7Fq+P+TgwAASo/TeipSUlLseiisEhIS5O7u7qxqAQBViMViUULSJfmHDpNf5/7KzDKUmUFQAQClzWlBxc0336wPPvjA9txkMikrK0sLFy5U7969nVUtAKAK8vYLlJdvnbJuRoHSr1xRTEyMoqOjFR0dnecCsQBQUTlt+NPChQvVp08f/fzzz7py5YqmTZumgwcPKiEhQT/88IOzqgUAoNxJS76okyeOa/K/Ztt66/1qeumjFe/K39+/jFsHAI5zWk9Fu3bt9Mcff+imm27S4MGDlZKSoqFDh2rfvn1q2rSps6oFAKDcSU+7rCyTi+r0GKpGgx6Vf+gwJSRdksViKeumAUCJcEpPRXp6uvr376+3335bzzzzjDOqAACgwvGq5S9zwNUEJgx+AlCZOKWnwtXVVb/99pszDg0AAACgnHHa8KdRo0bpvffec9bhAQAAAJQTTpuonZGRoeXLl2vbtm3q2rWrvL297fa/8sorzqoaAAAAQCkq8aDi+PHjatSokQ4cOKAuXbpIkv744w+7MiaTqaSrBQAAAFBGSjyoaN68uWJjY/XNN99IkoYPH67XX39dgYGBJV0VAAAAgHKgxOdUGIZh9/zrr79WSkpKSVcDAAAAoJxw2kRtq5xBBgAAAIDKpcSDCpPJlGvOBHMoAAAAgMqrxOdUGIahsWPHyt3dXZKUmpqqCRMm5Mr+tH79+pKuGgAAAEAZKPGgYsyYMXbPR40aVdJVAABQ4aVfuaKYmBhJktlslr+/fxm3CACuXYkHFStWrCjpQwIAUKmkJV/UyRPHNflfs+Xu7i6/ml76aMW7BBYAKiynT9QGAAD20tMuK8vkojo9hso/dJgSki7JYrGUdbMA4Jo5bUVtAABQMK9a/vL09FJ8WTcEABxU7nsqvv32W91+++2qV6+eTCaTvvjiC7v9Y8eOtWWcsj769+9vVyYhIUEjR46U2WyWr6+vxo8fr+TkZLsyv/32m26++WZ5eHgoODhYCxcudPapAQAAAJVCuQ8qUlJS1LFjRy1dujTfMv3791dsbKzt8fHHH9vtHzlypA4ePKjw8HBt3LhR3377rR566CHbfovFon79+ikkJER79+7VokWLNHv2bP373/922nkBAAAAlUW5H/40YMAADRgwoMAy7u7uCgoKynPf4cOHtXnzZv3000/q1q2bJOmNN97QwIED9fLLL6tevXpatWqVrly5ouXLl8vNzU1t27ZVVFSUXnnlFbvgAwAAAEBu5b6noih27NihgIAAtWzZUo888oj++usv277IyEj5+vraAgpJ6tu3r6pVq6bdu3fbyvTs2VNubm62MmFhYTpy5Ij+/vvvPOtMS0uTxWKxewAAAABVUYUPKvr3768PPvhAEREReumll7Rz504NGDBAmZmZkqS4uDgFBATYvcbFxUV+fn6Ki4uzlQkMDLQrY31uLZPT/Pnz5ePjY3sEBweX9KkBAAAAFUK5H/5UmBEjRtj+3759e3Xo0EFNmzbVjh071KdPH6fVO2PGDE2ZMsX23GKxEFgAAACgSqrwPRU5NWnSRHXq1NGxY8ckSUFBQbpw4YJdmYyMDCUkJNjmYQQFBen8+fN2ZazP85ur4e7uLrPZbPcAAAAAqqJKF1ScOXNGf/31l+rWrStJCg0NVWJiovbu3Wsrs337dmVlZal79+62Mt9++63S09NtZcLDw9WyZUvVqlWrdE8AAAAAqGDKfVCRnJysqKgoRUVFSZJOnDihqKgonTp1SsnJyZo6dap+/PFHnTx5UhERERo8eLCaNWumsLAwSVLr1q3Vv39/Pfjgg9qzZ49++OEHTZo0SSNGjFC9evUkSffee6/c3Nw0fvx4HTx4UGvXrtVrr71mN7wJAAAAQN7KfVDx888/q3PnzurcubMkacqUKercubOee+45Va9eXb/99pvuuOMOtWjRQuPHj1fXrl313Xffyd3d3XaMVatWqVWrVurTp48GDhyom266yW4NCh8fH23dulUnTpxQ165d9dRTT+m5554jnSwAAABQBOV+onavXr1kGEa++7ds2VLoMfz8/LR69eoCy3To0EHfffddsdsHAChd8fHxtjTeMTExykjPKOMWAQDKfVABAIBVfHy8Ro17QAlJlyRJqZcv6czZWDXMNicOAFD6CCoAABWGxWJRQtIl+YcOk7dfoC5EH1DM6eXKzCCoAICyVO7nVAAAkJO3X6DMAQ3k5VunrJsCABBBBQAAAAAHEVQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcIhLWTcAAICqLv3KFcXExNiem81m+fv7l2GLAKB4CCoAAChDackXdfLEcU3+12y5u7tLkvxqeumjFe8SWACoMAgqAAAoQ+lpl5VlclGdHkNVu16IUhLOKz7yM1ksFoIKABUGQQUAAOWAVy1/mQMaSJLiy7gtAFBcTNQGAAAA4BCCCgAAAAAOIagAAAAA4BCCCgAAAAAOIagAAAAA4BCCCgAAAAAOIagAAAAA4BCCCgAAAAAOIagAAAAA4BCCCgAAAAAOIagAAAAA4BCCCgAAAAAOIagAAAAA4BCCCgAAAAAOIagAAAAA4BCXsm4AAACwl37limJiYiRJZrNZ/v7+ZdwiACgYQQUAAOVIWvJFnTxxXJP/NVvu7u7yq+mlj1a8S2ABoFxj+BMAAOVIetplZZlcVKfHUPmHDlNC0iVZLJaybhYAFIieCgAAyiGvWv7y9PRSfFk3BACKgJ4KAAAAAA4hqAAAAADgEIIKAAAAAA4hqAAAAADgEIIKAAAAAA4p90HFt99+q9tvv1316tWTyWTSF198YbffMAw999xzqlu3rjw9PdW3b18dPXrUrkxCQoJGjhwps9ksX19fjR8/XsnJyXZlfvvtN918883y8PBQcHCwFi5c6OxTAwCgUNaF8KKjoxUdHa34ePJBASh/yn1QkZKSoo4dO2rp0qV57l+4cKFef/11vf3229q9e7e8vb0VFham1NRUW5mRI0fq4MGDCg8P18aNG/Xtt9/qoYcesu23WCzq16+fQkJCtHfvXi1atEizZ8/Wv//9b6efHwAA+cm+EN6I+ydoxP0TNGrcAwQWAMqdcr9OxYABAzRgwIA89xmGoSVLlujZZ5/V4MGDJUkffPCBAgMD9cUXX2jEiBE6fPiwNm/erJ9++kndunWTJL3xxhsaOHCgXn75ZdWrV0+rVq3SlStXtHz5crm5ualt27aKiorSK6+8Yhd8AABKX3x8vG3xt5iYGGWkZ5Rxi0pP9oXwatcLUUrCecVHfiaLxcIK2wDKlXIfVBTkxIkTiouLU9++fW3bfHx81L17d0VGRmrEiBGKjIyUr6+vLaCQpL59+6patWravXu3hgwZosjISPXs2VNubm62MmFhYXrppZf0999/q1atWqV6XgCAq+Lj4zVq3ANKSLokSUq9fElnzsaqYXp6GbesdHnV8pc5oIEksRgegHKpQgcVcXFxkqTAwEC77YGBgbZ9cXFxCggIsNvv4uIiPz8/uzKNGzfOdQzrvryCirS0NKWlpdmeW++iAQBKjsViUULSJfmHDpO3X6AuRB9QzOnlysyoWkEFAJR35X5ORXk1f/58+fj42B7BwcFl3SQAqLS8/QJlDmggL986Zd0UAEAeKnRQERQUJEk6f/683fbz58/b9gUFBenChQt2+zMyMpSQkGBXJq9jZK8jpxkzZujixYu2x+nTpx0/IQAAAKACqtBBRePGjRUUFKSIiAjbNovFot27dys0NFSSFBoaqsTERO3du9dWZvv27crKylL37t1tZb799lulZxujGx4erpYtW+Y7n8Ld3V1ms9nuAQCVVXx8PClNy4nsKWb5WQAoL8r9nIrk5GQdO3bM9vzEiROKioqSn5+fGjZsqMmTJ+v5559X8+bN1bhxY82cOVP16tXTnXfeKUlq3bq1+vfvrwcffFBvv/220tPTNWnSJI0YMUL16tWTJN17772aM2eOxo8fr+nTp+vAgQN67bXX9Oqrr5bFKQNAuZJzsrRfTS99tOJdsg+VgewpZt3d3flZACg3yn1Q8fPPP6t3796251OmTJEkjRkzRitXrtS0adOUkpKihx56SImJibrpppu0efNmeXh42F6zatUqTZo0SX369FG1atU0bNgwvf7667b9Pj4+2rp1qyZOnKiuXbuqTp06eu6550gnCwCynywtiZSmZSh7ilkPDw9+FgDKjXIfVPTq1UuGYeS732Qyae7cuZo7d26+Zfz8/LR69eoC6+nQoYO+++67a24nAFR23n5Xs+Ix4KbsedXyl6enFz8LAOVGhZ5TAQAAAKDslfueCgBA+WKdKGxlNpsZfgMAVRxBBQCgyHJOFJaYuA0AIKgAABRD9onCteuFKCXhPJOFyxC9RgDKC4IKAECxedXylzmggSQmbpcVeo0AlCcEFQCAcic+Pl4Wi0UxMTHKSM8o6+aUS/QaAShPCCoAAOVK9sX2Ui9f0pmzsWqYnl7WzSq36DUCUB6QUhYAUK5kX2zPr3N/ZWYZyswgqACA8oyeCgBAueTtF1jg4qcAgPKDngoAAAAADiGoAAAAAOAQggoAAAAADiGoAAAAAOAQJmoDAJzGut6ExGrPAFCZEVQAAJwi+3oTEqs9A0BlRlABAMglew9DcVa1zvm6CwkW1e05/Oq+AlZ7vtb6AADlA0EFAMBOzh6Goq5qne/ravrJzdU139Wer7U+AED5QVABAMi3h8HbL1AXog8o5vTyfFe1Tr9yRTExMZKU/+tcXfOtO/sK2kWpDwBQ/hBUAEAVZQ0k/vrrL01/draS065+ic/ew2AOaKDkv+LyPUZa8kWdPHFck/81W0ZWZpFflxdvv8Breh0AoOwRVABAFZR9yJE1iOg24kn5BjYoVk9BetplZZlcVKfHUGVeTqKHAQCqKIIKAKiCsg85Svk7XjGnl8vdfG09DJLkVctfme7uBZbJPkxKIsUsAFQmBBUAUIV5+wXKMAyn15N9mJT7/w8+rClmAQAVH0EFAMDpsg+Tql0vRCkJ520pZgEAFR9BBQCg1HjV8pc5oIEk5ZtiFgBQ8RBUAABQSWVPFcwcFgDORFABAEAllHNRQescFgILAM5QrawbAAAASl72DF/+ocOUkHSJOSwAnIaeCgAAKonsaXtjYmKUkZ4hb79AScxhAeBcBBUAAFQCOdP22lZGT0+Xm6trWTcPQCVHUAEAKBPZ76pnpGeUcWsqvpxpe+1WRnd1ZfFBAE5FUAEAKHXZ76obWZm2O+pwnDVtb/aV0QtafJDAAkBJIKgAAJS67HfVMy8n/e+OOpyioMUH/f397VLPSvRiACg+ggoAQJnxquWvzP9/5xzOl9figzlTz0r0YgAoPoIKAACqsOypZ739AnP1YgBAURBUAAAAefsF5urFAICiIqgAAKAKIvsWgJJEUAEAlVj2CbhMvoUV2bcAlDSCCgCopHJOwK3hVl0vvTBXtWvXtq22jKqpsOxb2XsxCgpGyRoFwIqgAgAqqewTcNMuJWnvJ6/rgcefzrXaMqquvLJv5VzTIr9MUGSNApAdQQUAVCLZ7xxbeyO8/QJlGEb+qy0D2WTvxfDw8MiVCcr6HouJidGFBIvq9hxO1igABBUAUFnkvHOcV29EXqstA3nxquUvT08vu0xQ2d9jtvdXTT+yRgEgqACAyiLnegP0RqCkZX+PpfwdX+BcDIk5FkBVQlABAJWMdb0BeiPgLNYhddnlnIshMccCqEoIKgAAgMOyz8WoXS8k1xwLMkUBlVu1sm6Ao2bPni2TyWT3aNWqlW1/amqqJk6cqNq1a6tGjRoaNmyYzp8/b3eMU6dOadCgQfLy8lJAQICmTp2qjAxSLQKoGOLj4xUdHU2aWJQL1nk73n6Btm3WuRgj7p9ge9x9733as2ePoqOjFR/PbAygoqsUPRVt27bVtm3bbM9dXP53Wk8++aS++uorrVu3Tj4+Ppo0aZKGDh2qH374QZKUmZmpQYMGKSgoSLt27VJsbKxGjx4tV1dXvfjii6V+LgBQHHlOnCVNLEpQ9nkS1xq45pzvk3DmmF2KY4ZJARVfpQgqXFxcFBQUlGv7xYsX9d5772n16tW69dZbJUkrVqxQ69at9eOPP6pHjx7aunWrDh06pG3btikwMFCdOnXSvHnzNH36dM2ePVtubm6lfToAUGSFTZwFHJFznoSjgWv2+T4Fpa0FUPFU+OFPknT06FHVq1dPTZo00ciRI3Xq1ClJ0t69e5Wenq6+ffvayrZq1UoNGzZUZGSkJCkyMlLt27dXYOD/umnDwsJksVh08ODBfOtMS0uTxWKxewBAWfH2C5SXb52ybgYqmezzJBoNelR+nfsrM8soscDVq5a/3TApABVXhQ8qunfvrpUrV2rz5s1atmyZTpw4oZtvvllJSUmKi4uTm5ubfH197V4TGBiouLirWVHi4uLsAgrrfuu+/MyfP18+Pj62R3BwcMmeGAAA5YR1nkRxA1fr0Cnm+wCVX4Uf/jRgwADb/zt06KDu3bsrJCREn3zyiTw9PZ1W74wZMzRlyhTbc4vFQmABAMD/l33olJGVyXwfoJKr8D0VOfn6+qpFixY6duyYgoKCdOXKFSUmJtqVOX/+vG0ORlBQUK5sUNbnec3TsHJ3d5fZbLZ7AACAq7IPnSrpYVMAyp9KF1QkJycrOjpadevWVdeuXeXq6qqIiAjb/iNHjujUqVMKDQ2VJIWGhmr//v26cOGCrUx4eLjMZrPatGlT6u0HAKAy8arlX+iwKeswqejoaEVHR+vw4cO2/5NuFqgYKvzwp6efflq33367QkJCdO7cOc2aNUvVq1fXPffcIx8fH40fP15TpkyRn5+fzGazHnvsMYWGhqpHjx6SpH79+qlNmza67777tHDhQsXFxenZZ5/VxIkTbSuCAgAA58iZYSr9yhWdPRWjBiGN5eLqQrpZoIKo8EHFmTNndM899+ivv/6Sv7+/brrpJv3444+2D59XX31V1apV07Bhw5SWlqawsDC99dZbttdXr15dGzdu1COPPKLQ0FB5e3trzJgxmjt3blmdEgAAVUbOlbgvRB/Q8ZPLVev6waSbBSqQCh9UrFmzpsD9Hh4eWrp0qZYuXZpvmZCQEG3atKmkmwYAAIrImmEq+a8423NPTy8x+AmoGCp8UAEAACqv7Ct6S9KVK1dsC9OazWZ6MIBygqACAMqh+Ph4u0U1+fKEqqiw+RY13KrrpRfmqnbt2pL4PQHKEkEFAJQz8fHxGjXuASUkXbJtY7IqqqKC5luYstK195PX9cDjT9sSq2QPMggwgNJFUIFrwl1UwHksFosSki7JP3SYvP0ClZJwXud2fqz9+/crJCSE3zdUOXnNt8i8ZLELOBLOHLMLMgjEgdJFUIFiy+suKl3QQMnz9guUOaBBriEg2X/fYmJilJGeUdZNBcpM9oDDGmSQNQoofQQVKLacd1Fz3h2SGKoBFEX2Hr+CAvHsQ0ByDvlIvXxJZ87GqmE6KxUDElmjgLJCUIFrZr2Lmv3uUO16IUpJOM8dIqAQOXv8itL7kNeQjwvRBxRzerkyMwgqAABlh6ACJcbaBS2JO0RAIbL3+KVdSip270POMeYA7OVMRZuzNzB7T2H2NLV5lQVQOIIKAChD3n6BMgyD3gegBOWchyTZD8vN3lOYM01tzrIAioagAkVmvavDxFDAOeh9AEpGzlS0OTOoxcTE6EKCRXV7DlfK3/G2NLUM4QWuHUEFiiT7XR0mhgIAKgJroJ6z58L2d6ymn7wMw66sVPQhvKRXB/6HoAJFkn38d8rf8YUOzcg+lpUPWeCq7F9A6PEDSk9ei+g5OsSQRSoBewQVKBbr+O+CFJRTXyLIQNWU8wsIPX5A6buWIYb59UbktUglw6ZQlRFUoMQVlFNf4k4OqqacX0CYjA2UX9be9r/++kvTn52t5LT//Z5ab5QlJycrIz3Dll5dIvMhqjaCCjhNXjn1uZODqi77+i4Ayp/sve1GVqbOnI1VtxFPyjewgd1ir9Z99DYCVxFUIF8lOf6bNSxQmRW0MjZZ04CKJXtve+blJMWcXi53s1+uxV6t++htBK4iqECenDn+m0ncqAiKmtUl5+9KfrnwmUMBVCxetfyV+f+H7RZnnxWZoVDVEFQgT84a/51zEjfzK1AeFZbVJWcvnjXfvaQCc+FzVxOo3Aqai8HfO1R2BBUoUEmP/87erezh4WH3BUziTg7Kh4KyukjKuxevpp+MtEuF5sIHUDkVNBcj5+J7knTlyhW5ublJ4m8fKgeCCpQJr1r+qpaZbvcFTOJODsqXvLK6FNSLl+mEXPgAKoaC5mLk7KVPv3JFZ0/FqEFIY7m4uvC3D5UCQQXslOak0pyLEZEZCmWtOO//gnrxriUXPoDKIa/5Fnktvnf85HLVun6wPDw8+NuHSoGgoorLPjY8+xjQ0pxUWtqZoQrK1IOqq7BJ1dkTDJDFCcC1yHnDwauWvzw9vXQu2+eLxN8mVEwEFVVYfhmeuo14UleSE8t82EbOzBlFHX9aUMaNgjL1oGrLPqwp56TqvMZKk8UJQEnIOTRK4m8TKiaCikriWlLX5Tc23N3sJxeXsn1r5PzyX9Txp4Vl7cl+zpKYKI5cvP0CZeSYVJ3XWGnmSQAoCYUNBSY1LSoKgopKoLAv0oUpTyv8Zh9iYk3FaQ14so8/zS+LRvYUntasPTlTfGakZ8jbL5C7Qyi2ouSmB4BrkX0o8DlS06ICIqioBPJKf5n9i3RFuauR5xCTmn65xp/mzBqVvRcjI+OK3etyBg7Zx8ozUbzq4Y4fgPKusNS0Bf2dKuqcQT4L4QwEFZWItcehoi4wV9QhJgVl0cj5urzK5jxuaU8UR+my/vHM645fDbfqeumFuapdu3apZDwDgMIUlJpWsv87lV+yFcn+8036X+Dg6OgGID8EFZVQzgXmct7VKM20sdeiqENM8sqikd/ripriM/vwK+7cVHx5ZXSy3vFLOHNMez95XQ88/nSuXiwAKGuF/S0sKNlKVnqq3eeb9L8gIzk5OdcwYXrpURIIKiqxvFLVlVXa2IqgovbwIH95ZXSy3vFL/iuOheoAVFgFJVvJvGSx+3zLfhMl5/BiiV56lAyCikouvzkF5SVtbHmSs4eHzFCVR14ZnaxYqA5ARVbUhTjJYAdnI6io5PKbU1Ae0saWV3lNBpfsx6cSYJStwtYise4rr0P8AOBaObIQZ35DqtJZfA8lgG+VVQR3Y4snZzCWc/x9zglwRV2YD44raJKhpDzHGDPED0Bl4IyFOAtLr36tC9Gi6iGoAAqQV9exKSvdLsDIuTBffhk3rLJ/QGf/cM75POe+oqYHLOxDvqCyxTmOMxSlbXmtRWKdZCgpzzHGdPMDqAycsRBnXunVrcN/a9SoYZdRqrCFaAv6+0YAUvkRVADF4FXLP9cEuOwpbXMGHFLuOz7WO+k5P5yzPzeMLLt9OY+TXc479wV9yBeUcjDnvuJMVL/WYKSgtuV33Wy9D9kmGZ7LMRygPC3oCAAlzRkLcVpvohW0TkZBC9Fm/wzP+fdNKp2/KShbBBXANcozpW2OgCOvFb2td9lT/o63fTjntd5G9n05U/7lnDdgPaakXOWKmnIw+z5XV9cipxgsLKjJWTa/IKKg+vPK4mS9O+eM4QAAUFUVtE5GQQvR5kwEk/NvWPa/hQX1YhTnRhkBR/lCUAE4QV53fOzWQqjpJ6//n42ooPU28lqYL79AoWFNP7m5utrdtc85VKiglIPZ9+VMRSwVPBk6v6Ame9mCggjrHbDs9eeV4jCvLE7OGA4AAFVdYb0hRUkEk9ffwmomU4G9GNlvIkkF3ygj9Xv5QlABOFFRVvQu8rGyfcnPK1DIzEhXWtqlfIOYoqQctCpo4p6Uz2To/x/U5FztNb/F57L/Acqrbek5gqPCspw4YzgAAKBgRUkEk/PmT0G9GNbPe2+/QEn2a2gUFHDkVFCWQDgHQQVQChzNvpXnEJ88AoWSCmLymrhXpMnQrq65goHsw70KCiLyO19WuwaAyiGvnviCPu/dXF3zPI414MivR72gLIF5ZbQqqYCjqg/NIqgAKoDiDvEpqRTCeQ2/ssprMnRRhnsVRUn28AAAyq8CP+/zuFFl7bXOq0fdmnwkOTk5V5bA/DJaZX9d7dq1rznzoiPJTioLggqgAinLIT5FWXCppIMB1lcBgKqhKENxs/diFLSeVM4e/YIyWmV/XV7zPfJLE19QIpTiJDupTAgqABSquBmWCAYAAI4qyo2qvNaTytmjX1hGq/zme+Rc+FbKvzekOMlOKiuCCgCFIsMSAKCsFOdGVUE9+kXdl1egUlhvSFGTnVTmwIKgAkCRkWEJAFDVFKU3JLuCkp1U5qCiWlk3oLxZunSpGjVqJA8PD3Xv3l179uwp6yYBAACgnPCq5S8v3zpFKmcOaGDLVlXZEVRks3btWk2ZMkWzZs3SL7/8oo4dOyosLEwXLlwo66YBAAAA5RZBRTavvPKKHnzwQY0bN05t2rTR22+/LS8vLy1fvrysmwYAAACUW8yp+P+uXLmivXv3asaMGbZt1apVU9++fRUZGZmrfFpamtLS0mzPL168KEl2i6mUlqSkJGVmZCgx9qTSUy/JcuGMjKwsWeJOKzM1yfZ/F5Ocvq806qiobavq9ZfntpV1/eW5bWVdf3luW1nXX57bVtb1l+e2lXX95bltZV2/s9qW8vcFZWZkKCkpqdS/J1rrM4qxVtS1MhmlUUsFcO7cOdWvX1+7du1SaGiobfu0adO0c+dO7d6926787NmzNWfOnNJuJgAAAFAsp0+fVoMGDZxaBz0V12jGjBmaMmWK7XlWVpYSEhJUu3ZtmUymMmxZ+WWxWBQcHKzTp0/LbDaXdXPKPa5X8XC9io5rVTxcr+LhehUP16t4uF5FZ71Whw4dUr169ZxeH0HF/1enTh1Vr15d58+ft9t+/vx5BQUF5Srv7u5uyz1s5evr68wmVhpms5kPgmLgehUP16vouFbFw/UqHq5X8XC9iofrVXT169dXtWrOn0bNRO3/z83NTV27dlVERIRtW1ZWliIiIuyGQwEAAACwR09FNlOmTNGYMWPUrVs3XX/99VqyZIlSUlI0bty4sm4aAAAAUG4RVGQzfPhwxcfH67nnnlNcXJw6deqkzZs3KzCwaixa4mzu7u6aNWtWrmFjyBvXq3i4XkXHtSoerlfxcL2Kh+tVPFyvoivta0X2JwAAAAAOYU4FAAAAAIcQVAAAAABwCEEFAAAAAIcQVAAAAABwCEEFrtmCBQtkMpk0efJkSVJCQoIee+wxtWzZUp6enmrYsKEef/xxXbx40e51JpMp12PNmjV2ZXbs2KEuXbrI3d1dzZo108qVK0vprJwn5/WSpF69euW6FhMmTLB73alTpzRo0CB5eXkpICBAU6dOVUZGhl2ZqnC9Tp48med7x2Qyad26dbbXVZX31+zZs3OdZ6tWrWz7U1NTNXHiRNWuXVs1atTQsGHDci3uWZXeWwVdLz67civs/cVn1/8UdK343Mrb2bNnNWrUKNWuXVuenp5q3769fv75Z9t+wzD03HPPqW7duvL09FTfvn119OhRu2MkJCRo5MiRMpvN8vX11fjx45WcnGxX5rffftPNN98sDw8PBQcHa+HChaVyfiWtoOuVnp6u6dOnq3379vL29la9evU0evRonTt3zu4YjRo1yvUeW7BggV0Zh6+XAVyDPXv2GI0aNTI6dOhgPPHEE4ZhGMb+/fuNoUOHGl9++aVx7NgxIyIiwmjevLkxbNgwu9dKMlasWGHExsbaHpcvX7btP378uOHl5WVMmTLFOHTokPHGG28Y1atXNzZv3lyap1ii8rpehmEYt9xyi/Hggw/aXYuLFy/a9mdkZBjt2rUz+vbta+zbt8/YtGmTUadOHWPGjBm2MlXlemVkZNhdp9jYWGPOnDlGjRo1jKSkJNtrq8r7a9asWUbbtm3tzjM+Pt62f8KECUZwcLARERFh/Pzzz0aPHj2MG264wba/qr23CrpefHblVtj7i8+u/ynoWvG5lVtCQoIREhJijB071ti9e7dx/PhxY8uWLcaxY8dsZRYsWGD4+PgYX3zxhfHrr78ad9xxh9G4cWO7a9K/f3+jY8eOxo8//mh89913RrNmzYx77rnHtv/ixYtGYGCgMXLkSOPAgQPGxx9/bHh6ehrvvPNOqZ6vowq7XomJiUbfvn2NtWvXGr///rsRGRlpXH/99UbXrl3tjhMSEmLMnTvX7j2WnJxs218S14ugAsWWlJRkNG/e3AgPDzduueUWuy/JOX3yySeGm5ubkZ6ebtsmyfj888/zfc20adOMtm3b2m0bPny4ERYW5mjTy0RB16uw67dp0yajWrVqRlxcnG3bsmXLDLPZbKSlpRmGUbWuV06dOnUy7r//frttVeX9NWvWLKNjx4557ktMTDRcXV2NdevW2bYdPnzYkGRERkYahlH13lsFXa+8VPXPrsKuF59d/1Pc91ZV/twyDMOYPn26cdNNN+W7PysrywgKCjIWLVpk25aYmGi4u7sbH3/8sWEYhnHo0CFDkvHTTz/Zynz99deGyWQyzp49axiGYbz11ltGrVq1bO83a90tW7Ys6VNyqsKuV1727NljSDJiYmJs20JCQoxXX30139eUxPVi+BOKbeLEiRo0aJD69u1baNmLFy/KbDbLxcV+ncWJEyeqTp06uv7667V8+XIZ2ZZLiYyMzHXssLAwRUZGlswJlLLCrteqVatUp04dtWvXTjNmzNClS5ds+yIjI9W+fXu7BRjDwsJksVh08OBBW5mqdL2s9u7dq6ioKI0fPz7PY1SF99fRo0dVr149NWnSRCNHjtSpU6ckXb026enpdufZqlUrNWzY0HaeVfG9ld/1ygufXYVfLz67/qeo7y0+t6Qvv/xS3bp101133aWAgAB17txZ//nPf2z7T5w4obi4OLvz9fHxUffu3e0+v3x9fdWtWzdbmb59+6patWravXu3rUzPnj3l5uZmKxMWFqYjR47o77//dvZplpjCrldeLl68KJPJJF9fX7vtCxYsUO3atdW5c2ctWrTIbjhiSVwvVtRGsaxZs0a//PKLfvrpp0LL/vnnn5o3b54eeughu+1z587VrbfeKi8vL23dulWPPvqokpOT9fjjj0uS4uLicq1iHhgYKIvFosuXL8vT07PkTsjJCrte9957r0JCQlSvXj399ttvmj59uo4cOaL169dLyv9aWPcVVKYyXq/s3nvvPbVu3Vo33HCD3faq8v7q3r27Vq5cqZYtWyo2NlZz5szRzTffrAMHDiguLk5ubm65/qAEBgYW+r6x7iuoTEW7VlLB16tmzZp2ZfnsKvx68dn1P8V5b1X1zy1JOn78uJYtW6YpU6boX//6l3766Sc9/vjjcnNz05gxY2zvj7zON/t7JyAgwG6/i4uL/Pz87Mo0btw41zGs+2rVquWU8ytphV2vnFJTUzV9+nTdc889MpvNtu2PP/64unTpIj8/P+3atUszZsxQbGysXnnlFUklc70IKlBkp0+f1hNPPKHw8HB5eHgUWNZisWjQoEFq06aNZs+ebbdv5syZtv937txZKSkpWrRoke3Ds7IoyvXK/qWlffv2qlu3rvr06aPo6Gg1bdq0tJpaLhTn/XX58mWtXr3a7r1kVVXeXwMGDLD9v0OHDurevbtCQkL0ySefVKgvGKWloOuV/a4xn11XFXa9+Oz6n6K+t/jcuiorK0vdunXTiy++KOnq+R44cEBvv/12nl+Sq7riXK/09HTdfffdMgxDy5Yts9s3ZcoU2/87dOggNzc3Pfzww5o/f77c3d1LpK0Mf0KR7d27VxcuXFCXLl3k4uIiFxcX7dy5U6+//rpcXFyUmZkpSUpKSlL//v1Vs2ZNff7553J1dS3wuN27d9eZM2eUlpYmSQoKCsqVpeb8+fMym80V6stSUa9Xdt27d5ckHTt2TFL+18K6r6Aylfl6ffrpp7p06ZJGjx5d6HEr6/srJ19fX7Vo0ULHjh1TUFCQrly5osTERLsy58+fL/R9Y91XUJmKfq0k++tlxWdX/vK6XtlV5c+unPK7VnxuXVW3bl21adPGblvr1q1tQ8as74+8zjf7e+fChQt2+zMyMpSQkFCsz7iKoLDrZWUNKGJiYhQeHm7XS5GX7t27KyMjQydPnpRUMteLoAJF1qdPH+3fv19RUVG2R7du3TRy5EhFRUWpevXqslgs6tevn9zc3PTll18WesdZkqKiolSrVi1bpBwaGqqIiAi7MuHh4QoNDXXKeTlLUa5XTlFRUZKufohIV6/F/v377T48rR8W1g+Zqni93nvvPd1xxx3y9/cv9LiV9f2VU3JysqKjo1W3bl117dpVrq6udud55MgRnTp1ynaeVem9lZfs10sSn12FyHm9cqrKn1055Xet+Ny66sYbb9SRI0fstv3xxx8KCQmRJDVu3FhBQUF252uxWLR79267z6/ExETt3bvXVmb79u3KysqyBbihoaH69ttvlZ6ebisTHh6uli1bVpihT1Lh10v6X0Bx9OhRbdu2TbVr1y70uFFRUapWrZptGFmJXK9iTScHcsieAeTixYtG9+7djfbt2xvHjh2zS1uWkZFhGIZhfPnll8Z//vMfY//+/cbRo0eNt956y/Dy8jKee+452zGtqfOmTp1qHD582Fi6dGmFTZ2XU/brdezYMWPu3LnGzz//bJw4ccLYsGGD0aRJE6Nnz5628ta0jP369TOioqKMzZs3G/7+/nmmZazs18vq6NGjhslkMr7++utc5avS++upp54yduzYYZw4ccL44YcfjL59+xp16tQxLly4YBjG1ZSyDRs2NLZv3278/PPPRmhoqBEaGmp7fVV7bxV0vfjsyq2g68Vnl73CfhcNg8+t7Pbs2WO4uLgYL7zwgnH06FFj1apVhpeXl/HRRx/ZyixYsMDw9fU1NmzYYPz222/G4MGD80wp27lzZ2P37t3G999/bzRv3twupWxiYqIRGBho3HfffcaBAweMNWvWGF5eXhUupWxh1+vKlSvGHXfcYTRo0MCIioqy+/yyZnLatWuX8eqrrxpRUVFGdHS08dFHHxn+/v7G6NGjbfWUxPUiqIBDsn/p++abbwxJeT5OnDhhGMbVlG+dOnUyatSoYXh7exsdO3Y03n77bSMzM9PuuN98843RqVMnw83NzWjSpImxYsWK0j0xJ8l+vU6dOmX07NnT8PPzM9zd3Y1mzZoZU6dOtcv1bhiGcfLkSWPAgAGGp6enUadOHeOpp56yS3NpGFXjelnNmDHDCA4OzvWeMYyq9f4aPny4UbduXcPNzc2oX7++MXz4cLs875cvXzYeffRRo1atWoaXl5cxZMgQIzY21u4YVem9VdD14rMrt4KuF59d9gr7XTQMPrdy+u9//2u0a9fOcHd3N1q1amX8+9//ttuflZVlzJw50wgMDDTc3d2NPn36GEeOHLEr89dffxn33HOPUaNGDcNsNhvjxo2zW/vDMAzj119/NW666SbD3d3dqF+/vrFgwQKnn5szFHS9Tpw4ke/n1zfffGMYhmHs3bvX6N69u+Hj42N4eHgYrVu3Nl588UUjNTXVrh5Hr5fJMLLlLAMAAACAYmJOBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQCg3OjVq5cmT55c1s0AABQTQQUAoETcfvvt6t+/f577vvvuO5lMJv3222+l3CoAQGkgqAAAlIjx48crPDxcZ86cybVvxYoV6tatmzp06FAGLQMAOBtBBQCgRPzjH/+Qv7+/Vq5cabc9OTlZ69at05133ql77rlH9evXl5eXl9q3b6+PP/64wGOaTCZ98cUXdtt8fX3t6jh9+rTuvvtu+fr6ys/PT4MHD9bJkydL5qQAAEVCUAEAKBEuLi4aPXq0Vq5cKcMwbNvXrVunzMxMjRo1Sl27dtVXX32lAwcO6KGHHtJ9992nPXv2XHOd6enpCgsLU82aNfXdd9/phx9+UI0aNdS/f39duXKlJE4LAFAEBBUAgBJz//33Kzo6Wjt37rRtW7FihYYNG6aQkBA9/fTT6tSpk5o0aaLHHntM/fv31yeffHLN9a1du1ZZWVl699131b59e7Vu3VorVqzQqVOntGPHjhI4IwBAURBUAABKTKtWrXTDDTdo+fLlkqRjx47pu+++0/jx45WZmal58+apffv28vPzU40aNbRlyxadOnXqmuv79ddfdezYMdWsWVM1atRQjRo15Ofnp9TUVEVHR5fUaQEACuFS1g0AAFQu48eP12OPPaalS5dqxYoVatq0qW655Ra99NJLeu2117RkyRK1b99e3t7emjx5coHDlEwmk91QKunqkCer5ORkde3aVatWrcr1Wn9//5I7KQBAgQgqAAAl6u6779YTTzyh1atX64MPPtAjjzwik8mkH374QYMHD9aoUaMkSVlZWfrjjz/Upk2bfI/l7++v2NhY2/OjR4/q0qVLtuddunTR2rVrFRAQILPZ7LyTAgAUiOFPAIASVaNGDQ0fPlwzZsxQbGysxo4dK0lq3ry5wsPDtWvXLh0+fFgPP/ywzp8/X+Cxbr31Vr355pvat2+ffv75Z02YMEGurq62/SNHjlSdOnU0ePBgfffddzpx4oR27Nihxx9/PM/UtgAA5yCoAACUuPHjx+vvv/9WWFiY6tWrJ0l69tln1aVLF4WFhalXr14KCgrSnXfeWeBxFi9erODgYN18882699579fTTT8vLy8u238vLS99++60aNmyooUOHqnXr1ho/frxSU1PpuQCAUmQycg5WBQAAAIBioKcCAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKoAKplGjRho7dmxZN6NCysjI0LRp0xQcHKxq1arpzjvvdGp9JpPJ9nj55ZedWheqrk6dOtneZ//4xz+K9JrTp0/Lw8NDP/zwg5NbB0cdOnRILi4uOnDgQFk3BSgQQQVQhlauXCmTyaSff/45z/29evVSu3btHK5n06ZNmj17tsPHqeiWL1+uRYsW6Z///Kfef/99Pfnkk06vc8iQIfrwww81aNCgXPuio6N17733KiAgQJ6enmrevLmeeeYZuzJ79uzRo48+qq5du8rV1VUmk6lI9X7//fe2L5p//vlnrv1r1qxRly5d5OHhIX9/f40fPz7PcjldunRJS5cuVb9+/VS3bl3VrFlTnTt31rJly5SZmWlX9uTJk3aBVfbHmjVrinQeebl48aKmTZum5s2by9PTUyEhIRo/frxOnTplV+7IkSN68skndcMNN8jDw0Mmk0knT57M85ipqamaP3++2rRpIy8vL9WvX1933XWXDh48aFeuV69e+Z6Tq6trkdp/+PBh9e/fXzVq1JCfn5/uu+8+xcfH5yqXlZWlhQsXqnHjxvLw8FCHDh308ccf5yr34osv6sMPP1SdOnWKVL8kzZ07V927d9eNN96Ya9+OHTs0dOhQBQUFyc3NTQEBAbr99tu1fv36Ih/fmS5duqTZs2drx44dDh0nIiJC999/v1q0aCEvLy81adJEDzzwgGJjY+3KFfQ+NplMevDBB21lk5OTNWvWLPXv319+fn4ymUxauXJlrrqzsrK0cuVK3XHHHQoODpa3t7fatWun559/XqmpqXZl27Rpo0GDBum5555z6HwBZ3Mp6wYAKJ4jR46oWrXi3Q/YtGmTli5dWuUDi+3bt6t+/fp69dVXS63ODh06aNSoUbm2R0VFqVevXqpfv76eeuop1a5dW6dOndLp06ftym3atEnvvvuuOnTooCZNmuiPP/4otM6srCw99thj8vb2VkpKSq79y5Yt06OPPqo+ffrolVde0ZkzZ/Taa6/p559/1u7du+Xh4ZHvsY8fP67HHntMffr00ZQpU2Q2m7VlyxY9+uij+vHHH/X+++/nes0999yjgQMH2m0LDQ0t9DzyO7fbbrtNhw4d0qOPPqoWLVro2LFjeuutt7RlyxYdPnxYNWvWlCRFRkbq9ddfV5s2bdS6dWtFRUXle9yRI0fqyy+/1IMPPqguXbro3LlzWrp0qUJDQ7V//36FhIRIkp555hk98MADdq9NSUnRhAkT1K9fv0Lbf+bMGfXs2VM+Pj568cUXlZycrJdffln79+/Xnj175ObmZiv7zDPPaMGCBXrwwQd13XXXacOGDbr33ntlMpk0YsQIWznrtX322WeLdA3j4+P1/vvv5/mzmjVrlubOnavmzZvr4YcfVkhIiP766y9t2rRJw4YN06pVq3TvvfcWqR5nuXTpkubMmSPpapB3raZPn66EhATdddddat68uY4fP64333xTGzduVFRUlIKCgiRJ/v7++vDDD3O9fvPmzVq1apXdz/3PP//U3Llz1bBhQ3Xs2DHfwOfSpUsaN26cevTooQkTJiggIECRkZGaNWuWIiIitH37drsbCBMmTNDAgQMVHR2tpk2bXvM5A05lACgzK1asMCQZP/30U577b7nlFqNt27YO1zNx4kSjLH/dk5OTy6zu7Hr37l0i19MqMzPTuHz5cr77JRmzZs3K83Xt2rUzunfvbly6dKnAOuLi4mxlivpzXLZsmVG7dm3jiSeeMCQZ8fHxtn1paWmGr6+v0bNnTyMrK8u2/b///a8hyXj99dcLPHZ8fLxx4MCBXNvHjRtnSDKOHj1q23bixAlDkrFo0aJC21xUP/zwgyHJePPNN+22L1++3JBkrF+/3rbtr7/+MiwWi2EYhrFo0SJDknHixIlcxzxz5owhyXj66afttm/fvt2QZLzyyisFtunDDz80JBmrVq0qtP2PPPKI4enpacTExNi2hYeHG5KMd955x65Nrq6uxsSJE23bsrKyjJtvvtlo0KCBkZGRkevYISEhxqBBgwptwyuvvGJ4enoaSUlJdtvXrVtnSDL++c9/GleuXMn1us2bNxv//e9/Cz2+s8XHx+f7u1UcO3fuNDIzM3Ntk2Q888wzhb6+T58+htlstvsMSE1NNWJjYw3DMIyffvrJkGSsWLEi12vT0tKMH374Idf2OXPmGJKM8PBwu+1XrlwxatWqZcycObMopwaUCYY/ARVMzjkV6enpmjNnjpo3by4PDw/Vrl1bN910k8LDwyVJY8eO1dKlSyXZj/G3SklJ0VNPPaXg4GC5u7urZcuWevnll2UYhl29ly9f1uOPP646deqoZs2auuOOO3T27FmZTCa7HpDZs2fLZDLp0KFDuvfee1WrVi3ddNNNkqTffvtNY8eOVZMmTeTh4aGgoCDdf//9+uuvv+zqsh7jjz/+0KhRo+Tj4yN/f3/NnDlThmHo9OnTGjx4sMxms4KCgrR48eICr5l1+MI333yjgwcP2q6B9S5iUa+ByWTSpEmTtGrVKrVt21bu7u7avHlz4T+0HLZu3aoDBw5o1qxZ8vT01KVLl3INHbIKDAyUp6dnkY+dkJCgZ599VnPnzpWvr2+u/QcOHFBiYqKGDx9u9z74xz/+oRo1ahQ6LKlOnTpq27Ztru1DhgyRdHVoT15SUlJ05cqVIp9HfiwWi6Sr1yW7unXrSpLdtfLz87P1WhQkKSmpyMfMy+rVq+Xt7a3BgwcXWtdnn32mf/zjH2rYsKFtW9++fdWiRQt98skntm0bNmxQenq6Hn30Uds2k8mkRx55RGfOnFFkZGShdeXniy++UPfu3VWjRg277TNnzpSfn5+WL1+e51CusLAwuzkbFy5c0Pjx4xUYGCgPDw917NgxV+/Hjh077H7XrKy/k9mHBo0dO1Y1atTQ2bNndeedd6pGjRry9/fX008/bfv9OHnypPz9/SVJc+bMsf0uWz+D0tPT9fvvv+cawpSXnj175ur17dmzp/z8/PJ9H1vFxsbqm2++0dChQ+169tzd3W09HAVxc3PTDTfckGt7fr9Hrq6u6tWrlzZs2FDosYGyQlABlAMXL17Un3/+meuRnp5e6Gtnz56tOXPmqHfv3nrzzTf1zDPPqGHDhvrll18kSQ8//LBuu+02SdKHH35oe0iSYRi644479Oqrr6p///565ZVX1LJlS02dOlVTpkyxq2fs2LF64403NHDgQL300kvy9PTMc56A1V133aVLly7pxRdftI05Dg8P1/HjxzVu3Di98cYbGjFihNasWaOBAwfm+gIvScOHD1dWVpYWLFig7t276/nnn9eSJUt02223qX79+nrppZfUrFkzPf300/r222/zbYt1+EKrVq3UoEED2zVo3bp1sa6BdHUI1ZNPPqnhw4frtddeU6NGjQr+AeVh27Ztkq5+AenWrZu8vb3l5eWlESNGKCEhodjHy27mzJkKCgrSww8/nOf+tLQ0SXl/Ufb09NS+ffuUlZVV7Hrj4uIkKc9x/XPmzFGNGjXk4eGh6667Tlu3bi328a2s12vmzJnavn27zp49q507d2ratGm67rrr1Ldv32Ifs2nTpmrQoIEWL16s//73vzpz5oz27NmjCRMmqHHjxnZDjXKKj49XeHi47rzzTnl7exdYz9mzZ3XhwgV169Yt177rr79e+/btsz3ft2+fvL291bp161zlrPuvRXp6un766Sd16dLFbvvRo0f1+++/68477yxSIHb58mX16tVLH374oUaOHKlFixbJx8dHY8eO1WuvvXZNbZOkzMxMhYWFqXbt2nr55Zd1yy23aPHixfr3v/8t6erv8rJlyyT9b77Shx9+qKFDh0q6eo1bt26tGTNmXFP9ycnJSk5OLnR+ypo1a5SVlaWRI0deUz35Kej3qGvXrjpw4IAtsAbKnbLsJgGqOuvwp4IeOYfrhISEGGPGjLE979ixY6FDHvIbNvPFF18Ykoznn3/ebvs///lPw2QyGceOHTMMwzD27t1rSDImT55sV27s2LG5hiHMmjXLkGTcc889uerLa6jPxx9/bEgyvv3221zHeOihh2zbMjIyjAYNGhgmk8lYsGCBbfvff/9teHp62l2T/OQ1nKyo18Awrg5nqlatmnHw4MFC67KWz2uIxh133GFIMmrXrm2MHDnS+PTTT42ZM2caLi4uxg033GA3LCm7woY//frrr0b16tWNLVu2GIbxv+uYffhTfHy8YTKZjPHjx9u99vfff7e95/78888inZ9VWlqa0aZNG6Nx48ZGenq6bXtMTIzRr18/Y9myZcaXX35pLFmyxGjYsKFRrVo1Y+PGjcWqI7uNGzcadevWtfs9CQsLyzWcJ7uChj8ZhmHs3r3baNq0qd0xu3btahvKkp833njDkGRs2rSp0HZbh8N88MEHufZNnTrVkGSkpqYahmEYgwYNMpo0aZKrXEpKiiHJ+L//+79c+4oy/OnYsWOGJOONN96w275hwwZDkvHqq68Weh6GYRhLliwxJBkfffSRbduVK1eM0NBQo0aNGrZhZ998840hyfjmm2/sXm8dGpd9aNCYMWMMScbcuXPtynbu3Nno2rWr7XlBw5+sxy3K50Fe5s2bZ0gyIiIiCizXtWtXo27durmGT2VX0PCn/PTt29cwm83G33//nWvf6tWrDUnG7t27i3w8oDTRUwGUA0uXLlV4eHiuR4cOHQp9ra+vrw4ePKijR48Wu95NmzapevXqevzxx+22P/XUUzIMQ19//bUk2Yb4ZB+KIUmPPfZYvseeMGFCrm3Z746npqbqzz//VI8ePSTJ1rOSXfYJsdWrV1e3bt1kGIbGjx9v2+7r66uWLVvq+PHj+balIEW9Bla33HKL2rRpc011WSUnJ0uSrrvuOn300UcaNmyY5s6dq3nz5mnXrl2KiIi4puM+/vjjGjBgQIEThuvUqaO7775b77//vhYvXqzjx4/ru+++0/Dhw21DXi5fvlyseidNmqRDhw7pzTfflIvL//J/NGzYUFu2bNGECRN0++2364knntC+ffvk7++vp5566prOUbp6t7pz58564YUX9MUXX2j27Nn67rvvNG7cuGs+Zq1atdSpUyf93//9n7744gu9/PLLOnnypO66665c2XiyW716tfz9/W29gQWxXld3d/dc+6xDaKxlLl++XKRyxWUdalirVi277da730XppZCu/t4EBQXpnnvusW1zdXXV448/ruTkZO3cufOa2ifl/uy4+eabi/z73ahRIxmGkWfGpcJ8++23mjNnju6++27deuut+Zb7448/tHfvXo0YMaLYSTMK8uKLL2rbtm1asGBBnkMXrT+zomRpA8oC2Z+AcuD666/Pc0hErVq1Cv0DMnfuXA0ePFgtWrRQu3bt1L9/f913331FCkhiYmJUr169XF8krEMuYmJibP9Wq1ZNjRs3tivXrFmzfI+ds6x0dbz/nDlztGbNGl24cMFu38WLF3OVzz7uXJJ8fHzk4eGRa2iAj49PrnkZRVXUa2CV13kVlzW4yv6FTJLuvfdezZgxQ7t27Sr2MJ61a9dq165dRcpl/8477+jy5ct6+umn9fTTT0uSRo0apaZNm2r9+vW5xtoXZNGiRfrPf/6jefPm5crwlBc/Pz+NGzdOCxYs0JkzZ9SgQYMi1yVdzT7Vu3dvffDBBxo2bJgkafDgwba5Rl9//bUGDBhQrGNevHhRN998s6ZOnWoX7HTr1k29evXSihUr9Mgjj+TZlsjISE2aNMkumMqP9eduHYKWnTVwsZbx9PQsUrlrZeQYbmg2myX9b35JYWJiYtS8efNcX6rz+70pKmuK4+xq1aqlv//++5qOV1S///67hgwZonbt2undd98tsOyqVaskqUSHPq1du1bPPvusxo8fn+d7Tfrfz6yoaaWB0kZPBVDB9ezZU9HR0Vq+fLntD2KXLl0K/cPobHl96bn77rv1n//8RxMmTND69eu1detWWy9IXuP4q1evXqRtUu4vSc7i6Jc5SapXr56k3BODAwICJOmavkBNnTpVd911l9zc3HTy5EmdPHlSiYmJkq4udHbu3DlbWR8fH23YsEExMTHauXOnTp48qQ8//FCxsbHy9/fP8y5pXlauXKnp06drwoQJRU5nKknBwcGSdE3zR1auXKnU1NRci7zdcccdknRNi7l99tlnOn/+vO0YVrfccovMZnO+x1y9erWkon+5tE78zmsScWxsrPz8/Gy9E3Xr1lVcXFyu97X1tdb3UHHVrl1bUu73WKtWrSRJ+/fvv6bj5ie/L8D5JSbI7/fbmU6fPq1+/frJx8dHmzZtKrS3ZvXq1WrZsqW6du1aIvWHh4dr9OjRGjRokN5+++18y1l/ZsVZjwQoTQQVQCVgvfv78ccf6/Tp0+rQoYNdRqb8/rCHhITo3Llzue5O/v7777b91n+zsrJ04sQJu3LHjh0rchv//vtvRURE6P/+7/80Z84cDRkyRLfddpuaNGlS5GM4Q1GvQUmyfhk5e/as3XbrF/+cd2qL4vTp01q9erUaN25se1gnzHbp0iXPXoSGDRuqZ8+eCgkJUWJiovbu3VvkHpINGzbogQce0NChQ23ZxYrKOpTlWs7z/PnzMgwj15dSa1KDjIyMazqmlPuLrrWe/I65evVqNW3a1DaErzD169eXv79/notd7tmzR506dbI979Spky5dupQrC9Du3btt+69Fw4YN5enpmet3uUWLFmrZsqU2bNhgG55XkJCQEB09ejTXzYCcvzfWITvWANfqWnsypJK9U//XX3+pX79+SktL05YtW2yBX352796tY8eOlVgvxe7duzVkyBB169ZNn3zySYE9XidOnFC1atXUokWLEqkbKGkEFUAFl3PYT40aNdSsWTO7oRPWrDQ5/7APHDhQmZmZevPNN+22v/rqqzKZTLZhJGFhYZKkt956y67cG2+8UeR2Wu9A5rzzumTJkiIfwxmKeg1K0uDBg+Xu7q4VK1bYfSmz9i4VZXx+Tp9//nmux/DhwyVJH3zwQaEL/s2YMUMZGRm5Vhn//fffc61U/e2332rEiBHq2bOnVq1ale+48rxWiT579qyWL1+uDh06FPoFLi8tWrSQYRh26Vcl2Vaa7ty58zUdU1KudLpffvmlUlJS8jzmvn37dPjw4QIXgouOjlZ0dLTdtmHDhmnjxo12ixxGRETojz/+0F133WXbNnjwYLm6utr9zhmGobffflv169fPMx1pUbi6uqpbt255BjZz5szRX3/9pQceeCDPQGrr1q3auHGjpKu/N3FxcVq7dq1tf0ZGht544w3VqFFDt9xyi6SrwUX16tVzZWfL+VlSHF5eXpJyf55JxUspm5KSooEDB+rs2bPatGmTmjdvXuhrrL1TJbEA4OHDhzVo0CA1atRIGzduLLQXdO/evWrbtq18fHwcrhtwBuZUABVcmzZt1KtXL3Xt2lV+fn76+eef9emnn2rSpEm2MtY7448//rjCwsJUvXp1jRgxQrfffrt69+6tZ555RidPnlTHjh21detWbdiwQZMnT7at3Nq1a1cNGzZMS5Ys0V9//aUePXpo586dttWdi3Ln0Gw2q2fPnlq4cKHS09NVv359bd26Ndcd09JW1GtQkoKCgvTMM8/oueeeU//+/XXnnXfq119/1X/+8x/dc889uu6662xlY2JibCmArV8En3/+eUlXv7Ddd999kqQ777wzVz3WFaQHDBhgN2RiwYIFOnDggLp37y4XFxd98cUX2rp1q55//nm7uqWrY+RvueUW2zoDMTExuuOOO2QymfTPf/5T69atsyvfoUMH23yeadOmKTo6Wn369FG9evV08uRJvfPOO0pJScmVdnTlypUaN26cVqxYYbcOS05jx47Vyy+/rIcfflj79u1T27Zt9csvv+jdd99V27ZtbXn+patzJayBr3UI05tvvilfX1/5+vrafkduv/12tW3bVnPnzlVMTIx69OihY8eO6c0331TdunXtEgNYFWVcfZ8+fSRdXVvB6l//+pfWrVun3r1764knnlBycrIWLVqk9u3b2000b9CggSZPnqxFixYpPT1d1113nb744gt99913WrVqlUPDhAYPHqxnnnlGFovFNpdCuprCef/+/XrhhRe0b98+3XPPPbYVtTdv3qyIiAjbl+qHHnpI77zzjsaOHau9e/eqUaNG+vTTT/XDDz9oyZIltiFEPj4+uuuuu/TGG2/IZDKpadOm2rhxY645VcXh6empNm3aaO3atWrRooX8/PzUrl07tWvXzpZSdsyYMYVO1h45cqT27Nmj+++/X4cPH7brFapRo0au36nMzEytXbtWPXr0KPBz4c0331RiYqKt59Gapli6mtzCx8dHSUlJCgsL099//62pU6fqq6++sjtG06ZN7VadT09P186dO3MlywDKlTLJOQXAMIxrW1E7Z0rZ559/3rj++usNX19fw9PT02jVqpXxwgsv2K2Im5GRYTz22GOGv7+/YTKZ7NKSJiUlGU8++aRRr149w9XV1WjevLmxaNGiXGlNU1JSjIkTJxp+fn5GjRo1jDvvvNM4cuSIIckuxWteaUytzpw5YwwZMsTw9fU1fHx8jLvuuss4d+5cvmlpcx5jzJgxhre3d5GuU17yK1fUayDJboXjwuQ8r+yysrKMN954w2jRooXh6upqBAcHG88++2yulYytKTnzetxyyy0F1p/fddy4caNx/fXXGzVr1jS8vLyMHj16GJ988km+55C9noLak/N8V69ebfTs2dPw9/c3XFxcjDp16hhDhgwx9u7dm6sea2rWzZs3F3hOhnH1fXT//fcbjRs3Ntzc3Iy6desaDz74YK7ztKYXzesREhJiVzYhIcF48sknjRYtWhju7u5GnTp1jBEjRhjHjx/PVX9mZqZRv359o0uXLgW2MyQkJFc9hmEYBw4cMPr162d4eXkZvr6+xsiRI424uLg863nxxReNkJAQw83NzWjbtq1dCte86ivKitrnz583XFxcjA8//DDP/REREcbgwYONgIAAw8XFxfD39zduv/12Y8OGDbmOM27cOKNOnTqGm5ub0b59+zzTp8bHxxvDhg0zvLy8jFq1ahkPP/ywceDAgTxTyub1+219H2e3a9cuo2vXroabm5vd+644KWVDQkKK/P4wjKsriqsIq84XdFxrSuOC3pt5tf/rr782lGPFeqC8MRlGKc1uBFDpREVFqXPnzvroo49KfBGoysBkMmnq1KmaNm2avL29S2SSd2V199136+TJk9qzZ09ZN6XCSUxMVEZGhrp06aIOHTrYhigVZPz48frjjz/03XfflUIL4ag777xTJpNJn3/+eVk3BcgXcyoAFEleefGXLFmiatWqqWfPnmXQooph0aJF8vf3L/Zk5qrEMAzt2LHDNqwLxdOrVy/5+/vbzdMozKxZs/TTTz9dU7YslK7Dhw9r48aNmjdvXlk3BSgQcyoAFMnChQu1d+9e9e7dWy4uLvr666/19ddf66GHHrKlCIW98PBw2//J2JI/k8nk0Bj7qu6dd96xZS8rakathg0bFrioH8qP1q1bX1NWM6C0MfwJQJGEh4drzpw5OnTokJKTk9WwYUPdd999euaZZ4q08BcAAKi8CCoAAAAAOIQ5FQAAAAAcQlABAAAAwCEMhC4hWVlZOnfunGrWrFmkhcAAAAAAZzIMQ0lJSapXr56qVXNuXwJBRQk5d+4cGXAAAABQ7pw+fVoNGjRwah0EFSWkZs2akq7+0Mxmcxm3BgAAAFWdxWJRcHCw7XuqMxFUlBDrkCez2UxQAQAAgHKjNIbmM1EbAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgEMIKgAAAAA4hKACAAAAgENcyroBAAAAKB1xcXFKTEws62ZUWb6+vgoKCirrZjhFhQ8qli1bpmXLlunkyZOSpLZt2+q5557TgAEDJEmpqal66qmntGbNGqWlpSksLExvvfWWAgMDbcc4deqUHnnkEX3zzTeqUaOGxowZo/nz58vFpcJfHgAAAElXA4p//vOfSk1NLeumVFkeHh769NNPK2VgUeG/NTdo0EALFixQ8+bNZRiG3n//fQ0ePFj79u1T27Zt9eSTT+qrr77SunXr5OPjo0mTJmno0KH64YcfJEmZmZkaNGiQgoKCtGvXLsXGxmr06NFydXXViy++WMZnBwAAUDISExOVmpqqefPmqXHjxmXdnCrnxIkTmjlzphITEwkqyqPbb7/d7vkLL7ygZcuW6ccff1SDBg303nvvafXq1br11lslSStWrFDr1q31448/qkePHtq6dasOHTqkbdu2KTAwUJ06ddK8efM0ffp0zZ49W25ubmVxWgAAAE7RuHFjtWrVqqybgUqmUk3UzszM1Jo1a5SSkqLQ0FDt3btX6enp6tu3r61Mq1at1LBhQ0VGRkqSIiMj1b59e7vhUGFhYbJYLDp48GCpnwMAAABQ0VT4ngpJ2r9/v0JDQ5WamqoaNWro888/V5s2bRQVFSU3Nzf5+vralQ8MDFRcXJykq+MLswcU1v3WfflJS0tTWlqa7bnFYimhs7k28fHxJdIGs9ksf3//EmgRAAAAqopKEVS0bNlSUVFRunjxoj799FONGTNGO3fudGqd8+fP15w5c5xaR1HFx8dr1LgHlJB0yeFj+dX00kcr3iWwAAAAQJFViqDCzc1NzZo1kyR17dpVP/30k1577TUNHz5cV65cUWJiol1vxfnz520TZIKCgrRnzx67450/f962Lz8zZszQlClTbM8tFouCg4NL6pSKxWKxKCHpkvxDh8nbL7DwF+QjJeG84iM/k8ViIagAAABAkVWqORVWWVlZSktLU9euXeXq6qqIiAjbviNHjujUqVMKDQ2VJIWGhmr//v26cOGCrUx4eLjMZrPatGmTbx3u7u4ym812j7Lm7Rcoc0CDa344EpAAAICKLS4uTo899piaNGkid3d3BQcH6/bbb7f7HlVaTCaTvvjii2K/bv369brtttvk7+8vs9ms0NBQbdmyxa5Mo0aNZDKZcj0mTpxoK/Pvf/9bvXr1ktlslslkyrW2x8mTJzV+/Hg1btxYnp6eatq0qWbNmqUrV65cy+lWChW+p2LGjBkaMGCAGjZsqKSkJK1evVo7duzQli1b5OPjo/Hjx2vKlCny8/OT2WzWY489ptDQUPXo0UOS1K9fP7Vp00b33XefFi5cqLi4OD377LOaOHGi3N3dy/jsAAAAnO/kyZO68cYb5evrq0WLFql9+/ZKT0/Xli1bNHHiRP3+++9l3cQi+fbbb3XbbbfpxRdflK+vr1asWKHbb79du3fvVufOnSVJP/30kzIzM22vOXDggG677Tbdddddtm2XLl1S//791b9/f82YMSNXPb///ruysrL0zjvvqFmzZjpw4IAefPBBpaSk6OWXX3b+iZZDFT6ouHDhgkaPHq3Y2Fj5+PioQ4cO2rJli2677TZJ0quvvqpq1app2LBhdovfWVWvXl0bN27UI488otDQUHl7e2vMmDGaO3duWZ0SAABAqXr00UdlMpm0Z88eeXt727a3bdtW999/v+35qVOn9NhjjykiIkLVqlVT//799cYbb9iS3IwdO1aJiYl2vQyTJ09WVFSUduzYIUnq1auXOnToIA8PD7377rtyc3PThAkTNHv2bElXexIkaciQIZKkkJAQ2yLHhVmyZInd8xdffFEbNmzQf//7X1tQkXOI94IFC9S0aVPdcsstdm2WZGtzTtaAw6pJkyY6cuSIli1bRlBRUb333nsF7vfw8NDSpUu1dOnSfMuEhIRo06ZNJd00AACAci8hIUGbN2/WCy+8YBdQWFnnpWZlZWnw4MGqUaOGdu7cqYyMDE2cOFHDhw/P98t3ft5//31NmTJFu3fvVmRkpMaOHasbb7xRt912m3766ScFBARoxYoV6t+/v6pXry7pam9K48aN9c0336hXr15FqicrK0tJSUny8/PLc/+VK1f00UcfacqUKTKZTMU6h5wuXryYbz1VQYUPKgAAAHDtjh07JsMwCl0QLyIiQvv379eJEydsyWk++OADtW3bVj/99JOuu+66ItfZoUMHzZo1S5LUvHlzvfnmm4qIiLDNh5CuBjPZk+a4urqqZcuW8vLyKnI9L7/8spKTk3X33Xfnuf+LL75QYmKixo4dW+Rj5uXYsWN64403qmwvhURQAQAAUKUZhlGkcocPH1ZwcLBdtss2bdrI19dXhw8fLnZQkV3dunXtkubkpX79+sWa27F69WrNmTNHGzZsUEBAQJ5l3nvvPQ0YMED16tUr8nFzOnv2rPr376+77rpLDz744DUfp6IjqAAAAKjCmjdvLpPJVCKTsatVq5YrSElPT89VztXV1e65yWRSVlaWw/VbrVmzRg888IDWrVunvn375lkmJiZG27Zt0/r166+5nnPnzql379664YYb9O9///uaj1MZVMqUsgAAACgaPz8/hYWFaenSpUpJScm135pOtXXr1jp9+rROnz5t23fo0CElJiba0vD7+/srNjbW7vVRUVHFbpOrq6tdhqbi+PjjjzVu3Dh9/PHHGjRoUL7lVqxYoYCAgALLFOTs2bPq1auXunbtqhUrVqhatar9tbpqnz0AAAC0dOlSZWZm6vrrr9dnn32mo0eP6vDhw3r99ddta3v17dtX7du318iRI/XLL79oz549Gj16tG655RZ169ZNknTrrbfq559/1gcffKCjR49q1qxZOnDgQLHb06hRI0VERCguLk5///23pKtf4lu1apVr0eLsVq9erdGjR2vx4sXq3r274uLiFBcXp4sXL9qVy8rK0ooVKzRmzBi5uOQeuBMXF6eoqCgdO3ZMkrR//35FRUUpISHB1pZevXqpYcOGevnllxUfH2+rq6oiqAAAAKjimjRpol9++UW9e/fWU089pXbt2um2225TRESEli1bJunqEKUNGzaoVq1a6tmzp/r27asmTZpo7dq1tuOEhYVp5syZmjZtmq677jolJSVp9OjRxW7P4sWLFR4eruDgYFsq2PT0dB05ckSXLl3K93X//ve/bVmp6tata3s88cQTduW2bdumU6dO2aXLze7tt99W586dbXMkevbsqc6dO+vLL7+UdHWh5GPHjikiIkINGjSwq6uqMhlFnZ2DAlksFvn4+OjixYulvrp2dHS0Rtw/QY0GPSpzQINrPo7lwhmd/OotrVn+tpo2bVqCLQQAAGXt999/16hRo/TRRx8VmukJJa8srn9pfj+lpwIAAACAQwgqAAAAADiEoAIAAACAQwgqAAAAADiEoAIAAACAQwgqAAAAADgk92ofAAAAqLROnDhR1k2okir7dSeoAAAAqAJ8fX3l4eGhmTNnlnVTqiwPDw/5+vqWdTOcgqACdtKvXFFMTIzDxzGbzfL39y+BFgEAgJIQFBSkTz/9VImJiWXdlCrL19dXQUFBZd0MpyCogE1a8kWdPHFck/81W+7u7g4dy6+mlz5a8S6BBQAA5UhQUFCl/VKLskVQAZv0tMvKMrmoTo+hql0v5JqPk5JwXvGRn8lisRBUAAAAVAEEFcjFq5a/zAENHDpGfAm1BQAAAOUfKWUBAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOISgAgAAAIBDCCoAAAAAOMSlrBuAyin9yhXFxMSUyLHMZrP8/f1L5FgAAAAoeQQVKHFpyRd18sRxTf7XbLm7uzt8PL+aXvpoxbsEFgAAAOUUQQVKXHraZWWZXFSnx1DVrhfi0LFSEs4rPvIzWSwWggoAAIByiqACTuNVy1/mgAYOHye+BNoCAAAA52GiNgAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHEFQAAAAAcAhBBQAAAACHVPigYv78+bruuutUs2ZNBQQE6M4779SRI0fsyvTq1Usmk8nuMWHCBLsyp06d0qBBg+Tl5aWAgABNnTpVGRkZpXkqAAAAQIXkUtYNcNTOnTs1ceJEXXfddcrIyNC//vUv9evXT4cOHZK3t7et3IMPPqi5c+fannt5edn+n5mZqUGDBikoKEi7du1SbGysRo8eLVdXV7344oulej4AAABARVPhg4rNmzfbPV+5cqUCAgK0d+9e9ezZ07bdy8tLQUFBeR5j69atOnTokLZt26bAwEB16tRJ8+bN0/Tp0zV79my5ubk59RwAAACAiqzCD3/K6eLFi5IkPz8/u+2rVq1SnTp11K5dO82YMUOXLl2y7YuMjFT79u0VGBho2xYWFiaLxaKDBw+WTsMBAACACqrC91Rkl5WVpcmTJ+vGG29Uu3btbNvvvfdehYSEqF69evrtt980ffp0HTlyROvXr5ckxcXF2QUUkmzP4+Li8qwrLS1NaWlptucWi6WkTwcAAACoECpVUDFx4kQdOHBA33//vd32hx56yPb/9u3bq27duurTp4+io6PVtGnTa6pr/vz5mjNnjkPtBQAAACqDSjP8adKkSdq4caO++eYbNWjQoMCy3bt3lyQdO3ZMkhQUFKTz58/blbE+z28exowZM3Tx4kXb4/Tp046eAgAAAFAhVfigwjAMTZo0SZ9//rm2b9+uxo0bF/qaqKgoSVLdunUlSaGhodq/f78uXLhgKxMeHi6z2aw2bdrkeQx3d3eZzWa7BwAAAFAVVfjhTxMnTtTq1au1YcMG1axZ0zYHwsfHR56enoqOjtbq1av/X3t3Hx1Vda9x/BkICQlhCHknAhERBSTI2xViUSnkkgBX0bBuBUFeTLVaUBBRilUL2hYrFbUWoa0QdCG+cGuttYryIooaUbgCYi1CCAlIElJiMgRIMkn2/cObKUMCJJxJTibz/ayVtZhzNnt+Z6+zknnmnH22xo4dq6ioKO3evVv33nuvrr32WvXv31+SNHr0aPXt21e33nqrnnjiCRUUFOihhx7SzJkzFRISYufhAQAAAC2e31+pWL58uUpLSzVixAh16dLF8/Pqq69KkoKDg7Vx40aNHj1avXv31n333acJEybob3/7m6ePtm3b6q233lLbtm2VnJysKVOmaOrUqV7rWgAAAACon99fqTDGnHN/t27d9MEHH5y3n8TERL399tu+KgsAAAAIGH5/pQIAAACAvQgVAAAAACwhVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAsIVQAAAAAsIRQAQAAAMASQgUAAAAASwgVAAAAACwhVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAsIVQAAAAAsCTI7gKA83FXVio3N9dyP06nUzExMT6oCAAAAKcjVKBFqygr1cGcA5rz4EKFhIRY6iuyY5jWZD5PsAAAAPAxQgVaNHfFKdU4ghQ9LF1RCYkX3M+J4kIVZf1ZLpeLUAEAAOBjhAr4hbDOMXLGdrXUR5GPagEAAIA3JmoDAAAAsIRQAQAAAMASQgUAAAAASwgVAAAAACwhVAAAAACwhKc/ATYpKiqSy+Wy3A+L+gEAALsRKgAbFBUVacqMH6v4+EnLfbGoHwAAsBuhArCBy+VS8fGTikmeoA6RcRfcD4v6AQCAloBQAdioQ2Qci/oBAAC/x0RtAAAAAJYQKgAAAABYQqgAAAAAYAmhAgAAAIAlTNQGGskX60vk5uaqyl3lo4oAAADsRagAGsFX60uUnzqpw9/mq7vb7aPKAAAA7EOoABrBV+tLHM3eo9xDq1RdRagAAAD+j1ABXACr60uUHSvwYTUAAAD2YqI2AAAAAEsIFQAAAAAsIVQAAAAAsIRQAQAAAMASQgUAAAAASwgVAAAAACwhVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALDE70PF4sWL9R//8R/q2LGjYmNjdeONN2rv3r1ebcrLyzVz5kxFRUUpPDxcEyZMUGFhoVebvLw8jRs3TmFhYYqNjdX999+vqqqq5jwUAAAAwC/5faj44IMPNHPmTH366afasGGD3G63Ro8erRMnTnja3Hvvvfrb3/6mdevW6YMPPtCRI0eUnp7u2V9dXa1x48apsrJSn3zyiV544QWtXr1ajzzyiB2HBAAAAPiVILsLsGr9+vVer1evXq3Y2Fjt2LFD1157rUpLS7Vy5UqtXbtWI0eOlCRlZmaqT58++vTTTzVs2DC99957+sc//qGNGzcqLi5OAwYM0GOPPab58+dr4cKFCg4OtuPQAAAAAL/g96HiTKWlpZKkyMhISdKOHTvkdruVkpLiadO7d291795dWVlZGjZsmLKyspSUlKS4uDhPm9TUVN1111366quvNHDgwOY9CDQJd2WlcnNzLfWRm5urKje3xQEAAJyuVYWKmpoazZkzRz/4wQ/Ur18/SVJBQYGCg4MVERHh1TYuLk4FBQWeNqcHitr9tfvqU1FRoYqKCs9rl8vlq8NAE6goK9XBnAOa8+BChYSEXHA/5adO6vC3+erudvuwOgAAAP/WqkLFzJkztWfPHn300UdN/l6LFy/WokWLmvx94BvuilOqcQQpeli6ohISL7ifo9l7lHtolaqrCBUAAAC1Wk2omDVrlt566y19+OGH6tq1q2d7fHy8KisrVVJS4nW1orCwUPHx8Z42n332mVd/tU+Hqm1zpgULFmju3Lme1y6XS926dfPV4aCJhHWOkTO26/kbnkXZsfqvXAEAAAQyv3/6kzFGs2bN0l/+8hdt3rxZPXr08No/ePBgtWvXTps2bfJs27t3r/Ly8pScnCxJSk5O1pdffqmjR4962mzYsEFOp1N9+/at931DQkLkdDq9fgAAAIBA5PdXKmbOnKm1a9fqr3/9qzp27OiZA9GpUyeFhoaqU6dOysjI0Ny5cxUZGSmn06m7775bycnJGjZsmCRp9OjR6tu3r2699VY98cQTKigo0EMPPaSZM2dauv8eAAAACAR+HyqWL18uSRoxYoTX9szMTE2fPl2S9NRTT6lNmzaaMGGCKioqlJqaqueee87Ttm3btnrrrbd01113KTk5WR06dNC0adP06KOPNtdhAAAAAH7L70OFMea8bdq3b69ly5Zp2bJlZ22TmJiot99+25elAQAAAAHB7+dUAAAAALAXoQIAAACAJYQKAAAAAJYQKgAAAABYQqgAAAAAYAmhAgAAAIAlhAoAAAAAlhAqAAAAAFhCqAAAAABgCaECAAAAgCWECgAAAACWECoAAAAAWEKoAAAAAGAJoQIAAACAJYQKAAAAAJYQKgAAAABYQqgAAAAAYImtoeLAgQN2vj0AAAAAH7A1VFx66aX64Q9/qDVr1qi8vNzOUgAAAABcIFtDxf/+7/+qf//+mjt3ruLj4/WTn/xEn332mZ0lAQAAAGgkW0PFgAED9Mwzz+jIkSNatWqV8vPzNXz4cPXr109Lly5VUVGRneUBAAAAaIAWMVE7KChI6enpWrdunX7zm99o//79mjdvnrp166apU6cqPz/f7hIBAAAAnEWLCBXbt2/XT3/6U3Xp0kVLly7VvHnzlJ2drQ0bNujIkSMaP3683SUCAAAAOIsgO9986dKlyszM1N69ezV27Fi9+OKLGjt2rNq0+T7r9OjRQ6tXr9bFF19sZ5kAAAAAzsHWULF8+XLddtttmj59urp06VJvm9jYWK1cubKZKwMAAADQULaGin379p23TXBwsKZNm9YM1QAAAAC4ELbOqcjMzNS6devqbF+3bp1eeOEFGyoCAAAA0Fi2horFixcrOjq6zvbY2Fj9+te/tqEiAAAAAI1la6jIy8tTjx496mxPTExUXl6eDRUBAAAAaCxbQ0VsbKx2795dZ/uuXbsUFRVlQ0UAAAAAGsvWUDFp0iTdc889ev/991VdXa3q6mpt3rxZs2fP1sSJE+0sDQAAAEAD2fr0p8cee0wHDx7UqFGjFBT0fSk1NTWaOnUqcyoAAAAAP2FrqAgODtarr76qxx57TLt27VJoaKiSkpKUmJhoZ1kAAAAAGsHWUFHrsssu02WXXWZ3GQAAAAAugK2horq6WqtXr9amTZt09OhR1dTUeO3fvHmzTZUBAAAAaChbQ8Xs2bO1evVqjRs3Tv369ZPD4bCzHAAAAAAXwNZQ8corr+i1117T2LFj7SwDAAAAgAW2PlI2ODhYl156qZ0lAAAAALDI1lBx33336ZlnnpExxs4yAAAAAFhg6+1PH330kd5//3298847uuKKK9SuXTuv/a+//rpNlQEAAABoKFtDRUREhG666SY7SwD8nruyUrm5uZb7cTqdiomJ8UFFAAAg0NgaKjIzM+18e8DvVZSV6mDOAc15cKFCQkIs9RXZMUxrMp8nWAAAgEazffG7qqoqbdmyRdnZ2brlllvUsWNHHTlyRE6nU+Hh4XaXB7Ro7opTqnEEKXpYuqISLnwl+hPFhSrK+rNcLhehAgAANJqtoSI3N1dpaWnKy8tTRUWF/vM//1MdO3bUb37zG1VUVGjFihV2lgf4jbDOMXLGdrXUR5GPagEAAIHH1qc/zZ49W0OGDNF3332n0NBQz/abbrpJmzZtsrEyAAAAAA1l65WKrVu36pNPPlFwcLDX9osvvljffvutTVUBAAAAaAxbr1TU1NSourq6zvbDhw+rY8eONlQEAAAAoLFsDRWjR4/W008/7XntcDhUVlamX/ziFxo7dqx9hQEAAABoMFtvf3ryySeVmpqqvn37qry8XLfccov27dun6Ohovfzyy3aWBgAAAKCBbA0VXbt21a5du/TKK69o9+7dKisrU0ZGhiZPnuw1cRsAAABAy2X7OhVBQUGaMmWK3WUAAAAAuEC2hooXX3zxnPunTp3aTJUAAAAAuFC2horZs2d7vXa73Tp58qSCg4MVFhZGqAAAAAD8gK1Pf/ruu++8fsrKyrR3714NHz6cidoAAACAn7A1VNSnV69eevzxx+tcxQAAAADQMrW4UCF9P3n7yJEjDW7/4Ycf6vrrr1dCQoIcDofeeOMNr/3Tp0+Xw+Hw+klLS/NqU1xcrMmTJ8vpdCoiIkIZGRkqKyvzxeEAAAAArZqtcyrefPNNr9fGGOXn5+v3v/+9fvCDHzS4nxMnTujKK6/UbbfdpvT09HrbpKWlKTMz0/M6JCTEa//kyZOVn5+vDRs2yO12a8aMGbrjjju0du3aRhwRAAAAEHhsDRU33nij12uHw6GYmBiNHDlSTz75ZIP7GTNmjMaMGXPONiEhIYqPj69339dff63169fr888/15AhQyRJzz77rMaOHavf/va3SkhIaHAtAAAAQKCxNVTU1NQ023tt2bJFsbGx6ty5s0aOHKlf/vKXioqKkiRlZWUpIiLCEygkKSUlRW3atNG2bdt00003NVudAAAAgL+xffG75pCWlqb09HT16NFD2dnZevDBBzVmzBhlZWWpbdu2KigoUGxsrNf/CQoKUmRkpAoKCurts6KiQhUVFZ7XLperSY8BAAAAaKlsDRVz585tcNulS5de8PtMnDjR8++kpCT1799fPXv21JYtWzRq1KgL6nPx4sVatGjRBdcEAAAAtBa2hoovvvhCX3zxhdxuty6//HJJ0jfffKO2bdtq0KBBnnYOh8On73vJJZcoOjpa+/fv16hRoxQfH6+jR496tamqqlJxcfFZ52EsWLDAKxS5XC5169bNp3UCAAAA/sDWUHH99derY8eOeuGFF9S5c2dJ3y+IN2PGDF1zzTW67777muR9Dx8+rGPHjqlLly6SpOTkZJWUlGjHjh0aPHiwJGnz5s2qqanR0KFD6+0jJCSkzhOkAAAAgEBka6h48skn9d5773kChSR17txZv/zlLzV69OgGh4qysjLt37/f8zonJ0c7d+5UZGSkIiMjtWjRIk2YMEHx8fHKzs7WAw88oEsvvVSpqamSpD59+igtLU233367VqxYIbfbrVmzZmnixIk8+QkAAAA4D1sXv3O5XCoqKqqzvaioSMePH29wP9u3b9fAgQM1cOBASd/P1Rg4cKAeeeQRtW3bVrt379YNN9ygyy67TBkZGRo8eLC2bt3qdaXhpZdeUu/evTVq1CiNHTtWw4cP1x//+EfrBwkAAAC0crZeqbjppps0Y8YMPfnkk7rqqqskSdu2bdP9999/1kXs6jNixAgZY866/9133z1vH5GRkSx0BwAAAFwAW0PFihUrNG/ePN1yyy1yu93fFxQUpIyMDC1ZssTO0gAAAAA0kK2hIiwsTM8995yWLFmi7OxsSVLPnj3VoUMHO8sCAAAA0Ai2zqmolZ+fr/z8fPXq1UsdOnQ4561MAAAAAFoWW0PFsWPHNGrUKF122WUaO3as8vPzJUkZGRlN9jhZAAAAAL5la6i499571a5dO+Xl5SksLMyz/eabb9b69ettrAwAAABAQ9k6p+K9997Tu+++q65du3pt79Wrl3Jzc22qCgAAAEBj2Hql4sSJE15XKGoVFxezWjUAAADgJ2wNFddcc41efPFFz2uHw6Gamho98cQT+uEPf2hjZQAAAAAaytbbn5544gmNGjVK27dvV2VlpR544AF99dVXKi4u1scff2xnaQAAAAAayNYrFf369dM333yj4cOHa/z48Tpx4oTS09P1xRdfqGfPnnaWBgAAAKCBbLtS4Xa7lZaWphUrVujnP/+5XWUAAAAAsMi2KxXt2rXT7t277Xp7AAAAAD5i6+1PU6ZM0cqVK+0sAQAAAIBFtk7Urqqq0qpVq7Rx40YNHjxYHTp08Nq/dOlSmyoDAAAA0FC2hIoDBw7o4osv1p49ezRo0CBJ0jfffOPVxuFw2FEaAAAAgEayJVT06tVL+fn5ev/99yVJN998s373u98pLi7OjnIAAAAAWGDLnApjjNfrd955RydOnLCjFAAAAAAW2TpRu9aZIQMAAACA/7AlVDgcjjpzJphDAQAAAPgnW+ZUGGM0ffp0hYSESJLKy8t155131nn60+uvv25HeQAAAAAawZZQMW3aNK/XU6ZMsaMMAAAAAD5gS6jIzMy0420BAAAANIEWMVEbAAAAgP8iVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAssWVFbQAtj7uyUrm5uT7py+l0KiYmxid9AQCAlo9QAUAVZaU6mHNAcx5cqJCQEMv9RXYM05rM5wkWAAAECEIFALkrTqnGEaToYemKSki01NeJ4kIVZf1ZLpeLUAEAQIAgVADwCOscI2dsV8v9FPmgFgAA4D+YqA0AAADAEkIFAAAAAEsIFQAAAAAsIVQAAAAAsIRQAQAAAMASQgUAAAAASwgVAAAAACwhVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAsIVQAAAAAsIRQAQAAAMASQgUAAAAASwgVAAAAACwhVAAAAACwpFWEig8//FDXX3+9EhIS5HA49MYbb3jtN8bokUceUZcuXRQaGqqUlBTt27fPq01xcbEmT54sp9OpiIgIZWRkqKysrBmPAgAAAPBPrSJUnDhxQldeeaWWLVtW7/4nnnhCv/vd77RixQpt27ZNHTp0UGpqqsrLyz1tJk+erK+++kobNmzQW2+9pQ8//FB33HFHcx0CAAAA4LeC7C7AF8aMGaMxY8bUu88Yo6effloPPfSQxo8fL0l68cUXFRcXpzfeeEMTJ07U119/rfXr1+vzzz/XkCFDJEnPPvusxo4dq9/+9rdKSEhotmMBAAAA/E2ruFJxLjk5OSooKFBKSopnW6dOnTR06FBlZWVJkrKyshQREeEJFJKUkpKiNm3aaNu2bc1eMwAAAOBPWsWVinMpKCiQJMXFxXltj4uL8+wrKChQbGys1/6goCBFRkZ62pypoqJCFRUVntcul8uXZQMAAAB+o9VfqWgqixcvVqdOnTw/3bp1s7skAAAAwBatPlTEx8dLkgoLC722FxYWevbFx8fr6NGjXvurqqpUXFzsaXOmBQsWqLS01PNz6NChJqgeAAAAaPlafajo0aOH4uPjtWnTJs82l8ulbdu2KTk5WZKUnJyskpIS7dixw9Nm8+bNqqmp0dChQ+vtNyQkRE6n0+sHAAAACEStYk5FWVmZ9u/f73mdk5OjnTt3KjIyUt27d9ecOXP0y1/+Ur169VKPHj308MMPKyEhQTfeeKMkqU+fPkpLS9Ptt9+uFStWyO12a9asWZo4cSJPfgIAAADOo1WEiu3bt+uHP/yh5/XcuXMlSdOmTdPq1av1wAMP6MSJE7rjjjtUUlKi4cOHa/369Wrfvr3n/7z00kuaNWuWRo0apTZt2mjChAn63e9+1+zHAgAAAPibVhEqRowYIWPMWfc7HA49+uijevTRR8/aJjIyUmvXrm2K8gAAAIBWrdXPqQAAAADQtAgVAAAAACwhVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAsCbK7AACtj7uyUrm5uZb7cTqdiomJ8UFFAACgKREqAPhURVmpDuYc0JwHFyokJMRSX5Edw7Qm83mCBQAALRyhAoBPuStOqcYRpOhh6YpKSLzgfk4UF6oo689yuVyECgAAWjhCBYAmEdY5Rs7Yrpb6KPJRLQAAoGkxURsAAACAJYQKAAAAAJYQKgAAAABYQqgAAAAAYAmhAgAAAIAlhAoAAAAAlhAqAAAAAFhCqAAAAABgCaECAAAAgCWECgAAAACWECoAAAAAWEKoAAAAAGAJoQIAAACAJYQKAAAAAJYQKgAAAABYQqgAAAAAYAmhAgAAAIAlhAoAAAAAlhAqAAAAAFhCqAAAAABgSZDdBQDA2bgrK5Wbm2u5H6fTqZiYGB9UBAAA6kOoANAiVZSV6mDOAc15cKFCQkIs9RXZMUxrMp8nWAAA0EQIFQBaJHfFKdU4ghQ9LF1RCYkX3M+J4kIVZf1ZLpeLUAEAQBMhVABo0cI6x8gZ29VSH0U+qgUAANSPidoAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAsIVQAAAAAsIRQAQAAAMASQgUAAAAASwgVAAAAACwhVAAAAACwhFABAAAAwBJCBQAAAABLCBUAAAAALCFUAAAAALCEUAEAAADAEkIFAAAAAEsIFQAAAAAsCYhQsXDhQjkcDq+f3r17e/aXl5dr5syZioqKUnh4uCZMmKDCwkIbKwYAAAD8R5DdBTSXK664Qhs3bvS8Dgr696Hfe++9+vvf/65169apU6dOmjVrltLT0/Xxxx/bUSoAH3NXVio3N9dyP06nUzExMT6oCACA1iVgQkVQUJDi4+PrbC8tLdXKlSu1du1ajRw5UpKUmZmpPn366NNPP9WwYcOau1QAPlRRVqqDOQc058GFCgkJsdRXZMcwrcl8nmABAMAZAiZU7Nu3TwkJCWrfvr2Sk5O1ePFide/eXTt27JDb7VZKSoqnbe/evdW9e3dlZWURKgA/5644pRpHkKKHpSsqIfGC+zlRXKiirD/L5XIRKgAAOENAhIqhQ4dq9erVuvzyy5Wfn69Fixbpmmuu0Z49e1RQUKDg4GBFRER4/Z+4uDgVFBSctc+KigpVVFR4XrtcrqYqH4APhHWOkTO2q6U+inxUCwAArU1AhIoxY8Z4/t2/f38NHTpUiYmJeu211xQaGnpBfS5evFiLFi3yVYkAAACA3wqIpz+dKSIiQpdddpn279+v+Ph4VVZWqqSkxKtNYWFhvXMwai1YsEClpaWen0OHDjVx1QAAAEDLFJChoqysTNnZ2erSpYsGDx6sdu3aadOmTZ79e/fuVV5enpKTk8/aR0hIiJxOp9cPAAAAEIgC4vanefPm6frrr1diYqKOHDmiX/ziF2rbtq0mTZqkTp06KSMjQ3PnzlVkZKScTqfuvvtuJScnM0kbAAAAaICACBWHDx/WpEmTdOzYMcXExGj48OH69NNPPU9weeqpp9SmTRtNmDBBFRUVSk1N1XPPPWdz1QAAAIB/CIhQ8corr5xzf/v27bVs2TItW7asmSoC4I9YRA8AgPoFRKgAAKtYRA8AgLMjVABAA7CIHgAAZ0eoAIBGYBE9AADqCshHygIAAADwHUIFAAAAAEsIFQAAAAAsIVQAAAAAsIRQAQAAAMASQgUAAAAAS3ikLAA0M1bmBgC0NoQKAGhGrMwNAGiNCBUA0IxYmbt5FRUVyeVyWe6Hq0IAcG6ECgCwAStzN72ioiJNmfFjFR8/abkvrgoBwLkRKgAArZLL5VLx8ZOKSZ6gDpFxF9wPV4UA4PwIFQAAn2mJtxt1iIyzfFXoCJPrAeCcCBUAAJ9orbcbMbkeAM6PUAEA8InWersRk+sB4PwIFQAAn/LF7UYtcRI6k+sB4OwIFQAAn8yFyM3NVZW7ykcVAQD8CaECAAKcr+ZClJ86qcPf5qu72+2jygAA/oJQAQABzldzIY5m71HuoVWqriJUnI3bR0+RkniSFICWhVABAJBkfS5E2bECH1bT+vjyKVIST5IC0LIQKgAAaAa+eoqUxJOkALQ8hAoA8FO+upWGCdbNyxdPkZJ4khSAloVQAQB+yJe30jDB2j/5KlT6am5GS1xNHUDzIVQAgB/y5a00TLD2Py1tle/Wupo6gIYjVACAH/PFrTQtcYK1L76Fb823dbW0Vb5b62rqABqOUAEAaFF89S18INzW1dJW+W6tq6kDOD9CBQCgRfHVt/Dc1gUAzYdQAQBokax+C98Sb+sCgNaqjd0FAAAAAPBvhAoAAAAAlnD7EwAAaDFa2vobOD/WKIFEqAAAAC1ES1t/A+fHGiWoRagAAAAtQktbfwPn1xLXKPHVlROJqyeNQagAACCAtcSFBlvS+ht8QG2YlrJGiS+vnEhcPWkMQgUAAAGqNS806IuwdOzYMc1/aKHKKnxzXHxAbXq+unIiccWrsQgVAAAEqNa60KCvw9KQifcqIs7at/B8QG1evrhyIrHCe2MQKgAACHCtbaFBX4elEGckH1CB8yBUAACAVqm1hSVf88V8EV/Pp4H/IlQAAAAEGF9NaG6J82lgD0IFAABAgPHVhOaWNp8G9iFUAAAABCirE5pb+y1iaLg2dhcAAAAAwL8RKgAAAABYQqgAAAAAYAmhAgAAAIAlTNQGAACA7dyVlcrNzbXUB+tm2IdQAQAAAFtVlJXqYM4BzXlwoUJCQi64H9bNsA+hAgAAoBn44pt4SaqsrFRwcLClPlraN/ruilOqcQQpeli6ohISL7gf1s2wD6ECAACgifnqm3h3ZaW+zctV18QeCmp34R/jWuo3+mGdY1g3w08RKgAAAJqYL7+JP3BwlTpfNZ5v9NGiECoAAACaia++iecbfbQ0hAoAAACgHr6aB+N0OhUTE+ODilouQgUAAABwBl/Ng5GkyI5hWpP5fKsOFoSKMyxbtkxLlixRQUGBrrzySj377LO66qqr7C4LAAAAzchX82BOFBeqKOvPcrlchIpA8eqrr2ru3LlasWKFhg4dqqefflqpqanau3evYmNj7S4PAAAAzczq/BVJKvJRLS1ZG7sLaEmWLl2q22+/XTNmzFDfvn21YsUKhYWFadWqVXaXBgAAALRYhIr/V1lZqR07diglJcWzrU2bNkpJSVFWVpaNlQEAAAAtG7c//b9//etfqq6uVlxcnNf2uLg4/fOf/6zTvqKiQhUVFZ7XpaWlkiSXy9W0hdbj+PHjqq6qUkn+QbnLT15wP66jh2VqauQqOKQgx4XX46t+WmJN9ON/NdGP/9VEP/5XE/34X03003x9nfjuqKqrqnT8+PFm/5xY+37GmCZ/L4dpjnfxA0eOHNFFF12kTz75RMnJyZ7tDzzwgD744ANt27bNq/3ChQu1aNGi5i4TAAAAaJRDhw6pa1dr80LOhysV/y86Olpt27ZVYWGh1/bCwkLFx8fXab9gwQLNnTvX87qmpkbFxcWKioqSw2ExGvsJl8ulbt266dChQ3I6nXaX0+IxXo3DeDUO49U4jFfDMVaNw3g1DuPVOI0dL2OMjh8/roSEhCavjVDx/4KDgzV48GBt2rRJN954o6Tvg8KmTZs0a9asOu1DQkLqPLM4IiKiGSpteZxOJ78IGoHxahzGq3EYr8ZhvBqOsWocxqtxGK/Gacx4derUqYmr+R6h4jRz587VtGnTNGTIEF111VV6+umndeLECc2YMcPu0gAAAIAWi1BxmptvvllFRUV65JFHVFBQoAEDBmj9+vV1Jm8DAAAA+DdCxRlmzZpV7+1OqCskJES/+MUvLC9dHygYr8ZhvBqH8WocxqvhGKvGYbwah/FqnJY8Xjz9CQAAAIAlLH4HAAAAwBJCBQAAAABLCBUAAAAALCFUQN9++62mTJmiqKgohYaGKikpSdu3b5ckud1uzZ8/X0lJSerQoYMSEhI0depUHTlyxKuPiy++WA6Hw+vn8ccf92qze/duXXPNNWrfvr26deumJ554otmO0ZfONV6SNH369DpjkZaW5tVHcXGxJk+eLKfTqYiICGVkZKisrMyrTWsYr/ON1ZnjVPuzZMkST5tAObfqO06Hw6GZM2dKksrLyzVz5kxFRUUpPDxcEyZMqLNYZ15ensaNG6ewsDDFxsbq/vvvV1VVlVebLVu2aNCgQQoJCdGll16q1atXN9ch+tS5xqu4uFh33323Lr/8coWGhqp79+665557VFpa6tVHff//lVde8WoTCOMlSSNGjKiz78477/Tqg/Pr+/E6ePDgWX93rVu3ztNHIJ1f1dXVevjhh9WjRw+FhoaqZ8+eeuyxx3T6tF1jjB555BF16dJFoaGhSklJ0b59+7z6CYS/jecbK7/+3GUQ0IqLi01iYqKZPn262bZtmzlw4IB59913zf79+40xxpSUlJiUlBTz6quvmn/+858mKyvLXHXVVWbw4MFe/SQmJppHH33U5Ofne37Kyso8+0tLS01cXJyZPHmy2bNnj3n55ZdNaGio+cMf/tCsx2vV+cbLGGOmTZtm0tLSvMaiuLjYq5+0tDRz5ZVXmk8//dRs3brVXHrppWbSpEme/a1hvBoyVqePUX5+vlm1apVxOBwmOzvb0yZQzq2jR496HeOGDRuMJPP+++8bY4y58847Tbdu3cymTZvM9u3bzbBhw8zVV1/t+f9VVVWmX79+JiUlxXzxxRfm7bffNtHR0WbBggWeNgcOHDBhYWFm7ty55h//+Id59tlnTdu2bc369eub+3AtO9d4ffnllyY9Pd28+eabZv/+/WbTpk2mV69eZsKECV59SDKZmZle/Zw6dcqzP1DGyxhjrrvuOnP77bd7tSktLfX8f86vf49XVVVVnd9dixYtMuHh4eb48eOePgLp/PrVr35loqKizFtvvWVycnLMunXrTHh4uHnmmWc8bR5//HHTqVMn88Ybb5hdu3aZG264wfTo0cNrTALhb+P5xsqfP3cRKgLc/PnzzfDhwxv1fz777DMjyeTm5nq2JSYmmqeeeuqs/+e5554znTt3NhUVFV7vffnllze6Zjs1ZLymTZtmxo8ff9b9//jHP4wk8/nnn3u2vfPOO8bhcJhvv/3WGNM6xutCzq3x48ebkSNHem0LlHPrTLNnzzY9e/Y0NTU1pqSkxLRr186sW7fOs//rr782kkxWVpYxxpi3337btGnTxhQUFHjaLF++3DidTs/YPPDAA+aKK67wep+bb77ZpKamNsMRNa3Tx6s+r732mgkODjZut9uzTZL5y1/+ctY+A2m8rrvuOjN79uyztuf8Ovf5NWDAAHPbbbd5bQuk82vcuHF1jj89Pd1MnjzZGGNMTU2NiY+PN0uWLPHsLykpMSEhIebll182xgTO38bzjVV9/OVzF7c/Bbg333xTQ4YM0X//938rNjZWAwcO1J/+9Kdz/p/S0lI5HA5FRER4bX/88ccVFRWlgQMHasmSJV6XxbOysnTttdcqODjYsy01NVV79+7Vd99959NjakoNHa8tW7YoNjZWl19+ue666y4dO3bMsy8rK0sREREaMmSIZ1tKSoratGmjbdu2edr4+3g19twqLCzU3//+d2VkZNTZFwjn1ukqKyu1Zs0a3XbbbXI4HNqxY4fcbrdSUlI8bXr37q3u3bsrKytL0vfjkJSU5LVYZ2pqqlwul7766itPm9P7qG1T24e/OnO86lNaWiqn06mgIO/lmWbOnKno6GhdddVVWrVqldftGoE2Xi+99JKio6PVr18/LViwQCdPnvTs4/w6+/m1Y8cO7dy5s97fXYFyfl199dXatGmTvvnmG0nSrl279NFHH2nMmDGSpJycHBUUFHgdb6dOnTR06FCv32GB8LfxfGNVH3/53MXidwHuwIEDWr58uebOnasHH3xQn3/+ue655x4FBwdr2rRpddqXl5dr/vz5mjRpkpxOp2f7Pffco0GDBikyMlKffPKJFixYoPz8fC1dulSSVFBQoB49enj1VfvHqaCgQJ07d27Co/SdhoxXWlqa0tPT1aNHD2VnZ+vBBx/UmDFjlJWVpbZt26qgoECxsbFe/QYFBSkyMlIFBQWSWsd4NfbceuGFF9SxY0elp6d7bQ+Uc+t0b7zxhkpKSjR9+nRJ3x9HcHBwnT8ocXFxXufM6R/4avfX7jtXG5fLpVOnTik0NLQJjqbpnTleZ/rXv/6lxx57THfccYfX9kcffVQjR45UWFiY3nvvPf30pz9VWVmZ7rnnHkmBNV633HKLEhMTlZCQoN27d2v+/Pnau3evXn/9dUmcX+c6v1auXKk+ffro6quv9toeSOfXz372M7lcLvXu3Vtt27ZVdXW1fvWrX2ny5MmS/n2O1He8p58/gfC38XxjdSZ/+txFqAhwNTU1GjJkiH79619LkgYOHKg9e/ZoxYoVdT74ud1u/ehHP5IxRsuXL/faN3fuXM+/+/fvr+DgYP3kJz/R4sWLW+SqjxeqIeM1ceJET/ukpCT1799fPXv21JYtWzRq1Chb6rZDY84tSVq1apUmT56s9u3be20PlHPrdCtXrtSYMWOUkJBgdyl+4Vzj5XK5NG7cOPXt21cLFy702vfwww97/j1w4ECdOHFCS5Ys8Xzoa63qG6/TA1dSUpK6dOmiUaNGKTs7Wz179rSjzBbjXOfXqVOntHbtWq9zqVYgnV+vvfaaXnrpJa1du1ZXXHGFdu7cqTlz5ighIaHe3/eBrDFj5W+fu7j9KcB16dJFffv29drWp08f5eXleW2rPbFzc3O1YcMGr7Rcn6FDh6qqqkoHDx6UJMXHx9d5Uk3t6/j4eItH0XwaOl6nu+SSSxQdHa39+/dL+v54jx496tWmqqpKxcXFnrFoDePVmLHaunWr9u7dqx//+Mfn7be1nlu1cnNztXHjRq+xiI+PV2VlpUpKSrzaFhYWNuqcOVsbp9PpV9+Knq6+8ap1/PhxpaWlqWPHjvrLX/6idu3anbOvoUOH6vDhw6qoqJAUeON1uqFDh0qS1+8tzq+6/ud//kcnT57U1KlTz9tXaz6/7r//fv3sZz/TxIkTlZSUpFtvvVX33nuvFi9eLOnf50h9x3v6+RMIfxvPN1a1/PFzF6EiwP3gBz/Q3r17vbZ98803SkxM9LyuPbH37dunjRs3Kioq6rz97ty5U23atPFcykxOTtaHH34ot9vtabNhwwZdfvnlfnG5slZDxutMhw8f1rFjx9SlSxdJ349FSUmJduzY4WmzefNm1dTUeP6Qt4bxasxYrVy5UoMHD9aVV1553n5b67lVKzMzU7GxsRo3bpxn2+DBg9WuXTtt2rTJs23v3r3Ky8tTcnKypO/H4csvv/T6o1z7h6g23CUnJ3v1Udumtg9/VN94Sd9foRg9erSCg4P15ptv1rkCVp+dO3eqc+fOnm/5Amm8zrRz505J8vq9xflV18qVK3XDDTcoJibmvH215vPr5MmTatPG+yNl27ZtVVNTI0nq0aOH4uPjvY7X5XJp27ZtXr/DAuFv4/nGSvLjz11NNgUcfuGzzz4zQUFB5le/+pXZt2+feemll0xYWJhZs2aNMcaYyspKc8MNN5iuXbuanTt3ej26rPaJAp988ol56qmnzM6dO012drZZs2aNiYmJMVOnTvW8T0lJiYmLizO33nqr2bNnj3nllVdMWFiYXz0Gzpjzj9fx48fNvHnzTFZWlsnJyTEbN240gwYNMr169TLl5eWeftLS0szAgQPNtm3bzEcffWR69erl9di81jBe5xurWqWlpSYsLMwsX768Th+BdG4ZY0x1dbXp3r27mT9/fp19d955p+nevbvZvHmz2b59u0lOTjbJycme/bWP/Bw9erTZuXOnWb9+vYmJian3kZ/333+/+frrr82yZcv89hGWxpx9vEpLS83QoUNNUlKS2b9/v9fvraqqKmOMMW+++ab505/+ZL788kuzb98+89xzz5mwsDDzyCOPePoJlPHav3+/efTRR8327dtNTk6O+etf/2ouueQSc+2113racH7VtW/fPuNwOMw777xTZ1+gnV/Tpk0zF110kecxqa+//rqJjo42DzzwgKfN448/biIiIsxf//pXs3v3bjN+/Ph6Hynb2v82nm+s/PlzF6EC5m9/+5vp16+fCQkJMb179zZ//OMfPftycnKMpHp/ap9vvmPHDjN06FDTqVMn0759e9OnTx/z61//2utDtDHG7Nq1ywwfPtyEhISYiy66yDz++OPNeZg+c67xOnnypBk9erSJiYkx7dq1M4mJieb222/3egyjMcYcO3bMTJo0yYSHhxun02lmzJjh9XxzY1rHeJ1rrGr94Q9/MKGhoaakpKTOvkA7t959910jyezdu7fOvlOnTpmf/vSnpnPnziYsLMzcdNNNJj8/36vNwYMHzZgxY0xoaKiJjo429913n9cjVI0x5v333zcDBgwwwcHB5pJLLjGZmZlNeUhN6mzj9f7775/191ZOTo4x5vtHVQ4YMMCEh4ebDh06mCuvvNKsWLHCVFdX1+mrtY9XXl6eufbaa01kZKQJCQkxl156qbn//vu91qkwhvPrTAsWLDDdunWrc84YE3jnl8vlMrNnzzbdu3c37du3N5dccon5+c9/7vU405qaGvPwww+buLg4ExISYkaNGlVnbAPhb+P5xsqfP3c5jDnt+WYAAAAA0EjMqQAAAABgCaECAAAAgCWECgAAAACWECoAAAAAWEKoAAAAAGAJoQIAAACAJYQKAAAAAJYQKgAAAABYQqgAALQYI0aM0Jw5c+wuAwDQSIQKAIBPXH/99UpLS6t339atW+VwOLR79+5mrgoA0BwIFQAAn8jIyNCGDRt0+PDhOvsyMzM1ZMgQ9e/f34bKAABNjVABAPCJ//qv/1JMTIxWr17ttb2srEzr1q3TjTfeqEmTJumiiy5SWFiYkpKS9PLLL5+zT4fDoTfeeMNrW0REhNd7HDp0SD/60Y8UERGhyMhIjR8/XgcPHvTNQQEAGoRQAQDwiaCgIE2dOlWrV6+WMcazfd26daqurtaUKVM0ePBg/f3vf9eePXt0xx136NZbb9Vnn312we/pdruVmpqqjh07auvWrfr4448VHh6utLQ0VVZW+uKwAAANQKgAAPjMbbfdpuzsbH3wwQeebZmZmZowYYISExM1b948DRgwQJdcconuvvtupaWl6bXXXrvg93v11VdVU1Oj559/XklJSerTp48yMzOVl5enLVu2+OCIAAANQagAAPhM7969dfXVV2vVqlWSpP3792vr1q3KyMhQdXW1HnvsMSUlJSkyMlLh4eF69913lZeXd8Hvt2vXLu3fv18dO3ZUeHi4wsPDFRkZqfLycmVnZ/vqsAAA5xFkdwEAgNYlIyNDd999t5YtW6bMzEz17NlT1113nX7zm9/omWee0dNPP62kpCR16NBBc+bMOedtSg6Hw+tWKun7W55qlZWVafDgwXrppZfq/N+YmBjfHRQA4JwIFQAAn/rRj36k2bNna+3atXrxxRd11113yeFw6OOPP9b48eM1ZcoUSVJNTY2++eYb9e3b96x9xcTEKD8/3/N63759OnnypOf1oEGD9Oqrryo2NlZOp7PpDgoAcE7c/gQA8Knw8HDdfPPNWrBggfLz8zV9+nRJUq9evbRhwwZ98skn+vrrr/WTn/xEhYWF5+xr5MiR+v3vf68vvvhC27dv15133ql27dp59k+ePFnR0dEaP368tm7dqpycHG3ZskX33HNPvY+2BQA0DUIFAMDnMjIy9N133yk1NVUJCQmSpIceekiDBg1SamqqRowYofj4eN14443n7OfJJ59Ut27ddM011+iWW27RvHnzFBYW5tkfFhamDz/8UN27d1d6err69OmjjIwMlZeXc+UCAJqRw5x5syoAAAAANAJXKgAAAABYQqgAAAAAYAmhAgAAAIAlhAoAAAAAlhAqAAAAAFhCqAAAAABgCaECAAAAgCWECgAAAACWECoAAAAAWEKoAAAAAGAJoQIAAACAJYQKAAAAAJb8H0CEawa9VV1eAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Interval Data Count Summary:\n", - "[36.00, 2073.75): 5223320\n", - "[2073.75, 4111.50): 205172\n", - "[4111.50, 6149.25): 89529\n", - "[6149.25, 8187.00]: 2712\n" - ] - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "# ---------------------- 1. 读取数据 ----------------------\n", - "values = []\n", - "with open(\"/workspace/data4packing/RiceVL/data_procs/token_info_v2_vqa_pretrain_5M_8k.txt\", 'r', encoding='utf-8') as f:\n", - " for line in f:\n", - " try:\n", - " num = int(line.split(':')[-1].strip())\n", - " values.append(num)\n", - " except:\n", - " continue # 避免异常数据影响，如有需可细化处理\n", - "\n", - "if not values:\n", - " raise ValueError(\"No valid numeric values found in tl.txt\")\n", - "\n", - "# ---------------------- 2. 计算关键点 ----------------------\n", - "min_val = min(values)\n", - "max_val = max(values)\n", - "# 等距划分三个中间点\n", - "a = min_val + (max_val - min_val) / 4\n", - "b = min_val + 2 * (max_val - min_val) / 4\n", - "c = min_val + 3 * (max_val - min_val) / 4\n", - "\n", - "# 区间划分（转为元组方便后续处理）\n", - "bins = [min_val, a, b, c, max_val]\n", - "labels = [\n", - " f\"[{min_val:.2f}, {a:.2f})\",\n", - " f\"[{a:.2f}, {b:.2f})\",\n", - " f\"[{b:.2f}, {c:.2f})\",\n", - " f\"[{c:.2f}, {max_val:.2f}]\"\n", - "]\n", - "\n", - "# 按区间分组\n", - "grouped_data = {label: [] for label in labels}\n", - "for num in values:\n", - " for i in range(len(bins) - 1):\n", - " if bins[i] <= num < bins[i + 1]:\n", - " grouped_data[labels[i]].append(num)\n", - " break\n", - " # 处理刚好等于 max_val 的情况（放入最后一组）\n", - " else:\n", - " if num == max_val:\n", - " grouped_data[labels[-1]].append(num)\n", - "\n", - "# ---------------------- 3. 绘制每个区间的直方图 ----------------------\n", - "plt.rcdefaults() # 重置为默认配置，避免字体等问题\n", - "fig, axes = plt.subplots(nrows=len(labels), ncols=1, figsize=(8, 5 * len(labels)))\n", - "\n", - "for i, (label, data) in enumerate(grouped_data.items()):\n", - " ax = axes[i] if len(labels) > 1 else axes\n", - " # 绘制直方图\n", - " ax.hist(data, bins='auto', edgecolor='black', alpha=0.7)\n", - " # 设置标题和标签\n", - " ax.set_title(f\"Histogram for {label} (Count: {len(data)})\")\n", - " ax.set_xlabel(\"Value\")\n", - " ax.set_ylabel(\"Frequency\")\n", - " # 添加数据量文本\n", - " ax.text(0.95, 0.95, f\"Count: {len(data)}\", \n", - " horizontalalignment='right', verticalalignment='top', \n", - " transform=ax.transAxes, fontsize=10, bbox=dict(facecolor='white', alpha=0.8))\n", - "\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "# 额外打印各区间数据量汇总\n", - "print(\"Interval Data Count Summary:\")\n", - "for label, data in grouped_data.items():\n", - " print(f\"{label}: {len(data)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "bc3bf8be-4e09-4b94-9f75-235d73ea7476", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxYAAAfGCAYAAACKzyibAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQABAABJREFUeJzs3XlcVFX/B/DPALIpw6ICmhuWO66YhuVOYvKUppUbLqj1WFgqpmaZuTxlapo+aVqPipVaZj/b1DTSNE3TNMmdSlFcQMdBGIZttvP7w+Y2MwzrDMwMfN6vFy/l3jN3zjDb/d5zzvcrE0IIEBERERER2cDN0R0gIiIiIiLXx8CCiIiIiIhsxsCCiIiIiIhsxsCCiIiIiIhsxsCCiIiIiIhsxsCCiIiIiIhsxsCCiIiIiIhsxsCCiIiIiIhsxsCCiIiIiIhsxsCCqJpq1qwZxo8f7+huVHvLli1D8+bN4e7ujk6dOhXbrlmzZpDJZJDJZJgyZUrVdbCaOH/+PDw8PHD27Nly3W7p0qVo3bo1DAZDJfWMyDEq+p4gqkwMLIhcwKZNmyCTyXDixAmr+/v06YPw8HCb72f37t2YP3++zcepKb7//nvMmjULDz/8MBITE/HWW2+V2L5nz5745JNPMG7cOGlbfn4+Jk6ciPDwcPj7+6NOnTro2LEjVq1aBa1Wa/U4P/zwA/r16wd/f3/4+fkhIiIC27Ztq9BjUKvVeOONNzBw4EAEBQVBJpNh06ZNxba/cOECBg4ciDp16iAoKAhjxoyBQqEwa3Px4kXMmjULnTp1gp+fHxo0aICYmJhiX7+mHn30UavBV9u2bRETE4N58+aV+bGpVCosWbIEs2fPhpvbP193xgBPJpPBw8MDQUFBiIiIwNSpU3H+/PkyH99SXl4e5s+fjwMHDlT4GCVRqVRYsGABOnbsiDp16sDHxwfh4eGYPXs2bt68WSn3WV6V8Rly6dIleHt7W/0MTE9PxyuvvIK+ffvCz88PMpnM6t8/Ly8Pa9aswYABA9CgQQP4+fmhc+fOWLt2LfR6vVnb+fPnm71GLH9+/vlnq/3UarVo27YtZDIZ3nnnnSL7DQYDli5dirCwMHh7e6NDhw749NNPi7T73//+h969eyMkJAReXl4ICwtDXFwcrly5YtauIu8Josrm4egOEFHlSElJMTuZKovdu3djzZo1DC7KaP/+/XBzc8OGDRvg6elZavvmzZsjNjbWbFt+fj7OnTuHQYMGoVmzZnBzc8ORI0cwffp0HDt2DFu3bjVrn5iYiIkTJ+LRRx/FW2+9BXd3d6SkpODatWsVegx37tzBwoUL0aRJE3Ts2LHEk+Lr16+jV69e8Pf3x1tvvQW1Wo133nkHZ86cwfHjx6W/wfr167FhwwYMGzYML7zwArKzs/HBBx/goYcewp49exAVFWX1+Dt27MDRo0eLvf/Jkydj0KBBuHTpEu6///5SH9vGjRuh0+kwcuTIIvseffRRjB07FkIIZGdn4/fff8dHH32E999/H0uWLEFCQkKpx7eUl5eHBQsWALgX7NvT5cuXERUVhbS0NDz99NN47rnn4OnpidOnT2PDhg348ssv8ccff9j1PiuiMj5Dpk+fDg8PDxQWFhbZl5KSgiVLlqBFixZo3759sa+fy5cv48UXX0T//v2RkJAAuVyOvXv34oUXXsAvv/yCjz76SGo7dOhQPPDAA0WO8eqrr0KtVuPBBx+0eh/vvfce0tLSin0cr732Gt5++208++yzePDBB/H1119j1KhRkMlkGDFihNTu1KlTCAsLwxNPPIHAwECkpqbif//7H3bu3Inff/8dDRs2lNqW9z1BVOkEETm9xMREAUD8+uuvVvf37t1btGvXzub7iY+PF478WFCr1Q6774qIi4sTtWvXLlPbpk2binHjxpX52FOmTBEARHp6urQtNTVV+Pj4iJdeeqm8XS1WQUGBdB+//vqrACASExOttn3++eeFj4+PuHr1qrQtKSlJABAffPCBtO3EiRMiJyfH7LZ37twR9evXFw8//LDVY+fn54tmzZqJhQsXCgAiPj6+SBuNRiMCAwPF66+/XqbH1qFDBxEbG1tke3HHv3PnjoiMjBQAxK5du8p0H6YUCoUAIN54441y37YkWq1WdOzYUfj6+opDhw4V2Z+dnS1effVVu95nRdn7M2TPnj3C09NTzJ071+pnoEqlEkqlUgghxPbt2wUA8eOPPxY5jkKhEGfPni2yPS4uTgAQf/75Z4n9SEtLEzKZTDz77LNW99+6dUv4+/tLr99ly5aZ7b9+/bqoVauW2evOYDCInj17ikaNGgmdTlfi/Z84cUIAEIsXLzbbXt73BFFl41QoomrKco2FVqvFggUL0KJFC3h7e6Nu3bp45JFHkJSUBAAYP3481qxZA8B8qohRbm4uZsyYgcaNG8PLywutWrXCO++8AyGE2f3m5+fjpZdeQr169eDn54cnnngCN27cgEwmM7uKaZxucP78eYwaNQqBgYF45JFHAACnT5/G+PHj0bx5c3h7eyM0NBQTJkyAUqk0uy/jMf744w/ExsbC398f9evXx+uvvw4hBK5du4bBgwdDLpcjNDQUy5cvL9PfTqfTYdGiRbj//vvh5eWFZs2a4dVXXzW7YiqTyZCYmIjc3Fzpb1XSFKLyatasGQAgKytL2rZu3Tro9XosXLgQwL1pTJZ///Ly8vJCaGhomdr+3//9H/71r3+hSZMm0raoqCi0bNkSn3/+ubQtIiICderUMbtt3bp10bNnT1y4cMHqsZcuXQqDwYCXX3652PuvVasW+vTpg6+//rrUvqampuL06dPFjo5YU7duXXz22Wfw8PDAm2++KW3XaDSYN28eIiIi4O/vj9q1a6Nnz5748ccfpTZXrlxB/fr1AQALFiyQXhPG13xZX9PW/N///R9+//13vPbaa9J7xJRcLjfrLwBs374dERER8PHxQb169RAbG4sbN26YtenTp4/VkZXx48dLrz/jYzNO7/nwww+l98WDDz6IX3/91ex2JX2GpKen4+LFi8VO8bOk1WoxdepUTJ06tdir8X5+fggKCir1WPXq1UO7du2KbH/yyScBoNjXpdGnn34KIQRGjx5tdf8rr7yCVq1aFRmRNPr666+h1WrxwgsvSNtkMhmef/55XL9+vcSROsD65wFQvvcEUVXgVCgiF5KdnY07d+4U2V6WL+r58+dj8eLFmDRpErp16waVSoUTJ07gt99+w6OPPop///vfuHnzJpKSkvDJJ5+Y3VYIgSeeeAI//vgjJk6ciE6dOmHv3r2YOXMmbty4gXfffVdqO378eHz++ecYM2YMHnroIRw8eBAxMTHF9uvpp59GixYt8NZbb0knyUlJSbh8+TLi4uIQGhqKc+fO4cMPP8S5c+fwyy+/mJ2sAMDw4cPRpk0bvP3229i1axf+85//ICgoCB988AH69euHJUuWYMuWLXj55Zfx4IMPolevXiX+rSZNmoSPPvoITz31FGbMmIFjx45h8eLFuHDhAr788ksAwCeffIIPP/wQx48fx/r16wEAPXr0KPV5KI5Go4FKpUJ+fj5OnDiBd955B02bNjWbkvHDDz+gdevW2L17t/S3DwwMRHx8PBYsWFDuqW/lcePGDdy+fRtdu3Ytsq9bt27YvXt3qcfIyMhAvXr1imxPS0vD22+/jY0bN8LHx6fEY0RERODrr7+GSqWCXC4vtt2RI0cAAF26dCm1X6aaNGmC3r1748cff5TuQ6VSYf369Rg5ciSeffZZ5OTkYMOGDYiOjsbx48fRqVMn1K9fH2vXrsXzzz+PJ598EkOHDgUAdOjQAUD5X9OmvvnmGwDAmDFjyvQYNm3ahLi4ODz44INYvHgxbt26hVWrVuHnn3/GqVOnEBAQUK6/idHWrVuRk5ODf//735DJZFi6dCmGDh2Ky5cvo1atWiV+hgDAnDlz8NFHHyE1NdUscCnOypUrcffuXcydOxc7duyoUJ9Lk5GRAQBWX5emtmzZgsaNG1v97Dh+/Dg++ugjHD58uNjn8dSpU6hduzbatGljtr1bt27SfsugUalUQq/XIy0tTbqY0L9//yLHLut7gqhKOHK4hIjKxjgVqqQfy6lQllNvOnbsKGJiYkq8n+KmMXz11VcCgPjPf/5jtv2pp54SMplM/PXXX0IIIU6ePCkAiGnTppm1Gz9+fJEpIm+88YYAIEaOHFnk/vLy8ops+/TTTwUA8dNPPxU5xnPPPSdt0+l0olGjRkImk4m3335b2n737l3h4+NT6nSk5ORkAUBMmjTJbPvLL78sAIj9+/dL28aNG2e3qVDGx2f86dq1qzh9+rRZG7lcLgIDA4WXl5d4/fXXxRdffCFGjRolAIhXXnmlTP0oSUlToYz7Pv744yL7Zs6cKQCIgoKCYo/9008/CZlMZnXKxlNPPSV69Ogh/Y5ipioJIcTWrVsFAHHs2LESH4tx6ozllKzSji+EEFOnThUAxO+//y6EuPeaKiwsNGtz9+5dERISIiZMmCBtK2kqVFlf09Z07txZ+Pv7l9jGSKPRiODgYBEeHi7y8/Ol7Tt37hQAxLx586RtvXv3Fr179y5yjHHjxommTZtKv6empgoAom7duiIzM1Pa/vXXXwsA4ttvv5W2lTQVaty4cQKASE1NLfVxpKenCz8/P2mKXWnTQYUoeSqUNYWFhaJt27YiLCxMaLXaYtudPXtWABCzZs0qss9gMIhu3bpJn2PGv5XlVKiYmBjRvHnzIrfPzc0t9v3r5eUlfR7UrVtX/Pe//7Xav7K+J4iqAqdCEbmQNWvWICkpqciP8apoSQICAnDu3Dn8+eef5b7f3bt3w93dHS+99JLZ9hkzZkAIge+++w4AsGfPHgAwG+4HgBdffLHYY0+ePLnINtOr1gUFBbhz5w4eeughAMBvv/1WpP2kSZOk/7u7u6Nr164QQmDixInS9oCAALRq1QqXL18uti8ApCvvlot3Z8yYAQDYtWtXibevqL59+yIpKQnbt2/H5MmTUatWLeTm5pq1UavVuHv3LhYsWICFCxdi2LBh2LJlCwYOHIhVq1YhJyenUvoG3JviBtybOmXJ29vbrI2l27dvY9SoUQgLC8OsWbPM9v3444/4v//7P6xcubJM/QgMDAQAqyN3ppRKJTw8PIpMySoL422Mf093d3dpYbrBYEBmZiZ0Oh26du1q9fVoTXlf06ZUKhX8/PzKdD8nTpzA7du38cILL0jPCwDExMSgdevWNr1+hw8fLv39gXtZzgCU+p4y2rRpE4QQZRqtmD17Npo3b2723ra3KVOm4Pz581i9ejU8PIqfwLFlyxYAsDoNatOmTThz5gyWLFlS4n3l5+eX+73z3XffYffu3Vi+fDmaNGlS5PPAqKzvCaKqwKlQRC6kW7duVqeiBAYGlvqlsnDhQgwePBgtW7ZEeHg4Bg4ciDFjxpQpKLl69SoaNmxY5OTGOKx/9epV6V83NzeEhYWZtbOWYcXIsi0AZGZmYsGCBfjss89w+/Zts33Z2dlF2pvO+QcAf39/eHt7F5ne4O/vX+qcduNjsOxzaGgoAgICpMdqbyEhIQgJCQEAPPXUU3jrrbfw6KOP4s8//5TWQPj4+CA3N7dIlqORI0diz549OHXqVKnTvCrKeGJsLTNPQUGBWRtTubm5+Ne//oWcnBwcPnzY7ERfp9PhpZdewpgxY4rNtGNJ/D1drqSpQ7ZSq9UAYPZ6/+ijj7B8+fIiawSsvX6tKe9r2pRcLi/zybvx9dmqVasi+1q3bo3Dhw+X6TjWWL7PjCe0d+/erfAxrfnll1/wySefYN++fZU2vW/ZsmX43//+h0WLFmHQoEHFthNCYOvWrQgPDy/yWalSqTBnzhzMnDkTjRs3LvH+fHx8yv3e6du3LwDgsccew+DBgxEeHo46deoUScVcFe8JorLiiAVRDdGrVy9cunQJGzduRHh4ONavX48uXbpI6wMcxdoX6jPPPIP//e9/mDx5Mnbs2IHvv/9eGg2xVujM3d29TNsAlHmxs6O/pJ966imo1WqzRZnGNJPGAMQoODgYgP1P8Ew1aNAAwL0FuJbS09MRFBRU5IqsRqPB0KFDcfr0aXz99ddFaq18/PHHSElJwb///W9cuXJF+gHujRZcuXIFeXl5ZrcxPsbS5sTXrVsXOp2uQqM4Z8+ehbu7uxQ0bN68GePHj8f999+PDRs2YM+ePUhKSkK/fv3KXHivvK9pU61bt0Z2dnaFUwoXp7jXuGVdByNb31NlNWvWLPTs2RNhYWHSa8J44SQ9Pb3ElK5lsWnTJsyePRuTJ0/G3LlzS2z7888/4+rVq1ZHK9555x1oNBoMHz5c6uf169cB3HudXrlyBRqNBsC9909GRkaRv5Xx/WSaQtaa+++/H507d5ZGT0yV9T1BVBUYWBDVIEFBQYiLi8Onn36Ka9euoUOHDmaZmoo70WjatClu3rxZ5CTt4sWL0n7jvwaDAampqWbt/vrrrzL38e7du9i3bx9eeeUVLFiwAE8++SQeffRRNG/evMzHsIXxMVhOGbt16xaysrKkx1rZjFMjTK9mR0REAECR7D7G4mjGrESV4b777kP9+vWtFrkzLmA2ZTAYMHbsWOzbtw9bt25F7969i9wuLS0NWq0WDz/8MMLCwqQf4F7QERYWhu+//97sNqmpqXBzc0PLli1L7G/r1q2l9uWRlpaGgwcPIjIyUhqx+OKLL9C8eXPs2LEDY8aMQXR0NKKioqSrzUbFvX9sfU0//vjjAO4FOKUxvj5TUlKK7EtJSTF7/QYGBhbJMgTAplE5ewTkaWlp+Omnn8xeEzNnzgQAPPHEE2UaZS3O119/jUmTJmHo0KFSBquSbNmyBTKZDKNGjbLaz7t376Jdu3ZSP43Tw9566y2EhYVJBRc7deqEvLy8Itmnjh07Ju0vTX5+vtXRrbK+J4iqAgMLohrCcgpQnTp18MADD5gNz9euXRtA0ZSGgwYNgl6vx+rVq822v/vuu5DJZHjssccAANHR0QCA999/36zde++9V+Z+Gq+KWl7ZK+scfFsZp0VY3t+KFSsAoMQMVxVx584dq1d8jSNJplPfhg8fDgDYsGGDtM1gMCAxMVGqHl2Zhg0bhp07d5pdOd+3bx/++OMPPP3002ZtX3zxRWzbtg3vv/++lCHJ0ogRI/Dll18W+QHuPQ9ffvklunfvbnabkydPol27dvD39y+xr5GRkQBQpmrfRpmZmRg5ciT0ej1ee+01abu11+SxY8eKpAj19fUFUPT9Y+tr+qmnnkL79u3x5ptvWk1LmpOTI/W3a9euCA4Oxrp168ze29999x0uXLhg9vq9//77cfHiRbPK6b///nuxlaXLorjPEKDs6WY//PDDIq8J4zqtd955x+pV+7L46aefMGLECPTq1QtbtmwpdZqVVqvF9u3b8cgjjxSZBgYAL730UpF+fvDBBwDuZcf78ssvpUB58ODBqFWrltlnoxAC69atw3333SdllNPpdFZHHo8fP44zZ85YnQpb1vcEUVXgGguiGqJt27bo06cPIiIiEBQUhBMnTuCLL74wm69rPDF96aWXEB0dDXd3d4wYMQKPP/44+vbti9deew1XrlxBx44d8f333+Prr7/GtGnTpBzzERERGDZsGFauXAmlUimlmzVWBC7L1Uy5XI5evXph6dKl0Gq1uO+++/D999+X+8pzRXXs2BHjxo3Dhx9+iKysLPTu3VtKJzlkyBBp3rO9bN68GevWrcOQIUPQvHlz5OTkYO/evUhKSsLjjz+Ofv36SW0HDx6M/v37Y/Hixbhz5w46duyIr776CocPH8YHH3xgNhVp/PjxZU7tuXr1amRlZUkjH99++600pePFF1+UTlheffVVbN++HX379sXUqVOhVquxbNkytG/fHnFxcdLxVq5ciffffx+RkZHw9fUtcqX9ySefRO3atdG6dWtpZMFSWFgYhgwZYrZNq9Xi4MGDRZIDWNO8eXOEh4fjhx9+wIQJE4rs/+OPP7B582YIIaBSqfD7779j+/btUKvVWLFiBQYOHCi1/de//oUdO3bgySefRExMDFJTU7Fu3Tq0bdtWWo8B3JvW17ZtW2zbtg0tW7ZEUFAQwsPDER4ebtNrulatWtixYweioqLQq1cvPPPMM3j44YdRq1YtnDt3Dlu3bkVgYCDefPNN1KpVC0uWLEFcXBx69+6NkSNHSulmmzVrhunTp0vHnTBhAlasWIHo6GhMnDgRt2/fxrp169CuXTuoVKoy9c1ScZ8hQNnTzQ4YMKDINmOg0rt37yIn1//5z38AAOfOnQNwLxW0cS2JcarT1atX8cQTT0Amk+Gpp57C9u3bzY7RoUOHIiMhe/fuhVKpLLZ2RZcuXYqkMzZO5WvXrp3Z67dRo0aYNm0ali1bBq1WiwcffBBfffUVDh06hC1btkjBp1qtRuPGjTF8+HC0a9cOtWvXxpkzZ5CYmAh/f3+8/vrrZvdXnvcEUZVwQCYqIiqnilTetkxv+p///Ed069ZNBAQECB8fH9G6dWvx5ptvCo1GI7XR6XTixRdfFPXr1xcymcwsbWROTo6YPn26aNiwoahVq5Zo0aKFWLZsmTAYDGb3m5ubK+Lj40VQUJCoU6eOGDJkiEhJSREAzNK/GlPFKhSKIo/n+vXr4sknnxQBAQHC399fPP300+LmzZvFpqy1PEZxaWDLWqFcq9WKBQsWiLCwMFGrVi3RuHFjMWfOnCLpVO2RbvbXX38VTz/9tGjSpInw8vIStWvXFl26dBErVqywmgIzJydHTJ06VYSGhgpPT0/Rvn17sXnz5iLthg0bJnx8fMTdu3fL1DcUk8bYMjXo2bNnxYABA4Svr68ICAgQo0ePFhkZGWZtjGlFy3pMSygmHex3331XpirJRitWrBB16tQpkurVtC9ubm4iICBAdO7cWUydOlWcO3euyHEMBoN46623RNOmTYWXl5fo3Lmz2LlzZ5G0rEIIceTIERERESE8PT3NXq9lfU2X5O7du2LevHmiffv2wtfXV3h7e4vw8HAxZ84cswrtQgixbds20blzZ+Hl5SWCgoLE6NGjxfXr14scc/PmzaJ58+bC09NTdOrUSezdu7fYdLOWKVSNf0vT/pf0GVKedLOWSvoMLOm1ZvTjjz+W2M7aczBixAhRq1YtqbJ3WZT0t9Lr9dLryNPTU7Rr167Ie7ewsFBMnTpVdOjQQcjlclGrVi3RtGlTMXHiRKt/t/K+J4gqm0wIO6+6IiKykJycjM6dO2Pz5s3FXv2r7po1a4bIyEi899578PHxkaaMVJaQkBCMHTsWy5Ytq9T7qUpDhgyBTCaTpkuVJjs7G82bN8fSpUvNUg8TVRflfU8QVTYGFkRkV/n5+UUyPY0fPx6ffPIJrly5UmpaxuqqWbNm0qLY+Pj4IutV7OncuXOIjIzE5cuXq02mmAsXLqB9+/ZITk4ukl2qJEuWLEFiYiLOnz9fqZXJiapaRd8TRJWJgQUR2dWCBQtw8uRJ9O3bFx4eHvjuu+/w3Xff4bnnnpMWNtZEP//8s5TpqXHjxlbrDBAREbkyBhZEZFdJSUlYsGABzp8/D7VajSZNmmDMmDF47bXXSqxuS0RERK6NgQUREREREdmME06JiIiIiMhmDCyIiIiIiMhmnPBsJwaDATdv3oSfn1+ZioARERERETk7IQRycnLQsGHDUrPrMbCwk5s3b9bYNJpEREREVL1du3YNjRo1KrENAws78fPzA3Dvjy6Xyx3cGyIiIiIi26lUKjRu3Fg61y0JAws7MU5/ksvlDCyIiIiIqFopy1R/Lt4mIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbMbAgIiIiIiKbeTi6A0RERERkm4yMDGRlZTm6G1QFAgICEBoa6uhuWMXAgoiIiMiFZWRk4KmnnkJBQYGju0JVwNvbG1988YVTBhcMLIiIiIhcWFZWFgoKCrBo0SKEhYU5ujtUiVJTU/H6668jKyuLgQURERERVY6wsDC0bt3a0d2gGoyBBRERERWhUCigUqmg0Wjg6ekJuVyO+vXrO7pbROTEGFgQERGRGYVCgdi4SbilzMKNtKto1DQMwUFybE5cz+CCiIrFdLNERERkRqVSITMnD55NO6PQIINXyx7IzMmDSqVydNeIyIk5NLBYu3YtOnToALlcDrlcjsjISHz33XfS/j59+kAmk5n9TJ482ewYaWlpiImJga+vL4KDgzFz5kzodDqzNgcOHECXLl3g5eWFBx54AJs2bSrSlzVr1qBZs2bw9vZG9+7dcfz48Up5zERERK7Cu4783r9+AY7tCNksIyMDL774Ipo3bw4vLy80btwYjz/+OPbt21flfZHJZPjqq69sOsbPP/8MDw8PdOrUyWz7Tz/9hMcffxwNGza0ej9arRazZ89G+/btUbt2bTRs2BBjx47FzZs3pTYHDhwocv5p/Pn111+L9OWvv/6Cn58fAgICiuzbvn07WrduDW9vb7Rv3x67d+822z9//ny0bt0atWvXRmBgIKKionDs2LEK/10czaGBRaNGjfD222/j5MmTOHHiBPr164fBgwfj3LlzUptnn30W6enp0s/SpUulfXq9HjExMdBoNDhy5Ag++ugjbNq0CfPmzZPapKamIiYmBn379kVycjKmTZuGSZMmYe/evVKbbdu2ISEhAW+88QZ+++03dOzYEdHR0bh9+3bV/CGIiIiIKsmVK1cQERGB/fv3Y9myZThz5gz27NmDvn37Ij4+3tHdK7esrCyMHTsW/fv3L7IvNzcXHTt2xJo1a6zeNi8vD7/99htef/11/Pbbb9ixYwdSUlLwxBNPSG169Ohhdu6Znp6OSZMmISwsDF27djU7nlarxciRI9GzZ88i93XkyBGMHDkSEydOxKlTpzBkyBAMGTIEZ8+eldq0bNkSq1evxpkzZ3D48GE0a9YMAwYMgEKhqOifx7GEkwkMDBTr168XQgjRu3dvMXXq1GLb7t69W7i5uYmMjAxp29q1a4VcLheFhYVCCCFmzZol2rVrZ3a74cOHi+joaOn3bt26ifj4eOl3vV4vGjZsKBYvXlzmfmdnZwsAIjs7u8y3ISIicia3b98Wf/31lzh27Jjo2itK9Jz4uvALbSZ6T1kquvaKEn/99Zeju0hWXLhwQURERIgLFy5Y3f/YY4+J++67T6jV6iL77t69K/3/6tWr4oknnhC1a9cWfn5+4umnnzY7xxo3bpwYPHiw2e2nTp0qevfuLf3eu3dv8eKLL4qZM2eKwMBAERISIt544w1pf9OmTQUA6adp06blfrzDhw8Xc+fOFW+88Ybo2LFjse0AiC+//LLU4x0/flwAEFevXrW6X6PRiPr164uFCxcW2Tdr1iwRGxsrEhMThb+/v9m+Z555RsTExJht6969u/j3v/9dbF+M55M//PCD1f2lPdeVoTznuE6zxkKv1+Ozzz5Dbm4uIiMjpe1btmxBvXr1EB4ejjlz5iAvL0/ad/ToUbRv3x4hISHStujoaKhUKmnU4+jRo4iKijK7r+joaBw9ehQAoNFocPLkSbM2bm5uiIqKktpYU1hYCJVKZfZDRETkqowLtkdMmIwZs1+FTqsr/Ubk9DIzM7Fnzx7Ex8ejdu3aRfYbp+8YDAYMHjwYmZmZOHjwIJKSknD58mUMHz683Pf50UcfoXbt2jh27BiWLl2KhQsXIikpCQCkqUSJiYlIT0+Xfr9y5QpkMhkOHDhQ4rETExNx+fJlvPHGG+XuV3Gys7Mhk8msTmUCgG+++QZKpRJxcXFm2/fv34/t27cXOzpS2jmoJY1Ggw8//BD+/v7o2LFj+R+IE3B4VqgzZ84gMjISBQUFqFOnDr788ku0bdsWADBq1Cg0bdoUDRs2xOnTpzF79mykpKRgx44dAO7NFzQNKgBIv2dkZJTYRqVSIT8/H3fv3oVer7fa5uLFi8X2e/HixViwYIFtD56IiMhJGBds+7bqiczkvZC5uTu6S2QHf/31F4QQpda32LdvH86cOYPU1FQ0btwYAPDxxx+jXbt2+PXXX/Hggw+W+T47dOggnfi3aNECq1evxr59+/Doo49KWcUCAgLMCrzVqlULrVq1gq+vb7HH/fPPP/HKK6/g0KFD8PCwzylsQUEBZs+ejZEjR0Iul1tts2HDBkRHR6NRo0bSNqVSifHjx2Pz5s3F3q64c1DjOarRzp07MWLECOTl5aFBgwZISkpCvXr1bHxkjuHwwKJVq1ZITk5GdnY2vvjiC4wbNw4HDx5E27Zt8dxzz0nt2rdvjwYNGqB///64dOkS7r//fgf2GpgzZw4SEhKk31UqlfRGJCIiclU+8kBHd4HsSAhRpnYXLlxA48aNzc5l2rZti4CAAFy4cKHcgYWpBg0alLpu9b777ivxgq5er8eoUaOwYMECtGzZssx9KYlWq8UzzzwDIQTWrl1rtc3169exd+9efP7552bbn332WYwaNQq9evWyuR/GdcB37tzB//73PzzzzDM4duwYgoODbT52VXP4VChPT0888MADiIiIwOLFi9GxY0esWrXKatvu3bsDuBd9A0BoaChu3bpl1sb4uzEKLq6NXC6Hj48P6tWrB3d3d6ttSiqV7uXlJWWzMv4QEREROZMWLVpAJpOVeNJeVm5ubkUCFa1WW6RdrVq1zH6XyWQwGAw23XdOTg5OnDiBKVOmwMPDAx4eHli4cCF+//13eHh4YP/+/eU6njGouHr1KpKSkoo9j0tMTETdunXNFncD96ZBvfPOO1JfJk6ciOzsbHh4eGDjxo0Aij8HtTy/rF27Nh544AE89NBD2LBhAzw8PLBhw4ZyPR5n4fDAwpLBYEBhYaHVfcnJyQDuRb4AEBkZiTNnzphFwcYXh3E6VWRkZJFUaklJSdI6Dk9PT0RERJi1MRgM2Ldvn9laDyIiIiJXExQUhOjoaKxZswa5ublF9mdlZQEA2rRpg2vXruHatWvSvvPnzyMrK0s6p6pfvz7S09PNbm88NyuPWrVqQa/Xl+s2crkcZ86cQXJysvQzefJkaeaL8eJzWRiDij///BM//PAD6tata7WdEAKJiYkYO3ZskWDp6NGjZn1ZuHAh/Pz8kJycjCeffBJA6eegxSnpXNjZOXQq1Jw5c/DYY4+hSZMmyMnJwdatW3HgwAHs3bsXly5dwtatWzFo0CDUrVsXp0+fxvTp09GrVy9piG3AgAFo27YtxowZg6VLlyIjIwNz585FfHw8vLy8AACTJ0/G6tWrMWvWLEyYMAH79+/H559/jl27dkn9SEhIwLhx49C1a1d069YNK1euRG5ubpFFOkRERDWZUqkEcO8kjxW4XceaNWvw8MMPo1u3bli4cCE6dOgAnU6HpKQkrF27FhcuXEBUVBTat2+P0aNHY+XKldDpdHjhhRfQu3dvKcVqv379sGzZMnz88ceIjIzE5s2bcfbsWXTu3Llc/WnWrBn27duHhx9+GF5eXggMDMSNGzfQv39/fPzxx+jWrVuR27i5uSE8PNxsW3BwMLy9vc22q9VqaWYLcK/sQHJyMoKCgtCkSRNotVo89dRT+O2337Bz507o9XppzUNQUBA8PT2l2+7fvx+pqamYNGlSkf60adPG7PcTJ04U6ePUqVPRu3dvLF++HDExMfjss89w4sQJfPjhhwDupcZ988038cQTT6BBgwa4c+cO1qxZgxs3buDpp58uz5/UaTh0xOL27dsYO3YsWrVqhf79++PXX3/F3r178eijj8LT0xM//PADBgwYgNatW2PGjBkYNmwYvv32W+n27u7u2LlzJ9zd3REZGYnY2FiMHTsWCxculNqEhYVh165dSEpKQseOHbF8+XKsX78e0dHRUpvhw4fjnXfewbx589CpUyckJydjz549RRbcEBER1VRajQYz5szFiAmTERs3yXXz7NdAzZs3x2+//Ya+fftixowZCA8Px6OPPop9+/ZJawtkMhm+/vprBAYGolevXoiKikLz5s2xbds26TjR0dF4/fXXMWvWLDz44IPIycnB2LFjy92f5cuXIykpCY0bN5aCEq1Wi5SUFLPsnxVx4sQJdO7cWTpuQkICOnfuLNU4u3HjBr755htcv34dnTp1QoMGDaSfI0eOmB1rw4YN6NGjR6kL34vTo0cPbN26FR9++CE6duyIL774Al999ZUUfLi7u+PixYsYNmwYWrZsiccffxxKpRKHDh1Cu3btbPgrOI5MlHVVD5VIpVLB398f2dnZXG9BREQuQ6FQQKVSQalUIn7ma6j74BO48dNnkLm5IyC8D5J3fYIuT70A5bGvIHNzR2CHKOSlHMJnG9c5PJEK3XPx4kXExsZi8+bNFT4JJtfgiOe6POe4Ds8KRURERI5hrF2RmZMHbzcBnbb0ee8+8kDYdk2ZiKorp1u8TURERFXDtHZFVm4+9IbyLaglIjLFwIKIiKiGY+0KIrIHBhZERERERGQzBhZERERERGQzBhZERERUIQqFgmlniUjCwIKIiIjKTalUIjZuEmtaEJGE6WaJiIio3NRqNTJz7iWeValUrMTtBFJTUx3dBapkzv4cM7AgIiIicmEBAQHw9vbG66+/7uiuUBXw9vZGQECAo7thFQMLIiIiIhcWGhqKL774AllZWY7uClWBgIAAhIaGOrobVjGwICIiInJxoaGhTnuySTUHAwsiIqIahoutiagyMCsUERFRDaJQKKRsTkql0tHdIaJqhCMWRERENYhKpZKyOanVagf3hoiqE45YEBERERGRzRhYEBERERGRzRhYEBERkc0UCgUXhRPVcAwsiIiIyCZKpVJaEM7ggqjmYmBBRERENlGr1cjMyUNmTh5UKpWju0NEDsLAgoiIiIiIbMZ0s0RERDUApygRUWXjiAUREVE1x6J4RFQVOGJBRERUzbEoHhFVBY5YEBERERGRzRhYEBERERGRzRhYEBERERGRzRhYEBERERGRzRhYEBERERGRzRhYEBEROZBCoWCNCSKqFphuloiIyEGM9SUAYHPietSvX9+ux1apVJDL5XY7JhFRSRhYEBERVRHjyIQxgDCtL6FSqaR/5XK5TUGGMWDJzMlDkJ8vFs17zcaeExGVjoEFERFRFTCe7GsKC7HkzYUICwsz269UKvHC1AQpGFi57G14enpWKMgwBiy+rXoiM+UQi+IRUZVgYEFERFQFVCoVbimzcCX1Mia99DIa1As0G0lQq9VSMHDrzD48N2UqCnQCQX6+RaZJWY58FMdHHoi8ynk4JTKdhmXP6V1E5Ny4eJuIiKiK6PU6GGQe8G3dE5k5eVZHEnzkgdDrdcjOLbg34pCTJ02TAv4Z+YiNm+SUi76VSiVi4yZhxITJTttHIqocDCyIiIiqmLdfQJna+cgDi2wzTnOyDDiAe0HHpUuXoFQq7dHNCjEdebHWRyKqvjgVioiIqBowXbDt7Sag0+od2h9HTcMiIsdhYEFERFSJ7DEVyDgCUVLqWLMF28l7IXNzt/l+iYjKg4EFERHVeJW12Ni0TkVFU75qNRrMmDNXWshteRzLwMXa9CkioqrAwIKIiGo0y5oP9ixUZ1qnoqIpX+8t5BYI7BBVJHWsMUUtUPHAhYjIXrh4m4iIajSzKUR2WGxsXEBt72xI1kYijAuli8swRURUlThiQUREBPssNmbFayKqyRhYEBFRjVTciILpQmlWvCYiKjsGFkREVGOYBhPWFlVbLpSu6HoLplolopqIayyIiKhGMK1YnZqaanVtQkkVr4mIqGQODSzWrl2LDh06QC6XQy6XIzIyEt999520v6CgAPHx8ahbty7q1KmDYcOG4datW2bHSEtLQ0xMDHx9fREcHIyZM2dCp9OZtTlw4AC6dOkCLy8vPPDAA9i0aVORvqxZswbNmjWDt7c3unfvjuPHj1fKYyYiIscwrVhd2hSl4lK2KhQKq1OonKHitbNSKpWVspidiJyPQ6dCNWrUCG+//TZatGgBIQQ++ugjDB48GKdOnUK7du0wffp07Nq1C9u3b4e/vz+mTJmCoUOH4ueffwYA6PV6xMTEIDQ0FEeOHEF6ejrGjh2LWrVq4a233gIApKamIiYmBpMnT8aWLVuwb98+TJo0CQ0aNEB0dDQAYNu2bUhISMC6devQvXt3rFy5EtHR0UhJSUFwcLDD/j5ERORYloXpjNOnNieuB3AvWNFoNJg28xWnqXjtTOw1tYyIXINDA4vHH3/c7Pc333wTa9euxS+//IJGjRphw4YN2Lp1K/r16wcASExMRJs2bfDLL7/goYcewvfff4/z58/jhx9+QEhICDp16oRFixZh9uzZmD9/Pjw9PbFu3TqEhYVh+fLlAIA2bdrg8OHDePfdd6XAYsWKFXj22WcRFxcHAFi3bh127dqFjRs34pVXXqnCvwgRETkLa4XpjDUpUlNT8frCN6VgQl2ohzy8LyteW7CswaFSqRhYEFVjTrPGQq/X47PPPkNubi4iIyNx8uRJaLVaREVFSW1at26NJk2a4OjRowCAo0ePon379ggJCZHaREdHQ6VS4dy5c1Ib02MY2xiPodFocPLkSbM2bm5uiIqKktoQEZFrsqWmhOV6C9PpU8b6Eb6teiIrNx96g54Vr0vAvw1RzeDwrFBnzpxBZGQkCgoKUKdOHXz55Zdo27YtkpOT4enpiYCAALP2ISEhyMjIAABkZGSYBRXG/cZ9JbVRqVTIz8/H3bt3odfrrba5ePFisf0uLCxEYWGh9DsX+BERORd71ZQoKcMTT5iJiP7h8BGLVq1aITk5GceOHcPzzz+PcePG4fz5847uVqkWL14Mf39/6adx48aO7hIREZmwrKjNmhJERJXL4YGFp6cnHnjgAURERGDx4sXo2LEjVq1ahdDQUGg0GmRlZZm1v3XrFkJDQwEAoaGhRbJEGX8vrY1cLoePjw/q1asHd3d3q22Mx7Bmzpw5yM7Oln6uXbtWocdPRESVi6MKRERVw+GBhSWDwYDCwkJERESgVq1a2Ldvn7QvJSUFaWlpiIyMBABERkbizJkzuH37ttQmKSkJcrkcbdu2ldqYHsPYxngMT09PREREmLUxGAzYt2+f1MYaLy8vKU2u8YeIiIiIqKZy6BqLOXPm4LHHHkOTJk2Qk5ODrVu34sCBA9i7dy/8/f0xceJEJCQkICgoCHK5HC+++CIiIyPx0EMPAQAGDBiAtm3bYsyYMVi6dCkyMjIwd+5cxMfHw8vLCwAwefJkrF69GrNmzcKECROwf/9+fP7559i1a5fUj4SEBIwbNw5du3ZFt27dsHLlSuTm5kpZooiIiIiIqGQODSxu376NsWPHIj09Hf7+/ujQoQP27t2LRx99FADw7rvvws3NDcOGDUNhYSGio6Px/vvvS7d3d3fHzp078fzzzyMyMhK1a9fGuHHjsHDhQqlNWFgYdu3ahenTp2PVqlVo1KgR1q9fL6WaBYDhw4dDoVBg3rx5yMjIQKdOnbBnz54iC7qJiIiIiMg6hwYWGzZsKHG/t7c31qxZgzVr1hTbpmnTpti9e3eJx+nTpw9OnTpVYpspU6ZgypQpJbYhIiLnp1AooFKpWAWbiKiKOTzdLBERkb2YpphlFWwioqrldIu3iYiIKso0xayxcB0REVUNBhZERFTtMMUsEVHVY2BBREREREQ2Y2BBREREREQ2Y2BBREREREQ2Y2BBREREREQ2Y7pZIiIiqhLG2iJyuRz169d3cG+IyN4YWBAREVGl02o0mDFnLgp0AkF+vticuJ7BBVE1w6lQREREVOn0eh2ycwvg26onMnPyoFKpHN0lIrIzBhZERERUZVhjhKj6YmBBREREREQ2Y2BBREREREQ24+JtIiJyeQqFwtFdICKq8ThiQURELk2hUCA2bhJi4yZJ6UyJiKjqccSCiIhcmkqlQmZOHgBArVY7uDdERDUXRyyIiIiIiMhmDCyIiIiIiMhmDCyIiIiIiMhmDCyIiIiIiMhmDCyIiIiIiMhmDCyIiIiIiMhmDCyIiIjIIRQKBYsbElUjDCyIiIioyimVSqmwIYMLouqBgQURERFVObVajcycPGTm5EGlUjm6O0RkB6y8TURELolXuYmInAtHLIiIyOUoFAppGo1SqXR0d4iICByxICIiF6RSqZCZkwfg3pQacn0KhQIqlQpyuRz169d3dHeIqAIYWBAREZFDKZVKvDA1AZk5eQjy88XmxPUMLohcEKdCERERkUMZF3L7turJxdxELoyBBRERETkFH3mgo7tARDZgYEFERC5FoVDg6tWr0Gl1ju4KERGZ4BoLIiJyGcZsUDdv38H1G+lo1rSpo7tERER/44gFERG5DGM2KJ/mD0JvENAb9I7uEhER/Y2BBRERuRzvOnJHd4GIiCwwsCAiIiIiIpsxsCAiIiIiIpsxsCAiIiIiIpsxKxQRERE5FaVSCQCQy+WswE3kQhhYEBERkdPQajSYMWcuCnQCQX6+2Jy4nsEFkYvgVCgiIiJyGnq9Dtm5BfBt1ROZOXlQqVSO7hIRlRFHLIiIiMjp+MgDkQdOiyJyJQwsiIiIyClxWhSRa+FUKCIicnoKhQIKhcLR3aAqxmlRRK6FgQURETk1hUKB2LhJiI2bJE2LoZrFRx7o6C4QURk4NLBYvHgxHnzwQfj5+SE4OBhDhgxBSkqKWZs+ffpAJpOZ/UyePNmsTVpaGmJiYuDr64vg4GDMnDkTOp3OrM2BAwfQpUsXeHl54YEHHsCmTZuK9GfNmjVo1qwZvL290b17dxw/ftzuj5mIiMpHpVIhMycPmTl5UKvVju4OEREVw6GBxcGDBxEfH49ffvkFSUlJ0Gq1GDBgAHJzc83aPfvss0hPT5d+li5dKu3T6/WIiYmBRqPBkSNH8NFHH2HTpk2YN2+e1CY1NRUxMTHo27cvkpOTMW3aNEyaNAl79+6V2mzbtg0JCQl444038Ntvv6Fjx46Ijo7G7du3K/8PQURERGXCaXFEzsuhi7f37Nlj9vumTZsQHByMkydPolevXtJ2X19fhIaGWj3G999/j/Pnz+OHH35ASEgIOnXqhEWLFmH27NmYP38+PD09sW7dOoSFhWH58uUAgDZt2uDw4cN49913ER0dDQBYsWIFnn32WcTFxQEA1q1bh127dmHjxo145ZVXKuPhExERUTkolUq8MDUBALiQm8gJOdUai+zsbABAUFCQ2fYtW7agXr16CA8Px5w5c5CXlyftO3r0KNq3b4+QkBBpW3R0NFQqFc6dOye1iYqKMjtmdHQ0jh49CgDQaDQ4efKkWRs3NzdERUVJbYiIiMix1Gq1NC2OC7mJnI/TpJs1GAyYNm0aHn74YYSHh0vbR40ahaZNm6Jhw4Y4ffo0Zs+ejZSUFOzYsQMAkJGRYRZUAJB+z8jIKLGNSqVCfn4+7t69C71eb7XNxYsXrfa3sLAQhYWF0u/8gCMiIiKimsxpAov4+HicPXsWhw8fNtv+3HPPSf9v3749GjRogP79++PSpUu4//77q7qbksWLF2PBggUOu38ioupMoVBApVJBLpc7uitERFRGTjEVasqUKdi5cyd+/PFHNGrUqMS23bt3BwD89ddfAIDQ0FDcunXLrI3xd+O6jOLayOVy+Pj4oF69enB3d7fapri1HXPmzEF2drb0c+3atTI+WiIiKokxveyICZOZYpaIyIU4NLAQQmDKlCn48ssvsX//foSFhZV6m+TkZABAgwYNAACRkZE4c+aMWfampKQkyOVytG3bVmqzb98+s+MkJSUhMjISAODp6YmIiAizNgaDAfv27ZPaWPLy8oJcLjf7ISIi2xnTyxqLojHFLBGRa3DoVKj4+Hhs3boVX3/9Nfz8/KQ1Ef7+/vDx8cGlS5ewdetWDBo0CHXr1sXp06cxffp09OrVCx06dAAADBgwAG3btsWYMWOwdOlSZGRkYO7cuYiPj4eXlxcAYPLkyVi9ejVmzZqFCRMmYP/+/fj888+xa9cuqS8JCQkYN24cunbtim7dumHlypXIzc2VskQREVHV8pEHIq/0ZkRE5CQcGlisXbsWwL0ieKYSExMxfvx4eHp64ocffpBO8hs3boxhw4Zh7ty5Ult3d3fs3LkTzz//PCIjI1G7dm2MGzcOCxculNqEhYVh165dmD59OlatWoVGjRph/fr1UqpZABg+fDgUCgXmzZuHjIwMdOrUCXv27CmyoJuIiIiIiIpyaGAhhChxf+PGjXHw4MFSj9O0aVPs3r27xDZ9+vTBqVOnSmwzZcoUTJkypdT7IyIiIsczFspjPQsi5+AUi7eJiIiIykOpVCI2bhJi4yaxEjeRk2BgQURERC6HxfKInI/T1LEgIqKazVi7gulliYhcEwMLIiJyOGPtisycPHi7Cei0ekd3iYiIyolToYiIyOFMa1dk5eZDb2BgQUTkajhiQURETsNHHujoLpCLMk6lk8vlzBJF5CAMLIiIiMilKZVKvDA1AZk5eQjy88XmxPUMLogcgFOhiIiIyKUZM0T5turJLFFEDsTAgoiIiKoFTqUjcixOhSIiIiIzOq0Wbu7C0d0gIhfDwIKIiIgkmrwcXL92DW6e3pCHMzsXEZUdAwsiIiKS6DQFEDJ36A0GGFw07a+xyCIzRBFVLQYWREREVG1oNRrMmDMXBTrBDFFEVYyLt4mIiAg6rRZ6nc7R3bCZXq9Ddm4BM0QROQBHLIiIyGGMRc2MU1fIMcqyrsLVFnT7yAOR5+hOENUwDCyIiMghFAoFYuMmITMnD95uAjqta87nrw5KW1ehyVdLgUeoC41qaDUaXL16lWstiKoIp0IREZFDqFQqqahZVm4+9C66ULgm0GsKpcBDr9dBq9Hg5s2b0GmdN8jQ5OXgSuplTHt1PmLjJkGhUDi6S0TVHgMLIiJyKBY1cy2afDWupF7GondW4crVq04bXOg0BTDIPODbmmstiKoKAwsiIiIqM72mEAaZB3yaPwi9QTj9SJO3X4Cju0BUYzCwICIionLzqiN3dBeIyMkwsCAiIiIiIpsxsCAiIiIiIpsxsCAiIqqBXCGzExG5FgYWRERENYxWozHL7KTVOfcCbCJyDQwsiIiIagjjKIWm0Dyzk7WieGWl07rGyIdSqcSlS5dYz4KoErHyNhERUQ1gOkqRnp4BIXO3ObOTwaDH9WvXsOidVbitUKJZ06Z26q19aTUazJgzFwU6gSA/X2xOXM9K3ESVgCMWRERUpRQKBa8aO4Ber5NGKQwGAQFh+0ENBhjcnL+mhV6vQ3ZuAXxbsVgeUWViYEFERFVGoVAgNm4SYuMmQalUOro7NVJl1J9wlZoWrPJOVLk4FYqIiKqMSqVCZk4eAECtVju4N0REZE8csSAiIiIiIpsxsCAiIiIiIptVKLC4fPmyvftBREREREQurEKBxQMPPIC+ffti8+bNKCgosHefiIiIqr2qqnxtvB99FRTBc5WaFkRUOSoUWPz222/o0KEDEhISEBoain//+984fvy4vftGRERULWny1WaVryvrRFyTlyPdz/UbNyCEHVLMFsO0psW9at4MLohqmgoFFp06dcKqVatw8+ZNbNy4Eenp6XjkkUcQHh6OFStWMD85ERFRCfQa88rXlVX/QacpsH/tiuJY1rTQM7AgqmlsWrzt4eGBoUOHYvv27ViyZAn++usvvPzyy2jcuDHGjh2L9PR0e/WTiIio2qms+g86rRZ6kxGDqqwz4Qo1LZRKJS5dusQLoUR2ZlNgceLECbzwwgto0KABVqxYgZdffhmXLl1CUlISbt68icGDB9urn0RERFQGmrwcXL92DddvpkNbBesqXI1Wo8GMOXMxYsJkxMZNYnBBZEcVKpC3YsUKJCYmIiUlBYMGDcLHH3+MQYMGwc3tXpwSFhaGTZs2oVmzZvbsKxERkUsqbQG15aJnnVYLN/eSpy2ZLv72qPXP17lOUwAhc4feYIChkqZYuTK9XofsXIHADlHITDkElUqF+vXrO7pbRNVChQKLtWvXYsKECRg/fjwaNGhgtU1wcDA2bNhgU+eIiIhcnVajkRZQp6dnQMjczfabLnq+rVCiTngWrl+7BjdPb4TqdFaDDNNj3lYo0axp06p8SNWCjzwQeY7uBFE1U6HA4s8//yy1jaenJ8aNG1eRwxMREVUber3unwXUN74BLEciDAYY3P9e9HzrO2gKcqURh3y1eZBh7Zj6W99V2uJvIqLyqNAai8TERGzfvr3I9u3bt+Ojjz6yuVNERETVTWmLmq3t12sKpSBDr9cVmVJlvA3rRxCRM6hQYLF48WLUq1evyPbg4GC89dZbNneKiIioslRVYTp7M619YVqTomj9COcZvbDMTkVE1VuFAou0tDSEhYUV2d60aVOkpaXZ3CkiIqpeFAqFU2TfMV2bUJmF6SqDae0Ls5oUFvUjnGXBtiZfbZKdynX+zkRUcRUKLIKDg3H69Oki23///XfUrVvX5k4REVH1oVAoEBs3CbFxk6BUKh3aF7O1CZVYmK4yFTelytnqR1hO4yKi6q9Ci7dHjhyJl156CX5+fujVqxcA4ODBg5g6dSpGjBhh1w4SEZFrU6lUyMy5l39HrVY7uDf3lGVtQllSvpaktBSzRETVTYVGLBYtWoTu3bujf//+8PHxgY+PDwYMGIB+/fqVa43F4sWL8eCDD8LPzw/BwcEYMmQIUlJSzNoUFBQgPj4edevWRZ06dTBs2DDcunXLrE1aWhpiYmLg6+uL4OBgzJw5EzqLYdcDBw6gS5cu8PLywgMPPIBNmzYV6c+aNWvQrFkzeHt7o3v37jh+/HjZ/yhERORSiq5N0ElrAsyLzJX/arsmL8fqeggiouqsQoGFp6cntm3bhosXL2LLli3YsWMHLl26hI0bN8LT07PMxzl48CDi4+Pxyy+/ICkpCVqtFgMGDEBubq7UZvr06fj222+xfft2HDx4EDdv3sTQoUOl/Xq9HjExMdBoNDhy5Ag++ugjbNq0CfPmzZPapKamIiYmBn379kVycjKmTZuGSZMmYe/evVKbbdu2ISEhAW+88QZ+++03dOzYEdHR0bh9+3ZF/kRERORErI4eWKxNMKZ2vX4zHfl5ucVO4ylp8bcxMNFpCqyvhyAiqsYqNBXKqGXLlmjZsmWFb79nzx6z3zdt2oTg4GCcPHkSvXr1QnZ2NjZs2ICtW7eiX79+AO6lum3Tpg1++eUXPPTQQ/j+++9x/vx5/PDDDwgJCUGnTp2waNEizJ49G/Pnz4enpyfWrVuHsLAwLF++HADQpk0bHD58GO+++y6io6MB3Ksm/uyzzyIuLg4AsG7dOuzatQsbN27EK6+8UuHHSEREjmU6emCtQJ1xWpTpmoDiFkBbFqYL6vZPcGEc5XDz9IY83DwdLDk3Y2IBVuAmsk2FRiz0ej02bNiAUaNGISoqCv369TP7qajs7GwAQFBQEADg5MmT0Gq1iIqKktq0bt0aTZo0wdGjRwEAR48eRfv27RESEiK1iY6Ohkqlwrlz56Q2pscwtjEeQ6PR4OTJk2Zt3NzcEBUVJbWxVFhYCJVKZfZDRETOx56jB0UWf+t1ZqMUpQUm5HyUSqWUXMAZMpcRubIKjVhMnToVmzZtQkxMDMLDwyGTyWzuiMFgwLRp0/Dwww8jPDwcAJCRkQFPT08EBASYtQ0JCUFGRobUxjSoMO437iupjUqlQn5+Pu7evQu9Xm+1zcWLF632d/HixViwYEHFHiwREVU5W0YPiitMZ0ypajpKQa5FrVZLyQVUKhVHLYhsUKHA4rPPPsPnn3+OQYMG2a0j8fHxOHv2LA4fPmy3Y1amOXPmICEhQfpdpVKhcePGDuwRERFVBtPCdJZTqcoyfYqIqKao8OLtBx54wG6dmDJlCnbu3Ikff/wRjRo1kraHhoZCo9EgKyvLrP2tW7cQGhoqtbHMEmX8vbQ2crkcPj4+qFevHtzd3a22MR7DkpeXF+RyudmPq1IoFLh06RKHgImIrCi2MB0REZmpUGAxY8YMrFq1yub0eUIITJkyBV9++SX2799fpJp3REQEatWqhX379knbUlJSkJaWhsjISABAZGQkzpw5Y5a9KSkpCXK5HG3btpXamB7D2MZ4DE9PT0RERJi1MRgM2Ldvn9SmujIWrhoxYTLnlxIRlYALsW1jXItCRNVXhaZCHT58GD/++CO+++47tGvXDrVq1TLbv2PHjjIdJz4+Hlu3bsXXX38NPz8/aU2Ev78/fHx84O/vj4kTJyIhIQFBQUGQy+V48cUXERkZiYceeggAMGDAALRt2xZjxozB0qVLkZGRgblz5yI+Ph5eXl4AgMmTJ2P16tWYNWsWJkyYgP379+Pzzz/Hrl27pL4kJCRg3Lhx6Nq1K7p164aVK1ciNzdXyhJVXRkLV/m26onMlEOcX0pE1YatBe7IfkzXooQyuCCqtioUWAQEBODJJ5+0+c7Xrl0LAOjTp4/Z9sTERIwfPx4A8O6778LNzQ3Dhg1DYWEhoqOj8f7770tt3d3dsXPnTjz//POIjIxE7dq1MW7cOCxcuFBqExYWhl27dmH69OlYtWoVGjVqhPXr10upZgFg+PDhUCgUmDdvHjIyMtCpUyfs2bOnyILu6spHHog8R3eCiMhOrKV+JccxXYtiWReEiKqPCgUWiYmJdrnzskyl8vb2xpo1a7BmzZpi2zRt2hS7d+8u8Th9+vTBqVOnSmwzZcoUTJkypdQ+ERGRc2PqVyKiqlehNRYAoNPp8MMPP+CDDz5ATk4OAODmzZtQq9V26xwREREREbmGCo1YXL16FQMHDkRaWhoKCwvx6KOPws/PD0uWLEFhYSHWrVtn734SEREREZETq9CIxdSpU9G1a1fcvXsXPj4+0vYnn3yySPYlIiKqeRyRxtpYxE6n5Rx+IiJHqNCIxaFDh3DkyBF4enqabW/WrBlu3Lhhl44REZFrMqaxzszJQ5CfLxbNe63S71Or0UhF7G4rlGjdskel3ycREZmr0IiFwWCAXl90Mdz169fh5+dnc6fIsRQKBetZEFGFmaWxzsmrkrV3er1OKmKnNwgu2KYK43cgUcVVKLAYMGAAVq5cKf0uk8mgVqvxxhtvYNCgQfbqGzmAUqlEbNwkFssjIpv5yAOr/D5ZxI5swe9AIttUaCrU8uXLER0djbZt26KgoACjRo3Cn3/+iXr16uHTTz+1dx+pCqnVamTm3KtowWJ5ROTsdNp76yr0Oo5QkO34HUhkmwoFFo0aNcLvv/+Ozz77DKdPn4ZarcbEiRMxevRos8XcRERElcVg0OP6tWtY9M4qpKdnQMjcHd0lKiPThfYetSp0KkJETqjC72YPDw/Exsbasy9ERERlZzDA4H5vXYXhxjeAe+lFV8nxNPlqs4X2zZo2dXSXiMhOKhRYfPzxxyXuHzt2bIU6Q0REVF5cV+Fa9JrCfxba3/oOei60J6o2KhRYTJ061ex3rVaLvLw8eHp6wtfXl4EFERERlYgBIVH1U6GsUHfv3jX7UavVSElJwSOPPMLF20RERERENVCFAgtrWrRogbfffrvIaAYREREREVV/dgssgHsLum/evGnPQxIRERERkQuo0BqLb775xux3IQTS09OxevVqPPzww3bpGBERERERuY4KBRZDhgwx+10mk6F+/fro168fli9fbo9+ERERETmUQqGASqWCXC5nsTyiMqhQYGEwGOzdDyIiIqqBjNXTdVqdo7tiRqlU4oWpCcjMyUOQny82J65ncEFUCruusSAiIiIqK9Pq6VeuXoVW5zzBhVqtRmZOHnxb9URmTh5UKpWju0Tk9Co0YpGQkFDmtitWrKjIXRAREVF1Z1I9XX/rO+j1Oui0Wrg5URV1H3kg8hzdCSIXUaHA4tSpUzh16hS0Wi1atWoFAPjjjz/g7u6OLl26SO1kMpl9eklERE7POB9dqVRWyvG1GuecMkO2MxbL0+Srcf3aNbh5eiNU53xBBhGVrEKBxeOPPw4/Pz989NFHCAwMBHCvaF5cXBx69uyJGTNm2LWTRETk3BQKBWLjJiEzJw/ebgI6rd6ux9dqNLiSehmL3lmF2wolWrfsYdfjk3PQawohZO7QGwzIV2eZBRlE5PwqtMZi+fLlWLx4sRRUAEBgYCD+85//MCsUEVENpFKppPnoWbn50BvsG1jo9ToYZH9PmTEIGOx8fHI+pkGGXs/AgsgVVCiwUKlUUCgURbYrFArk5OTY3CkiInJNPvLA0hvZwDhlhoiInE+FAosnn3wScXFx2LFjB65fv47r16/j//7v/zBx4kQMHTrU3n0kIiIiIiInV6E1FuvWrcPLL7+MUaNGQavV3juQhwcmTpyIZcuW2bWDRERERKaL9z1qVej0hYgqWYXemb6+vnj//fexbNkyXLp0CQBw//33o3bt2nbtHBEREZEmX222eL9Z06aO7hIRWWFTgbz09HSkp6ejRYsWqF27NoRgSjgiIiKyL72m0Gzxvr2TAxCRfVQosFAqlejfvz9atmyJQYMGIT09HQAwceJEppolIiKiSsHF+0TOrUKBxfTp01GrVi2kpaXB19dX2j58+HDs2bPHbp0jIiIiIiLXUKE1Ft9//z327t2LRo0amW1v0aIFrl69apeOERERERGR66jQiEVubq7ZSIVRZmYmvLy8bO4UERER8E8mIL2Oc+qJiJxdhQKLnj174uOPP5Z+l8lkMBgMWLp0Kfr27Wu3zlHlUSgUuHTpEpRKpaO7QkRklSYvR8oEdP3GDSYIIYlO+0/q2aqiVCpx6dIlqwWCieieCk2FWrp0Kfr3748TJ05Ao9Fg1qxZOHfuHDIzM/Hzzz/bu49kZwqFArFxk5CZkwdvNwGdllcCicj56DQFUiYgw41vAHcGFgQYDHpcv3atSlPPajUazJgzFwU6gSA/X2xOXI/69etX+v0SuZoKjViEh4fjjz/+wCOPPILBgwcjNzcXQ4cOxalTp3D//ffbu49kZyqVCpk5efBt1RNZuflM20dETkWn1UKv++dKNDMBkRmDAQa3qk09q9frkJ1bAN9WPZGZkweVSlXp90nkiso9YqHVajFw4ECsW7cOr732WmX0iaqIjzzQ0V0gIjKjycvB9WvX4ObpDXk4L3pQ8RwRcPrIA5FX5fdK5DrKHVjUqlULp0+froy+EBGRi7H3fHOdpgBC5g69wQADR1OpDByx3oKIrKvQVKjY2Fhs2LDB3n0hIiIXYlyvFRs3iYkgyCFM11tcuXoVWh2DCyJHqtDibZ1Oh40bN+KHH35AREQEateubbZ/xYoVdukcERE5L+N6LQBQq9UO7g3VSAYDDO5/r7e49R30egYWRI5UrsDi8uXLaNasGc6ePYsuXboAAP744w+zNjKZzH69IyIiIioFF/gTOYdyBRYtWrRAeno6fvzxRwDA8OHD8d///hchISGV0jkiIqoZdFot3JhOluyAryUixylXYGFZnOi7775Dbm6uXTtEREQ1CzNBkb1o8tV8LRE5UIUWbxuxCioREdmKmaDIXvSaQr6WiByoXIGFTCYrsoaCayqIiIiIiKjcU6HGjx8PLy8vAEBBQQEmT55cJCvUjh077NdDIiIiIidjrOFSv359B/eEyHmUa8Ri3LhxCA4Ohr+/P/z9/REbG4uGDRtKvxt/yuqnn37C448/joYNG0Imk+Grr74y2z9+/HhplMT4M3DgQLM2mZmZGD16NORyOQICAjBx4sQiaQ9Pnz6Nnj17wtvbG40bN8bSpUuL9GX79u1o3bo1vL290b59e+zevbvsfxgiIiKqMZRKpVTDxd5FIolcWblGLBITE+1657m5uejYsSMmTJiAoUOHWm0zcOBAs/s1jpYYjR49Gunp6UhKSoJWq0VcXByee+45bN26FcC9POsDBgxAVFQU1q1bhzNnzmDChAkICAjAc889BwA4cuQIRo4cicWLF+Nf//oXtm7diiFDhuC3335DeHi4XR8zERERuTa1Wi3VcFGpVBy1IPpbhQrk2ctjjz2Gxx57rMQ2Xl5eCA0NtbrvwoUL2LNnD3799Vd07doVAPDee+9h0KBBeOedd9CwYUNs2bIFGo0GGzduhKenJ9q1a4fk5GSsWLFCCixWrVqFgQMHYubMmQCARYsWISkpCatXr8a6devs+IiJiIiIiKonm7JCVYUDBw4gODgYrVq1wvPPPw+lUintO3r0KAICAqSgAgCioqLg5uaGY8eOSW169eoFT09PqU10dDRSUlJw9+5dqU1UVJTZ/UZHR+Po0aOV+dCIiIiIiKoNh45YlGbgwIEYOnQowsLCcOnSJbz66qt47LHHcPToUbi7uyMjIwPBwcFmt/Hw8EBQUBAyMjIAABkZGQgLCzNrYyzol5GRgcDAQGRkZBQp8hcSEiIdw5rCwkIUFhZKv6tUKpseKxERERGRK3PqwGLEiBHS/9u3b48OHTrg/vvvx4EDB9C/f38H9gxYvHgxFixY4NA+EBERUelYjZuoajj9VChTzZs3R7169fDXX38BAEJDQ3H79m2zNjqdDpmZmdK6jNDQUNy6dcusjfH30toUt7YDAObMmYPs7Gzp59q1a7Y9OCKiGkSr0eDmzZvQaXWO7gpVc8Zq3NdvpkOr4+uNqDK5VGBx/fp1KJVKNGjQAAAQGRmJrKwsnDx5Umqzf/9+GAwGdO/eXWrz008/QavVSm2SkpLQqlUrBAYGSm327dtndl9JSUmIjIwsti9eXl6Qy+VmP0REVDqtRoMrqZex6J1VuHL1KrQ6VkimymNajVuvZ2BBVJkcGlio1WokJycjOTkZAJCamork5GSkpaVBrVZj5syZ+OWXX3DlyhXs27cPgwcPxgMPPIDo6GgAQJs2bTBw4EA8++yzOH78OH7++WdMmTIFI0aMQMOGDQEAo0aNgqenJyZOnIhz585h27ZtWLVqFRISEqR+TJ06FXv27MHy5ctx8eJFzJ8/HydOnMCUKVOq/G9CRFTd6fU6GGQe8Gn+IPQGAYOBgQURUXXg0MDixIkT6Ny5Mzp37gwASEhIQOfOnTFv3jy4u7vj9OnTeOKJJ9CyZUtMnDgREREROHTokFktiy1btqB169bo378/Bg0ahEceeQQffvihtN/f3x/ff/89UlNTERERgRkzZmDevHlSqlkA6NGjB7Zu3YoPP/wQHTt2xBdffIGvvvqq2tWwUCgUuHTpkllmLSIiR/Gqw5FeIqLqxKGLt/v06QMhil9MtXfv3lKPERQUJBXDK06HDh1w6NChEts8/fTTePrpp0u9P1elUCgQGzcJmTl58HYT0Gl5hZCIiMgejNW3WSiPajqXWmNBFadSqZCZkwffVj2RlZsPPaceEFEFKRQK6USKyNXotFro7biIW6lUIjZuEmLjJvF9QTUeA4saxkce6OguEJELM45+xsZN4rRKcjmVkSFKrVYjMycPmTl5rGlFNZ5T17EgIiLnYhz9BO6dUBG5EmaIIqpcHLEgIiIiIiKbMbAgIqIqYSyKp2fdCiKiaomBBUlYCZeIKotpUbzrN26UmBGQiIhcEwMLAgBo8nLMKuEyuCAiezItimcwCAgwsCDH4wU1IvtiYEEAAJ2mwKwSLtPRElFlYFE8chaafDUvqBHZGQMLMsMvfSIiqgn0mkJeUCOyMwYWREREVGPxghqR/bCOBREREZGdKBQKqFQqyOVy1K9f39HdIapSDCyIiIiI7ECpVOKFqQnIzMlDkJ8vNieuZ3BBNQqnQlGJFAoFFAqFo7tBRC6KtSuoJlGr1cjMyYNvq57IzMmDSqVydJeIqhQDCyqWUqlEbNwkxMZNYnBBROVmmsaatSuoJvGRBzq6C0QOwcCCimW88sKrLkRUEaZprFm7gpydTsuaFkS2YmBBRESVill3yNkZDHpcv3aNNS2IbMTAgoiIiGo2gwEGN9a0ILIVs0IREVGJTNNnElVnHF0jsg0DCyIiKpZCoUBs3CQpfeaiea85uktEROSkOBWKiIiKpVKpzNJnqtVqR3eJiIicFEcsqhlW/CSiyuAjD0SeoztB5GKUSiUA8DuZagwGFtWI5ZQFVvwkIiJyDK1Ggxlz5qJAJ/idTDUGp0JVI5ZTFlh7gogcQafVQq9juk6q2fR6HbJzC/idTDUKA4tqiBU/ichRNHk5uH7tGq7fTIdWx5Sd5JrsWSyP38lUkzCwICIiu9FpCiBk7tAbDDCwFgC5IMtieVqOvhGVGQOLGo5TFohcn0KhwKVLl6BQKKzus7adiIphWSxPz+9IorLi4u0azDhlwc3TG/JwXlkkciXGDHAajQbTZr5iNWmDMaEDAC4cJSonFssjKj8GFjUYpywQuSbTDHDebgLqQj3k4X2RmXIIKpVKCiCMCR2M/2dgQURElYmBBRGRizHLAJe8FzI3d6nOhGnefFsYR0SMxyMiIioNA4tqjnOriaovy2wzlnnzF817rULHtRwR0Wk5oklERKXj4u1qTKlUIjZuEmLjJvGqI1E1UNpCbMu8+Wq1ukL3YzoikpWbDz2nShIRURlwxKIaU6vV0vzqip5gEJFzMF2IXdpIhHFalK2Yf5/oHp1WCzd34ehuEDk9BhZERC7AdCE2LxQQVR1NvlrKoBhqQ3p20/VPTKRA1RUDCyKiGsI4jYonNURlp9cUShkUK1rTwnL9E9M/U3XFNRZERE6qpMJ35WW65qq44xnvr7xrsrQaDW7evAmdloXEiKyxXP+kUqkc3SWiSsERCyIiJ2SamcmWDE9GpmuuLGtaKBQKpKamYvbc+VAXasuVCUqr0eBK6mUsemcVbiuUaN2yh039JKrO7LX+ichZccSCiMgJmdWqsCHDU2mMAczEKdORfO4iPO+PLFcmKL1eB4PMAz7NH4TeIFhsk4ioBmNgQUTkxCorM5Nx2lNqaioyc/KkwKBWbb8KHc+rjm0F+YiIyPVxKhQRUQ2jVCrxwtQEswJ4fgwMiIjIRhyxICKqYYzrLVgAj4iI7ImBBVml0zLLC1F1xwJ4RERkTwwsqAiDQY/r165h0TurcOXqVQYXREREf7NXemWFQmGXVNJEzoSBBRVlMMDg9k+WF06TICIiuleF25he2ZYLb2WpK0PkihhYULGY5YWIiOgfek2hWXrlil54M65zYrE8qm4YWBARERGVAy+8EVnn0MDip59+wuOPP46GDRtCJpPhq6++MtsvhMC8efPQoEED+Pj4ICoqCn/++adZm8zMTIwePRpyuRwBAQGYOHFikUJSp0+fRs+ePeHt7Y3GjRtj6dKlRfqyfft2tG7dGt7e3mjfvj12795t98dLRERERFRdOTSwyM3NRceOHbFmzRqr+5cuXYr//ve/WLduHY4dO4batWsjOjoaBQUFUpvRo0fj3LlzSEpKws6dO/HTTz/hueeek/arVCoMGDAATZs2xcmTJ7Fs2TLMnz8fH374odTmyJEjGDlyJCZOnIhTp05hyJAhGDJkCM6ePVt5D56IyApj4TqlUunorhAREZWLQwvkPfbYY3jssces7hNCYOXKlZg7dy4GDx4MAPj4448REhKCr776CiNGjMCFCxewZ88e/Prrr+jatSsA4L333sOgQYPwzjvvoGHDhtiyZQs0Gg02btwIT09PtGvXDsnJyVixYoUUgKxatQoDBw7EzJkzAQCLFi1CUlISVq9ejXXr1lXBX6Jq6bRauLkLR3eDiCwoFArExk0yK1znrIyZcfQ65+0jERFVLaddY5GamoqMjAxERUVJ2/z9/dG9e3ccPXoUAHD06FEEBARIQQUAREVFwc3NDceOHZPa9OrVC56enlKb6OhopKSk4O7du1Ib0/sxtjHeT3WiycvB9WvXcP1mOrQ8ISByKiqVyiUK12k1GikzzvUbNyAEL1RQzcSaT0TmnDawyMjIAACEhISYbQ8JCZH2ZWRkIDg42Gy/h4cHgoKCzNpYO4bpfRTXxrjfmsLCQqhUKrMfV6DTFEDI3KE3GGBw0pMWoprO2QvX6fU6KTOOwSAgwMCCah571nwyToFk6llydU4bWDi7xYsXw9/fX/pp3Lixo7tERFSlmBmHajQ71Xwy1rQYMWEy61qQy3PawCI0NBQAcOvWLbPtt27dkvaFhobi9u3bZvt1Oh0yMzPN2lg7hul9FNfGuN+aOXPmIDs7W/q5du1aeR8iERERuThbA2xjTQvfVj1Z14JcntMGFmFhYQgNDcW+ffukbSqVCseOHUNkZCQAIDIyEllZWTh58qTUZv/+/TAYDOjevbvU5qeffoJWq5XaJCUloVWrVggMDJTamN6PsY3xfqzx8vKCXC43+yEiIiKqCB95ILQaDTPCkUtzaGChVquRnJyM5ORkAPcWbCcnJyMtLQ0ymQzTpk3Df/7zH3zzzTc4c+YMxo4di4YNG2LIkCEAgDZt2mDgwIF49tlncfz4cfz888+YMmUKRowYgYYNGwIARo0aBU9PT0ycOBHnzp3Dtm3bsGrVKiQkJEj9mDp1Kvbs2YPly5fj4sWLmD9/Pk6cOIEpU6ZU9Z+EiIiIaiBNXg6upF7GjDlzOR2KXJZD082eOHECffv2lX43nuyPGzcOmzZtwqxZs5Cbm4vnnnsOWVlZeOSRR7Bnzx54e3tLt9myZQumTJmC/v37w83NDcOGDcN///tfab+/vz++//57xMfHIyIiAvXq1cO8efPMal306NEDW7duxdy5c/Hqq6+iRYsW+OqrrxAeHl4FfwUiIiKq6XSaAhhkHsjOLYBKpUL9+vUd3SWicnNoYNGnT58S0xTKZDIsXLgQCxcuLLZNUFAQtm7dWuL9dOjQAYcOHSqxzdNPP42nn3665A4TEREREZFVTrvGgoiIiMhVsKYFEQOLastYFZcfcERERJXLsqaFVsfvXqqZGFhUQ8YFYPYo2mOKBXyIKofxvcVsMEQuyrKmhZ6BBdVMDl1jQZXDuADMp/mD0N/6rsJFe0wplUq8MDUBmTl5CPLzxebE9VxYRmQHCoUCsXGTkJmTB283AZ3W9vcrETmGPYtGGi/i8buWXAlHLKoxe37AsYAPUeVQqVTSeysrN98uFwIqi3GKpV7nvH0kqg6M1bhZiZtcDQMLKhcfeaCju0BULTn7e8t0iuX1GzdKzOhHRLYxXszjhTxyNQwsiIioVKZTLA0GAQEGFkREZI5rLIiIqMzsOcWSiMpGoVBApVJBLpdzzQU5NQYWRERERE6KyVPIlXAqFBEREZGd6bRa6O1Qz4LJU8iVcMSiBmA1UCIioqqjyVfj+rVrcPP0RqidiuX5yAORZ5cjEVUejlhUc0WrgTJNJBERUWXSawohZO7QGwwslkc1CgOL6s6iGqjBiXPkExEREZHr4lSoGoKZXIiIiBxDp9XCzZ0pmqn6Y2BBRETF4gkRkW0qY70FkbNiYEFE5AAKhaJCt9NqzJMxmJ74m/7ftJ1HrYp91GvycqQTInk4p1ESVYS911solUoAYE0LckoMLIiIqphCoUBs3CQAwKJ5r5X5dlqNBldSL2PRO6twW6FEnfAs6cQ/SGX+f9N2jRo2qFCQodMUSCdEXJ9F5HhajQYz5sxFgU6wpgU5JS7eJiKqYiqVCpk5ecjMyYNarS61vXH0QVNYCIPsn2QMmoJc6cTf8v/GdlqdDlevXpEywzHtNJHr0ut1yM4tYE0LcloMLIiInJgmL0cafbh+4waEEGVOxuBVR14kM5y+DCMP9irsRUSVw0ceKP1foVBUeGolkb0xsCAicmI6TYE0+mAwCAhUbCF1WYMR40LT6zfTWfeGyMkplUrExk1CbNwkBhfkFBhYEBG5gKpKGW260JTrKojszzIBgy3UarU0rZLTosgZMLAgIqpBdFr7ndQQUflo8tXS1EaueaLqiIEFEVENYTDocf3aNemkRst1FERVSq8xT8BQljVPRK6EgQURUU1huZBbr+NCbSIHqKqpjURVjXUsqFScOkFU9Sqz4rXxpMa0IjAL4BG5PuMCbta2IEdhYEElMp06cVuhRFC3e8EFK38SVZ6qqnjNhdpEjmXPC3dKpRIvTE0AABbOI4dhYEElMxhgcP976sSt76DX61j5k6iSseI1UfVneeGudcseNh3PmCEKuFeEk9/L5AhcY0FlYjoflJU/iSqGhayISGKx5okXEag6YGBBFWZa+ZOISqZQKKRCVsaphEREXMhN1QmnQhERVQGVSiVNU1Cr1Q7uDRFVdwqFAiqVimshqUoxsCAiIiKqRowLuTNz8rgWkqoUp0IRERERVSPGhdxcC0lVjSMWRERERNWQjzwQeWCKeKo6DCyIiJxEZRbFI6KaiSniqSpxKhQRkRMwFsW7fjMdWh3TThLVVPYsmgcwRTxVLY5YEBE5ARbFIyLLonnNmja127E5LYqqAgMLIiIiImdgMMDg/nfRvFvfQW/niwycFkWVjVOhiIiIiJxIZRXN47QoqmwcsSAiqiSmBaqIiMrL3ustjIzTooB7n1MAOHJBdsHAguyCH0xE5hQKBWLjJkkFqhbNe83RXSIiF2K53iKom32DC+CfQnoAOC2K7IJTochmSqUSsXGTEBs3SQowiGo6lUplVqBKrVYXaaPVVM7VSCKqBgwGGNz+Xm9hENDr7f85YSykx2lRZC8MLMhm/GAiKp6PPNDqdq1Ggyupl7HonVW4cvUqU8wSkVWVtd6CqDIwsCAicgC9XgeD7J+rkUwxS0Sl0Wm10Osqb4RToVDg0qVLnH1AFcY1FkREDsSrkURUFpp8Na5fuwY3T2+EVkJwYVxvYVwXxjUXVBEcsSAiIiJycnpNoVREszLXWxjXhaWmpnL0gsrNqQOL+fPnQyaTmf20bt1a2l9QUID4+HjUrVsXderUwbBhw3Dr1i2zY6SlpSEmJga+vr4IDg7GzJkzobOI9A8cOIAuXbrAy8sLDzzwADZt2lQVD89lVfZQLBERETmGjzxQKqQ3YsJkJmahcnHqwAIA2rVrh/T0dOnn8OHD0r7p06fj22+/xfbt23Hw4EHcvHkTQ4cOlfbr9XrExMRAo9HgyJEj+Oijj7Bp0ybMmzdPapOamoqYmBj07dsXycnJmDZtGiZNmoS9e/dW6eN0Fcah2Os306FlcEFUhHGOslKpdHRXiKgaq8yscpaF9Dh6QWXl9GssPDw8EBoaWmR7dnY2NmzYgK1bt6Jfv34AgMTERLRp0wa//PILHnroIXz//fc4f/48fvjhB4SEhKBTp05YtGgRZs+ejfnz58PT0xPr1q1DWFgYli9fDgBo06YNDh8+jHfffRfR0dFV+lhdQWUPxRK5MtPaFd5uAjpt0QXZxpMBPbNAEVEFafLVUla52wolmjVtWin34yMPRPbfoxcFOiGtvTDiGgyy5PQjFn/++ScaNmyI5s2bY/To0UhLSwMAnDx5ElqtFlFRUVLb1q1bo0mTJjh69CgA4OjRo2jfvj1CQkKkNtHR0VCpVDh37pzUxvQYxjbGYxSnsLAQKpXK7IeIajbT2hVZufnQW2R6Mk0xe/3GDQghHNRTInJlek2hWVY5y88au96XldEL1q6i4jh1YNG9e3ds2rQJe/bswdq1a5GamoqePXsiJycHGRkZ8PT0REBAgNltQkJCkJGRAQDIyMgwCyqM+437SmqjUqmQn59fbN8WL14Mf39/6adx48a2PlwiqiaKq11hmmLWYBAQYGBBRBVnzCqn01Z+sU3j55pl7SqFQoELFy5U2lQppsB1LU49Feqxxx6T/t+hQwd0794dTZs2xeeffw4fHx8H9gyYM2cOEhISpN9VKhWDC3JKCoUCKpUKcrnc6rB1SfuNH+Qc7rYvppglInsxGPS4fu1apU+LskapVGLS5HicPX8RjZqGIThIbtc0tabTS5kC1zU4dWBhKSAgAC1btsRff/2FRx99FBqNBllZWWajFrdu3ZLWZISGhuL48eNmxzBmjTJtY5lJ6tatW5DL5SUGL15eXvDy8rLHwyKyO9MrO9Y+lEvbbzxGbNwkAOCHeRkoFApcvXq1Uq8YEhEVYTDA4P73tKhb31XqtChLarUad7JzUGiQwatlD2SmHjebGm78vinp4lZJTKeXZqYcQmpqqtmxePHL+bhUYKFWq3Hp0iWMGTMGERERqFWrFvbt24dhw4YBAFJSUpCWlobIyEgAQGRkJN58803cvn0bwcHBAICkpCTI5XK0bdtWarN7926z+0lKSpKOQeRqTAOCRfNekz6Ub53Zh9TUVACwut/yQ9v4gQ7c+3DnB3fxjH/zm7fv4PqNdAR1Y3BBRFXL0SOh3n4ByMU/hfYAYOWytzFt5ivSxauVy96Gp6cn5PJ7fS1rwGFtEbnx2ED1v/jlSgGUUwcWL7/8Mh5//HE0bdoUN2/exBtvvAF3d3eMHDkS/v7+mDhxIhISEhAUFAS5XI4XX3wRkZGReOihhwAAAwYMQNu2bTFmzBgsXboUGRkZmDt3LuLj46XRhsmTJ2P16tWYNWsWJkyYgP379+Pzzz/Hrl27HPnQXZorvQGqC9MrQqYBgVqtBgC4e3jgSuplzJgzF8sX/6fIfmsf2ovmveaYB+OCjH9zn+YPQn/ta2ZMIyKHqYr1FiUxrsEAgPT0dLOLW89NmYoCnUAdT3dA5gZ1obbMmabuLSIXCOwQhcyUQ9KxARQZJXFlludQrjZ7wKkDi+vXr2PkyJFQKpWoX78+HnnkEfzyyy/SH/Xdd9+Fm5sbhg0bhsLCQkRHR+P999+Xbu/u7o6dO3fi+eefR2RkJGrXro1x48Zh4cKFUpuwsDDs2rUL06dPx6pVq9CoUSOsX7+eqWYryPRKhSu8AVyZMZjQaDRmV4SsBQQ6TQEMMg9k5xZIwYQlyw9ty3amH3a2DG1XZ95cO0FEDmS53sJZRk995IHINPmOuZO8FzI3d+n7JjU1Fa8vfBNA6ecOPvJA5Flsqy7nHpZBBACcOXMGtzNV8Kjl4RKzB5w6sPjss89K3O/t7Y01a9ZgzZo1xbZp2rRpkalOlvr06YNTp05VqI9kzvRKhSu8AVyNtWDC201AXaiHPLyv1YCgvEr70LYc2nblD3EiomrFcr2FXgedVgs3d+fJQGeaNc/4fWN57lBe1m5vvPhl+n9n/64ynXFgDLaMU2yrclG+LZw6sCCif1gWX5OCib+v/FgLCOyluKHtzJRDDCDLgEXxiKgqGddbaPLVuH7tGtw8vRGqc47Ri9KYXsiqyJRc4+0zc/KKTLkyXePhyO8ty1F/a6l0jd+70hTbKlyUbwsGFmQT4wmTTquDRy2+nCqTWXYMk2DCUYyBjFKpBACHf1A7gvHLwfg3sEaTlyMVxUtPz4CQuVdhD4moJtNrCiFk7tAbDC6z9sv0QlZFRuCNt/dt1dNsypXlGo8lby5EWFhYpX5vWZs2bJlC13QRurVAytWm2PJMkCpMk6+WTpiqOnd2TWJ5JcORwYQlrZUsHc5wNagyFTcdTae1fjXJuL7Fp/mDMNz4BnCiKQlEVLM427SoymQ55cq4xqN2i0icStqKSS+9jAb1Akv83irL2sKStltL526ZQtd0EbqtU5mdAQMLqjC9plA6Yarq3Nk1hWXqWGdjuuDb9GpQdV17Udp0tJI4OhUkEdVsltOiauqMg1peXjDIPODbuidupRwu8r1lyvj9W1zaXMvkKabfe8XV4DCOcFfm9GVHqjmvJKo0PGGqPNZSxzojy4wf1XXthbNNRyMiKivTaVH56qwaP+PA2y8AuRbfW6bZqYx1noDi0+ZaJk+xtvDcMp17SSPc1YGboztARNVLTTjRrgmPkYiqL7MZBwaBwsJ8h9a+cDTjZ7pxfUZmTp7Vi3k+8sC/R+oL4NuqJ7Jy86E36KXbK5VKxMZNQmzcJLO1d9ZuU11xxILICVnLEOGKnL1YYlnrcVTk+ahJc5mJyDV51ZFbrX3Bz6/SWbvAVNrC85pwUYqBBZETMD3BBeDU6yrKypkKFlkuwDOytrDO8jbGdkDZnw/Tuczy8Op7ZYqIqgGL2hf56iyXS1FLzoOBBZGDWWaOMJ3X6czrKkpTUsGiyg4yigvULNP6WVtYZy24K+/zYTqX2VCNh7yJqPowrpcsKUVtTV3wTWXHVwXZjU5b9AOnrFNNajLLzBGuHEwUx7RgUWWlpbWWBtYyULOW1s9yYV11Cu6IiGxlnBZlWpOnpi74ptIxsCC7sJyj2axp0yInk46eDuNsLIurVdfUc4B5wSJ7pqUtrqaEaZaOsgQGeovMIAwmiIgspnXm5ZqlmDcu+NbrOCpL/2BgQfZhMUdTb9CbnUxW1/SjFWVZD8HZUs9V1nC3tbS0plOQTNdAGP9/584ds9ENa2skrNWUqEigVp2DOyKi8rI2rdNywXd6egaE7F4dH06VIj7r1YhSqXR4qjhrNS14slaUtXoIVcF0upoQBrP0glU93G1tCpLpGoiVy97GCy9Nx9nzF9GoaRiCg+TFr5FgTQkioqpjcjHRcOMbwF1Ak682++5o1LCBXVPYmmaqYgDjvFjHoppQKBSYMftVXLl6FVoOSzothUJhdsXdHifCOq0WeiuZO0w/eAHz6WqXL19G6qVLWPTOKly5ehW5qntZQK7fTEe+6XB3Jec3N83tnZmTJ62BMP7/TnYOCg0yeLXsUWS/6RoJRyvuOSAiqs5MLyaa1sbQ6nS4evWK9B2jtfHzUZOXI31H5ar+Ke535epVFOTl1egaHM6GgUU1oVKp/i66IpiFxkkZpz9ZFs6xhemHrVank4KJgrw8sw9erU5/7wqT298Bg14PvZu7FDxoCnJLHe6+cvVqpX1wlxYcePsFVMr92oP5c8D3HhHVbF515ObfNwYBvV5ndgHG9P+mF8GK+79OUyB9R2kKcosNYBhkOB7Hj4iqiHH6E2B7piHjkLDph22++p+rOMY5r8Y1L6bBpukVJmtT18xYrJ2p6dVZrTF9DhjUExH9w/gdY7oIPEiVZfZ/4/dWRsZtwKAv8v/bCiVat+xh/dgm31Ha9J1SkOGKhf5MAynLqcquhCMWRJXMcvqTrYq7Qm46DG0wCAiI0gOHMrI2eqHV6TgFiIiISmW6CNx0hNx09MFsJN1iVL20izbWRkmMhf6MI/rWOOI7zNpojeksA8upyq42Es7AgqgS2Wv6U3FDwtY+bO0VTBRRyod2TQsyatrjJSKqLMWNpFfk+6y4Qn+Wn9nGURTLqcQlTcmyVNY1jsZ2xa0VuZqWBj0qFlQ5G06Fokphmn2oJrPH9CetRmOWacPakHBVsvahbQwy3Dy9EWrxAV0dM3YYvxzcPL0hD3etD30ioprGrB7H35/Zlt9hZZmSZTq9yvR7INRijeO1a2nSbeqEZ5nVArE2WmPMrGVrUOUMOGJBdmdtygyVn/FDSlNYaJalyRmvXhT3AV2ZC76rWnlGjYiIyHlYq8dhub8sU7JMR+rzTYIE0+89s9GHYpKjmHLVAKI4DCzI/qxkg6hJFAoFLl26VKF1FdbmW16/cQNC2G+9RGUz+4Cu5HS11pR1aLo8TEeNXHHOKxERla60KVnFBSiVucbR1VS/OQrkNGrSm0qhUEClUkGj0WDazFeQmZOHID9fLJr3WpmPYVqYzjSrk3GI1NVYLvg2DiNXprIOTVsWbiqu8JIxM4fZqJFFli0iIiKgZp33FIeBBVUZ4+JluVyO+vXrO7g3FWcaRHh6epoFE95uAupCPeThfZGZcqhM6ypMU8eWNN/SJVmkq62s0avypN+1TEloOv+1uNSHxtu7/PNBRERUiRhYUJXQajSYMWcuCnQCQX6+2Jy43qWCC8sRiVvKLNxIu4pGTcNQx8v9n2AieS9kbu7wkQcirwzHtbYIuDqevJo+prLkFTeOGuhNphwVl+O7uIXUZkPTpoGaRbBTbOrDG98A7q49akRERFSVGFhQldDrdcjOFQjsEIXMlENQqVROH1iYrpGIjZtkNiLh2bQzCq+kwatlD2SdPyAFE2VRkxcBm2bmCNXprE5BMp22ZBwp0OSrrY4k3FYo4d2yYgvjSgrgqkNmDiIioqrGxdtUpcp68u1opvUnUlNTkZmTB99WPZGVmw+9QQ/vv082vf0CynQ8a4uya+IiYGspakvK521cBFdsxg4nzZJFRERUEzGwoCrhKsXEjFWyjfUnMnPypHUSFQ2KTDMKWaahq8knxZaVWMuSUYMjCURERM6LgQVVOsvqls7KXlWyLen1OqahKyP+XYiIiFwX11hQpTO9Mm3MCuTIDFHGEQm5/N5JrPH/9qiSbW3NgHEBMk+aiYiIqDpjYEFVzpEZooyjEpk5eajj6Q7I3KAu1Ja75oQ1ptmJTNOWGhcgExEREVVnnApFVe5ehqgC+LbqicycPKhUqiq7b+OohG+rnriTnQOlSi31o6w1J4pbK2Ka4cnamgEiIiKi6oyBBTmMIzNEmd53WfthHJG4t1bkn0XXJQUbnP5ERERENQWnQlGNYFxXUZZF2cUVYrNWc8K0LoNpcTYiIiKimoaBBVU5ywXOV69erdRF3KbrKrzdBHTa4gMA09SwloXYWrfsUaS96cL0mpw6loiIiIhToahKmaaeNRZFm/bqfMTGTTKrdG0PljUpTAvcWTKOUmgKWYiNiIiIqCIYWFCVslYUzbe1/RdxW6tJ4SMPNFsPYfy/Ji9HGqW4fuMGhBAsxEZERERUTpwKRQ7n7ReAXDscx7Q+hbWaFJbpYKW1EXkmGZxufAO4M4MTERERUXkxsCCnUZGiecZgQqPRYNrMV5CZk1dsTQrLdLCWayM4MkFERERUcQwsyClYK5pnVL9+fWn9hfH/lsGEt5uAulAPeXhfZKYckkYpTDM8EREREVHlYWBBTqGwIB9arRb1ugxEZsohpKam4vWFbwIAVi57G9NmvmL2/yLBRPJeyNzc4SMPRN7fxzTN8FRcViciIiIisg8u3iaHM80U5eHrB+DeuojMnDxk5uQhPT29yP9NMzxZFrizmuGJWZ2IiIiIKhVHLMjhTDNF6fW6YgvUedT65+VqzPBkrIdh/L9phqf09AwImTvXThARERFVAQYW5FQ0+epiC9Q1athACjKY4YmIiIjIuXAqFDkVvcZ6gTqtToerV69g0TurcOXqVeTn5TLDExEREZETYWBhYc2aNWjWrBm8vb3RvXt3HD9+3NFdqpGKFKgzGGBw43oJIiIiImfFwMLEtm3bkJCQgDfeeAO//fYbOnbsiOjoaNy+fdvRXaO/cSSCiIiIyDkxsDCxYsUKPPvss4iLi0Pbtm2xbt06+Pr6YuPGjY7uGhERERGRU2Ng8TeNRoOTJ08iKipK2ubm5oaoqCgcPXrUgT0jIiIiInJ+zAr1tzt37kCv1yMkJMRse0hICC5evFikfWFhIQoLC6Xfs7OzAQAqlapyO1qMnJwc6PU6CIMBeXfvmP0LmazEbbbu5zGr5zGdpR88Jo9Z3frBY/KY1a0fPGblH1Ov0yEnJ8ch55nG+xSi9EybMlGWVjXAzZs3cd999+HIkSOIjIyUts+aNQsHDx7EsWPHzNrPnz8fCxYsqOpuEhERERFVuWvXrqFRo0YltuGIxd/q1asHd3d33Lp1y2z7rVu3EBoaWqT9nDlzkJCQIP1uMBiQmZmJunXrQiaTWb0PlUqFxo0b49q1a5DL5fZ9AOT0+PzXbHz+ia+Bmo3PP7nqa0AIgZycHDRs2LDUtgws/ubp6YmIiAjs27cPQ4YMAXAvWNi3bx+mTJlSpL2Xlxe8vLzMtgUEBJTpvuRyuUu9oMi++PzXbHz+ia+Bmo3PP7nia8Df379M7RhYmEhISMC4cePQtWtXdOvWDStXrkRubi7i4uIc3TUiIiIiIqfGwMLE8OHDoVAoMG/ePGRkZKBTp07Ys2dPkQXdRERERERkjoGFhSlTplid+mQPXl5eeOONN4pMoaKagc9/zcbnn/gaqNn4/FNNeA0wKxQREREREdmMBfKIiIiIiMhmDCyIiIiIiMhmDCyIiIiIiMhmDCyIiIiIiMhmDCyqyJo1a9CsWTN4e3uje/fuOH78uKO7RHawePFiPPjgg/Dz80NwcDCGDBmClJQUszYFBQWIj49H3bp1UadOHQwbNqxIhfe0tDTExMTA19cXwcHBmDlzJnQ6XVU+FLKDt99+GzKZDNOmTZO28fmv/m7cuIHY2FjUrVsXPj4+aN++PU6cOCHtF0Jg3rx5aNCgAXx8fBAVFYU///zT7BiZmZkYPXo05HI5AgICMHHiRKjV6qp+KFROer0er7/+OsLCwuDj44P7778fixYtgmleHD7/1ctPP/2Exx9/HA0bNoRMJsNXX31ltt9ez/fp06fRs2dPeHt7o3Hjxli6dGllPzT7EFTpPvvsM+Hp6Sk2btwozp07J5599lkREBAgbt265eiukY2io6NFYmKiOHv2rEhOThaDBg0STZo0EWq1WmozefJk0bhxY7Fv3z5x4sQJ8dBDD4kePXpI+3U6nQgPDxdRUVHi1KlTYvfu3aJevXpizpw5jnhIVEHHjx8XzZo1Ex06dBBTp06VtvP5r94yMzNF06ZNxfjx48WxY8fE5cuXxd69e8Vff/0ltXn77beFv7+/+Oqrr8Tvv/8unnjiCREWFiby8/OlNgMHDhQdO3YUv/zyizh06JB44IEHxMiRIx3xkKgc3nzzTVG3bl2xc+dOkZqaKrZv3y7q1KkjVq1aJbXh81+97N69W7z22mtix44dAoD48ssvzfbb4/nOzs4WISEhYvTo0eLs2bPi008/FT4+PuKDDz6oqodZYQwsqkC3bt1EfHy89LterxcNGzYUixcvdmCvqDLcvn1bABAHDx4UQgiRlZUlatWqJbZv3y61uXDhggAgjh49KoS49yHl5uYmMjIypDZr164VcrlcFBYWVu0DoArJyckRLVq0EElJSaJ3795SYMHnv/qbPXu2eOSRR4rdbzAYRGhoqFi2bJm0LSsrS3h5eYlPP/1UCCHE+fPnBQDx66+/Sm2+++47IZPJxI0bNyqv82SzmJgYMWHCBLNtQ4cOFaNHjxZC8Pmv7iwDC3s93++//74IDAw0+w6YPXu2aNWqVSU/IttxKlQl02g0OHnyJKKioqRtbm5uiIqKwtGjRx3YM6oM2dnZAICgoCAAwMmTJ6HVas2e/9atW6NJkybS83/06FG0b9/erMJ7dHQ0VCoVzp07V4W9p4qKj49HTEyM2fMM8PmvCb755ht07doVTz/9NIKDg9G5c2f873//k/anpqYiIyPD7DXg7++P7t27m70GAgIC0LVrV6lNVFQU3NzccOzYsap7MFRuPXr0wL59+/DHH38AAH7//XccPnwYjz32GAA+/zWNvZ7vo0ePolevXvD09JTaREdHIyUlBXfv3q2iR1MxrLxdye7cuQO9Xm920gAAISEhuHjxooN6RZXBYDBg2rRpePjhhxEeHg4AyMjIgKenJwICAszahoSEICMjQ2pj7fVh3EfO7bPPPsNvv/2GX3/9tcg+Pv/V3+XLl7F27VokJCTg1Vdfxa+//oqXXnoJnp6eGDdunPQcWnuOTV8DwcHBZvs9PDwQFBTE14CTe+WVV6BSqdC6dWu4u7tDr9fjzTffxOjRowGAz38NY6/nOyMjA2FhYUWOYdwXGBhYKf23BwYWRHYSHx+Ps2fP4vDhw47uClWRa9euYerUqUhKSoK3t7eju0MOYDAY0LVrV7z11lsAgM6dO+Ps2bNYt24dxo0b5+DeUWX7/PPPsWXLFmzduhXt2rVDcnIypk2bhoYNG/L5pxqJU6EqWb169eDu7l4kC8ytW7cQGhrqoF6RvU2ZMgU7d+7Ejz/+iEaNGknbQ0NDodFokJWVZdbe9PkPDQ21+vow7iPndfLkSdy+fRtdunSBh4cHPDw8cPDgQfz3v/+Fh4cHQkJC+PxXcw0aNEDbtm3NtrVp0wZpaWkA/nkOS/oOCA0Nxe3bt83263Q6ZGZm8jXg5GbOnIlXXnkFI0aMQPv27TFmzBhMnz4dixcvBsDnv6ax1/Ptyt8LDCwqmaenJyIiIrBv3z5pm8FgwL59+xAZGenAnpE9CCEwZcoUfPnll9i/f3+RocuIiAjUqlXL7PlPSUlBWlqa9PxHRkbizJkzZh80SUlJkMvlRU5YyLn0798fZ86cQXJysvTTtWtXjB49Wvo/n//q7eGHHy6SYvqPP/5A06ZNAQBhYWEIDQ01ew2oVCocO3bM7DWQlZWFkydPSm32798Pg8GA7t27V8GjoIrKy8uDm5v5qZS7uzsMBgMAPv81jb2e78jISPz000/QarVSm6SkJLRq1cqpp0EBYLrZqvDZZ58JLy8vsWnTJnH+/Hnx3HPPiYCAALMsMOSann/+eeHv7y8OHDgg0tPTpZ+8vDypzeTJk0WTJk3E/v37xYkTJ0RkZKSIjIyU9hvTjQ4YMEAkJyeLPXv2iPr16zPdqIsyzQolBJ//6u748ePCw8NDvPnmm+LPP/8UW7ZsEb6+vmLz5s1Sm7ffflsEBASIr7/+Wpw+fVoMHjzYavrJzp07i2PHjonDhw+LFi1aMN2oCxg3bpy47777pHSzO3bsEPXq1ROzZs2S2vD5r15ycnLEqVOnxKlTpwQAsWLFCnHq1Clx9epVIYR9nu+srCwREhIixowZI86ePSs+++wz4evry3Sz9I/33ntPNGnSRHh6eopu3bqJX375xdFdIjsAYPUnMTFRapOfny9eeOEFERgYKHx9fcWTTz4p0tPTzY5z5coV8dhjjwkfHx9Rr149MWPGDKHVaqv40ZA9WAYWfP6rv2+//VaEh4cLLy8v0bp1a/Hhhx+a7TcYDOL1118XISEhwsvLS/Tv31+kpKSYtVEqlWLkyJGiTp06Qi6Xi7i4OJGTk1OVD4MqQKVSialTp4omTZoIb29v0bx5c/Haa6+ZpQnl81+9/Pjjj1a/98eNGyeEsN/z/fvvv4tHHnlEeHl5ifvuu0+8/fbbVfUQbSITwqQ8JBERERERUQVwjQUREREREdmMgQUREREREdmMgQUREREREdmMgQUREREREdmMgQUREREREdmMgQUREREREdmMgQUREREREdmMgQUREbmUPn36YNq0aY7uBhERWWBgQUREVebxxx/HwIEDre47dOgQZDIZTp8+XcW9IiIie2BgQUREVWbixIlISkrC9evXi+xLTExE165d0aFDBwf0jIiIbMXAgoiIqsy//vUv1K9fH5s2bTLbrlarsX37dgwZMgQjR47EfffdB19fX7Rv3x6ffvppiceUyWT46quvzLYFBASY3ce1a9fwzDPPICAgAEFBQRg8eDCuXLlinwdFREQAGFgQEVEV8vDwwNixY7Fp0yYIIaTt27dvh16vR2xsLCIiIrBr1y6cPXsWzz33HMaMGYPjx49X+D61Wi2io6Ph5+eHQ4cO4eeff0adOnUwcOBAaDQaezwsIiICAwsiIqpiEyZMwKVLl3Dw4EFpW2JiIoYNG4amTZvi5ZdfRqdOndC8eXO8+OKLGDhwID7//PMK39+2bdtgMBiwfv16tG/fHm3atEFiYiLS0tJw4MABOzwiIiICGFgQEVEVa926NXr06IGNGzcCAP766y8cOnQIEydOhF6vx6JFi9C+fXsEBQWhTp062Lt3L9LS0ip8f7///jv++usv+Pn5oU6dOqhTpw6CgoJQUFCAS5cu2ethERHVeB6O7gAREdU8EydOxIsvvog1a9YgMTER999/P3r37o0lS5Zg1apVWLlyJdq3b4/atWtj2rRpJU5ZkslkZtOqgHvTn4zUajUiIiKwZcuWIretX7++/R4UEVENx8CCiIiq3DPPPIOpU6di69at+Pjjj/H8889DJpPh559/xuDBgxEbGwsAMBgM+OOPP9C2bdtij1W/fn2kp6dLv//555/Iy8uTfu/SpQu2bduG4OBgyOXyyntQREQ1HKdCERFRlatTpw6GDx+OOXPmID09HePHjwcAtGjRAklJSThy5AguXLiAf//737h161aJx+rXrx9Wr16NU6dO4cSJE5g8eTJq1aol7R89ejTq1auHwYMH49ChQ0hNTcWBAwfw0ksvWU17S0REFcPAgoiIHGLixIm4e/cuoqOj0bBhQwDA3Llz0aVLF0RHR6NPnz4IDQ3FkCFDSjzO8uXL0bhxY/Ts2ROjRo3Cyy+/DF9fX2m/r68vfvrpJzRp0gRDhw5FmzZtMHHiRBQUFHAEg4jIjmTCcmIqERERERFROXHEgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgoiIiIiIbMbAgsjFNGvWDOPHj3d0N6q9ZcuWoXnz5nB3d0enTp2KbdesWTPIZDLIZDJMmTKl6jpIxTp//jw8PDxw9uzZct1u6dKlaN26NQwGQyX1jMgxKvqeICovBhZEDrRp0ybIZDKcOHHC6v4+ffogPDzc5vvZvXs35s+fb/Nxaorvv/8es2bNwsMPP4zExES89dZbJbbv2bMnPvnkE4wbN85s+9q1a/H000+jSZMmkMlkJQaEWVlZeO6551C/fn3Url0bffv2xW+//WbWRqlUYtmyZejVqxfq16+PgIAAPPTQQ9i2bVupj+nNN9+ETCaz6fV08eJFzJo1C506dYKfnx8aNGiAmJiYYl+/N27cwDPPPIOAgADI5XIMHjwYly9fLvE+Dh8+LAVqd+7cKbL/hx9+QN++fVGvXj0EBASgW7du+OSTT8zatG3bFjExMZg3b16ZH5tKpcKSJUswe/ZsuLn989Vo7ItMJoOHhweCgoIQERGBqVOn4vz582U+vqW8vDzMnz8fBw4cqPAxSqJSqbBgwQJ07NgRderUgY+PD8LDwzF79mzcvHmzUu6zvOz1ubRt2zbExsaiRYsWkMlk6NOnT7FtCwsLMXv2bDRs2BA+Pj7o3r07kpKSirQzGAxYt24dOnXqhDp16iAkJASPPfYYjhw5YtbuwIEDZq8R059ffvlFapeXl4c1a9ZgwIABaNCgAfz8/NC5c2esXbsWer2+yP2/+eabeOKJJxASEgKZTFbi36my3hNEFSKIyGESExMFAPHrr79a3d+7d2/Rrl07s20FBQVCo9GU637i4+MF3+5lN3v2bOHm5iYKCwtLbdu0aVMxbty4YvcFBQWJgQMHCg8Pj2Lb6fV60aNHD1G7dm0xf/58sXr1atG2bVvh5+cn/vjjD6ndt99+K2rVqiUGDx4sVq5cKVavXi369u0rAIh58+YV28dr164JX19fUbt27SKvp/KYMWOGCAgIEBMnThQffPCBWLp0qbj//vuFu7u7SEpKMmubk5MjWrRoIYKDg8WSJUvEihUrROPGjUWjRo3EnTt3iv07dOrUSdSuXVsAEAqFwmz/119/LWQymejRo4d47733xOrVq0WvXr0EALFixQqztrt37xYAxF9//VWmx/buu+8KuVwu8vPzzbYDEI8++qj45JNPxMcffyzee+89MWnSJOHv7y88PDzE8uXLy3R8SwqFQgAQb7zxRoVuX5JLly6JsLAw4e7uLkaMGCFWr14tPvzwQzFlyhRRt25d0aJFC7vfZ0XY63Opd+/eok6dOqJv374iMDBQ9O7du9i2I0aMEB4eHuLll18WH3zwgYiMjBQeHh7i0KFDZu0SEhIEABEbGys++OADsWTJEtG8eXPh4eEhjh07JrX78ccfBQDx0ksviU8++cTsx/T1e+bMGSGTyURUVJRYunSpWLdunXjyyScFADF27Ngi/QQgQkNDRXR0dImvk8p8TxBVBM80iByoIoFFRTg6sFCr1Q6774qIi4sTtWvXLlPbkgKLK1euCIPBIIQQonbt2sW227ZtmwAgtm/fLm27ffu2CAgIECNHjpS2Xb58WVy5csXstgaDQfTr1094eXkV+3cePny46Nevn82vpxMnToicnByzbXfu3BH169cXDz/8sNn2JUuWCADi+PHj0rYLFy4Id3d3MWfOHKvHX7t2rahbt66YOnWq1cDi0UcfFQ0bNhQFBQXSNq1WK+6//37RoUMHs7YajUYEBgaK119/vUyPrUOHDiI2NrbIdgAiPj6+yPY7d+6IyMhIAUDs2rWrTPdhqrICC61WKzp27Ch8fX2LnCwLIUR2drZ49dVX7XqfFWWvz6W0tDSh/3/27jwuqur/H/hr2BcZ9kUSEXcF3DANS8UyUckld80dMwv3ND+aGmrlllsukR8TKjVNM+uTZuJuiZYLGu4ioCkgiTKyCAOc3x/+5n65zLAOO6/n4zGPmnPPPffcuXPxvudsOTlCCCE8PT0LDCzOnj0rAIiVK1dKaRkZGaJRo0bC19dXSlOr1cLc3FwMGjRItv+dO3ekIEJDE1jkvXd1SUpKElFRUVrp48aNEwDErVu3ZOkxMTHSfoV9T8rzniAqDXaFIqpm8o+xUKvVWLRoEZo0aQIzMzPY29vjlVdekZr3x44di40bNwKQd+vQSEtLw/vvvw83NzeYmpqiWbNm+OyzzyCEkB03IyMDU6dOhYODA6ysrNC3b1/cv39fq5k+ODgYCoUCV69exYgRI2Bra4tXXnkFAHD58mWMHTsWDRs2hJmZGVxcXDB+/Hg8evRIdixNGTdv3sTIkSNhbW0NR0dHLFiwAEII3Lt3D/369YNSqYSLiwtWrVpVrM8uOzsbS5YsQaNGjWBqaooGDRpg3rx5yMzMlPIoFAqEhoYiLS1N+qzCwsKKVX5+7u7uss+6IHv27IGzszMGDBggpTk6OmLIkCH46aefpPp5eHjA3d1dtq9CoUD//v2RmZmps5vRyZMnsWfPHqxdu7ZU55CXj48P6tSpI0uzt7dH586dce3aNa1zevHFF/Hiiy9Kac2bN8drr72G77//Xqvs5ORkzJ8/H4sXL4aNjY3O46tUKtja2sLU1FRKMzIygoODA8zNzWV5jY2N4efnh59++qnI84qJicHly5fRvXv3IvNq2NvbY+fOnTAyMsInn3wipWdlZWHhwoXw8fGBtbU1LC0t0blzZxw7dkzKExsbC0dHRwDAokWLpO+Z5j4q7n2iyw8//IBLly7hww8/lO67vJRKpay+ALB79274+PjA3NwcDg4OGDlyJO7fvy/L4+fnp7OL0dixY9GgQQPZuSkUCnz22WfYvHmzdK+9+OKL+Ouvv2T7FfZ3KT4+HtevX4darS7ynN3c3GTd1wqyZ88eGBoaYuLEiVKamZkZAgMDERERgXv37gF4/jc1IyMDzs7Osv2dnJxgYGCg9V3TePr0KbKzs3Vuc3BwgKenp1b6m2++CQBa90/ez7Qw5XVPEJUWAwuiKiAlJQX//vuv1qs4/6gGBwdj0aJF6NatGzZs2IAPP/wQ9evXl/rnv/POO3j99dcBAN9++630AgAhBPr27Ys1a9agZ8+eWL16NZo1a4bZs2dj5syZsuOMHTsW69evR+/evbF8+XKYm5sjICCgwHoNHjwY6enp+PTTT/H2228DAMLDw3Hnzh2MGzcO69evx7Bhw7Bz50707t1bK5ABgKFDhyI3NxfLli1Dx44d8fHHH2Pt2rV4/fXX8cILL2D58uVo3LgxZs2ahZMnTxb5WU2YMAELFy5Eu3btsGbNGnTt2hVLly7FsGHDpDzffvstOnfuDFNTU+mz6tKlS5Fl6+PixYto166d1sNRhw4dkJ6ejps3bxa6f0JCAoDnDy955eTkYMqUKZgwYQK8vb3LttL5jp/32Lm5ubh8+TLat2+vlbdDhw6Ijo7G06dPZekLFiyAi4sL3nnnnQKP4+fnhytXrmDBggW4ffs2oqOjsWTJEpw7dw4ffPCBVn4fHx9ERUVBpVIVWn9Nv/l27doVmi+/+vXro2vXrjhz5ox0DJVKhS1btsDPzw/Lly9HcHAwkpKS4O/vj8jISADPg8YvvvgCwPMHS833TBNYlvQ+yevnn38GAIwaNapY5xAWFoYhQ4bA0NAQS5cuxdtvv429e/filVdewZMnT0r0eeS1Y8cOrFy5Eu+88w4+/vhjxMbGYsCAAdLftML+LgHA3Llz0aJFC60ARx8XL15E06ZNoVQqZekdOnQAAOn6aMZehIWFYfv27bh7964U7Nna2soCE41x48ZBqVTCzMwM3bp1K3DcUX4F3bvFVV73BFGpVWp7CVEtp+kKVdgrf9eV/F1vWrduLQICAgo9TkFdDvbt2ycAiI8//liWPmjQIKFQKKS+uOfPnxcAxPTp02X5xo4dq9VM/9FHHwkAsi48Gunp6Vpp3333nQAgTp48qVXGxIkTpbTs7GxRr149oVAoxLJly6T0x48fC3Nz8wK7GWlERkYKAGLChAmy9FmzZgkA4ujRo1LamDFjyqQrVF6FdYWytLQU48eP10rfv3+/ACAOHjxYYLmPHj0STk5OonPnzlrbNmzYIKytrcXDhw+FEGXXtS6vkydPCoVCIeteoem+sXjxYq38GzduFADE9evXpbRLly4JQ0ND8dtvvwkh/u/65+8KlZqaKoYMGSIUCoV0f1hYWIh9+/bprNuOHTsEAFmfeF3mz58vAGh18xKi4K5QGppuW5cuXRJCPP+e5h+b8/jxY+Hs7Cy7xoV1cSnufaJL27ZthbW1daF5NLKysoSTk5Pw8vKSjS355ZdftMbtdO3aVWcXozFjxgh3d3fpfUxMjAAg7O3tRXJyspT+008/CQDif//7n5RWWFeoMWPGCABSl6DiKqwrlKenp3j11Ve10q9cuSIAiJCQECnt1q1bol27drK/xQ0bNpR9b4UQ4o8//hADBw4UX331lfjpp5/E0qVLhb29vTAzMxMXLlwotK6ZmZmiZcuWwsPDQ6jVap15iuoKVV73BFFpscWCqArYuHEjwsPDtV6tWrUqcl8bGxtcuXIFt27dKvFxDxw4AENDQ0ydOlWW/v7770MIgV9//RUAcPDgQQDAe++9J8s3ZcqUAsueNGmSVlrepvlnz57h33//xUsvvQQAWjMgAc9bGDQMDQ3Rvn17CCEQGBgopdvY2KBZs2ZFzjZ04MABANBqiXn//fcBAPv37y90//KUkZEh68qgYWZmJm3XJTc3F2+99RaePHmC9evXy7Y9evQICxcuxIIFC6RuN2Xt4cOHGDFiBDw8PGS/jmrqW9xzmjp1Knr16oUePXoUejxTU1M0bdoUgwYNwnfffYdt27ahffv2GDlypGwGHg1bW1sA0Dm7VF6PHj2CkZGRVjev4tDso2mBMTQ0hImJCYDn1yc5ORnZ2dlo3769zu+4LiW9T/JSqVSwsrIq1nHOnTuHhw8f4r333pOuCwAEBASgefPmet0TQ4cOlT5/4PnMaQCKvE81wsLCIIQodpeg4ijJfWZlZQVPT08EBQVh79692LRpE7Kzs9G/f3/Z96lTp07Ys2cPxo8fj759++I///kPzpw5A4VCgblz5xZan8mTJ+Pq1avYsGEDjIyMSnVO5XVPEJVW6b7JRFSmOnTooLPbiK2tbZH/ACxevBj9+vVD06ZN4eXlhZ49e2LUqFHFCkri4uLg6uqq9SDSokULabvmvwYGBvDw8JDla9y4cYFl588LPO9Hv2jRIuzcuRMPHz6UbUtJSdHKX79+fdl7a2trmJmZaXUbsLa2LrL/ueYc8tfZxcUFNjY20rlWBnNzc9k4D41nz55J23WZMmUKDh48iG+++QatW7eWbZs/fz7s7OwKDf70kZaWhjfeeANPnz7F77//Lnso19S3OOe0a9cunD59uljz60+ePBlnzpzBhQsXpG5jQ4YMgaenJ6ZNm4azZ8/K8ov/322oOONcSis1NRUAZPfQ119/jVWrVmmNEdB1T+hS0vskL6VSWeyHd813vlmzZlrbmjdvjt9//71Y5eiS/97VPNA+fvy41GXqq7j3WXZ2Nrp37w4/Pz9ZwN69e3d4enpi5cqVWL58eYHHady4Mfr164e9e/ciJycHhoaGWnlWrlyJ//73v1iyZAl69+5d6nOqivcE1W5ssSCq5rp06YLo6Ghs3boVXl5e2LJlC9q1a4ctW7ZUar10PQwPGTIE//3vfzFp0iTs3bsXhw4dklpDdC1KpusfZF1pAIrse65RFf9BrVu3LuLj47XSNWmurq5a2xYtWoRNmzZh2bJlWv3pb926hc2bN2Pq1Kl48OABYmNjERsbi2fPnkGtViM2NhbJycmlrm9WVhYGDBiAy5cv46efftJaG8POzg6mpqbFOqfZs2dj8ODBMDExkeqp6dt/7949ac2FrKwsfPXVVwgICJCNRTE2NkavXr1w7tw5ZGVlyY6leYgtqv+6vb09srOztcZ9FEdUVBQMDQ2loGHbtm0YO3YsGjVqhK+++goHDx5EeHg4Xn311WIvvFfS+ySv5s2bIyUlRRqIXFYKum90rcEA6H+flofi3mcnT55EVFQU+vbtK8vXpEkTtGjRAn/88UeRx3Jzc0NWVhbS0tK0toWFhWHOnDmYNGkS5s+fX5pTAVC+9wRRaTGwIKoB7OzsMG7cOHz33Xe4d+8eWrVqJZupqaCHAnd3dzx48EDrger69evSds1/c3NzERMTI8t3+/btYtfx8ePHOHLkCP7zn/9g0aJFePPNN/H666+jYcOGxS5DH5pzyN9lLDExEU+ePNGabakitWnTBhcuXNB6aDx79iwsLCzQtGlTWfrGjRsRHByM6dOnY86cOVrl3b9/H7m5uZg6dSo8PDyk19mzZ3Hz5k14eHhg8eLFpaprbm4uRo8ejSNHjmDHjh3o2rWrVh4DAwN4e3vrHMB69uxZNGzYUPqF/969e9ixY4esnuvWrQPwfDC15tfcR48eITs7W+eDrFqtRm5urta2mJgYGBgYaH1++TVv3lzKXxJ3797FiRMn4OvrK53Pnj170LBhQ+zduxejRo2Cv78/unfvLv0qrlHQPanvfdKnTx8AzwOcomi+8zdu3NDaduPGDdk9YWtrq3Mwtz4tfRUd5Ldp0wY3b97UGris+VW/TZs2AJ7/TQB0B01qtbrAmZ/yunPnDszMzLS61/3000+YMGECBgwYIM2KVVrleU8QlRYDC6JqLn8XoDp16qBx48ayJn9LS0sA0How6N27N3JycrBhwwZZ+po1a6BQKNCrVy8AgL+/PwBg06ZNsnz5+/UXRvMLZv5fLMtiGtTi0Dyg5j/e6tWrAaDQGa7K26BBg5CYmIi9e/dKaf/++y92796NPn36yPqF79q1C1OnTsVbb70l1T0/Ly8v/Pjjj1ovT09P1K9fHz/++KNsnEpJTJkyBbt27cKmTZtk0+PqOqe//vpLFlzcuHEDR48exeDBg6U0XfUcOnQoAOCbb77BmjVrADyf6tPGxgY//vij7FfY1NRU/O9//0Pz5s21WsnOnz8PT09PWFtbF3pOvr6+AFDsmXyA592Vhg8fjpycHHz44YdSuq7v+dmzZxERESHb38LCAoD2PanvfTJo0CB4e3vjk08+0Tom8HwsiKa+7du3h5OTE0JCQmR/L3799Vdcu3ZNdk80atQI169fR1JSkpR26dKlYv16X5CC/i4BJZtutrgGDRqEnJwcbN68WUrLzMxEaGgoOnbsCDc3NwCQHrp37twp2//ChQu4ceMG2rZtK6Xl/Tw0Ll26hJ9//hk9evSQtSScPHkSw4YNQ5cuXbB9+/ZiTZFbmPK8J4hKi2MsiKq5li1bws/PDz4+PrCzs8O5c+ewZ88eTJ48Wcrj4+MD4PkgWX9/fxgaGmLYsGHo06cPunXrhg8//BCxsbFo3bo1Dh06hJ9++gnTp09Ho0aNpP0HDhyItWvX4tGjR3jppZdw4sQJaRrU4vzyqFQq0aVLF6xYsQJqtRovvPACDh06VOJfiUurdevWGDNmDDZv3ownT56ga9eu+PPPP/H111+jf//+6NatW5kf83//+x8uXboE4PkviJcvX8bHH38MAOjbt680DmbQoEF46aWXMG7cOFy9ehUODg7YtGkTcnJysGjRIqm8P//8E6NHj4a9vT1ee+01bN++XXa8Tp06oWHDhnBwcED//v216qN5OM2/TTNl8bFjx3SuVZB3/02bNsHX1xcWFhZav4q/+eab0sPie++9h//+978ICAjArFmzYGxsjNWrV8PZ2VkaMK+rLsD/TfvZq1cvqcuGoaEhZs2ahfnz5+Oll17C6NGjkZOTg6+++gr//POPVl3UajVOnDihNeGALg0bNoSXlxcOHz6M8ePHa22/efMmtm3bBiEEVCoVLl26hN27dyM1NRWrV69Gz549pbxvvPEG9u7dizfffBMBAQGIiYlBSEgIWrZsKY3HAJ53FWzZsiV27dqFpk2bws7ODl5eXvDy8tLrPjE2NsbevXvRvXt3dOnSBUOGDMHLL78MY2NjXLlyBTt27ICtrS0++eQTGBsbY/ny5Rg3bhy6du2K4cOHIzExEevWrUODBg0wY8YMqdzx48dj9erV8Pf3R2BgIB4+fIiQkBB4enqWeurSgv4uAc+nm/36668RExNT5ADukydPStNNJyUlIS0tTbrPunTpIk0X3bFjRwwePBhz587Fw4cP0bhxY3z99deIjY3FV199JavX66+/jq+//hoqlQo9evRAfHw81q9fD3Nzc0yfPl3KO3ToUJibm6NTp05wcnLC1atXsXnzZlhYWGDZsmVSvri4OPTt2xcKhQKDBg3C7t27ZefQqlUr2bi4b7/9FnFxcUhPT5fOUXNOo0aNgru7e7neE0SlVkmzURGRKN3K2/mnN/34449Fhw4dhI2NjTA3NxfNmzcXn3zyicjKypLyZGdniylTpghHR0dpWkKNp0+fihkzZghXV1dhbGwsmjRpIlauXCmtGK2RlpYmgoKChJ2dnahTp47o37+/uHHjhgAgm/61oKlChRDin3/+EW+++aawsbER1tbWYvDgweLBgwcFTlmbv4yCpoEt7jSqarVaLFq0SHh4eAhjY2Ph5uYm5s6dK1u1trDj6FLYdLOaKTN1vUJDQ2V5k5OTRWBgoLC3txcWFhaia9euWt+LoqYnzl9mfgV9Tu+//75QKBTi2rVrhe5f2PlAx9Sg9+7dE4MGDRJKpVLUqVNHvPHGG1orDOtS2Hdo+/btsu97x44dxZ49e7Ty/frrrzpXNC7I6tWrRZ06dbSmes17fgYGBsLGxka0bdtWTJs2TVy5ckWrnNzcXPHpp58Kd3d3YWpqKtq2bSt++eUXrWlZhRDi9OnTwsfHR5iYmMjugeLeJ4V5/PixWLhwofD29hYWFhbCzMxMeHl5iblz54r4+HhZ3l27dom2bdsKU1NTYWdnJ9566y3xzz//aJW5bds20bBhQ2FiYiLatGkjfvvttwKnm827unXezzJv/Qv7u1SS6WY13xddr/yfV0ZGhpg1a5ZwcXERpqam4sUXX9Q5nXN6erpYvHixaNmypTA3NxfW1tbijTfeEBcvXpTlW7dunejQoYOws7MTRkZGom7dumLkyJFa3zvNCt3FrWfXrl0LzHvs2DFZ3vK6J4hKQyFEJY6kIqJqLTIyEm3btsW2bdvw1ltvVXZ1KkWDBg3g6+sr/Zqp+cW+OunQoQPc3d21fkWtzvr37w+FQoEff/yxWPlTUlLQsGFDrFixotTdxIiqspLeE0SlwcCCiIolIyNDq7/u2LFj8e233yI2Nlbqn1zbNGjQQBrAGhQUpDVepapTqVRwdHREZGSkNM1wdXft2jV4e3sjMjJSa8aqwixfvhyhoaG4evWq3v3fiaqS0t4TRCXFwIKIimXRokU4f/48unXrBiMjI/z666/49ddfMXHiRHz55ZeVXb1K88cff0gLa7m5uelcE4CIiKg2YGBBRMUSHh6ORYsW4erVq0hNTUX9+vUxatQofPjhh6VeNZaIiIhqDgYWRERERESkN3YiJSIiIiIivTGwICIiIiIivTGwICIiIiIivXHEZRnJzc3FgwcPYGVlVaxViImIiIiIqjohBJ4+fQpXV9cip+JmYFFGHjx4UGvn8SciIiKimu3evXuoV69eoXkYWJQRKysrAM8/dKVSWcm1ISIiIiLSn0qlgpubm/SsWxgGFmVE0/1JqVQysCAiIiKiGqU4Xf05eJuIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPRmVNkVICIiIiL9JCQk4MmTJ5VdDaoANjY2cHFxqexq6MTAgoiIiKgaS0hIwKBBg/Ds2bPKrgpVADMzM+zZs6dKBhcMLIiIiIiqsSdPnuDZs2dYsmQJPDw8Krs6VI5iYmKwYMECPHnyhIEFEREREZUPDw8PNG/evLKrQbUYAwuqUElJSVCpVLI0pVIJR0fHSqoREREREZUFBhZUYZKSkjBy3AQkP02XpdtZWWBb6BYGF0RERETVGAMLqjAqlQrJT9Ph6DsQlnbOAIC05EQkRfwAlUrFwIKIiIioGuM6FlThLO2coXSqB6VTPSnAICIiovKRkJCAKVOmoGHDhjA1NYWbmxv69OmDI0eOVHhdFAoF9u3bV+L94uPjMWLECDRt2hQGBgaYPn26zny7d+9G8+bNYWZmBm9vbxw4cEC2PTU1FZMnT0a9evVgbm6Oli1bIiQkRJbHz88PCoVC9po0aZK0/dKlSxg+fDjc3Nxgbm6OFi1aYN26daWq79q1a9GsWTOYm5vDzc0NM2bMqNaze7HFgoiIiKiGio2NxcsvvwwbGxusXLkS3t7eUKvV+O233xAUFITr169XdhWLJTMzE46Ojpg/fz7WrFmjM8/p06cxfPhwLF26FG+88QZ27NiB/v3748KFC/Dy8gIAzJw5E0ePHsW2bdvQoEEDHDp0CO+99x5cXV3Rt29fqay3334bixcvlt5bWFhI/3/+/Hk4OTlh27ZtcHNzw+nTpzFx4kQYGhpi8uTJxa7vjh078J///Adbt25Fp06dcPPmTYwdOxYKhQKrV6/W+zOrDAwsiIiIiGqo9957DwqFAn/++ScsLS2ldE9PT4wfP156f/fuXUyZMgVHjhyBgYEBevbsifXr18PZ+XnPgrFjx+LJkyey1obp06cjMjISx48fB/D8l/5WrVrBzMwMW7ZsgYmJCSZNmoTg4GAAQIMGDQAAb775JgDA3d0dsbGxxTqPBg0aSK0CW7du1Zln3bp16NmzJ2bPng0AWLJkCcLDw7FhwwapVeL06dMYM2YM/Pz8AAATJ07El19+iT///FMWWFhYWBQ4nWvezw0AGjZsiIiICOzdu1cKLIpT39OnT+Pll1/GiBEjpH2GDx+Os2fPFvl5VFXsCkVERERUAyUnJ+PgwYMICgqSBRUaNjY2AIDc3Fz069cPycnJOHHiBMLDw3Hnzh0MHTq0xMf8+uuvYWlpibNnz2LFihVYvHgxwsPDAQB//fUXACA0NBTx8fHS+9jYWCgUCilAKa2IiAh0795dlubv74+IiAjpfadOnfDzzz/j/v37EELg2LFjuHnzJnr06CHbb/v27XBwcICXlxfmzp2L9HT5xDP5paSkwM7OrkT17dSpE86fP48///wTAHDnzh0cOHAAvXv3LlE5VQlbLIiIiIhqoNu3b0MIUeTaFkeOHMHff/+NmJgYuLm5AQC++eYbeHp64q+//sKLL75Y7GO2atUKH330EQCgSZMm2LBhA44cOYLXX39dmqTFxsZG1hpgbGyMZs2ayboblUZCQoLUwqLh7OyMhIQE6f369esxceJE1KtXD0ZGRjAwMMB///tfdOnSRcozYsQIuLu7w9XVFZcvX8acOXNw48YN7N27V+dxT58+jV27dmH//v0lqu+IESPw77//4pVXXoEQAtnZ2Zg0aRLmzZtXonKqEgYWRERERDWQEKJY+a5duwY3NzcpqACAli1bwsbGBteuXStxYJFX3bp18fDhw0L3eeGFFypsrMf69etx5swZ/Pzzz3B3d8fJkycRFBQEV1dXqbVj4sSJUn5vb2/UrVsXr732GqKjo9GoUSNZeVFRUejXrx8++ugjrVaPohw/fhyffvopNm3ahI4dO+L27duYNm0alixZggULFuh/spWAgQURERFRDdSkSRMoFIoyeWg3MDDQClTUarVWPmNjY9l7hUKB3NxcvY9fHC4uLkhMTJSlJSYmSq0jGRkZmDdvHn788UcEBAQAeB4IRUZG4rPPPtPqRqXRsWNHAM9bgPIGFlevXsVrr72GiRMnYv78+SWu74IFCzBq1ChMmDABwPMgJi0tDRMnTsSHH34IA4PqN2Kh+tWYiIiIiIpkZ2cHf39/bNy4EWlpaVrbnzx5AgBo0aIF7t27h3v37knbrl69iidPnqBly5YAAEdHR8THx8v2j4yMLHGdjI2NkZOTU+L9isPX11drCt3w8HD4+voCeB4IqdVqrQd2Q0PDQoMfzXnWrVtXSrty5Qq6deuGMWPG4JNPPilVfdPT03XWBSh+a1NVw8CCiIiIqIbauHEjcnJy0KFDB/zwww+4desWrl27hs8//1x64O7evTu8vb3x1ltv4cKFC/jzzz8xevRodO3aFe3btwcAvPrqqzh37hy++eYb3Lp1Cx999BGioqJKXJ8GDRrgyJEjSEhIwOPHjwEA9+/fR/PmzaVBzAWJjIxEZGQkUlNTkZSUhMjISFy9elXaPm3aNBw8eBCrVq3C9evXERwcjHPnzkkzNSmVSnTt2hWzZ8/G8ePHERMTg7CwMHzzzTfSTFXR0dFYsmQJzp8/j9jYWPz8888YPXo0unTpInXzioqKQrdu3dCjRw/MnDkTCQkJSEhIQFJSUonq26dPH3zxxRfYuXMnYmJiEB4ejgULFqBPnz5SgFHtiEr06aefivbt24s6deoIR0dH0a9fP3H9+nVZnoyMDPHee+8JOzs7YWlpKQYMGCASEhJkeeLi4kTv3r2Fubm5cHR0FLNmzRJqtVqW59ixY6Jt27bCxMRENGrUSISGhmrVZ8OGDcLd3V2YmpqKDh06iLNnzxb7XFJSUgQAkZKSUvwPoJa5ffu2aN+luxi0fK8YH/qnGB/6pxi0fK9o36W7uH37dmVXj4iIqFq6du2a8PHxEdeuXdO5/cGDByIoKEi4u7sLExMT8cILL4i+ffuKY8eOSXni4uJE3759haWlpbCyshKDBw/Wet5auHChcHZ2FtbW1mLGjBli8uTJomvXrtL2rl27imnTpsn26devnxgzZoz0/ueffxaNGzcWRkZGwt3dXQghRExMjAAgq48uALRemjI0vv/+e9G0aVNhYmIiPD09xf79+2Xb4+PjxdixY4Wrq6swMzMTzZo1E6tWrRK5ublCCCHu3r0runTpIuzs7ISpqalo3LixmD17tuz57qOPPipWXYrKo1arRXBwsGjUqJEwMzMTbm5u4r333hOPHz8u8DMo6lqXh5I84yqEqLy2lp49e2LYsGF48cUXkZ2djXnz5iEqKgpXr16VpkV79913sX//foSFhcHa2hqTJ0+GgYEB/vjjDwBATk4O2rRpAxcXF6xcuRLx8fEYPXo03n77bXz66acAgJiYGHh5eWHSpEmYMGECjhw5gunTp2P//v3w9/cHAOzatQujR49GSEgIOnbsiLVr12L37t24ceMGnJycijwXlUoFa2trpKSkQKlUltMnVr1FR0dj2PhJaBDwHpRO9QAAqof/IHb/JuzcGqI1IIqIiIiKdv36dYwcORLbtm0rcgYoqt4q41qX5Bm3UgdvHzx4UPY+LCwMTk5OOH/+PLp06YKUlBR89dVX2LFjB1599VUAz+c+btGiBc6cOYOXXnoJhw4dwtWrV3H48GE4OzujTZs2WLJkCebMmYPg4GCYmJggJCQEHh4eWLVqFYDnfQl///13rFmzRgosVq9ejbfffhvjxo0DAISEhGD//v3YunUr/vOf/1Tgp0JEREREVP1UqTEWKSkpACAtMHL+/Hmo1WrZKP3mzZujfv360mInERER8Pb2ls1b7O/vD5VKhStXrkh5ClswJSsrC+fPn5flMTAwQPfu3WWLqhARERERkW5VZrrZ3NxcTJ8+HS+//DK8vLwAPF/oxMTERFoZUiPvYicFLYai2VZYHpVKhYyMDDx+/Bg5OTk68xQ0RVtmZiYyMzOl9yqVqoRnTERERERUc1SZFougoCBERUVh586dlV2VYlm6dCmsra2lV95FZYiIiIiIapsqEVhMnjwZv/zyC44dO4Z69epJ6S4uLsjKypLmWdbIu9hJQYuhaLYVlkepVMLc3BwODg4wNDQsdFGV/ObOnYuUlBTplXfuZyIiIiKi2qZSAwshBCZPnowff/wRR48ehYeHh2y7j48PjI2NZYud3LhxA3fv3pXmXvb19cXff/8tWy4+PDwcSqVSWtSlqAVTTExM4OPjI8uTm5uLI0eOSHnyMzU1hVKplL2IiIiIiGqrSh1jERQUhB07duCnn36ClZWVNCbC2toa5ubmsLa2RmBgIGbOnAk7OzsolUpMmTIFvr6+eOmllwAAPXr0QMuWLTFq1CisWLECCQkJmD9/PoKCgmBqagoAmDRpEjZs2IAPPvgA48ePx9GjR/H9999j//79Ul1mzpyJMWPGoH379ujQoQPWrl2LtLQ0aZYoIiIiIiIqWKUGFl988QUAwM/PT5YeGhqKsWPHAgDWrFkDAwMDDBw4EJmZmfD398emTZukvIaGhvjll1/w7rvvwtfXF5aWlhgzZgwWL14s5fHw8MD+/fsxY8YMrFu3DvXq1cOWLVukqWYBYOjQoUhKSsLChQuRkJCANm3a4ODBg1oDuomIiIiqopiYmMquApWzqn6NK3WBvJqEC+QVjQvkERERlb2EhAQMGjQIz549q+yqUAUwMzPDnj17ChwHXNaqzQJ5RERERKQfFxcX7NmzR2uyG6qZbGxsKiyoKCkGFkRERETVnIuLS5V92KTao0pMN0tERERERNUbAwsiIiIiItIbu0JRgZKSkqBSqWRpSqUSjo6OlVQjIiIiIqqqGFiQTklJSRg5bgKSn6bL0u2sLLAtdAuDCyIiIiKSYWBBOqlUKiQ/TYej70BY2j1fyyMtORFJET9ApVIxsCAiIiIiGQYWVChLO2dpzQkASKrEuhARERFR1cXB20REREREpDcGFkREREREpDcGFkREREREpDcGFkREREREpDcGFkREREREpDcGFkREREREpDdON0vlJv/K3XFxcchWZ1dijYiIiIiovDCwoHKha+XuZxnp+Od+POqr1ZVYMyIiIiIqDwwsqFzoWrn7YXQU4u5tRU42AwsiIiKimoaBBZWrvCt3pz5KqOTaEBEREVF54eBtIiIiIiLSG1ssSG/5B2kDHKhNREREVNswsCC96BqkDXCgNhEREVFtw8CC9KJrkDZQcQO1dbWWKJVKODo6lutxiYiIiEiOgQWVibyDtIGKGahdUGuJnZUFtoVuYXBBREREVIEYWFClU2dlIS4uTpZWnFYHXa0lacmJSIr4ASqVioEFERERUQViYEElkj8I0HeQdmZqCmJj7mD6vGCYmppK6SVpdcjfWpJU6toQERERUWkxsKBi0xUE6DtIW52ZgVyFERxeGgB7V3cAbHUgIiIiqo4YWFCx6QoCymqQtoWtI1sdiIiIiKoxBhZUYnmDAK6mTUREREQAV94mIiIiIqIywMCCiIiIiIj0xsCCiIiIiIj0xsCCiIiIiIj0xsCCiIiIiIj0xsCCiIiIiIj0xulmqUrKv8I3ACiVSi6YR0RERFRFMbCgKkfXCt8AYGdlgW2hWxhcEBEREVVBDCyoytG1wndaciKSIn6ASqViYEFERERUBVXqGIuTJ0+iT58+cHV1hUKhwL59+2TbFQqFztfKlSulPA0aNNDavmzZMlk5ly9fRufOnWFmZgY3NzesWLFCqy67d+9G8+bNYWZmBm9vbxw4cKBczpmKT7PCt9KpHiztnCu7OkRERERUiEoNLNLS0tC6dWts3LhR5/b4+HjZa+vWrVAoFBg4cKAs3+LFi2X5pkyZIm1TqVTo0aMH3N3dcf78eaxcuRLBwcHYvHmzlOf06dMYPnw4AgMDcfHiRfTv3x/9+/dHVFRU+Zw4EREREVENU6ldoXr16oVevXoVuN3FxUX2/qeffkK3bt3QsGFDWbqVlZVWXo3t27cjKysLW7duhYmJCTw9PREZGYnVq1dj4sSJAIB169ahZ8+emD17NgBgyZIlCA8Px4YNGxASEqLPKRIRERER1QrVZrrZxMRE7N+/H4GBgVrbli1bBnt7e7Rt2xYrV65Edna2tC0iIgJdunSBiYmJlObv748bN27g8ePHUp7u3bvLyvT390dERESB9cnMzIRKpZK9iIiIiIhqq2ozePvrr7+GlZUVBgwYIEufOnUq2rVrBzs7O5w+fRpz585FfHw8Vq9eDQBISEiAh4eHbB9nZ2dpm62tLRISEqS0vHkSEhIKrM/SpUuxaNGisjg1IiIiIqJqr9oEFlu3bsVbb70FMzMzWfrMmTOl/2/VqhVMTEzwzjvvYOnSpbKpSsva3LlzZcdWqVRwc3Mrt+MREREREVVl1SKwOHXqFG7cuIFdu3YVmbdjx47Izs5GbGwsmjVrBhcXFyQmJsryaN5rxmUUlKegcRsAYGpqWq6BCxERERFRdVItxlh89dVX8PHxQevWrYvMGxkZCQMDAzg5OQEAfH19cfLkSajVailPeHg4mjVrBltbWynPkSNHZOWEh4fD19e3DM+CiIiIiKjmqtTAIjU1FZGRkYiMjAQAxMTEIDIyEnfv3pXyqFQq7N69GxMmTNDaPyIiAmvXrsWlS5dw584dbN++HTNmzMDIkSOloGHEiBEwMTFBYGAgrly5gl27dmHdunWybkzTpk3DwYMHsWrVKly/fh3BwcE4d+4cJk+eXL4fABERERFRDVGpXaHOnTuHbt26Se81D/tjxoxBWFgYAGDnzp0QQmD48OFa+5uammLnzp0IDg5GZmYmPDw8MGPGDFnQYG1tjUOHDiEoKAg+Pj5wcHDAwoULpalmAaBTp07YsWMH5s+fj3nz5qFJkybYt28fvLy8yunMqTTUWVmIi4uT3sfFxSFbnV3IHkRERERUUSo1sPDz84MQotA8EydOlAUBebVr1w5nzpwp8jitWrXCqVOnCs0zePBgDB48uMiyqHJkpqYgNuYOps8Llsa2PMtIxz/341E/Tzc3IiIiIqoc1WLwNpE6MwO5CiM4vDQA9q7uAICH0VGIu7cVOdkMLIiIiIgqGwMLqlYsbB2hdKoHAEh9VPA6I0RERERUsarFrFBERERERFS1MbAgIiIiIiK9MbAgIiIiIiK9cYwFAQCSkpKgUqmk95zKlYiIiIhKgoEFISkpCSPHTUDy03QpjVO5EhEREVFJMLAgqFQqJD9Nh6PvQFjaOQPgVK5EREREVDIMLEhiaefMqVyJiIiIqFQ4eJuIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGwIKIiIiIiPTGWaFqIS6GR0RERERljYFFLcPF8IiIiIioPDCwqGW4GB4RERERlQcGFrUUF8MjIiIiorLEwIIon/xjUABAqVTC0dGxkmpEREREVPUxsCDKQ9cYFACws7LAttAtDC6IiIiICsDAgigPXWNQ0pITkRTxA1QqFQMLIiIiogIwsCDSIe8YFABIqsS6EBEREVUHDCyIyhnHbBAREVFtwMCCqBxxzAYRERHVFgwsiMoRx2wQERFRbcHAgmocdVYW4uLiZGmV3fWIYzaIiIiopmNgQTVKZmoKYmPuYPq8YJiamkrp7HpEREREVL4YWFCNos7MQK7CCA4vDYC9qzsAdj0iIiIiqggMLKhGsrB1ZNcjIiIiogpkUNkVICIiIiKi6o+BBRERERER6Y2BBRERERER6Y2BBRERERER6Y2BBRERERER6Y2BBRERERER6Y3TzdZwSUlJUKlU0vu4uDhkq7MrsUZEREREVBNVaovFyZMn0adPH7i6ukKhUGDfvn2y7WPHjoVCoZC9evbsKcuTnJyMt956C0qlEjY2NggMDERqaqosz+XLl9G5c2eYmZnBzc0NK1as0KrL7t270bx5c5iZmcHb2xsHDhwo8/OtaElJSRg5bgKGjZ8kvab9ZwFi4+KQpVZXdvWIiIiIqAap1BaLtLQ0tG7dGuPHj8eAAQN05unZsydCQ0Ol96amprLtb731FuLj4xEeHg61Wo1x48Zh4sSJ2LFjBwBApVKhR48e6N69O0JCQvD3339j/PjxsLGxwcSJEwEAp0+fxvDhw7F06VK88cYb2LFjB/r3748LFy7Ay8urnM6+/KlUKiQ/TYej70BY2jkDAB5GRyHu3lbkZNeuwEKdlYW4uDhZmlKp5ErcRERERGWkUgOLXr16oVevXoXmMTU1hYuLi85t165dw8GDB/HXX3+hffv2AID169ejd+/e+Oyzz+Dq6ort27cjKysLW7duhYmJCTw9PREZGYnVq1dLgcW6devQs2dPzJ49GwCwZMkShIeHY8OGDQgJCSnDMy5fBXV7srRzllahTn2UUFnVqzSZqSmIjbmD6fOCZYGpnZUFtoVuYXBBREREVAaq/BiL48ePw8nJCba2tnj11Vfx8ccfw97eHgAQEREBGxsbKagAgO7du8PAwABnz57Fm2++iYiICHTp0gUmJiZSHn9/fyxfvhyPHz+Gra0tIiIiMHPmTNlx/f39tbpmVWWabk/JT9OltGcZ6fjnfjzq1/JuT+rMDOQqjODw0gDYu7oDANKSE5EU8QNUKhUDCyIiIqIyUKUDi549e2LAgAHw8PBAdHQ05s2bh169eiEiIgKGhoZISEiAk5OTbB8jIyPY2dkhIeH5L/MJCQnw8PCQ5XF2dpa22draIiEhQUrLm0dThi6ZmZnIzMyU3udtKagM7PZUNAtbR6nlBgCSKrEuRERERDVNlQ4shg0bJv2/t7c3WrVqhUaNGuH48eN47bXXKrFmwNKlS7Fo0aJKrYMutb3bExERERFVjmq1jkXDhg3h4OCA27dvAwBcXFzw8OFDWZ7s7GwkJydL4zJcXFyQmJgoy6N5X1SegsZ2AMDcuXORkpIive7du6ffyRERERERVWPVKrD4559/8OjRI9StWxcA4OvriydPnuD8+fNSnqNHjyI3NxcdO3aU8pw8eRLqPOMMwsPD0axZM9ja2kp5jhw5IjtWeHg4fH19C6yLqakplEql7EVEREREVFtVamCRmpqKyMhIREZGAgBiYmIQGRmJu3fvIjU1FbNnz8aZM2cQGxuLI0eOoF+/fmjcuDH8/f0BAC1atEDPnj3x9ttv488//8Qff/yByZMnY9iwYXB1dQUAjBgxAiYmJggMDMSVK1ewa9curFu3TjZYe9q0aTh48CBWrVqF69evIzg4GOfOncPkyZMr/DMhIiIiIqqOKnWMxblz59CtWzfpveZhf8yYMfjiiy9w+fJlfP3113jy5AlcXV3Ro0cPLFmyRDZl6Pbt2zF58mS89tprMDAwwMCBA/H5559L262trXHo0CEEBQXBx8cHDg4OWLhwoTTVLAB06tQJO3bswPz58zFv3jw0adIE+/btq9ZrWFDRdK1twZXJiYiIiEqnUgMLPz8/CCEK3P7bb78VWYadnZ20GF5BWrVqhVOnThWaZ/DgwRg8eHCRx6OaoaC1LThFLxEREVHpVOlZoYjKi661LQBO0UtERERUWgwsqFbLv7YFp+glIiIiKp1qNSsUERERERFVTQwsiIiIiIhIbwwsiIiIiIhIbxxjQVSGkpKSoFKppPflMX1t/mNoKJVKODo6lumxiIiIiIqLgQVRGUlKSsLIcROQ/DRdSivr6Wt1HUPDzsoC20K3MLggIiKiSsHAgqiMqFQqJD9Nh6PvQFjaOQMo++lrdR0DANKSE5EU8QNUKhUDCyIiIqoUDCyISqmgbk+Wds7SFLblNX1t3mNI9SmXIxEREREVDwMLomJQZ2UhLi5Oev/o0SPMmR+M1Mz/a4ngqt1ERERUmzGwICpCZmoKYmPuYPq8YJiamgL4vyCi/bAZsHF+3nLAVbuJiIioNmNgQVQEdWYGchVGcHhpAOxd3QH8XxBhqrQr925PRERERNUBAwuiYrKwdWQQQURERFQALpBHRERERER6Y4sFUSXIPxgc4AJ3REREVL0xsCCqYLoGgwNc4I6IiIiqNwYWRBVM12BwLnBHRERE1R0DC6JKkncwOMAF7oiIiKh64+BtIiIiIiLSGwMLIiIiIiLSGwMLIiIiIiLSGwMLIiIiIiLSGwdvE1URXNuCiIiIqjMGFkRVANe2ICIiouqOgQVRFcC1LYiIiKi6Y2BBVIXkX9viQb7uUXFxcchWZ1dG1YiIiIgKxcCCqIrS1T3qWUY6/rkfj/pqdSXXjoiIiEiOgQVRFaWre9TD6CjE3duKnGwGFkRERFS1MLAgquLydo9KfZRQybUhIiIi0o3rWBARERERkd4YWBARERERkd4YWBARERERkd4YWBARERERkd4YWBARERERkd4YWBARERERkd4YWBARERERkd4YWBARERERkd4qNbA4efIk+vTpA1dXVygUCuzbt0/aplarMWfOHHh7e8PS0hKurq4YPXo0Hjx4ICujQYMGUCgUsteyZctkeS5fvozOnTvDzMwMbm5uWLFihVZddu/ejebNm8PMzAze3t44cOBAuZwzEREREVFNVKrA4s6dO2Vy8LS0NLRu3RobN27U2paeno4LFy5gwYIFuHDhAvbu3YsbN26gb9++WnkXL16M+Ph46TVlyhRpm0qlQo8ePeDu7o7z589j5cqVCA4OxubNm6U8p0+fxvDhwxEYGIiLFy+if//+6N+/P6KiosrkPImIiIiIajqj0uzUuHFjdO3aFYGBgRg0aBDMzMxKdfBevXqhV69eOrdZW1sjPDxclrZhwwZ06NABd+/eRf369aV0KysruLi46Cxn+/btyMrKwtatW2FiYgJPT09ERkZi9erVmDhxIgBg3bp16NmzJ2bPng0AWLJkCcLDw7FhwwaEhISU6tyIiIiIiGqTUrVYXLhwAa1atcLMmTPh4uKCd955B3/++WdZ101LSkoKFAoFbGxsZOnLli2Dvb092rZti5UrVyI7O1vaFhERgS5dusDExERK8/f3x40bN/D48WMpT/fu3WVl+vv7IyIiosC6ZGZmQqVSyV5ERERERLVVqQKLNm3aYN26dXjw4AG2bt2K+Ph4vPLKK/Dy8sLq1auRlJRU1vXEs2fPMGfOHAwfPhxKpVJKnzp1Knbu3Iljx47hnXfewaeffooPPvhA2p6QkABnZ2dZWZr3CQkJhebRbNdl6dKlsLa2ll5ubm56nyMRERERUXWl1+BtIyMjDBgwALt378by5ctx+/ZtzJo1C25ubhg9ejTi4+PLpJJqtRpDhgyBEAJffPGFbNvMmTPh5+eHVq1aYdKkSVi1ahXWr1+PzMzMMjl2QebOnYuUlBTpde/evXI9HhERERFRVaZXYHHu3Dm89957qFu3LlavXo1Zs2YhOjoa4eHhePDgAfr166d3BTVBRVxcHMLDw2WtFbp07NgR2dnZiI2NBQC4uLggMTFRlkfzXjMuo6A8BY3bAABTU1MolUrZi4iIiIiotipVYLF69Wp4e3ujU6dOePDgAb755hvExcXh448/hoeHBzp37oywsDBcuHBBr8ppgopbt27h8OHDsLe3L3KfyMhIGBgYwMnJCQDg6+uLkydPQq1WS3nCw8PRrFkz2NraSnmOHDkiKyc8PBy+vr561Z+IiIiIqLYo1axQX3zxBcaPH4+xY8eibt26OvM4OTnhq6++KrSc1NRU3L59W3ofExODyMhI2NnZoW7duhg0aBAuXLiAX375BTk5OdKYBzs7O5iYmCAiIgJnz55Ft27dYGVlhYiICMyYMQMjR46UgoYRI0Zg0aJFCAwMxJw5cxAVFYV169ZhzZo10nGnTZuGrl27YtWqVQgICMDOnTtx7tw52ZS0RERERERUsFIFFrdu3Soyj4mJCcaMGVNonnPnzqFbt27S+5kzZwIAxowZg+DgYPz8888Ang8Wz+vYsWPw8/ODqakpdu7cieDgYGRmZsLDwwMzZsyQygGeT1t76NAhBAUFwcfHBw4ODli4cKE01SwAdOrUCTt27MD8+fMxb948NGnSBPv27YOXl1eR50lERERERKUMLEJDQ1GnTh0MHjxYlr57926kp6cXGVBo+Pn5QQhR4PbCtgFAu3btcObMmSKP06pVK5w6darQPIMHD9Y6H6LqRJ2Vhbi4OFmaUqmEo6NjJdWIiIiIapNSBRZLly7Fl19+qZXu5OSEiRMnFjuwIKKykZmagtiYO5g+LximpqZSup2VBbaFbmFwQUREROWuVIHF3bt34eHhoZXu7u6Ou3fv6l0pIioZdWYGchVGcHhpAOxd3QEAacmJSIr4ASqVioEFERERlbtSzQrl5OSEy5cva6VfunSpWDM3EVH5sLB1hNKpHpRO9WBp51z0DkRERERlpFSBxfDhwzF16lQcO3YMOTk5yMnJwdGjRzFt2jQMGzasrOtIRERERERVXKm6Qi1ZsgSxsbF47bXXYGT0vIjc3FyMHj0an376aZlWkIhKr7gDupOSkqBSqYrMR0RERFSQUgUWJiYm2LVrF5YsWYJLly7B3Nwc3t7ecHd3L+v6EVEpFXdAd1JSEkaOm4Dkp+my/Tnwm4iIiEqiVIGFRtOmTdG0adOyqgsRlaHiDuhWqVRIfpoOR9+B0rgMDvwmIiKikipVYJGTk4OwsDAcOXIEDx8+RG5urmz70aNHy6RyRKQ/zYBujaQC8lnaORcrHxEREZEupQospk2bhrCwMAQEBMDLywsKhaKs60VERERERNVIqQKLnTt34vvvv0fv3r3Luj5ERERERFQNlWq6WRMTEzRu3Lis60JERERERNVUqQKL999/H+vWrYMQoqzrQ0RERERE1VCpukL9/vvvOHbsGH799Vd4enrC2NhYtn3v3r1lUjkiIiIiIqoeShVY2NjY4M033yzruhARERERUTVVqsAiNDS0rOtBRBUk/2rccXFxyFZnV2KNiIiIqCYo9QJ52dnZOH78OKKjozFixAhYWVnhwYMHUCqVqFOnTlnWkYjKiK7VuJ9lpOOf+/Gor1ZXcu2IiIioOitVYBEXF4eePXvi7t27yMzMxOuvvw4rKyssX74cmZmZCAkJKet6ElEZ0LUa98PoKMTd24qcbAYWREREVHqlmhVq2rRpaN++PR4/fgxzc3Mp/c0338SRI0fKrHJEVD40q3ErnerBwsahsqtDRERENUCpWixOnTqF06dPw8TERJbeoEED3L9/v0wqRkRERERE1UepAovc3Fzk5ORopf/zzz+wsrLSu1JERAQkJSVBpVLJ0pRKJRwdHcusPH3LJCIi0ihVYNGjRw+sXbsWmzdvBgAoFAqkpqbio48+Qu/evcu0gkREtVFSUhJGjpuA5KfpsnQ7KwtsC91S4kCgoPL0KZOIiCivUgUWq1atgr+/P1q2bIlnz55hxIgRuHXrFhwcHPDdd9+VdR2JiGodlUqF5KfpcPQdCEs7ZwBAWnIikiJ+gEqlKnEQoKs8fcskIiLKq1SBRb169XDp0iXs3LkTly9fRmpqKgIDA/HWW2/JBnMTEZF+LO2coXSqJ71PKuPyyqJMIiIiQI91LIyMjDBy5MiyrAsREREREVVTpQosvvnmm0K3jx49ulSVISIiIiKi6qlUgcW0adNk79VqNdLT02FiYgILCwsGFkREREREtUypFsh7/Pix7JWamoobN27glVde4eBtIiIiIqJaqNRjLPJr0qQJli1bhpEjR+L69etlVSwRUbVR1utOEBERVSdlFlgAzwd0P3jwoCyLJCKqFsp63QkiIqLqplSBxc8//yx7L4RAfHw8NmzYgJdffrlMKkZE1Vdt/OW+rNedICIiqm5KFVj0799f9l6hUMDR0RGvvvoqVq1aVRb1IqJqqrb/cl/W604QERFVF6UKLHJzc8u6HkRUQ/CXeyIiotqpTMdYEBFp8Jf72qE2dnsjIiLdShVYzJw5s9h5V69eXZpDEBFRFVfbu70REZFcqQKLixcv4uLFi1Cr1WjWrBkA4ObNmzA0NES7du2kfAqFomxqSUREVQ67vRERUV6lCiz69OkDKysrfP3117C1tQXwfNG8cePGoXPnznj//ffLtJJERFR1sdsbEREBpQwsVq1ahUOHDklBBQDY2tri448/Ro8ePRhYENUA6qwsxMXFydLYd56IiIgKYlCanVQqFZKStH+TSkpKwtOnT4tdzsmTJ9GnTx+4urpCoVBg3759su1CCCxcuBB169aFubk5unfvjlu3bsnyJCcn46233oJSqYSNjQ0CAwORmpoqy3P58mV07twZZmZmcHNzw4oVK7Tqsnv3bjRv3hxmZmbw9vbGgQMHin0eRDVNZmoKYmPuYPq8YAwbP0l6jRw3Qee9Xx0lJSUhOjpa9qop50ZERFQZStVi8eabb2LcuHFYtWoVOnToAAA4e/YsZs+ejQEDBhS7nLS0NLRu3Rrjx4/Xud+KFSvw+eef4+uvv4aHhwcWLFgAf39/XL16FWZmZgCAt956C/Hx8QgPD4darca4ceMwceJE7NixA8DzIKhHjx7o3r07QkJC8Pfff2P8+PGwsbHBxIkTAQCnT5/G8OHDsXTpUrzxxhvYsWMH+vfvjwsXLsDLy6s0HxFRtabOzECuwggOLw2Avas7gJrVd56DjomIiMpeqQKLkJAQzJo1CyNGjIBarX5ekJERAgMDsXLlymKX06tXL/Tq1UvnNiEE1q5di/nz56Nfv34AgG+++QbOzs7Yt28fhg0bhmvXruHgwYP466+/0L59ewDA+vXr0bt3b3z22WdwdXXF9u3bkZWVha1bt8LExASenp6IjIzE6tWrpcBi3bp16NmzJ2bPng0AWLJkCcLDw7FhwwaEhISU5iMiqhEsbB1rZN95DjomIiIqe6XqCmVhYYFNmzbh0aNH0gxRycnJ2LRpEywtLcukYjExMUhISED37t2lNGtra3Ts2BEREREAgIiICNjY2EhBBQB0794dBgYGOHv2rJSnS5cuMDExkfL4+/vjxo0bePz4sZQn73E0eTTH0SUzMxMqlUr2IqrpNOMuakr3Ic2gY6VTPSnAqCnyd/WKi4tDtjq7sqtFREQ1mF4L5MXHxyM+Ph5dunSBubk5hBBlNsVsQkICAMDZWf6PvbOzs7QtISEBTk5Osu1GRkaws7OT5fHw8NAqQ7PN1tYWCQkJhR5Hl6VLl2LRokWlODOi6invuAtTU1Mpnd2HKl/+ReoePXqEOfODkZqpltKeZaTjn/vxqK9W6yqCiIhIb6UKLB49eoQhQ4bg2LFjUCgUuHXrFho2bIjAwEDY2tpi1apVZV3PKmfu3LmyhQJVKhXc3NwqsUZE5aumj7uornSNF9EEEe2HzYCN8/OubA+joxB3bytyshlYEBFR+ShVYDFjxgwYGxvj7t27aNGihZQ+dOhQzJw5s0wCCxcXFwBAYmIi6tatK6UnJiaiTZs2Up6HDx/K9svOzkZycrK0v4uLCxITE2V5NO+LyqPZroupqansV1ui2qKmjruornSNF9EEEaZKO+lapT4quAWWUwsTEVFZKFVgcejQIfz222+oV6+eLL1JkyZa/ziVloeHB1xcXHDkyBEpkFCpVDh79izeffddAICvry+ePHmC8+fPw8fHBwBw9OhR5ObmomPHjlKeDz/8EGq1GsbGxgCA8PBwNGvWTFqHw9fXF0eOHMH06dOl44eHh8PX17dMzoWI9Je/uw9QfR9+y+Nc8i5SV1gQkR+7uBERUVkpVWCRlpYGCwsLrfTk5OQS/YqfmpqK27dvS+9jYmIQGRkJOzs71K9fH9OnT8fHH3+MJk2aSNPNurq6on///gCAFi1aoGfPnnj77bcREhICtVqNyZMnY9iwYXB1dQUAjBgxAosWLUJgYCDmzJmDqKgorFu3DmvWrJGOO23aNHTt2hWrVq1CQEAAdu7ciXPnzmHz5s2l+XiIqIzVpOlhCzqXOiaGWP7JYtjb2wNAgYOt87cu6Dsom13ciIiorJQqsOjcuTO++eYbLFmyBACgUCiQm5uLFStWoFu3bsUu59y5c7L8mjELY8aMQVhYGD744AOkpaVh4sSJePLkCV555RUcPHhQWsMCALZv347Jkyfjtddeg4GBAQYOHIjPP/9c2m5tbY1Dhw4hKCgIPj4+cHBwwMKFC6WpZgGgU6dO2LFjB+bPn4958+ahSZMm2LdvH9ewIKoiatL0sLrOJfmf2zj//eeYMHWW9OOMrsHWuloXympQNru4ERGRvkoVWKxYsQKvvfYazp07h6ysLHzwwQe4cuUKkpOT8ccffxS7HD8/PwghCtyuUCiwePFiLF68uMA8dnZ20mJ4BWnVqhVOnTpVaJ7Bgwdj8ODBhVeYiCpV3u4+QPV++M3fdSl/q4Guwda6Whc4KJuIiKqKUgUWXl5euHnzJjZs2AArKyukpqZiwIABCAoKkg20JiKq7UrSdSlvq0Fh4ySKm686qkljaYiIapsSBxZqtRo9e/ZESEgIPvzww/KoExFRjVCeXZfKW2XMFFWTxtIQEdVGJQ4sjI2Ncfny5fKoCxFRjVJduy5V1kxRNWksDRFRbVSqrlAjR47EV199hWXLlpV1fYiominrWYpqourWdamgmaIenPgOf//9N9zdn6eV17WuSWNpiIhqk1IFFtnZ2di6dSsOHz4MHx8fWFpayravXr26TCpHRFVbde7qQ0XLGxDxWhMRUVFKFFjcuXMHDRo0QFRUFNq1awcAuHnzpiyPQqEou9oRUZVWXbv6UMnxWhMRUVFKFFg0adIE8fHxOHbsGABg6NCh+Pzzz+Hs7FwulSOi6qG6dfWh0uO1JiKigpQosMi/5sSvv/6KtLS0Mq0QEVFVomv6U4BToBIREeVXqjEWGoUtbkdEVN0VNP0pANQxMcTyTxbD3t4eAAetExERlSiwUCgUWmMoOKaCiIpD17oIQNX+5V/X9KcAkPzPbZz//nNMmDqLA5mLiQvfERHVfCXuCjV27Nj/+4f02TNMmjRJa1aovXv3ll0NiajaK2hdBKBqLX5W0NS5+ac/TX2UwIHMJcCF74iIaocSBRZjxoyRvR85cmSZVoaIaiZdMwoBVWvxs9JMp8qBzLrpCtAeJqtQt8tQLnxHRFSDlSiwCA0NLa96EFEtkPdBXKOqLH7G6VTLRqEBmpVdpSx8x25YREQVQ6/B20RENQ1bIfRT1QI0dsMiIqo4DCyIqFLpGtTNX5Orv6oSoOkagM9uWERE5YOBBRFVmoIGdfPX5NqhoMHyReXTKEkAmn8AflXpgkdEVJMwsCCiSqOr20xaciIenPgOf//9N9zdn6dxjYiap7iD5avLjGJERMTAgoiqgLzdZkozOxNVP8Udi1EdZhQjIqLnGFgQUZVS1Qb/Uvkq7liMqjyjGBERPcfAgoiqpKoy+JeIiIiKh4EFEdV4+dcx4JgNIiKissfAgohqlPxBxKNHjzBnfjBSM/+vGxXHbBAREZU9BhZEVG3ln4a0sCCi/bAZsHF+3rWKYzaIiIjKHgMLIqqWCps9SlcQYaq045iNGkbX+hZZWVkwMTGR3rPbGxFRxWFgQUTVUmGzRzGIqPl0BZbqrCzcvxuHeu4eMDJ+/s8bu70REVUcBhZEVK1x9qjaqaDA8k7sVth26MepiomIKgEDCyIiqrZ0BZYMNomIKodBZVeAiIiIiIiqPwYWRERERESkNwYWRERERESkNwYWRERERESkNwYWRERERESkNwYWRERERESkN043S0REtY6uVbuVSiUcHR2L3DcpKQkqlapU+xIR1WQMLIiIqFbRtWo3ANhZWWBb6JZCA4SkpCSMHDcByU/TZenF2ZeIqKZjYEFERLWKrlW705ITkRTxA1QqVaHBgUqlQvLTdDj6DoSlnXOJ9iUiqumq/BiLBg0aQKFQaL2CgoIAAH5+flrbJk2aJCvj7t27CAgIgIWFBZycnDB79mxkZ2fL8hw/fhzt2rWDqakpGjdujLCwsIo6RSIiqgSaFbqVTvWkIKG4LO2cS70vEVFNVeVbLP766y/k5ORI76OiovD6669j8ODBUtrbb7+NxYsXS+8tLCyk/8/JyUFAQABcXFxw+vRpxMfHY/To0TA2Nsann34KAIiJiUFAQAAmTZqE7du348iRI5gwYQLq1q0Lf3//CjhLIiIiIqLqrcoHFvmblZctW4ZGjRqha9euUpqFhQVcXFx07n/o0CFcvXoVhw8fhrOzM9q0aYMlS5Zgzpw5CA4OhomJCUJCQuDh4YFVq1YBAFq0aIHff/8da9asYWBBRERERFQMVb4rVF5ZWVnYtm0bxo8fD4VCIaVv374dDg4O8PLywty5c5Ge/n+D6iIiIuDt7Q1n5/9rqvb394dKpcKVK1ekPN27d5cdy9/fHxEREeV8RkREVFVoZoqKjo6WXklJSZVdLSKiaqPKt1jktW/fPjx58gRjx46V0kaMGAF3d3e4urri8uXLmDNnDm7cuIG9e/cCABISEmRBBQDpfUJCQqF5VCoVMjIyYG5urlWXzMxMZGZmSu/zTz1IRETVhz4zRRER0XPVKrD46quv0KtXL7i6ukppEydOlP7f29sbdevWxWuvvYbo6Gg0atSo3OqydOlSLFq0qNzKJyKiiqPPTFFERPRctekKFRcXh8OHD2PChAmF5uvYsSMA4Pbt2wAAFxcXJCYmyvJo3mvGZRSUR6lU6mytAIC5c+ciJSVFet27d6/kJ0VERFWKPjNFERHVdtUmsAgNDYWTkxMCAgIKzRcZGQkAqFu3LgDA19cXf//9Nx4+fCjlCQ8Ph1KpRMuWLaU8R44ckZUTHh4OX1/fAo9jamoKpVIpexERERER1VbVoitUbm4uQkNDMWbMGBgZ/V+Vo6OjsWPHDvTu3Rv29va4fPkyZsyYgS5duqBVq1YAgB49eqBly5YYNWoUVqxYgYSEBMyfPx9BQUFSP9pJkyZhw4YN+OCDDzB+/HgcPXoU33//Pfbv318p50tERFWDZkC3RlxcHLLV2YXsQURUe1WLwOLw4cO4e/cuxo8fL0s3MTHB4cOHsXbtWqSlpcHNzQ0DBw7E/PnzpTyGhob45Zdf8O6778LX1xeWlpYYM2aMbN0LDw8P7N+/HzNmzMC6detQr149bNmyhVPNEhHVYroGdD/LSMc/9+NRX62u5NoREVU91SKw6NGjB4QQWulubm44ceJEkfu7u7vjwIEDhebx8/PDxYsXS11HIiKqWXQN6H4YHYW4e1uRk83Agogov2oRWBAREVUWzYBuAEh9lFDJtSEiqrqqzeBtIiIiIiKquhhYEBERERGR3hhYEBERERGR3hhYEBERERGR3hhYEBERERGR3hhYEBERERGR3jjdLBERUSVLSkqCSqWSpSmVSjg6OlZSjYiISo6BBRERkZ7UWVmIi4uTpRU3MEhKSsLIcROQ/DRdlm5nZYFtoVsYXBBRtcHAgoiISA+ZqSmIjbmD6fOCYWpqKqUXNzBQqVRIfpoOR9+BsLRzBgCkJSciKeIHqFQqBhbFpKvVB2DLD1FFYmBBRESkB3VmBnIVRnB4aQDsXd0BlC4wsLRzllb4BoCkcqltzVRQqw/Alh+iisTAgoiIqAxY2DoyMKgkulp9ALb8EFU0BhZERERUI+Rv9QEY4BFVJAYWRERE5UDXgG6Aff6JqOZiYEFERFTGChrQDQB1TAyx/JPFsLe3BwDExcUhW51dGdUkIipTDCyIiIjKmK4B3QCQ/M9tnP/+c0yYOksKOJ5lpOOf+/Gor1ZXVnWJiMoEAwsiIqJykn9Ad+qjBK2A42F0FOLubUVOdtkFFlxwj4gqAwMLIiKiCpY34Eh9lFCmZXPBPSKqLAwsiIiIahAuuEdElYWBBRERUQ3EBfeIqKIZVHYFiIiIiIio+mNgQUREREREemNgQUREREREeuMYCyIiIqpwnBKXqOZhYEFERFQFqbOyEBcXJ0urKQ/enBKXqGZiYEFERFTFZKamIDbmDqbPC5ZW6AZqzoO3vlPi5m/tiIuLQ7Y6u1zrTERFY2BBRERUxagzM7RW6E5LTsSDE9/h77//hru7u5S3OrdilGZKXF2tHc8y0vHP/XjUV5fd6uVEVHIMLIiIiKqovCt0V3YrRmWNidDVOvEwWYW6XYZKrR0Po6MQd28rcrIZWBBVJgYWRERE1UBBrRgVsaJ2ZY2JKLR1wspOCrpSHyUUWEZNHqtCVNUwsCAiIqpG8rZiABWzora+YyKKK38QoG/rhL6tPJy5iqhkGFgQERFVY7oexstrIHNpxkQUl64goKStE/np08rDmauISo6BBRERUTVV6MN4NRvIrCsIKKuxE6Vp5amoVhqimoSBBRERUTVVng/jlSVvEFCS1omSKMm4i/JspSGqaRhYEBERVXPFeRjnIObnKnt2LaKajIEFERFRDVeSh2ldA5Zr0gJ0lTm7FlFNx8CCiIiohivugnuPHj3CnPnBSM2Ud6OqruM2ClMZs2sR1XQMLIiIiGqJohbc0wQQ7YfNgI3z/z10V/dxG0RUMQwquwKFCQ4OhkKhkL2aN28ubX/27BmCgoJgb2+POnXqYODAgUhMTJSVcffuXQQEBMDCwgJOTk6YPXs2srPlzbnHjx9Hu3btYGpqisaNGyMsLKwiTo+IiKjS5G3FaBDwHhoEvAe7tj2Rkytgqnw+vavmZWHjUNnVLXeaMSjR0dGIjo6uUd2/iCpKlW+x8PT0xOHDh6X3Rkb/V+UZM2Zg//792L17N6ytrTF58mQMGDAAf/zxBwAgJycHAQEBcHFxwenTpxEfH4/Ro0fD2NgYn376KQAgJiYGAQEBmDRpErZv344jR45gwoQJqFu3Lvz9/Sv2ZImIiCpYRczCVNXVpGl7iSpTlQ8sjIyM4OLiopWekpKCr776Cjt27MCrr74KAAgNDUWLFi1w5swZvPTSSzh06BCuXr2Kw4cPw9nZGW3atMGSJUswZ84cBAcHw8TEBCEhIfDw8MCqVasAAC1atMDvv/+ONWvWMLAgIiIqA/kHhFe11oCaOG0vUWWo8oHFrVu34OrqCjMzM/j6+mLp0qWoX78+zp8/D7Vaje7du0t5mzdvjvr16yMiIgIvvfQSIiIi4O3tDWdnZymPv78/3n33XVy5cgVt27ZFRESErAxNnunTpxdar8zMTGRmZkrv88+gQUREVNPpmsI2KysLJiYm0ntdA8KramsAW2+I9FOlA4uOHTsiLCwMzZo1Q3x8PBYtWoTOnTsjKioKCQkJMDExgY2NjWwfZ2dnJCQ8/2OQkJAgCyo02zXbCsujUqmQkZEBc3NznXVbunQpFi1aVBanSUREVO3o6j6kzsrC/btxqOfuASPj548YugaEszWAqGaq0oFFr169pP9v1aoVOnbsCHd3d3z//fcFPvBXlLlz52LmzJnSe5VKBTc3t0qsERERUcUpqPvQnditsO3QT6tLkWZAOMDWAKKaqkrPCpWfjY0NmjZtitu3b8PFxQVZWVl48uSJLE9iYqI0JsPFxUVrlijN+6LyKJXKQoMXU1NTKJVK2YuIiKi20XQfyjt7lK40Iqr5qlVgkZqaiujoaNStWxc+Pj4wNjbGkSNHpO03btzA3bt34evrCwDw9fXF33//jYcPH0p5wsPDoVQq0bJlSylP3jI0eTRlEBEREWnkn5Y2OjoaSUnFW14vKSlJtl9J9iWqDqp0V6hZs2ahT58+cHd3x4MHD/DRRx/B0NAQw4cPh7W1NQIDAzFz5kzY2dlBqVRiypQp8PX1xUsvvQQA6NGjB1q2bIlRo0ZhxYoVSEhIwPz58xEUFCT1B500aRI2bNiADz74AOPHj8fRo0fx/fffY//+/ZV56kRERFTF6BpXAgB2VhbYFroFjo6OBe6blJSEkeMmIPlpuiy9OPsSVRdVOrD4559/MHz4cDx69AiOjo545ZVXcObMGenmW7NmDQwMDDBw4EBkZmbC398fmzZtkvY3NDTEL7/8gnfffRe+vr6wtLTEmDFjsHjxYimPh4cH9u/fjxkzZmDdunWoV68etmzZwqlmiYiISEbXuJK05EQ8OPEd/v77b7i7u0t5lUqlLFhQqVRIfpoOR9+BsLRzlvZNivgBKpWKgQXVCFU6sNi5c2eh283MzLBx40Zs3LixwDzu7u44cOBAoeX4+fnh4sWLpaojERER1S55p6UtaSuGpZ2ztC8AsCMU1SRVOrAgIiIiqsoKasVgSwTVRgwsiIiIiPSUtxUDYEsE1U7ValYoIiIiIiKqmhhYEBERERGR3tgVioiIiKiMada70IiLi0O2OrvMj5OUlASVSiVLyz8jFVFFYWBBREREVIZ0zRT1LCMd/9yPR321usyOw7UxqKphYEFERERUhnTNFPUwOgpx97YiJ7vsAguujUFVDQMLIiIionKQd6ao1EcJ5Xacslwbozy6VrG7Vu3BwIKIiIiokuQfiwFU3kN3eXStYnet2oWBBREREVElKOmq3fl/+S/rAeHl0bWK3bVqFwYWRERERJWgoFW7H5z4Dn///Tfc3d2lvI8ePcKc+cFIzfy/MRrlMSAcKNuuVeVZJlU9DCyIiIiIKlHesRgFtWJogoj2w2bAxvl53vIYEF4dcMxG1cXAgoiIiKiK0NWKAfxfEGGqtCtyQHhVGrdR1qrLmI3aGvwwsCAiIiKqYvK2YgDFn1WqJOM2ynvMRnmoDmM29A1+qnNQwsCCiIiIqIYo7riNihyzUR6q8pgNfYKf6tIiUxAGFkREREQ1TFHjNkoyZqM8ulbV5O5aGqUJfqpDi0xhGFgQERER1WCFrQRe1JiNkk6JWxzlUaY+dHU9Aio30KnKLTKFYWBBREREVAuUZiXwkkyJm5WVBRMTE9n+usZtFFSmrl/ly3scSEFdj4Dq0/2oKmFgQURERESFKqprlTorC/fvxqGeuweMjP/v8bKwcRv5B6g/yNc9qiLGgejqegRUr+5HVQkDCyIiIiIqtoK6Vt2J3QrbDv10TpNb1Fob+o4D0Vf+rkdA9el+VJUwsCAiIiKiEtPVtaq00+TqMw6kutA1YD1/97HqMOVvYRhYEBEREVGVUJpxIEDVn2WquN3HqtOUv7owsCAiIiKiaqugWabqmBhi+SeLYW9vL6VVVrBR3O5j5dXVq6IwsCAiIiKiakvXQ3vyP7dx/vvPMWHqrFJPaVucrku60grrzlRU97Hq3NULYGBBRERERDVA/gf04kyTW1AQUNyuSzWxO5M+GFgQERERUY1U3BXI8wcBJem6VNO6M+mDgQURERER1XiFzTxVUBBQ3K5LNak7kz4YWBARERFRrcEgoPwYVHYFiIiIiIio+mNgQUREREREemNgQUREREREemNgQUREREREemNgQUREREREemNgQUREREREemNgQUREREREemNgQUREREREeqvSgcXSpUvx4osvwsrKCk5OTujfvz9u3Lghy+Pn5weFQiF7TZo0SZbn7t27CAgIgIWFBZycnDB79mxkZ2fL8hw/fhzt2rWDqakpGjdujLCwsPI+PSIiIiKiGqNKBxYnTpxAUFAQzpw5g/DwcKjVavTo0QNpaWmyfG+//Tbi4+Ol14oVK6RtOTk5CAgIQFZWFk6fPo2vv/4aYWFhWLhwoZQnJiYGAQEB6NatGyIjIzF9+nRMmDABv/32W4WdKxERERFRdWZU2RUozMGDB2Xvw8LC4OTkhPPnz6NLly5SuoWFBVxcXHSWcejQIVy9ehWHDx+Gs7Mz2rRpgyVLlmDOnDkIDg6GiYkJQkJC4OHhgVWrVgEAWrRogd9//x1r1qyBv79/+Z0gEREREVENUaVbLPJLSUkBANjZ2cnSt2/fDgcHB3h5eWHu3LlIT0+XtkVERMDb2xvOzs5Smr+/P1QqFa5cuSLl6d69u6xMf39/RERElNepEBERERHVKFW6xSKv3NxcTJ8+HS+//DK8vLyk9BEjRsDd3R2urq64fPky5syZgxs3bmDv3r0AgISEBFlQAUB6n5CQUGgelUqFjIwMmJuba9UnMzMTmZmZ0nuVSlU2J0pEREREVA1Vm8AiKCgIUVFR+P3332XpEydOlP7f29sbdevWxWuvvYbo6Gg0atSo3OqzdOlSLFq0qNzKJyIiIiKqTqpFV6jJkyfjl19+wbFjx1CvXr1C83bs2BEAcPv2bQCAi4sLEhMTZXk07zXjMgrKo1QqdbZWAMDcuXORkpIive7du1fyEyMiIiIiqiGqdGAhhMDkyZPx448/4ujRo/Dw8Chyn8jISABA3bp1AQC+vr74+++/8fDhQylPeHg4lEolWrZsKeU5cuSIrJzw8HD4+voWeBxTU1MolUrZi4iIiIiotqrSgUVQUBC2bduGHTt2wMrKCgkJCUhISEBGRgYAIDo6GkuWLMH58+cRGxuLn3/+GaNHj0aXLl3QqlUrAECPHj3QsmVLjBo1CpcuXcJvv/2G+fPnIygoCKampgCASZMm4c6dO/jggw9w/fp1bNq0Cd9//z1mzJhRaedORERERFSdVOnA4osvvkBKSgr8/PxQt25d6bVr1y4AgImJCQ4fPowePXqgefPmeP/99zFw4ED873//k8owNDTEL7/8AkNDQ/j6+mLkyJEYPXo0Fi9eLOXx8PDA/v37ER4ejtatW2PVqlXYsmULp5olIiIiIiqmKj14WwhR6HY3NzecOHGiyHLc3d1x4MCBQvP4+fnh4sWLJaofERERERE9V6VbLIiIiIiIqHpgYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYEFERERERHpjYJHPxo0b0aBBA5iZmaFjx474888/K7tKRERERERVHgOLPHbt2oWZM2fio48+woULF9C6dWv4+/vj4cOHlV01IiIiIqIqjYFFHqtXr8bbb7+NcePGoWXLlggJCYGFhQW2bt1a2VUjIiIiIqrSjCq7AlVFVlYWzp8/j7lz50ppBgYG6N69OyIiIrTyZ2ZmIjMzU3qfkpICAFCpVOVfWR2ePn2KnOxsPImPhfpZ+vO6PPwHIjcXqoR7MFKgwtIq89isd+XXh/WuPmlVrT6sd9VMq2r1qen1rknnUtXqU13rnfb4IXKys/H06dNKec7UHFMIUWRehShOrlrgwYMHeOGFF3D69Gn4+vpK6R988AFOnDiBs2fPyvIHBwdj0aJFFV1NIiIiIqIKd+/ePdSrV6/QPGyxKKW5c+di5syZ0vvc3FwkJyfD3t4eCoWikD0LplKp4Obmhnv37kGpVJZVVamK4vWuXXi9axde79qD17p2qY3XWwiBp0+fwtXVtci8DCz+PwcHBxgaGiIxMVGWnpiYCBcXF638pqamMDU1laXZ2NiUSV2USmWt+bISr3dtw+tdu/B61x681rVLbbve1tbWxcrHwdv/n4mJCXx8fHDkyBEpLTc3F0eOHJF1jSIiIiIiIm1sschj5syZGDNmDNq3b48OHTpg7dq1SEtLw7hx4yq7akREREREVRoDizyGDh2KpKQkLFy4EAkJCWjTpg0OHjwIZ2fnCjm+qakpPvroI60uVlQz8XrXLrzetQuvd+3Ba1278HoXjrNCERERERGR3jjGgoiIiIiI9MbAgoiIiIiI9MbAgoiIiIiI9MbAgoiIiIiI9MbAooydPHkSffr0gaurKxQKBfbt2yfbLoTAwoULUbduXZibm6N79+64deuWLE9ycjLeeustKJVK2NjYIDAwEKmpqbI8ly9fRufOnWFmZgY3NzesWLGivE+NdCjseqvVasyZMwfe3t6wtLSEq6srRo8ejQcPHsjK4PWuPoq6v/OaNGkSFAoF1q5dK0vn9a4einOtr127hr59+8La2hqWlpZ48cUXcffuXWn7s2fPEBQUBHt7e9SpUwcDBw7UWoT17t27CAgIgIWFBZycnDB79mxkZ2eX9+lRPkVd79TUVEyePBn16tWDubk5WrZsiZCQEFkeXu/qY+nSpXjxxRdhZWUFJycn9O/fHzdu3JDlKavrefz4cbRr1w6mpqZo3LgxwsLCyvv0KhUDizKWlpaG1q1bY+PGjTq3r1ixAp9//jlCQkJw9uxZWFpawt/fH8+ePZPyvPXWW7hy5QrCw8Pxyy+/4OTJk5g4caK0XaVSoUePHnB3d8f58+excuVKBAcHY/PmzeV+fiRX2PVOT0/HhQsXsGDBAly4cAF79+7FjRs30LdvX1k+Xu/qo6j7W+PHH3/EmTNn4OrqqrWN17t6KOpaR0dH45VXXkHz5s1x/PhxXL58GQsWLICZmZmUZ8aMGfjf//6H3bt348SJE3jw4AEGDBggbc/JyUFAQACysrJw+vRpfP311wgLC8PChQvL/fxIrqjrPXPmTBw8eBDbtm3DtWvXMH36dEyePBk///yzlIfXu/o4ceIEgoKCcObMGYSHh0OtVqNHjx5IS0uT8pTF9YyJiUFAQAC6deuGyMhITJ8+HRMmTMBvv/1WoedboQSVGwDixx9/lN7n5uYKFxcXsXLlSintyZMnwtTUVHz33XdCCCGuXr0qAIi//vpLyvPrr78KhUIh7t+/L4QQYtOmTcLW1lZkZmZKeebMmSOaNWtWzmdEhcl/vXX5888/BQARFxcnhOD1rs4Kut7//POPeOGFF0RUVJRwd3cXa9askbbxeldPuq710KFDxciRIwvc58mTJ8LY2Fjs3r1bSrt27ZoAICIiIoQQQhw4cEAYGBiIhIQEKc8XX3whlEql7PpTxdJ1vT09PcXixYtlae3atRMffvihEILXu7p7+PChACBOnDghhCi76/nBBx8IT09P2bGGDh0q/P39y/uUKg1bLCpQTEwMEhIS0L17dynN2toaHTt2REREBAAgIiICNjY2aN++vZSne/fuMDAwwNmzZ6U8Xbp0gYmJiZTH398fN27cwOPHjyvobKg0UlJSoFAoYGNjA4DXu6bJzc3FqFGjMHv2bHh6empt5/WuGXJzc7F//340bdoU/v7+cHJyQseOHWXdZ86fPw+1Wi37e9+8eXPUr19f9vfe29tbtgirv78/VCoVrly5UmHnQ0Xr1KkTfv75Z9y/fx9CCBw7dgw3b95Ejx49APB6V3cpKSkAADs7OwBldz0jIiJkZWjyaMqoiRhYVKCEhAQA0FrJ29nZWdqWkJAAJycn2XYjIyPY2dnJ8ugqI+8xqOp59uwZ5syZg+HDh0OpVALg9a5pli9fDiMjI0ydOlXndl7vmuHhw4dITU3FsmXL0LNnTxw6dAhvvvkmBgwYgBMnTgB4fq1MTEykHxE08v+957WuHtavX4+WLVuiXr16MDExQc+ePbFx40Z06dIFAK93dZabm4vp06fj5ZdfhpeXF4Cyu54F5VGpVMjIyCiP06l0RpVdAaLaQK1WY8iQIRBC4Isvvqjs6lA5OH/+PNatW4cLFy5AoVBUdnWoHOXm5gIA+vXrhxkzZgAA2rRpg9OnTyMkJARdu3atzOpROVi/fj3OnDmDn3/+Ge7u7jh58iSCgoLg6uqq9Ys0VS9BQUGIiorC77//XtlVqRHYYlGBXFxcAEBrVoHExERpm4uLCx4+fCjbnp2djeTkZFkeXWXkPQZVHZqgIi4uDuHh4VJrBcDrXZOcOnUKDx8+RP369WFkZAQjIyPExcXh/fffR4MGDQDwetcUDg4OMDIyQsuWLWXpLVq0kGaFcnFxQVZWFp48eSLLk//vPa911ZeRkYF58+Zh9erV6NOnD1q1aoXJkydj6NCh+OyzzwDweldXkydPxi+//IJjx46hXr16UnpZXc+C8iiVSpibm5f16VQJDCwqkIeHB1xcXHDkyBEpTaVS4ezZs/D19QUA+Pr64smTJzh//ryU5+jRo8jNzUXHjh2lPCdPnoRarZbyhIeHo1mzZrC1ta2gs6Hi0AQVt27dwuHDh2Fvby/bzutdc4waNQqXL19GZGSk9HJ1dcXs2bOlGUB4vWsGExMTvPjii1rTU968eRPu7u4AAB8fHxgbG8v+3t+4cQN3796V/b3/+++/ZcGm5seH/EELVR61Wg21Wg0DA/kjk6GhodR6xetdvQghMHnyZPz44484evQoPDw8ZNvL6nr6+vrKytDk0ZRRI1Xy4PEa5+nTp+LixYvi4sWLAoBYvXq1uHjxojQL0LJly4SNjY346aefxOXLl0W/fv2Eh4eHyMjIkMro2bOnaNu2rTh79qz4/fffRZMmTcTw4cOl7U+ePBHOzs5i1KhRIioqSuzcuVNYWFiIL7/8ssLPt7Yr7HpnZWWJvn37inr16onIyEgRHx8vvfLOAMLrXX0UdX/nl39WKCF4vauLoq713r17hbGxsdi8ebO4deuWWL9+vTA0NBSnTp2Sypg0aZKoX7++OHr0qDh37pzw9fUVvr6+0vbs7Gzh5eUlevToISIjI8XBgweFo6OjmDt3boWfb21X1PXu2rWr8PT0FMeOHRN37twRoaGhwszMTGzatEkqg9e7+nj33XeFtbW1OH78uOzf5vT0dClPWVzPO3fuCAsLCzF79mxx7do1sXHjRmFoaCgOHjxYoedbkRhYlLFjx44JAFqvMWPGCCGeTzm7YMEC4ezsLExNTcVrr70mbty4ISvj0aNHYvjw4aJOnTpCqVSKcePGiadPn8ryXLp0SbzyyivC1NRUvPDCC2LZsmUVdYqUR2HXOyYmRuc2AOLYsWNSGbze1UdR93d+ugILXu/qoTjX+quvvhKNGzcWZmZmonXr1mLfvn2yMjIyMsR7770nbG1thYWFhXjzzTdFfHy8LE9sbKzo1auXMDc3Fw4ODuL9998XarW6Ik6R8ijqesfHx4uxY8cKV1dXYWZmJpo1ayZWrVolcnNzpTJ4vauPgv5tDg0NlfKU1fU8duyYaNOmjTAxMRENGzaUHaMmUgghRHm1hhARERERUe3AMRZERERERKQ3BhZERERERKQ3BhZERERERKQ3BhZERERERKQ3BhZERERERKQ3BhZERERERKQ3BhZERERERKQ3BhZERERERKQ3BhZERFSt+Pn5Yfr06ZVdDSIiyoeBBRERVZg+ffqgZ8+eOredOnUKCoUCly9fruBaERFRWWBgQUREFSYwMBDh4eH4559/tLaFhoaiffv2aNWqVSXUjIiI9MXAgoiIKswbb7wBR0dHhIWFydJTU1Oxe/du9O/fH8OHD8cLL7wACwsLeHt747vvviu0TIVCgX379snSbGxsZMe4d+8ehgwZAhsbG9jZ2aFfv36IjY0tm5MiIiIADCyIiKgCGRkZYfTo0QgLC4MQQkrfvXs3cnJyMHLkSPj4+GD//v2IiorCxIkTMWrUKPz555+lPqZarYa/vz+srKxw6tQp/PHHH6hTpw569uyJrKyssjgtIiICAwsiIqpg48ePR3R0NE6cOCGlhYaGYuDAgXB3d8esWbPQpk0bNGzYEFOmTEHPnj3x/fffl/p4u3btQm5uLrZs2QJvb2+0aNECoaGhuHv3Lo4fP14GZ0RERAADCyIiqmDNmzdHp06dsHXrVgDA7du3cerUKQQGBiInJwdLliyBt7c37OzsUKdOHfz222+4e/duqY936dIl3L59G1ZWVqhTpw7q1KkDOzs7PHv2DNHR0WV1WkREtZ5RZVeAiIhqn8DAQEyZMgUbN25EaGgoGjVqhK5du2L58uVYt24d1q5dC29vb1haWmL69OmFdllSKBSyblXA8+5PGqmpqfDx8cH27du19nV0dCy7kyIiquUYWBARUYUbMmQIpk2bhh07duCbb77Bu+++C4VCgT/++AP9+vXDyJEjAQC5ubm4efMmWrZsWWBZjo6OiI+Pl97funUL6enp0vt27dph165dcHJyglKpLL+TIiKq5dgVioiIKlydOnUwdOhQzJ07F/Hx8Rg7diwAoEmTJggPD8fp06dx7do1vPPOO0hMTCy0rFdffRUbNmzAxYsXce7cOUyaNAnGxsbS9rfeegsODg7o168fTp06hZiYGBw/fhxTp07VOe0tERGVDgMLIiKqFIGBgXj8+DH8/f3h6uoKAJg/fz7atWsHf39/+Pn5wcXFBf379y+0nFWrVsHNzQ2dO3fGiBEjMGvWLFhYWEjbLSwscPLkSdSvXx8DBgxAixYtEBgYiGfPnrEFg4ioDClE/o6pREREREREJcQWCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDCyIiIiIi0hsDC6IaoEGDBhg7dmxlV6PGW7lyJRo2bAhDQ0O0adOmwHwNGjSAQqGAQqHA5MmTK66CVKCrV6/CyMgIUVFRJdpvxYoVaN68OXJzc8upZkRl6z//+Q86duxY2dWgWoqBBVEVExYWBoVCgXPnzunc7ufnBy8vL72Pc+DAAQQHB+tdTm1x6NAhfPDBB3j55ZcRGhqKTz/9tND8nTt3xrfffosxY8ZIaffu3cOiRYvQoUMH2NrawsHBAX5+fjh8+LDOMp48eYKJEyfC0dERlpaW6NatGy5cuFDocaOjo2FmZlbgd+j8+fN444034OLigjp16qBVq1b4/PPPkZOTU4xPQbdbt25h2LBhqFevHiwsLNC8eXMsXrwY6enpsnyHDh1CYGAgvLy8YGhoiAYNGhRY5u3btzFo0CDY2trCwsICr7zyCo4dO6aV77///S+6du0KZ2dnmJqawsPDA+PGjUNsbKwsX8uWLREQEICFCxcW+7xUKhWWL1+OOXPmwMDg//651ASNCoUCRkZGsLOzg4+PD6ZNm4arV68Wu/z80tPTERwcjOPHj5e6jMKoVCosWrQIrVu3Rp06dWBubg4vLy/MmTMHDx48KJdjllRZ/V2aMWMG2rVrBzs7O1hYWKBFixYIDg5GamqqVt7MzEzMmTMHrq6uMDc3R8eOHREeHq6VLzc3FyEhIWjTpg3q1KkDZ2dn9OrVC6dPn5blS01NxUcffYSePXvCzs4OCoUCYWFhOuv5559/4r333oOPjw+MjY2hUCh05ivJ347p06fj0qVL+Pnnn4vxSRGVMUFEVUpoaKgAIP766y+d27t27So8PT1lac+ePRNZWVklOk5QUJDgn4DimzNnjjAwMBCZmStuH0YAAK/aSURBVJlF5nV3dxdjxozRSl+/fr0wNzcXw4cPFxs2bBBr164V7dq1EwDE1q1bZXlzcnJEp06dhKWlpQgODhYbNmwQLVu2FFZWVuLmzZsFHrtPnz7C0tJS53fo3LlzwsTERHh6eorVq1eLkJAQ0a9fPwFATJ06tXgfRD53794VNjY2wt3dXSxdulR8+eWXYuzYsQKA6Nu3ryzvmDFjhJmZmejUqZOoV6+ecHd3L7BMBwcH4ezsLD755BOxdu1a0bp1a2FkZCROnDghy/vuu++KMWPGiM8++0x89dVXYv78+cLZ2Vk4ODiI+/fvy/IeOHBAABC3b98u1rmtWbNGKJVKkZGRIUsHIF5//XXx7bffim+++UasX79eTJgwQVhbWwsjIyOxatWqYpWfX1JSkgAgPvroo1LtX5jo6Gjh4eEhDA0NxbBhw8SGDRvE5s2bxeTJk4W9vb1o0qRJmR+zNMrq79LLL78spk6dKj7//HOxefNm8e677wpTU1Px8ssvi5ycHFneYcOGCSMjIzFr1izx5ZdfCl9fX2FkZCROnTolyzdz5kwBQIwcOVJ8+eWXYvny5aJhw4bCyMhInD17VsoXExMjAIj69esLPz8/AUCEhobqrOdHH30kjI2NhY+Pj2jatGmB516Svx1CCDFkyBDRuXPnEn5qRPrjUwVRFVOawKI0KjuwSE1NrbRjl8a4ceOEpaVlsfIWFFhERUWJpKQkWdqzZ89E8+bNRb169WTpu3btEgDE7t27pbSHDx8KGxsbMXz4cJ3HPXjwoDAxMRHz58/X+R16++23hYmJiXj06JEsvUuXLkKpVBbr3PL75JNPBAARFRUlSx89erQAIJKTk6W0+/fvSwFwQEBAgYHFe++9J4yMjMT169eltLS0NOHm5ibatWtXZJ3OnTsnAIilS5fK0rOysoStra1YsGBBsc6tVatWYuTIkVrpAERQUJBW+r///it8fX0FALF///5iHSOv8gos1Gq1aN26tbCwsNB6WBZCiJSUFDFv3rwyPWZpleffpc8++0wAEBEREVLa2bNnBQCxcuVKKS0jI0M0atRI+Pr6SmlqtVqYm5uLQYMGycq8c+eOVmD+7NkzER8fL4QQ4q+//io0sEhISBDp6elCiMLPvSR/O4QQYs+ePUKhUIjo6Gid5RGVF3aFIqoB8o+xUKvVWLRoEZo0aQIzMzPY29vjlVdekZr3x44di40bNwKQd+vQSEtLw/vvvw83NzeYmpqiWbNm+OyzzyCEkB03IyMDU6dOhYODA6ysrNC3b1/cv38fCoVC1p0hODgYCoUCV69exYgRI2Bra4tXXnkFAHD58mWMHTsWDRs2hJmZGVxcXDB+/Hg8evRIdixNGTdv3sTIkSNhbW0NR0dHLFiwAEII3Lt3D/369YNSqYSLiwtWrVpVrM8uOzsbS5YsQaNGjWBqaooGDRpg3rx5yMzMlPIoFAqEhoYiLS1N+qwK6tpQGE9PTzg4OMjSTE1N0bt3b/zzzz94+vSplL5nzx44OztjwIABUpqjoyOGDBmCn376SVY/4Pk1nzZtGqZNm4ZGjRrpPL5KpYKZmRlsbGxk6XXr1oW5uXmJz0dTJgA4OztrlWlgYAATExMpzdXVFcbGxkWWeerUKbRt2xbNmjWT0iwsLNC3b19cuHABt27dKnR/TRerJ0+eyNKNjY3h5+eHn376qcg6xMTE4PLly+jevXuReTXs7e2xc+dOGBkZ4ZNPPpHSs7KysHDhQvj4+MDa2hqWlpbo3LmzrGtXbGwsHB0dAQCLFi2Svmea+6i494kuP/zwAy5duoQPP/xQuu/yUiqVsvoCwO7du+Hj4wNzc3M4ODhg5MiRuH//viyPn58f/Pz8tMobO3asrJtbbGwsFAoFPvvsM2zevFm611588UX89ddfsv0K+7sUHx+P69evQ61WF3nOuuj6XuzZsweGhoaYOHGilGZmZobAwEBERETg3r17AJ7fXxkZGVrfcycnJxgYGMjuH1NTU7i4uBSrTs7OzsW690rytwOA9L0tznedqCwxsCCqolJSUvDvv/9qvYrzj2pwcDAWLVqEbt26YcOGDfjwww9Rv359qX/+O++8g9dffx0A8O2330ovABBCoG/fvlizZg169uyJ1atXo1mzZpg9ezZmzpwpO87YsWOxfv169O7dG8uXL4e5uTkCAgIKrNfgwYORnp6OTz/9FG+//TYAIDw8HHfu3MG4ceOwfv16DBs2DDt37kTv3r21AhkAGDp0KHJzc7Fs2TJ07NgRH3/8MdauXYvXX38dL7zwApYvX47GjRtj1qxZOHnyZJGf1YQJE7Bw4UK0a9cOa9asQdeuXbF06VIMGzZMyvPtt9+ic+fOMDU1lT6rLl26FFl2cSUkJMDCwgIWFhZS2sWLF9GuXTtZ334A6NChA9LT03Hz5k1Z+tq1a/H48WPMnz+/wOP4+flBpVLhnXfewbVr1xAXF4eQkBDs3bsXc+fOLVXdNQ+WgYGBiIyMxL1797Br1y588cUXmDp1KiwtLUtcZmZmps6HLc3nc/78ea1tjx49wsOHD3Hu3DmMGzcOAPDaa69p5fPx8UFUVJQUEBVE02++Xbt2Jap7/fr10bVrV5w5c0Y6hkqlwpYtW+Dn54fly5cjODgYSUlJ8Pf3R2RkJIDnQeMXX3wBAHjzzTel75kmsCzpfZKXpq/9qFGjinUOYWFhGDJkCAwNDbF06VK8/fbb2Lt3L1555RWtYK0kduzYgZUrV+Kdd97Bxx9/jNjYWAwYMED6m1bY3yUAmDt3Llq0aKEV4BQkOzsb//77Lx48eIBDhw5h/vz5sLKyQocOHaQ8Fy9eRNOmTaFUKmX7avJoro9m7EVYWBi2b9+Ou3fvSsGera2tLDCpSLr+dgCAtbU1GjVqhD/++KNS6kW1WKW2lxCRFk1XqMJe+btC5e9607p1axEQEFDocQpqdt+3b58AID7++GNZ+qBBg4RCoZD6p58/f14AENOnT5fl0/Svz9ud46OPPhIAdHbh0XQDyOu7774TAMTJkye1ypg4caKUlp2dLerVqycUCoVYtmyZlP748WNhbm6usztSXpGRkQKAmDBhgix91qxZAoA4evSolDZmzBi9u0LpcuvWLWFmZiZGjRolS7e0tBTjx4/Xyr9//34BQBw8eFBKi4+PF1ZWVuLLL78UQhTcnS47O1tMnjxZGBsbS98lQ0ND8cUXXxSrrgVZsmSJMDc3l31HP/zww0L3KawrVJ8+fYSNjY1QqVSydE03o88++0xrH1NTU+nY9vb24vPPP9dZ9o4dOwQAWZ94XTTdyZ4+faq1DQV0hdKYNm2aACAuXbokhHj+uecfm/P48WPh7Owsu8aFdYUq7n2iS9u2bYW1tXWheTSysrKEk5OT8PLyko0t+eWXXwQAsXDhQimta9euomvXrlpljBkzRnZtNWMO7O3tZV3jfvrpJwFA/O9//5PSCusONGbMGAFAxMTEFOtcIiIiZN/JZs2aiWPHjsnyeHp6ildffVVr3ytXrggAIiQkREq7deuWNK5B82rYsKGsy15+RXWFyquk3cAK+tuh0aNHD9GiRYtil0dUFthiQVRFbdy4EeHh4VqvVq1aFbmvjY0Nrly5UmSXEV0OHDgAQ0NDTJ06VZb+/vvvQwiBX3/9FQBw8OBBAMB7770nyzdlypQCy540aZJWWt5fpp89e4Z///0XL730EgDonAFpwoQJ0v8bGhqiffv2EEIgMDBQSrexsUGzZs1w586dAusCPD9XAFotMe+//z4AYP/+/YXur6/09HQMHjwY5ubmWLZsmWxbRkYGTE1NtfYxMzOTtmvMmTMHDRs2lH02uhgaGqJRo0bw9/fH119/jV27dqFPnz6YMmUK9u3bV+rzaNCgAbp06YLNmzfjhx9+wPjx4/Hpp59iw4YNpSrv3XffxZMnTzB06FBcvHgRN2/exPTp06VZrvKeu8avv/6KAwcOYNWqVahfvz7S0tJ0lm1rawsA+Pfffwutw6NHj2BkZIQ6deqUuP6afTTdUwwNDaUuYbm5uUhOTkZ2djbat29f5CxfGiW9T/JSqVSwsrIq1nHOnTuHhw8f4r333pO+awAQEBCA5s2b63VPDB06VPr8geczpwEo8j7VCAsLgxCi0NnE8mrZsiXCw8Oxb98+fPDBB7C0tNSaFaok95mVlRU8PT0RFBSEvXv3YtOmTcjOzkb//v2L/D6VtcL+dmjY2tpWeL2IjCq7AkSkW4cOHdC+fXut9OL8Y7F48WL069cPTZs2hZeXF3r27IlRo0YVKyiJi4uDq6ur1oNIixYtpO2a/xoYGMDDw0OWr3HjxgWWnT8vACQnJ2PRokXYuXMnHj58KNuWkpKilb9+/fqy99bW1jAzM9Pqf2xtbV1k/3PNOeSvs4uLC2xsbKRzLQ85OTkYNmwYrl69il9//RWurq6y7ebm5lrjKIDnD5Wa7QBw5swZfPvttzhy5IhWt6n8li1bhnXr1uHWrVvSw++QIUPQrVs3BAUF4Y033oCRUcn+Wdi5cycmTpyImzdvol69egCAAQMGIDc3F3PmzMHw4cNhb29fojJ79eqF9evX4z//+Y/UFalx48b45JNP8MEHH+h82O/WrZu0b79+/eDl5YU6deporSMi/n+3oYKm9SwLmofXvPfQ119/jVWrVmmNEdB1T+hS0vskL6VSWeyHd813Pu/4Fo3mzZvj999/L1Y5uuS/dzVBxuPHj0tdZmGUSqU01qBfv37YsWMH+vXrhwsXLqB169YAin+fZWdno3v37vDz88P69eulfN27d4enpydWrlyJ5cuXl8t55FfU3w4NIUS5fs+JdGGLBVEN1KVLF0RHR2Pr1q3w8vLCli1b0K5dO2zZsqVS66Wr3/yQIUPw3//+F5MmTcLevXtx6NAhqTVE16JkhoaGxUoDUGTfc43K+Mf37bffxi+//IKwsDC8+uqrWtvr1q2L+Ph4rXRNmuZh4oMPPkDnzp3h4eGB2NhYxMbGSoFnfHw87t69K+27adMmvPrqq1oP5n379sWDBw+01n4ojk2bNqFt27ZSUJG3zPT0dFy8eLHEZQLA5MmTkZiYiNOnT+PcuXO4fv06rK2tAQBNmzYtdN9GjRqhbdu22L59u9Y2zUNs/kA0P3t7e2RnZ2sNii2OqKgoGBoaSkHDtm3bMHbsWDRq1AhfffUVDh48iPDwcLz66qvFXnivpPdJXs2bN0dKSoo0ELmsFHTfFLQmir73qb4041V27twppRX3Pjt58iSioqLQt29fWb4mTZqgRYsWFTqWoai/HRqPHz8u8ntOVNbYYkFUQ9nZ2WHcuHEYN24cUlNT0aVLFwQHB0vdZQp6KHB3d8fhw4fx9OlT2S+u169fl7Zr/pubm4uYmBg0adJEynf79u1i1/Hx48c4cuQIFi1aJFu4rDRduEpDcw63bt2SWmQAIDExEU+ePJHOtazNnj0boaGhWLt2LYYPH64zT5s2bXDq1Cnk5ubKWiLOnj0LCwsL6eH67t27iIuL0/nLd9++fWFtbS0NuE1MTNT50Kf5BT07O7vE55KYmCjr3lIWZWpYWlrC19dXen/48GGYm5vj5ZdfLnLfjIwMnb9Ex8TEwMDAoMjgpHnz5lL+4rT0ady9excnTpyAr6+vdP/s2bMHDRs2xN69e2X33UcffSTbt6B7Ut/7pE+fPvjuu++wbdu2Igfpa77zN27c0HpovXHjhuyesLW11dkSok9LX3kG+ZmZmcjNzZW18LRp0wbHjh2DSqWSDeA+e/astB14/j0HdAdNarVar+95SRTnb4dGTEyM1DJDVFHYYkFUA+XvAlSnTh00btxY9qClma0n/ywvvXv3Rk5Ojlb/+DVr1kChUKBXr14AAH9/fwDPf7HOK283gaJofsHM/4vl2rVri12GPnr37q3zeKtXrwaAQme4Kq2VK1fis88+w7x58zBt2rQC8w0aNAiJiYnYu3evlPbvv/9i9+7d6NOnj9QvfPPmzfjxxx9lL804l88++0z2q33Tpk0RHh4u+37k5OTg+++/h5WVVYHT1BamadOm0jiIvL777jsYGBiU6KG8MKdPn8bevXsRGBgotVxkZ2fr7Ebz559/4u+//9bZlfD8+fPw9PSUyiiIJqDRtXp5QZKTkzF8+HDk5OTgww8/lNJ1fc/Pnj2LiIgI2f6amX3y35P63ieDBg2Ct7c3PvnkE61jAs/Hgmjq2759ezg5OSEkJET29+LXX3/FtWvXZPdEo0aNcP36dSQlJUlply5d0uvX+4L+LgHFn272yZMnOvNoWmzzfi8GDRqEnJwcbN68WUrLzMxEaGgoOnbsCDc3NwD/10qWt7UDeD6+5caNG2jbtm0xzk4/xf3bATzvHhcdHY1OnTqVe72I8mKLBVEN1LJlS/j5+cHHxwd2dnY4d+4c9uzZI+tv7uPjAwCYOnUq/P39YWhoiGHDhqHP/2PvvuOjqvL/j78H0oEkBFJAQgjSO4JALBSNBkWlqYh0UBcFpagoihRRUVwQFJR10YALSFEsCwLGUERAkRI6SAlNSMgKyRBKEpLz+8Nf5suQQsgNKfB6Ph7z2J17ztz7uTMZvO+555778MNq166dXn/9dR0+fFiNGzfWjz/+qO+++05Dhw51HHw2a9ZMXbt21ZQpU/TXX3+pVatWWrNmjeMAMy+/PHp7e6t169aaOHGi0tLSdMstt+jHH39UbGzsdXhXsmrcuLH69OmjTz/9VImJiWrTpo02btyo2bNnq1OnTo5x+wXlm2++0YgRIxzDJ+bMmePUft999znmyX/00UfVqlUr9evXT7t371bFihX18ccfKz09XePGjXO85v7778+yncyDsjZt2jgdRL366qvq2bOnWrZsqWeeeUaenp768ssvtXnzZr311ltO95jo27evZs+erdjY2Fwvln355Ze1bNky3X333Ro8eLAqVKigJUuWaNmyZXrqqaecxn9v377dMfXpgQMHlJSUpLfeekvS35/Fww8/LOnvX7wff/xxPfLIIwoKCtKuXbs0Y8YMNWrUSO+8845jfcnJyQoODla3bt1Uv359lSlTRjt27FBkZKR8fHz0xhtvONWalpamNWvWZJlwIDvVq1dXgwYN9NNPP6l///5Z2v/44w/NmTNHxhjZ7XZt27ZNixYtUnJysiZPnqz27ds7+j700ENavHixOnfurA4dOig2NlYzZsxQvXr1nC4m9vT0VL169bRgwQLVqlVLfn5+atCggRo0aGDpe+Lq6qrFixcrPDxcrVu31uOPP64777xTrq6u2rVrl+bNm6fy5cvr7bfflqurq9577z3169dPbdq0Uffu3RUfH6+pU6eqWrVqGjZsmGO9/fv31+TJkxUREaEBAwbo1KlTmjFjhurXr3/V6XxzktO/S9Lf083m5W9y9erVeuGFF/Too4+qZs2aSk1N1dq1a7V48WI1b95cPXv2dPRt2bKlHnvsMY0cOVKnTp1SjRo1NHv2bB0+fFifffaZU1333XefZs+eLbvdrvvvv18nT57URx99JE9PTw0dOtSphmnTpikxMVEnTpyQJP33v//V8ePHJf09wUVmsD1y5IhjSt3MEJv5nQgJCXFMEXwt/3ZIf5/dM8aoY8eOeXjXgQJUNJNRAchJfu68feX0pm+99ZZp0aKF8fX1NZ6enqZOnTrm7bffdtz12Ji/p8B8/vnnjb+/v7HZbE7THJ49e9YMGzbMVK5c2bi6upqaNWua999/32RkZDht99y5c2bQoEHGz8/PlC1b1nTq1Mns27fPSHKa/jVzqtgr7xxrjDHHjx83nTt3Nr6+vsbHx8c89thj5sSJEzlOWXvlOnKaBjavdyhPS0sz48aNM6GhocbV1dUEBwebkSNHmosXL+ZpO9nJabrZzH3I6XHlVJinT582AwYMMBUqVDBeXl6mTZs2Of5dXC63v6Hly5ebNm3amIoVKxo3NzfTsGFDpyk1M3Xt2tV4enqaM2fOXHV7v/32m3nggQdMUFCQcXV1NbVq1TJvv/22SUtLy7au7B6Xv1+nT582HTt2NEFBQcbNzc2EhoaaV155Jcv0sykpKWbIkCGmUaNGxtvb27i6upqQkBAzYMCAbKckXbZsmZFk9u/ff9V9MsaYyZMnm7Jly2aZ6vXyukuVKmV8fX1N06ZNzZAhQ8yuXbuyrCcjI8O88847JiQkxLi7u5umTZuaJUuWZJmW1Rhj1q9fb5o1a2bc3NycvgN5/Z7k5syZM2b06NGmYcOGxsvLy3h4eJgGDRqYkSNHOu4UnWnBggWmadOmxt3d3fj5+ZkePXqY48ePZ1nnnDlzTPXq1Y2bm5tp0qSJWbFiRY7TzV5+d+vL38vL68/t36W8Tjd74MAB07t3b1O9enXj6elpPDw8TP369c2YMWNMcnJylv4XLlwwL730kgkKCjLu7u7m9ttvd5rOOdP58+fNm2++aerVq2c8PT2Nj4+Peeihh8zWrVuz9A0JCcnxb/3y+letWpVjv8un8r3Wfzu6detm7rrrrlzfJ+B6sBlTSFdNAbgpxMTEqGnTppozZ4569OhR1OUUiWrVqiksLMzxa2Z+bhJX1AIDA9W7d2+9//77RV1KgenUqZNsNpu++eabPPVPSkpS9erVNXHiRKfpjIHiLC4uTqGhoZo/fz5nLFDouMYCQL5ldz+BKVOmqFSpUgV6Z+qSaP78+fL399crr7xS1KVcs127dunChQslsvac7NmzR0uWLNH48ePz/BofHx+NGDFC77//fp5nbwKK2pQpU9SwYUNCBYoEZywA5Nu4ceO0efNmtWvXTi4uLlq2bJmWLVumZ555Rv/617+Kurwis27dOkfoCg4OzvaeAAAA3GgIFgDyLSoqSuPGjdPu3buVnJysqlWrqlevXnr99dev+UZrAACgZCNYAAAAALCMaywAAAAAWEawAAAAAGAZg6ALSEZGhk6cOKFy5crl6cZgAAAAQHFnjNHZs2dVuXJllSqV+zkJgkUBOXHihIKDg4u6DAAAAKDAHTt2TFWqVMm1D8GigJQrV07S32+6t7d3EVcDAAAAWGe32xUcHOw41s0NwaKAZA5/8vb2JlgAAADghpKXof5cvA0AAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACwjWAAAAACwjGABAAAAwDKCBQAAAADLCBYAAAAALHMp6gIAAABwfcXFxSkxMbGoy0AB8PX1VVBQUFGXkS2CBQAAwA0sLi5Ojz76qC5evFjUpaAAeHh46KuvviqW4YJgAQAAcANLTEzUxYsXNX78eIWGhhZ1ObAgNjZWb7zxhhITEwkWAAAAKBqhoaGqU6dOUZeBGxgXbwMAAACwjDMWN5CEhATZ7fYc2729veXv71+IFQEAAOBmQbC4QSQkJKhnv6d0+uz5HPv4lfPSnMiZhAsAAAAUOILFDcJut+v02fPyD+uqMn6BWdrPnY5XwoavZbfbCRYAAAAocEV6jcUnn3yiRo0aydvbW97e3goLC9OyZcsc7RcvXtSgQYNUoUIFlS1bVl27dlV8fLzTOo4ePaoOHTrIy8tLAQEBevnll3Xp0iWnPqtXr9Ztt90md3d31ahRQ7NmzcpSy/Tp01WtWjV5eHioZcuW2rhx43XZ5+utjF+gvAOqZHlkFzYAAACkv6ekff7551W9enW5u7srODhYDz/8sKKjowu9FpvNpm+//faaX/fLL7/ozjvvVIUKFeTp6ak6derogw8+yNLvasd8cXFx6tWrl4KCglSmTBnddttt+vrrr536vP3227rjjjvk5eUlX1/fbOt54YUX1KxZM7m7u6tJkyZZ2levXq2OHTuqUqVKKlOmjJo0aaK5c+de834XJ0UaLKpUqaJ3331Xmzdv1qZNm3TPPfeoY8eO2rVrlyRp2LBh+u9//6tFixZpzZo1OnHihLp06eJ4fXp6ujp06KDU1FStX79es2fP1qxZszR69GhHn9jYWHXo0EHt2rVTTEyMhg4dqqeeekorVqxw9FmwYIGGDx+uMWPGaMuWLWrcuLEiIiJ06tSpwnszAAAAisDhw4fVrFkzrVy5Uu+//7527Nih5cuXq127dho0aFBRl5dnZcqU0eDBg/Xzzz9rz549GjVqlEaNGqVPP/3U0Scvx3y9e/fWvn379P3332vHjh3q0qWLHn/8cW3dutXRJzU1VY899pieffbZXGvq37+/unXrlm3b+vXr1ahRI3399dfavn27+vXrp969e2vJkiUW34kiZIqZ8uXLm5kzZ5rExETj6upqFi1a5Gjbs2ePkWQ2bNhgjDHmhx9+MKVKlTJxcXGOPp988onx9vY2KSkpxhhjRowYYerXr++0jW7dupmIiAjH8xYtWphBgwY5nqenp5vKlSubCRMm5LnupKQkI8kkJSVd2w4XkAMHDpjmrcPNo+8tNv0jN2Z5PPreYtO8dbg5cOBAkdQHAACKxp49e0yzZs3Mnj17sm1/4IEHzC233GKSk5OztJ05c8bx/48cOWIeeeQRU6ZMGVOuXDnz2GOPOR2D9enTx3Ts2NHp9UOGDDFt2rRxPG/Tpo15/vnnzcsvv2zKly9vAgMDzZgxYxztISEhRpLjERISkp9ddujcubPp2bOn43lejvnKlCljvvjiC6f1+Pn5mX//+99Z1h8ZGWl8fHxyrWHMmDGmcePGear3wQcfNP369cux/Wqf5fVwLce4xWa62fT0dM2fP1/nzp1TWFiYNm/erLS0NIWHhzv61KlTR1WrVtWGDRskSRs2bFDDhg0VGPh/w3wiIiJkt9sdZz02bNjgtI7MPpnrSE1N1ebNm536lCpVSuHh4Y4+AAAAN6LTp09r+fLlGjRokMqUKZOlPXOYT0ZGhjp27KjTp09rzZo1ioqK0qFDh3L8NT43s2fPVpkyZfTbb79p4sSJevPNNxUVFSVJ+v333yVJkZGROnnypOP54cOHZbPZtHr16jxvZ+vWrVq/fr3atGkjKe/HfHfccYcWLFig06dPKyMjQ/Pnz9fFixfVtm3ba97Xa5WUlCQ/P7/rvp3rpcgv3t6xY4fCwsJ08eJFlS1bVt98843q1aunmJgYubm5ZRm3FhgYqLi4OEl/j4G7PFRktme25dbHbrfrwoULOnPmjNLT07Pts3fv3hzrTklJUUpKiuN5btO8AgAAFEcHDhyQMeaqN86Ljo7Wjh07FBsbq+DgYEnSF198ofr16+v333/X7bffnudtNmrUSGPGjJEk1axZU9OmTVN0dLTuu+8+xwQzvr6+TneWdnV1Ve3ateXl5XXV9VepUkUJCQm6dOmSxo4dq6eeekqS9L///S9Px3wLFy5Ut27dVKFCBbm4uMjLy0vffPONatSoked9zI+FCxfq999/17/+9a/rup3rqciDRe3atRUTE6OkpCR99dVX6tOnj9asWVPUZV3VhAkTNG7cuKIuAwAAIN+MMXnqt2fPHgUHBztChSTVq1dPvr6+2rNnzzUHi8tVqlTpqte13nLLLbn+4Hu5tWvXKjk5Wb/++qteffVV1ahRQ927d89zfW+88YYSExP1008/qWLFivr222/1+OOPa+3atWrYsGGe13MtVq1apX79+unf//636tevf122URiKPFi4ubk5EmCzZs30+++/a+rUqerWrZtSU1OVmJjodNYiPj7ekWCDgoKyXMmfOWvU5X2unEkqPj5e3t7e8vT0VOnSpVW6dOls+1yelK80cuRIDR8+3PHcbrc7fdkAAACKu5o1a8pms+X5oD03pUqVyhJU0tLSsvRzdXV1em6z2ZSRkWF5+5lCQ0MlSQ0bNlR8fLzGjh2r7t27q2LFilc95jt48KCmTZumnTt3Og7wGzdurLVr12r69OmaMWNGgdWZac2aNXr44Yf1wQcfqHfv3gW+/sJUbK6xyJSRkaGUlBQ1a9ZMrq6uTtOc7du3T0ePHlVYWJgkKSwsTDt27HBKuVFRUfL29la9evUcfa6cKi0qKsqxDjc3NzVr1sypT0ZGhqKjox19suPu7u6YJjfzAQAAUJL4+fkpIiJC06dP17lz57K0JyYmSpLq1q2rY8eO6dixY4623bt3KzEx0XHM5e/vr5MnTzq9PiYm5pprcnV1VXp6+jW/LjuZx5VS3o75zp//+0bDpUo5HyKXLl26QMNPptWrV6tDhw5677339MwzzxT4+gtbkQaLkSNH6ueff9bhw4e1Y8cOjRw5UqtXr1aPHj3k4+OjAQMGaPjw4Vq1apU2b96sfv36KSwsTK1atZIk3X///apXr5569eqlbdu2acWKFRo1apQGDRokd3d3SdLAgQN16NAhjRgxQnv37tXHH3+shQsXatiwYY46hg8frn//+9+aPXu29uzZo2effVbnzp1Tv379iuR9AQAAKCzTp09Xenq6WrRooa+//lr79+/Xnj179OGHHzoOuMPDw9WwYUP16NFDW7Zs0caNG9W7d2+1adNGzZs3lyTdc8892rRpk7744gvt379fY8aM0c6dO6+5nmrVqik6OlpxcXE6c+aMJOnPP/9UnTp1cr3P2PTp0/Xf//5X+/fv1/79+/XZZ5/pn//8p3r27Onoc7Vjvjp16qhGjRr6xz/+oY0bN+rgwYOaNGmSoqKi1KlTJ8d6jh49qpiYGB09elTp6emKiYlRTEyMkpOTHX0OHDigmJgYxcXF6cKFC44+qampkv4e/tShQwe98MIL6tq1q+Li4hQXF6fTp09f83tWbFzvKapy079/fxMSEmLc3NyMv7+/uffee82PP/7oaL9w4YJ57rnnTPny5Y2Xl5fp3LmzOXnypNM6Dh8+bB544AHj6elpKlasaF588UWTlpbm1GfVqlWmSZMmxs3NzVSvXt1ERkZmqeWjjz4yVatWNW5ubqZFixbm119/vaZ9YbpZAABQHOVlitITJ06YQYMGOY7LbrnlFvPII4+YVatWOfpcbbpZY4wZPXq0CQwMND4+PmbYsGFm8ODBWaabHTJkiNNrOnbsaPr06eN4/v3335saNWoYFxcXx3SzsbGxRpJTPVf68MMPTf369Y2Xl5fx9vY2TZs2NR9//LFJT0936ne1Y74//vjDdOnSxQQEBBgvLy/TqFGjLNPP9unTx2la3MzH5fW1adMm2z6xsbG5ruPy9+tKxX26WZsxebxqB7my2+3y8fFRUlJSkQyLOnjwoJ7oP1DVOjwn74AqWes7dVyHl36s+Z/P0K233lro9QEAgKKxd+9e9ezZU3PmzLnq7E8o3oris7yWY9xid40FAAAAgJKHYAEAAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyl6IuAAAAANdfbGxsUZcAi4r7Z0iwAAAAuIH5+vrKw8NDb7zxRlGXggLg4eEhX1/foi4jWwQLAACAG1hQUJC++uorJSYmFnUpKAC+vr4KCgoq6jKyRbAAAAC4wQUFBRXbg1HcOLh4GwAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZS5FXQAKT1pqqo4cOZJju7e3t/z9/QuxIgAAANwoCBY3iZTkJB2OPaShr42Vu7t7tn38ynlpTuRMwgUAAACuGcHiJpGWckEZNhdVbNVFFSqHZGk/dzpeCRu+lt1uJ1gAAADgmhEsbjJe5f3lHVAl27aEQq4FAAAANw4u3gYAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYVabCYMGGCbr/9dpUrV04BAQHq1KmT9u3b59Snbdu2stlsTo+BAwc69Tl69Kg6dOggLy8vBQQE6OWXX9alS5ec+qxevVq33Xab3N3dVaNGDc2aNStLPdOnT1e1atXk4eGhli1bauPGjQW+zwAAAMCNqEiDxZo1azRo0CD9+uuvioqKUlpamu6//36dO3fOqd/TTz+tkydPOh4TJ050tKWnp6tDhw5KTU3V+vXrNXv2bM2aNUujR4929ImNjVWHDh3Url07xcTEaOjQoXrqqae0YsUKR58FCxZo+PDhGjNmjLZs2aLGjRsrIiJCp06duv5vBAAAAFDCuRTlxpcvX+70fNasWQoICNDmzZvVunVrx3IvLy8FBQVlu44ff/xRu3fv1k8//aTAwEA1adJE48eP1yuvvKKxY8fKzc1NM2bMUGhoqCZNmiRJqlu3rn755Rd98MEHioiIkCRNnjxZTz/9tPr16ydJmjFjhpYuXarPP/9cr7766vXYfQAAAOCGUayusUhKSpIk+fn5OS2fO3euKlasqAYNGmjkyJE6f/68o23Dhg1q2LChAgMDHcsiIiJkt9u1a9cuR5/w8HCndUZERGjDhg2SpNTUVG3evNmpT6lSpRQeHu7oAwAAACBnRXrG4nIZGRkaOnSo7rzzTjVo0MCx/Mknn1RISIgqV66s7du365VXXtG+ffu0ePFiSVJcXJxTqJDkeB4XF5drH7vdrgsXLujMmTNKT0/Pts/evXuzrTclJUUpKSmO53a7PZ97DgAAAJR8xSZYDBo0SDt37tQvv/zitPyZZ55x/P+GDRuqUqVKuvfee3Xw4EHdeuuthV2mw4QJEzRu3Lgi2z4AAABQnBSLoVCDBw/WkiVLtGrVKlWpUiXXvi1btpQkHThwQJIUFBSk+Ph4pz6ZzzOvy8ipj7e3tzw9PVWxYkWVLl062z45XdsxcuRIJSUlOR7Hjh3L494CAAAAN54iDRbGGA0ePFjffPONVq5cqdDQ0Ku+JiYmRpJUqVIlSVJYWJh27NjhNHtTVFSUvL29Va9ePUef6Ohop/VERUUpLCxMkuTm5qZmzZo59cnIyFB0dLSjz5Xc3d3l7e3t9AAAAABuVkU6FGrQoEGaN2+evvvuO5UrV85xTYSPj488PT118OBBzZs3Tw8++KAqVKig7du3a9iwYWrdurUaNWokSbr//vtVr1499erVSxMnTlRcXJxGjRqlQYMGyd3dXZI0cOBATZs2TSNGjFD//v21cuVKLVy4UEuXLnXUMnz4cPXp00fNmzdXixYtNGXKFJ07d84xSxQAAACAnBVpsPjkk08k/X0TvMtFRkaqb9++cnNz008//eQ4yA8ODlbXrl01atQoR9/SpUtryZIlevbZZxUWFqYyZcqoT58+evPNNx19QkNDtXTpUg0bNkxTp05VlSpVNHPmTMdUs5LUrVs3JSQkaPTo0YqLi1OTJk20fPnyLBd0AwAAAMiqSIOFMSbX9uDgYK1Zs+aq6wkJCdEPP/yQa5+2bdtq69atufYZPHiwBg8efNXtAQAAAHBWLC7eBgAAAFCyESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFjmUtQFAABwPSQkJMhut+fY7u3tLX9//0KsCABubAQLAMANJyEhQT37PaXTZ8/n2MevnJfmRM4kXABAASFYwCEtNVVHjhzJsZ1f9wCUFHa7XafPnpd/WFeV8QvM0n7udLwSNnwtu93Ov2sAUEAIFpAkpSQn6XDsIQ19bazc3d2z7cOvewBKmjJ+gfIOqJJtW0Ih1wIANzqCBSRJaSkXlGFzUcVWXVShckiWdn7dAwAAQG4IFnDiVd6fX/cAAABwzZhuFgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhVpsJgwYYJuv/12lStXTgEBAerUqZP27dvn1OfixYsaNGiQKlSooLJly6pr166Kj4936nP06FF16NBBXl5eCggI0Msvv6xLly459Vm9erVuu+02ubu7q0aNGpo1a1aWeqZPn65q1arJw8NDLVu21MaNGwt8nwEAAIAbUZEGizVr1mjQoEH69ddfFRUVpbS0NN1///06d+6co8+wYcP03//+V4sWLdKaNWt04sQJdenSxdGenp6uDh06KDU1VevXr9fs2bM1a9YsjR492tEnNjZWHTp0ULt27RQTE6OhQ4fqqaee0ooVKxx9FixYoOHDh2vMmDHasmWLGjdurIiICJ06dapw3owSIC01VUeOHNHBgwezfSQkJBR1iQAAACgiLkW58eXLlzs9nzVrlgICArR582a1bt1aSUlJ+uyzzzRv3jzdc889kqTIyEjVrVtXv/76q1q1aqUff/xRu3fv1k8//aTAwEA1adJE48eP1yuvvKKxY8fKzc1NM2bMUGhoqCZNmiRJqlu3rn755Rd98MEHioiIkCRNnjxZTz/9tPr16ydJmjFjhpYuXarPP/9cr776aiG+K8VTSnKSDsce0tDXxsrd3T3bPn7lvDQncqb8/f0LuToAAAAUtSINFldKSkqSJPn5+UmSNm/erLS0NIWHhzv61KlTR1WrVtWGDRvUqlUrbdiwQQ0bNlRgYKCjT0REhJ599lnt2rVLTZs21YYNG5zWkdln6NChkqTU1FRt3rxZI0eOdLSXKlVK4eHh2rBhQ7a1pqSkKCUlxfHcbrdb2/liLi3lgjJsLqrYqosqVA7J0n7udLwSNnwtu91OsAAAALgJFZtgkZGRoaFDh+rOO+9UgwYNJElxcXFyc3OTr6+vU9/AwEDFxcU5+lweKjLbM9ty62O323XhwgWdOXNG6enp2fbZu3dvtvVOmDBB48aNy9/OlmBe5f3lHVAl2zYGQgEAANy8is2sUIMGDdLOnTs1f/78oi4lT0aOHKmkpCTH49ixY0VdEgAAAFBkisUZi8GDB2vJkiX6+eefVaXK//0aHhQUpNTUVCUmJjqdtYiPj1dQUJCjz5WzN2XOGnV5nytnkoqPj5e3t7c8PT1VunRplS5dOts+meu4kru7e47XGgAAAAA3myI9Y2GM0eDBg/XNN99o5cqVCg0NdWpv1qyZXF1dFR0d7Vi2b98+HT16VGFhYZKksLAw7dixw2n2pqioKHl7e6tevXqOPpevI7NP5jrc3NzUrFkzpz4ZGRmKjo529AEAAACQsyI9YzFo0CDNmzdP3333ncqVK+e4JsLHx0eenp7y8fHRgAEDNHz4cPn5+cnb21vPP/+8wsLC1KpVK0nS/fffr3r16qlXr16aOHGi4uLiNGrUKA0aNMhxRmHgwIGaNm2aRowYof79+2vlypVauHChli5d6qhl+PDh6tOnj5o3b64WLVpoypQpOnfunGOWKAAAAAA5K9Jg8cknn0iS2rZt67Q8MjJSffv2lSR98MEHKlWqlLp27aqUlBRFRETo448/dvQtXbq0lixZomeffVZhYWEqU6aM+vTpozfffNPRJzQ0VEuXLtWwYcM0depUValSRTNnznRMNStJ3bp1U0JCgkaPHq24uDg1adJEy5cvz3JBNwAAAICsijRYGGOu2sfDw0PTp0/X9OnTc+wTEhKiH374Idf1tG3bVlu3bs21z+DBgzV48OCr1gQAAADAWbGZFQoAAABAyUWwAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACW5StYHDp0qKDrAAAAAFCC5StY1KhRQ+3atdOcOXN08eLFgq4JAAAAQAmTr2CxZcsWNWrUSMOHD1dQUJD+8Y9/aOPGjQVdGwAAAIASIl/BokmTJpo6dapOnDihzz//XCdPntRdd92lBg0aaPLkyUpISCjoOgEAAAAUY5Yu3nZxcVGXLl20aNEivffeezpw4IBeeuklBQcHq3fv3jp58mRB1QkAAACgGLMULDZt2qTnnntOlSpV0uTJk/XSSy/p4MGDioqK0okTJ9SxY8eCqhMAAABAMeaSnxdNnjxZkZGR2rdvnx588EF98cUXevDBB1Wq1N85JTQ0VLNmzVK1atUKslYAAAAAxVS+gsUnn3yi/v37q2/fvqpUqVK2fQICAvTZZ59ZKg4AAABAyZCvYLF///6r9nFzc1OfPn3ys3oAAAAAJUy+rrGIjIzUokWLsixftGiRZs+ebbkoAAAAACVLvoLFhAkTVLFixSzLAwIC9M4771guCgAAAEDJkq9gcfToUYWGhmZZHhISoqNHj1ouCgAAAEDJkq9gERAQoO3bt2dZvm3bNlWoUMFyUQAAAABKlnwFi+7du+uFF17QqlWrlJ6ervT0dK1cuVJDhgzRE088UdA1AgAAACjm8jUr1Pjx43X48GHde++9cnH5exUZGRnq3bs311jcxNJSU3XkyJEc2729veXv71+IFQEAAKCw5CtYuLm5acGCBRo/fry2bdsmT09PNWzYUCEhIQVdH0qIlOQkHY49pKGvjZW7u3u2ffzKeWlO5EzCBQAAwA0oX8EiU61atVSrVq2CqgUlWFrKBWXYXFSxVRdVqJw1YJ47Ha+EDV/LbrcTLAAAAG5A+QoW6enpmjVrlqKjo3Xq1CllZGQ4ta9cubJAikPJ41XeX94BVbJtSyjkWgAAAFB48hUshgwZolmzZqlDhw5q0KCBbDZbQdcFAAAAoATJV7CYP3++Fi5cqAcffLCg6wEAAABQAuVrulk3NzfVqFGjoGsBAAAAUELlK1i8+OKLmjp1qowxBV0PAAAAgBIoX0OhfvnlF61atUrLli1T/fr15erq6tS+ePHiAikOAAAAQMmQr2Dh6+urzp07F3QtAAAAAEqofAWLyMjIgq4DAAAAQAmWr2ssJOnSpUv66aef9K9//Utnz56VJJ04cULJyckFVhwAAACAkiFfZyyOHDmi9u3b6+jRo0pJSdF9992ncuXK6b333lNKSopmzJhR0HUCAAAAKMbydcZiyJAhat68uc6cOSNPT0/H8s6dOys6OrrAigMAAABQMuTrjMXatWu1fv16ubm5OS2vVq2a/vzzzwIpDAAAAEDJka8zFhkZGUpPT8+y/Pjx4ypXrpzlogAAAACULPkKFvfff7+mTJnieG6z2ZScnKwxY8bowQcfLKjaAAAAAJQQ+RoKNWnSJEVERKhevXq6ePGinnzySe3fv18VK1bUl19+WdA1AgAAACjm8hUsqlSpom3btmn+/Pnavn27kpOTNWDAAPXo0cPpYm4AAAAAN4d8BQtJcnFxUc+ePQuyFgAAAAAlVL6CxRdffJFre+/evfNVDAAAAICSKV/BYsiQIU7P09LSdP78ebm5ucnLy4tgAQAAANxk8jUr1JkzZ5weycnJ2rdvn+666y4u3gYAAABuQvkKFtmpWbOm3n333SxnMwAAAADc+AosWEh/X9B94sSJglwlAAAAgBIgX9dYfP/9907PjTE6efKkpk2bpjvvvLNACgMAAABQcuQrWHTq1Mnpuc1mk7+/v+655x5NmjSpIOoCAAAAUILkK1hkZGQUdB0AAAAASrACvcbiWv388896+OGHVblyZdlsNn377bdO7X379pXNZnN6tG/f3qnP6dOn1aNHD3l7e8vX11cDBgxQcnKyU5/t27fr7rvvloeHh4KDgzVx4sQstSxatEh16tSRh4eHGjZsqB9++KHA9xcAAAC4UeXrjMXw4cPz3Hfy5Mk5tp07d06NGzdW//791aVLl2z7tG/fXpGRkY7n7u7uTu09evTQyZMnFRUVpbS0NPXr10/PPPOM5s2bJ0my2+26//77FR4erhkzZmjHjh3q37+/fH199cwzz0iS1q9fr+7du2vChAl66KGHNG/ePHXq1ElbtmxRgwYN8ryvAAAAwM0qX8Fi69at2rp1q9LS0lS7dm1J0h9//KHSpUvrtttuc/Sz2Wy5rueBBx7QAw88kGsfd3d3BQUFZdu2Z88eLV++XL///ruaN28uSfroo4/04IMP6p///KcqV66suXPnKjU1VZ9//rnc3NxUv359xcTEaPLkyY5gMXXqVLVv314vv/yyJGn8+PGKiorStGnTNGPGjLy9KQAAAMBNLF9DoR5++GG1bt1ax48f15YtW7RlyxYdO3ZM7dq100MPPaRVq1Zp1apVWrlypeUCV69erYCAANWuXVvPPvus/vrrL0fbhg0b5Ovr6wgVkhQeHq5SpUrpt99+c/Rp3bq13NzcHH0iIiK0b98+nTlzxtEnPDzcabsRERHasGFDjnWlpKTIbrc7PQAAAICbVb6CxaRJkzRhwgSVL1/esax8+fJ66623CnRWqPbt2+uLL75QdHS03nvvPa1Zs0YPPPCA0tPTJUlxcXEKCAhweo2Li4v8/PwUFxfn6BMYGOjUJ/P51fpktmdnwoQJ8vHxcTyCg4Ot7SwAAABQguVrKJTdbldCQkKW5QkJCTp79qzlojI98cQTjv/fsGFDNWrUSLfeeqtWr16te++9t8C2kx8jR450utbEbrcTLgAAAHDTytcZi86dO6tfv35avHixjh8/ruPHj+vrr7/WgAEDcrwIuyBUr15dFStW1IEDByRJQUFBOnXqlFOfS5cu6fTp047rMoKCghQfH+/UJ/P51frkdG2H9Pe1H97e3k4PAAAA4GaVr2AxY8YMPfDAA3ryyScVEhKikJAQPfnkk2rfvr0+/vjjgq7R4fjx4/rrr79UqVIlSVJYWJgSExO1efNmR5+VK1cqIyNDLVu2dPT5+eeflZaW5ugTFRWl2rVrO4ZyhYWFKTo62mlbUVFRCgsLu277AgAAANxI8hUsvLy89PHHH+uvv/5yzBB1+vRpffzxxypTpkye15OcnKyYmBjFxMRIkmJjYxUTE6OjR48qOTlZL7/8sn799VcdPnxY0dHR6tixo2rUqKGIiAhJUt26ddW+fXs9/fTT2rhxo9atW6fBgwfriSeeUOXKlSVJTz75pNzc3DRgwADt2rVLCxYs0NSpU52GMQ0ZMkTLly/XpEmTtHfvXo0dO1abNm3S4MGD8/P2AAAAADcdSzfIO3nypE6ePKmaNWuqTJkyMsZc0+s3bdqkpk2bqmnTppL+vj9G06ZNNXr0aJUuXVrbt2/XI488olq1amnAgAFq1qyZ1q5d63Qvi7lz56pOnTq699579eCDD+quu+7Sp59+6mj38fHRjz/+qNjYWDVr1kwvvviiRo8e7ZhqVpLuuOMOzZs3T59++qkaN26sr776St9++y33sAAAAADyKF8Xb//11196/PHHtWrVKtlsNu3fv1/Vq1fXgAEDVL58+TzPDNW2bdtcw8iKFSuuug4/Pz/HzfBy0qhRI61duzbXPo899pgee+yxq24PAAAAQFb5OmMxbNgwubq66ujRo/Ly8nIs79atm5YvX15gxQEAAAAoGfJ1xuLHH3/UihUrVKVKFaflNWvW1JEjRwqkMAAAAAAlR77OWJw7d87pTEWm06dPO13/AAAAAODmkK9gcffdd+uLL75wPLfZbMrIyNDEiRPVrl27AisOAAAAQMmQr6FQEydO1L333qtNmzYpNTVVI0aM0K5du3T69GmtW7euoGsEAAAAUMzl64xFgwYN9Mcff+iuu+5Sx44dde7cOXXp0kVbt27VrbfeWtA1AgAAACjmrvmMRVpamtq3b68ZM2bo9ddfvx41AQAAAChhrvmMhaurq7Zv3349agEAAABQQuXrGouePXvqs88+07vvvlvQ9eAGlpaaetXpiL29veXv719IFQEAAKCg5CtYXLp0SZ9//rl++uknNWvWTGXKlHFqnzx5coEUhxtHSnKSDsce0tDXxuY6JbFfOS/NiZxJuAAAAChhrilYHDp0SNWqVdPOnTt12223SZL++OMPpz42m63gqsMNIy3lgjJsLqrYqosqVA7Jts+50/FK2PC17HY7wQIAAKCEuaZgUbNmTZ08eVKrVq2SJHXr1k0ffvihAgMDr0txuPF4lfeXd0CVHNsTCrEWAAAAFJxrunjbGOP0fNmyZTp37lyBFgQAAACg5MnXfSwyXRk0AAAAANycrilY2Gy2LNdQcE0FAAAAgGu6xsIYo759+zpm9bl48aIGDhyYZVaoxYsXF1yFAAAAAIq9awoWffr0cXres2fPAi0GAAAAQMl0TcEiMjLyetUBAAAAoASzdPE2AAAAAEgECwAAAAAFgGABAAAAwDKCBQAAAADLCBYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACxzKeoCAAAoCmmpqTpy5EiO7d7e3vL39y/EigCgZCNYAABuOinJSToce0hDXxsrd3f3bPv4lfPSnMiZhAsAyCOCBQDgppOWckEZNhdVbNVFFSqHZGk/dzpeCRu+lt1uJ1gAQB4RLAAANy2v8v7yDqiSbVtCIdcCACUdF28DAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACwjWAAAAACwjGABAAAAwDKCBQAAAADLCBYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFgAAAAAscynqAoDiJCEhQXa7Pcd2b29v+fv7F2JFAAAAJQPBAvj/EhIS1LPfUzp99nyOffzKeWlO5EzCBQAAwBUIFsD/Z7fbdfrsefmHdVUZv8As7edOxythw9ey2+0ECwAAgCsQLIArlPELlHdAlWzbEgq5FgAAgJKCi7cBAAAAWEawAAAAAGBZkQaLn3/+WQ8//LAqV64sm82mb7/91qndGKPRo0erUqVK8vT0VHh4uPbv3+/U5/Tp0+rRo4e8vb3l6+urAQMGKDk52anP9u3bdffdd8vDw0PBwcGaOHFilloWLVqkOnXqyMPDQw0bNtQPP/xQ4PuLq0tLTdWRI0d08ODBbB8JCQxGAgAAKI6K9BqLc+fOqXHjxurfv7+6dOmSpX3ixIn68MMPNXv2bIWGhuqNN95QRESEdu/eLQ8PD0lSjx49dPLkSUVFRSktLU39+vXTM888o3nz5kn6+4Lc+++/X+Hh4ZoxY4Z27Nih/v37y9fXV88884wkaf369erevbsmTJighx56SPPmzVOnTp20ZcsWNWjQoPDekJtcSnKSDsce0tDXxsrd3T3bPszKBAAAUDwVabB44IEH9MADD2TbZozRlClTNGrUKHXs2FGS9MUXXygwMFDffvutnnjiCe3Zs0fLly/X77//rubNm0uSPvroIz344IP65z//qcqVK2vu3LlKTU3V559/Ljc3N9WvX18xMTGaPHmyI1hMnTpV7du318svvyxJGj9+vKKiojRt2jTNmDGjEN4JSFJaygVl2FxUsVUXVagckqWdWZkAAACKr2J7jUVsbKzi4uIUHh7uWObj46OWLVtqw4YNkqQNGzbI19fXESokKTw8XKVKldJvv/3m6NO6dWu5ubk5+kRERGjfvn06c+aMo8/l28nsk7kdFC6v8v7yDqiS5VHGL5ChUgAAAMVUsZ1uNi4uTpIUGOh8P4HAwEBHW1xcnAICApzaXVxc5Ofn59QnNDQ0yzoy28qXL6+4uLhct5OdlJQUpaSkOJ7ndrdmFAyGSgEAABRfxTZYFHcTJkzQuHHjirqMmwpDpQAAAIqvYjsUKigoSJIUHx/vtDw+Pt7RFhQUpFOnTjm1X7p0SadPn3bqk906Lt9GTn0y27MzcuRIJSUlOR7Hjh271l1EPuU2VAoAAABFo9gGi9DQUAUFBSk6OtqxzG6367ffflNYWJgkKSwsTImJidq8ebOjz8qVK5WRkaGWLVs6+vz8889KS0tz9ImKilLt2rVVvnx5R5/Lt5PZJ3M72XF3d5e3t7fTAwAAALhZFWmwSE5OVkxMjGJiYiT9fcF2TEyMjh49KpvNpqFDh+qtt97S999/rx07dqh3796qXLmyOnXqJEmqW7eu2rdvr6efflobN27UunXrNHjwYD3xxBOqXLmyJOnJJ5+Um5ubBgwYoF27dmnBggWaOnWqhg8f7qhjyJAhWr58uSZNmqS9e/dq7Nix2rRpkwYPHlzYbwkAAABQIhXpNRabNm1Su3btHM8zD/b79OmjWbNmacSIETp37pyeeeYZJSYm6q677tLy5csd97CQpLlz52rw4MG69957VapUKXXt2lUffviho93Hx0c//vijBg0apGbNmqlixYoaPXq0Y6pZSbrjjjs0b948jRo1Sq+99ppq1qypb7/9lntYAEAxlpCQkOPEGUeOHNGltEuFXBEA3NyKNFi0bdtWxpgc2202m9588029+eabOfbx8/Nz3AwvJ40aNdLatWtz7fPYY4/psccey71gAECxkJCQoJ79ntLps+ezbb944byO/3lSVS8bBgsAuL6YFQoAUOLY7XadPnte/mFds5244dTBnTpy7HOlXyJYAEBhIVgAAEqsMn6B8g6okmV58l8534cIAHB9FNtZoQAAAACUHAQLAAAAAJYRLAAAAABYRrAAAAAAYBkXb+OGkpaaqiNHjuTY7u3tLX9//0KsCAAA4OZAsMANIyU5SYdjD2noa2Pl7u6ebR+/cl6aEzmTcAEAAFDACBa4YaSlXFCGzUUVW3VRhcohWdrPnY5XwoavZbfbCRYAAAAFjGCBG45Xef9s57WXpIRCrgUAAOBmwcXbAAAAACwjWAAAAACwjGABAAAAwDKCBQAAAADLCBYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACwjWAAAAACwjGABAAAAwDKCBQAAAADLCBYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyl6IuAChJ0lJTdeTIkRzbvb295e/vX4gVAQAAFA8ECyCPUpKTdDj2kIa+Nlbu7u7Z9vEr56U5kTMJFwAA4KZDsADyKC3lgjJsLqrYqosqVA7J0n7udLwSNnwtu91OsAAAADcdggVwjbzK+8s7oEq2bQmFXAsAAEBxwcXbAAAAACwjWAAAAACwjGABAAAAwDKCBQAAAADLCBYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACwjWAAAAACwjGABAAAAwDKXoi4AAIArJSQkyG6359h+5MgRXUq7VIgVAQCuhmABAChWEhIS1LPfUzp99nyOfS5eOK/jf55U1bS061ZHWmqqjhw5kmO7t7e3/P39r9v2AaCkKdbBYuzYsRo3bpzTstq1a2vv3r2SpIsXL+rFF1/U/PnzlZKSooiICH388ccKDAx09D969KieffZZrVq1SmXLllWfPn00YcIEubj8366vXr1aw4cP165duxQcHKxRo0apb9++hbKPAABndrtdp8+el39YV5XxC8y2z6mDO3Xk2OdKv3R9gkVKcpIOxx7S0NfGyt3dPds+fuW8NCdyJuECAP6/Yh0sJKl+/fr66aefHM8vDwTDhg3T0qVLtWjRIvn4+Gjw4MHq0qWL1q1bJ0lKT09Xhw4dFBQUpPXr1+vkyZPq3bu3XF1d9c4770iSYmNj1aFDBw0cOFBz585VdHS0nnrqKVWqVEkRERGFu7MAAIcyfoHyDqiSbVvyX3HXddtpKReUYXNRxVZdVKFySJb2c6fjlbDha9ntdoIFAPx/xT5YuLi4KCgoKMvypKQkffbZZ5o3b57uueceSVJkZKTq1q2rX3/9Va1atdKPP/6o3bt366efflJgYKCaNGmi8ePH65VXXtHYsWPl5uamGTNmKDQ0VJMmTZIk1a1bV7/88os++OADggUA3OS8yvvnGG4SCrkWACjuiv2sUPv371flypVVvXp19ejRQ0ePHpUkbd68WWlpaQoPD3f0rVOnjqpWraoNGzZIkjZs2KCGDRs6DY2KiIiQ3W7Xrl27HH0uX0dmn8x14MaSOWb64MGDWR5cDAoAAJB/xfqMRcuWLTVr1izVrl1bJ0+e1Lhx43T33Xdr586diouLk5ubm3x9fZ1eExgYqLi4v0+Rx8XFOYWKzPbMttz62O12XbhwQZ6entnWlpKSopSUFMfz3GYvQfFwtTHThXExKAAAwI2qWAeLBx54wPH/GzVqpJYtWyokJEQLFy7M8YC/sEyYMCHLheUo3q42Zvp6XwwKAABwIyv2Q6Eu5+vrq1q1aunAgQMKCgpSamqqEhMTnfrEx8c7rskICgpSfHx8lvbMttz6eHt75xpeRo4cqaSkJMfj2LFjVncPhSRzzPSVDy/fikVdGgAAQIlVooJFcnKyDh48qEqVKqlZs2ZydXVVdHS0o33fvn06evSowsLCJElhYWHasWOHTp065egTFRUlb29v1atXz9Hn8nVk9slcR07c3d3l7e3t9AAAAABuVsU6WLz00ktas2aNDh8+rPXr16tz584qXbq0unfvLh8fHw0YMEDDhw/XqlWrtHnzZvXr109hYWFq1aqVJOn+++9XvXr11KtXL23btk0rVqzQqFGjNGjQIMcY+4EDB+rQoUMaMWKE9u7dq48//lgLFy7UsGHDinLXAQAAgBKlWF9jcfz4cXXv3l1//fWX/P39ddddd+nXX391zBn+wQcfqFSpUuratavTDfIylS5dWkuWLNGzzz6rsLAwlSlTRn369NGbb77p6BMaGqqlS5dq2LBhmjp1qqpUqaKZM2cy1SwAAABwDYp1sJg/f36u7R4eHpo+fbqmT5+eY5+QkBD98MMPua6nbdu22rp1a75qBABcu4SEhBxn02PqZwAomYp1sAAA3HgSEhLUs99TOn32fLbtTP0MACUTwQIAUKjsdrtOnz0v/7CuKuMXmKWdqZ8BoGQiWAAAikQZv0B5B1TJsjz5r7giqAYAYFWxnhUKAAAAQMlAsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWMasUEAhyu2mYJm8vb0dd5cHUHylpabqyJEjObbzXQZwsyFYAIXkajcFy+RXzktzImdyQAIUYynJSToce0hDXxsrd3f3bPvwXQZwsyFYAIXkajcFk6Rzp+OVsOFr2e12DkaAYiwt5YIybC6q2KqLKlQOydLOdxnAzYhgARSynG4KlimhEGsBYI1Xef8cv898lwHcbLh4GwAAAIBlBAsAAAAAlhEsAAAAAFjGNRZAAcpt+skjR47oUtqlQq4IAACgcBAsgAJyteknL144r+N/nlTVtLQiqA4AAOD6IlgABeRq00+eOrhTR459rvRL1zdYXO0mfNy0Cygc3EAPwM2GYAEUsJymn0z+K+66bzsvN+Hjpl3A9ccN9ADcjAgWwA3kajfh46ZdQOHgBnoAbkYEC+AGlNtN+E4wPAMoNNxAD8DNhGAB3EQYngEAAK4XggVQzFzPCz4ZngEAAK4XggVQjBTWGQWGZwAAgIJGsACKEc4o4EaR27TH3CwSAG5MBAugGLqRzyhc7T4bEheQl3RXm/aYm0X+jftcALjRECwAFJq83GdD4gLyku5q0x4X1s0iizMmUgBwIyJYACg0VzvglBjudSPJadrjwrhZZHHHsEcANyKCBYBCl9t9NqSSP9wLyKsbedgjgJsPwQLANbnaNRIFMS68uI89L4z3AACAkoZgASDP8nKNRFm30nrv7TdVoUKFLG15mQ2ouI89z8t7wNh4AMDNiGABlDC5/Zp/vafxvNo1EqePH9DmhR/qqRdeyjYU5GU2oOI+9vxq70FR1wcAQFEhWAAlyNV+zS+saTxzuyg3t1BwLbMBFfex57ldJ1Ic6kPJd7UhgRLD7gAULwQLoAS52q/5BTGNZ0GcEckpFDAbEJA3eRkSKDHsDkDxQrAASqDrdeBeXM6IoPjjztrX19V+RJAYdgeg+CFYAHAojDMiKPm4s3bhyW1IoMSwOwDFC8ECQBYMZUJuuLM2ACA7BAsAKGDF/T4cBYU7awMALkewAIACVNzvwwEAwPVCsABw07med84u7vfhAADgeiFYALjh5BYc/vrrL70yaqySU3Ie/2/17uFS7hfdnrhJhkrh+rtZht0BKBkIFgBuKHmdsaj5E8PkG5j1wL8g7h6eG4ZKoaDwtwSguCFYALih5HXGIndvv+t+9/DsMFQKBYW/JQDFDcECQImTl7uDW52x6HpPuZvbUKnicG8CboBXchT3vyUANw+CBYAShbuDX3/cAO/GwTUYAAoTwQJAicLdwa8/boB3Y+AaDACFjWABoES6ke8OfrVfmaXC+aWZG+CVbFyDAaCwESwAoBjJy6/MUu5T4koMccH/YepjAIWFYAEAxcjVfmWWrj4lrnT14JGamio3N7ds27g4++bAUCkABY1gAQDFUG6/Ml9tStyrBY+01FT9efSIqoSEysU1638GuDj75sBQKQAFjWABACVUbteZXO0C90OHP1f5Fh25AB4MlQJQYAgWAHCDutoF7jfyBfCwLi9DpbjWB8DlCBYAACCLqw2VKohrfQgewI2FYAEAAHKU3yF313uSgUyEE6D4IFhcYfr06Xr//fcVFxenxo0b66OPPlKLFi2KuiwAAIql6xU8rjbJQCar4eRq7QQXIO8IFpdZsGCBhg8frhkzZqhly5aaMmWKIiIitG/fPgUEBBR1eQAAlDjXa5IByXo4yUt4YTgXkHcEi8tMnjxZTz/9tPr16ydJmjFjhpYuXarPP/9cr776ahFXBwDAjSe/kwxk9rE6A1pu7YUxnItgghsJweL/S01N1ebNmzVy5EjHslKlSik8PFwbNmzI0j8lJUUpKSmO50lJSZIku91+/YvNxtmzZ5V+6ZISTx5W2sXzWdrtp47LZGTIHndMLrasry/q9uJQQ1G3F4cairq9ONRQ3NuLQw1F3V4caijq9uJQQ1G3X97nUsrFbP/bdyn1oqX2i2cTlW5Kya16C/lUyDpyIenUCW1ZtUj9nhua4xmTE8eP6ZbgkJzPiLi7aOyokfLz88t+JwFJvr6+RfY3knlsa4y5al+byUuvm8CJEyd0yy23aP369QoLC3MsHzFihNasWaPffvvNqf/YsWM1bty4wi4TAAAAKHTHjh1TlSrZnz3MxBmLfBo5cqSGDx/ueJ6RkaHTp0+rQoUKstly+GmlgNjtdgUHB+vYsWPy9va+rtvCteGzKb74bIovPpvijc+n+OKzKb5upM/GGKOzZ8+qcuXKV+1LsPj/KlasqNKlSys+Pt5peXx8vIKCgrL0d3d3z3La09fX93qWmIW3t3eJ/2O9UfHZFF98NsUXn03xxudTfPHZFF83ymfj4+OTp36lrnMdJYabm5uaNWum6Ohox7KMjAxFR0c7DY0CAAAAkBVnLC4zfPhw9enTR82bN1eLFi00ZcoUnTt3zjFLFAAAAIDsESwu061bNyUkJGj06NGKi4tTkyZNtHz5cgUGBhZ1aU7c3d01ZsyYHKe+Q9Hhsym++GyKLz6b4o3Pp/jisym+btbPhlmhAAAAAFjGNRYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFkVgwoQJuv3221WuXDkFBASoU6dO2rdvn1OfixcvatCgQapQoYLKli2rrl27Zrl539GjR9WhQwd5eXkpICBAL7/8si5duuTUZ/Xq1brtttvk7u6uGjVqaNasWdd790q8Tz75RI0aNXLc1CYsLEzLli1ztPPZFA/vvvuubDabhg4d6ljGZ1N0xo4dK5vN5vSoU6eOo53Ppmj9+eef6tmzpypUqCBPT081bNhQmzZtcrQbYzR69GhVqlRJnp6eCg8P1/79+53Wcfr0afXo0UPe3t7y9fXVgAEDlJyc7NRn+/btuvvuu+Xh4aHg4GBNnDixUPavpKpWrVqW743NZtOgQYMk8b0pSunp6XrjjTcUGhoqT09P3XrrrRo/frwun/OI7002DApdRESEiYyMNDt37jQxMTHmwQcfNFWrVjXJycmOPgMHDjTBwcEmOjrabNq0ybRq1crccccdjvZLly6ZBg0amPDwcLN161bzww8/mIoVK5qRI0c6+hw6dMh4eXmZ4cOHm927d5uPPvrIlC5d2ixfvrxQ97ek+f77783SpUvNH3/8Yfbt22dee+014+rqanbu3GmM4bMpDjZu3GiqVatmGjVqZIYMGeJYzmdTdMaMGWPq169vTp486XgkJCQ42vlsis7p06dNSEiI6du3r/ntt9/MoUOHzIoVK8yBAwccfd59913j4+Njvv32W7Nt2zbzyCOPmNDQUHPhwgVHn/bt25vGjRubX3/91axdu9bUqFHDdO/e3dGelJRkAgMDTY8ePczOnTvNl19+aTw9Pc2//vWvQt3fkuTUqVNO35moqCgjyaxatcoYw/emKL399tumQoUKZsmSJSY2NtYsWrTIlC1b1kydOtXRh+9NVgSLYuDUqVNGklmzZo0xxpjExETj6upqFi1a5OizZ88eI8ls2LDBGGPMDz/8YEqVKmXi4uIcfT755BPj7e1tUlJSjDHGjBgxwtSvX99pW926dTMRERHXe5duOOXLlzczZ87ksykGzp49a2rWrGmioqJMmzZtHMGCz6ZojRkzxjRu3DjbNj6bovXKK6+Yu+66K8f2jIwMExQUZN5//33HssTEROPu7m6+/PJLY4wxu3fvNpLM77//7uizbNkyY7PZzJ9//mmMMebjjz825cuXd3xemduuXbt2Qe/SDWvIkCHm1ltvNRkZGXxviliHDh1M//79nZZ16dLF9OjRwxjD9yYnDIUqBpKSkiRJfn5+kqTNmzcrLS1N4eHhjj516tRR1apVtWHDBknShg0b1LBhQ6eb90VERMhut2vXrl2OPpevI7NP5jpwdenp6Zo/f77OnTunsLAwPptiYNCgQerQoUOW94/Ppujt379flStXVvXq1dWjRw8dPXpUEp9NUfv+++/VvHlzPfbYYwoICFDTpk3173//29EeGxuruLg4p/fWx8dHLVu2dPp8fH191bx5c0ef8PBwlSpVSr/99pujT+vWreXm5uboExERoX379unMmTPXezdLvNTUVM2ZM0f9+/eXzWbje1PE7rjjDkVHR+uPP/6QJG3btk2//PKLHnjgAUl8b3LCnbeLWEZGhoYOHao777xTDRo0kCTFxcXJzc1Nvr6+Tn0DAwMVFxfn6HPlHcEzn1+tj91u14ULF+Tp6Xk9dumGsGPHDoWFhenixYsqW7asvvnmG9WrV08xMTF8NkVo/vz52rJli37//fcsbXxvilbLli01a9Ys1a5dWydPntS4ceN09913a+fOnXw2RezQoUP65JNPNHz4cL322mv6/fff9cILL8jNzU19+vRxvL/ZvbeXv/cBAQFO7S4uLvLz83PqExoammUdmW3ly5e/Lvt3o/j222+VmJiovn37SuLftKL26quvym63q06dOipdurTS09P19ttvq0ePHpLE9yYHBIsiNmjQIO3cuVO//PJLUZeCy9SuXVsxMTFKSkrSV199pT59+mjNmjVFXdZN7dixYxoyZIiioqLk4eFR1OXgCpm/4klSo0aN1LJlS4WEhGjhwoUcuBSxjIwMNW/eXO+8844kqWnTptq5c6dmzJihPn36FHF1yPTZZ5/pgQceUOXKlYu6FEhauHCh5s6dq3nz5ql+/fqKiYnR0KFDVblyZb43uWAoVBEaPHiwlixZolWrVqlKlSqO5UFBQUpNTVViYqJT//j4eAUFBTn6XDkzRObzq/Xx9vbmP/RX4ebmpho1aqhZs2aaMGGCGjdurKlTp/LZFKHNmzfr1KlTuu222+Ti4iIXFxetWbNGH374oVxcXBQYGMhnU4z4+vqqVq1aOnDgAN+bIlapUiXVq1fPaVndunUdQ9Uy39/s3tvL3/tTp045tV+6dEmnT5++ps8Q2Tty5Ih++uknPfXUU45lfG+K1ssvv6xXX31VTzzxhBo2bKhevXpp2LBhmjBhgiS+NzkhWBQBY4wGDx6sb775RitXrsxyCqxZs2ZydXVVdHS0Y9m+fft09OhRhYWFSZLCwsK0Y8cOpz/YqKgoeXt7O/4DEhYW5rSOzD6Z60DeZWRkKCUlhc+mCN17773asWOHYmJiHI/mzZurR48ejv/PZ1N8JCcn6+DBg6pUqRLfmyJ25513ZpnS/I8//lBISIgkKTQ0VEFBQU7vrd1u12+//eb0+SQmJmrz5s2OPitXrlRGRoZatmzp6PPzzz8rLS3N0ScqKkq1a9cuccM5CltkZKQCAgLUoUMHxzK+N0Xr/PnzKlXK+TC5dOnSysjIkMT3JkdFffX4zejZZ581Pj4+ZvXq1U7TzJ0/f97RZ+DAgaZq1apm5cqVZtOmTSYsLMyEhYU52jOnmLv//vtNTEyMWb58ufH39892irmXX37Z7Nmzx0yfPp0p5vLg1VdfNWvWrDGxsbFm+/bt5tVXXzU2m838+OOPxhg+m+Lk8lmhjOGzKUovvviiWb16tYmNjTXr1q0z4eHhpmLFiubUqVPGGD6borRx40bj4uJi3n77bbN//34zd+5c4+XlZebMmePo8+677xpfX1/z3Xffme3bt5uOHTtmO21m06ZNzW+//WZ++eUXU7NmTadpMxMTE01gYKDp1auX2blzp5k/f77x8vIqsdNmFpb09HRTtWpV88orr2Rp43tTdPr06WNuueUWx3SzixcvNhUrVjQjRoxw9OF7kxXBoghIyvYRGRnp6HPhwgXz3HPPmfLlyxsvLy/TuXNnc/LkSaf1HD582DzwwAPG09PTVKxY0bz44osmLS3Nqc+qVatMkyZNjJubm6levbrTNpC9/v37m5CQEOPm5mb8/f3Nvffe6wgVxvDZFCdXBgs+m6LTrVs3U6lSJePm5mZuueUW061bN6f7JPDZFK3//ve/pkGDBsbd3d3UqVPHfPrpp07tGRkZ5o033jCBgYHG3d3d3HvvvWbfvn1Off766y/TvXt3U7ZsWePt7W369etnzp4969Rn27Zt5q677jLu7u7mlltuMe++++5137eSbsWKFUZSlvfbGL43Rclut5shQ4aYqlWrGg8PD1O9enXz+uuvO00Ly/cmK5sxl91CEAAAAADygWssAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAJUrbtm01dOjQoi4DAHAFggUAoNA8/PDDat++fbZta9eulc1m0/bt2wu5KgBAQSBYAAAKzYABAxQVFaXjx49naYuMjFTz5s3VqFGjIqgMAGAVwQIAUGgeeugh+fv7a9asWU7Lk5OTtWjRInXq1Endu3fXLbfcIi8vLzVs2FBffvllruu02Wz69ttvnZb5+vo6bePYsWN6/PHH5evrKz8/P3Xs2FGHDx8umJ0CAEgiWAAACpGLi4t69+6tWbNmyRjjWL5o0SKlp6erZ8+eatasmZYuXaqdO3fqmWeeUa9evbRx48Z8bzMtLU0REREqV66c1q5dq3Xr1qls2bJq3769UlNTC2K3AAAiWAAACln//v118OBBrVmzxrEsMjJSXbt2VUhIiF566SU1adJE1atX1/PPP6/27dtr4cKF+d7eggULlJGRoZkzZ6phw4aqW7euIiMjdfToUa1evboA9ggAIBEsAACFrE6dOrrjjjv0+eefS5IOHDigtWvXasCAAUpPT9f48ePVsGFD+fn5qWzZslqxYoWOHj2a7+1t27ZNBw4cULly5VS2bFmVLVtWfn5+unjxog4ePFhQuwUANz2Xoi4AAHDzGTBggJ5//nlNnz5dkZGRuvXWW9WmTRu99957mjp1qqZMmaKGDRuqTJkyGjp0aK5Dlmw2m9OwKunv4U+ZkpOT1axZM82dOzfLa/39/QtupwDgJkewAAAUuscff1xDhgzRvHnz9MUXX+jZZ5+VzWbTunXr1LFjR/Xs2VOSlJGRoT/++EP16tXLcV3+/v46efKk4/n+/ft1/vx5x/PbbrtNCxYsUEBAgLy9va/fTgHATY6hUACAQle2bFl169ZNI0eO1MmTJ9W3b19JUs2aNRUVFaX169drz549+sc//qH4+Phc13XPPfdo2rRp2rp1qzZt2qSBAwfK1dXV0d6jRw9VrFhRHTt21Nq1axUbG6vVq1frhRdeyHbaWwBA/hAsAABFYsCAATpz5owiIiJUuXJlSdKoUaN02223KSIiQm3btlVQUJA6deqU63omTZqk4OBg3X333XryySf10ksvycvLy9Hu5eWln3/+WVWrVlWXLl1Ut25dDRgwQBcvXuQMBgAUIJu5cmAqAAAAAFwjzlgAAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACwjWAAAAACwjGABAAAAwDKCBQAAAADLCBYAAAAALCNYAAAAALCMYAEAAADAMoIFAAAAAMsIFgAAAAAsI1gAAAAAsIxgAQAAAMAyggUAAAAAywgWAAAAACwjWAAAAACwjGABIM+qVaumvn37FnUZN7z3339f1atXV+nSpdWkSZMc+1WrVk02m002m02DBw8uvAKRo5iYGMdnYrPZ9NVXX+XpdRMnTlSdOnWUkZFxnStEUXj11VfVsmXLoi4DuO4IFsBNatasWbLZbNq0aVO27W3btlWDBg0sb+eHH37Q2LFjLa/nZvHjjz9qxIgRuvPOOxUZGal33nkn1/533323/vOf/6hPnz5Oy5OSkjRixAjVrFlTnp6eCgkJ0YABA3T06FGnfvv27dOwYcN0xx13yMPDQzabTYcPH852W8nJyRo6dKiqVKkid3d31a1bV5988kmWfj///LMeeeQRBQcHy8PDQ0FBQWrfvr3WrVt3bW/GFU6ePKlnnnlGoaGh8vT01K233qrhw4frr7/+cuq3ceNGPffcc2rWrJlcXV1ls9lyXGde36fLQ9yVj5o1azr6hYSE6D//+Y9ee+21PO+X3W7Xe++9p1deeUWlSv3ff5Yv34aLi4v8/PzUrFkzDRkyRLt3787z+q90/vx5jR07VqtXr873OnJjt9s1btw4NW7cWGXLlpWnp6caNGigV155RSdOnLgu27xWBfnv0vfff6/bbrtNHh4eqlq1qsaMGaNLly459Rk6dKi2bdum77//vkC2CRRXLkVdAICSY9++fU4HPnnxww8/aPr06YSLPFq5cqVKlSqlzz77TG5ublftX716dfXs2dNpWUZGhu677z7t3r1bzz33nGrVqqUDBw7o448/1ooVK7Rnzx6VK1dOkrRhwwZ9+OGHqlevnurWrauYmJhst5Oenq6IiAht2rRJgwYNUs2aNbVixQo999xzOnPmjNOB9B9//KFSpUpp4MCBCgoK0pkzZzRnzhy1bt1aS5cuVfv27a/5fUlOTlZYWJjOnTun5557TsHBwdq2bZumTZumVatWafPmzY6/zR9++EEzZ85Uo0aNVL16df3xxx/ZrvNa3qcpU6YoOTnZ6fVHjhzRqFGjdP/99zuWlS9fXj179tTq1auvGgozff7557p06ZK6d++epe2+++5T7969ZYxRUlKStm3bptmzZ+vjjz/We++9p+HDh+dpG5c7f/68xo0bJ+nvHxAK0qFDhxQeHq6jR4/qscce0zPPPCM3Nzdt375dn332mb755pscP4/CVFD/Li1btkydOnVS27Zt9dFHH2nHjh166623dOrUKafQHRQUpI4dO+qf//ynHnnkEYvVA8WYAXBTioyMNJLM77//nm17mzZtTP369S1vZ9CgQaYo/6lJTk4usm3nR79+/UyZMmXy1DckJMT06dMny/J169YZSWbatGlOyz///HMjySxevNix7K+//jJ2u90YY8z7779vJJnY2Ngs61y4cKGRZD777DOn5V27djUeHh4mPj4+11rPnTtnAgMDTURERJ727Upz5841ksySJUuclo8ePdpIMlu2bHEsi4uLM+fPnzfG5P73dy3vU3bGjx9vJJl169ZlaVu1apWRZBYtWnTVfWvUqJHp2bNnluWSzKBBg7Is/9///mfCwsKMJLN06dKrrv9KCQkJRpIZM2bMNb82N2lpaaZx48bGy8vLrF27Nkt7UlKSee211wp0m/lVUP8u1atXzzRu3NikpaU5lr3++uvGZrOZPXv2OPX96quvjM1mMwcPHrS8XaC4YigUgDy78hqLtLQ0jRs3TjVr1pSHh4cqVKigu+66S1FRUZKkvn37avr06ZKch3VkOnfunF588UUFBwfL3d1dtWvX1j//+U8ZY5y2e+HCBb3wwguqWLGiypUrp0ceeUR//vmnbDab0y+OY8eOlc1m0+7du/Xkk0+qfPnyuuuuuyRJ27dvV9++fVW9enXH8Jz+/ftnGUaTuY4//vhDPXv2lI+Pj/z9/fXGG2/IGKNjx46pY8eO8vb2VlBQkCZNmpSn9+7SpUsaP368br31Vrm7u6tatWp67bXXlJKS4uhjs9kUGRmpc+fOOd6rWbNm5Wn9l7Pb7ZKkwMBAp+WVKlWSJHl6ejqW+fn5OX6Vz83atWslSU888YTT8ieeeEIXL17Ud999l+vrvby85O/vr8TExKtuKzvXsk+BgYFOzwtindmZN2+eQkNDdccdd1x1WzmJjY3V9u3bFR4enufXVKhQQfPnz5eLi4vefvttx/LU1FSNHj1azZo1k4+Pj8qUKaO7775bq1atcvQ5fPiw/P39JUnjxo1z/J1lfo/y+j3Jztdff61t27bp9ddfd3zvLuft7e1UryQtWrRIzZo1k6enpypWrKiePXvqzz//dOrTtm3bbM+s9O3bV9WqVXPaN5vNpn/+85/69NNPHd+122+/Xb///rvT63L7d+nkyZPau3ev0tLSct3f3bt3a/fu3XrmmWfk4vJ/A0Cee+45GWOyXF+T+Rlf7bsClGQMhQJucklJSfrf//6XZfnV/qMq/X0QPmHCBD311FNq0aKF7Ha7Nm3apC1btui+++7TP/7xD504cUJRUVH6z3/+4/RaY4weeeQRrVq1SgMGDFCTJk20YsUKvfzyy/rzzz/1wQcfOPr27dtXCxcuVK9evdSqVSutWbNGHTp0yLGuxx57TDVr1tQ777zjCClRUVE6dOiQ+vXrp6CgIO3atUuffvqpdu3apV9//TXLOPxu3bqpbt26evfdd7V06VK99dZb8vPz07/+9S/dc889eu+99zR37ly99NJLuv3229W6detc36unnnpKs2fP1qOPPqoXX3xRv/32myZMmKA9e/bom2++kST95z//0aeffqqNGzdq5syZkpSvg9bmzZurTJkyeuONN+Tn56fatWvrwIEDGjFihG6//fZrOojNlJKSotKlS2cZnuXl5SVJ2rx5s55++mmnNrvdrtTUVP3vf//TF198oZ07d17TtQeXa926tUqVKqUhQ4Zo0qRJqlKlirZv3663335bnTp1Up06da55nVbep61bt2rPnj16/fXX87U/mdavXy9Juu22267pdVWrVlWbNm20atUq2e12eXt7y263a+bMmerevbuefvppnT17Vp999pkiIiK0ceNGNWnSRP7+/vrkk0/07LPPqnPnzurSpYskqVGjRpKu/XtyuczrB3r16pWnfZg1a5b69eun22+/XRMmTFB8fLymTp2qdevWaevWrfL19b2m9yTTvHnzdPbsWf3jH/+QzWbTxIkT1aVLFx06dEiurq65/rskSSNHjtTs2bMVGxvrFFyutHXrVkl//x1drnLlyqpSpYqjPZOPj49uvfVWrVu3TsOGDcvXvgHFXpGeLwFQZDKHQuX2uHIo1JVDbxo3bmw6dOiQ63ZyGnLw7bffGknmrbfeclr+6KOPGpvNZg4cOGCMMWbz5s1Gkhk6dKhTv759+2YZzjFmzBgjyXTv3j3L9jKHxlzuyy+/NJLMzz//nGUdzzzzjGPZpUuXTJUqVYzNZjPvvvuuY/mZM2eMp6dntsORLhcTE2Mkmaeeespp+UsvvWQkmZUrVzqW9enTx/JQKGOMWbJkialUqZLT5xkREWHOnj2b4/pyGwo1adIkIynLEJdXX33VSDIPPfRQltdEREQ4tu3m5mb+8Y9/mAsXLuRp37Izc+ZM4+vr67RPffr0cRqGcqWrDXnJz/tkjDEvvviikWR2796dbXteh0KNGjXKSMp2e8phKFSmIUOGGElm27Ztxpi//05TUlKc+pw5c8YEBgaa/v37O5blNhQqr9+T7DRt2tT4+Pjk2idTamqqCQgIMA0aNHD6m1iyZImRZEaPHu1Y1qZNG9OmTZss6+jTp48JCQlxPI+NjTWSTIUKFczp06cdy7/77jsjyfz3v/91LMvt76JPnz45fg8ul/l9OXr0aJa222+/3bRq1SrL8vvvv9/UrVs31/UCJRlDoYCb3PTp0xUVFZXlkfkLZm58fX21a9cu7d+//5q3+8MPP6h06dJ64YUXnJa/+OKLMsZo2bJlkqTly5dL+nt4weWef/75HNc9cODALMsuH9Zy8eJF/e9//1OrVq0kSVu2bMnS/6mnnnL8/9KlS6t58+YyxmjAgAGO5b6+vqpdu7YOHTqUYy3S3/sqKcuFti+++KIkaenSpbm+Pj/8/f3VtGlTvf322/r22281duxYrV27Vv369cvX+p588kn5+Piof//+ioqK0uHDh/Xpp5/q448/lvT3cLUrvfvuu/rxxx/12WefqVWrVkpNTc0yW861uOWWW9SiRQtNmTJF33zzjYYPH665c+fq1Vdfzfc68/M+ZWRkaP78+WratKnq1q2b721L0l9//SUXFxeVLVv2ml+b+ZqzZ89KktMZpYyMDJ0+fVqXLl1S8+bNs/0bz861fk8uZ7fb8zSsTpI2bdqkU6dO6bnnnpOHh4djeYcOHVSnTh1L34lu3bqpfPnyjud33323JF31e5pp1qxZMsbkerZC+r+/eXd39yxtHh4e2X4nypcvn+0ZYuBGwVAo4CbXokWLLKfypbz9B/DNN99Ux44dVatWLTVo0EDt27dXr1698hRKjhw5osqVK2c5EMk8UDty5Ijjf0uVKqXQ0FCnfjVq1Mhx3Vf2laTTp09r3Lhxmj9/vk6dOuXUlpSUlKV/1apVnZ77+PjIw8NDFStWzLL8auPPM/fhypqDgoLk6+vr2NeCcujQIbVr105ffPGFunbtKknq2LGj4xqZZcuW6YEHHrimdQYFBen7779Xr169HLMgeXt766OPPlKfPn2yPTC+/B4cPXv21G233aa+ffvm+d4Ol1u3bp0eeugh/frrr46/106dOsnb21vjxo1T//79Va9evWtaZ37fpzVr1ujPP/8s8uEsmbNUXf4dmj17tiZNmpTlGoHsvhPZudbvyeW8vb3zfPCe+Tdfu3btLG116tTRL7/8kqf1ZOfK725myDhz5ky+15mdzBB2+XVSmS5evJjtNTrGmFyHkwElHWcsAORb69atdfDgQX3++edq0KCBZs6cqdtuu81xfUBRye4/6I8//rj+/e9/a+DAgVq8eLF+/PFHx9mQ7G5KVrp06Twtk5TlYvOcFNYBxaxZs3Tx4kU99NBDTsszp7nM7/0kWrdurUOHDmnr1q365Zdf9Oeffzp+za5Vq1aur3Vzc9MjjzyixYsXZ/tL7tX861//UmBgYJYQ/Mgjj8gY47hW4Vrk932aO3euSpUqle30sNeqQoUKunTpkuOsw7XYuXOnSpcu7QgNc+bMUd++fXXrrbfqs88+0/LlyxUVFaV77rknzzfeu9bvyeXq1KmjpKQkHTt27Jr3JTc5fW/S09OzXW71e5pXmRf5nzx5MkvbyZMnVbly5SzLz5w5k+XHCeBGQrAAYImfn5/69eunL7/8UseOHVOjRo2cZmrK6aAgJCREJ06cyHJAtXfvXkd75v9mZGQoNjbWqd+BAwfyXOOZM2cUHR2tV199VePGjVPnzp113333qXr16nlehxWZ+3DlkLH4+HglJiY69rWgxMfHyxiT5cAr8xdsK8ORMu8Gfuedd6ps2bL66aefJClPF4RfuHBBxph8HUTHx8dneyBpZZ/y8z6lpKTo66+/Vtu2bbM9cLxWmRedX/n3fTVHjx7VmjVrFBYW5jhj8dVXX6l69epavHixevXqpYiICIWHh+vixYtOr83pO2n1e/Lwww9L+jvgXE3m3/y+ffuytO3bt8/pO1G+fPlsZxOzcqavIEJ+5hm5K28yeuLECR0/ftzpjF2m2NhYy8PngOKMYAEg364cAlS2bFnVqFHDaWhAmTJlJCnLgcGDDz6o9PR0TZs2zWn5Bx98IJvN5hiCEhERIUmOsfyZPvroozzXmfkL5pW/WE6ZMiXP67DiwQcfzHZ7kydPlqRcZ7jKj1q1askYo4ULFzot//LLLyVJTZs2LZDtJCQk6L333lOjRo2cgsWVQ2ikvz//r7/+WsHBwQoICLjmbdWqVUvx8fFZ7hZtZZ/y8z798MMPSkxMVI8ePa55e9kJCwuTlPXgNDenT59W9+7dlZ6e7jQrVXZ/57/99ps2bNjg9PrMmbyu/E5a/Z48+uijatiwod5+++0s25T+vhYks97mzZsrICBAM2bMcPr3YtmyZdqzZ4/Td+LWW2/V3r17lZCQ4Fi2bds2S3dyz+nfJSnv083Wr19fderU0aeffuoUTj/55BPZbDY9+uijTv2TkpJ08OBBS9MTA8Ud11gAyLd69eqpbdu2atasmfz8/LRp0yZ99dVXGjx4sKNPs2bNJEkvvPCCIiIiVLp0aT3xxBN6+OGH1a5dO73++us6fPiwGjdurB9//FHfffedhg4dqltvvdXx+q5du2rKlCn666+/HNPNZt69Ny+/PHp7e6t169aaOHGi0tLSdMstt+jHH3+85l+J86tx48bq06ePPv30UyUmJqpNmzbauHGjZs+erU6dOqldu3YFur2+ffvqn//8p/7xj39o69atql+/vrZs2aKZM2eqfv366ty5s6NvUlKSI6RlHqhNmzZNvr6+8vX1dfos27Rpo7CwMNWoUUNxcXH69NNPlZycrCVLljjdkf2BBx5QlSpV1LJlSwUEBOjo0aOKjIzUiRMntGDBAqdax44dq3HjxmnVqlW53gV68ODBioyM1MMPP6znn39eISEhWrNmjb788kvdd999atmypaPvkSNHHNOIZh6wv/XWW5L+/qU8czrUa3mfMs2dO1fu7u6OazKsql69uho0aKCffvpJ/fv3z9L+xx9/aM6cOTLGyG63a9u2bVq0aJGSk5M1efJkp7uYP/TQQ1q8eLE6d+6sDh06KDY2VjNmzFC9evWc7hru6empevXqacGCBapVq5b8/PzUoEEDNWjQwNL3xNXVVYsXL1Z4eLhat26txx9/XHfeeadcXV21a9cuzZs3T+XLl9fbb78tV1dXvffee+rXr5/atGmj7t27O6abrVatmtP1K/3799fkyZMVERGhAQMG6NSpU5oxY4bq16/vuBfJtcrp3yUp79PNStL777+vRx55RPfff7+eeOIJ7dy5U9OmTdNTTz2V5czETz/9JGOMOnbsmK+agRKh0OehAlAs5OfO21dOb/rWW2+ZFi1aGF9fX+Pp6Wnq1Klj3n77bZOamuroc+nSJfP8888bf39/Y7PZnKZ4PHv2rBk2bJipXLmycXV1NTVr1jTvv/++ycjIcNruuXPnzKBBg4yfn58pW7as6dSpk9m3b5+R5DT9a+ZUsQkJCVn25/jx46Zz587G19fX+Pj4mMcee8ycOHEixylrr1xHTtPA5vUO5WlpaWbcuHEmNDTUuLq6muDgYDNy5Ehz8eLFPG0nO7lNN3v8+HHTv39/Exoaatzc3EylSpXM008/nWW/MqfozO5x+VSexhgzbNgwU716dePu7m78/f3Nk08+me1dhKdNm2buuusuU7FiRePi4mL8/f3Nww8/nO10pS+++GK2dynOzt69e82jjz5qgoODjaurqwkJCTEvvfSSOXfunFO/zKles3tcOW1pXt8nY/6+c7SHh4fp0qXLVWu9ljtvT5482ZQtWzbLVK+X112qVCnj6+trmjZtaoYMGWJ27dqVZT0ZGRnmnXfeMSEhIcbd3d00bdrULFmyJMu0rMYYs379etOsWTPj5ubm9B3I6/ckN2fOnDGjR482DRs2NF5eXsbDw8M0aNDAjBw50pw8edKp74IFC0zTpk2Nu7u78fPzMz169DDHjx/Pss45c+aY6tWrGzc3N9OkSROzYsWKHKebff/997O8/sr6c/t3Ka/TzWb65ptvTJMmTYy7u7upUqWKGTVqlNO/gZm6detm7rrrrjytEyipbMYU8NVMAFAIYmJi1LRpU82ZM6fAhqWUNNWqVVNYWJg++ugjeXp6OoZ3lCQtWrRQSEiIFi1aVNSlFIj09HSdOXNG69atU6dOnbRo0aIsQ2KulJSUpOrVq2vixIlO0xnjxhEXF6fQ0FDNnz+fMxa4oXGNBYBiL7tZhKZMmaJSpUpd9Y7XN7r58+fL399fr7zySlGXcs0yh/a8+eabRV1KgdmxY4f8/f3VqVOnPL/Gx8dHI0aM0Pvvv5/n2ZtQskyZMkUNGzYkVOCGxxkLAMXeuHHjtHnzZrVr104uLi5atmyZli1bpmeeeUb/+te/irq8IrNu3TpH6AoODs72ngAoXMnJyfr1118dzxs1apSvi9UBoCQiWAAo9qKiojRu3Djt3r1bycnJqlq1qnr16qXXX39dLi7MQQEAQHFAsAAAAABgGddYAAAAALCMYAEAAADAMgYnF5CMjAydOHFC5cqVy9MNuwAAAIDizhijs2fPqnLlyk43Q80OwaKAnDhxQsHBwUVdBgAAAFDgjh07pipVquTah2BRQMqVKyfp7zfd29u7iKsBAAAArLPb7QoODnYc6+aGYFFAMoc/eXt7EywAAABwQ8nLUH8u3gYAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlrkUdQEAAADAleLi4pSYmFjUZRQ7vr6+CgoKKuoyskWwAAAAQLESFxenRx99VBcvXizqUoodDw8PffXVV8UyXBAsAAAAUKwkJibq4sWLGj9+vEJDQ4u6nGIjNjZWb7zxhhITEwkWAAAAQF6FhoaqTp06RV0G8oiLtwEAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAUKLExcXp+eefV/Xq1eXu7q7g4GA9/PDDio6OLvRabDabvv3223y9dvXq1brtttvk7u6uGjVqaNasWQVaW2FjVigAAACUGIcPH9add94pX19fvf/++2rYsKHS0tK0YsUKDRo0SHv37i3qEvMkNjZWHTp00MCBAzV37lxFR0frqaeeUqVKlRQREVHU5eULZywAAABQYjz33HOy2WzauHGjunbtqlq1aql+/foaPny4fv31V0e/o0ePqmPHjipbtqy8vb31+OOPKz4+3tHet29fderUyWndQ4cOVdu2bR3P27ZtqxdeeEEjRoyQn5+fgoKCNHbsWEd7tWrVJEmdO3eWzWZzPM+LGTNmKDQ0VJMmTVLdunU1ePBgPfroo/rggw+u5e0oVggWAAAAKBFOnz6t5cuXa9CgQSpTpkyWdl9fX0lSRkaGOnbsqNOnT2vNmjWKiorSoUOH1K1bt2ve5uzZs1WmTBn99ttvmjhxot58801FRUVJkn7//XdJUmRkpE6ePOl4fvjwYdlsNq1evTrH9W7YsEHh4eFOyyIiIrRhw4ZrrrG4YCgUAAAASoQDBw7IGHPVm+ZFR0drx44dio2NVXBwsCTpiy++UP369fX777/r9ttvz/M2GzVqpDFjxkiSatasqWnTpik6Olr33Xef/P39Jf0daC6/E7arq6tq164tLy+vHNcbFxenwMBAp2WBgYGy2+26cOGCPD0981xjcUGwAAAAQIlgjMlTvz179ig4ONgRKiSpXr168vX11Z49e645WFyuUqVKOnXqVK6vueWWW0rMtR4FiaFQAAAAKBFq1qwpm81WIAftpUqVyhJU0tLSsvRzdXV1em6z2ZSRkWF5+0FBQU7XfEhSfHy8vL29S+TZColgAQAAgBLCz89PERERmj59us6dO5elPTExUZJUt25dHTt2TMeOHXO07d69W4mJiapXr54kyd/fXydPnnR6fUxMzDXX5OrqqvT09Gt+XVhYWJbpcaOiohQWFnbN6youCBYAAAAoMaZPn6709HS1aNFCX3/9tfbv3689e/boww8/dByUh4eHq2HDhurRo4e2bNmijRs3qnfv3mrTpo2aN28uSbrnnnu0adMmffHFF9q/f7/GjBmjnTt3XnM91apVU3R0tOLi4nTmzBlJ0p9//qk6depo48aNOb5u4MCBOnTokEaMGKG9e/fq448/1sKFCzVs2LB8vCvFA8ECAAAAJUb16tW1ZcsWtWvXTi+++KIaNGig++67T9HR0frkk08k/T1c6bvvvlP58uXVunVrhYeHq3r16lqwYIFjPREREXrjjTc0YsQI3X777Tp79qx69+59zfVMmjRJUVFRCg4OVtOmTSX9PaRq3759On/+fI6vCw0N1dKlSxUVFaXGjRtr0qRJmjlzZom9h4Uk2Uxer4JBrux2u3x8fJSUlCRvb++iLgcAAKDE2rt3r3r27Kk5c+ZcdQaom0lRvC/XcozLGQsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWOZS1AUAAAAA2YmNjS3qEoqV4v5+ECwAAABQrPj6+srDw0NvvPFGUZdS7Hh4eMjX17eoy8gWwQIAAADFSlBQkL766islJiYWdSnFjq+vr4KCgoq6jGwRLAAAAFDsBAUFFdsDaGSPi7cBAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhWYoPF9OnTVa1aNXl4eKhly5bauHFjrv0XLVqkOnXqyMPDQw0bNtQPP/yQY9+BAwfKZrNpypQpBVw1AAAAcGMqkcFiwYIFGj58uMaMGaMtW7aocePGioiI0KlTp7Ltv379enXv3l0DBgzQ1q1b1alTJ3Xq1Ek7d+7M0vebb77Rr7/+qsqVK1/v3QAAAABuGCUyWEyePFlPP/20+vXrp3r16mnGjBny8vLS559/nm3/qVOnqn379nr55ZdVt25djR8/XrfddpumTZvm1O/PP//U888/r7lz58rV1bUwdgUAAAC4IZS4YJGamqrNmzcrPDzcsaxUqVIKDw/Xhg0bsn3Nhg0bnPpLUkREhFP/jIwM9erVSy+//LLq169/fYoHAAAAblAuRV3Atfrf//6n9PR0BQYGOi0PDAzU3r17s31NXFxctv3j4uIcz9977z25uLjohRdeyFMdKSkpSklJcTy32+153QUAAADghlPizlhcD5s3b9bUqVM1a9Ys2Wy2PL1mwoQJ8vHxcTyCg4Ovc5UAAABA8VXigkXFihVVunRpxcfHOy2Pj49XUFBQtq8JCgrKtf/atWt16tQpVa1aVS4uLnJxcdGRI0f04osvqlq1atmuc+TIkUpKSnI8jh07Zn3nAAAAgBKqxAULNzc3NWvWTNHR0Y5lGRkZio6OVlhYWLavCQsLc+ovSVFRUY7+vXr10vbt2xUTE+N4VK5cWS+//LJWrFiR7Trd3d3l7e3t9AAAAABuViXuGgtJGj58uPr06aPmzZurRYsWmjJlis6dO6d+/fpJknr37q1bbrlFEyZMkCQNGTJEbdq00aRJk9ShQwfNnz9fmzZt0qeffipJqlChgipUqOC0DVdXVwUFBal27dqFu3MAAABACVQig0W3bt2UkJCg0aNHKy4uTk2aNNHy5csdF2gfPXpUpUr938mYO+64Q/PmzdOoUaP02muvqWbNmvr222/VoEGDotoFAAAA4IZiM8aYoi7iRmC32+Xj46OkpCSGRQEAAOCGcC3HuCXuGgsAAAAAxQ/BAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgWYkNFtOnT1e1atXk4eGhli1bauPGjbn2X7RokerUqSMPDw81bNhQP/zwg6MtLS1Nr7zyiho2bKgyZcqocuXK6t27t06cOHG9dwMAAAC4IZTIYLFgwQINHz5cY8aM0ZYtW9S4cWNFRETo1KlT2fZfv369unfvrgEDBmjr1q3q1KmTOnXqpJ07d0qSzp8/ry1btuiNN97Qli1btHjxYu3bt0+PPPJIYe4WAAAAUGLZjDGmqIu4Vi1bttTtt9+uadOmSZIyMjIUHBys559/Xq+++mqW/t26ddO5c+e0ZMkSx7JWrVqpSZMmmjFjRrbb+P3339WiRQsdOXJEVatWvWpNdrtdPj4+SkpKkre3dz73DAAAACg+ruUYt8SdsUhNTdXmzZsVHh7uWFaqVCmFh4drw4YN2b5mw4YNTv0lKSIiIsf+kpSUlCSbzSZfX99s21NSUmS3250eAAAAwM2qxAWL//3vf0pPT1dgYKDT8sDAQMXFxWX7mri4uGvqf/HiRb3yyivq3r17jslswoQJ8vHxcTyCg4PzsTcAAADAjaHEBYvrLS0tTY8//riMMfrkk09y7Ddy5EglJSU5HseOHSvEKgEAAIDixaWoC7hWFStWVOnSpRUfH++0PD4+XkFBQdm+JigoKE/9M0PFkSNHtHLlylzHkbm7u8vd3T2fewEAAADcWErcGQs3Nzc1a9ZM0dHRjmUZGRmKjo5WWFhYtq8JCwtz6i9JUVFRTv0zQ8X+/fv1008/qUKFCtdnBwAAAIAbUIk7YyFJw4cPV58+fdS8eXO1aNFCU6ZM0blz59SvXz9JUu/evXXLLbdowoQJkqQhQ4aoTZs2mjRpkjp06KD58+dr06ZN+vTTTyX9HSoeffRRbdmyRUuWLFF6errj+gs/Pz+5ubkVzY4CAAAAJUSJDBbdunVTQkKCRo8erbi4ODVp0kTLly93XKB99OhRlSr1fydj7rjjDs2bN0+jRo3Sa6+9ppo1a+rbb79VgwYNJEl//vmnvv/+e0lSkyZNnLa1atUqtW3btlD2CwAAACipSuR9LIoj7mMBAACAG80NfR8LAAAAAMUPwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWEawAAAAAGAZwQIAAACAZQQLAAAAAJYRLAAAAABYRrAAAAAAYBnBAgAAAIBlBAsAAAAAlhEsAAAAAFhGsAAAAABgGcECAAAAgGUECwAAAACWESwAAAAAWGYpWBw6dKig6gAAAABQglkKFjVq1FC7du00Z84cXbx4saBqAgAAAFDCWAoWW7ZsUaNGjTR8+HAFBQXpH//4hzZu3FhQtQEAAAAoISwFiyZNmmjq1Kk6ceKEPv/8c508eVJ33XWXGjRooMmTJyshIaGg6gQAAABQjBXIxdsuLi7q0qWLFi1apPfee08HDhzQSy+9pODgYPXu3VsnT54siM0AAAAAKKYKJFhs2rRJzz33nCpVqqTJkyfrpZde0sGDBxUVFaUTJ06oY8eOBbEZAAAAAMWUi5UXT548WZGRkdq3b58efPBBffHFF3rwwQdVqtTfeSU0NFSzZs1StWrVCqJWAAAAAMWUpWDxySefqH///urbt68qVaqUbZ+AgAB99tlnVjYDAAAAoJizGWNMURdxI7Db7fLx8VFSUpK8vb2LuhwAAADAsms5xrV0jUVkZKQWLVqUZfmiRYs0e/ZsK6sGAAAAUIJYChYTJkxQxYoVsywPCAjQO++8Y2XVAAAAAEoQS8Hi6NGjCg0NzbI8JCRER48etbJqAAAAACWIpWAREBCg7du3Z1m+bds2VahQwcqqAQAAAJQgloJF9+7d9cILL2jVqlVKT09Xenq6Vq5cqSFDhuiJJ54oqBoBAAAAFHOWppsdP368Dh8+rHvvvVcuLn+vKiMjQ7179+YaCwAAAOAmUiDTzf7xxx/atm2bPD091bBhQ4WEhBREbSUK080CAADgRnMtx7iWzlhkqlWrlmrVqlUQqwIAAABQAlkKFunp6Zo1a5aio6N16tQpZWRkOLWvXLnSUnEAAAAASgZLwWLIkCGaNWuWOnTooAYNGshmsxVUXQAAAABKEEvBYv78+Vq4cKEefPDBgqoHAAAAQAlkabpZNzc31ahRo6BqAQAAAFBCWQoWL774oqZOnaoCmFgKAAAAQAlmaSjUL7/8olWrVmnZsmWqX7++XF1dndoXL15sqTgAAAAAJYOlYOHr66vOnTsXVC0AAAAASihLwSIyMvL/tXfvUVVW+R/HPweR442LJIIoZjqWWDYuYSQspxImHBtH05bmLbUzWaZO3mo0LX9dNcsupsV001xpmmWtMocyzDRlrAgz73kpNIE0FRDjJvv3R8vz+5GIwj5cDrxfa5016+xn7+d8n/Zo59N+9nM8VQcAAAAAL2a1x0KSiouL9emnn+rf//63cnNzJUlHjhzRqVOnrIsDAAAA4B2sVix+/PFH9e7dW+np6SooKNBf/vIX+fv768knn1RBQYESExM9VScAAACAWsxqxeLee+9VdHS0Tpw4ocaNG7vbb7nlFiUnJ1sXBwAAAMA7WAWLjRs3aubMmfLz8yvV3q5dO/30009WhV3IwoUL1a5dOzVq1EgxMTH68ssvy+2/cuVKderUSY0aNVKXLl20Zs2aUseNMXrooYfUqlUrNW7cWPHx8fr++++r8hIAAACAOsMqWJSUlOjMmTPntB8+fFj+/v42py7XihUrNHnyZM2aNUvffPON/vjHPyohIUE///xzmf03b96sIUOGyOVyKS0tTf3791f//v21fft2d5+5c+dq/vz5SkxM1JYtW9S0aVMlJCQoPz+/yq4DAAAAqCscxuLX7QYPHqzAwEC9/PLL8vf317Zt2xQSEqJ+/fqpbdu2VfbUqJiYGP3pT3/SggULJP0WcCIiIjRhwgRNmzatzDrz8vK0evVqd9s111yjrl27KjExUcYYhYeHa8qUKZo6daokKTs7W6GhoVq8eLFuu+22C9aUk5OjwMBAZWdnKyAgwENXCgAAANScinzHtVqxmDdvnjZt2qTOnTsrPz9fQ4cOdd8G9eSTT9qc+rwKCwuVmpqq+Ph4d5uPj4/i4+OVkpJS5piUlJRS/SUpISHB3f/gwYPKzMws1ScwMFAxMTHnPScAAACA/2P1VKg2bdro22+/1fLly7Vt2zadOnVKLpdLw4YNK7WZ25OOHTumM2fOKDQ0tFR7aGiodu/eXeaYzMzMMvtnZma6j59tO1+f3ysoKFBBQYH7fU5OTsUupAocP35cJ06cqOkyAAAA4EHNmzdXcHBwTZdxQVbBQpJ8fX01fPhwT9TiVWbPnq2HH364pssoJSkpSW+99VZNlwEAAAAPGjJkiIYOHVrTZVyQVbBYsmRJucdvv/12m9OXqUWLFmrQoIGysrJKtWdlZSksLKzMMWFhYeX2P/u/WVlZatWqVak+Xbt2LfOc06dP1+TJk93vc3JyFBERUeHr8aTevXsrJiamRmsAAACAZzVv3rymS7goVsHi3nvvLfW+qKhIp0+flp+fn5o0aVIlwcLPz09RUVFKTk5W//79Jf22eTs5OVnjx48vc0xsbKySk5M1ceJEd9vatWsVGxsrSbrssssUFham5ORkd5DIycnRli1bNHbs2DLP6XQ65XQ6PXZdnhAcHOwVy2QAAACoe6yCRVn383///fcaO3as7rvvPptTl2vy5MkaOXKkoqOj1b17dz333HPKy8vT6NGjJf22UtK6dWvNnj1b0m8B6Prrr9e8efN08803a/ny5fr666/18ssvS5IcDocmTpyoxx57TB07dtRll12mBx98UOHh4e7wAgAAAOD8rPdY/F7Hjh01Z84cDR8+/LybqW0NHjxYR48e1UMPPaTMzEx17dpVSUlJ7s3X6enp8vH5vwde9ejRQ8uWLdPMmTP1wAMPqGPHjnr//fd11VVXufvcf//9ysvL05gxY3Ty5Eldd911SkpKUqNGjarkGgAAAIC6xOp3LM5n69at+vOf/1wrnpRUXfgdCwAAANQ1FfmOa7Vi8cEHH5R6b4xRRkaGFixYoGuvvdbm1AAAAAC8iFWw+P3+A4fDoZCQEPXq1Uvz5s2zOTUAAAAAL2IVLEpKSjxVBwAAAAAv5nPhLgAAAABQPqsVi///A3EX8swzz9h8FAAAAIBazCpYpKWlKS0tTUVFRbriiiskSXv37lWDBg3UrVs3dz+Hw2FXJQAAAIBazSpY9O3bV/7+/nrjjTfcPzV+4sQJjR49Wj179tSUKVM8UiQAAACA2s3qdyxat26tTz75RFdeeWWp9u3bt+umm27SkSNHrAv0FvyOBQAAAOqainzHtdq8nZOTo6NHj57TfvToUeXm5tqcGgAAAIAXsQoWt9xyi0aPHq1Vq1bp8OHDOnz4sN599125XC4NGDDAUzUCAAAAqOWs9lgkJiZq6tSpGjp0qIqKin47oa+vXC6XnnrqKY8UCAAAAKD2s9pjcVZeXp72798vSerQoYOaNm1qXZi3YY8FAAAA6ppq22NxVkZGhjIyMtSxY0c1bdpUHsgqAAAAALyIVbD45ZdfFBcXp8svv1x9+vRRRkaGJMnlcvGoWQAAAKAesQoWkyZNUsOGDZWenq4mTZq42wcPHqykpCTr4gAAAAB4B6vN25988ok+/vhjtWnTplR7x44d9eOPP1oVBgAAAMB7WK1Y5OXllVqpOOv48eNyOp02pwYAAADgRayCRc+ePbVkyRL3e4fDoZKSEs2dO1c33nijdXEAAAAAvIPVrVBz585VXFycvv76axUWFur+++/Xjh07dPz4cW3atMlTNQIAAACo5axWLK666irt3btX1113nfr166e8vDwNGDBAaWlp6tChg6dqBAAAAFDLVXrFoqioSL1791ZiYqJmzJjhyZoAAAAAeJlKr1g0bNhQ27Zt82QtAAAAALyU1a1Qw4cP12uvveapWgAAAAB4KavN28XFxXr99df16aefKioqSk2bNi11/JlnnrEqDgAAAIB3qFSwOHDggNq1a6ft27erW7dukqS9e/eW6uNwOOyrAwAAAOAVKhUsOnbsqIyMDH322WeSpMGDB2v+/PkKDQ31aHEAAAAAvEOl9lgYY0q9/89//qO8vDyPFAQAAADA+1ht3j7r90EDAAAAQP1SqWDhcDjO2UPBngoAAACg/qrUHgtjjEaNGiWn0ylJys/P1913333OU6FWrVplXyEAAACAWq9SwWLkyJGl3g8fPtwjxQAAAADwTpUKFosWLfJ0HQAAAAC8mEc2bwMAAACo3wgWAAAAAKwRLAAAAABYI1gAAAAAsEawAAAAAGCNYAEAAADAGsECAAAAgDWCBQAAAABrBAsAAAAA1ggWAAAAAKwRLAAAAABYI1gAAAAAsEawAAAAAGCNYAEAAADAGsECAAAAgDWCBQAAAABrBAsAAAAA1ggWAAAAAKwRLAAAAABYI1gAAAAAsEawAAAAAGCNYAEAAADAGsECAAAAgDWCBQAAAABrBAsAAAAA1ggWAAAAAKwRLAAAAABYI1gAAAAAsEawAAAAAGCNYAEAAADAGsECAAAAgDWCBQAAAABrBAsAAAAA1rwuWBw/flzDhg1TQECAgoKC5HK5dOrUqXLH5Ofna9y4cbrkkkvUrFkzDRw4UFlZWe7j3377rYYMGaKIiAg1btxYkZGRev7556v6UgAAAIA6w+uCxbBhw7Rjxw6tXbtWq1ev1oYNGzRmzJhyx0yaNEkffvihVq5cqc8//1xHjhzRgAED3MdTU1PVsmVLvfnmm9qxY4dmzJih6dOna8GCBVV9OQAAAECd4DDGmJou4mLt2rVLnTt31ldffaXo6GhJUlJSkvr06aPDhw8rPDz8nDHZ2dkKCQnRsmXLdOutt0qSdu/ercjISKWkpOiaa64p87PGjRunXbt2ad26dRdVW05OjgIDA5Wdna2AgIBKXiEAAABQe1TkO65XrVikpKQoKCjIHSokKT4+Xj4+PtqyZUuZY1JTU1VUVKT4+Hh3W6dOndS2bVulpKSc97Oys7MVHBx83uMFBQXKyckp9QIAAADqK68KFpmZmWrZsmWpNl9fXwUHByszM/O8Y/z8/BQUFFSqPTQ09LxjNm/erBUrVpR7i9Xs2bMVGBjofkVERFTsYgAAAIA6pFYEi2nTpsnhcJT72r17d7XUsn37dvXr10+zZs3STTfddN5+06dPV3Z2tvt16NChaqkPAAAAqI18a7oASZoyZYpGjRpVbp/27dsrLCxMP//8c6n24uJiHT9+XGFhYWWOCwsLU2FhoU6ePFlq1SIrK+ucMTt37lRcXJzGjBmjmTNnlluP0+mU0+kstw8AAABQX9SKYBESEqKQkJAL9ouNjdXJkyeVmpqqqKgoSdK6detUUlKimJiYMsdERUWpYcOGSk5O1sCBAyVJe/bsUXp6umJjY939duzYoV69emnkyJF6/PHHPXBVAAAAQP3hVU+FkqS//vWvysrKUmJiooqKijR69GhFR0dr2bJlkqSffvpJcXFxWrJkibp37y5JGjt2rNasWaPFixcrICBAEyZMkPTbXgrpt9ufevXqpYSEBD311FPuz2rQoMFFBR6Jp0IBAACg7qnId9xasWJREUuXLtX48eMVFxcnHx8fDRw4UPPnz3cfLyoq0p49e3T69Gl327PPPuvuW1BQoISEBL344ovu4++8846OHj2qN998U2+++aa7/dJLL9UPP/xQLdcFAAAAeDOvW7GorVixAAAAQF1TZ3/HAgAAAEDtRLAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACw5nXB4vjx4xo2bJgCAgIUFBQkl8ulU6dOlTsmPz9f48aN0yWXXKJmzZpp4MCBysrKKrPvL7/8ojZt2sjhcOjkyZNVcAUAAABA3eN1wWLYsGHasWOH1q5dq9WrV2vDhg0aM2ZMuWMmTZqkDz/8UCtXrtTnn3+uI0eOaMCAAWX2dblcuvrqq6uidAAAAKDOchhjTE0XcbF27dqlzp0766uvvlJ0dLQkKSkpSX369NHhw4cVHh5+zpjs7GyFhIRo2bJluvXWWyVJu3fvVmRkpFJSUnTNNde4+7700ktasWKFHnroIcXFxenEiRMKCgq6qNpycnIUGBio7OxsBQQE2F8sAAAAUMMq8h3Xq1YsUlJSFBQU5A4VkhQfHy8fHx9t2bKlzDGpqakqKipSfHy8u61Tp05q27atUlJS3G07d+7UI488oiVLlsjH58L/WAoKCpSTk1PqBQAAANRXXhUsMjMz1bJly1Jtvr6+Cg4OVmZm5nnH+Pn5nbPyEBoa6h5TUFCgIUOG6KmnnlLbtm0vqpbZs2crMDDQ/YqIiKj4BQEAAAB1RK0IFtOmTZPD4Sj3tXv37ir7/OnTpysyMlLDhw+v0Jjs7Gz369ChQ1VWHwAAAFDb+dZ0AZI0ZcoUjRo1qtw+7du3V1hYmH7++edS7cXFxTp+/LjCwsLKHBcWFqbCwkKdPHmy1KpFVlaWe8y6dev03Xff6Z133pEknd120qJFC82YMUMPP/zwOed1Op1yOp0Xe4kAAABAnVYrgkVISIhCQkIu2C82NlYnT55UamqqoqKiJP0WCkpKShQTE1PmmKioKDVs2FDJyckaOHCgJGnPnj1KT09XbGysJOndd9/Vr7/+6h7z1Vdf6Y477tDGjRvVoUMH28sDAAAA6rxaESwuVmRkpHr37q0777xTiYmJKioq0vjx43Xbbbe5nwj1008/KS4uTkuWLFH37t0VGBgol8ulyZMnKzg4WAEBAZowYYJiY2PdT4T6fXg4duyY+/Mu9qlQAAAAQH3mVcFCkpYuXarx48crLi5OPj4+GjhwoObPn+8+XlRUpD179uj06dPutmeffdbdt6CgQAkJCXrxxRdronwAAACgTvKq37GozfgdCwAAANQ1dfZ3LAAAAADUTgQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAa741XUBdYYyRJOXk5NRwJQAAAIBnnP1ue/a7bnkIFh6Sm5srSYqIiKjhSgAAAADPys3NVWBgYLl9HOZi4gcuqKSkREeOHJG/v78cDke1f35OTo4iIiJ06NAhBQQEVPvno+Yw9/UT815/Mff1F3NfP9X0vBtjlJubq/DwcPn4lL+LghULD/Hx8VGbNm1qugwFBATwl009xdzXT8x7/cXc11/Mff1Uk/N+oZWKs9i8DQAAAMAawQIAAACANYJFHeF0OjVr1iw5nc6aLgXVjLmvn5j3+ou5r7+Y+/rJm+adzdsAAAAArLFiAQAAAMAawQIAAACANYIFAAAAAGsECy+ycOFCtWvXTo0aNVJMTIy+/PLLcvuvXLlSnTp1UqNGjdSlSxetWbOmmiqFJ1Vk3l955RX17NlTzZs3V/PmzRUfH3/B/5+g9qron/mzli9fLofDof79+1dtgagyFZ37kydPaty4cWrVqpWcTqcuv/xy/s73UhWd++eee05XXHGFGjdurIiICE2aNEn5+fnVVC08YcOGDerbt6/Cw8PlcDj0/vvvX3DM+vXr1a1bNzmdTv3hD3/Q4sWLq7zOi2LgFZYvX278/PzM66+/bnbs2GHuvPNOExQUZLKyssrsv2nTJtOgQQMzd+5cs3PnTjNz5kzTsGFD891331Vz5bBR0XkfOnSoWbhwoUlLSzO7du0yo0aNMoGBgebw4cPVXDlsVXTuzzp48KBp3bq16dmzp+nXr1/1FAuPqujcFxQUmOjoaNOnTx/zxRdfmIMHD5r169ebrVu3VnPlsFXRuV+6dKlxOp1m6dKl5uDBg+bjjz82rVq1MpMmTarmymFjzZo1ZsaMGWbVqlVGknnvvffK7X/gwAHTpEkTM3nyZLNz507zwgsvmAYNGpikpKTqKbgcBAsv0b17dzNu3Dj3+zNnzpjw8HAze/bsMvsPGjTI3HzzzaXaYmJizF133VWldcKzKjrvv1dcXGz8/f3NG2+8UVUloopUZu6Li4tNjx49zKuvvmpGjhxJsPBSFZ37l156ybRv394UFhZWV4moIhWd+3HjxplevXqVaps8ebK59tprq7ROVJ2LCRb333+/ufLKK0u1DR482CQkJFRhZReHW6G8QGFhoVJTUxUfH+9u8/HxUXx8vFJSUsock5KSUqq/JCUkJJy3P2qfysz7750+fVpFRUUKDg6uqjJRBSo794888ohatmwpl8tVHWWiClRm7j/44APFxsZq3LhxCg0N1VVXXaUnnnhCZ86cqa6y4QGVmfsePXooNTXVfbvUgQMHtGbNGvXp06daakbNqM3f8XxrugBc2LFjx3TmzBmFhoaWag8NDdXu3bvLHJOZmVlm/8zMzCqrE55VmXn/vX/9618KDw8/5y8g1G6VmfsvvvhCr732mrZu3VoNFaKqVGbuDxw4oHXr1mnYsGFas2aN9u3bp3vuuUdFRUWaNWtWdZQND6jM3A8dOlTHjh3TddddJ2OMiouLdffdd+uBBx6ojpJRQ873HS8nJ0e//vqrGjduXEOVsXkbqLPmzJmj5cuX67333lOjRo1quhxUodzcXI0YMUKvvPKKWrRoUdPloJqVlJSoZcuWevnllxUVFaXBgwdrxowZSkxMrOnSUMXWr1+vJ554Qi+++KK++eYbrVq1Sh999JEeffTRmi4N9RQrFl6gRYsWatCggbKyskq1Z2VlKSwsrMwxYWFhFeqP2qcy837W008/rTlz5ujTTz/V1VdfXZVlogpUdO7379+vH374QX379nW3lZSUSJJ8fX21Z88edejQoWqLhkdU5s99q1at1LBhQzVo0MDdFhkZqczMTBUWFsrPz69Ka4ZnVGbuH3zwQY0YMUL/+Mc/JEldunRRXl6exowZoxkzZsjHh/9+XBed7zteQEBAja5WSKxYeAU/Pz9FRUUpOTnZ3VZSUqLk5GTFxsaWOSY2NrZUf0lau3btefuj9qnMvEvS3Llz9eijjyopKUnR0dHVUSo8rKJz36lTJ3333XfaunWr+/X3v/9dN954o7Zu3aqIiIjqLB8WKvPn/tprr9W+ffvcYVKS9u7dq1atWhEqvEhl5v706dPnhIezAdMYU3XFokbV6u94Nb17HBdn+fLlxul0msWLF5udO3eaMWPGmKCgIJOZmWmMMWbEiBFm2rRp7v6bNm0yvr6+5umnnza7du0ys2bN4nGzXqii8z5nzhzj5+dn3nnnHZORkeF+5ebm1tQloJIqOve/x1OhvFdF5z49Pd34+/ub8ePHmz179pjVq1ebli1bmscee6ymLgGVVNG5nzVrlvH39zdvvfWWOXDggPnkk09Mhw4dzKBBg2rqElAJubm5Ji0tzaSlpRlJ5plnnjFpaWnmxx9/NMYYM23aNDNixAh3/7OPm73vvvvMrl27zMKFC3ncLCruhRdeMG3btjV+fn6me/fu5r///a/72PXXX29GjhxZqv/bb79tLr/8cuPn52euvPJK89FHH1VzxfCEisz7pZdeaiSd85o1a1b1Fw5rFf0z//8RLLxbRed+8+bNJiYmxjidTtO+fXvz+OOPm+Li4mquGp5QkbkvKioy//M//2M6dOhgGjVqZCIiIsw999xjTpw4Uf2Fo9I+++yzMv/dfXauR44caa6//vpzxnTt2tX4+fmZ9u3bm0WLFlV73WVxGMNaGQAAAAA77LEAAAAAYI1gAQAAAMAawQIAAACANYIFAAAAAGsECwAAAADWCBYAAAAArBEsAAAAAFgjWAAAAACwRrAAAHiVG264QRMnTqzpMgAAv0OwAABUm759+6p3795lHtu4caMcDoe2bdtWzVUBADyBYAEAqDYul0tr167V4cOHzzm2aNEiRUdH6+qrr66BygAAtggWAIBq87e//U0hISFavHhxqfZTp05p5cqV6t+/v4YMGaLWrVurSZMm6tKli956661yz+lwOPT++++XagsKCir1GYcOHdKgQYMUFBSk4OBg9evXTz/88INnLgoAIIlgAQCoRr6+vrr99tu1ePFiGWPc7StXrtSZM2c0fPhwRUVF6aOPPtL27ds1ZswYjRgxQl9++WWlP7OoqEgJCQny9/fXxo0btWnTJjVr1ky9e/dWYWGhJy4LACCCBQCgmt1xxx3av3+/Pv/8c3fbokWLNHDgQF166aWaOnWqunbtqvbt22vChAnq3bu33n777Up/3ooVK1RSUqJXX31VXbp0UWRkpBYtWqT09HStX7/eA1cEAJAIFgCAatapUyf16NFDr7/+uiRp37592rhxo1wul86cOaNHH31UXbp0UXBwsJo1a6aPP/5Y6enplf68b7/9Vvv27ZO/v7+aNWumZs2aKTg4WPn5+dq/f7+nLgsA6j3fmi4AAFD/uFwuTZgwQQsXLtSiRYvUoUMHXX/99XryySf1/PPP67nnnlOXLl3UtGlTTZw4sdxblhwOR6nbqqTfbn8669SpU4qKitLSpUvPGRsSEuK5iwKAeo5gAQCodoMGDdK9996rZcuWacmSJRo7dqwcDoc2bdqkfv36afjw4ZKkkpIS7d27V507dz7vuUJCQpSRkeF+//333+v06dPu9926ddOKFSvUsmVLBQQEVN1FAUA9x61QAIBq16xZMw0ePFjTp09XRkaGRo0aJUnq2LGj1q5dq82bN2vXrl266667lJWVVe65evXqpQULFigtLU1ff/217r77bjVs2NB9fNiwYWrRooX69eunjRs36uDBg1q/fr3++c9/lvnYWwBA5RAsAAA1wuVy6cSJE0pISFB4eLgkaebMmerWrZsSEhJ0ww03KCwsTP379y/3PPPmzVNERIR69uypoUOHaurUqWrSpIn7eJMmTbRhwwa1bdtWAwYMUGRkpFwul/Lz81nBAAAPcpjf35gKAAAAABXEigUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWCNYAAAAALBGsAAAAABgjWABAAAAwBrBAgAAAIA1ggUAAAAAawQLAAAAANYIFgAAAACsESwAAAAAWPtfaGpUZEQCXRoAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "各区间数据量统计：\n", - "[36, 1024): 4127403 条\n", - "[1024, 2048): 1085218 条\n", - "[2048, 8193): 308112 条\n", - "[8193, 8187]: 0 条\n" - ] - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "\n", - "def plot_interval_histograms(file_path, split_points):\n", - " \"\"\"\n", - " 读取文件中的数值，按 [min, a), [a, b), [b, c), [c, max] 划分区间，绘制每个区间直方图并统计数据量\n", - " :param file_path: 包含数值的文件路径，每行格式建议为 \"xxx: 数值\"\n", - " :param split_points: 分割关键点列表，如 [a, b, c]，需满足 a < b < c\n", - " \"\"\"\n", - " # ---------------------- 1. 读取并提取数值 ----------------------\n", - " values = []\n", - " with open(file_path, 'r', encoding='utf-8') as f:\n", - " for line in f:\n", - " try:\n", - " # 假设每行是类似 \"xxx: 数值\" 格式，提取最后面的数值部分并转成整数\n", - " num = int(line.strip().split(':')[-1].strip())\n", - " values.append(num)\n", - " except (IndexError, ValueError):\n", - " # 遇到格式异常的行，跳过处理\n", - " continue\n", - " if not values:\n", - " raise ValueError(\"文件中未提取到有效数值，请检查文件内容格式\")\n", - "\n", - " # ---------------------- 2. 确定区间边界 ----------------------\n", - " min_val = min(values)\n", - " max_val = max(values)\n", - " # 结合传入的分割点，构建完整区间边界\n", - " bins = [min_val] + split_points + [max_val]\n", - " # 构建区间标签，方便识别\n", - " interval_labels = [\n", - " f\"[{min_val}, {split_points[0]})\",\n", - " f\"[{split_points[0]}, {split_points[1]})\",\n", - " f\"[{split_points[1]}, {split_points[2]})\",\n", - " f\"[{split_points[2]}, {max_val}]\"\n", - " ]\n", - "\n", - " # ---------------------- 3. 按区间分组数据 ----------------------\n", - " grouped_data = {label: [] for label in interval_labels}\n", - " for num in values:\n", - " for i in range(len(bins) - 1):\n", - " lower_bound = bins[i]\n", - " upper_bound = bins[i + 1]\n", - " # 处理左闭右开区间，最后一个区间是左闭右闭\n", - " if i < len(bins) - 2 and lower_bound <= num < upper_bound:\n", - " grouped_data[interval_labels[i]].append(num)\n", - " break\n", - " elif i == len(bins) - 2 and lower_bound <= num <= upper_bound:\n", - " grouped_data[interval_labels[i]].append(num)\n", - " break\n", - "\n", - " # ---------------------- 4. 绘制每个区间的直方图 ----------------------\n", - " plt.rcdefaults() # 重置 matplotlib 配置，避免字体等潜在问题\n", - " fig, axes = plt.subplots(nrows=len(interval_labels), ncols=1, figsize=(8, 5 * len(interval_labels)))\n", - " # 处理单轴和多轴情况，让代码更鲁棒\n", - " axes = np.array(axes).reshape(-1)\n", - "\n", - " for i, (label, data) in enumerate(grouped_data.items()):\n", - " ax = axes[i]\n", - " # 绘制直方图，用 auto 自动适配 bins\n", - " ax.hist(data, bins='auto', edgecolor='black', alpha=0.7)\n", - " # 设置标题，带上该区间的数据量\n", - " ax.set_title(f\"Histogram of {label} (Data Count: {len(data)})\")\n", - " ax.set_xlabel(\"Value\")\n", - " ax.set_ylabel(\"Frequency\")\n", - " # 在图中添加数据量文本标注，放在右上角，带半透明背景\n", - " ax.text(\n", - " 0.95, 0.95, \n", - " f\"Count: {len(data)}\", \n", - " horizontalalignment='right', \n", - " verticalalignment='top', \n", - " transform=ax.transAxes, \n", - " fontsize=10, \n", - " bbox=dict(facecolor='white', alpha=0.8)\n", - " )\n", - "\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n", - " # ---------------------- 5. 打印区间数据量汇总 ----------------------\n", - " print(\"各区间数据量统计：\")\n", - " for label, data_list in grouped_data.items():\n", - " print(f\"{label}: {len(data_list)} 条\")\n", - "\n", - "\n", - "file_name = \"/workspace/data4packing/RiceVL/data_procs/token_info_v2_vqa_pretrain_5M_8k.txt\"\n", - "split_points = [1024, 2048, 8193] # 你可以根据实际需求调整这三个分割点\n", - "try:\n", - " plot_interval_histograms(file_name, split_points)\n", - "except Exception as e:\n", - " print(f\"执行过程中出现错误：{e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "2a459f04-924e-4927-bf1b-078a46aa930e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "5497705" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "4127332+1085218+285155" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e3216da-1881-431c-ae08-0993cf332615", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ae5ccb7e-edf4-4a96-b1c8-3e3eb3b4de6a", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e461e2f0-8592-4eef-9c1b-8e65effde478", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "66d8b229-2eca-4741-bf2f-fb873fb395ad", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e4f5d11f-4568-49e9-85eb-fab70eeabdfd", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9236575f-bcc0-4747-b118-b2f927668526", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/tools/data_preprocess/offline_packing/s2_prepare_rawsamples-emova.py b/tools/data_preprocess/offline_packing/s2_prepare_rawsamples-emova.py deleted file mode 100644 index f01e2d15..00000000 --- a/tools/data_preprocess/offline_packing/s2_prepare_rawsamples-emova.py +++ /dev/null @@ -1,396 +0,0 @@ -# ### 所有代码放到一起，只运行这一块就可以 -# Step1: -# python -u s2_prepare_rawsamples-emova.py 2>&1 | tee s2_proc.log - -import bisect -import os -import json -import sys -import shutil -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union -import threading -from concurrent.futures import ThreadPoolExecutor, as_completed - -# ### 参数配置 -target_directory = "/mnt/cluster" # 最终数据存放的位置 - -# current_file = Path(__file__).resolve() -# target_directory = current_file.parent - -newDir = "raw_packing_data_emova_3000tk" # 转 webdataset 之前数据的存放位置 (jpg, json) -SRC_DIR_IMGS = "/vlm/xiangan/datasets/aiak_caption_emova_300k/" # 原始 img 数据的存放位置 -SRC_DIR_JSONS = "/vlm/xiangan/datasets/aiak_caption_emova_300k/" # 原始 json 数据的存放位置 -SRC_DST_EXTENSIONS = ("jpg", "json") -# f_toklens_originalsample = os.path.join(target_directory, "token_info_v2_emova_3000tk.txt") -# s1 的结果 -f_toklens_originalsample = "/workspace/gits/train_rice_vl/tools/data_preprocess/offline_packing/token_info_v2_emova_3000tk.txt" -PACKED_LENGTH = 8192 -# s2 的结果位置 -dst_dir = os.path.join(target_directory,newDir) -MAX_WORKERS = 96 # 线程池大小（根据CPU核心数和IO性能调整） - -f_TEST=False # test 示例输出（仅做测试用：生成少量数据） -n_packed_samples=100 # 测试用，输出几条打包后的数据 - -# PROMPTS = # Creating a list of the provided English prompts -PROMPTS = [ - "What about this picture?", - "Please provide a vivid description of the image.", - "Please Depict the image in words." - "Could you please transcribe thr image into a descriptive paragraph?" - "What is the content of this figure?", - "What do you see here?", - "Tell me about this image.", - "What's going on in this artwork?", - "What is depicted in this painting?", - "What is the subject matter here?", - "What can you make out in this picture?", - "What's the main thing shown in this image?", - "What's the gist of this artwork?", - "What's the essence of this figure?", - "What's the general idea here?", - "What does this image show?", - "What's the core element in this painting?", - "What's the overview of this scene?", - "What's the primary focus of this artwork?", - "What's the fundamental subject matter?", - "What's the general view presented?", - "What's the main impression given by this picture?", - "What's the central theme shown?", - "What's the overall presentation here?", - "What's the key element you notice?", - "What's the fundamental concept in this image?", - "What's the overall content?", - "What's the main thing you get from this?", - "What's the general subject?", - "What's the core idea conveyed?", - "What's the basic representation?", - "What's the main point of this figure?" -] - -import random - -def get_random_prompts(prompts, n): - if n > len(prompts): - # 允许重复 - return random.choices(prompts, k=n) - else: - # 不允许重复 - return random.sample(prompts, n) - -# 全局变量 - 用元组存储（不可变，效率更高） -BASE_NAMES = [] # 初始化为空元组，后续会被替换 (所有在原始数据集中的 sample 名称， 已经按照 tokens 长度排序) - -def search_for_fit(numbers: List[int], capacity: int) -> int: - """Finds the index of largest number that fits into the knapsack with the given capacity.""" - index = bisect.bisect(numbers, capacity) - return -1 if index == 0 else (index - 1) - -def greedy_knapsack(numbers: List[int], capacity: int) -> Tuple[List[List[int]], List[List[int]]]: - r"""Implement efficient greedy algorithm with binary search for the knapsack problem. - 参数 - ---- - numbers : List[int] - 物品大小列表，可以为任意顺序 - capacity : int - 背包容量 - - 返回 - ---- - Tuple[List[List[int]], List[List[int]]] - 第一个列表：每个背包里的物品大小 - 第二个列表：每个背包里物品对应的原始下标 - - """ - # 保存原始索引，与输入的numbers一一对应 - indexed_numbers = [(val, idx) for idx, val in enumerate(numbers)] - # 由于输入已排序，直接使用即可（保持与原逻辑一致的处理方式） - knapsacks = [] - index_knapsacks = [] - iii = int(0) - while indexed_numbers: - current_knapsack = [] - current_indices = [] - remaining_capacity = capacity - - while True: - # 提取当前数值列表用于查找（保持升序） - current_values = [val for val, idx in indexed_numbers] - index = search_for_fit(current_values, remaining_capacity) - if index == -1: - break # 没有可放入当前背包的物品了 - - # 取出找到的物品及其原始索引 - val, idx = indexed_numbers.pop(index) - remaining_capacity -= val - current_knapsack.append(val) - current_indices.append(idx) - - if iii%1000==0: - print(f"---------第{iii}个pack----------") - print(f"{current_knapsack}--->{sum(current_knapsack)}") - print(current_indices) - print(f"\n") - iii+=1 - knapsacks.append(tuple(current_knapsack)) - index_knapsacks.append(tuple(current_indices)) - - return tuple(knapsacks), tuple(index_knapsacks) # 使用了 tuple - -def extract_content(json_file): - try: - # 打开并加载JSON文件 - with open(json_file, 'r', encoding='utf-8') as f: - data = json.load(f) - - # 提取content内容 - # 假设captions数组至少有一个元素 - if data.get('captions') and len(data['captions']) > 0: - return data['captions'][0].get('content', "") - else: - return "未找到有效的caption内容" - - except FileNotFoundError: - return f"错误：文件 {json_file} 不存在" - except json.JSONDecodeError: - return f"错误：文件 {json_file} 不是有效的JSON格式" - except Exception as e: - return f"提取过程中发生错误：{str(e)}" - -def prepare_dirs(target_dir, new_dir): - os.chdir(target_dir) - print(f"--------change to directory {target_dir}--------") - # 创建新目录 - if not os.path.exists(new_dir): - os.makedirs(new_dir) - print(f"Directory '{new_dir}' created.") - else: - print(f"Directory '{new_dir}' already exists.") - - -def dataset_tokinfo_generator(f_name): - """ - 数据集token信息生成器，逐行读取并解析文件内容 - - 参数: - f_name (str): 包含token信息的文件路径 - - 生成: - tuple: (base_name, token_len) - 解析后的基础文件名和token长度 - """ - try: - with open(f_name, 'r', encoding='utf-8') as f: - for line in f: - # 跳过空行 - stripped_line = line.strip() - if not stripped_line: - continue - - # 按冒号分割并验证格式 - parts = stripped_line.split(':') - if len(parts) == 2: - base_name = parts[0].strip() - token_len_str = parts[1].strip() - - try: - token_len = int(token_len_str) - yield (base_name, token_len) - except ValueError: - print( - f"警告: 无法将 '{token_len_str}' 转换为整数，已跳过该行", - file=sys.stderr - ) - continue - - except FileNotFoundError: - print(f"错误: 文件 '{f_name}' 不存在", file=sys.stderr) - return - except Exception as e: - print(f"处理文件时发生错误: {str(e)}", file=sys.stderr) - return - - -class TokenInfoReader: - """ - Token信息读取器 - - 支持分批读取、全量读取和断点续读功能，适用于处理包含token信息的文本文件。 - 文件格式要求: 每行一条记录，格式为 "base_name: token_len" - """ - - def __init__(self, f_name): - """ - 初始化读取器 - - 参数: - f_name (str): 包含token信息的文件路径 - """ - self.f_name = f_name - self.generator = dataset_tokinfo_generator(f_name) - self._current_position = 0 # 记录已读取的记录数量 - - def read(self, count=None): - """ - 读取记录 - - 参数: - count (int, optional): 要读取的记录数量，默认为None（读取全部剩余记录） - - 返回: - tuple: (base_names列表, token_lens列表, 实际读取数量) - """ - base_names = [] - token_lens = [] - read_count = 0 - - # 循环读取直到达到指定数量或文件结束 - while True: - # 检查是否已达到指定读取数量 - if count is not None and read_count >= count: - break - - try: - # 从生成器获取下一条记录 - base_name, token_len = next(self.generator) - base_names.append(base_name) - token_lens.append(token_len) - read_count += 1 - self._current_position += 1 - - except StopIteration: - # 已读取到文件末尾 - break - - return base_names, token_lens, read_count - - def get_current_position(self): - """ - 获取当前读取位置 - - 返回: - int: 已读取的记录总数 - """ - return self._current_position - - -def process_knapsack(s1, idx_knapsack, dst_dir): - """ - 处理单个 packing 数据 - - 参数: - s1: 当前处理组的索引 - idx_knapsack: 背包中包含的索引列表 - dst_dir: 目标目录路径 - """ - global BASE_NAMES - - packed_imgs, packed_caps = [], [] # 单个 packed-sample 的构成 - - # 获取基础文件名 - packed_b_names = (BASE_NAMES[idx] for idx in idx_knapsack) - - # 构建源文件信息 - packed_info = ( - (os.path.join(SRC_DIR_IMGS, f"{b_name}.{SRC_DST_EXTENSIONS[0]}"), - extract_content(os.path.join(SRC_DIR_JSONS, f"{b_name}.{SRC_DST_EXTENSIONS[1]}"))) - for b_name in packed_b_names - ) - - # 目标JSON文件路径 - json_dst = os.path.join(dst_dir, f"ps_{s1:08d}.{SRC_DST_EXTENSIONS[1]}") - - # 处理每张图片和对应的描述 - for s2, (img_src, cap_src) in enumerate(packed_info): - # 目标图片路径 - img_name_dst = f"ps_{s1:08d}.img{s2:03d}.{SRC_DST_EXTENSIONS[0]}" - # img_name_dst = f"img{s2:03d}.{SRC_DST_EXTENSIONS[0]}" # 看后面具体需求决定使用哪一个 - img_dst = os.path.join(dst_dir, img_name_dst) - - # 收集信息 - # packed_imgs.append(img_name_dst) - packed_imgs.append(f"img{s2:03d}.{SRC_DST_EXTENSIONS[0]}") - packed_caps.append(cap_src) - - # 复制图片 - shutil.copyfile(img_src, img_dst) - # 此处也可以调用大模型来生成 提问, 更加多样性 - selected_prompts = get_random_prompts(PROMPTS, len(packed_imgs)) - # 生成JSON文件 - json_data = { - "images": packed_imgs, - "captions": packed_caps, - "prompts": selected_prompts - } - # print(packed_imgs) - - try: - with open(json_dst, 'w', encoding='utf-8') as f: - json.dump(json_data, f, indent=4, ensure_ascii=False) - # json.dump(json_data, f) - except Exception as e: - print(f"线程 {threading.current_thread().name} 生成JSON文件 {json_dst} 失败: {str(e)}") - return s1 - - -if __name__ == "__main__": - ## 1. 创建工作目录 - print("Step1-----------------已创建工作环境-----------------Start") - prepare_dirs(target_directory, newDir) - print("Step1-----------------已创建工作环境-----------------Stop\n\n") - - ## 2. 获取原始数据集信息（没有处理之前） - # 可以用于构建多个 pool，分块 packing（read的参数决定 packing cache size） - print("Step2-----------------读取原ds的 tokenlen 信息-----------------Start") - info_reader = TokenInfoReader(f_toklens_originalsample) - base_names, token_lens, n_count = info_reader.read() - - # global BASE_NAMES - BASE_NAMES=tuple(base_names) - print(f"已读取{n_count}条数据") - print("Step2-----------------读取原ds的 tokenlen 信息-----------------Stop\n\n") - - ## 3. packing分组 - # 调用 packing-group 进行分组 - print("Step3-----------------packing 分组-----------------Start") - knapsacks, idx_knapsacks= greedy_knapsack(token_lens, PACKED_LENGTH) - print(idx_knapsacks[10]) - print(knapsacks[10]) - - total_knapsacks = len(idx_knapsacks) - print(f"原始数据----{n_count}----条，packing后变为----{total_knapsacks}----条") - print("Step3-----------------packing 分组-----------------Stop\n\n") - - print("Step4----------------- 开始构建新数据集 -----------------Start") - print(f"开始处理 {total_knapsacks} 组数据，使用 {MAX_WORKERS} 个线程") - - # 4. 使用线程池处理所有pack - with ThreadPoolExecutor(max_workers=MAX_WORKERS, thread_name_prefix="PackThread") as executor: - # 提交所有任务 - if f_TEST: - futures = { - executor.submit(process_knapsack, s1, idx_knapsack, dst_dir): s1 - for s1, idx_knapsack in enumerate(idx_knapsacks[0:n_packed_samples]) - } - else: - futures = { - executor.submit(process_knapsack, s1, idx_knapsack, dst_dir): s1 - for s1, idx_knapsack in enumerate(idx_knapsacks) - } - - # 跟踪进度 - completed = 0 - for future in as_completed(futures): - s1 = futures[future] - try: - result = future.result() - completed += 1 - # 每完成1%输出一次进度 - if completed % (max(1, total_knapsacks // 10)) == 0: - print(f"已完成 {completed}/{total_knapsacks} 组数据 ({completed/total_knapsacks*100:.1f}%)") - except Exception as e: - print(f"处理第 {s1} 组数据时发生错误: {str(e)}") - - print(f"所有数据处理完成，共处理 {completed} 组") - - print("Step4-----------------Sccessful！！！！---- 构建新数据集成功 -----------------Stop") diff --git a/tools/data_preprocess/offline_packing/s2_prepare_rawsamples-vqa_5500k-16k-fast.py b/tools/data_preprocess/offline_packing/s2_prepare_rawsamples-vqa_5500k-16k-fast.py deleted file mode 100644 index 0f115a6c..00000000 --- a/tools/data_preprocess/offline_packing/s2_prepare_rawsamples-vqa_5500k-16k-fast.py +++ /dev/null @@ -1,489 +0,0 @@ -# ### 所有代码放到一起，只运行这一块就可以 -# Step1: -# python -u s2_prepare_rawsamples-emova.py 2>&1 | tee s2_proc.log -# python -u s2_prepare_rawsamples-llava_vqa.py 2>&1 | tee s2_proc_llava.log -# python -u s2_prepare_rawsamples-vqa_1000k.py 2>&1 | tee ./logs/s2_proc_vqa_1000k.log -# python -u s2_prepare_rawsamples-vqa_1000k-16k.py 2>&1 | tee ./logs/s2_proc_vqa_1000k-16k.log -# python -u s2_prepare_rawsamples-vqa_5500k-16k.py 2>&1 | tee ./logs/s2_proc_vqa_5500k-16k.log -# python -u s2_prepare_rawsamples-vqa_5500k-16k-fast.py 2>&1 | tee ./logs/s2_proc_vqa_5500k-16k-fast.log - -import bisect -import os -import json -import sys -import shutil -from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union -import threading -from concurrent.futures import ThreadPoolExecutor, as_completed - -# ### 参数配置 -# target_directory = "/workspace/test/packing" # 最终数据存放的位置 - -current_file = Path(__file__).resolve() -target_directory = current_file.parent -newDir = "raw_packing_data_vqa_5500k-16k-fast" # 转 webdataset 之前数据的存放位置 (jpg, json) -SRC_DIR_IMGS = "/data_3/aiak_pretrain_vqa_5500k" # 原始 img 数据的存放位置 -SRC_DIR_JSONS = "/data_3/aiak_pretrain_vqa_5500k" # 原始 json 数据的存放位置 -SRC_DST_EXTENSIONS = ("jpg", "json") -f_toklens_originalsample = os.path.join(target_directory, "token_info_v2_vqa_5500k_s16k.txt") -PACKED_LENGTH = 16384 -dst_dir = os.path.join(target_directory,newDir) -MAX_WORKERS = 96 # 线程池大小（根据CPU核心数和IO性能调整） - -task_type = "sft" - - - -f_TEST=False # test 示例输出（仅做测试用：生成少量数据） -n_packed_samples=100 # 测试用，输出几条打包后的数据 - -# PROMPTS = # Creating a list of the provided English prompts -PROMPTS = [ - "What about this picture?", - "Please provide a vivid description of the image.", - "Please Depict the image in words." - "Could you please transcribe thr image into a descriptive paragraph?" - "What is the content of this figure?", - "What do you see here?", - "Tell me about this image.", - "What's going on in this artwork?", - "What is depicted in this painting?", - "What is the subject matter here?", - "What can you make out in this picture?", - "What's the main thing shown in this image?", - "What's the gist of this artwork?", - "What's the essence of this figure?", - "What's the general idea here?", - "What does this image show?", - "What's the core element in this painting?", - "What's the overview of this scene?", - "What's the primary focus of this artwork?", - "What's the fundamental subject matter?", - "What's the general view presented?", - "What's the main impression given by this picture?", - "What's the central theme shown?", - "What's the overall presentation here?", - "What's the key element you notice?", - "What's the fundamental concept in this image?", - "What's the overall content?", - "What's the main thing you get from this?", - "What's the general subject?", - "What's the core idea conveyed?", - "What's the basic representation?", - "What's the main point of this figure?" -] - -import random - -def get_random_prompts(prompts, n): - if n > len(prompts): - # 允许重复 - return random.choices(prompts, k=n) - else: - # 不允许重复 - return random.sample(prompts, n) - -# 全局变量 - 用元组存储（不可变，效率更高） -BASE_NAMES = [] # 初始化为空元组，后续会被替换 (所有在原始数据集中的 sample 名称， 已经按照 tokens 长度排序) - -def search_for_fit(numbers: List[int], capacity: int) -> int: - """Finds the index of largest number that fits into the knapsack with the given capacity.""" - index = bisect.bisect(numbers, capacity) - return -1 if index == 0 else (index - 1) - -def greedy_knapsack(numbers: List[int], capacity: int) -> Tuple[List[List[int]], List[List[int]]]: - r"""Implement efficient greedy algorithm with binary search for the knapsack problem. - 参数 - ---- - numbers : List[int] - 物品大小列表，可以为任意顺序（这里是升序输入进来的） - capacity : int - 背包容量 - - 返回 - ---- - Tuple[List[List[int]], List[List[int]]] - 第一个列表：每个背包里的物品大小 - 第二个列表：每个背包里物品对应的原始下标 - - """ - # 保存原始索引，与输入的numbers一一对应 - indexed_numbers = [(val, idx) for idx, val in enumerate(numbers)] - # 由于输入已排序，直接使用即可（保持与原逻辑一致的处理方式） - knapsacks = [] - index_knapsacks = [] - iii = int(0) - while indexed_numbers: - current_knapsack = [] - current_indices = [] - remaining_capacity = capacity - - while True: - # 提取当前数值列表用于查找（保持升序） - current_values = [val for val, idx in indexed_numbers] - index = search_for_fit(current_values, remaining_capacity) - if index == -1: - break # 没有可放入当前背包的物品了 - - # 取出找到的物品及其原始索引 - val, idx = indexed_numbers.pop(index) - remaining_capacity -= val - current_knapsack.append(val) - current_indices.append(idx) - - if iii%1000==0: - print(f"---------第{iii}个pack----------") - print(f"{current_knapsack}--->{sum(current_knapsack)}") - print(current_indices) - print(f"\n") - iii+=1 - knapsacks.append(tuple(current_knapsack)) - index_knapsacks.append(tuple(current_indices)) - - return tuple(knapsacks), tuple(index_knapsacks) # 使用了 tuple - -def extract_content(json_file): - try: - # 打开并加载JSON文件 - with open(json_file, 'r', encoding='utf-8') as f: - data = json.load(f) - - if task_type=="sft": - try: - user_content = next(msg["content"] for msg in data["messages"] if msg["role"] == "assistant") - return user_content - except Exception as e: - pass - # 提取content内容 - # 假设captions数组至少有一个元素 - elif task_type=="pretrain": - if data.get('captions') and len(data['captions']) > 0: - return data['captions'][0].get('content', "") - else: - assert 0, "未找到有效的caption内容" - # return "未找到有效的caption内容" - - except FileNotFoundError: - return f"错误：文件 {json_file} 不存在" - except json.JSONDecodeError: - return f"错误：文件 {json_file} 不是有效的JSON格式" - except Exception as e: - return f"提取过程中发生错误：{str(e)}" - -def extract_prompt(json_file): - try: - # 打开并加载JSON文件 - with open(json_file, 'r', encoding='utf-8') as f: - data = json.load(f) - - # 提取助手回复 - assistant_content = next(msg["content"] for msg in data["messages"] if msg["role"] == "user") - return assistant_content - - # # 提取图片路径（可选） - # image_path = data["images"][0] if data["images"] else None - - except FileNotFoundError: - return f"错误：文件 {json_file} 不存在" - except json.JSONDecodeError: - return f"错误：文件 {json_file} 不是有效的JSON格式" - except Exception as e: - return f"提取过程中发生错误：{str(e)}" - - -def prepare_dirs(target_dir, new_dir): - os.chdir(target_dir) - print(f"--------change to directory {target_dir}--------") - # 创建新目录 - if not os.path.exists(new_dir): - os.makedirs(new_dir) - print(f"Directory '{new_dir}' created.") - else: - print(f"Directory '{new_dir}' already exists.") - - -def dataset_tokinfo_generator(f_name): - """ - 数据集token信息生成器，逐行读取并解析文件内容 - - 参数: - f_name (str): 包含token信息的文件路径 - - 生成: - tuple: (base_name, token_len) - 解析后的基础文件名和token长度 - """ - try: - with open(f_name, 'r', encoding='utf-8') as f: - for line in f: - # 跳过空行 - stripped_line = line.strip() - if not stripped_line: - continue - - # 按冒号分割并验证格式 - parts = stripped_line.split(':') - if len(parts) == 2: - base_name = parts[0].strip() - token_len_str = parts[1].strip() - - try: - token_len = int(token_len_str) - yield (base_name, token_len) - except ValueError: - print( - f"警告: 无法将 '{token_len_str}' 转换为整数，已跳过该行", - file=sys.stderr - ) - continue - - except FileNotFoundError: - print(f"错误: 文件 '{f_name}' 不存在", file=sys.stderr) - return - except Exception as e: - print(f"处理文件时发生错误: {str(e)}", file=sys.stderr) - return - - -class TokenInfoReader: - """ - Token信息读取器 - - 支持分批读取、全量读取和断点续读功能，适用于处理包含token信息的文本文件。 - 文件格式要求: 每行一条记录，格式为 "base_name: token_len" - """ - - def __init__(self, f_name): - """ - 初始化读取器 - - 参数: - f_name (str): 包含token信息的文件路径 - """ - self.f_name = f_name - self.generator = dataset_tokinfo_generator(f_name) - self._current_position = 0 # 记录已读取的记录数量 - - def read(self, count=None): - """ - 读取记录 - - 参数: - count (int, optional): 要读取的记录数量，默认为None（读取全部剩余记录） - - 返回: - tuple: (base_names列表, token_lens列表, 实际读取数量) - """ - base_names = [] - token_lens = [] - read_count = 0 - - # 循环读取直到达到指定数量或文件结束 - while True: - # 检查是否已达到指定读取数量 - if count is not None and read_count >= count: - break - - try: - # 从生成器获取下一条记录 - base_name, token_len = next(self.generator) - base_names.append(base_name) - token_lens.append(token_len) - read_count += 1 - self._current_position += 1 - - except StopIteration: - # 已读取到文件末尾 - break - - return base_names, token_lens, read_count - - def get_current_position(self): - """ - 获取当前读取位置 - - 返回: - int: 已读取的记录总数 - """ - return self._current_position - - -def process_knapsack(s1, idx_knapsack, dst_dir): - """ - 处理单个 packing 数据 - - 参数: - s1: 当前处理组的索引 - idx_knapsack: 背包中包含的索引列表 - dst_dir: 目标目录路径 - """ - # global BASE_NAMES - - packed_imgs, packed_caps = [], [] # 单个 packed-sample 的构成 - - # 获取基础文件名 - # packed_b_names = (BASE_NAMES[idx] for idx in idx_knapsack) - packed_b_names = (idx["name"] for idx in idx_knapsack) - - # 构建源文件信息 - if task_type == "pretrain": - packed_info = ( - (os.path.join(SRC_DIR_IMGS, f"{b_name}.{SRC_DST_EXTENSIONS[0]}"), - extract_content(os.path.join(SRC_DIR_JSONS, f"{b_name}.{SRC_DST_EXTENSIONS[1]}"))) - for b_name in packed_b_names - ) - elif task_type == "sft": - packed_info = ( - (os.path.join(SRC_DIR_IMGS, f"{b_name}.{SRC_DST_EXTENSIONS[0]}"), - extract_content(os.path.join(SRC_DIR_JSONS, f"{b_name}.{SRC_DST_EXTENSIONS[1]}")), - extract_prompt(os.path.join(SRC_DIR_JSONS, f"{b_name}.{SRC_DST_EXTENSIONS[1]}"))) - for b_name in packed_b_names - ) - - # 目标JSON文件路径 - json_dst = os.path.join(dst_dir, f"ps_{s1:08d}.{SRC_DST_EXTENSIONS[1]}") - - # 处理每张图片和对应的描述 - if task_type=="pretrain": - for s2, (img_src, cap_src) in enumerate(packed_info): - # 目标图片路径 - img_name_dst = f"ps_{s1:08d}.img{s2:03d}.{SRC_DST_EXTENSIONS[0]}" - # img_name_dst = f"img{s2:03d}.{SRC_DST_EXTENSIONS[0]}" # 看后面具体需求决定使用哪一个 - img_dst = os.path.join(dst_dir, img_name_dst) - - # 收集信息 - # packed_imgs.append(img_name_dst) - packed_imgs.append(f"img{s2:03d}.{SRC_DST_EXTENSIONS[0]}") - packed_caps.append(cap_src) - - # 复制图片 - shutil.copyfile(img_src, img_dst) - # 此处也可以调用大模型来生成 提问(对于 纯 captioning 数据) - selected_prompts = get_random_prompts(PROMPTS, len(packed_imgs)) - elif task_type=="sft": - selected_prompts = [] - for s2, (img_src, cap_src, prompt_src) in enumerate(packed_info): - # 目标图片路径 - img_name_dst = f"ps_{s1:08d}.img{s2:03d}.{SRC_DST_EXTENSIONS[0]}" - # img_name_dst = f"img{s2:03d}.{SRC_DST_EXTENSIONS[0]}" # 看后面具体需求决定使用哪一个 - img_dst = os.path.join(dst_dir, img_name_dst) - - # 收集信息 - # packed_imgs.append(img_name_dst) - packed_imgs.append(f"img{s2:03d}.{SRC_DST_EXTENSIONS[0]}") - packed_caps.append(cap_src) - - # 复制图片 - shutil.copyfile(img_src, img_dst) - - # prompts - selected_prompts.append(prompt_src) - pass - # 生成JSON文件 - json_data = { - "images": packed_imgs, - "captions": packed_caps, - "prompts": selected_prompts - } - # print(packed_imgs) - - try: - with open(json_dst, 'w', encoding='utf-8') as f: - json.dump(json_data, f, indent=4, ensure_ascii=False) - # json.dump(json_data, f) - except Exception as e: - print(f"线程 {threading.current_thread().name} 生成JSON文件 {json_dst} 失败: {str(e)}") - return s1 - - -if __name__ == "__main__": - ## 1. 创建工作目录 - print("Step1-----------------已创建工作环境-----------------Start") - prepare_dirs(target_directory, newDir) - print("Step1-----------------已创建工作环境-----------------Stop\n\n") - - ## 2. 获取原始数据集信息（没有处理之前） - # 可以用于构建多个 pool，分块 packing（read的参数决定 packing cache size） - print("Step2-----------------读取原ds的 tokenlen 信息-----------------Start") - info_reader = TokenInfoReader(f_toklens_originalsample) - base_names, token_lens, n_count = info_reader.read() - - # global BASE_NAMES - BASE_NAMES=tuple(base_names) - print(f"已读取{n_count}条数据") - # print(BASE_NAMES) - print("Step2-----------------读取原ds的 tokenlen 信息-----------------Stop\n\n") - - # 3. packing分组 - #调用 packing-group 进行分组 - print("Step3-----------------packing 分组-----------------Start") - # knapsacks, idx_knapsacks= greedy_knapsack(token_lens, PACKED_LENGTH) - # print(idx_knapsacks[10]) - # print(knapsacks[10]) - import pickle - def load_bin_boxes(file_path: str): - """ - 加载单步装箱结果 - """ - with open(file_path, 'rb') as f: - bin_boxes = pickle.load(f) - print(f"已加载装箱结果: {file_path}") - return bin_boxes - - bin_boxs = load_bin_boxes("./s2_ckpt/bins_boxs.pkl") - - # total_knapsacks = len(idx_knapsacks) - total_knapsacks = len(bin_boxs) - - print(f"原始数据----{n_count}----条，packing后变为----{total_knapsacks}----条") - print("Step3-----------------packing 分组-----------------Stop\n\n") - - print("Step4----------------- 开始构建新数据集 -----------------Start") - print(f"开始处理 {total_knapsacks} 组数据，使用 {MAX_WORKERS} 个线程") - - # 4. 使用线程池处理所有pack - with ThreadPoolExecutor(max_workers=MAX_WORKERS, thread_name_prefix="PackThread") as executor: - # 提交所有任务 - if f_TEST: - futures = { - executor.submit(process_knapsack, s1, idx_knapsack, dst_dir): s1 - for s1, idx_knapsack in enumerate(bin_boxs[0:n_packed_samples]) - } - else: - futures = { - executor.submit(process_knapsack, s1, idx_knapsack, dst_dir): s1 - for s1, idx_knapsack in enumerate(bin_boxs) - } - - # tqdm 自动跟踪完成数 - from tqdm import tqdm - tty = open(os.devnull, 'w') if os.name == 'nt' else open('/dev/tty', 'w') - for future in tqdm(as_completed(futures), - total=len(futures), - desc="Packing progress", - unit="pack", - file=tty - ): - try: - future.result() - except Exception as e: - s1 = futures[future] - print(f"\n处理第 {s1} 组数据时发生错误: {e}") - - - # # 跟踪进度 - # completed = 0 - # for future in as_completed(futures): - # s1 = futures[future] - # try: - # result = future.result() - # completed += 1 - # # 每完成1%输出一次进度 - # if completed % (max(1, total_knapsacks // 10)) == 0: - # print(f"已完成 {completed}/{total_knapsacks} 组数据 ({completed/total_knapsacks*100:.1f}%)") - # except Exception as e: - # print(f"处理第 {s1} 组数据时发生错误: {str(e)}") - - # print(f"所有数据处理完成，共处理 {completed} 组") - - print("Step4-----------------Sccessful！！！！---- 构建新数据集成功 -----------------Stop") diff --git a/tools/data_preprocess/offline_packing/s3_test_emova.sh b/tools/data_preprocess/offline_packing/s3_test_emova.sh deleted file mode 100644 index 071edec9..00000000 --- a/tools/data_preprocess/offline_packing/s3_test_emova.sh +++ /dev/null @@ -1 +0,0 @@ -python -u ./convert_packedsample_to_wds.py --output_dir /mnt/cluster/emova_wds_3000tk --json_file /mnt/cluster/raw_packing_data_emova_3000tk --video_dir /mnt/cluster/raw_packing_data_emova_3000tk --image_dir /mnt/cluster/raw_packing_data_emova_3000tk --mode caption_pack --maxcount 5000 2>&1 | tee s3_proc.log diff --git a/tools/data_preprocess/preprocess_pretrain_data.py b/tools/data_preprocess/preprocess_pretrain_data.py deleted file mode 100644 index 0879a7fc..00000000 --- a/tools/data_preprocess/preprocess_pretrain_data.py +++ /dev/null @@ -1,444 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. - -"""Processing large data for pretraining.""" -import argparse -import math -import json -import os -import sys -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir))) -import time -import gzip -import glob -import torch -import numpy as np -import multiprocessing -try: - import nltk - nltk_available = True -except ImportError: - nltk_available = False - -from aiak_training_llm.tokenizer import build_tokenizer -from megatron.core.datasets import indexed_dataset -from aiak_training_llm.utils import constants - - -# https://stackoverflow.com/questions/33139531/preserve-empty-lines-with-nltks-punkt-tokenizer -class CustomLanguageVars(nltk.tokenize.punkt.PunktLanguageVars): - - _period_context_fmt = r""" - \S* # some word material - %(SentEndChars)s # a potential sentence ending - \s* # <-- THIS is what I changed - (?=(?P - %(NonWord)s # either other punctuation - | - (?P\S+) # <-- Normally you would have \s+ here - ))""" - -class IdentitySplitter(object): - def tokenize(self, *text): - return text - - -class Encoder(object): - def __init__(self, args): - self.args = args - - def initializer(self): - # Use Encoder class as a container for global data - Encoder.tokenizer = build_tokenizer(self.args) - if self.args.split_sentences: - if not nltk_available: - print("NLTK is not available to split sentences.") - exit() - if os.environ.get("NLTK_DATA"): - library = os.path.join(os.environ.get("NLTK_DATA"), "tokenizers", "punkt", f"{self.args.lang}.pickle") - url = f"file:{library}" - else: - library = os.path.join("tokenizers", "punkt", f"{self.args.lang}.pickle") - url = f"nltk:{library}" - splitter = nltk.load(url) - if self.args.keep_newlines: - # this prevents punkt from eating newlines after sentences - Encoder.splitter = nltk.tokenize.punkt.PunktSentenceTokenizer( - train_text = splitter._params, - lang_vars = CustomLanguageVars()) - else: - Encoder.splitter = splitter - - else: - Encoder.splitter = IdentitySplitter() - - def split(self, json_line): - data = json.loads(json_line) - output = {} - for key in self.args.json_keys: - text = data[key] - max_len = 1000000 - tokens_list = [Encoder.splitter.tokenize(text[i:i+max_len]) for i in range(0, len(text), max_len)] - output[key] = [tokens for partial in tokens_list for tokens in partial] - return json.dumps(output), len(json_line) - - def encode(self, json_line): - data = json.loads(json_line) - ids = {} - lens = {} - for key in self.args.json_keys: - text = data[key] - if isinstance(text, list): - sentences = text - else: - sentences = [text] - doc_ids = [] - sentence_lens = [] - for sentence in sentences: - sentence_ids = Encoder.tokenizer.tokenize(sentence) - if len(sentence_ids) > 0: - doc_ids.extend(sentence_ids) - sentence_lens.append(len(sentence_ids)) - if len(doc_ids) > 0 and self.args.append_eod: - doc_ids.append(Encoder.tokenizer.eod) - sentence_lens[-1] += 1 - ids[key] = doc_ids - lens[key] = sentence_lens - return ids, lens, len(json_line) - - -class Partition(object): - def __init__(self, args, workers): - self.args = args - self.workers = workers - - def print_processing_stats(self, count, proc_start, total_bytes_processed): - if count % self.args.log_interval == 0: - current = time.time() - elapsed = current - proc_start - mbs = total_bytes_processed/elapsed/1024/1024 - print(f"Processed {count} documents", - f"({count/elapsed} docs/s, {mbs} MB/s).", - file=sys.stderr) - - def split_sentences(self, file_name): - input_file_name, output_file_name = file_name - print("Opening", input_file_name) - fin = open(input_file_name, 'r', encoding='utf-8') - fout = open(output_file_name, 'w') - - encoder = Encoder(self.args) - pool = multiprocessing.Pool(self.workers, initializer=encoder.initializer) - split_docs = pool.imap(encoder.split, fin, 32) - - proc_start = time.time() - total_bytes_processed = 0 - for i, (doc, bytes_processed) in enumerate(split_docs, start=1): - total_bytes_processed += bytes_processed - fout.write(doc + "\n") - self.print_processing_stats(i, proc_start, total_bytes_processed) - - fin.close() - fout.close() - - - def process_json_file(self, file_name): - input_file_name, output_prefix = file_name - print("Opening", input_file_name) - fin = open(input_file_name, 'r', encoding='utf-8') - - startup_start = time.time() - encoder = Encoder(self.args) - tokenizer = build_tokenizer(self.args) - pool = multiprocessing.Pool(self.workers, initializer=encoder.initializer) - encoded_docs = pool.imap(encoder.encode, fin, 32) - - level = "document" - if self.args.split_sentences: - level = "sentence" - - output_bin_files = {} - output_idx_files = {} - builders = {} - - for key in self.args.json_keys: - output_bin_files[key] = "{}_{}_{}.bin".format(output_prefix, - key, level) - output_idx_files[key] = "{}_{}_{}.idx".format(output_prefix, - key, level) - builders[key] = indexed_dataset.IndexedDatasetBuilder( - output_bin_files[key], - dtype=indexed_dataset.DType.optimal_dtype(tokenizer.vocab_size), - ) - - startup_end = time.time() - proc_start = time.time() - total_bytes_processed = 0 - total_sentence_lens = {key: 0 for key in self.args.json_keys} - - print("Time to startup:", startup_end - startup_start) - for i, (doc, sentence_lens, bytes_processed) in enumerate(encoded_docs, start=1): - total_bytes_processed += bytes_processed - for key in doc.keys(): - builders[key].add_document(doc[key], sentence_lens[key]) - total_sentence_lens[key] += sum(sentence_lens[key]) - - self.print_processing_stats(i, proc_start, total_bytes_processed) - - fin.close() - builders[key].finalize(output_idx_files[key]) - print(f"{input_file_name} total tokens: {total_sentence_lens}") - - -def get_args(): - parser = argparse.ArgumentParser() - group = parser.add_argument_group(title='input data') - group.add_argument('--input', type=str, required=True, - help='Path to input JSON') - group.add_argument('--json-keys', nargs='+', default=['text'], - help='space separate listed of keys to extract from json') - group.add_argument('--split-sentences', action='store_true', - help='Split documents into sentences.') - group.add_argument('--keep-newlines', action='store_true', - help='Keep newlines between sentences when splitting.') - - group = parser.add_argument_group(title='tokenizer') - group.add_argument('--model-family', type=str, - choices=["llama", "llama2", "llama3", "llama3.1", - "baichuan", "baichuan2", - "qwen", "qwen1.5", "qwen2", - "mixtral"], - help='model family name. note: for the qwen model, this parameter needs to be passed, ' - 'otherwise an error will be reported') - group.add_argument('--tokenizer-type', type=str, required=True, - choices=[ - 'NullTokenizer', - 'Llama2Tokenizer', # megatron verison - 'HFTokenizer', # recommended - ], - help='What type of tokenizer to use.') - group.add_argument('--tokenizer-model', type=str, default=None, - help='YTTM tokenizer model.') - group.add_argument('--vocab-file', type=str, default=None, - help='Path to the vocab file') - group.add_argument('--vocab-size', default=786, - help='size of vocab for use with NullTokenizer') - group.add_argument('--merge-file', type=str, default=None, - help='Path to the BPE merge file (if necessary).') - group.add_argument('--append-eod', action='store_true', - help='Append an token to the end of a document.') - group.add_argument('--lang', type=str, default='english', - help='Language to use for NLTK-powered sentence splitting.') - - group.add_argument('--seq-length', type=int, default=None, help='sequence length') - group.add_argument('--hf-tokenizer-path', type=str, default=None, - help='HuggingFace tokenizer path: ' - '1) A string, the *model id* of a predefined tokenizer hosted inside a model repo ' - 'on huggingface.co' - '2) A path to a *directory* containing vocabulary files required by the tokenizer') - group.add_argument('--use-fast-tokenizer', action='store_true', - help='Whether to use the fast tokenizer when --tokenizer-type=HFTokenizer. Default: False') - group.add_argument('--split-special-tokens', action='store_true', - help="Whether the special tokens should be split during the tokenization process " - "when --tokenizer-type=HFTokenizer. Default: False") - group.add_argument("--additional-special-tokens", - type=str, - default=None, - help="Additional special tokens to add to the tokenizer. Use commas to separate multiple tokens") - - group = parser.add_argument_group(title='output data') - group.add_argument('--output-prefix', type=str, required=True, - help='Path to binary output file without suffix') - - group = parser.add_argument_group(title='runtime') - group.add_argument('--workers', type=int, required=True, - help=('Number of worker processes to launch.' - 'A good default for fast pre-processing ' - 'is: (workers * partitions) = available CPU cores.')) - group.add_argument('--partitions', type=int, default=1, - help='Number of file partitions') - group.add_argument('--log-interval', type=int, default=1000, - help='Interval between progress updates') - group.add_argument('--keep-sequential-samples', action='store_true', - help='Ensure ordering of samples in .jsonl files is ' - 'preserved when using partitions>1.') - args = parser.parse_args() - args.keep_empty = False - - if args.tokenizer_type.lower().startswith('bert') and not args.split_sentences: - print("Are you sure you don't want to split sentences?") - - # some default/dummy values for the tokenizer - args.rank = 1 - args.make_vocab_size_divisible_by = 128 - args.tensor_model_parallel_size = 1 - args.vocab_extra_ids = 0 - args.training_phase=constants.TrainingPhase.PRETRAIN - args.padding_side = "right" - - return args - - -def get_file_name(args, file_id): - file_name, extension = os.path.splitext(args.input) - input_file_name = file_name + "_" + str(file_id) + extension - sentence_split_file = file_name + "_ss_" + str(file_id) + extension - output_prefix = args.output_prefix + "_" + str(file_id) - file_names = { - 'partition': input_file_name, - 'sentence_split': sentence_split_file, - 'output_prefix': output_prefix} - return file_names - - -def check_files_exist(in_ss_out_names, key, num_partitions): - for i in range(num_partitions): - if not os.path.exists(in_ss_out_names[i][key]): - return False - return True - - -def main(): - args = get_args() - - if args.split_sentences: - if nltk_available: - nltk.download("punkt", quiet=True, download_dir=os.environ.get("NLTK_DATA")) - else: - raise Exception( - "nltk library required for sentence splitting is not available.") - - in_ss_out_names = [] - if args.partitions == 1: - file_name, extension = os.path.splitext(args.input) - sentence_split_file = file_name + "_ss" + extension - file_names = { - 'partition': args.input, - 'sentence_split': sentence_split_file, - 'output_prefix': args.output_prefix} - in_ss_out_names.append(file_names) - else: - in_file_names = glob.glob(args.input) - - # Count total number of lines across .jsonl files - if args.keep_sequential_samples: - total_sample_count = 0 - for filename in in_file_names: - with open(filename, "r") as fin: - for fc, _ in enumerate(fin): - pass - total_sample_count += (fc + 1) - partition_size = math.ceil(total_sample_count / args.partitions) - - # create .jsonl parition files - for idx in range(args.partitions): - in_ss_out_name = get_file_name(args, idx) - in_ss_out_names.append(in_ss_out_name) - - # check to see if paritions were already created - partitions_present = check_files_exist(in_ss_out_names, 'partition', args.partitions) - - # check to see if paritions with split sentences already created - split_sentences_present = check_files_exist(in_ss_out_names, 'sentence_split', args.partitions) - - if not partitions_present and not split_sentences_present: - # populate .jsonl partition files from parent files - partitioned_input_files = [] - for idx in range(args.partitions): - partitioned_input_file = open(in_ss_out_names[idx]['partition'], 'w') - partitioned_input_files.append(partitioned_input_file) - - index = 0 - if args.keep_sequential_samples: line_count = 0 - for in_file_name in in_file_names: - # support for gzip files - if in_file_name.endswith(".gz"): - fin = gzip.open(in_file_name, 'rt') - else: - fin = open(in_file_name, 'r', encoding='utf-8') - - for line in fin: - partitioned_input_files[index].write(line) - if args.keep_sequential_samples: - line_count += 1 - if line_count % partition_size == 0: - index += 1 - else: - index = (index + 1)%args.partitions - - fin.close() - - for idx in range(args.partitions): - partitioned_input_files[idx].close() - - assert args.workers % args.partitions == 0 - partition = Partition(args, args.workers//args.partitions) - - # check to see if paritions with split sentences already created - split_sentences_present = check_files_exist(in_ss_out_names, 'sentence_split', args.partitions) - - # split sentences in partition files - if args.split_sentences and not split_sentences_present: - processes = [] - for name in in_ss_out_names: - p = multiprocessing.Process(target=partition.split_sentences, - args=((name['partition'], name['sentence_split']),)) - p.start() - processes.append(p) - - for p in processes: - p.join() - - if args.partitions == 1: - return - - - # encode partition files in parallel - processes = [] - input_key = 'sentence_split' if args.split_sentences else 'partition' - for name in in_ss_out_names: - p = multiprocessing.Process(target=partition.process_json_file, - args=((name[input_key], name['output_prefix']),)) - p.start() - processes.append(p) - - for p in processes: - p.join() # 等待子进程结束 - if p.exitcode != 0: # 如果子进程的退出码不为 0 - sys.exit(1) # 让主进程以非零码退出 - - if args.partitions == 1: - return - - # merge bin/idx partitions - level = "document" - if args.split_sentences: - level = "sentence" - - output_bin_files = {} - output_idx_files = {} - builders = {} - tokenizer = build_tokenizer(args) - - for key in args.json_keys: - output_bin_files[key] = "{}_{}_{}.bin".format(args.output_prefix, - key, level) - output_idx_files[key] = "{}_{}_{}.idx".format(args.output_prefix, - key, level) - builders[key] = indexed_dataset.IndexedDatasetBuilder( - output_bin_files[key], - dtype=indexed_dataset.DType.optimal_dtype(tokenizer.vocab_size), - ) - - for name in in_ss_out_names: - parition_output_prefix = name['output_prefix'] - full_partition_output_prefix = "{}_{}_{}".format(parition_output_prefix, - key, level) - builders[key].add_index(full_partition_output_prefix) - builders[key].finalize(output_idx_files[key]) - - -if __name__ == '__main__': - - main() - diff --git a/tools/data_preprocess/preprocess_sft_data.py b/tools/data_preprocess/preprocess_sft_data.py deleted file mode 100644 index f4fcc2c3..00000000 --- a/tools/data_preprocess/preprocess_sft_data.py +++ /dev/null @@ -1,200 +0,0 @@ -"""preprocess sft data""" - -import argparse - -from transformers import AutoProcessor -from datasets import DatasetDict - -from megatron.core.datasets.utils import get_blend_from_list, Split - -from aiak_training_llm.data.sft_dataset import SFTDatasetConfig, SFTDataset -from aiak_training_llm.data import ChatTemplate -from aiak_training_llm.tokenizer import build_tokenizer -from aiak_training_llm.utils import constants -from aiak_training_llm.utils.utils import get_default_sft_dataset_config -from aiak_training_llm.train.sft.utils import get_dataset_blend_from_list - - -def build_sft_dataset(args): - """build sft dataset""" - tokenizer = build_tokenizer(args, chat_template=args.template) - processor = AutoProcessor.from_pretrained(args.hf_tokenizer_path, trust_remote_code=True) - if args.image_resolution: - setattr(processor, "image_resolution", args.image_resolution) - - config = SFTDatasetConfig( - random_seed=args.seed, - sequence_length=args.seq_length, # max sequence length - enable_discard_sample=args.enable_discard_sample, - blend=get_blend_from_list([args.input]), - split=args.split, - tokenizer=tokenizer, - dataset=get_dataset_blend_from_list([args.sft_dataset]), - dataset_config_file=args.sft_dataset_config, - streaming=False, - chat_template=args.template, - processor=processor, - num_preprocess_workers=args.workers, - train_on_prompt=args.train_on_prompt, - ignore_index=constants.IGNORE_INDEX, - eod_mask_loss=args.eod_mask_loss, - path_to_cache=None, - is_tokenized=False, - packing=args.packing_sft_data, - sort_batch=args.sft_sort_batch, - packing_batch_size=args.packing_batch_size, - context_parallel_size=args.context_parallel_size, - ) - - dataset = SFTDataset(args.sft_dataset, args.input, config) - split_dataset = dataset.split(config.split_matrix) - - num_samples = 0 - - dataset_dict = DatasetDict() - for i, split in enumerate(Split): - if split_dataset[i] is not None: - dataset_dict[split.name] = split_dataset[i] - num_samples += len(split_dataset[i]) - - dataset_dict.save_to_disk(args.output_path) - - print(f">>> Saved preprocessed dataset to {args.output_path} with total samples: {num_samples}") - for i, split in enumerate(Split): - print(f">>> {split.name} samples: {len(dataset_dict[split.name]) if split.name in dataset_dict else 0 }") - - print(f"NOTE: Please run sft with `--data-path {args.output_path}` and `--is-tokenized-data`") - - -def _add_arguments(parser: argparse.ArgumentParser): - """Add arguments""" - group = parser.add_argument_group(title='input data') - group.add_argument('--input', type=str, required=True, help='Path to input JSON') - - group.add_argument('--seq-length', type=int, default=None, help='max sequence length') - - group.add_argument("--enable-discard-sample", - action='store_true', - help="Whether to discard sample when its length is greater than seq-length.") - - group.add_argument('--sft-dataset-config', type=str, default=None, - help="A json file that contains the dataset configuration." - "default: configs/dataset_config.jsoin") - - group.add_argument('--sft-dataset', type=str, default="default", - help='the dataset name should be defined in the dataset config file (--sft-dataset-config)') - - group.add_argument('--output-path', type=str, required=True, - help='Output directory where the processed dataset will be saved') - - group.add_argument("--packing-sft-data", - action='store_true', - help="Whether to pack multiple sft data into one.") - group.add_argument("--packing-batch-size", - type=int, - default=10000, - help="Perform packing in batches, deciding how many samples each batch contains;" - "if the --sft-sort-batch option is enabled, the samples will be sorted after packing.") - group.add_argument('--sft-sort-batch', - action='store_true', - help='Sort the entire dataset from smallest to largest; ' - 'if the --packing-sft-data option is enabled, sort the data after packing. Default: False') - - group.add_argument("--context-parallel-size", - type=int, default=None, - help="If packing is enabled, and context-parallel is enabled during the training phase, " - "it is necessary to set the corresponding context_parallel_size " - "to correctly pad the data.") - - group.add_argument('--split', type=str, default="100,0,0", - help='Comma-separated list of proportions for training,' - ' validation, and test split. For example the split ' - '`90,5,5` will use 90%% of data for training, 5%% for ' - 'validation and 5%% for test.') - - group = parser.add_argument_group(title='model&tokenizer') - group.add_argument('--chat-template', type=str, required=True, - choices=["llama2", "llama2_zh", "llama3", "llama3.1", - "baichuan", "baichuan2", - "qwen", - "mistral", - "qwen2-vl", - "alpaca", - "deepseek", "deepseek3"], - help='The template to apply to instruction data.') - - group.add_argument('--tokenizer-type', type=str, default='HFTokenizer', - choices=['HFTokenizer'], - help='What type of tokenizer to use.') - - group.add_argument('--hf-tokenizer-path', type=str, required=True, - help='HuggingFace tokenizer path: ' - '1) A string, the *model id* of a predefined tokenizer hosted inside a model repo ' - 'on huggingface.co' - '2) A path to a *directory* containing vocabulary files required by the tokenizer') - - group.add_argument('--image-resolution', type=int, help='Resolution of image inputs') - - group.add_argument('--use-fast-tokenizer', action='store_true', - help='Whether to use the fast tokenizer when --tokenizer-type=HFTokenizer. Default: False') - - group.add_argument('--split-special-tokens', action='store_true', - help="Whether the special tokens should be split during the tokenization process " - "when --tokenizer-type=HFTokenizer. Default: False") - - group.add_argument("--additional-special-tokens", - type=str, - default=None, - help="Additional special tokens to add to the tokenizer. Use commas to separate multiple tokens") - - group.add_argument('--train-on-prompt', action='store_true', - help='Whether compute loss on prompt. Default: False') - - group.add_argument('--eod-mask-loss', action='store_true', - help='Mask loss for the end of document tokens.') - - group = parser.add_argument_group(title='preprocess-runtime') - group.add_argument('--workers', type=int, required=True, help='Number of worker processes to launch.') - return parser - - -def parse_args(): - """arguments""" - parser = argparse.ArgumentParser() - _add_arguments(parser) - - args = parser.parse_args() - - # some default/dummy values - args.rank = 0 - args.training_phase = constants.TrainingPhase.SFT - args.make_vocab_size_divisible_by = 128 - args.tensor_model_parallel_size = 1 - args.vocab_extra_ids = 0 - args.seed = 42 - args.padding_side = "right" - - if args.sft_dataset_config is None: - args.sft_dataset_config = get_default_sft_dataset_config() - assert args.sft_dataset_config is not None, "No default sft dataset config found, please specify one" - - assert args.chat_template is not None, "chat_template not specified" - template = ChatTemplate.from_name(args.chat_template) - assert template is not None, f"chat_template {args.chat_template} not supported." - args.template = template - - args.variable_seq_lengths = True - if args.packing_sft_data: - print(f"Enable to pack multiple sft data with max length {args.seq_length} ...") - - return args - - -def main(): - """main function""" - args = parse_args() - build_sft_dataset(args) - - -if __name__ == '__main__': - main() \ No newline at end of file From 4ef568031b455dcd30516adec7f82c4ad4792d00 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 1 May 2026 17:37:50 +0400 Subject: [PATCH 27/39] Add mobile-LLM integration: FastViT encoder + MobileLLM-R1-140M backbone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace upstream vision encoder and LLM with mobile-optimized components: Vision encoder - Swap RICE ViT-Large (560px) for Apple FastViT MobileCLIP-L (1024px) - Output: [B, 256 tokens, 3072 dim] per image - Loaded from ml-fastvlm pretrained weights (mobileclip_l_1024) Language model - Swap Qwen3-4B for Facebook MobileLLM-R1-140M (140M params) - 15 layers, hidden=576, 9 attn heads / 3 KV heads (GQA), FFN=2048 - QK LayerNorm enabled (use_qk_norm=true from HF config) - Vocab: 128256, context: 32k, Llama3 tokenizer Adapter - 2-layer MLP: Linear(3072,3072)+GELU → Linear(3072,576) - Only trainable component in Stage 1 (~3.5M params) Key bug fixes - llavaov_1_5_layer_spec.py: was hard-coding q_layernorm=IdentityOp, ignoring config.qk_layernorm=True — fixed by importing the correct implementation from mobilellm/mobilellm_layer_spec.py - dot_product_attention.py: GQA shape fix for non-square head counts - apex/csrc: PyTorch 2.x API patches (.scalar_type(), .options()) - transformer_engine.py: ROCm/AMD compatibility (skip CUDA arch checks) Pipeline logging - Clean 6-step per-iteration output: BATCH → VISION ENCODER → ADAPTER → TOKEN FUSION → LANGUAGE MODEL → LOSS/LOGITS - Per-step top-1 token accuracy via no-grad logit pass Training artifacts - Stage 1 alignment: 500 steps, GBS=4, LR=1e-4, BF16 - Loss: 11.78 (step 1) → 11.30 (step 500) - All 4 run logs included (20-step, 20-step, 100-step, 500-step runs) - Checkpoint weights on server at: stage_1_alignment_mobilellm_140m/iter_0000500/ (not in git, 658 MB) Documentation - Full README: architecture diagram, pipeline trace, file map, install guide, dataset download, training instructions, demo usage, modifications vs upstream, credits and citations --- README.md | 889 +- .../core/extensions/transformer_engine.py | 7 +- .../core/extensions/transformer_engine2.py | 3 +- .../megatron/core/transformer/attention.py | 5 - .../core/transformer/dot_product_attention.py | 6 - .../data/multimodal/qwen2vl_task_encoder.py | 70 +- .../data/multimodal/task_encoder.py | 13 +- .../models/fastvit/fastvit_vision_model.py | 2 - .../models/fastvit/mobileclip/__init__.py | 1 - .../models/fastvit/mobileclip/mci.py | 10 +- .../models/fastvit/mobileclip_encoder.py | 11 +- .../models/llavaov_1_5/adapter.py | 74 - .../llavaov_1_5/llavaov_1_5_layer_spec.py | 90 +- .../llavaov_1_5/llavaov_1_5_layer_spec_org.py | 177 - .../models/llavaov_1_5/llavaov_1_5_model.py | 150 +- .../llavaov_1_5/llavaov_mobilellm_config.py | 193 - .../models/mobilellm/mobilellm_config.py | 9 +- .../models/mobilellm/mobilellm_model.py | 32 +- .../train/pretrain/pretrain_llavaov_1_5.py | 192 +- aiak_training_llm/train/training_utils.py | 7 +- apex/csrc/fused_dense.cpp | 50 +- apex/csrc/mlp.cpp | 14 +- .../stage_1_alignment_mobilellm_140m.sh | 17 +- inference_fastvlm.py | 462 + stage_1_alignment_mobilellm_140m/.gitignore | 9 + .../latest_checkpointed_iteration.txt | 1 + .../latest_wandb_artifact_path.txt | 1 + .../launch_100_gpu4.out | 0 .../launch_500_wandb_gpu4.sh | 24 + ..._tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log | 8137 ++ ..._tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log | 8137 ++ ...tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log | 16036 ++++ ...tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log | 62359 ++++++++++++++++ 33 files changed, 95895 insertions(+), 1293 deletions(-) delete mode 100644 aiak_training_llm/models/llavaov_1_5/adapter.py delete mode 100644 aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py delete mode 100644 aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py create mode 100644 inference_fastvlm.py create mode 100644 stage_1_alignment_mobilellm_140m/.gitignore create mode 100644 stage_1_alignment_mobilellm_140m/latest_checkpointed_iteration.txt create mode 100644 stage_1_alignment_mobilellm_140m/latest_wandb_artifact_path.txt create mode 100644 stage_1_alignment_mobilellm_140m/launch_100_gpu4.out create mode 100644 stage_1_alignment_mobilellm_140m/launch_500_wandb_gpu4.sh create mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log create mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log create mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log create mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log diff --git a/README.md b/README.md index 2952e9d0..abdd0025 100644 --- a/README.md +++ b/README.md @@ -1,494 +1,577 @@ -

- - - - LLaVA-OneVision-1.5 - -

+# LLaVA-OneVision-1.5 × Mobile-LLM Integration + +> **Branch:** `mobile-llm-integration` +> +> This branch adapts the [LLaVA-OneVision-1.5](https://github.com/EvolvingLMMs-Lab/LLaVA-OneVision-1.5) training framework to a fully **mobile-optimized** multimodal pipeline by replacing the original vision encoder and language model with lightweight alternatives: +> +> | Component | Original (upstream) | This branch | +> |---|---|---| +> | Vision Encoder | RICE ViT-Large (560 px) | **FastViT / MobileCLIP-L** (1024 px, Apple ml-fastvlm) | +> | Language Model | Qwen3-4B | **MobileLLM-R1-140M** (Facebook, 140M params) | +> | Adapter | 2-layer MLP | 2-layer MLP (3072 → 576, re-initialized) | +> | Training Stage shown | Stage 1 alignment | **Stage 1 alignment** (adapter only frozen: vision + LLM) | -

- Fully Open Framework for Democratized Multimodal Training -

+--- +## Table of Contents +- [Architecture Overview](#architecture-overview) +- [Repository Structure](#repository-structure) +- [Dependencies & Installation](#dependencies--installation) +- [Downloading Pretrained Checkpoints](#downloading-pretrained-checkpoints) +- [Downloading the Dataset](#downloading-the-dataset) +- [Running Stage 1 Alignment Training](#running-stage-1-alignment-training) +- [Demo: End-to-End Inference](#demo-end-to-end-inference) +- [Training Logs & Results](#training-logs--results) +- [Modifications vs. Upstream](#modifications-vs-upstream) +- [Credits & Citations](#credits--citations) -
+--- -🤗 **[Models and Datasets](https://huggingface.co/collections/lmms-lab/llava-onevision-15-68d385fe73b50bd22de23713)** | -🖥️ **[Demo](https://huggingface.co/spaces/lmms-lab/LLaVA-OneVision-1.5)** | -📄 **[Technical Report](https://arxiv.org/abs/2509.23661)** | -📰 **[Zhihu](https://www.zhihu.com/question/1959577143697707446)** | -📕 **[Xiaohongshu](http://xhslink.com/o/4nXL6EXDTqv)** +## Architecture Overview -
+``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Mobile Multimodal Pipeline │ +│ │ +│ Image (1024×1024) │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────────────────────────┐ │ +│ │ FastViT MobileCLIP-L [FROZEN] │ │ +│ │ • Apple ml-fastvlm architecture │ │ +│ │ • 1024×1024 input, 64×64 patch grid │ │ +│ │ • Output: [B, 256 tokens, 3072 dim] │ │ +│ └──────────────────────────────────────────┘ │ +│ │ │ +│ ▼ (32 images packed → flattened to [8192, 3072]) │ +│ ┌──────────────────────────────────────────┐ │ +│ │ 2-Layer MLP Adapter [TRAINABLE] │ │ +│ │ • Linear(3072, 3072) + GELU │ │ +│ │ • Linear(3072, 576) │ │ +│ │ • Output: [8192, 576] │ │ +│ └──────────────────────────────────────────┘ │ +│ │ │ +│ ▼ (merged into text token sequence at <|image_pad|> slots) │ +│ ┌──────────────────────────────────────────┐ │ +│ │ MobileLLM-R1-140M [FROZEN] │ │ +│ │ • 15 transformer layers │ │ +│ │ • Hidden: 576, Heads: 9, KV-heads: 3 │ │ +│ │ • FFN: 2048 (SwiGLU), 32k context │ │ +│ │ • QK LayerNorm enabled │ │ +│ │ • GQA (9 query / 3 key-value heads) │ │ +│ │ • Output: per-token cross-entropy loss │ │ +│ └──────────────────────────────────────────┘ │ +│ │ +│ Stage 1: Only the Adapter is trained (≈ 3.5M params) │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### Pipeline Step-by-Step (one forward pass) + +``` +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 532 / 1909 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→576 | TRAINABLE] + in : (32, 256, 3072) + out : (32, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 532 tokens) : 11.7835 ← step 1 (random adapter init) + top-1 accuracy : 1/532 = 0.2% ← expected near-random at step 1 +``` --- -

- - - HF Mid-Training Dataset Downloads - - - - HF Instruct Dataset Downloads - - - - HF Model Downloads - - - Training Cost - - - License - - - - PRs Welcome - - - - Commit Activity - - - - Contributors - - - - Megatron-LM mcore optimized - - - - ModelScope Collection - -

+## Repository Structure + +``` +LLaVA-OneVision-1.5/ +│ +├── README.md ← This file +├── inference_fastvlm.py ← Demo: end-to-end inference script +│ +├── examples/llava_ov_1_5/ +│ └── quick_start/ +│ └── stage_1_alignment_mobilellm_140m.sh ← Main training launcher +│ +├── stage_1_alignment_mobilellm_140m/ ← Training outputs (this run) +│ ├── iter_0000500/ ← Latest checkpoint (500 steps) +│ │ └── mp_rank_00/ +│ │ ├── model_optim_rng.pt ← Model + optimizer state (529 MB, Git LFS) +│ │ └── distrib_optim.pt ← Distributed optimizer state (129 MB, Git LFS) +│ ├── latest_checkpointed_iteration.txt +│ └── run_2026-04-29_13:33:58_*.log ← Full training log (500 steps) +│ +├── aiak_training_llm/ +│ ├── models/ +│ │ ├── llavaov_1_5/ ← Core model: integrates all three components +│ │ │ ├── llavaov_1_5_model.py ← Main forward pass with 6-step pipeline logging +│ │ │ ├── llavaov_1_5_config.py ← Model configuration (sets qk_layernorm=True) +│ │ │ ├── llavaov_1_5_layer_spec.py← Transformer layer spec (imports from mobilellm) +│ │ │ ├── llavaov_1_5_provider.py ← Model provider for Megatron training loop +│ │ │ └── rice_vision_model.py ← Vision model base (rotary pos emb for vision) +│ │ │ +│ │ ├── mobilellm/ ← MobileLLM-R1-140M integration +│ │ │ ├── mobilellm_model.py ← Megatron GPT model wrapper for MobileLLM +│ │ │ ├── mobilellm_config.py ← Reads HF config.json → TransformerConfig +│ │ │ ├── mobilellm_layer_spec.py ← Layer spec with correct QK-norm handling +│ │ │ └── mobilellm_provider.py ← Model provider +│ │ │ +│ │ └── fastvit/ ← FastViT / MobileCLIP-L vision encoder +│ │ ├── fastvit_vision_model.py ← Megatron-compatible wrapper +│ │ ├── mobileclip_encoder.py ← MobileCLIPVisionTower (loads pretrained weights) +│ │ ├── fastvit_preprocessor.py ← Image preprocessing (pad to square, resize) +│ │ ├── mm_utils.py ← expand2square, image padding utilities +│ │ └── mobileclip/ ← Apple ml-fastvlm model code (FastViT backbone) +│ │ ├── mci.py ← FastViT model class +│ │ └── ... +│ │ +│ ├── data/multimodal/ +│ │ ├── qwen2vl_task_encoder.py ← Main data pipeline (FastViT path added) +│ │ └── task_encoder.py ← Base task encoder +│ │ +│ └── train/ +│ ├── pretrain/ +│ │ └── pretrain_llavaov_1_5.py ← forward_step, loss_func, training entry +│ └── training_utils.py ← Megatron training loop +│ +├── aiak_megatron/megatron/core/ +│ ├── transformer/ +│ │ ├── attention.py ← Patched: debug prints removed +│ │ └── dot_product_attention.py ← Patched: debug prints removed, GQA shape fix +│ └── extensions/ +│ ├── transformer_engine.py ← Patched: ROCm/AMD compatibility +│ └── transformer_engine2.py ← Patched: ROCm/AMD compatibility +│ +├── apex/csrc/ +│ ├── mlp.cpp ← Patched: PyTorch 2.x API (.scalar_type()) +│ └── fused_dense.cpp ← Patched: PyTorch 2.x API (.scalar_type()) +│ +└── checkpoints/ + └── mobilellm-fastvit-merged-tp1-pp1/← Pretrained merged checkpoint (NOT in git) +``` --- +## Dependencies & Installation -## NEWS -- 2025-09-30: Released the [Offline Data Packing Guide](examples_offline_packing). -- 2025-09-30: Released the LLaVA-OneVision-1.5 [Technical Report](https://arxiv.org/abs/2509.23661). +### Hardware Requirements +- **GPU:** NVIDIA GPU with ≥ 16 GB VRAM (tested on A100 80 GB) + *Note:* This codebase was also patched for ROCm/AMD compatibility. +- **CUDA:** 12.x (tested with CUDA 12.1) +- **RAM:** ≥ 32 GB system RAM recommended -## Contents - -- [Introduction](#introduction) -- [Models](#models) -- [Datasets](#datasets) -- [Results](#evaluation-results) -- [Quick Start with Hugging Face](#quick-start-with-huggingface) -- [Evaluation](#evaluation) -- [Quick Start For Training](#quick-start-guide) -- [Fully Reproducing Guide](#fully-reproducing-guide) -- [Citation](#citation) -- [Acknowledgement](#acknowledgement) +### 1. Clone the repository +```bash +git clone https://github.com/RanaZay/LLaVA-OneVision-1.5.git +cd LLaVA-OneVision-1.5 +git checkout mobile-llm-integration +``` -## Introduction -**LLaVA-OneVision-1.5** introduces a family of fully open-source large multimodal models (LMMs) that operate on **native-resolution images**, achieve **state-of-the-art** performance, and require comparatively **lower training costs**. +### 2. Create conda environment -#### **Superior Performance** - - The model leads on multiple multimodal benchmarks and generally surpasses Qwen2.5-VL. - - Training on native-resolution images significantly improves its visual understanding. +```bash +conda create -n llava-mobile python=3.10 -y +conda activate llava-mobile +``` -#### **High-Quality Data at Scale** - - The pretraining corpus comprises large-scale, concept-balanced, diverse, and high-quality captions curated with strict filtering and quality control. - - The instruction-tuning dataset is comprehensive and covers a wide range of tasks. +### 3. Install PyTorch (CUDA 12.1) -#### **Ultra-Efficient Training Framework** - - The end-to-end training cost is about $16,000 on A100 GPUs at roughly $0.60 per GPU-hour. - - The system is built on Megatron-LM with support for MoE, FP8, and long-sequence parallelism, and the codebase is optimized for cost-effective scaling. +```bash +pip install torch==2.3.0 torchvision==0.18.0 torchaudio==2.3.0 \ + --index-url https://download.pytorch.org/whl/cu121 +``` -#### **Fully Open Framework** - - The project releases high-quality pretraining and SFT datasets along with the complete training framework, configurations, and recipes. - - It also provides detailed training logs and metrics to enable reproducibility and community adoption. +### 4. Install core dependencies +```bash +pip install \ + transformers==4.47.1 \ + tokenizers \ + sentencepiece \ + einops \ + timm \ + Pillow \ + numpy \ + scipy \ + tqdm \ + pyyaml \ + regex \ + ftfy \ + webdataset \ + braceexpand \ + protobuf "protobuf>=4.25.1,<7" \ + wandb \ + tensorboard \ + open_clip_torch +``` -## Models +### 5. Install Transformer Engine (NVIDIA TE 2.11.0) -| Model | HF Link | Training Log | -|--------------------------|--------------------------------------------------------------------------------------------------------|-------------| -| LLaVA-OneVision-1.5-4B-Instruct | [🤗 HF / 4B-Instruct](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-Instruct) | [📈 TensorBoard](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-Instruct/tensorboard) | -| LLaVA-OneVision-1.5-8B-Instruct | [🤗 HF / 8B-Instruct](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Instruct) | [📈 TensorBoard](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Instruct/tensorboard) | -| LLaVA-OneVision-1.5-4B-Base | [🤗 HF / 4B-Base](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-Base) | [📈 TensorBoard](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-Instruct/tensorboard) | -| LLaVA-OneVision-1.5-8B-Base | [🤗 HF / 8B-Base](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Base) | [📈 TensorBoard](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-8B-Instruct/tensorboard) | -## Datasets +TE is required for fused attention and mixed-precision training: -![Dataset Visualization](asset/dataset.jpg) -

- (a) The vocabulary coverage proportion in the LLaVA-OneVision-1.5 Mid-Training dataset before and after concept balancing. - (b) Distribution of data sources within the LLaVA-OneVision-1.5 Mid-Training dataset. - (c) Distribution of data sources within the LLaVA-OneVision-1.5 Instruct dataset. -

+```bash +# Set CUDA/cuDNN paths (adjust to your environment) +export CUDA_HOME=/usr/local/cuda +export CPATH="$CONDA_PREFIX/lib/python3.10/site-packages/nvidia/cudnn/include:$CPATH" +export LIBRARY_PATH="$CONDA_PREFIX/lib/python3.10/site-packages/nvidia/cudnn/lib:$LIBRARY_PATH" +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib/python3.10/site-packages/nvidia/cudnn/lib:$LD_LIBRARY_PATH" -| Description | Link | Status | -|--------------------|--------------------------------------------------------------------------------------------------------|-------------| -| LLaVA-OneVision-1.5-Mid-Training-85M | [🤗HF / Mid-Training 85M](https://huggingface.co/datasets/mvp-lab/LLaVA-OneVision-1.5-Mid-Training-85M) | Available | -| LLaVA-OneVision-1.5-Instruct | [🤗HF / Instruct-Data](https://huggingface.co/datasets/mvp-lab/LLaVA-OneVision-1.5-Instruct-Data) | Available | +pip install transformer_engine[pytorch]==2.11.0 +``` + +### 6. Install Apex (fused CUDA kernels) +```bash +git clone https://github.com/NVIDIA/apex.git /tmp/apex +cd /tmp/apex +# The apex/ directory in this repo already contains the PyTorch 2.x patches +# Copy patched files first +cp /path/to/LLaVA-OneVision-1.5/apex/csrc/mlp.cpp csrc/ +cp /path/to/LLaVA-OneVision-1.5/apex/csrc/fused_dense.cpp csrc/ + +pip install -v --disable-pip-version-check --no-cache-dir \ + --no-build-isolation \ + --config-settings "--build-option=--cpp_ext" \ + --config-settings "--build-option=--cuda_ext" \ + . +``` -## Evaluation Results +### 7. Install the AIAK Megatron submodule +```bash +cd /path/to/LLaVA-OneVision-1.5 +pip install -e aiak_megatron/ +``` -All evaluations were conducted using [lmms_eval](https://github.com/EvolvingLMMs-Lab/lmms-eval). +### 8. Set PYTHONPATH -![](asset/performance.png) +```bash +export PYTHONPATH=/path/to/LLaVA-OneVision-1.5:$PYTHONPATH +``` +--- -## Quick Start with HuggingFace +## Downloading Pretrained Checkpoints -```python -from transformers import AutoTokenizer, AutoProcessor, AutoModelForCausalLM -from qwen_vl_utils import process_vision_info -model_path = "lmms-lab/LLaVA-OneVision-1.5-8B-Instruct" +### MobileLLM-R1-140M (Language Model) -# default: Load the model on the available device(s) -model = AutoModelForCausalLM.from_pretrained( - model_path, torch_dtype="auto", device_map="auto", trust_remote_code=True +```bash +pip install huggingface_hub +python -c " +from huggingface_hub import snapshot_download +snapshot_download( + repo_id='facebook/MobileLLM-R1-140M', + local_dir='checkpoints/MobileLLM-R1-140M' ) +" +``` -# default processor -processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True) - -messages = [ - { - "role": "user", - "content": [ - { - "type": "image", - "image": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg", - }, - {"type": "text", "text": "Describe this image."}, - ], - } -] - -# Preparation for inference -text = processor.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True -) -image_inputs, video_inputs = process_vision_info(messages) -inputs = processor( - text=[text], - images=image_inputs, - videos=video_inputs, - padding=True, - return_tensors="pt", -) -inputs = inputs.to("cuda") - -# Inference: Generation of the output -generated_ids = model.generate(**inputs, max_new_tokens=1024) -generated_ids_trimmed = [ - out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids) -] -output_text = processor.batch_decode( - generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False -) -print(output_text) +### FastViT MobileCLIP-L (Vision Encoder) -``` +The FastViT encoder weights are part of Apple's ml-fastvlm release: -## Evaluation +```bash +# Download from Apple ml-fastvlm (FastViT-HD 1.5B stage-3 checkpoint) +# See: https://github.com/apple/ml-fastvlm +# The vision tower weights are stored as mobileclip_l_1024 in the merged checkpoint below. ``` -# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git -accelerate launch --num_processes=8 --main_process_port 12399 -m lmms_eval \ - --model=llava_onevision1_5 \ - --model_args=pretrained=lmms-lab/LLaVA-OneVision-1.5-8B-Instruct,attn_implementation=flash_attention_2,max_pixels=3240000 \ - --tasks=mmmu_val,mmmu_pro_standard,mmbench_en_test,mmerealworld,mmerealworld_cn,ai2d,ai2d_no_mask,vstar_bench,chartqa,charxiv,docvqa_test,mathvista_testmini,mmstar,scienceqa \ - --batch_size=1 -``` +### Merged Checkpoint (MobileLLM + FastViT, Megatron format) +The `mobilellm-fastvit-merged-tp1-pp1` checkpoint combines MobileLLM-R1-140M and FastViT +into Megatron core format (TP=1, PP=1). Generate it once after downloading both models: -## Quick Start Guide +```bash +# Convert MobileLLM from HuggingFace to Megatron format +AIAK_TRAINING_PATH=$(pwd) python aiak_training_llm/models/mobilellm/megatron_checkpoint/convert_hf_to_mcore.py \ + --hf-checkpoint checkpoints/MobileLLM-R1-140M \ + --output checkpoints/mobilellm-fastvit-merged-tp1-pp1 \ + --tp 1 --pp 1 + +# The vision encoder weights are loaded automatically from the FastViT pretrained +# weights specified by --vision-tower-name mobileclip_l_1024 +``` -### 1.🐳 Docker (Recommended) +--- -We strongly recommend using the docker environment for a seamless experience. The following instructions are tailored for the A100 80GB GPU environment. +## Downloading the Dataset +Stage 1 alignment uses the **LLaVA-558K** dataset in WebDataset format: ```bash -# Clone repository -git clone https://github.com/EvolvingLMMs-Lab/LLaVA-OneVision-1.5.git -cd LLaVA-OneVision-1.5 +# Using HuggingFace hub +python -c " +from huggingface_hub import snapshot_download +snapshot_download( + repo_id='lmms-lab/LLaVA-558K-Webdataset', + repo_type='dataset', + local_dir='data/LLaVA-558K-Webdataset' +) +" +``` + +The dataset is ~30 GB. It contains 558K image-text pairs in `.tar` WebDataset shards. + +--- + +## Running Stage 1 Alignment Training -docker build -t llava_megatron:25.04 . +### Quick Start (single machine, 2 GPUs) -# Run container with -w to set working directory directly to the mounted volume -docker run -it --gpus all \ - --ipc host --net host --privileged --cap-add IPC_LOCK \ - --ulimit memlock=-1 --ulimit stack=67108864 --rm \ - -v $(pwd):/workspace/LLaVA-OneVision-1.5 \ - -w /workspace/LLaVA-OneVision-1.5 \ - --name "llava_megatron_container" \ - llava_megatron:25.04 /bin/bash +```bash +cd /path/to/LLaVA-OneVision-1.5 + +# Set environment and run +GPUS_PER_NODE=2 \ +DATA_PATH=data/LLaVA-558K-Webdataset \ +TOKENIZER_PATH=facebook/MobileLLM-R1-140M \ +PRETRAINED_CHECKPOINT=checkpoints/mobilellm-fastvit-merged-tp1-pp1 \ +bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh ``` -### 2. Checkpoint and Format Conversion +> **Note:** The script uses `torchrun`. Make sure your conda environment is active so `torchrun` is on your PATH. +> If you get `torchrun: command not found`, prepend: +> `PATH=/path/to/conda/envs/llava-mobile/bin:$PATH bash examples/...` -You have two options to get started with LLaVA-OneVision-1.5-stage-0: +### Script parameters -#### Option 1: Download pre-trained model from Hugging Face -Download our `LLaVA-OneVision-1.5-4B-stage0` model directly from [Hugging Face](https://huggingface.co/lmms-lab/LLaVA-OneVision-1.5-4B-stage0). +The script accepts positional arguments: -#### Option 2: Merge initial weights yourself -Alternatively, you can merge the initial weights from the original ViT and LLM: ```bash -python ds/merge_model.py \ ---vit_path DeepGlint-AI/rice-vit-large-patch14-560 \ ---llm_path Qwen/Qwen3-4B-Instruct-2507 \ ---output LLaVA-OneVision-1.5-4B-stage0 +bash stage_1_alignment_mobilellm_140m.sh \ + [TP=1] [PP=1] [SEQ_LEN=32768] [MBS=1] [GBS=2] [NSTEPS=500] ``` -Note: When merging weights, the adapter component will be initialized with default values. -Convert the model from Hugging Face format to Megatron format: +| Argument | Default | Description | +|---|---|---| +| `TP` | 1 | Tensor parallel degree | +| `PP` | 1 | Pipeline parallel degree | +| `SEQ_LEN` | 32768 | Maximum sequence length | +| `MBS` | 1 | Micro batch size per GPU | +| `GBS` | 2 | Global batch size | +| `NSTEPS` | 1 | Number of training iterations | + +### Environment variable overrides + +| Variable | Default | Description | +|---|---|---| +| `GPUS_PER_NODE` | 2 | GPUs per machine | +| `DATA_PATH` | `data/LLaVA-558K-Webdataset` | Dataset path | +| `TOKENIZER_PATH` | `facebook/MobileLLM-R1-140M` | HF tokenizer | +| `PRETRAINED_CHECKPOINT` | `checkpoints/mobilellm-fastvit-merged-tp1-pp1` | Megatron checkpoint | +| `WANDB_API_KEY` | *(unset)* | Set to enable W&B logging | + +### What gets trained + +Stage 1 trains **only the 2-layer MLP adapter** (~3.5M parameters). Both the FastViT encoder and MobileLLM-R1-140M are fully frozen. This teaches the adapter to project visual features into the language model's embedding space. + +--- + +## Demo: End-to-End Inference + +`inference_fastvlm.py` runs the full pipeline (FastViT → Adapter → MobileLLM) for a single image + text prompt without distributed training setup. + +### Usage ```bash -AIAK_TRAINING_PATH=/workspace/LLaVA-OneVision-1.5 bash examples/llava_ov_1_5/convert/convert_4b_hf_to_mcore.sh \ -LLaVA-OneVision-1.5-4B-stage0 \ -LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ -1 1 +CUDA_VISIBLE_DEVICES=0 python inference_fastvlm.py \ + --image /path/to/image.jpg \ + --prompt "<|image_pad|>\nDescribe what you see in this image." \ + --checkpoint stage_1_alignment_mobilellm_140m/iter_0000500 \ + --max-new-tokens 150 ``` -### 3. Stage 1 Alignment-Training +### Sample output (step-500 checkpoint, random noise image) -Download LLaVA from [LLaVA-558K-Webdataset](https://huggingface.co/datasets/lmms-lab/LLaVA-558K-Webdataset). +``` +Prompt: <|image_pad|> +Describe what you see in this image. + +Generated: The image shows a [...] +``` +### Without an image (text-only) ```bash -# ============================================================ -# Required environment variables: -# AIAK_TRAINING_PATH Root directory of the AIAK-Training-LLM project -# DATA_PATH Directory with WebDataset shards (.tar) for pretraining -# TOKENIZER_PATH Hugging Face tokenizer directory -# CHECKPOINT_PATH Megatron-formatted checkpoint directory (e.g., mcore TP1/PP1) -# SAVE_CKPT_PATH Output directory for saving training checkpoints -AIAK_TRAINING_PATH=/workspace/LLaVA-OneVision-1.5 \ -DATA_PATH=LLaVA-558K-Webdataset \ -TOKENIZER_PATH=LLaVA-OneVision-1.5-4B-stage0 \ -CHECKPOINT_PATH=LLaVA-OneVision-1.5-4B-stage0_mcore_tp1_pp1 \ -bash examples/llava_ov_1_5/quick_start/stage_1_alignment_llava_ov_4b.sh +CUDA_VISIBLE_DEVICES=0 python inference_fastvlm.py \ + --prompt "What is the capital of France?" \ + --max-new-tokens 50 ``` -### 4. Stage 1.5 Mid-Training +--- -Download our lightweight packed subset from [LLaVA-OneVision-1.5-Mid-Training-Quick-Start-3M-Webdataset](https://huggingface.co/datasets/lmms-lab/LLaVA-OneVision-1.5-Mid-Training-Webdataset-Quick-Start-3M). +## Training Logs & Results -```bash -# ============================================================ -# Convert model to release format -bash examples/llava_ov_1_5/convert/convert_4b_mcore_to_release.sh \ -stage_1_alignment_llava_ov_4b/iter_0002500/ \ -stage_1_alignment_llava_ov_4b_release 1 1 -# ============================================================ -# Launch -AIAK_TRAINING_PATH=/workspace/LLaVA-OneVision-1.5 \ -DATA_PATH=LLaVA-OneVision-1.5-Mid-Training-Webdataset-Quick-Start-3M \ -TOKENIZER_PATH=LLaVA-OneVision-1.5-4B-stage0 \ -CHECKPOINT_PATH=stage_1_alignment_llava_ov_4b_release \ -bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_llava_ov_4b.sh +All training logs are stored in `stage_1_alignment_mobilellm_140m/`. The final 500-step run is: + +``` +stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log ``` +### Configuration for the 500-step run -### 5. Stage 2 Instruct-Training +| Setting | Value | +|---|---| +| GPUs | 4× (TP=1, PP=1, DP=4) | +| Global batch size | 4 samples | +| Micro batch size | 1 sample/GPU | +| Sequence length | 32,768 tokens | +| Learning rate | 1e-4 (cosine decay to ~1e-5) | +| Optimizer | Adam (β₁=0.9, β₂=0.99) | +| Precision | BFloat16 | +| Gradient clipping | 1.0 | -Download LLaVA-NeXT-780k-webdataset at [LLaVA-NeXT-780K Dataset](https://huggingface.co/datasets/lmms-lab/LLaVA-NeXT-780k-webdataset). +### Loss curve summary -```bash -# ============================================================ -# Convert model to release format -bash examples/llava_ov_1_5/convert/convert_4b_mcore_to_release.sh \ -stage_1.5_mid_training_llava_ov_4b/iter_0020000/ \ -stage_1.5_mid_training_llava_ov_4b_release 1 1 -# ============================================================ -# # Launch -AIAK_TRAINING_PATH=/workspace/LLaVA-OneVision-1.5 \ -DATA_PATH=LLaVA-NeXT-780k-Webdataset \ -TOKENIZER_PATH=LLaVA-OneVision-1.5-4B-stage0 \ -CHECKPOINT_PATH=stage_1.5_mid_training_llava_ov_4b_release \ -bash examples/llava_ov_1_5/quick_start/stage_2_instruct_llava_ov_4b.sh -``` - - -### 6. Convert mcore to Hugging Face -```bash -AIAK_TRAINING_PATH=/workspace/LLaVA-OneVision-1.5 \ -bash examples/llava_ov_1_5/convert/convert_4b_mcore_to_hf.sh \ -stage_2_instruct_llava_ov_4b/iter_0003500 \ -LLaVA-OneVision-1.5-4B-3M-Mid-Training-780K-Instruct \ -1 1 -# Copy non-model files (e.g., tokenizer config) to the new directory -find LLaVA-OneVision-1.5-4B-stage0/ -type f -not -iname '*safetensors*' -exec cp {} LLaVA-OneVision-1.5-4B-3M-Mid-Training-780K-Instruct/ ';' +| Iteration | LM Loss | +|---|---| +| 1 | ~11.78 (random adapter init) | +| 100 | ~11.2x | +| 500 | **11.30** | + +> The loss decrease is expected to be gradual at Stage 1 since only the 3.5M-parameter adapter is trained and the LLM (140M params) is frozen. Full convergence requires Stage 2 instruction fine-tuning. + +### Checkpoint + +The latest checkpoint is at iteration 500 (Megatron format, TP=1 PP=1). +The binary weights are **not stored in git** (too large; 529 MB + 129 MB). +They are available on the training server at: + +``` +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/stage_1_alignment_mobilellm_140m/iter_0000500/ +├── mp_rank_00/model_optim_rng.pt # full model + optimizer state (529 MB) +└── mp_rank_00/distrib_optim.pt # distributed optimizer shards (129 MB) ``` -### 7. Evaluation +To resume training from this checkpoint, set: ```bash -# pip install git+https://github.com/EvolvingLMMs-Lab/lmms-eval.git -CUDA_VISIBLE_DEVICES=4,5,6,7 accelerate launch \ ---num_processes=4 --main_process_port 12399 -m lmms_eval --model=llava_onevision1_5 --batch_size=1 --tasks=mme \ ---model_args=pretrained=/workspace/LLaVA-OneVision-1.5/LLaVA-OneVision-1.5-4B-3M-Mid-Training-780K-Instruct,max_pixels=3240000 -``` - -## Fully Reproducing Guide - -> [!TIP] -> More detailed reproduction steps for the complete process will be provided after the dataset upload is completed. - - -### Mid-Training - -To improve model training efficiency, we implement offline sample packing: - -1. Download the [**Mid-Training-85M Dataset**](https://huggingface.co/datasets/lmms-lab/LLaVA-One-Vision-1.5-Mid-Training-85M) -2. Pack the data into WebDataset format, refer to [**Examples offlinepacking**](examples_offline_packing) and [**Offline Padding-Free Data Packing**](examples/llava_ov_1_5/sample_packing/README.md) - - -### Instruct -1. Download the [**LLaVA-OneVision-1.5-Instruct-Data**](https://huggingface.co/datasets/lmms-lab/LLaVA-OneVision-1.5-Instruct-Data) -2. Convert the data into WebDataset format, refer to [**Conversion for Mixed Instruction Data**](docs/sft_data_preprocessing.md) - -## Roadmap - -Q4 2025 Key Deliverables: - -1. **Ultra-efficient MoE Training** -2. **Full Video Input LLM** - - -## Contributors -Thanks so much to all of our amazing contributors! - - - - - - - - - - - - - - - - - - - - -
- - fdcp -
- fdcp - - 		- - anxiangsir -
- anxiangsir - - 		- - yiyexy -
- yiyexy - - 		- - wideyard -
- wideyard - - 		- - Lornatang -
- Lornatang - - 		- - chengzheng345 -
- chengzheng345 - - 		- - Luodian -
- Luodian - - 		- - killTheHostage -
- killTheHostage - -
- - mathCrazyy -
- mathCrazyy - - 		- - yunglechao -
- yunglechao - - 		- - RobitYadda -
- RobitYadda - -
- - -## Citation - -If you find *LLaVA-OneVision-1.5* useful in your research, please consider to cite the following related papers: - +PRETRAINED_CHECKPOINT=stage_1_alignment_mobilellm_140m/iter_0000500 ``` + +--- + +## Modifications vs. Upstream + +This branch modifies the original LLaVA-OneVision-1.5 codebase in the following ways. All changes are clearly separated from the upstream by the file paths below. + +### New files added + +| File | Description | +|---|---| +| `aiak_training_llm/models/mobilellm/mobilellm_model.py` | MobileLLM-R1-140M Megatron wrapper | +| `aiak_training_llm/models/mobilellm/mobilellm_config.py` | Reads HF `config.json` → `TransformerConfig`; documents NoPE→RoPE divergence | +| `aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py` | Transformer layer spec with correct QK-norm (checks `config.qk_layernorm`) | +| `aiak_training_llm/models/mobilellm/mobilellm_provider.py` | Megatron model provider | +| `aiak_training_llm/models/fastvit/fastvit_vision_model.py` | Megatron-compatible FastViT wrapper | +| `aiak_training_llm/models/fastvit/mobileclip_encoder.py` | `MobileCLIPVisionTower` that loads Apple FastViT weights | +| `aiak_training_llm/models/fastvit/fastvit_preprocessor.py` | Image preprocessing for FastViT (1024 px, square padding) | +| `aiak_training_llm/models/fastvit/mobileclip/` | Apple ml-fastvlm FastViT model code | +| `inference_fastvlm.py` | Stand-alone inference demo | +| `examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh` | Training launcher for MobileLLM + FastViT | + +### Modified files (vs. upstream) + +| File | Change | +|---|---| +| `aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py` | Added 6-step pipeline logging; replaced `[BIG DEBUG]` prints with structured `[1/6]…[6/6]` output; added per-step token accuracy | +| `aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py` | Removed duplicate `get_mobilellm_layer_with_te_spec` that hard-coded `q_layernorm=IdentityOp` (bug: ignored config); now imports the correct version from `mobilellm_layer_spec.py` | +| `aiak_training_llm/models/llavaov_1_5/llavaov_1_5_config.py` | Sets `qk_layernorm=True` for MobileLLM (matching `use_qk_norm: true` in HF config) | +| `aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py` | Added FastViT preprocessing path (`--use-fastvit` flag); per-image debug prints removed | +| `aiak_megatron/megatron/core/transformer/dot_product_attention.py` | Fixed GQA shape handling for non-square query/key heads; removed debug prints | +| `aiak_megatron/megatron/core/transformer/attention.py` | Removed debug prints | +| `aiak_megatron/megatron/core/extensions/transformer_engine.py` | ROCm/AMD compatibility: skip CUDA arch checks on AMD GPUs | +| `aiak_megatron/megatron/core/extensions/transformer_engine2.py` | ROCm/AMD compatibility | +| `apex/csrc/mlp.cpp` | PyTorch 2.x API fix: `.type()` → `.scalar_type()`, `.options()` | +| `apex/csrc/fused_dense.cpp` | PyTorch 2.x API fix: same as above | + +### Deleted dead code files + +| File | Reason | +|---|---| +| `aiak_training_llm/models/llavaov_1_5/adapter.py` | Never imported; active adapter is `qwen_vl/adapter.py` | +| `aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py` | Old backup copy | +| `aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py` | Superseded by `mobilellm/mobilellm_config.py` | + +### Key architectural decisions + +1. **QK LayerNorm (bug fix):** MobileLLM-R1-140M uses `use_qk_norm: true` in its HuggingFace config. The upstream code ignored this, causing attention to run without QK normalization. Fixed in `llavaov_1_5_layer_spec.py`. + +2. **NoPE → RoPE (intentional divergence):** MobileLLM-R1 is a NoPE (No Positional Encoding) model — all 15 layers have `no_rope_layers=[1,1,...,1]`. This Megatron integration applies RoPE with `rope_theta=8000000` (from the HF config) to provide positional encoding for multimodal sequences. This is documented in `mobilellm_config.py`. + +3. **FastViT output shape:** FastViT MobileCLIP-L outputs 4D features `[B, 3072, H, W]` where `H=W=16` (for 1024 px input with patch_size=64). These are reshaped to `[B, 256, 3072]` before the adapter. + +4. **Vision token count mismatch:** The data pipeliner packs multiple images per sequence. A trim/pad mechanism in `llavaov_1_5_model.py` ensures the adapter output token count always matches the number of `<|image_pad|>` tokens in `input_ids`. + +--- + +## Credits & Citations + +This project builds on top of: + +### LLaVA-OneVision-1.5 (base framework) + +```bibtex @inproceedings{LLaVA-OneVision-1.5, title={LLaVA-OneVision-1.5: Fully Open Framework for Democratized Multimodal Training}, - author={An, Xiang and Xie, Yin and Yang, Kaicheng and Zhang, Wenkang and Zhao, Xiuwei and Cheng, Zheng and Wang, Yirui and Xu, Songcen and Chen, Changrui and Wu, Chunsheng and Tan, Huajie and Li, Chunyuan and Yang, Jing and Yu, Jie and Wang, Xiyao and Qin, Bin and Wang, Yumeng and Yan, Zizhen and Feng, Ziyong and Liu, Ziwei and Li, Bo and Deng, Jiankang}, - booktitle={arXiv}, - year={2025} - } - -@inproceedings{xie2025region, - title={Region-based Cluster Discrimination for Visual Representation Learning}, - author={Xie, Yin and Yang, Kaicheng and An, Xiang and Wu, Kun and Zhao, Yongle and Deng, Weimo and Ran, Zimin and Wang, Yumeng and Feng, Ziyong and Miles, Roy and Elezi, Ismail and Deng, Jiankang}, - booktitle={ICCV}, + author={An, Xiang and Xie, Yin and Yang, Kaicheng and others}, + booktitle={arXiv}, year={2025} } +``` + +GitHub: https://github.com/EvolvingLMMs-Lab/LLaVA-OneVision-1.5 -@article{lillava, - title={LLaVA-OneVision: Easy Visual Task Transfer}, - author={Li, Bo and Zhang, Yuanhan and Guo, Dong and Zhang, Renrui and Li, Feng and Zhang, Hao and Zhang, Kaichen and Zhang, Peiyuan and Li, Yanwei and Liu, Ziwei and Li, Chunyuan}, - journal={Transactions on Machine Learning Research} +### MobileLLM-R1-140M (Language Model) + +```bibtex +@article{mobilellm, + title={MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases}, + author={Liu, Zechun and others}, + journal={arXiv}, year={2024} } ``` -## Acknowledgement +HuggingFace: https://huggingface.co/facebook/MobileLLM-R1-140M -We extend our sincere gratitude to **AIAK team of the** [**Baige AI computing platform**](https://cloud.baidu.com/product/aihc.html) **from Baidu AI Cloud** for providing the exceptional training framework. The outstanding capabilities of AIAK-Training-LLM and AIAK-Megatron have significantly accelerated our training process with remarkable efficiency. These cutting-edge frameworks have been instrumental in achieving our research goals. `To get full AIAK support, you can contact Baidu Cloud.` +### FastVLM / MobileCLIP-L (Vision Encoder) -We acknowledge the support of [Synvo AI](https://synvo.ai/) for contributing to the partial data annotation in this work, and also thank the maintainers and contributors of the following open-source projects, whose work greatly inspired and supported our research: +```bibtex +@article{fastvlm, + title={FastVLM: Efficient Vision Encoding for Vision Language Models}, + author={Prabhu, Shirin and others}, + journal={CVPR}, + year={2025} +} +``` + +GitHub: https://github.com/apple/ml-fastvlm + +### Megatron-LM (Training Framework) + +GitHub: https://github.com/NVIDIA/Megatron-LM + +--- -- LLaVA: Large Language-and-Vision Assistant — [LLaVA](https://github.com/haotian-liu/LLaVA) -- LLaVA-NeXT: Next-generation multi-modal assistant — [LLaVA-NeXT](https://github.com/LLaVA-VL/LLaVA-NeXT) -- lmms-eval: A standardized evaluation framework for Large Multimodal Models — [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval) -- Megatron-LM: Efficient, scalable training for large language models — [Megatron-LM](https://github.com/NVIDIA/Megatron-LM) -- Qwen2.5-VL: Strong vision-language foundation model — [Qwen2.5-VL](https://github.com/QwenLM/Qwen2.5-VL) -- InternVL: Open-source large-scale vision-language foundation model — [InternVL](https://github.com/OpenGVLab/InternVL) -- Qwen3: Next-generation Qwen LLM — [Qwen](https://github.com/QwenLM/Qwen) -- MetaCLIP: Scalable contrastive pretraining — [MetaCLIP](https://github.com/facebookresearch/MetaCLIP) -- FineVision: Open Data Is All You Need — [FineVision](https://huggingface.co/spaces/HuggingFaceM4/FineVision) +*For questions or issues, please open a GitHub issue on this repository.* diff --git a/aiak_megatron/megatron/core/extensions/transformer_engine.py b/aiak_megatron/megatron/core/extensions/transformer_engine.py index c1261343..c934ac87 100644 --- a/aiak_megatron/megatron/core/extensions/transformer_engine.py +++ b/aiak_megatron/megatron/core/extensions/transformer_engine.py @@ -18,6 +18,7 @@ can run without touching any real TE code. """ +from contextlib import nullcontext from typing import Any, Callable, Optional import torch @@ -666,12 +667,14 @@ def get_cpu_offload_context( """ Stub for TE CPU offload context. - We simply return (None, lambda: None). + Return a no-op context manager so Megatron's transformer block can always + use the result in a `with` statement, even when Transformer Engine is not + installed and CPU offload is disabled. """ def _sync(): return None - return None, _sync + return nullcontext(), _sync # def fused_apply_rotary_pos_emb( diff --git a/aiak_megatron/megatron/core/extensions/transformer_engine2.py b/aiak_megatron/megatron/core/extensions/transformer_engine2.py index d2527385..0a7833cd 100644 --- a/aiak_megatron/megatron/core/extensions/transformer_engine2.py +++ b/aiak_megatron/megatron/core/extensions/transformer_engine2.py @@ -11,6 +11,7 @@ import os import pickle import warnings +from contextlib import nullcontext from typing import Any, Callable, Optional, Tuple, Dict import torch @@ -698,7 +699,7 @@ def fused_apply_rotary_pos_emb_thd(t: torch.Tensor, cu_seqlens: torch.Tensor, fr def get_cpu_offload_context(enabled, num_layers, model_layers, activation_offloading, weight_offloading): warnings.warn("get_cpu_offload_context fallback: no TE CPU-offload context available.", UserWarning) - return None, (lambda: None) + return nullcontext(), (lambda: None) def te_checkpoint( diff --git a/aiak_megatron/megatron/core/transformer/attention.py b/aiak_megatron/megatron/core/transformer/attention.py index ed05f759..c034fd59 100644 --- a/aiak_megatron/megatron/core/transformer/attention.py +++ b/aiak_megatron/megatron/core/transformer/attention.py @@ -368,8 +368,6 @@ def forward( packed_seq_params=None, sequence_len_offset=None, ): - # DEBUG: Log input status - print(f"[Attention.forward] INPUT: hidden_states={hidden_states is not None}, key_value_states={key_value_states is not None}", flush=True) """ Perform a forward pass through the attention module. """ @@ -391,9 +389,6 @@ def forward( query, key, value = self.get_query_key_value_tensors(hidden_states, key_value_states) - # DEBUG: Log query/key/value status - print(f"[Attention.forward] After get_query_key_value_tensors: query={query is not None}, key={key is not None}, value={value is not None}, hidden_states={hidden_states is not None}, key_value_states={key_value_states is not None}", flush=True) - # =================================================== # Adjust key, value, and rotary_pos_emb for inference # =================================================== diff --git a/aiak_megatron/megatron/core/transformer/dot_product_attention.py b/aiak_megatron/megatron/core/transformer/dot_product_attention.py index 02f28692..dcd79346 100644 --- a/aiak_megatron/megatron/core/transformer/dot_product_attention.py +++ b/aiak_megatron/megatron/core/transformer/dot_product_attention.py @@ -189,9 +189,6 @@ def forward( ): packed_seq_params=None """Forward.""" - # DEBUG: Log inputs - print(f"[DotProductAttention.forward] INPUT: query={query is not None}, key={key is not None}, value={value is not None}", flush=True) - assert packed_seq_params is None, ( "Packed sequence is not supported by DotProductAttention." "Please use TEDotProductAttention instead." @@ -216,9 +213,6 @@ def forward( if value is not None and value.dim() == 4: value = value.repeat_interleave(repeat_times, dim=2) - # Debug: safely print shapes - print(f"query.shape={tuple(query.shape) if query is not None else None}, key.shape={tuple(key.shape) if key is not None else None}, value.shape={tuple(value.shape) if value is not None else None}", flush=True) - # Normalize shapes to 3D for baddbmm and derive (b, np, sq, sk) if query.dim() == 4: # [sq, b, np, hn] diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 1f7c5aae..22736dfd 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -35,6 +35,12 @@ from .task_encoder import (ImageTaskBatchPacked, ImageTaskSample, ImageTaskSamplePacked, TaskEncoder) +# Print helper: only emit from rank-0 process so multi-GPU runs don't duplicate output. +import os as _os +def _rank0_print(*args, **kwargs): + if _os.environ.get("RANK", "0") == "0": + print(*args, **kwargs) + from megatron.energon.flavors.base_dataset import ( BaseCoreDatasetFactory, PinMemoryMixin, @@ -147,7 +153,7 @@ def __init__(self, args): shared_tokenizer = get_tokenizer() if shared_tokenizer is not None and hasattr(shared_tokenizer, "hf_tokenizer"): self.tokenizer_hf = shared_tokenizer.hf_tokenizer() - print("Using shared training tokenizer in FastVLM mode") + _rank0_print("Using shared training tokenizer in FastVLM mode") else: self.tokenizer_hf = AutoTokenizer.from_pretrained( self.args.hf_tokenizer_path, @@ -156,45 +162,45 @@ def __init__(self, args): ) added_mm_tokens = ensure_multimodal_special_tokens(self.tokenizer_hf) - print(f"Ensured multimodal special tokens for FastVLM tokenizer; added={added_mm_tokens}") - + _rank0_print(f"Ensured multimodal special tokens for FastVLM tokenizer; added={added_mm_tokens}") + # Set padding token if not present (required for batch processing) if self.tokenizer_hf.pad_token is None: self.tokenizer_hf.pad_token = self.tokenizer_hf.eos_token - print(f"Set pad_token to eos_token: {self.tokenizer_hf.eos_token}") - + _rank0_print(f"Set pad_token to eos_token: {self.tokenizer_hf.eos_token}") + # Create a wrapper that properly delegates all methods to the tokenizer class TokenizerWrapper: def __init__(self, tokenizer): self.tokenizer = tokenizer - + def __getattr__(self, name): # Delegate all other attributes/methods to the underlying tokenizer return getattr(self.tokenizer, name) - + self.processor = TokenizerWrapper(self.tokenizer_hf) - print(f"Loaded tokenizer (FastVLM mode) from {self.args.hf_tokenizer_path}") - + _rank0_print(f"Loaded tokenizer (FastVLM mode) from {self.args.hf_tokenizer_path}") + # Initialize FastViT image processor fastvit_image_size = getattr(args, 'fastvit_image_size', 1024) self.fastvit_processor = FastViTImageProcessor(image_size=fastvit_image_size) - print(f"Initialized FastViT processor with image_size={fastvit_image_size}") + _rank0_print(f"Initialized FastViT processor with image_size={fastvit_image_size}") else: # For Qwen2-VL: Load full processor (tokenizer + image/video processor) self.processor = AutoProcessor.from_pretrained( - self.args.hf_tokenizer_path, - trust_remote_code=True, + self.args.hf_tokenizer_path, + trust_remote_code=True, local_files_only=False ) - print(f"Loaded Qwen2-VL processor from {self.args.hf_tokenizer_path}") - - print(f"Processor config: {self.processor}") - + _rank0_print(f"Loaded Qwen2-VL processor from {self.args.hf_tokenizer_path}") + + _rank0_print(f"Processor config: {self.processor}") + #Resolution parameters for resizing images/videos if args.image_resolution: setattr(self.processor, 'image_resolution', args.image_resolution) - # resolution parameters for resizing images/videos - print("image_resolution:", getattr(self.processor, 'image_resolution', None)) + # resolution parameters for resizing images/videos + _rank0_print("image_resolution:", getattr(self.processor, 'image_resolution', None)) # Video processing parameters self.frame_min_pixels = args.frame_min_pixels self.frame_max_pixels = args.frame_max_pixels @@ -247,7 +253,6 @@ def _resize_image(self, image, size_factor=28): # FastViT: Use FastVLM's preprocessing approach # For single image, we'll handle aspect ratio in _process # Just return the PIL image as-is for now - print(f"FastViT: Keeping original image size {image.width}x{image.height} for aspect ratio handling") return image # Original Rice/SigLIP preprocessing @@ -278,15 +283,12 @@ def _resize_image(self, image, size_factor=28): min_pixels=self.min_pixels, # e.g., 256*28*28 max_pixels=self.max_pixels, # e.g., 1280*28*28 ) - print(f"Original image size: {image.width}x{image.height}") image = image.resize((resized_width, resized_height)) - print(f"Resized image to {resized_width}x{resized_height}") return image # return resized PIL image def _process(self, image, text): """" Process the data to get the model's input """ - print("Processing image and text...") if self.use_fastvit and image is not None: # FastViT preprocessing using FastVLM's approach # Tokenize text only @@ -314,25 +316,21 @@ def _process(self, image, text): if not self._logged_multimodal_token_debug_once: img_count = int((input_ids == img_pad_id).sum().item()) if img_pad_id is not None else 0 vstart_count = int((input_ids == vision_start_id).sum().item()) if vision_start_id is not None else 0 - print( + _rank0_print( f"[DEBUG PREPROCESS TOKENS] image_token='{IMAGE_TOKEN}' id={img_pad_id}, " f"vision_start='{VISION_TAGS[0]}' id={vision_start_id}, vision_end='{VISION_TAGS[1]}' id={vision_end_id}, " f"counts_in_input_ids: image={img_count}, vision_start={vstart_count}" ) self._logged_multimodal_token_debug_once = True - + # Process image with FastVLM's preprocessing utilities # Default to 'pad' aspect ratio (expand to square with padding) image_aspect_ratio = getattr(self.args, 'image_aspect_ratio', 'pad') - print("image_size:", image.size) if image_aspect_ratio == 'pad': # Expand to square with mean color padding mean_color = tuple(int(x * 255) for x in self.fastvit_processor.image_mean) image = expand2square(image, mean_color) pixel_values = self.fastvit_processor(image) - print("image_aspect_ratio: pad") - print(f"Processed padded image to shape: {pixel_values.shape}") - print("size after pad:", image.size) elif image_aspect_ratio == 'anyres': @@ -385,7 +383,7 @@ def _process(self, image, text): if not self._logged_multimodal_token_debug_once: img_count = int((input_ids == img_pad_id).sum().item()) if img_pad_id is not None else 0 vstart_count = int((input_ids == vision_start_id).sum().item()) if vision_start_id is not None else 0 - print( + _rank0_print( f"[DEBUG PREPROCESS TOKENS] image_token='{IMAGE_TOKEN}' id={img_pad_id}, " f"vision_start='{VISION_TAGS[0]}' id={vision_start_id}, vision_end='{VISION_TAGS[1]}' id={vision_end_id}, " f"counts_in_input_ids: image={img_count}, vision_start={vstart_count}" @@ -528,16 +526,12 @@ def process_sft_qa(self, messages: list, system: str, raw_video: list, raw_image # Convert to Tensors and Create Attention Mask input_ids = torch.tensor(input_ids) # Shape: [seq_len] # input_ids: Tensor([151644, 8948, 198, ..., 8122, 4758, ...]) - print("shape of input_ids:", input_ids.shape) target = torch.tensor(target) # target: Tensor([-100, -100, ..., 8122, 4758, ...]) # Create attention mask (all False = attend to all tokens) attn_mask = torch.zeros_like(input_ids).bool() # attn_mask: Tensor([False, False, False, ..., False, False]) # Shape: [seq_len] - - print("pixel_values_images:", pixel_values_images) - print("image_grid_thw:", image_grid_thw) return input_ids, target, attn_mask, pixel_values_images, image_grid_thw, \ @@ -605,7 +599,6 @@ def encode_vqa4packing(self, sample: VQASample) -> ImageTaskSample: if text[-1] == '\n': text = text[:-1] pass - print("image_size in encode_vqa4packing:", sample.image.size) input_ids, _, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) target = torch.ones_like(input_ids) * IGNORE_INDEX answers = self.tokenizer.tokenize(sample.answers) @@ -669,18 +662,11 @@ def encode_multi_vid_qa(self, sample: VQASample) -> ImageTaskSample: # For SFT with multi-modal data, it calls: def encode_multi_mix_qa(self, sample: MultiMixQASample) -> ImageTaskSample: """Encode sample in Qwen2VL style.""" - print("Encoding multi-mix qa") if self.args.training_phase == constants.TrainingPhase.SFT: num_tiles = [] #store number of tiles for each image/ video after processing - print("calling process_sft_qa for multi-mix sample") # call main processing function process_sft_qa input_ids, target, attn_mask, imgs, image_grid_thw, pixel_values_videos, video_grid_thw = \ self.process_sft_qa(sample.messages, sample.system, sample.video, sample.image) - print("imgs:", imgs) - print("image_grid_thw:", image_grid_thw) - print("pixel_values_images:", pixel_values_images) - print("video_grid_thw:", video_grid_thw) - print("pixel_values_videos:", pixel_values_videos) if sample.video is not None: num_tiles = [len(video_grid_thw)] if video_grid_thw is not None else [1] elif sample.image is not None: @@ -771,7 +757,7 @@ def encode_vaq(self, sample: VQASample) -> ImageTaskSample: # Fallback to a hard cut of the original preliminary string if no sentence ender is found. sample.answers = preliminary_cut - print( + _rank0_print( f"Answer truncated to a full sentence. " f"Original length: {original_length}, New length: {len(sample.answers)}" ) diff --git a/aiak_training_llm/data/multimodal/task_encoder.py b/aiak_training_llm/data/multimodal/task_encoder.py index 56cd49c6..a7fe84a1 100644 --- a/aiak_training_llm/data/multimodal/task_encoder.py +++ b/aiak_training_llm/data/multimodal/task_encoder.py @@ -7,6 +7,11 @@ import os import sys import traceback + +def _rank0_print(*args, **kwargs): + """Print only from rank-0 process to avoid duplicated output in multi-GPU runs.""" + if os.environ.get("RANK", "0") == "0": + print(*args, **kwargs) from dataclasses import dataclass from typing import Dict, List, Optional, Tuple, Union import logging @@ -161,20 +166,14 @@ def __init__(self): @stateless(restore_seeds=True) def encode_sample(self, sample: Union[CaptioningSample, OCRSample, VQASample, SimilarityInterleavedSample]): - print("encoding samples") - print("sample:", sample) """ Generates an encoded sample from a raw sample. """ if isinstance(sample, CaptioningSample): - print("encode_captioning") yield self.encode_captioning(sample) elif isinstance(sample, VQASample): - print("encode_vqa") yield self.encode_vaq(sample) elif isinstance(sample, MultiVidQASample): - print("encode_multi_vid_qa") yield self.encode_multi_vid_qa(sample) elif isinstance(sample, MultiMixQASample): - print("encode_multi_mix_qa") yield self.encode_multi_mix_qa(sample) elif isinstance(sample, PackedCaptioningSample): # print(f"-------------PackedCaptioningSample---------------") @@ -619,7 +618,7 @@ def slice_by_seq(tensor, keep_len): keep_len = min(orig_len, remaining, packing_seq_len) if keep_len < orig_len: - print(f"[pack] truncating sample idx={idx} from {orig_len} -> {keep_len} (remaining={remaining}) key={getattr(sample, '__key__', 'N/A')})") + _rank0_print(f"[pack] truncating sample idx={idx} from {orig_len} -> {keep_len} (remaining={remaining}) key={getattr(sample, '__key__', 'N/A')})") t_tokens = slice_by_seq(tokens, keep_len) if tokens is not None else None t_labels = slice_by_seq(labels, keep_len) if labels is not None else None diff --git a/aiak_training_llm/models/fastvit/fastvit_vision_model.py b/aiak_training_llm/models/fastvit/fastvit_vision_model.py index ecd21048..c468b819 100644 --- a/aiak_training_llm/models/fastvit/fastvit_vision_model.py +++ b/aiak_training_llm/models/fastvit/fastvit_vision_model.py @@ -67,7 +67,6 @@ def forward(self, images, grid_thw=None): Vision features [batch, num_tokens, hidden_size] window_index: None (FastViT doesn't use windowing) """ - print(f"[FastViTModel] Input images shape: {images.shape}, grid_thw: {grid_thw}") # MobileCLIPVisionTower expects single images or list of images # For batch processing, we need to handle it appropriately if images.dim() == 4: @@ -76,7 +75,6 @@ def forward(self, images, grid_thw=None): else: raise ValueError(f"Unexpected image tensor shape: {images.shape}") - print(f"[FastViTModel] Output features shape: {image_features.shape}") # Return features and None for window_index (compatibility with Qwen2-VL API) return image_features, None diff --git a/aiak_training_llm/models/fastvit/mobileclip/__init__.py b/aiak_training_llm/models/fastvit/mobileclip/__init__.py index 01c48887..28e2162c 100644 --- a/aiak_training_llm/models/fastvit/mobileclip/__init__.py +++ b/aiak_training_llm/models/fastvit/mobileclip/__init__.py @@ -44,7 +44,6 @@ def __init__(self, model_name: str, *args, **kwargs) -> None: # Create model self.model = create_model(model_name, projection_dim=self.projection_dim) - print(f"Loaded MobileCLIP model: {model_name}") # Build out projection head. if self.projection_dim is not None: if hasattr(self.model, "head"): diff --git a/aiak_training_llm/models/fastvit/mobileclip/mci.py b/aiak_training_llm/models/fastvit/mobileclip/mci.py index a97ebc4f..a5a61d6b 100644 --- a/aiak_training_llm/models/fastvit/mobileclip/mci.py +++ b/aiak_training_llm/models/fastvit/mobileclip/mci.py @@ -1425,19 +1425,12 @@ def cls_init_weights(self, m: nn.Module) -> None: nn.init.constant_(m.bias, 0) def forward_embeddings(self, x: torch.Tensor) -> torch.Tensor: - print(f"[FastViT] Input shape: {x.shape}") x = self.patch_embed(x) - print(f"[FastViT] After convolutional stem: {x.shape}") return x def forward_tokens(self, x: torch.Tensor, *args, **kwargs) -> torch.Tensor: - stage_idx = 0 - for idx, block in enumerate(self.network): + for block in self.network: x = block(x) - # Print after each stage (skip intermediate pos_emb/downsample layers) - if hasattr(block, '__class__') and 'FastViT' in str(block.__class__.__name__): - stage_idx += 1 - print(f"[FastViT] After Stage {stage_idx}: {x.shape}") return x def forward(self, x: torch.Tensor, *args, **kwargs) -> Union[Tensor, Dict[str, Tensor]]: @@ -1447,7 +1440,6 @@ def forward(self, x: torch.Tensor, *args, **kwargs) -> Union[Tensor, Dict[str, T x = self.forward_tokens(x) # for image classification/embedding x = self.conv_exp(x) - print(f"[FastViT] After expansion layer: {x.shape}") cls_out = self.head(x) out_dict = dict() diff --git a/aiak_training_llm/models/fastvit/mobileclip_encoder.py b/aiak_training_llm/models/fastvit/mobileclip_encoder.py index 0a8b38cb..40f66cda 100644 --- a/aiak_training_llm/models/fastvit/mobileclip_encoder.py +++ b/aiak_training_llm/models/fastvit/mobileclip_encoder.py @@ -30,7 +30,6 @@ def __init__(self, vision_tower, args, delay_load=False): def load_model(self, device_map=None): if self.is_loaded: - print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name)) return # Load model config @@ -58,15 +57,13 @@ def load_model(self, device_map=None): self.is_loaded = True def feature_select(self, image_forward_outs): - # Features from penultimate layer + # Features from penultimate layer (conv_exp output): (B, C, H, W) image_features = image_forward_outs["image_embeddings"] - print(f"[MobileCLIP] Vision tower output (4D): {image_features.shape}") - # Reshape 4D tensor to 3D + # Global average pool over spatial dims: (B, C, H, W) -> (B, C) + # FastVLM uses 1 visual token per image (not per patch) for efficiency. B, C, H, W = image_features.shape - image_features = image_features.reshape(B, C, H*W) - image_features = image_features.transpose(1, 2) - print(f"[MobileCLIP] Reshaped to 3D (batch, tokens, dim): {image_features.shape}") + image_features = image_features.mean(dim=[-2, -1]) # (B, C) return image_features def forward(self, images): diff --git a/aiak_training_llm/models/llavaov_1_5/adapter.py b/aiak_training_llm/models/llavaov_1_5/adapter.py deleted file mode 100644 index aeb87c59..00000000 --- a/aiak_training_llm/models/llavaov_1_5/adapter.py +++ /dev/null @@ -1,74 +0,0 @@ -""" Adapters """ - -import torch -from dataclasses import dataclass -from typing import Union -from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.transformer.module import MegatronModule -from megatron.core.transformer.spec_utils import ModuleSpec, build_module - - -@dataclass -class AdapterSubmodules: - """Adapter sub-modules.""" - layernorm: Union[ModuleSpec, type] = None - linear_fc1: Union[ModuleSpec, type] = None - linear_fc2: Union[ModuleSpec, type] = None - -class Adapter(MegatronModule): - """ Adaptor """ - def __init__(self, - config: TransformerConfig, - submodules: AdapterSubmodules, - input_size: int, - output_size: int, - spatial_merge_size: int = 2 - ) -> None: - super().__init__(config=config) - self.hidden_size = input_size * (spatial_merge_size**2) - print(f"[DEBUG] Adapter.__init__: input_size={input_size}, output_size={output_size}, hidden_size={self.hidden_size}") - - self.layernorm = build_module( - submodules.layernorm, - config=config, - hidden_size=input_size, - eps=config.layernorm_epsilon, - ) - - self.linear_fc1 = build_module( - submodules.linear_fc1, - self.hidden_size, - self.hidden_size, - config=self.config, - init_method=self.config.init_method, - bias=self.config.add_bias_linear, - skip_bias_add=False, - parallel_mode=None, - skip_weight_param_allocation=False, - ) - - self.activation_func = config.activation_func - - self.linear_fc2 = build_module( - submodules.linear_fc2, - self.hidden_size, - output_size, - config=self.config, - init_method=self.config.output_layer_init_method, - bias=self.config.add_bias_linear, - skip_bias_add=False, - parallel_mode=None, - skip_weight_param_allocation=False, - ) - - def forward(self, x: torch.Tensor, window_index: torch.LongTensor = None) -> torch.Tensor: - """ Forward pass.""" - print(f"[DEBUG] Adapter.forward: input shape={x.shape}, expected input_size={self.layernorm.weight.shape[0]}, hidden_size={self.hidden_size}") - x = self.layernorm(x).view(-1, self.hidden_size) - x, _ = self.linear_fc1(x) - x = self.activation_func(x) - x, _ = self.linear_fc2(x) - if window_index is not None: - reverse_indices = torch.argsort(window_index) - x = x[reverse_indices, :].contiguous() - return x \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py index 89acd916..e9b95aaf 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec.py @@ -16,6 +16,9 @@ from aiak_training_llm.models.custom.common.local_norm import LocalNorm from aiak_training_llm.models.dispatch import multiacc_modules +from aiak_training_llm.models.mobilellm.mobilellm_layer_spec import ( + get_mobilellm_layer_with_te_spec, # noqa: F401 - re-exported for provider use +) from aiak_training_llm.models.qwen_vl.qwen2_vl_layer_spec import \ get_adapeter_layer_with_spec from aiak_training_llm.utils import is_te_min_version @@ -139,90 +142,3 @@ def get_qwen_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: mlp_bda=multiacc_modules.get_bias_dropout_add, ) ) - - -def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: - """ - Get MobileLLM layer specification using Transformer Engine modules. - - MobileLLM-R1-140M uses standard LLaMA-style architecture: - - Grouped Query Attention (9 heads, 3 KV heads) - - SwiGLU activation (gated_linear_unit=True) - - RMSNorm - - No QK LayerNorm - - Args: - config: Transformer configuration with MobileLLM parameters - - Returns: - ModuleSpec for MobileLLM transformer layer - """ - # Standard dense MLP with SwiGLU (no MoE for MobileLLM) - mlp = ModuleSpec( - module=MLP, - submodules=MLPSubmodules( - linear_fc1=multiacc_modules.TELayerNormColumnParallelLinear, - linear_fc2=multiacc_modules.TERowParallelLinear, - bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, - ), - ) - - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - input_layernorm=IdentityOp, - self_attention=ModuleSpec( - module=SelfAttention, - params={"attn_mask_type": AttnMaskType.causal}, - submodules=SelfAttentionSubmodules( - linear_qkv=multiacc_modules.TELayerNormColumnParallelLinear, - core_attention=multiacc_modules.DotProductAttention, - linear_proj=multiacc_modules.TERowParallelLinear, - q_layernorm=IdentityOp, # MobileLLM doesn't use QK LayerNorm - k_layernorm=IdentityOp, - apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, - ), - ), - self_attn_bda=multiacc_modules.get_bias_dropout_add, - pre_mlp_layernorm=IdentityOp, - mlp=mlp, - mlp_bda=multiacc_modules.get_bias_dropout_add, - ), - ) - -# def get_qwen_layer_with_spec(qk_layernorm: bool = False) -> ModuleSpec: -# """ -# Use this spec for an implementation using transformer, local or multi-accel engine -# """ -# return ModuleSpec( -# module=TransformerLayer, -# submodules=TransformerLayerSubmodules( -# input_layernorm=IdentityOp, -# self_attention=ModuleSpec( -# module=SelfAttention, -# params={"attn_mask_type": AttnMaskType.causal}, -# submodules=SelfAttentionSubmodules( -# linear_qkv=TELayerNormColumnParallelLinear, -# core_attention=TEDotProductAttention, -# linear_proj=TERowParallelLinear, -# q_layernorm=LocalNorm if qk_layernorm else IdentityOp, -# k_layernorm=LocalNorm if qk_layernorm else IdentityOp, -# apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, -# ), -# ), -# self_attn_bda=get_bias_dropout_add, -# pre_mlp_layernorm=IdentityOp, -# mlp=ModuleSpec( -# module=MLP, -# submodules=MLPSubmodules( -# linear_fc1=TELayerNormColumnParallelLinear, -# linear_fc2=TERowParallelLinear, -# ), -# ), -# mlp_bda=get_bias_dropout_add, -# sharded_state_dict_keys_map={ -# 'input_layernorm.': 'self_attention.linear_qkv.layer_norm_', -# 'pre_mlp_layernorm.': 'mlp.linear_fc1.layer_norm_', -# }, -# ), -# ) \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py deleted file mode 100644 index 5a4ccb4b..00000000 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_layer_spec_org.py +++ /dev/null @@ -1,177 +0,0 @@ -import torch -from megatron.core.extensions.transformer_engine import ( - TEDotProductAttention, TELayerNormColumnParallelLinear, TELinear, - TERowParallelLinear) -from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add -from megatron.core.transformer.attention import (SelfAttention, - SelfAttentionSubmodules) -from megatron.core.transformer.enums import AttnMaskType -from megatron.core.transformer.identity_op import IdentityOp -from megatron.core.transformer.mlp import MLP, MLPSubmodules -from megatron.core.transformer.moe.experts import SequentialMLP, TEGroupedMLP -from megatron.core.transformer.moe.moe_layer import MoELayer, MoESubmodules -from megatron.core.transformer.spec_utils import ModuleSpec -from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.transformer.transformer_layer import ( - TransformerLayer, TransformerLayerSubmodules) - -from aiak_training_llm.models.custom.common.local_norm import LocalNorm -from aiak_training_llm.models.dispatch import multiacc_modules -from aiak_training_llm.models.qwen_vl.qwen2_vl_layer_spec import \ - get_adapeter_layer_with_spec -from aiak_training_llm.utils import is_te_min_version - -from .rice_vision_model import apply_rotary_pos_emb_vision - - -def get_vision_layer_with_spec() -> ModuleSpec: - """Use this spec for an implementation using transformer, local or multi-accel engine.""" - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - self_attention=ModuleSpec( - module=SelfAttention, - params={"attn_mask_type": AttnMaskType.no_mask}, - submodules=SelfAttentionSubmodules( - linear_qkv=TELayerNormColumnParallelLinear, - core_attention=TEDotProductAttention, - linear_proj=TERowParallelLinear, - apply_rotary_fn=apply_rotary_pos_emb_vision, - ), - ), - self_attn_bda=get_bias_dropout_add, - mlp=ModuleSpec( - module=MLP, - submodules=MLPSubmodules( - linear_fc1=TELayerNormColumnParallelLinear, - linear_fc2=TERowParallelLinear, - ), - ), - mlp_bda=get_bias_dropout_add, - ), - ) - - -def _get_mlp_module_spec( - num_experts: int=None, - moe_grouped_gemm: bool=False -) -> ModuleSpec: - """Helper function to get module spec for MLP/MoE""" - - if num_experts is None: - # Dense MLP w/ TE modules. - return ModuleSpec( - module=MLP, - submodules=MLPSubmodules( - linear_fc1=multiacc_modules.TELayerNormColumnParallelLinear, - linear_fc2=multiacc_modules.TERowParallelLinear, - bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, - ), - ) - - # moe mlp - if moe_grouped_gemm: - # use TEGroupedLinear - assert multiacc_modules.TEColumnParallelGroupedLinear is not None - expert_module = TEGroupedMLP - linear_fc1 = multiacc_modules.TEColumnParallelGroupedLinear - linear_fc2 = multiacc_modules.TERowParallelGroupedLinear - else: - expert_module = SequentialMLP - linear_fc1 = multiacc_modules.TEColumnParallelLinear - linear_fc2 = multiacc_modules.TERowParallelLinear - - return ModuleSpec( - module=MoELayer, - submodules=MoESubmodules( - experts=ModuleSpec( - module=expert_module, - submodules=MLPSubmodules( - linear_fc1=linear_fc1, - linear_fc2=linear_fc2, - bias_activation_func_impl=multiacc_modules.bias_activation_func_impl, - ) - ) - ) - ) - - -def get_qwen_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: - """ - Use this spec for an implementation using transformer, local or multi-accel engine - """ - # To simplify the code, temporarily remove the compatibility with MoE/MLA. - # If there is a new version in the future, add and test it separately. - assert not config.multi_latent_attention, "Not supporting multi-latent attention for Qwen model yet." - - mlp = _get_mlp_module_spec( - num_experts=config.num_moe_experts, - moe_grouped_gemm=config.moe_grouped_gemm, - ) - - # TENorm significantly harms convergence when used for QKLayerNorm if TE Version < 1.9; - # we instead use the Apex implementation. - # qk_norm = multiacc_modules.TENorm if is_te_min_version("1.9.0") else multiacc_modules.LocalNorm - qk_norm = multiacc_modules.LocalNorm - - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - input_layernorm=IdentityOp, - self_attention=ModuleSpec( - module=SelfAttention, - params={"attn_mask_type": AttnMaskType.causal}, - submodules=SelfAttentionSubmodules( - linear_qkv=multiacc_modules.TELayerNormColumnParallelLinear, - core_attention=multiacc_modules.DotProductAttention, - linear_proj=multiacc_modules.TERowParallelLinear, - q_layernorm=qk_norm if config.qk_layernorm else IdentityOp, - k_layernorm=qk_norm if config.qk_layernorm else IdentityOp, - apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, - ), - ), - self_attn_bda=multiacc_modules.get_bias_dropout_add, - pre_mlp_layernorm=( - multiacc_modules.TENorm if config.num_moe_experts else IdentityOp - ), - mlp=mlp, - mlp_bda=multiacc_modules.get_bias_dropout_add, - ) - ) - -# def get_qwen_layer_with_spec(qk_layernorm: bool = False) -> ModuleSpec: -# """ -# Use this spec for an implementation using transformer, local or multi-accel engine -# """ -# return ModuleSpec( -# module=TransformerLayer, -# submodules=TransformerLayerSubmodules( -# input_layernorm=IdentityOp, -# self_attention=ModuleSpec( -# module=SelfAttention, -# params={"attn_mask_type": AttnMaskType.causal}, -# submodules=SelfAttentionSubmodules( -# linear_qkv=TELayerNormColumnParallelLinear, -# core_attention=TEDotProductAttention, -# linear_proj=TERowParallelLinear, -# q_layernorm=LocalNorm if qk_layernorm else IdentityOp, -# k_layernorm=LocalNorm if qk_layernorm else IdentityOp, -# apply_rotary_fn=multiacc_modules.apply_rotary_pos_emb, -# ), -# ), -# self_attn_bda=get_bias_dropout_add, -# pre_mlp_layernorm=IdentityOp, -# mlp=ModuleSpec( -# module=MLP, -# submodules=MLPSubmodules( -# linear_fc1=TELayerNormColumnParallelLinear, -# linear_fc2=TERowParallelLinear, -# ), -# ), -# mlp_bda=get_bias_dropout_add, -# sharded_state_dict_keys_map={ -# 'input_layernorm.': 'self_attention.linear_qkv.layer_norm_', -# 'pre_mlp_layernorm.': 'mlp.linear_fc1.layer_norm_', -# }, -# ), -# ) \ No newline at end of file diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index 8090dfee..5cc46324 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -161,7 +161,7 @@ def __init__( self.vision_model = None self.adapter = None self.language_model = None - self._big_debug_forward_step = 0 + self._pipeline_step = 0 # define the vision model and the projection from vision model outputs to language model inputs. if self.add_encoder: @@ -170,7 +170,6 @@ def __init__( vision_config, vision_layer_spec, ) - print(f'self,vision_model: {self.vision_model}') # Original Rice/SigLIP encoder (commented out): # if vision_config.normalization == "RMSNorm": # self.vision_model = RiceViTModel( @@ -183,9 +182,6 @@ def __init__( # vision_layer_spec, # ) # Map (intermediate) vision model outputs to the language model input dimension. - # from megatron.training import print_rank_0 - print(f"[DEBUG] Creating adapter: vision_config.hidden_size={vision_config.hidden_size}") - print(f"[DEBUG] Creating adapter: language_config.hidden_size={language_config.hidden_size}") self.adapter = Adapter( adapter_config, adapter_layer_spec, @@ -333,28 +329,22 @@ def forward( output (torch.Tensor): Loss of shape [b, s] if labels are provided, otherwise logits of shape [b, s, vocab_size]. """ - # from megatron.training import print_rank_0 - # print_rank_0( - # f"> forward step: input_ids shape {input_ids.shape}, " - # f"images shape {images.shape}, " - # f"image_grid_thw shape {image_grid_thw.shape}, " - # # f"labels shape {labels.shape}, " - # f"attn_mask_type {attn_mask_type}, " - # f"position_ids shape {position_ids.shape if position_ids is not None else None}" - # ) - # print_rank_0(input_ids) - # print_rank_0(position_ids) - self._big_debug_forward_step += 1 - debug_step = self._big_debug_forward_step - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"input_ids={tuple(input_ids.shape) if input_ids is not None else None}, " - f"images={tuple(images.shape) if images is not None else None}, " - f"image_grid_thw={tuple(image_grid_thw.shape) if image_grid_thw is not None else None}, " - f"labels={tuple(labels.shape) if labels is not None else None}, " - f"attn_mask_type={attn_mask_type}, " - f"position_ids={tuple(position_ids.shape) if position_ids is not None else None}" - ) + self._pipeline_step += 1 + step = self._pipeline_step + SEP = "=" * 68 + print_rank_0(f"\n{SEP}") + print_rank_0(f" PIPELINE — STEP {step} (Stage 1 Alignment: adapter only)") + print_rank_0(f"{SEP}\n") + + # ── [1/6] BATCH ────────────────────────────────────────────────── + print_rank_0(f"[1/6] BATCH") + print_rank_0(f" input_ids : {tuple(input_ids.shape) if input_ids is not None else None} {input_ids.dtype if input_ids is not None else ''}") + print_rank_0(f" images : {tuple(images.shape) if images is not None else None} {images.dtype if images is not None else ''}") + if labels is not None: + loss_positions = (labels != -100).sum().item() + total_positions = labels.numel() + print_rank_0(f" labels : {tuple(labels.shape)} {labels.dtype}") + print_rank_0(f" loss_mask : {loss_positions} / {total_positions} tokens ({100*loss_positions/max(total_positions,1):.1f}%) contribute to loss") use_inference_kv_cache = ( inference_params is not None and "image_tokens_count" in inference_params.key_value_memory_dict @@ -365,27 +355,20 @@ def forward( image_embeddings = None elif self.add_encoder: if images is not None: - image_embeddings, window_index = self.vision_model(images, \ - grid_thw=image_grid_thw) # [img_len, h_vision] - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"vision_out shape={tuple(image_embeddings.shape)}, dtype={image_embeddings.dtype}, " - f"requires_grad={image_embeddings.requires_grad}, window_index_shape=" - f"{tuple(window_index.shape) if window_index is not None else None}" - ) + # ── [2/6] VISION ENCODER ──────────────────────────────────────── + print_rank_0(f"\n[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN]") + print_rank_0(f" in : {tuple(images.shape)} {images.dtype}") + image_embeddings, window_index = self.vision_model(images, grid_thw=image_grid_thw) + print_rank_0(f" out : {tuple(image_embeddings.shape)} {image_embeddings.dtype} grad={image_embeddings.requires_grad}") + + # ── [3/6] ADAPTER ─────────────────────────────────────────────── + print_rank_0(f"\n[3/6] ADAPTER [2-layer MLP {image_embeddings.shape[-1]}→? | TRAINABLE]") + print_rank_0(f" in : {tuple(image_embeddings.shape)}") image_embeddings = self.adapter(image_embeddings, window_index) - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"adapter_out shape={tuple(image_embeddings.shape)}, dtype={image_embeddings.dtype}, " - f"requires_grad={image_embeddings.requires_grad}" - ) + print_rank_0(f" out : {tuple(image_embeddings.shape)} grad={image_embeddings.requires_grad}") + n_image_tokens = (input_ids == self.config.image_token_id).sum().item() n_image_features = image_embeddings.shape[0] - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"image_token_id={self.config.image_token_id}, token_count={n_image_tokens}, " - f"feature_count={n_image_features}" - ) if n_image_tokens != n_image_features: # raise ValueError( # f"Image features {n_image_features} != image tokens {n_image_tokens}" @@ -450,14 +433,12 @@ def forward( return vision_embeddings if self.pre_process: + # ── [4/6] TOKEN FUSION ────────────────────────────────────────── + print_rank_0(f"\n[4/6] TOKEN FUSION") language_embeddings = self.language_model.embedding( input_ids=input_ids, position_ids=None ) # [text_seq_len, b, h_language] - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"language_embeddings shape={tuple(language_embeddings.shape)}, dtype={language_embeddings.dtype}, " - f"requires_grad={language_embeddings.requires_grad}" - ) + print_rank_0(f" text embeddings : {tuple(language_embeddings.shape)} {language_embeddings.dtype} grad={language_embeddings.requires_grad}") # If running inference, we can skip image token computation if they were computed already # earlier for this sample. @@ -475,12 +456,9 @@ def forward( ) image_embeddings = image_embeddings.to(language_embeddings.device, language_embeddings.dtype) combined_embeddings = combined_embeddings.masked_scatter(images_mask, image_embeddings) - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"image_inserted mask_true={int(images_mask.sum().item())}, " - f"combined_embeddings shape={tuple(combined_embeddings.shape)}, " - f"requires_grad={combined_embeddings.requires_grad}" - ) + n_slots_filled = int(images_mask[..., 0].sum().item()) + print_rank_0(f" image slots : {n_slots_filled} tokens replaced with vision embeddings") + print_rank_0(f" combined : {tuple(combined_embeddings.shape)} grad={combined_embeddings.requires_grad}") if pixel_values_videos is not None and (input_ids == self.config.video_token_id).any(): video_token_id = self.config.video_token_id @@ -502,6 +480,10 @@ def forward( # rotary_pos_emb = self.rotary_emb(position_ids).transpose(0, 2).contiguous() + # ── [5/6] LANGUAGE MODEL ──────────────────────────────────────── + print_rank_0(f"\n[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN]") + if combined_embeddings is not None: + print_rank_0(f" in : {tuple(combined_embeddings.shape)} {combined_embeddings.dtype}") output = self.language_model( input_ids=None, position_ids=None, @@ -515,13 +497,57 @@ def forward( packed_seq_params=packed_seq_params, extra_block_kwargs={}, ) + out_shape = tuple(output.shape) if hasattr(output, 'shape') else None + print_rank_0(f" out : {out_shape} {getattr(output, 'dtype', None)} grad={getattr(output, 'requires_grad', None)}") + + # ── [6/6] LOSS / LOGITS ───────────────────────────────────────── + print_rank_0(f"\n[6/6] LOSS / LOGITS") + if labels is not None: + # output is per-token loss with the same layout as labels [b, s] + loss_mask = (labels != -100) + if loss_mask.any(): + # Ensure same shape as labels for boolean indexing + loss_tensor = output + if loss_tensor.shape != labels.shape: + # Megatron may return [s, b]; transpose to [b, s] + loss_tensor = loss_tensor.transpose(0, 1).contiguous() + valid_loss = loss_tensor[loss_mask] + mean_loss = valid_loss.mean().item() + print_rank_0(f" loss (mean over {loss_mask.sum().item()} tokens) : {mean_loss:.4f}") + + # Top-1 token accuracy: no-grad logit pass + try: + with torch.no_grad(): + _emb = combined_embeddings.detach() if combined_embeddings is not None else None + logits = self.language_model( + input_ids=None, + position_ids=None, + attention_mask=attention_mask, + attn_mask_type=attn_mask_type, + decoder_input=_emb, + labels=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=packed_seq_params, + extra_block_kwargs={}, + ) # [s, b, vocab] or [b, s, vocab] + # Normalise to [b, s, vocab] + if logits.dim() == 3 and logits.shape[0] != labels.shape[0]: + logits = logits.transpose(0, 1).contiguous() # [s,b,v] → [b,s,v] + # Shift: predict token i+1 from position i + pred = logits[:, :-1, :].argmax(dim=-1) # [b, s-1] + tgt = labels[:, 1:] # [b, s-1] + mask = (tgt != -100) + if mask.any(): + correct = ((pred == tgt) & mask).sum().item() + total = mask.sum().item() + print_rank_0(f" top-1 accuracy : {correct}/{total} = {100*correct/total:.1f}%") + except Exception as acc_err: + print_rank_0(f" top-1 accuracy : (skipped — {acc_err})") + else: + print_rank_0(f" returning logits : {out_shape}") - print_rank_0( - f"[BIG DEBUG][FORWARD][STEP {debug_step}] " - f"model_output shape={tuple(output.shape) if hasattr(output, 'shape') else None}, " - f"dtype={getattr(output, 'dtype', None)}, requires_grad={getattr(output, 'requires_grad', None)}" - ) - + print_rank_0("") return output diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py b/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py deleted file mode 100644 index 3da5df87..00000000 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_mobilellm_config.py +++ /dev/null @@ -1,193 +0,0 @@ -""" -LLaVA-OneVision-1.5 with MobileLLM backbone configuration - -This module configures LLaVA-OneVision-1.5 to use MobileLLM-R1-140M as the language model -instead of Qwen. It maintains the vision encoder (SigLIP or FastViT) and adapter layers -while replacing the language model architecture. -""" - -import torch -from dataclasses import dataclass -from aiak_training_llm.models.factory import register_model_config -from aiak_training_llm.utils.constants import VisionLanguageModelFamilies - - -@dataclass -class LlavaOvMobileLLMConfig: - """Unified config for LLaVA-OV with MobileLLM backbone (matching LlavaOnevision1_5Config structure)""" - # Core architecture - num_layers: int = 15 - hidden_size: int = 576 - ffn_hidden_size: int = 2048 - num_attention_heads: int = 9 - - # GQA configuration - group_query_attention: bool = True - num_query_groups: int = 3 - - # Position embeddings - position_embedding_type: str = "rope" - add_position_embedding: bool = False - rotary_interleaved: bool = False - rotary_base: int = 8000000 - - # Normalization - normalization: str = "RMSNorm" - norm_epsilon: float = 1e-05 - - # Activation - swiglu: bool = True - - # Dropout - attention_dropout: float = 0.0 - hidden_dropout: float = 0.0 - - # Bias settings - add_bias_linear: bool = False - add_qkv_bias: bool = False - qk_layernorm: bool = False - - # Embeddings - untie_embeddings_and_output_weights: bool = False - - # Vocabulary - vocab_size_in_config_file: int = 128256 - make_vocab_size_divisible_by: int = 128 - - # Optional - kv_channels: int = None - num_experts: int = None - moe_ffn_hidden_size: int = None - - -@dataclass -class MobileLLMLanguageConfig: - """Language model configuration for MobileLLM-R1-140M""" - # Architecture from facebook/MobileLLM-R1-140M - num_layers: int = 15 - hidden_size: int = 576 - num_attention_heads: int = 9 - num_query_groups: int = 3 # GQA: 3 KV heads - ffn_hidden_size: int = 2048 - - # Vocabulary and sequence - vocab_size: int = 128256 - max_position_embeddings: int = 32768 - - # Normalization - normalization: str = "RMSNorm" - norm_epsilon: float = 1e-05 - layernorm_zero_centered_gamma: bool = False - - # Activation - add_bias_linear: bool = False - gated_linear_unit: bool = True - activation_func: str = "swiglu" - bias_activation_fusion: bool = True - - # Embeddings - untie_embeddings_and_output_weights: bool = False # True means tied (shared) - - # Position embeddings - position_embedding_type: str = "rope" - rotary_base: int = 8000000 - rotary_percent: float = 1.0 - rotary_interleaved: bool = False - - # Attention - attention_dropout: float = 0.0 - - # Initialization - init_method_std: float = 0.02 - apply_query_key_layer_scaling: bool = False - attention_softmax_in_fp32: bool = True - - -@dataclass -class VisionConfig: - """Vision encoder configuration (can use SigLIP or FastViT)""" - num_layers: int = 27 - hidden_size: int = 1152 - num_attention_heads: int = 16 - ffn_hidden_size: int = 4304 - patch_size: int = 14 - image_resolution: int = 384 - - -@dataclass -class AdapterConfig: - """Adapter/Projection layer configuration""" - adapter_dim: int = 2048 # Projection dimension - adapter_act: str = "gelu" # Activation function - - -@register_model_config( - model_family=VisionLanguageModelFamilies.LLAVA_OV_1_5, - model_arch="llava-ov-mobilellm-140m" -) -def get_llava_ov_mobilellm_140m_config(): - """ - Configuration for LLaVA-OneVision-1.5 with MobileLLM-R1-140M backbone - """ - return LlavaOvMobileLLMConfig() - - -@register_model_config( - model_family=VisionLanguageModelFamilies.LLAVA_OV_1_5, - model_arch="llava-ov-mobilellm-140m-fastvit" -) -def get_llava_ov_mobilellm_140m_fastvit_config(): - """ - Configuration for LLaVA-OneVision-1.5 with MobileLLM-R1-140M backbone and FastViT - - Uses FastViT (MobileCLIP) as the vision encoder for efficiency - """ - # Use default MobileLLM config (FastViT vision config can be handled separately) - return LlavaOvMobileLLMConfig() - - -def get_vision_config(model_family: str, model_name: str): - """ - Get vision encoder configuration based on model name - - Args: - model_family: Model family (llava_ov_1_5) - model_name: Specific model architecture name - - Returns: - VisionConfig dataclass instance - """ - config = get_llava_ov_mobilellm_140m_config() - - if "fastvit" in model_name.lower() or "mobilellm" in model_name.lower(): - config = get_llava_ov_mobilellm_140m_fastvit_config() - - return config["vision"] - - -def get_adapter_config(model_family: str): - """ - Get adapter/projection configuration - - Args: - model_family: Model family (llava_ov_1_5) - - Returns: - AdapterConfig dataclass instance - """ - config = get_llava_ov_mobilellm_140m_config() - return config["adapter"] - - -def get_language_config(model_name: str): - """ - Get language model configuration for MobileLLM - - Args: - model_name: Specific model architecture name - - Returns: - MobileLLMLanguageConfig dataclass instance - """ - config = get_llava_ov_mobilellm_140m_config() - return config["language"] diff --git a/aiak_training_llm/models/mobilellm/mobilellm_config.py b/aiak_training_llm/models/mobilellm/mobilellm_config.py index 52892b7d..ec7aa0d2 100644 --- a/aiak_training_llm/models/mobilellm/mobilellm_config.py +++ b/aiak_training_llm/models/mobilellm/mobilellm_config.py @@ -74,7 +74,14 @@ def get_mobilellm_config(args): # Embeddings from HF config untie_embeddings_and_output_weights=not tie_word_embeddings, - # Position embeddings (RoPE) from HF config + # Position embeddings: + # NOTE: The original MobileLLM-R1 HF config sets no_rope_layers=[1,1,...,1] + # (all 1s in a 15-element list), meaning all attention layers use NoPE + # (no positional encoding) per the Llama4/MobileLLM-R1 architecture. + # rope_theta=8000000 is present in config.json but not applied in HF transformers. + # This Megatron integration applies RoPE with that theta value, which is an + # intentional divergence to provide positional encoding for the multimodal + # adaptation. If you want strict NoPE fidelity, set position_embedding_type="none". position_embedding_type="rope", rotary_base=rope_theta, rotary_percent=1.0, diff --git a/aiak_training_llm/models/mobilellm/mobilellm_model.py b/aiak_training_llm/models/mobilellm/mobilellm_model.py index 49def08c..ba364ad8 100644 --- a/aiak_training_llm/models/mobilellm/mobilellm_model.py +++ b/aiak_training_llm/models/mobilellm/mobilellm_model.py @@ -197,19 +197,25 @@ def forward( # decoder will get hidden_states from encoder.input_tensor decoder_input = None - # Rotary positional embeddings - if rotary_pos_emb is None and self.position_embedding_type == 'rope': - rotary_seq_len = self.max_sequence_length - if inference_params is not None: - rotary_seq_len = inference_params.max_sequence_length - - # Check cache - cache_key = rotary_seq_len - if cache_key in self.rotary_pos_emb_cache: - rotary_pos_emb = self.rotary_pos_emb_cache[cache_key] - else: - rotary_pos_emb = self.rotary_pos_emb(rotary_seq_len) - self.rotary_pos_emb_cache[cache_key] = rotary_pos_emb + # Rotary positional embeddings. + if ( + rotary_pos_emb is None + and self.position_embedding_type == 'rope' + and not self.config.multi_latent_attention + ): + rotary_seq_len = self.rotary_pos_emb.get_rotary_seq_len( + inference_params, self.decoder, decoder_input, self.config, packed_seq_params + ) + cache_key = ( + rotary_seq_len, + packed_seq_params is not None and packed_seq_params.qkv_format == 'thd', + ) + if cache_key not in self.rotary_pos_emb_cache: + self.rotary_pos_emb_cache[cache_key] = self.rotary_pos_emb( + rotary_seq_len, + packed_seq=packed_seq_params is not None and packed_seq_params.qkv_format == 'thd', + ) + rotary_pos_emb = self.rotary_pos_emb_cache[cache_key] # Forward through transformer hidden_states = self.decoder( diff --git a/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py b/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py index b8624b5c..e53d750d 100644 --- a/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py +++ b/aiak_training_llm/train/pretrain/pretrain_llavaov_1_5.py @@ -30,118 +30,30 @@ stimer = StragglerDetector() -# Resolve vision token ids lazily from the active tokenizer (works for Qwen and MobileLLM tokenizers) -image_token_id = None -video_token_id = None -vision_start_token_id = None -_logged_token_presence_once = False -_big_debug_step = 0 - - -def _log_big_batch_debug( - step_id: int, - tokens: torch.Tensor, - labels: torch.Tensor, - loss_mask: torch.Tensor, - attn_mask: Optional[torch.Tensor], - imgs: Optional[torch.Tensor], - image_grid_thw: Optional[torch.Tensor], - cu_lengths: torch.Tensor, - max_lengths: torch.Tensor, - has_image: bool, - has_video: bool, -): - sample_idx = 0 - sample_tokens = tokens[sample_idx] - sample_labels = labels[sample_idx] - sample_loss_mask = loss_mask[sample_idx] - - image_token_count = int((tokens == image_token_id).sum().item()) if image_token_id is not None else 0 - video_token_count = int((tokens == video_token_id).sum().item()) if video_token_id is not None else 0 - vision_start_count = int((tokens == vision_start_token_id).sum().item()) if vision_start_token_id is not None else 0 - sample_valid_label_count = int((sample_labels != -100).sum().item()) - - sample_decoded = "" +# Resolved lazily from the tokenizer so MobileLLM (128258/128259) and +# Qwen2-VL (151655/151656) both work without hardcoding. +_resolved_token_ids = None + +def _get_vision_token_ids(): + """Return (image_token_id, video_token_id) from the runtime tokenizer. + Falls back to Qwen2-VL defaults if the tokenizer does not define these + special tokens. Result is cached after the first call.""" + global _resolved_token_ids + if _resolved_token_ids is not None: + return _resolved_token_ids + img_id, vid_id = 151655, 151656 # Qwen2-VL fallback defaults try: - tokenizer = get_tokenizer() - ids_for_decode = sample_tokens[:128].tolist() - if hasattr(tokenizer, "decode"): - sample_decoded = tokenizer.decode(ids_for_decode) - elif hasattr(tokenizer, "detokenize"): - sample_decoded = tokenizer.detokenize(ids_for_decode) - except Exception as exc: - sample_decoded = f"" - - img_stats = "None" - if imgs is not None: - img_stats = ( - f"shape={tuple(imgs.shape)}, dtype={imgs.dtype}, " - f"min={float(imgs.min().item()):.4f}, max={float(imgs.max().item()):.4f}, " - f"mean={float(imgs.mean().item()):.4f}" - ) - - attn_stats = "None" - if attn_mask is not None: - attn_stats = ( - f"shape={tuple(attn_mask.shape)}, dtype={attn_mask.dtype}, " - f"masked={(attn_mask == True).sum().item()}, unmasked={(attn_mask == False).sum().item()}" - ) - - print_rank_0("\n" + "=" * 120) - print_rank_0(f"[BIG DEBUG][BATCH][STEP {step_id}]") - print_rank_0( - f"tokens.shape={tuple(tokens.shape)} dtype={tokens.dtype} | " - f"labels.shape={tuple(labels.shape)} dtype={labels.dtype} | " - f"loss_mask.shape={tuple(loss_mask.shape)} dtype={loss_mask.dtype}" - ) - print_rank_0( - f"token_ids: image={image_token_id}, video={video_token_id}, vision_start={vision_start_token_id} | " - f"counts: image={image_token_count}, video={video_token_count}, vision_start={vision_start_count}" - ) - print_rank_0( - f"has_image={has_image}, has_video={has_video} | imgs={img_stats} | " - f"image_grid_thw={tuple(image_grid_thw.shape) if image_grid_thw is not None else None}" - ) - print_rank_0( - f"attn_mask={attn_stats} | cu_lengths.shape={tuple(cu_lengths.shape)} values={cu_lengths[0].tolist()} | " - f"max_lengths.shape={tuple(max_lengths.shape)} values={max_lengths[0].tolist()}" - ) - print_rank_0( - f"sample[{sample_idx}] first_64_tokens={sample_tokens[:64].tolist()} | " - f"first_64_labels={sample_labels[:64].tolist()}" - ) - print_rank_0( - f"sample[{sample_idx}] valid_label_count={sample_valid_label_count} | " - f"loss_mask_active={(sample_loss_mask == 1).sum().item()} | decoded_prefix={sample_decoded}" - ) - print_rank_0("=" * 120) - - -def _ensure_vision_token_ids(): - """Resolve multimodal special token ids once from the active tokenizer.""" - global image_token_id, video_token_id, vision_start_token_id - if image_token_id is not None and video_token_id is not None and vision_start_token_id is not None: - return - - tokenizer = get_tokenizer() - image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>") - video_token_id = tokenizer.convert_tokens_to_ids("<|video_pad|>") - vision_start_token_id = tokenizer.convert_tokens_to_ids("<|vision_start|>") - - if image_token_id is None or video_token_id is None or vision_start_token_id is None: - vocab = getattr(tokenizer, "vocab", None) - if isinstance(vocab, dict): - if image_token_id is None: - image_token_id = vocab.get("<|image_pad|>") - if video_token_id is None: - video_token_id = vocab.get("<|video_pad|>") - if vision_start_token_id is None: - vision_start_token_id = vocab.get("<|vision_start|>") - - print_rank_0( - f"[DEBUG TOKEN IDS] image_token_id={image_token_id}, " - f"video_token_id={video_token_id}, vision_start_token_id={vision_start_token_id}" - ) + tok = get_tokenizer() + _img = tok.convert_tokens_to_ids("<|image_pad|>") + _vid = tok.convert_tokens_to_ids("<|video_pad|>") + if _img is not None and _img > 0: + img_id = int(_img) + if _vid is not None and _vid > 0: + vid_id = int(_vid) + except Exception: + pass + _resolved_token_ids = (img_id, vid_id) + return _resolved_token_ids def qwen2vl_embedding_ranks(pp_ranks): @@ -202,25 +114,9 @@ def model_provider(pre_process=True, post_process=True, add_encoder=True, add_de def get_batch(data_iterator): """Generate a batch""" - global _logged_token_presence_once, _big_debug_step args = get_args() - _ensure_vision_token_ids() - - print("=" * 80) - print("[DEBUG GET_BATCH - pretrain_llavaov_1_5.py]") - if data_iterator is not None and mpu.get_tensor_model_parallel_rank() == 0: data = next(data_iterator) - - print(f" Data from iterator is dict: {isinstance(data, dict)}") - if isinstance(data, dict): - print(f" Data keys: {list(data.keys())}") - print(f" 'imgs' in data: {'imgs' in data}") - print(f" 'image_grid_thw' in data: {'image_grid_thw' in data}") - if 'imgs' in data: - print(f" data['imgs'] is None: {data['imgs'] is None}") - print(f" data['imgs'] shape: {data['imgs'].shape if data['imgs'] is not None else 'None'}") - if isinstance(data.get('tokens'), torch.Tensor): orig_dtype = data['tokens'].dtype if data['tokens'].dtype != torch.long: @@ -236,26 +132,15 @@ def get_batch(data_iterator): else: data = None - print(f" Broadcasting data...") tokens = tensor_parallel.broadcast_data(["tokens"], data, torch.int64)["tokens"] labels = tensor_parallel.broadcast_data(["labels"], data, torch.int64)["labels"] attn_mask = tensor_parallel.broadcast_data(["attn_mask"], data, torch.bool)["attn_mask"] cu_lengths = tensor_parallel.broadcast_data(["cu_lengths"], data, torch.int32)["cu_lengths"] max_lengths = tensor_parallel.broadcast_data(["max_lengths"], data, torch.int32)["max_lengths"] - - has_video = False # Video path intentionally disabled for this training setup. - has_image = image_token_id is not None and image_token_id in tokens - if not _logged_token_presence_once: - image_token_count = int((tokens == image_token_id).sum().item()) if image_token_id is not None else 0 - vision_start_count = int((tokens == vision_start_token_id).sum().item()) if vision_start_token_id is not None else 0 - print_rank_0( - f"[DEBUG TRAIN TOKENS] image_token_id={image_token_id} count={image_token_count}, " - f"vision_start_token_id={vision_start_token_id} count={vision_start_count}, " - f"first_32_ids={tokens[0, :32].tolist()}" - ) - _logged_token_presence_once = True - print(f" has_image token in batch: {has_image}") - print(f" has_video token in batch: {has_video}") + + image_token_id, video_token_id = _get_vision_token_ids() + has_video = video_token_id in tokens + has_image = image_token_id in tokens thw = None video_grid_thw = None imgs = None @@ -263,10 +148,6 @@ def get_batch(data_iterator): if has_image: imgs = tensor_parallel.broadcast_data(["imgs"], data, torch.float32)["imgs"] thw = tensor_parallel.broadcast_data(["image_grid_thw"], data, torch.int32)["image_grid_thw"] - print(f" Broadcasted imgs: {imgs.shape if imgs is not None else 'None'}") - print(f" Broadcasted image_grid_thw: {thw.shape if thw is not None else 'None'}") - else: - print(" No image tokens in this packed batch (imgs not forwarded to model).") if has_video: pixel_values_videos = tensor_parallel.broadcast_data( ["pixel_values_videos"], @@ -277,10 +158,8 @@ def get_batch(data_iterator): data, torch.int32)["video_grid_thw"] - print("=" * 80) - packed_seq_params = None - is_video = False + is_video = video_token_id in tokens attn_mask_type = AttnMaskType.padding_causal if attn_mask.any() else AttnMaskType.causal @@ -309,21 +188,6 @@ def get_batch(data_iterator): labels = get_inputs_on_this_cp_rank(labels.transpose(0, 1)).transpose(0, 1) loss_mask = get_inputs_on_this_cp_rank(loss_mask.transpose(0, 1)).transpose(0, 1) - _big_debug_step += 1 - _log_big_batch_debug( - step_id=_big_debug_step, - tokens=tokens, - labels=labels, - loss_mask=loss_mask, - attn_mask=attn_mask, - imgs=imgs, - image_grid_thw=thw, - cu_lengths=cu_lengths, - max_lengths=max_lengths, - has_image=bool(has_image), - has_video=bool(has_video), - ) - # TODO attn_mask_type = AttnMaskType.causal attn_mask = None diff --git a/aiak_training_llm/train/training_utils.py b/aiak_training_llm/train/training_utils.py index 10211a19..bd5273cf 100644 --- a/aiak_training_llm/train/training_utils.py +++ b/aiak_training_llm/train/training_utils.py @@ -459,7 +459,12 @@ def setup_model_and_optimizer(model_provider_func, print_rank_0(f'[DEBUG] FastViT enabled: use_fastvit={getattr(args, "use_fastvit", False)}') print_rank_0(f'[DEBUG] Before handling: args.load={args.load}, args.pretrained_checkpoint={args.pretrained_checkpoint}') - if args.pretrained_checkpoint is not None: + if args.load is not None: + # Keep explicit resume checkpoints. If the resume directory does not + # contain a checkpoint yet, Megatron can still fall back to + # args.pretrained_checkpoint below. + print_rank_0(f'FastViT enabled: Will resume/load checkpoint from: {args.load}') + elif args.pretrained_checkpoint is not None: # We have a pretrained checkpoint - load only vision tower from HF checkpoint print_rank_0(f'FastViT enabled: Will load vision tower from pretrained checkpoint: {args.pretrained_checkpoint}') # Keep args.pretrained_checkpoint for vision tower loading diff --git a/apex/csrc/fused_dense.cpp b/apex/csrc/fused_dense.cpp index 7e99b460..59c4af06 100644 --- a/apex/csrc/fused_dense.cpp +++ b/apex/csrc/fused_dense.cpp @@ -27,12 +27,12 @@ at::Tensor linear_bias_forward(at::Tensor input, at::Tensor weight, at::Tensor b //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); // create output/workspace tensor - auto out = at::empty({batch_size, out_features}, input.type()); - //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + auto out = at::empty({batch_size, out_features}, input.options()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].options()); // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB - auto lt_workspace = at::empty({1 << 22}, input.type()); + auto lt_workspace = at::empty({1 << 22}, input.options()); - AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_bias_forward", [&] { + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.scalar_type(), "linear_bias_forward", [&] { scalar_t* w_ptr = weight.data_ptr(); scalar_t* b_ptr = bias.data_ptr(); auto result = linear_bias_forward_cuda( @@ -61,18 +61,18 @@ std::vector linear_bias_backward(at::Tensor input, at::Tensor weight //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); // create output/workspace tensor - auto d_weight = at::empty({out_features, in_features}, input.type()); + auto d_weight = at::empty({out_features, in_features}, input.options()); #if defined(CUBLAS_VERSION) && CUBLAS_VERSION < 11600 auto d_bias = d_output.view({-1, out_features}).sum(0, false); #else - auto d_bias = at::empty({out_features}, input.type()); + auto d_bias = at::empty({out_features}, input.options()); #endif - auto d_input = at::empty({batch_size, in_features}, input.type()); - //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + auto d_input = at::empty({batch_size, in_features}, input.options()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].options()); // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB - auto lt_workspace = at::empty({1 << 22}, input.type()); + auto lt_workspace = at::empty({1 << 22}, input.options()); - AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_bias_backward", [&] { + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.scalar_type(), "linear_bias_backward", [&] { scalar_t* w_ptr = weight.data_ptr(); scalar_t* d_b_ptr = d_bias.data_ptr(); auto result = linear_bias_backward_cuda( @@ -103,14 +103,14 @@ std::vector linear_gelu_linear_forward(at::Tensor input, at::Tensor //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); // create output/workspace tensor - auto output1 = at::empty({batch_size, hidden_features}, input.type()); - auto gelu_in = at::empty({batch_size, hidden_features}, input.type()); - auto output2 = at::empty({batch_size, out_features}, input.type()); - //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + auto output1 = at::empty({batch_size, hidden_features}, input.options()); + auto gelu_in = at::empty({batch_size, hidden_features}, input.options()); + auto output2 = at::empty({batch_size, out_features}, input.options()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].options()); // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB - auto lt_workspace = at::empty({1 << 22}, input.type()); + auto lt_workspace = at::empty({1 << 22}, input.options()); - AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_gelu_linear_forward", [&] { + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.scalar_type(), "linear_gelu_linear_forward", [&] { scalar_t* w1_ptr = weight1.data_ptr(); scalar_t* b1_ptr = bias1.data_ptr(); scalar_t* w2_ptr = weight2.data_ptr(); @@ -146,17 +146,17 @@ std::vector linear_gelu_linear_backward(at::Tensor input, at::Tensor //auto reserved_size = get_mlp_reserved_space(batch_size, num_layers, output_features.data()); // create output/workspace tensor - auto d_weight1 = at::empty({hidden_features, in_features}, input.type()); - auto d_weight2 = at::empty({out_features, hidden_features}, input.type()); - auto d_bias1 = at::empty({hidden_features}, input.type()); - auto d_bias2 = at::empty({out_features}, input.type()); - auto d_input = at::empty({batch_size, in_features}, input.type()); - auto d_output1 = at::empty({batch_size, hidden_features}, input.type()); - //auto reserved_space = at::empty({reserved_size}, inputs[0].type()); + auto d_weight1 = at::empty({hidden_features, in_features}, input.options()); + auto d_weight2 = at::empty({out_features, hidden_features}, input.options()); + auto d_bias1 = at::empty({hidden_features}, input.options()); + auto d_bias2 = at::empty({out_features}, input.options()); + auto d_input = at::empty({batch_size, in_features}, input.options()); + auto d_output1 = at::empty({batch_size, hidden_features}, input.options()); + //auto reserved_space = at::empty({reserved_size}, inputs[0].options()); // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB - auto lt_workspace = at::empty({1 << 22}, input.type()); + auto lt_workspace = at::empty({1 << 22}, input.options()); - AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.type(), "linear_bias_backward", [&] { + AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, input.scalar_type(), "linear_bias_backward", [&] { //scalar_t* w_ptr = weight.data_ptr(); //scalar_t* d_b_ptr = d_bias.data_ptr(); diff --git a/apex/csrc/mlp.cpp b/apex/csrc/mlp.cpp index 830d6062..1f983bbd 100644 --- a/apex/csrc/mlp.cpp +++ b/apex/csrc/mlp.cpp @@ -61,12 +61,12 @@ std::vector mlp_forward(int use_bias, int activation, std::vector(reserved_size)}, inputs[0].type()); + auto out = at::empty({batch_size, output_features.back()}, inputs[0].scalar_type()); + auto reserved_space = at::empty({static_cast(reserved_size)}, inputs[0].scalar_type()); // allocate fixed 4MB workspace for cublaslt for now, and this gets at least 4 MB - auto lt_workspace = at::empty({1 << 22}, inputs[0].type()); + auto lt_workspace = at::empty({1 << 22}, inputs[0].scalar_type()); - AT_DISPATCH_FLOATING_TYPES_AND_HALF(inputs[0].type(), "mlp_forward", [&] { + AT_DISPATCH_FLOATING_TYPES_AND_HALF(inputs[0].scalar_type(), "mlp_forward", [&] { std::vector w_ptr; std::vector b_ptr; for (int i = 0; i < num_layers; i++) { @@ -118,10 +118,10 @@ std::vector mlp_backward( // create outputs, length of inputs std::vector outputs; for (int i = 0; i < inputs.size(); i++) { - outputs.push_back(at::empty(inputs[i].sizes(), inputs[i].type())); // clone for testing now + outputs.push_back(at::empty(inputs[i].sizes(), inputs[i].options())); // clone for testing now } - AT_DISPATCH_FLOATING_TYPES_AND_HALF(inputs[0].type(), "mlp_backward", [&] { + AT_DISPATCH_FLOATING_TYPES_AND_HALF(inputs[0].scalar_type(), "mlp_backward", [&] { std::vector w_ptr; for (int i = 0; i < num_layers; i++) { w_ptr.push_back(inputs[i + 1].data_ptr()); @@ -135,7 +135,7 @@ std::vector mlp_backward( get_mlp_bp_workspace_in_bytes(batch_size, num_layers, output_features.data()); // auto work_space = at::empty({work_size*4}, at::kByte); - auto work_space = at::empty({static_cast(work_size / sizeof(scalar_t))}, inputs[0].type()); + auto work_space = at::empty({static_cast(work_size / sizeof(scalar_t))}, inputs[0].scalar_type()); auto result = mlp_bp( inputs[0].data_ptr(), diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index 16bce8e3..91582e14 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -13,8 +13,8 @@ TP="${1:-1}" # Tensor parallel PP="${2:-1}" # Pipeline parallel SEQ_LEN="${3:-32768}" # Sequence length (reduced for testing) MBS="${4:-1}" # Micro batch size -GBS="${5:-8}" # Global batch size (4 examples for testing) -NSTEP="${6:-20}" # Number of training iterations (1 step with 4 examples) +GBS="${5:-4}" # Global batch size (for TP=1, PP=1, 8 GPUs -> DP=8) +NSTEP="${6:-100}" # Number of training iterations (1 step with 4 examples) # Data paths - UPDATE THESE FOR YOUR SETUP DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" @@ -80,8 +80,9 @@ mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=${GPUS_PER_NODE:-8} +GPUS_PER_NODE=${GPUS_PER_NODE:-4} MASTER_PORT=${MASTER_PORT:-26000} +SAVE_INTERVAL=${SAVE_INTERVAL:-1} if [[ $SINGLE_NODE -eq 1 ]]; then DISTRIBUTED_ARGS=( @@ -158,9 +159,10 @@ TRAINING_ARGS=( --lr-warmup-fraction 0.002 --initial-loss-scale 65536 --bf16 + --load "$SAVE_CKPT_PATH" --save "$SAVE_CKPT_PATH" # --save-interval 2000 - --save-interval 1 + --save-interval "$SAVE_INTERVAL" --ckpt-format torch --dataloader-save "${SAVE_CKPT_PATH}/dataloader" --ckpt-fully-parallel-load @@ -169,6 +171,13 @@ TRAINING_ARGS=( --recompute-num-layers 3 # Must divide evenly into 15 layers (15/3=5 chunks) ) +if [[ "${NO_LOAD_OPTIM_RNG:-0}" == "1" ]]; then + TRAINING_ARGS+=( + --no-load-optim + --no-load-rng + ) +fi + # ======================================== # MODEL PARALLELISM CONFIGURATION # ======================================== diff --git a/inference_fastvlm.py b/inference_fastvlm.py new file mode 100644 index 00000000..b094d59d --- /dev/null +++ b/inference_fastvlm.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +""" +FastVLM End-to-End Inference Test +Tests FastViT + Adapter + MobileLLM-R1-140M pipeline. + +Run with a single GPU (no torchrun needed, handles dist init internally): + CUDA_VISIBLE_DEVICES=0 /home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin/python3 \ + inference_fastvlm.py \ + [--image /path/to/image.jpg] \ + [--prompt "Describe what you see."] \ + [--checkpoint stage_1_alignment_mobilellm_140m/iter_0000019] \ + [--max-new-tokens 150] +""" + +# ────────────────────────────────────────────────────────────────────────────── +# Step 0 — Handle sys.argv BEFORE any imports that touch argparse / Megatron. +# We split our inference args out of argv and rebuild argv for Megatron. +# ────────────────────────────────────────────────────────────────────────────── +import sys +import os +import argparse + +REPO_ROOT = os.path.dirname(os.path.abspath(__file__)) + +# ── Our inference args ──────────────────────────────────────────────────────── +_inf_parser = argparse.ArgumentParser(add_help=False) +_inf_parser.add_argument("--image", type=str, default=None, + help="Path to image file (PNG/JPG). Defaults to random noise.") +_inf_parser.add_argument("--prompt", type=str, + default="<|image_pad|>\nDescribe what you see in this image.", + help="Prompt. Keep <|image_pad|> where the image should go.") +_inf_parser.add_argument("--checkpoint", type=str, default=None, + help="Megatron checkpoint dir. Defaults to latest stage-1 ckpt.") +_inf_parser.add_argument("--max-new-tokens", type=int, default=150) +_inf_parser.add_argument("--temperature", type=float, default=1.0) +our_args, _ = _inf_parser.parse_known_args() + +# ── Resolve checkpoint path ─────────────────────────────────────────────────── +_STAGE1_CKPT = os.path.join(REPO_ROOT, "stage_1_alignment_mobilellm_140m/iter_0000019") +_MERGED_CKPT = os.path.join(REPO_ROOT, "checkpoints/mobilellm-fastvit-merged-tp1-pp1/release") + +if our_args.checkpoint: + _LOAD_CKPT = our_args.checkpoint if os.path.isabs(our_args.checkpoint) \ + else os.path.join(REPO_ROOT, our_args.checkpoint) +elif os.path.exists(_STAGE1_CKPT): + _LOAD_CKPT = _STAGE1_CKPT +else: + _LOAD_CKPT = _MERGED_CKPT + +_DATA_PATH = os.path.join(REPO_ROOT, "data/LLaVA-558K-Webdataset") + +# ── Rebuild sys.argv for Megatron argument parser ──────────────────────────── +# (Megatron parses sys.argv directly via argparse) +sys.argv = [ + sys.argv[0], + # ── model ────────────────────────────────────────────────────────────── + "--model-name", "llava-ov-mobilellm-140m", + # ── tokenizer ────────────────────────────────────────────────────────── + "--tokenizer-type", "HFTokenizer", + "--hf-tokenizer-path", "facebook/MobileLLM-R1-140M", + # ── vision ───────────────────────────────────────────────────────────── + "--use-fastvit", + "--fastvit-image-size", "1024", + "--vision-tower-name", "mobileclip_l_1024", + "--image-aspect-ratio", "pad", + # ── training phase (sets dataloader-type=external) ───────────────────── + "--training-phase", "sft", + "--chat-template", "llama3", + "--trainable-modules", "adapter", + # ── required training dimensions ─────────────────────────────────────── + "--micro-batch-size", "1", + "--global-batch-size", "1", + "--train-iters", "1", + "--seq-length", "4096", + "--max-position-embeddings", "32768", + # ── architecture ─────────────────────────────────────────────────────── + "--attention-backend", "local", + "--transformer-impl", "local", + "--no-gradient-accumulation-fusion", + "--no-rope-fusion", + "--norm-epsilon", "1e-05", + "--init-method-std", "0.02", + "--training-rice-vl-max-answer-length", "32768", + # ── precision ────────────────────────────────────────────────────────── + "--bf16", + # ── model parallelism ────────────────────────────────────────────────── + "--tensor-model-parallel-size", "1", + "--pipeline-model-parallel-size", "1", + "--use-distributed-optimizer", + "--distributed-backend", "nccl", + # ── checkpoint ───────────────────────────────────────────────────────── + "--pretrained-checkpoint", _LOAD_CKPT, + "--no-load-optim", + "--no-load-rng", + # ── data (dummy — not used for inference) ────────────────────────────── + "--data-path", _DATA_PATH, + "--split", "100,0,0", + "--num-workers", "0", + "--dataloader-type", "external", + # ── logging ──────────────────────────────────────────────────────────── + "--log-interval", "1", +] + +# ────────────────────────────────────────────────────────────────────────────── +# Step 1 — Add repo to path, then import Megatron and model code. +# ────────────────────────────────────────────────────────────────────────────── +sys.path.insert(0, REPO_ROOT) +sys.path.insert(0, os.path.join(REPO_ROOT, "aiak_megatron")) + +# Single-GPU distributed bootstrap (needed before Megatron init) +os.environ.setdefault("RANK", "0") +os.environ.setdefault("LOCAL_RANK", "0") +os.environ.setdefault("WORLD_SIZE", "1") +os.environ.setdefault("MASTER_ADDR", "127.0.0.1") +os.environ.setdefault("MASTER_PORT", "29600") + +import torch +import numpy as np +from PIL import Image + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 2 — Initialize Megatron (parses sys.argv, inits distributed, tokenizer…) +# ────────────────────────────────────────────────────────────────────────────── +def megatron_init(): + from aiak_training_llm.train.arguments import ( + aiak_extra_train_args_provider, + validate_aiak_extra_args, + ) + from aiak_training_llm.utils.initialize import ( + parse_arguments, + initialize_aiak_megatron, + ) + # parse_arguments sets global Megatron args; validate_aiak_extra_args + # also applies MobileLLM config from the model registry. + args = parse_arguments( + extra_args_provider=aiak_extra_train_args_provider, + validate_extra_args_provider=validate_aiak_extra_args, + ) + initialize_aiak_megatron(args) + return args + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 3 — Build model with the existing provider (same as training) +# ────────────────────────────────────────────────────────────────────────────── +def build_model(): + from megatron.training.training import get_model + from aiak_training_llm.models import get_model_provider, get_model_family + from aiak_training_llm.utils import get_args + + args = get_args() + model_family = get_model_family(args.model_name) + provider = get_model_provider(model_family) + + # get_model wraps provider in DDP etc., returns a list of model chunks + models = get_model(provider, model_type=None, wrap_with_ddp=False) + # Unwrap list/tuple; for PP=1 there is exactly one chunk + model = models[0] if isinstance(models, (list, tuple)) else models + return model + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 4 — Load checkpoint weights into the model +# ────────────────────────────────────────────────────────────────────────────── +def load_weights(model, ckpt_dir: str): + """ + Direct weight copy from a Megatron checkpoint. + Supports both the merged pre-train checkpoint and stage-1 checkpoints. + """ + ckpt_file = os.path.join(ckpt_dir, "mp_rank_00", "model_optim_rng.pt") + if not os.path.exists(ckpt_file): + raise FileNotFoundError(f"Checkpoint not found: {ckpt_file}") + + print(f"\n[ckpt] Loading weights from:\n {ckpt_file}") + raw = torch.load(ckpt_file, map_location="cpu", weights_only=False) + saved = raw["model"] + + # Stage-1 checkpoints have 'language_model.*' prefix. + # Merged pre-train has top-level keys without that prefix. + has_lm_prefix = any(k.startswith("language_model.") for k in saved) + + def remap(k): + if has_lm_prefix: + return k + if k.startswith(("embedding.", "decoder.", "output_layer.")): + return "language_model." + k + return k + + remapped = {remap(k): v for k, v in saved.items()} + + model_sd = model.state_dict() + loaded_n = 0 + skipped = [] + missing = [] + + for name, param in model_sd.items(): + if name in remapped: + src = remapped[name].to(dtype=param.dtype) + if src.shape == param.shape: + model_sd[name].copy_(src) + loaded_n += 1 + else: + skipped.append(f"{name}: saved {src.shape} vs model {param.shape}") + else: + missing.append(name) + + model.load_state_dict(model_sd, strict=False) + print(f"[ckpt] Loaded : {loaded_n} / {len(model_sd)} tensors") + if missing: + pfx = sorted(set(n.split(".")[0] for n in missing)) + print(f"[ckpt] Missing : {len(missing)} tensors (kept random init) — prefixes: {pfx}") + if skipped: + print(f"[ckpt] Skipped (shape mismatch): {skipped[:5]}") + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 5 — Image preprocessing +# ────────────────────────────────────────────────────────────────────────────── +def preprocess_image(path, size, device, dtype): + if path and os.path.exists(path): + img = Image.open(path).convert("RGB") + print(f"[img] Loaded : {path} original size {img.size}") + else: + if path: + print(f"[img] WARNING: {path!r} not found — using random noise") + arr = np.random.randint(0, 256, (size, size, 3), dtype=np.uint8) + img = Image.fromarray(arr) + print(f"[img] Using random noise image ({size}×{size})") + + img_resized = img.resize((size, size), Image.BICUBIC) + arr = np.array(img_resized, dtype=np.float32) / 255.0 # [H, W, 3] in [0,1] + tensor = torch.from_numpy(arr).permute(2, 0, 1) # [3, H, W] + tensor = tensor.unsqueeze(0).to(device=device, dtype=dtype) + print(f"[img] Tensor : {tuple(tensor.shape)} " + f"min={tensor.min():.3f} max={tensor.max():.3f}") + return tensor + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 6 — Tokenisation +# ────────────────────────────────────────────────────────────────────────────── +def tokenise_prompt(prompt, tokenizer, image_token_id, device): + # tokenizer is the AutoTokenizerFromHF wrapper; use inner HF tokenizer for encode/decode + inner = tokenizer.tokenizer + ids = inner.encode(prompt, add_special_tokens=True) + input_ids = torch.tensor(ids, dtype=torch.long, device=device).unsqueeze(0) + n_img = (input_ids == image_token_id).sum().item() + print(f"[tok] Prompt tokens : {input_ids.shape[1]} | image placeholders : {n_img}") + print(f"[tok] Decoded : {inner.decode(ids)!r}") + return input_ids + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 7 — Dimension diagnostic (one forward pass, no generation) +# ────────────────────────────────────────────────────────────────────────────── +@torch.no_grad() +def dimension_check(model, images, input_ids, tokenizer, image_token_id): + print("\n" + "═" * 68) + print(" DIMENSION CHECK") + print("═" * 68) + + # Monkey-patch feature_select to print the raw spatial shape before pooling + vt = model.vision_model.vision_tower + _orig = vt.feature_select + def _patched(out): + raw = out["image_embeddings"] + B, C, H, W = raw.shape + print(f" [dim] FastViT spatial (pre-pool) : {(B, C, H, W)} " + f"= {B} images × {H*W} patches × {C}-dim") + result = _orig(out) + print(f" [dim] After global avg pool : {tuple(result.shape)} " + f"= {result.shape[0]} images × {result.shape[1]}-dim") + return result + vt.feature_select = _patched + + seq_len = input_ids.shape[1] + pos_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0) + + logits = model( + images=images, + image_grid_thw=None, + input_ids=input_ids, + position_ids=pos_ids, + attention_mask=None, + attn_mask_type=None, + labels=None, + packed_seq_params=None, + ) + + # Normalise shape to [batch, seq, vocab] + if logits.dim() == 3 and logits.shape[0] != input_ids.shape[0]: + logits = logits.transpose(0, 1).contiguous() + + print(f"\n Final logits : {tuple(logits.shape)}") + + # Top-5 predicted next tokens + top5 = logits[0, -1, :].topk(5) + print(f"\n Top-5 predicted next tokens (after prompt):") + _inner = tokenizer.tokenizer # AutoTokenizerFromHF → inner HF tokenizer + for val, idx in zip(top5.values.tolist(), top5.indices.tolist()): + tok_text = _inner.decode([idx], skip_special_tokens=False) + print(f" token_id={idx:6d} logit={val:8.3f} decoded={tok_text!r}") + + vt.feature_select = _orig # restore + return logits + + +# ────────────────────────────────────────────────────────────────────────────── +# Step 8 — Autoregressive generation (greedy, full recompute — no KV cache) +# ────────────────────────────────────────────────────────────────────────────── +@torch.no_grad() +def generate(model, images, input_ids, tokenizer, image_token_id, + max_new_tokens=150, temperature=1.0): + """ + Greedy decode without KV cache. + + On every step we pass `images` so the model correctly substitutes + image token positions with visual features (those positions persist in + input_ids throughout generation). This is correct but slow; it is + fine for a pipeline sanity-check. + """ + eos_ids = {tokenizer.eos_token_id} + # Llama3 / MobileLLM may have multiple EOS ids + if hasattr(tokenizer, "eos_token_ids"): + eos_ids |= set(tokenizer.eos_token_ids) + + generated_ids = [] + print(f"\n[gen] Generating up to {max_new_tokens} tokens …") + print("─" * 60) + print("RESPONSE: ", end="", flush=True) + + for step in range(max_new_tokens): + seq_len = input_ids.shape[1] + pos_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0) + + logits = model( + images=images, # always pass images (for correct visual sub.) + image_grid_thw=None, + input_ids=input_ids, + position_ids=pos_ids, + attention_mask=None, + attn_mask_type=None, + labels=None, + packed_seq_params=None, + ) + + # Normalise to [batch, seq, vocab] + if logits.dim() == 3 and logits.shape[0] != 1: + logits = logits.transpose(0, 1).contiguous() + + last_logit = logits[0, -1, :] # [vocab] + if temperature != 1.0: + last_logit = last_logit / temperature + + next_id = int(last_logit.argmax()) + generated_ids.append(next_id) + + tok_text = tokenizer.decode([next_id], skip_special_tokens=False) + print(tok_text, end="", flush=True) + + if next_id in eos_ids: + print() + break + + next_tensor = torch.tensor([[next_id]], dtype=torch.long, device=input_ids.device) + input_ids = torch.cat([input_ids, next_tensor], dim=1) + + print("\n" + "─" * 60) + return generated_ids + + +# ────────────────────────────────────────────────────────────────────────────── +# Main +# ────────────────────────────────────────────────────────────────────────────── +def main(): + print("\n" + "═" * 68) + print(" FastVLM Inference — FastViT + Adapter + MobileLLM-R1-140M") + print("═" * 68 + "\n") + + # 1. Init Megatron + print("[init] Initialising Megatron (single GPU mode) …") + args = megatron_init() + print(f"[init] Done. dtype=bfloat16, TP=1, PP=1, device=cuda:0") + + device = torch.device("cuda:0") + dtype = torch.bfloat16 + + # 2. Tokenizer (initialised by Megatron, just fetch it) + from aiak_training_llm.utils import get_tokenizer + tokenizer = get_tokenizer() + # AutoTokenizerFromHF wraps the inner HF tokenizer in .tokenizer + _inner_tok = tokenizer.tokenizer + IMAGE_TOKEN_ID = tokenizer.convert_tokens_to_ids("<|image_pad|>") + EOS_ID = _inner_tok.eos_token_id + print(f"[tok] image_token_id = {IMAGE_TOKEN_ID} (eos={EOS_ID})") + + # 3. Build model + print("\n[model] Building LlavaOnevision1_5 (FastViT + Adapter + MobileLLM) …") + model = build_model() + model = model.to(device=device, dtype=dtype) + model.eval() + for p in model.parameters(): + p.requires_grad_(False) + + total = sum(p.numel() for p in model.parameters()) / 1e6 + print(f"[model] Total parameters : {total:.1f}M") + + # 4. Load checkpoint + load_weights(model, _LOAD_CKPT) + print(f"[ckpt] Source: {_LOAD_CKPT}") + + # 5. Image + images = preprocess_image(our_args.image, 1024, device, dtype) + + # 6. Prompt + prompt = our_args.prompt + if "<|image_pad|>" not in prompt: + prompt = "<|image_pad|>\n" + prompt + print("[tok] Prepended <|image_pad|> to prompt") + input_ids = tokenise_prompt(prompt, tokenizer, IMAGE_TOKEN_ID, device) + _inner_tok_ref = tokenizer.tokenizer + + # 7. Dimension check + _ = dimension_check(model, images, input_ids, tokenizer, IMAGE_TOKEN_ID) + # (dimension_check only uses tokenizer.tokenizer for decode internally) + + # 8. Generation + print("\n" + "═" * 68) + print(" GENERATION") + print("═" * 68) + print(f"PROMPT: {prompt!r}\n") + + gen_ids = generate( + model = model, + images = images, + input_ids = input_ids, + tokenizer = _inner_tok, + image_token_id = IMAGE_TOKEN_ID, + max_new_tokens = our_args.max_new_tokens, + temperature = our_args.temperature, + ) + + response = _inner_tok.decode(gen_ids, skip_special_tokens=True) + print(f"\nFull decoded response:\n {response!r}") + + # 9. Summary + print("\n" + "═" * 68) + print(" SUMMARY") + print("═" * 68) + print(f" Checkpoint : {_LOAD_CKPT}") + print(f" Image : {our_args.image or 'random noise (1024×1024)'}") + print(f" Prompt tokens : {input_ids.shape[1]}") + print(f" Gen tokens : {len(gen_ids)}") + print(f" Vision : FastViT MobileCLIP-L @1024 → global avg pool → 3072-dim") + print(f" Adapter : 3072 → 576 (2-layer MLP, SiLU)") + print(f" Language : MobileLLM-R1-140M (15 layers, 576-dim, GQA 9/3 heads)") + print() + + +if __name__ == "__main__": + main() diff --git a/stage_1_alignment_mobilellm_140m/.gitignore b/stage_1_alignment_mobilellm_140m/.gitignore new file mode 100644 index 00000000..ef7b1f99 --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/.gitignore @@ -0,0 +1,9 @@ +# Exclude all checkpoint binary weights (too large for GitHub without LFS on fork) +# Checkpoint weights are stored on the training server at: +# /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/stage_1_alignment_mobilellm_140m/iter_0000500/ +iter_*/ + +# Exclude large binary / ephemeral directories +wandb/ +tensorboard/ +dataloader/ diff --git a/stage_1_alignment_mobilellm_140m/latest_checkpointed_iteration.txt b/stage_1_alignment_mobilellm_140m/latest_checkpointed_iteration.txt new file mode 100644 index 00000000..eb1f4948 --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/latest_checkpointed_iteration.txt @@ -0,0 +1 @@ +500 \ No newline at end of file diff --git a/stage_1_alignment_mobilellm_140m/latest_wandb_artifact_path.txt b/stage_1_alignment_mobilellm_140m/latest_wandb_artifact_path.txt new file mode 100644 index 00000000..4ba41ef3 --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/latest_wandb_artifact_path.txt @@ -0,0 +1 @@ +rana-zayed-mbzuai/llava-ov-1_5 \ No newline at end of file diff --git a/stage_1_alignment_mobilellm_140m/launch_100_gpu4.out b/stage_1_alignment_mobilellm_140m/launch_100_gpu4.out new file mode 100644 index 00000000..e69de29b diff --git a/stage_1_alignment_mobilellm_140m/launch_500_wandb_gpu4.sh b/stage_1_alignment_mobilellm_140m/launch_500_wandb_gpu4.sh new file mode 100644 index 00000000..220bc780 --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/launch_500_wandb_gpu4.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5 + +export PATH="/home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin:${PATH}" +export CUDA_VISIBLE_DEVICES=4 +export GPUS_PER_NODE=1 +export MASTER_PORT=26050 +export SAVE_INTERVAL=50 +export NO_LOAD_OPTIM_RNG=1 + +if [[ -z "${WANDB_API_KEY:-}" ]]; then + WANDB_API_KEY="$( + sed -n 's/^export WANDB_API_KEY="\([^"]*\)".*/\1/p' Stage1/alignment.sh | head -n 1 + )" + export WANDB_API_KEY +fi + +export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" +export WANDB_NAME="${WANDB_NAME:-mobilellm_integration}" + +exec bash examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh \ + 1 1 32768 1 4 500 diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log new file mode 100644 index 00000000..1a1667eb --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log @@ -0,0 +1,8137 @@ +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] +W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] ***************************************** +W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] ***************************************** +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +False +False +False +False +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +parse arguments function called +parse arguments function called +parse arguments function called +parse arguments function called +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +-------------- Configure model to llava-ov-mobilellm-140m -------------- +[DEBUG] Model config type: LlavaOnevision1_5Config +[DEBUG] Is MobileLLM: True + num_layers = 15 + hidden_size = 576 + ffn_hidden_size = 2048 + num_attention_heads = 9 + group_query_attention = True + num_query_groups = 3 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = False + vocab_size_in_config_file = 128256 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-05 + rotary_base = 8000000 + kv_channels = None + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 4, data-parallel size: 4, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.local + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... llama3 + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 4 + data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 15 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 32768 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + fastvit_image_size .............................. 1024 + ffn_hidden_size ................................. 2048 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 4 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M + hidden_dropout .................................. 0 + hidden_size ..................................... 576 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_aspect_ratio .............................. pad + image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ None + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 20 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-mobilellm-140m + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 9 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3Using unified model type for training. + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 15 + + num_layers_at_end_in_bf16 ....................... 1 +Starting training... num_layers_at_start_in_bf16 ..................... 1 + + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 3 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 16 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 3 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 8000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_mobilellm_140m + save_ema ........................................ None + save_interval ................................... 1 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 32768 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 20 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 32768Using unified model type for training. + training_rice_vl_max_image_area ................. 1806336 + + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. False + +Starting training... use_checkpoint_args ............................. False + + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_fastvit ..................................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vision_tower_name ............................... mobileclip_l_1024 + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 128256 + vpp_scheduler ................................... None + wandb_exp_name .................................. mobilellm_integration + wandb_project ................................... llava-ov-1_5 + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 4 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +Using unified model type for training. + +Starting training... +Using unified model type for training. + +Starting training... +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> AIAK building HFTokenizer tokenizer ... +> setting tensorboard ... +INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 +WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. +WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. +WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). + > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +wandb: Currently logged in as: rana-zayed (rana-zayed-mbzuai) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: creating run +wandb: Tracking run with wandb version 0.21.0 +wandb: Run data is saved locally in stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_085758-7dva7qn6 +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run mobilellm_integration +wandb: ⭐️ View project at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 +wandb: 🚀 View run at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/7dva7qn6 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank [Gloo] Rank 32 is connected to 3 is connected to peer ranks. Expected number of connected peer ranks is : 33 peer ranks. +Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank [Gloo] Rank 32 is connected to 3 is connected to peer ranks. Expected number of connected peer ranks is : 33 peer ranks. +Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to [Gloo] Rank 3 peer ranks. Expected number of connected peer ranks is : 33 + is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so +make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.080 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W429 08:58:05.082128572 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.933 seconds +time to initialize megatron (seconds): 27.517 +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2026-04-29 08:58:22 +building llava-ov-mobilellm-140m model ... +[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m +[DEBUG PROVIDER] Args num_layers: 15 +[DEBUG PROVIDER] Args hidden_size: 576 +[DEBUG PROVIDER] Args vocab_size: 128260 +[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 +[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 +[DEBUG PROVIDER] Model family: llava_ov_1_5 +[DEBUG PROVIDER] use_mobilellm flag: True +[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone +[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 +[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Loading vision config... +[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 +[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json +[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd +[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ================================================ +[DEBUG PROVIDER] Loading adapter config... +[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 +[DEBUG PROVIDER] Building layer specs... +[DEBUG PROVIDER] ✓ Using MobileLLM layer specification +[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] MobileLLM layer spec created successfully +[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... +[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 +[INFO] Using MobileLLM as language backbone +[INFO] Using MobileLLM as language backbone +[INFO] Using MobileLLM as language backbone +[INFO] Using MobileLLM as language backbone +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (11216448 elements, 11216512 padded size): + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias + module.adapter.layernorm.bias + module.adapter.layernorm.weight +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[DEBUG] FastViT enabled: use_fastvit=True +[DEBUG] Before handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +FastViT enabled: Will load vision tower from pretrained checkpoint: /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +[DEBUG] After handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +Checkpoint file not found in load directory None attempting to finetune with checkpoint in /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + loading release checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +could not find arguments in the checkpoint ... +Loading checkpoint with device: cuda:0, strict=False +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel + checkpoint version 3.0 + successfully loaded checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 [ t 1/1, p 1/1 ] at iteration 0 + failed to find checkpoint /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1/release/mp_rank_00/model_optim_rng.pt in wandb +(min, max) time across ranks (ms): + load-checkpoint ................................: (3234.68, 3235.04) +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 08:58:28 +================================================================================ +FULL MODEL STRUCTURE: +================================================================================ + +--- Model 0 (pipeline rank 0) --- +DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +) +================================================================================ + +================================================================================ +PARAMETER TRAINABILITY STATUS: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +================================================================================ +Total trainable parameters: 11,216,448 (11.22M) +Total frozen parameters: 265,417,440 (265.42M) +Total parameters: 276,633,888 (276.63M) +Trainable percentage: 4.05% +================================================================================ +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 80 + validation: 400 + test: 400 +Using shared training tokenizer in FastVLM mode +Ensured multimodal special tokens for FastVLM tokenizer; added=0 +Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M +Initialized FastViT processor with image_size=1024 +Processor config: .TokenizerWrapper object at 0x7f762c463e20> +image_resolution: None +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 375), ] sum(count)=375 +rank=0, worker=1: shard_range=[pretrain-000003.tar[375, 750), ] sum(count)=375 +rank=0, worker=2: shard_range=[pretrain-000003.tar[750, 1125), ] sum(count)=375 +rank=0, worker=3: shard_range=[pretrain-000003.tar[1125, 1500), ] sum(count)=375 +rank=0, worker=4: shard_range=[pretrain-000003.tar[1500, 1875), ] sum(count)=375 +rank=0, worker=5: shard_range=[pretrain-000003.tar[1875, 2250), ] sum(count)=375 +rank=0, worker=6: shard_range=[pretrain-000003.tar[2250, 2625), ] sum(count)=375 +rank=0, worker=7: shard_range=[pretrain-000003.tar[2625, 2999), ] sum(count)=374 +rank=0, worker=8: shard_range=[pretrain-000003.tar[2999, 3374), ] sum(count)=375 +rank=0, worker=9: shard_range=[pretrain-000003.tar[3374, 3749), ] sum(count)=375 +rank=0, worker=10: shard_range=[pretrain-000003.tar[3749, 4124), ] sum(count)=375 +rank=0, worker=11: shard_range=[pretrain-000003.tar[4124, 4498), ] sum(count)=374 +rank=0, worker=12: shard_range=[pretrain-000003.tar[4498, 4873), ] sum(count)=375 +rank=0, worker=13: shard_range=[pretrain-000003.tar[4873, 5058), pretrain-000001.tar[0, 190), ] sum(count)=375 +rank=0, worker=14: shard_range=[pretrain-000001.tar[190, 565), ] sum(count)=375 +rank=0, worker=15: shard_range=[pretrain-000001.tar[565, 939), ] sum(count)=374 +rank=3, worker=0: shard_range=[pretrain-000004.tar[2858, 3233), ] sum(count)=375 +rank=3, worker=1: shard_range=[pretrain-000004.tar[3233, 3608), ] sum(count)=375 +rank=3, worker=2: shard_range=[pretrain-000004.tar[3608, 3821), pretrain-000000.tar[0, 162), ] sum(count)=375 +rank=3, worker=3: shard_range=[pretrain-000000.tar[162, 537), ] sum(count)=375 +rank=3, worker=4: shard_range=[pretrain-000000.tar[537, 912), ] sum(count)=375 +rank=3, worker=5: shard_range=[pretrain-000000.tar[912, 1287), ] sum(count)=375 +rank=3, worker=6: shard_range=[pretrain-000000.tar[1287, 1662), ] sum(count)=375 +rank=3, worker=7: shard_range=[pretrain-000000.tar[1662, 2036), ] sum(count)=374 +rank=3, worker=8: shard_range=[pretrain-000000.tar[2036, 2411), ] sum(count)=375 +rank=3, worker=9: shard_range=[pretrain-000000.tar[2411, 2786), ] sum(count)=375 +rank=3, worker=10: shard_range=[pretrain-000000.tar[2786, 3161), ] sum(count)=375 +rank=3, worker=11: shard_range=[pretrain-000000.tar[3161, 3535), ] sum(count)=374rank=2, worker=0: shard_range=[pretrain-000002.tar[1900, 2275), ] sum(count)=375 + +rank=2, worker=1: shard_range=[pretrain-000002.tar[2275, 2650), ] sum(count)=375 +rank=2, worker=2: shard_range=[pretrain-000002.tar[2650, 3025), ] sum(count)=375rank=3, worker=12: shard_range=[pretrain-000000.tar[3535, 3910), ] sum(count)=375 +rank=2, worker=3: shard_range=[pretrain-000002.tar[3025, 3400), ] sum(count)=375 + +rank=2, worker=4: shard_range=[pretrain-000002.tar[3400, 3775), ] sum(count)=375 +rank=2, worker=5: shard_range=[pretrain-000002.tar[3775, 4150), ] sum(count)=375 +rank=2, worker=6: shard_range=[pretrain-000002.tar[4150, 4525), ] sum(count)=375rank=3, worker=13: shard_range=[pretrain-000000.tar[3910, 4285), ] sum(count)=375 +rank=2, worker=7: shard_range=[pretrain-000002.tar[4525, 4899), ] sum(count)=374 + +rank=2, worker=8: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 235), ] sum(count)=375 +rank=2, worker=9: shard_range=[pretrain-000004.tar[235, 610), ] sum(count)=375 +rank=3, worker=14: shard_range=[pretrain-000000.tar[4285, 4660), ] sum(count)=375rank=2, worker=10: shard_range=[pretrain-000004.tar[610, 985), ] sum(count)=375 +rank=2, worker=11: shard_range=[pretrain-000004.tar[985, 1359), ] sum(count)=374 + +rank=2, worker=12: shard_range=[pretrain-000004.tar[1359, 1734), ] sum(count)=375 +rank=2, worker=13: shard_range=[pretrain-000004.tar[1734, 2109), ] sum(count)=375rank=3, worker=15: shard_range=[pretrain-000000.tar[4660, 5034), ] sum(count)=374 +rank=2, worker=14: shard_range=[pretrain-000004.tar[2109, 2484), ] sum(count)=375 + +rank=2, worker=15: shard_range=[pretrain-000004.tar[2484, 2858), ] sum(count)=374 +rank=1, worker=0: shard_range=[pretrain-000001.tar[939, 1314), ] sum(count)=375 +rank=1, worker=1: shard_range=[pretrain-000001.tar[1314, 1689), ] sum(count)=375 +rank=1, worker=2: shard_range=[pretrain-000001.tar[1689, 2064), ] sum(count)=375 +rank=1, worker=3: shard_range=[pretrain-000001.tar[2064, 2439), ] sum(count)=375 +rank=1, worker=4: shard_range=[pretrain-000001.tar[2439, 2814), ] sum(count)=375 +rank=1, worker=5: shard_range=[pretrain-000001.tar[2814, 3189), ] sum(count)=375 +rank=1, worker=6: shard_range=[pretrain-000001.tar[3189, 3564), ] sum(count)=375 +rank=1, worker=7: shard_range=[pretrain-000001.tar[3564, 3938), ] sum(count)=374 +rank=1, worker=8: shard_range=[pretrain-000001.tar[3938, 4313), ] sum(count)=375 +rank=1, worker=9: shard_range=[pretrain-000001.tar[4313, 4688), ] sum(count)=375 +rank=1, worker=10: shard_range=[pretrain-000001.tar[4688, 5036), pretrain-000002.tar[0, 27), ] sum(count)=375 +rank=1, worker=11: shard_range=[pretrain-000002.tar[27, 401), ] sum(count)=374 +rank=1, worker=12: shard_range=[pretrain-000002.tar[401, 776), ] sum(count)=375 +rank=1, worker=13: shard_range=[pretrain-000002.tar[776, 1151), ] sum(count)=375 +rank=1, worker=14: shard_range=[pretrain-000002.tar[1151, 1526), ] sum(count)=375 +rank=1, worker=15: shard_range=[pretrain-000002.tar[1526, 1900), ] sum(count)=374 +[after dataloaders are built] datetime: 2026-04-29 08:58:28 +done with setup ... +training ...(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (5522.85, 5523.23) + train/valid/test-data-iterators-setup ..........: (94.62, 94.80) + +================================================================================ +training with the following parameter status: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 + +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2026-04-29 08:58:28 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1430]) +Dataloader batch - labels shape: torch.Size([1, 1430]) +Dataloader batch - attn_mask shape: torch.Size([1, 1430]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1757]) +Dataloader batch - labels shape: torch.Size([1, 1757]) +Dataloader batch - attn_mask shape: torch.Size([1, 1757]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1909]) +Dataloader batch - labels shape: torch.Size([1, 1909]) +Dataloader batch - attn_mask shape: torch.Size([1, 1909]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1356]) +Dataloader batch - labels shape: torch.Size([1, 1356]) +Dataloader batch - attn_mask shape: torch.Size([1, 1356]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 532 / 1909 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 256, 3072) + out : (32, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 532 tokens) : 11.7835 + top-1 accuracy : 1/532 = 0.2% + +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +Number of parameters in transformer layers in billions: 0.07 +Number of parameters in embedding layers in billions: 0.07 +Total number of parameters in billions: 0.14 +Number of parameters in most loaded shard in billions: 0.1403 +Theoretical memory footprints: weight and optimizer=1204.55 MB + [2026-04-29 08:58:52] iteration 1/ 20 | consumed samples: 4 | elapsed time per iteration (ms): 24530.1 | throughput (token/sec/GPU): 65.8 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.179645E+01 | loss scale: 1.0 | grad norm: 2.994 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 |[Rank 0] (after 1 iterations) memory (MB) | allocated: 709.1826171875 | max allocated: 5398.64892578125 | reserved: 6388.0 | max reserved: 6388.0 + +saving checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 1 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2495.67, 2499.58) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1781]) +Dataloader batch - labels shape: torch.Size([1, 1781]) +Dataloader batch - attn_mask shape: torch.Size([1, 1781]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + Dataloader batch - tokens shape: torch.Size([1, 1186]) +Dataloader batch - labels shape: torch.Size([1, 1186]) +Dataloader batch - attn_mask shape: torch.Size([1, 1186]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +torch.Size([1, 942]) +Dataloader batch - labels shape:Dataloader batch - tokens shape: torch.Size([1, 1114]) +Dataloader batch - labels shape: torch.Size([1, 1114]) +Dataloader batch - attn_mask shape: torch.Size([1, 1114]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +torch.Size([1, 942]) +Dataloader batch - attn_mask shape: torch.Size([1, 942]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1186) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1186) torch.int64 + loss_mask : 273 / 1186 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 256, 3072) + out : (22, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1186, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1186, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1186, 1, 576) torch.bfloat16 + out : (1, 1186) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 273 tokens) : 11.8206 + top-1 accuracy : 0/273 = 0.0% + + [2026-04-29 08:58:58] iteration 2/ 20 | consumed samples: 8 | elapsed time per iteration (ms): 3313.5 | throughput (token/sec/GPU): 379.0 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.182033E+01 | loss scale: 1.0 | grad norm: 3.165 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 2 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2544.79, 2545.35) +Dataloader batch - tokens shape: torch.Size([1, 1554]) +Dataloader batch - labels shape: torch.Size([1, 1554]) +Dataloader batch - attn_mask shape: torch.Size([1, 1554]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1849]) +Dataloader batch - labels shape: torch.Size([1, 1849]) +Dataloader batch - attn_mask shape: torch.Size([1, 1849]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1399]) +Dataloader batch - labels shape: torch.Size([1, 1399]) +Dataloader batch - attn_mask shape: torch.Size([1, 1399]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1741]) +Dataloader batch - labels shape: torch.Size([1, 1741]) +Dataloader batch - attn_mask shape: torch.Size([1, 1741]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1399) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1399) torch.int64 + loss_mask : 342 / 1399 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 256, 3072) + out : (26, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1399, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1399, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1399, 1, 576) torch.bfloat16 + out : (1, 1399) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.7172 + top-1 accuracy : 0/342 = 0.0% + + [2026-04-29 08:59:03] iteration 3/ 20 | consumed samples: 12 | elapsed time per iteration (ms): 2636.4 | throughput (token/sec/GPU): 620.4 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.168310E+01 | loss scale: 1.0 | grad norm: 1.864 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 3 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2523.37, 2523.95) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1585]) +torch.Size([1, 1544])Dataloader batch - labels shape: torch.Size([1, 1585]) + +Dataloader batch - attn_mask shape:Dataloader batch - labels shape: torch.Size([1, 1585])torch.Size([1, 1544]) + +Dataloader batch - image_grid_thw shape:Dataloader batch - attn_mask shape: torch.Size([1, 1544])torch.Size([29, 3]) + +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1042]) +Dataloader batch - labels shape: torch.Size([1, 1042]) +Dataloader batch - attn_mask shape: torch.Size([1, 1042]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1585) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1585) torch.int64 + loss_mask : 350 / 1585 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 256, 3072) + out : (29, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1585, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1585, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1585, 1, 576) torch.bfloat16 + out : (1, 1585) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.5905 + top-1 accuracy : 0/350 = 0.0% + + [2026-04-29 08:59:08] iteration 4/ 20 | consumed samples: 16 | elapsed time per iteration (ms): 2075.7 | throughput (token/sec/GPU): 676.3 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.163278E+01 | loss scale: 1.0 | grad norm: 1.456 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 4 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (3088.82, 3089.15) +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1412])Dataloader batch - tokens shape: torch.Size([1, 1214]) +Dataloader batch - labels shape: +torch.Size([1, 1214]) +Dataloader batch - attn_mask shape:Dataloader batch - labels shape: torch.Size([1, 1214]) + Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 367 / 1541 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 256, 3072) + out : (28, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.5435 + top-1 accuracy : 0/367 = 0.0% + + [2026-04-29 08:59:13] iteration 5/ 20 | consumed samples: 20 | elapsed time per iteration (ms): 2098.3 | throughput (token/sec/GPU): 638.4 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.153166E+01 | loss scale: 1.0 | grad norm: 1.349 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 5 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2403.65, 2403.98) +Dataloader batch - tokens shape: torch.Size([1, 1344]) +Dataloader batch - labels shape: torch.Size([1, 1344]) +Dataloader batch - attn_mask shape: torch.Size([1, 1344]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1498]) +Dataloader batch - labels shape: torch.Size([1, 1498]) +Dataloader batch - attn_mask shape: torch.Size([1, 1498]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1689]) +Dataloader batch - labels shape: torch.Size([1, 1689]) +Dataloader batch - attn_mask shape: torch.Size([1, 1689]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1344) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1344) torch.int64 + loss_mask : 399 / 1344 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 256, 3072) + out : (22, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1344, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1344, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1344, 1, 576) torch.bfloat16 + out : (1, 1344) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.5155 + top-1 accuracy : 1/399 = 0.3% + + [2026-04-29 08:59:18] iteration 6/ 20 | consumed samples: 24 | elapsed time per iteration (ms): 2397.1 | throughput (token/sec/GPU): 625.5 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.151996E+01 | loss scale: 1.0 | grad norm: 1.014 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 6 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2450.32, 2450.99) +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) +Dataloader batch - tokens shape: Dataloader batch - tokens shape:torch.Size([1, 1352]) +torch.Size([1, 1824])Dataloader batch - labels shape: + Dataloader batch - labels shape:torch.Size([1, 1352]) +torch.Size([1, 1824])Dataloader batch - attn_mask shape: + Dataloader batch - attn_mask shape: torch.Size([1, 1352])torch.Size([1, 1824]) + +Dataloader batch - image_grid_thw shape: torch.Size([32, 3])Dataloader batch - image_grid_thw shape: + torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1352) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1352) torch.int64 + loss_mask : 318 / 1352 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 256, 3072) + out : (25, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1352, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1352, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1352, 1, 576) torch.bfloat16 + out : (1, 1352) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.5107 + top-1 accuracy : 0/318 = 0.0% + + [2026-04-29 08:59:23] iteration 7/ 20 | consumed samples: 28 | elapsed time per iteration (ms): 2798.8 | throughput (token/sec/GPU): 557.4 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.149603E+01 | loss scale: 1.0 | grad norm: 0.963 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 7 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2402.95, 2403.50) +Dataloader batch - tokens shape: torch.Size([1, 1612]) +Dataloader batch - labels shape: torch.Size([1, 1612]) +Dataloader batch - attn_mask shape: torch.Size([1, 1612]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 995]) +Dataloader batch - labels shape: torch.Size([1, 995]) +Dataloader batch - attn_mask shape:torch.Size([1, 1527]) torch.Size([1, 995]) +Dataloader batch - image_grid_thw shape: + torch.Size([18, 3]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1558]) +Dataloader batch - labels shape: torch.Size([1, 1558]) +Dataloader batch - attn_mask shape: torch.Size([1, 1558]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 995) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 995) torch.int64 + loss_mask : 227 / 995 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 256, 3072) + out : (18, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (995, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (995, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (995, 1, 576) torch.bfloat16 + out : (1, 995) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 227 tokens) : 11.4816 + top-1 accuracy : 0/227 = 0.0% + + [2026-04-29 08:59:28] iteration 8/ 20 | consumed samples: 32 | elapsed time per iteration (ms): 2316.3 | throughput (token/sec/GPU): 614.3 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.145122E+01 | loss scale: 1.0 | grad norm: 0.860 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 8 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2311.63, 2311.82) +Dataloader batch - tokens shape: torch.Size([1, 1026]) +Dataloader batch - labels shape: torch.Size([1, 1026]) +Dataloader batch - attn_mask shape: torch.Size([1, 1026]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1529]) +Dataloader batch - labels shape: torch.Size([1, 1529]) +Dataloader batch - attn_mask shape: torch.Size([1, 1529]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1673]) +Dataloader batch - labels shape: torch.Size([1, 1673]) +Dataloader batch - attn_mask shape: torch.Size([1, 1673]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1175]) +Dataloader batch - labels shape: torch.Size([1, 1175]) +Dataloader batch - attn_mask shape: torch.Size([1, 1175]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1026) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1026) torch.int64 + loss_mask : 200 / 1026 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 256, 3072) + out : (20, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1026, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1026, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1026, 1, 576) torch.bfloat16 + out : (1, 1026) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 200 tokens) : 11.3715 + top-1 accuracy : 0/200 = 0.0% + + [2026-04-29 08:59:32] iteration 9/ 20 | consumed samples: 36 | elapsed time per iteration (ms): 1970.5 | throughput (token/sec/GPU): 685.5 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.143318E+01 | loss scale: 1.0 | grad norm: 0.920 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 9 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2160.61, 2160.88) +Dataloader batch - tokens shape: torch.Size([1, 1809]) +Dataloader batch - labels shape: torch.Size([1, 1809]) +Dataloader batch - attn_mask shape: torch.Size([1, 1809]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1396]) +Dataloader batch - labels shape: torch.Size([1, 1396]) +Dataloader batch - attn_mask shape: torch.Size([1, 1396]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1658]) +Dataloader batch - labels shape: torch.Size([1, 1658]) +Dataloader batch - attn_mask shape: torch.Size([1, 1658]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1804]) +Dataloader batch - labels shape: torch.Size([1, 1804]) +Dataloader batch - attn_mask shape: torch.Size([1, 1804]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1804) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1804) torch.int64 + loss_mask : 416 / 1804 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 256, 3072) + out : (32, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1804, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1804, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1804, 1, 576) torch.bfloat16 + out : (1, 1804) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.4491 + top-1 accuracy : 0/416 = 0.0% + + [2026-04-29 08:59:37] iteration 10/ 20 | consumed samples: 40 | elapsed time per iteration (ms): 2433.4 | throughput (token/sec/GPU): 684.9 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.144508E+01 | loss scale: 1.0 | grad norm: 0.838 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader +saving checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 10 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2352.68, 2353.32) +Dataloader batch - tokens shape: torch.Size([1, 1899]) +Dataloader batch - labels shape: torch.Size([1, 1899]) +Dataloader batch - attn_mask shape: torch.Size([1, 1899]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1134]) +Dataloader batch - labels shape: torch.Size([1, 1134]) +Dataloader batch - attn_mask shape: torch.Size([1, 1134]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1204]) +Dataloader batch - labels shape: torch.Size([1, 1204]) +Dataloader batch - attn_mask shape: torch.Size([1, 1204]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1134) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1134) torch.int64 + loss_mask : 319 / 1134 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 256, 3072) + out : (19, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1134, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1134, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1134, 1, 576) torch.bfloat16 + out : (1, 1134) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.5059 + top-1 accuracy : 3/319 = 0.9% + + [2026-04-29 08:59:41] iteration 11/ 20 | consumed samples: 44 | elapsed time per iteration (ms): 2028.4 | throughput (token/sec/GPU): 731.2 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.146556E+01 | loss scale: 1.0 | grad norm: 0.775 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 11 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1975.71, 1976.00) +Dataloader batch - tokens shape: torch.Size([1, 1115]) +Dataloader batch - labels shape: torch.Size([1, 1115]) +Dataloader batch - attn_mask shape: torch.Size([1, 1115]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1458]) +Dataloader batch - labels shape: torch.Size([1, 1458]) +Dataloader batch - attn_mask shape: torch.Size([1, 1458]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 295 / 1475 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 256, 3072) + out : (28, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.3431 + top-1 accuracy : 1/295 = 0.3% + + [2026-04-29 08:59:46] iteration 12/ 20 | consumed samples: 48 | elapsed time per iteration (ms): 2429.4 | throughput (token/sec/GPU): 562.7 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.136467E+01 | loss scale: 1.0 | grad norm: 0.852 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m in torch formatsaving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 12 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1950.65, 1951.35) +Dataloader batch - tokens shape: torch.Size([1, 1535]) +Dataloader batch - labels shape: torch.Size([1, 1535]) +Dataloader batch - attn_mask shape: torch.Size([1, 1535]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 963]) +Dataloader batch - labels shape: torch.Size([1, 963]) +Dataloader batch - attn_mask shape: torch.Size([1, 963]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1481]) +Dataloader batch - labels shape: torch.Size([1, 1481]) +Dataloader batch - attn_mask shape: torch.Size([1, 1481]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1481) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1481) torch.int64 + loss_mask : 350 / 1481 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 256, 3072) + out : (28, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1481, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1481, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1481, 1, 576) torch.bfloat16 + out : (1, 1481) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.3549 + top-1 accuracy : 6/350 = 1.7% + + [2026-04-29 08:59:50] iteration 13/ 20 | consumed samples: 52 | elapsed time per iteration (ms): 2254.6 | throughput (token/sec/GPU): 604.2 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.138722E+01 | loss scale: 1.0 | grad norm: 0.812 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 13 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1940.31, 1940.49) +Dataloader batch - tokens shape: torch.Size([1, 1848]) +Dataloader batch - labels shape: torch.Size([1, 1848]) +Dataloader batch - tokens shape:Dataloader batch - attn_mask shape: torch.Size([1, 1848]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +torch.Size([1, 1588]) +Dataloader batch - labels shape: torch.Size([1, 1588]) +Dataloader batch - attn_mask shape: torch.Size([1, 1588]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1534]) +Dataloader batch - labels shape: torch.Size([1, 1534]) +Dataloader batch - attn_mask shape: torch.Size([1, 1534]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1588) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1588) torch.int64 + loss_mask : 455 / 1588 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 256, 3072) + out : (27, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1588, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1588, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1588, 1, 576) torch.bfloat16 + out : (1, 1588) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 455 tokens) : 11.4433 + top-1 accuracy : 0/455 = 0.0% + + [2026-04-29 08:59:54] iteration 14/ 20 | consumed samples: 56 | elapsed time per iteration (ms): 2173.7 | throughput (token/sec/GPU): 735.3 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.144041E+01 | loss scale: 1.0 | grad norm: 0.641 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 14 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1797.54, 1797.87) +Dataloader batch - tokens shape: torch.Size([1, 1577]) +Dataloader batch - labels shape: torch.Size([1, 1577]) +Dataloader batch - attn_mask shape: torch.Size([1, 1577]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1007]) +Dataloader batch - labels shape: torch.Size([1, 1007]) +Dataloader batch - attn_mask shape: torch.Size([1, 1007]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1577) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1577) torch.int64 + loss_mask : 389 / 1577 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 256, 3072) + out : (28, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1577, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1577, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1577, 1, 576) torch.bfloat16 + out : (1, 1577) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.4508 + top-1 accuracy : 3/389 = 0.8% + + [2026-04-29 08:59:57] iteration 15/ 20 | consumed samples: 60 | elapsed time per iteration (ms): 1600.5 | throughput (token/sec/GPU): 841.4 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.138847E+01 | loss scale: 1.0 | grad norm: 0.718 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 15 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1845.50, 1845.65) +Dataloader batch - tokens shape: torch.Size([1, 1067]) +Dataloader batch - labels shape: torch.Size([1, 1067]) +Dataloader batch - attn_mask shape: torch.Size([1, 1067]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1575]) +Dataloader batch - labels shape: torch.Size([1, 1575]) +Dataloader batch - attn_mask shape: torch.Size([1, 1575]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1163]) +Dataloader batch - labels shape: torch.Size([1, 1163]) +Dataloader batch - attn_mask shape: torch.Size([1, 1163]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1441]) +Dataloader batch - labels shape: torch.Size([1, 1441]) +Dataloader batch - attn_mask shape: torch.Size([1, 1441]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1575) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1575) torch.int64 + loss_mask : 384 / 1575 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 256, 3072) + out : (28, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1575, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1575, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1575, 1, 576) torch.bfloat16 + out : (1, 1575) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.3409 + top-1 accuracy : 0/384 = 0.0% + + [2026-04-29 09:00:01] iteration 16/ 20 | consumed samples: 64 | elapsed time per iteration (ms): 2136.7 | throughput (token/sec/GPU): 613.8 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.142639E+01 | loss scale: 1.0 | grad norm: 0.756 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 16 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2297.64, 2297.76) +Dataloader batch - tokens shape: torch.Size([1, 1793]) +Dataloader batch - labels shape: torch.Size([1, 1793]) +Dataloader batch - attn_mask shape: torch.Size([1, 1793]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1395]) +Dataloader batch - labels shape: torch.Size([1, 1395]) +Dataloader batch - attn_mask shape: torch.Size([1, 1395]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1498]) +Dataloader batch - labels shape: torch.Size([1, 1498]) +Dataloader batch - attn_mask shape: torch.Size([1, 1498]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1581]) +Dataloader batch - labels shape: torch.Size([1, 1581]) +Dataloader batch - attn_mask shape: torch.Size([1, 1581]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1395) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1395) torch.int64 + loss_mask : 357 / 1395 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 256, 3072) + out : (25, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1395, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1395, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1395, 1, 576) torch.bfloat16 + out : (1, 1395) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3814 + top-1 accuracy : 3/357 = 0.8% + + [2026-04-29 09:00:06] iteration 17/ 20 | consumed samples: 68 | elapsed time per iteration (ms): 2179.7 | throughput (token/sec/GPU): 718.8 | learning rate: 6.416419E-06 | global batch size: 4 | lm loss: 1.139987E+01 | loss scale: 1.0 | grad norm: 0.644 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 17 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1828.11, 1828.17) +Dataloader batch - tokens shape: torch.Size([1, 1871]) +Dataloader batch - labels shape: torch.Size([1, 1871]) +Dataloader batch - attn_mask shape: torch.Size([1, 1871]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1825]) +Dataloader batch - labels shape: torch.Size([1, 1825]) +Dataloader batch - attn_mask shape: Dataloader batch - tokens shape:torch.Size([1, 1825])Dataloader batch - tokens shape: +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + torch.Size([1, 943]) +Dataloader batch - labels shape: torch.Size([1, 943]) +Dataloader batch - attn_mask shape: torch.Size([1, 943]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + torch.Size([1, 1451]) +Dataloader batch - labels shape: torch.Size([1, 1451]) +Dataloader batch - attn_mask shape: torch.Size([1, 1451]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1825) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1825) torch.int64 + loss_mask : 463 / 1825 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 256, 3072) + out : (32, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1825, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1825, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1825, 1, 576) torch.bfloat16 + out : (1, 1825) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 463 tokens) : 11.3953 + top-1 accuracy : 1/463 = 0.2% + + [2026-04-29 09:00:10] iteration 18/ 20 | consumed samples: 72 | elapsed time per iteration (ms): 2223.1 | throughput (token/sec/GPU): 684.9 | learning rate: 3.432342E-06 | global batch size: 4 | lm loss: 1.139225E+01 | loss scale: 1.0 | grad norm: 0.651 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader + + +saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 18 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1950.87, 1951.92) +Dataloader batch - tokens shape: torch.Size([1, 1478]) +Dataloader batch - labels shape: torch.Size([1, 1478]) +Dataloader batch - attn_mask shape: torch.Size([1, 1478]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1747]) +Dataloader batch - labels shape: torch.Size([1, 1747]) +Dataloader batch - attn_mask shape: torch.Size([1, 1747]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1394]) +Dataloader batch - labels shape: torch.Size([1, 1394]) +Dataloader batch - attn_mask shape: torch.Size([1, 1394]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + torch.Size([1, 1382]) +Dataloader batch - labels shape: torch.Size([1, 1382]) +Dataloader batch - attn_mask shape: torch.Size([1, 1382]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1747) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1747) torch.int64 + loss_mask : 390 / 1747 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 256, 3072) + out : (32, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1747, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1747, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1747, 1, 576) torch.bfloat16 + out : (1, 1747) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3102 + top-1 accuracy : 5/390 = 1.3% + + [2026-04-29 09:00:14] iteration 19/ 20 | consumed samples: 76 | elapsed time per iteration (ms): 2114.2 | throughput (token/sec/GPU): 709.6 | learning rate: 1.611867E-06 | global batch size: 4 | lm loss: 1.134015E+01 | loss scale: 1.0 | grad norm: 0.719 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader + + successfully saved checkpoint from iteration 19 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2256.53, 2256.75) +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1897]) +Dataloader batch - labels shape: torch.Size([1, 1897]) +Dataloader batch - attn_mask shape: torch.Size([1, 1897]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1457]) +Dataloader batch - labels shape: torch.Size([1, 1457]) +Dataloader batch - attn_mask shape: torch.Size([1, 1457]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1936]) +Dataloader batch - labels shape: torch.Size([1, 1936]) +Dataloader batch - attn_mask shape: torch.Size([1, 1936]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1936) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1936) torch.int64 + loss_mask : 569 / 1936 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 256, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 256, 3072) + out : (32, 256, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1936, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1936, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1936, 1, 576) torch.bfloat16 + out : (1, 1936) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 569 tokens) : 11.4828 + top-1 accuracy : 11/569 = 1.9% + + [2026-04-29 09:00:18] iteration 20/ 20 | consumed samples: 80 | elapsed time per iteration (ms): 1895.5 | throughput (token/sec/GPU): 898.0 | learning rate: 1.000000E-06 | global batch size: 4 | lm loss: 1.141360E+01 | loss scale: 1.0 | grad norm: 0.637 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (1857.53, 1857.59) + forward-compute ................................: (1405.63, 1406.53) + backward-compute ...............................: (322.51, 437.95) + batch-generator ................................: (275.09, 393.60) + layernorm-grads-all-reduce .....................: (0.02, 0.06) + embedding-grads-all-reduce .....................: (0.04, 0.09) + all-grads-sync .................................: (9.51, 10.53) + params-all-gather ..............................: (1.28, 1.36) + optimizer-copy-to-main-grad ....................: (0.10, 0.30) + optimizer-clip-main-grad .......................: (1.04, 1.19) + optimizer-count-zeros ..........................: (0.01, 0.04) + optimizer-inner-step ...........................: (0.64, 2.00) + optimizer-copy-main-to-model-params ............: (0.12, 0.45) + optimizer ......................................: (7.24, 7.32) +saving checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 20 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1846.10, 1846.25) +[after training is done] datetime: 2026-04-29 09:00:20 +wandb: uploading output.log; uploading config.yaml +wandb: uploading history steps 18-19, summary, console lines 1127-1145 +wandb: +wandb: +wandb: Run history: +wandb: Token throughput (per-sec-per-GPU) ▁▄▆▆▆▆▅▆▆▆▇▅▆▇█▆▆▆▆█ +wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: grad-norm ██▄▃▃▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁ +wandb: iteration-time █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: learning-rate ███▇▇▇▆▆▅▅▄▃▃▂▂▂▁▁▁▁ +wandb: lm loss ██▆▅▄▄▃▃▂▃▃▁▂▂▂▂▂▂▁▂ +wandb: loss-scale ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: num-zeros ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: samples vs steps ▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██ +wandb: +wandb: Run summary: +wandb: Token throughput (per-sec-per-GPU) 898.04171 +wandb: batch-size 4 +wandb: grad-norm 0.6374 +wandb: iteration-time 1.89551 +wandb: learning-rate 0.0 +wandb: lm loss 11.4136 +wandb: loss-scale 1 +wandb: num-zeros 0 +wandb: samples vs steps 80 +wandb: +wandb: 🚀 View run mobilellm_integration at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/7dva7qn6 +wandb: ⭐️ View project at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_085758-7dva7qn6/logs +[rank0]:[W429 09:01:42.132834283 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W429 09:01:43.891072258 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W429 09:01:43.383694399 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W429 09:01:46.131858054 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log new file mode 100644 index 00000000..f9ac6d7a --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log @@ -0,0 +1,8137 @@ +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] +W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] ***************************************** +W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] ***************************************** +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +FalseFalse + +False +False +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +parse arguments function called +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +parse arguments function called +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +Using unified model type for training. + +Starting training... +Using unified model type for training. + +Starting training... +> setting tensorboard ... +parse arguments function called +-------------- Configure model to llava-ov-mobilellm-140m -------------- +[DEBUG] Model config type: LlavaOnevision1_5Config +[DEBUG] Is MobileLLM: True + num_layers = 15 + hidden_size = 576 + ffn_hidden_size = 2048 + num_attention_heads = 9 + group_query_attention = True + num_query_groups = 3 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = False + vocab_size_in_config_file = 128256 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-05 + rotary_base = 8000000 + kv_channels = None + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 4, data-parallel size: 4, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.local + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... llama3 + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 4 + data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 15 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 32768 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + fastvit_image_size .............................. 1024 + ffn_hidden_size ................................. 2048 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 4 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M + hidden_dropout .................................. 0 + hidden_size ..................................... 576 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_aspect_ratio .............................. pad + image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ None + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 20 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-mobilellm-140m + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 9 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 15 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 3 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 16 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 3 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 8000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_mobilellm_140m + save_ema ........................................ None + save_interval ................................... 1 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 32768 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 20 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 32768 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. False + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_fastvit ..................................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vision_tower_name ............................... mobileclip_l_1024 + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 128256 + vpp_scheduler ................................... None + wandb_exp_name .................................. mobilellm_integration + wandb_project ................................... llava-ov-1_5 + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 4 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +Using unified model type for training. + +Starting training... +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> AIAK building HFTokenizer tokenizer ... +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +parse arguments function called +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +Using unified model type for training. + +Starting training... +wandb: Currently logged in as: rana-zayed (rana-zayed-mbzuai) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 +WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. +WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. +WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). + > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +wandb: creating run +wandb: Tracking run with wandb version 0.21.0 +wandb: Run data is saved locally in stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_091210-1ll8kzkt +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run mobilellm_integration +wandb: ⭐️ View project at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 +wandb: 🚀 View run at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/1ll8kzkt +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 3 +3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so +make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.043 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W429 09:12:13.409480472 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.338 seconds +time to initialize megatron (seconds): 7.571 +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2026-04-29 09:12:16 +building llava-ov-mobilellm-140m model ... +[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m +[DEBUG PROVIDER] Args num_layers: 15 +[DEBUG PROVIDER] Args hidden_size: 576 +[DEBUG PROVIDER] Args vocab_size: 128260 +[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 +[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 +[DEBUG PROVIDER] Model family: llava_ov_1_5 +[DEBUG PROVIDER] use_mobilellm flag: True +[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone +[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 +[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Loading vision config... +[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 +[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json +[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd +[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ================================================ +[DEBUG PROVIDER] Loading adapter config... +[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 +[DEBUG PROVIDER] Building layer specs... +[DEBUG PROVIDER] ✓ Using MobileLLM layer specification +[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] MobileLLM layer spec created successfully +[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... +[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 +[INFO] Using MobileLLM as language backbone +[INFO] Using MobileLLM as language backbone +[INFO] Using MobileLLM as language backbone +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[INFO] Using MobileLLM as language backbone +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (11216448 elements, 11216512 padded size): + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.bias + module.adapter.layernorm.weight + module.adapter.linear_fc2.linear.weight + module.adapter.linear_fc1.linear.bias +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[DEBUG] FastViT enabled: use_fastvit=True +[DEBUG] Before handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +FastViT enabled: Will load vision tower from pretrained checkpoint: /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +[DEBUG] After handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +Checkpoint file not found in load directory None attempting to finetune with checkpoint in /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + loading release checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +could not find arguments in the checkpoint ... +Loading checkpoint with device: cuda:0, strict=False +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel +WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel + checkpoint version 3.0 + successfully loaded checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 [ t 1/1, p 1/1 ] at iteration 0 + failed to find checkpoint /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1/release/mp_rank_00/model_optim_rng.pt in wandb +(min, max) time across ranks (ms): + load-checkpoint ................................: (3038.02, 3038.45) +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 09:12:21 +================================================================================ +FULL MODEL STRUCTURE: +================================================================================ + +--- Model 0 (pipeline rank 0) --- +DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +) +================================================================================ + +================================================================================ +PARAMETER TRAINABILITY STATUS: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +================================================================================ +Total trainable parameters: 11,216,448 (11.22M) +Total frozen parameters: 265,417,440 (265.42M) +Total parameters: 276,633,888 (276.63M) +Trainable percentage: 4.05% +================================================================================ +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 80 + validation: 400 + test: 400 +Using shared training tokenizer in FastVLM mode +Ensured multimodal special tokens for FastVLM tokenizer; added=0 +Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M +Initialized FastViT processor with image_size=1024 +Processor config: .TokenizerWrapper object at 0x7f1df8173eb0> +image_resolution: None +rank=3, worker=0: shard_range=[pretrain-000004.tar[2858, 3233), ] sum(count)=375rank=1, worker=0: shard_range=[pretrain-000001.tar[939, 1314), ] sum(count)=375 +rank=1, worker=1: shard_range=[pretrain-000001.tar[1314, 1689), ] sum(count)=375 +rank=1, worker=2: shard_range=[pretrain-000001.tar[1689, 2064), ] sum(count)=375 +rank=1, worker=3: shard_range=[pretrain-000001.tar[2064, 2439), ] sum(count)=375 +rank=1, worker=4: shard_range=[pretrain-000001.tar[2439, 2814), ] sum(count)=375 +rank=1, worker=5: shard_range=[pretrain-000001.tar[2814, 3189), ] sum(count)=375 +rank=1, worker=6: shard_range=[pretrain-000001.tar[3189, 3564), ] sum(count)=375 + +rank=1, worker=7: shard_range=[pretrain-000001.tar[3564, 3938), ] sum(count)=374 +rank=1, worker=8: shard_range=[pretrain-000001.tar[3938, 4313), ] sum(count)=375 +rank=3, worker=1: shard_range=[pretrain-000004.tar[3233, 3608), ] sum(count)=375rank=1, worker=9: shard_range=[pretrain-000001.tar[4313, 4688), ] sum(count)=375 +rank=1, worker=10: shard_range=[pretrain-000001.tar[4688, 5036), pretrain-000002.tar[0, 27), ] sum(count)=375 + +rank=1, worker=11: shard_range=[pretrain-000002.tar[27, 401), ] sum(count)=374 +rank=1, worker=12: shard_range=[pretrain-000002.tar[401, 776), ] sum(count)=375 +rank=3, worker=2: shard_range=[pretrain-000004.tar[3608, 3821), pretrain-000000.tar[0, 162), ] sum(count)=375rank=1, worker=13: shard_range=[pretrain-000002.tar[776, 1151), ] sum(count)=375 +rank=1, worker=14: shard_range=[pretrain-000002.tar[1151, 1526), ] sum(count)=375 + +rank=1, worker=15: shard_range=[pretrain-000002.tar[1526, 1900), ] sum(count)=374 +rank=3, worker=3: shard_range=[pretrain-000000.tar[162, 537), ] sum(count)=375 +rank=3, worker=4: shard_range=[pretrain-000000.tar[537, 912), ] sum(count)=375 +rank=3, worker=5: shard_range=[pretrain-000000.tar[912, 1287), ] sum(count)=375 +rank=3, worker=6: shard_range=[pretrain-000000.tar[1287, 1662), ] sum(count)=375 +rank=3, worker=7: shard_range=[pretrain-000000.tar[1662, 2036), ] sum(count)=374 +rank=3, worker=8: shard_range=[pretrain-000000.tar[2036, 2411), ] sum(count)=375 +rank=3, worker=9: shard_range=[pretrain-000000.tar[2411, 2786), ] sum(count)=375 +rank=3, worker=10: shard_range=[pretrain-000000.tar[2786, 3161), ] sum(count)=375 +rank=3, worker=11: shard_range=[pretrain-000000.tar[3161, 3535), ] sum(count)=374 +rank=3, worker=12: shard_range=[pretrain-000000.tar[3535, 3910), ] sum(count)=375 +rank=3, worker=13: shard_range=[pretrain-000000.tar[3910, 4285), ] sum(count)=375 +rank=3, worker=14: shard_range=[pretrain-000000.tar[4285, 4660), ] sum(count)=375 +rank=3, worker=15: shard_range=[pretrain-000000.tar[4660, 5034), ] sum(count)=374 +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 375), ] sum(count)=375 +rank=0, worker=1: shard_range=[pretrain-000003.tar[375, 750), ] sum(count)=375 +rank=0, worker=2: shard_range=[pretrain-000003.tar[750, 1125), ] sum(count)=375 +rank=0, worker=3: shard_range=[pretrain-000003.tar[1125, 1500), ] sum(count)=375 +rank=0, worker=4: shard_range=[pretrain-000003.tar[1500, 1875), ] sum(count)=375 +rank=0, worker=5: shard_range=[pretrain-000003.tar[1875, 2250), ] sum(count)=375 +rank=0, worker=6: shard_range=[pretrain-000003.tar[2250, 2625), ] sum(count)=375 +rank=0, worker=7: shard_range=[pretrain-000003.tar[2625, 2999), ] sum(count)=374 +rank=0, worker=8: shard_range=[pretrain-000003.tar[2999, 3374), ] sum(count)=375 +rank=0, worker=9: shard_range=[pretrain-000003.tar[3374, 3749), ] sum(count)=375 +rank=0, worker=10: shard_range=[pretrain-000003.tar[3749, 4124), ] sum(count)=375 +rank=0, worker=11: shard_range=[pretrain-000003.tar[4124, 4498), ] sum(count)=374 +rank=0, worker=12: shard_range=[pretrain-000003.tar[4498, 4873), ] sum(count)=375 +rank=0, worker=13: shard_range=[pretrain-000003.tar[4873, 5058), pretrain-000001.tar[0, 190), ] sum(count)=375 +rank=0, worker=14: shard_range=[pretrain-000001.tar[190, 565), ] sum(count)=375 +rank=0, worker=15: shard_range=[pretrain-000001.tar[565, 939), ] sum(count)=374 +rank=2, worker=0: shard_range=[pretrain-000002.tar[1900, 2275), ] sum(count)=375 +rank=2, worker=1: shard_range=[pretrain-000002.tar[2275, 2650), ] sum(count)=375 +rank=2, worker=2: shard_range=[pretrain-000002.tar[2650, 3025), ] sum(count)=375 +rank=2, worker=3: shard_range=[pretrain-000002.tar[3025, 3400), ] sum(count)=375 +rank=2, worker=4: shard_range=[pretrain-000002.tar[3400, 3775), ] sum(count)=375 +rank=2, worker=5: shard_range=[pretrain-000002.tar[3775, 4150), ] sum(count)=375 +rank=2, worker=6: shard_range=[pretrain-000002.tar[4150, 4525), ] sum(count)=375 +rank=2, worker=7: shard_range=[pretrain-000002.tar[4525, 4899), ] sum(count)=374 +rank=2, worker=8: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 235), ] sum(count)=375 +rank=2, worker=9: shard_range=[pretrain-000004.tar[235, 610), ] sum(count)=375 +rank=2, worker=10: shard_range=[pretrain-000004.tar[610, 985), ] sum(count)=375 +rank=2, worker=11: shard_range=[pretrain-000004.tar[985, 1359), ] sum(count)=374 +rank=2, worker=12: shard_range=[pretrain-000004.tar[1359, 1734), ] sum(count)=375 +rank=2, worker=13: shard_range=[pretrain-000004.tar[1734, 2109), ] sum(count)=375 +rank=2, worker=14: shard_range=[pretrain-000004.tar[2109, 2484), ] sum(count)=375 +rank=2, worker=15: shard_range=[pretrain-000004.tar[2484, 2858), ] sum(count)=374 +[after dataloaders are built] datetime: 2026-04-29 09:12:21 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (5402.02, 5415.21) + train/valid/test-data-iterators-setup ..........: (75.65, 81.61)training ... + +================================================================================ +training with the following parameter status: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 + +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +Setting rerun_state_machine.current_iteration to 0... +[before the start of training step] datetime: 2026-04-29 09:12:21 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 + +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1757]) +Dataloader batch - labels shape: torch.Size([1, 1757]) +Dataloader batch - attn_mask shape: torch.Size([1, 1757]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1356]) +Dataloader batch - labels shape: torch.Size([1, 1356]) +Dataloader batch - attn_mask shape: torch.Size([1, 1356]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1909]) +Dataloader batch - labels shape: torch.Size([1, 1909]) +Dataloader batch - attn_mask shape: torch.Size([1, 1909]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 532 / 1909 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1430]) +Dataloader batch - labels shape: torch.Size([1, 1430]) +Dataloader batch - attn_mask shape: torch.Size([1, 1430]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 532 tokens) : 11.7829 + top-1 accuracy : 1/532 = 0.2% + +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +Number of parameters in transformer layers in billions: 0.07 +Number of parameters in embedding layers in billions: 0.07 +Total number of parameters in billions: 0.14 +Number of parameters in most loaded shard in billions: 0.1403 +Theoretical memory footprints: weight and optimizer=1204.55 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 709.1826171875 | max allocated: 5398.64892578125 | reserved: 6388.0 | max reserved: 6388.0 + [2026-04-29 09:12:42] iteration 1/ 20 | consumed samples: 4 | elapsed time per iteration (ms): 20330.5 | throughput (token/sec/GPU): 79.3 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.179576E+01 | loss scale: 1.0 | grad norm: 3.000 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 1 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (3113.10, 3115.03) +Dataloader batch - tokens shape: torch.Size([1, 1781]) +Dataloader batch - labels shape: torch.Size([1, 1781]) +Dataloader batch - attn_mask shape: torch.Size([1, 1781]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1186]) +Dataloader batch - labels shape: torch.Size([1, 1186]) +Dataloader batch - attn_mask shape: torch.Size([1, 1186]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1114]) +Dataloader batch - labels shape: torch.Size([1, 1114]) +Dataloader batch - attn_mask shape: torch.Size([1, 1114]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 942]) +Dataloader batch - labels shape: torch.Size([1, 942]) +Dataloader batch - attn_mask shape: torch.Size([1, 942]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1186) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1186) torch.int64 + loss_mask : 273 / 1186 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1186, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1186, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1186, 1, 576) torch.bfloat16 + out : (1, 1186) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 273 tokens) : 11.8201 + top-1 accuracy : 0/273 = 0.0% + + [2026-04-29 09:12:48] iteration 2/ 20 | consumed samples: 8 | elapsed time per iteration (ms): 2850.4 | throughput (token/sec/GPU): 440.6 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.181961E+01 | loss scale: 1.0 | grad norm: 3.164 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m in torch formatsaving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 2 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2986.92, 2988.67) +Dataloader batch - tokens shape: torch.Size([1, 1849]) +Dataloader batch - labels shape: torch.Size([1, 1849]) +Dataloader batch - attn_mask shape: torch.Size([1, 1849]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1741]) +Dataloader batch - labels shape: torch.Size([1, 1741]) +Dataloader batch - attn_mask shape: torch.Size([1, 1741]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1554]) +Dataloader batch - labels shape: torch.Size([1, 1554]) +Dataloader batch - attn_mask shape: torch.Size([1, 1554]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1399]) +Dataloader batch - labels shape: torch.Size([1, 1399]) +Dataloader batch - attn_mask shape: torch.Size([1, 1399]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1399) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1399) torch.int64 + loss_mask : 342 / 1399 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1399, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1399, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1399, 1, 576) torch.bfloat16 + out : (1, 1399) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.7174 + top-1 accuracy : 0/342 = 0.0% + + [2026-04-29 09:12:53] iteration 3/ 20 | consumed samples: 12 | elapsed time per iteration (ms): 2824.4 | throughput (token/sec/GPU): 579.1 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.168303E+01 | loss scale: 1.0 | grad norm: 1.876 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 3 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (3043.68, 3044.00) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1444]) +torch.Size([1, 1585])Dataloader batch - labels shape: +torch.Size([1, 1444])Dataloader batch - labels shape: + Dataloader batch - attn_mask shape:torch.Size([1, 1585]) +torch.Size([1, 1444])Dataloader batch - attn_mask shape: + Dataloader batch - image_grid_thw shape:torch.Size([1, 1585]) +torch.Size([24, 3])Dataloader batch - image_grid_thw shape: + torch.Size([29, 3]) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1544]) +Dataloader batch - labels shape: torch.Size([1, 1544]) +Dataloader batch - attn_mask shape: torch.Size([1, 1544]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3])torch.Size([1, 1042]) + +Dataloader batch - labels shape: torch.Size([1, 1042]) +Dataloader batch - attn_mask shape: torch.Size([1, 1042]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1585) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1585) torch.int64 + loss_mask : 350 / 1585 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1585, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1585, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1585, 1, 576) torch.bfloat16 + out : (1, 1585) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.5907 + top-1 accuracy : 0/350 = 0.0% + + [2026-04-29 09:12:58] iteration 4/ 20 | consumed samples: 16 | elapsed time per iteration (ms): 1773.9 | throughput (token/sec/GPU): 791.3 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.163304E+01 | loss scale: 1.0 | grad norm: 1.483 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 4 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2776.61, 2777.07) +Dataloader batch - tokens shape: torch.Size([1, 1214]) +Dataloader batch - labels shape: torch.Size([1, 1214]) +Dataloader batch - attn_mask shape: torch.Size([1, 1214]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 367 / 1541 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.5424 + top-1 accuracy : 0/367 = 0.0% + + [2026-04-29 09:13:03] iteration 5/ 20 | consumed samples: 20 | elapsed time per iteration (ms): 2378.8 | throughput (token/sec/GPU): 563.1 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.153108E+01 | loss scale: 1.0 | grad norm: 1.398 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 5 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2793.07, 2793.32) +Dataloader batch - tokens shape: torch.Size([1, 1498]) +Dataloader batch - labels shape: torch.Size([1, 1498]) +Dataloader batch - attn_mask shape: torch.Size([1, 1498]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1344]) +Dataloader batch - labels shape: torch.Size([1, 1344]) +Dataloader batch - attn_mask shape: torch.Size([1, 1344]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1689]) +Dataloader batch - labels shape: torch.Size([1, 1689]) +Dataloader batch - attn_mask shape: torch.Size([1, 1689]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1344) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1344) torch.int64 + loss_mask : 399 / 1344 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1344, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1344, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1344, 1, 576) torch.bfloat16 + out : (1, 1344) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.5134 + top-1 accuracy : 1/399 = 0.3% + + [2026-04-29 09:13:09] iteration 6/ 20 | consumed samples: 24 | elapsed time per iteration (ms): 2612.5 | throughput (token/sec/GPU): 574.0 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.151871E+01 | loss scale: 1.0 | grad norm: 1.023 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 6 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2767.65, 2767.84) +Dataloader batch - tokens shape: torch.Size([1, 1824]) +Dataloader batch - labels shape: torch.Size([1, 1824]) +Dataloader batch - attn_mask shape: torch.Size([1, 1824]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1352]) +Dataloader batch - labels shape: torch.Size([1, 1352]) +Dataloader batch - attn_mask shape: torch.Size([1, 1352]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1352) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1352) torch.int64 + loss_mask : 318 / 1352 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1352, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1352, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1352, 1, 576) torch.bfloat16 + out : (1, 1352) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.5097 + top-1 accuracy : 0/318 = 0.0% + + [2026-04-29 09:13:13] iteration 7/ 20 | consumed samples: 28 | elapsed time per iteration (ms): 2011.4 | throughput (token/sec/GPU): 775.6 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.149438E+01 | loss scale: 1.0 | grad norm: 0.972 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 7 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2697.84, 2698.53) +Dataloader batch - tokens shape: torch.Size([1, 1612]) +Dataloader batch - labels shape: torch.Size([1, 1612]) +Dataloader batch - attn_mask shape: torch.Size([1, 1612]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) +Dataloader batch - tokens shape: torch.Size([1, 995]) +Dataloader batch - labels shape: torch.Size([1, 995]) +Dataloader batch - attn_mask shape: torch.Size([1, 995]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1558]) +Dataloader batch - labels shape: torch.Size([1, 1558]) +Dataloader batch - attn_mask shape: torch.Size([1, 1558]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 1527]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 995) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 995) torch.int64 + loss_mask : 227 / 995 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (995, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (995, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (995, 1, 576) torch.bfloat16 + out : (1, 995) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 227 tokens) : 11.4815 + top-1 accuracy : 0/227 = 0.0% + + [2026-04-29 09:13:19] iteration 8/ 20 | consumed samples: 32 | elapsed time per iteration (ms): 2354.6 | throughput (token/sec/GPU): 604.4 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.145043E+01 | loss scale: 1.0 | grad norm: 0.868 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 8 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2993.46, 2993.78) +Dataloader batch - tokens shape: torch.Size([1, 1529]) +Dataloader batch - labels shape: torch.Size([1, 1529]) +Dataloader batch - attn_mask shape: torch.Size([1, 1529]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1175]) +Dataloader batch - tokens shape: Dataloader batch - labels shape:torch.Size([1, 1026]) +Dataloader batch - labels shape: torch.Size([1, 1026]) +Dataloader batch - attn_mask shape: torch.Size([1, 1026])torch.Size([1, 1175]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +Dataloader batch - attn_mask shape: torch.Size([1, 1175]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1673]) +Dataloader batch - labels shape: torch.Size([1, 1673]) +Dataloader batch - attn_mask shape: torch.Size([1, 1673]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1026) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1026) torch.int64 + loss_mask : 200 / 1026 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1026, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1026, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1026, 1, 576) torch.bfloat16 + out : (1, 1026) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 200 tokens) : 11.3712 + top-1 accuracy : 0/200 = 0.0% + + [2026-04-29 09:13:24] iteration 9/ 20 | consumed samples: 36 | elapsed time per iteration (ms): 2256.3 | throughput (token/sec/GPU): 598.7 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.143266E+01 | loss scale: 1.0 | grad norm: 0.944 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloadersaving checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m in torch format + + +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 9 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2398.93, 2399.30) +Dataloader batch - tokens shape: torch.Size([1, 1396]) +Dataloader batch - labels shape: torch.Size([1, 1396]) +Dataloader batch - attn_mask shape: torch.Size([1, 1396]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1804]) +Dataloader batch - labels shape: torch.Size([1, 1804]) +Dataloader batch - attn_mask shape: torch.Size([1, 1804]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1809]) +Dataloader batch - labels shape: torch.Size([1, 1809]) +Dataloader batch - attn_mask shape: torch.Size([1, 1809]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1658]) +Dataloader batch - labels shape: torch.Size([1, 1658]) +Dataloader batch - attn_mask shape: torch.Size([1, 1658]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1804) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1804) torch.int64 + loss_mask : 416 / 1804 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1804, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1804, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1804, 1, 576) torch.bfloat16 + out : (1, 1804) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.4488 + top-1 accuracy : 0/416 = 0.0% + + [2026-04-29 09:13:28] iteration 10/ 20 | consumed samples: 40 | elapsed time per iteration (ms): 2172.5 | throughput (token/sec/GPU): 767.2 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.144409E+01 | loss scale: 1.0 | grad norm: 0.851 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 10 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2500.80, 2501.73) +Dataloader batch - tokens shape: torch.Size([1, 1899]) +Dataloader batch - labels shape: torch.Size([1, 1899]) +Dataloader batch - attn_mask shape: torch.Size([1, 1899]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1134]) +Dataloader batch - labels shape: torch.Size([1, 1134]) +Dataloader batch - attn_mask shape: torch.Size([1, 1134]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1204]) +Dataloader batch - labels shape: torch.Size([1, 1204]) +Dataloader batch - attn_mask shape: torch.Size([1, 1204]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1134) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1134) torch.int64 + loss_mask : 319 / 1134 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1134, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1134, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1134, 1, 576) torch.bfloat16 + out : (1, 1134) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.5044 + top-1 accuracy : 3/319 = 0.9% + + [2026-04-29 09:13:33] iteration 11/ 20 | consumed samples: 44 | elapsed time per iteration (ms): 2299.5 | throughput (token/sec/GPU): 645.0 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.146383E+01 | loss scale: 1.0 | grad norm: 0.789 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 11 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2844.03, 2844.77) +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1115]) +Dataloader batch - labels shape: torch.Size([1, 1115]) +Dataloader batch - attn_mask shape: torch.Size([1, 1115]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1458]) +Dataloader batch - labels shape: torch.Size([1, 1458]) +Dataloader batch - attn_mask shape: torch.Size([1, 1458]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 295 / 1475 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.3404 + top-1 accuracy : 1/295 = 0.3% + + [2026-04-29 09:13:38] iteration 12/ 20 | consumed samples: 48 | elapsed time per iteration (ms): 2071.3 | throughput (token/sec/GPU): 660.0 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.136227E+01 | loss scale: 1.0 | grad norm: 0.882 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 12 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2613.00, 2613.48) +Dataloader batch - tokens shape: torch.Size([1, 1535]) +Dataloader batch - labels shape: torch.Size([1, 1535]) +Dataloader batch - attn_mask shape: torch.Size([1, 1535]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1481]) +Dataloader batch - labels shape: torch.Size([1, 1481]) +Dataloader batch - attn_mask shape: Dataloader batch - tokens shape:torch.Size([1, 1481]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 963]) +Dataloader batch - labels shape: torch.Size([1, 963]) +Dataloader batch - attn_mask shape: torch.Size([1, 963]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1481) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1481) torch.int64 + loss_mask : 350 / 1481 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1481, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1481, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1481, 1, 576) torch.bfloat16 + out : (1, 1481) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.3526 + top-1 accuracy : 6/350 = 1.7% + + [2026-04-29 09:13:43] iteration 13/ 20 | consumed samples: 52 | elapsed time per iteration (ms): 1814.6 | throughput (token/sec/GPU): 750.7 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.138488E+01 | loss scale: 1.0 | grad norm: 0.831 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloadersaving checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 13 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2329.09, 2329.61) +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1848]) +Dataloader batch - labels shape: torch.Size([1, 1848]) +Dataloader batch - attn_mask shape: torch.Size([1, 1848]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1588]) +Dataloader batch - labels shape: torch.Size([1, 1588]) +Dataloader batch - attn_mask shape: torch.Size([1, 1588]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1534]) +Dataloader batch - labels shape: torch.Size([1, 1534]) +Dataloader batch - attn_mask shape: torch.Size([1, 1534]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1588) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1588) torch.int64 + loss_mask : 455 / 1588 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1588, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1588, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1588, 1, 576) torch.bfloat16 + out : (1, 1588) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 455 tokens) : 11.4415 + top-1 accuracy : 0/455 = 0.0% + + [2026-04-29 09:13:47] iteration 14/ 20 | consumed samples: 56 | elapsed time per iteration (ms): 1856.3 | throughput (token/sec/GPU): 861.0 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.143828E+01 | loss scale: 1.0 | grad norm: 0.664 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m in torch formatsaving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 14 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2617.99, 2619.03) +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1577]) +Dataloader batch - labels shape: torch.Size([1, 1577]) +Dataloader batch - attn_mask shape: torch.Size([1, 1577]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1007]) +Dataloader batch - labels shape: torch.Size([1, 1007]) +Dataloader batch - attn_mask shape: torch.Size([1, 1007]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1577) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1577) torch.int64 + loss_mask : 389 / 1577 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1577, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1577, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1577, 1, 576) torch.bfloat16 + out : (1, 1577) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.4468 + top-1 accuracy : 6/389 = 1.5% + + [2026-04-29 09:13:51] iteration 15/ 20 | consumed samples: 60 | elapsed time per iteration (ms): 1877.1 | throughput (token/sec/GPU): 717.5 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.138525E+01 | loss scale: 1.0 | grad norm: 0.740 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader + + successfully saved checkpoint from iteration 15 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2424.06, 2425.03) +Dataloader batch - tokens shape: torch.Size([1, 1067]) +Dataloader batch - labels shape: torch.Size([1, 1067]) +Dataloader batch - attn_mask shape: torch.Size([1, 1067]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1441]) +Dataloader batch - labels shape: torch.Size([1, 1441]) +Dataloader batch - attn_mask shape: torch.Size([1, 1441]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1575]) +Dataloader batch - labels shape: torch.Size([1, 1575]) +Dataloader batch - attn_mask shape: torch.Size([1, 1575]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1163]) +Dataloader batch - labels shape: torch.Size([1, 1163]) +Dataloader batch - attn_mask shape: torch.Size([1, 1163]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1575) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1575) torch.int64 + loss_mask : 384 / 1575 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1575, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1575, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1575, 1, 576) torch.bfloat16 + out : (1, 1575) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.3348 + top-1 accuracy : 0/384 = 0.0% + + [2026-04-29 09:13:56] iteration 16/ 20 | consumed samples: 64 | elapsed time per iteration (ms): 1966.1 | throughput (token/sec/GPU): 667.0 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.142319E+01 | loss scale: 1.0 | grad norm: 0.773 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 16 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2601.56, 2601.82) +Dataloader batch - tokens shape: torch.Size([1, 1498]) +Dataloader batch - labels shape: torch.Size([1, 1498]) +Dataloader batch - attn_mask shape: torch.Size([1, 1498]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: Dataloader batch - tokens shape: torch.Size([1, 1793]) +Dataloader batch - labels shape: torch.Size([1, 1793]) +Dataloader batch - attn_mask shape: torch.Size([1, 1793]) +Dataloader batch - image_grid_thw shape:torch.Size([1, 1581]) torch.Size([32, 3]) + +Dataloader batch - labels shape: torch.Size([1, 1581]) +Dataloader batch - attn_mask shape: torch.Size([1, 1581]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1395]) +Dataloader batch - labels shape: torch.Size([1, 1395]) +Dataloader batch - attn_mask shape: torch.Size([1, 1395]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1395) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1395) torch.int64 + loss_mask : 357 / 1395 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1395, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1395, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1395, 1, 576) torch.bfloat16 + out : (1, 1395) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3790 + top-1 accuracy : 3/357 = 0.8% + + [2026-04-29 09:14:00] iteration 17/ 20 | consumed samples: 68 | elapsed time per iteration (ms): 1828.7 | throughput (token/sec/GPU): 856.7 | learning rate: 6.416419E-06 | global batch size: 4 | lm loss: 1.139731E+01 | loss scale: 1.0 | grad norm: 0.660 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 17 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2378.09, 2378.67) +Dataloader batch - tokens shape: torch.Size([1, 943]) +Dataloader batch - labels shape: torch.Size([1, 943]) +Dataloader batch - attn_mask shape: torch.Size([1, 943]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1871]) +Dataloader batch - labels shape: torch.Size([1, 1871]) +Dataloader batch - attn_mask shape: torch.Size([1, 1871]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1825]) +Dataloader batch - labels shape: torch.Size([1, 1825]) +Dataloader batch - attn_mask shape: torch.Size([1, 1825]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1451]) +Dataloader batch - labels shape: torch.Size([1, 1451]) +Dataloader batch - attn_mask shape: torch.Size([1, 1451]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1825) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1825) torch.int64 + loss_mask : 463 / 1825 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1825, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1825, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1825, 1, 576) torch.bfloat16 + out : (1, 1825) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 463 tokens) : 11.3931 + top-1 accuracy : 6/463 = 1.3% + + [2026-04-29 09:14:05] iteration 18/ 20 | consumed samples: 72 | elapsed time per iteration (ms): 2162.8 | throughput (token/sec/GPU): 703.9 | learning rate: 3.432342E-06 | global batch size: 4 | lm loss: 1.139007E+01 | loss scale: 1.0 | grad norm: 0.671 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 18 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2485.46, 2485.70) +Dataloader batch - tokens shape: torch.Size([1, 1394]) +Dataloader batch - labels shape: torch.Size([1, 1394]) +Dataloader batch - attn_mask shape: torch.Size([1, 1394]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1747]) +Dataloader batch - labels shape: torch.Size([1, 1747]) +Dataloader batch - attn_mask shape: torch.Size([1, 1747]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1382]) +Dataloader batch - labels shape: torch.Size([1, 1382]) +Dataloader batch - attn_mask shape: Dataloader batch - tokens shape:torch.Size([1, 1382]) torch.Size([1, 1478]) + +Dataloader batch - labels shape: torch.Size([1, 1478]) +Dataloader batch - attn_mask shape: torch.Size([1, 1478]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1747) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1747) torch.int64 + loss_mask : 390 / 1747 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1747, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1747, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1747, 1, 576) torch.bfloat16 + out : (1, 1747) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3062 + top-1 accuracy : 5/390 = 1.3% + + [2026-04-29 09:14:09] iteration 19/ 20 | consumed samples: 76 | elapsed time per iteration (ms): 2270.7 | throughput (token/sec/GPU): 660.7 | learning rate: 1.611867E-06 | global batch size: 4 | lm loss: 1.133604E+01 | loss scale: 1.0 | grad norm: 0.744 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloadersaving checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader + +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 19 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2558.12, 2561.94) +Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1897]) +Dataloader batch - labels shape: torch.Size([1, 1897]) +torch.Size([1, 1519])Dataloader batch - attn_mask shape: + Dataloader batch - labels shape:torch.Size([1, 1897]) + Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) +torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1457]) +Dataloader batch - labels shape: torch.Size([1, 1457]) +Dataloader batch - attn_mask shape: torch.Size([1, 1457]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) +Dataloader batch - tokens shape: torch.Size([1, 1936]) +Dataloader batch - labels shape: torch.Size([1, 1936]) +Dataloader batch - attn_mask shape: torch.Size([1, 1936]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1936) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1936) torch.int64 + loss_mask : 569 / 1936 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1936, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1936, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1936, 1, 576) torch.bfloat16 + out : (1, 1936) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 569 tokens) : 11.4807 + top-1 accuracy : 11/569 = 1.9% + + [2026-04-29 09:14:14] iteration 20/ 20 | consumed samples: 80 | elapsed time per iteration (ms): 2226.9 | throughput (token/sec/GPU): 764.4 | learning rate: 1.000000E-06 | global batch size: 4 | lm loss: 1.141078E+01 | loss scale: 1.0 | grad norm: 0.661 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (2193.58, 2194.32) + forward-compute ................................: (1623.25, 1625.61) + backward-compute ...............................: (388.65, 558.12) + batch-generator ................................: (299.87, 523.16) + layernorm-grads-all-reduce .....................: (0.02, 0.18) + embedding-grads-all-reduce .....................: (0.05, 0.15) + all-grads-sync .................................: (5.97, 6.40) + params-all-gather ..............................: (1.33, 1.39) + optimizer-copy-to-main-grad ....................: (0.10, 0.56) + optimizer-clip-main-grad .......................: (1.46, 2.83) + optimizer-count-zeros ..........................: (0.01, 0.29) + optimizer-inner-step ...........................: (0.70, 2.98) + optimizer-copy-main-to-model-params ............: (0.11, 0.67) + optimizer ......................................: (16.01, 16.23) +saving checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader +saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 20 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2488.37, 2488.65) +[after training is done] datetime: 2026-04-29 09:14:17 +wandb: uploading artifact stage_1_alignment_mobilellm_140m; updating run metadata; uploading output.log +wandb: uploading artifact stage_1_alignment_mobilellm_140m +wandb: +wandb: +wandb: Run history: +wandb: Token throughput (per-sec-per-GPU) ▁▄▅▇▅▅▇▆▆▇▆▆▇█▇▆█▇▆▇ +wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: grad-norm ██▄▃▃▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁ +wandb: iteration-time █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: learning-rate ███▇▇▇▆▆▅▅▄▃▃▂▂▂▁▁▁▁ +wandb: lm loss ██▆▅▄▄▃▃▂▃▃▁▂▂▂▂▂▂▁▂ +wandb: loss-scale ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: num-zeros ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: samples vs steps ▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██ +wandb: +wandb: Run summary: +wandb: Token throughput (per-sec-per-GPU) 764.39014 +wandb: batch-size 4 +wandb: grad-norm 0.66135 +wandb: iteration-time 2.22694 +wandb: learning-rate 0.0 +wandb: lm loss 11.41078 +wandb: loss-scale 1 +wandb: num-zeros 0 +wandb: samples vs steps 80 +wandb: +wandb: 🚀 View run mobilellm_integration at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/1ll8kzkt +wandb: ⭐️ View project at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 +wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) +wandb: Find logs at: stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_091210-1ll8kzkt/logs +[rank0]:[W429 09:15:39.766225268 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W429 09:15:40.686778180 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W429 09:15:40.170121945 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W429 09:15:43.622688209 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log new file mode 100644 index 00000000..f848dedc --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log @@ -0,0 +1,16036 @@ +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +False +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +parse arguments function called +-------------- Configure model to llava-ov-mobilellm-140m -------------- +[DEBUG] Model config type: LlavaOnevision1_5Config +[DEBUG] Is MobileLLM: True + num_layers = 15 + hidden_size = 576 + ffn_hidden_size = 2048 + num_attention_heads = 9 + group_query_attention = True + num_query_groups = 3 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = False + vocab_size_in_config_file = 128256 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-05 + rotary_base = 8000000 + kv_channels = None + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.local + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... llama3 + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 15 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 32768 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + fastvit_image_size .............................. 1024 + ffn_hidden_size ................................. 2048 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 4 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M + hidden_dropout .................................. 0 + hidden_size ..................................... 576 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_aspect_ratio .............................. pad + image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ stage_1_alignment_mobilellm_140m + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 100 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-mobilellm-140m + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... True + no_load_rng ..................................... True + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 9 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 15 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 3 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 16 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 3 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 8000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_mobilellm_140m + save_ema ........................................ None + save_interval ................................... 1 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 32768 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 100 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 32768 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. False + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_fastvit ..................................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vision_tower_name ............................... mobileclip_l_1024 + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 128256 + vpp_scheduler ................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +Using unified model type for training. + +Starting training... +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 4 +> setting tensorboard ... +> AIAK building HFTokenizer tokenizer ... +INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 +WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. +WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. +WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). + > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so +make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.062 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W429 12:22:17.464874121 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.221 seconds +time to initialize megatron (seconds): 3.815 +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2026-04-29 12:22:19 +building llava-ov-mobilellm-140m model ... +[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m +[DEBUG PROVIDER] Args num_layers: 15 +[DEBUG PROVIDER] Args hidden_size: 576 +[DEBUG PROVIDER] Args vocab_size: 128260 +[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 +[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 +[DEBUG PROVIDER] Model family: llava_ov_1_5 +[DEBUG PROVIDER] use_mobilellm flag: True +[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone +[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 +[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Loading vision config... +[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 +[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json +[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd +[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ================================================ +[DEBUG PROVIDER] Loading adapter config... +[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 +[DEBUG PROVIDER] Building layer specs... +[DEBUG PROVIDER] ✓ Using MobileLLM layer specification +[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] MobileLLM layer spec created successfully +[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... +[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 +[INFO] Using MobileLLM as language backbone +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (11216448 elements, 11216512 padded size): + module.adapter.layernorm.bias + module.adapter.linear_fc1.linear.weight + module.adapter.layernorm.weight + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.bias + module.adapter.linear_fc2.linear.weight +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[DEBUG] FastViT enabled: use_fastvit=True +[DEBUG] Before handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +FastViT enabled: Will resume/load checkpoint from: stage_1_alignment_mobilellm_140m +[DEBUG] After handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + loading checkpoint from stage_1_alignment_mobilellm_140m at iteration 20 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine disabled via CLI, ignoring machine state saved in checkpoint + successfully loaded checkpoint from stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] at iteration 20 +(min, max) time across ranks (ms): + load-checkpoint ................................: (514.72, 514.72) +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 12:22:22 +================================================================================ +FULL MODEL STRUCTURE: +================================================================================ + +--- Model 0 (pipeline rank 0) --- +DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +) +================================================================================ + +================================================================================ +PARAMETER TRAINABILITY STATUS: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +================================================================================ +Total trainable parameters: 11,216,448 (11.22M) +Total frozen parameters: 265,417,440 (265.42M) +Total parameters: 276,633,888 (276.63M) +Trainable percentage: 4.05% +================================================================================ +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 400 + validation: 400 + test: 400 +Using shared training tokenizer in FastVLM mode +Ensured multimodal special tokens for FastVLM tokenizer; added=0 +Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M +Initialized FastViT processor with image_size=1024 +Processor config: .TokenizerWrapper object at 0x7f57ec6afe20> +image_resolution: None +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 1500), ] sum(count)=1500 +rank=0, worker=1: shard_range=[pretrain-000003.tar[1500, 2999), ] sum(count)=1499 +rank=0, worker=2: shard_range=[pretrain-000003.tar[2999, 4498), ] sum(count)=1499 +rank=0, worker=3: shard_range=[pretrain-000003.tar[4498, 5058), pretrain-000001.tar[0, 939), ] sum(count)=1499 +rank=0, worker=4: shard_range=[pretrain-000001.tar[939, 2439), ] sum(count)=1500 +rank=0, worker=5: shard_range=[pretrain-000001.tar[2439, 3938), ] sum(count)=1499 +rank=0, worker=6: shard_range=[pretrain-000001.tar[3938, 5036), pretrain-000002.tar[0, 401), ] sum(count)=1499 +rank=0, worker=7: shard_range=[pretrain-000002.tar[401, 1900), ] sum(count)=1499 +rank=0, worker=8: shard_range=[pretrain-000002.tar[1900, 3400), ] sum(count)=1500 +rank=0, worker=9: shard_range=[pretrain-000002.tar[3400, 4899), ] sum(count)=1499 +rank=0, worker=10: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 1359), ] sum(count)=1499 +rank=0, worker=11: shard_range=[pretrain-000004.tar[1359, 2858), ] sum(count)=1499 +rank=0, worker=12: shard_range=[pretrain-000004.tar[2858, 3821), pretrain-000000.tar[0, 537), ] sum(count)=1500 +rank=0, worker=13: shard_range=[pretrain-000000.tar[537, 2036), ] sum(count)=1499 +rank=0, worker=14: shard_range=[pretrain-000000.tar[2036, 3535), ] sum(count)=1499 +rank=0, worker=15: shard_range=[pretrain-000000.tar[3535, 5034), ] sum(count)=1499 +loading dataset state failed. Skipping. Shard ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=0, count=375, byte_offset=0, byte_size=240076800) not found in [[[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=0, count=1500, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=1500, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=2999, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=4498, count=560, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=0, count=939, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=939, count=1500, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=2439, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=3938, count=1098, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=0, count=401, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=401, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=1900, count=1500, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=3400, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=4899, count=140, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000004.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000004.tar'), offset=0, count=1359, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000004.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000004.tar'), offset=1359, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000004.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000004.tar'), offset=2858, count=963, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=0, count=537, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=537, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=2036, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=3535, count=1499, byte_offset=None, byte_size=None)]]], states differ, not recoverable +[after dataloaders are built] datetime: 2026-04-29 12:22:22 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3066.93, 3066.93) + train/valid/test-data-iterators-setup ..........: (79.38, 79.38) +training ... + +================================================================================ +training with the following parameter status: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +Setting rerun_state_machine.current_iteration to 20... +[before the start of training step] datetime: 2026-04-29 12:22:23 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 1662]) +Dataloader batch - labels shape: torch.Size([1, 1662]) +Dataloader batch - attn_mask shape: torch.Size([1, 1662]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1662) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1662) torch.int64 + loss_mask : 412 / 1662 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1662, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1662, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1662, 1, 576) torch.bfloat16 + out : (1, 1662) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3501 + top-1 accuracy : 10/412 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1412) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1412) torch.int64 + loss_mask : 358 / 1412 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1412, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1412, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1412, 1, 576) torch.bfloat16 + out : (1, 1412) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.3680 + top-1 accuracy : 3/358 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1175]) +Dataloader batch - labels shape: torch.Size([1, 1175]) +Dataloader batch - attn_mask shape: torch.Size([1, 1175]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1175) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1175) torch.int64 + loss_mask : 278 / 1175 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1175, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1175, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1175, 1, 576) torch.bfloat16 + out : (1, 1175) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.3444 + top-1 accuracy : 11/278 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1470) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1470) torch.int64 + loss_mask : 421 / 1470 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1470, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1470, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1470, 1, 576) torch.bfloat16 + out : (1, 1470) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 421 tokens) : 11.4015 + top-1 accuracy : 0/421 = 0.0% + +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once + [2026-04-29 12:22:43] iteration 21/ 100 | consumed samples: 84 | elapsed time per iteration (ms): 20059.9 | throughput (token/sec/GPU): 285.1 | learning rate: 9.998430E-05 | global batch size: 4 | lm loss: 1.136812E+01 | loss scale: 1.0 | grad norm: 0.652 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer layers in billions: 0.07 +Number of parameters in embedding layers in billions: 0.07 +Total number of parameters in billions: 0.14 +Number of parameters in most loaded shard in billions: 0.1403 +Theoretical memory footprints: weight and optimizer=2409.10 MB +[Rank 0] (after 21 iterations) memory (MB) | allocated: 789.76123046875 | max allocated: 5130.18212890625 | reserved: 6046.0 | max reserved: 6046.0 +saving checkpoint at iteration 21 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 21 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 21 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1668.41, 1668.41) +Dataloader batch - tokens shape: torch.Size([1, 1909]) +Dataloader batch - labels shape: torch.Size([1, 1909]) +Dataloader batch - attn_mask shape: torch.Size([1, 1909]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 532 / 1909 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 532 tokens) : 11.4621 + top-1 accuracy : 0/532 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 367 / 1541 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.3489 + top-1 accuracy : 15/367 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1026]) +Dataloader batch - labels shape: torch.Size([1, 1026]) +Dataloader batch - attn_mask shape: torch.Size([1, 1026]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1026) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1026) torch.int64 + loss_mask : 200 / 1026 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1026, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1026, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1026, 1, 576) torch.bfloat16 + out : (1, 1026) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 200 tokens) : 11.2604 + top-1 accuracy : 4/200 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1481]) +Dataloader batch - labels shape: torch.Size([1, 1481]) +Dataloader batch - attn_mask shape: torch.Size([1, 1481]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1481) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1481) torch.int64 + loss_mask : 350 / 1481 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1481, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1481, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1481, 1, 576) torch.bfloat16 + out : (1, 1481) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.3202 + top-1 accuracy : 11/350 = 3.1% + + [2026-04-29 12:22:49] iteration 22/ 100 | consumed samples: 88 | elapsed time per iteration (ms): 4845.6 | throughput (token/sec/GPU): 1229.4 | learning rate: 9.992056E-05 | global batch size: 4 | lm loss: 1.137132E+01 | loss scale: 1.0 | grad norm: 0.740 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 22 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 22 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 22 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1050.60, 1050.60) +Dataloader batch - tokens shape: torch.Size([1, 1757]) +Dataloader batch - labels shape: torch.Size([1, 1757]) +Dataloader batch - attn_mask shape: torch.Size([1, 1757]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1757) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1757) torch.int64 + loss_mask : 383 / 1757 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1757, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1757, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1757, 1, 576) torch.bfloat16 + out : (1, 1757) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.3092 + top-1 accuracy : 14/383 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1214]) +Dataloader batch - labels shape: torch.Size([1, 1214]) +Dataloader batch - attn_mask shape: torch.Size([1, 1214]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1214) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1214) torch.int64 + loss_mask : 289 / 1214 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1214, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1214, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1214, 1, 576) torch.bfloat16 + out : (1, 1214) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 289 tokens) : 11.3197 + top-1 accuracy : 2/289 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1673]) +Dataloader batch - labels shape: torch.Size([1, 1673]) +Dataloader batch - attn_mask shape: torch.Size([1, 1673]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1673) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1673) torch.int64 + loss_mask : 394 / 1673 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1673, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1673, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1673, 1, 576) torch.bfloat16 + out : (1, 1673) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3795 + top-1 accuracy : 10/394 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 963]) +Dataloader batch - labels shape: torch.Size([1, 963]) +Dataloader batch - attn_mask shape: torch.Size([1, 963]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 963) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 963) torch.int64 + loss_mask : 223 / 963 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (963, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (963, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (963, 1, 576) torch.bfloat16 + out : (1, 963) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 223 tokens) : 11.3192 + top-1 accuracy : 17/223 = 7.6% + + [2026-04-29 12:22:55] iteration 23/ 100 | consumed samples: 92 | elapsed time per iteration (ms): 4616.3 | throughput (token/sec/GPU): 1214.6 | learning rate: 9.980785E-05 | global batch size: 4 | lm loss: 1.133477E+01 | loss scale: 1.0 | grad norm: 0.832 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 23 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 23 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 23 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1180.81, 1180.81) +Dataloader batch - tokens shape: torch.Size([1, 1430]) +Dataloader batch - labels shape: torch.Size([1, 1430]) +Dataloader batch - attn_mask shape: torch.Size([1, 1430]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1430) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1430) torch.int64 + loss_mask : 382 / 1430 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1430, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1430, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1430, 1, 576) torch.bfloat16 + out : (1, 1430) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.3642 + top-1 accuracy : 16/382 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1191) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1191) torch.int64 + loss_mask : 280 / 1191 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1191, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1191, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1191, 1, 576) torch.bfloat16 + out : (1, 1191) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 280 tokens) : 11.2816 + top-1 accuracy : 19/280 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 392 / 1431 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3694 + top-1 accuracy : 21/392 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1535]) +Dataloader batch - labels shape: torch.Size([1, 1535]) +Dataloader batch - attn_mask shape: torch.Size([1, 1535]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1535) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1535) torch.int64 + loss_mask : 349 / 1535 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1535, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1535, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1535, 1, 576) torch.bfloat16 + out : (1, 1535) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.3177 + top-1 accuracy : 14/349 = 4.0% + + [2026-04-29 12:23:00] iteration 24/ 100 | consumed samples: 96 | elapsed time per iteration (ms): 4313.0 | throughput (token/sec/GPU): 1295.4 | learning rate: 9.964628E-05 | global batch size: 4 | lm loss: 1.133762E+01 | loss scale: 1.0 | grad norm: 1.035 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 24 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 24 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 24 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1271.39, 1271.39) +Dataloader batch - tokens shape: torch.Size([1, 1394]) +Dataloader batch - labels shape: torch.Size([1, 1394]) +Dataloader batch - attn_mask shape: torch.Size([1, 1394]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1394) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1394) torch.int64 + loss_mask : 317 / 1394 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1394, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1394, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1394, 1, 576) torch.bfloat16 + out : (1, 1394) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2622 + top-1 accuracy : 25/317 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1190]) +Dataloader batch - labels shape: torch.Size([1, 1190]) +Dataloader batch - attn_mask shape: torch.Size([1, 1190]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1190) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1190) torch.int64 + loss_mask : 278 / 1190 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1190, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1190, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1190, 1, 576) torch.bfloat16 + out : (1, 1190) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.2963 + top-1 accuracy : 22/278 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1554]) +Dataloader batch - labels shape: torch.Size([1, 1554]) +Dataloader batch - attn_mask shape: torch.Size([1, 1554]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1554) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1554) torch.int64 + loss_mask : 333 / 1554 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1554, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1554, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1554, 1, 576) torch.bfloat16 + out : (1, 1554) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.2884 + top-1 accuracy : 26/333 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 392 / 1443 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3861 + top-1 accuracy : 17/392 = 4.3% + + [2026-04-29 12:23:06] iteration 25/ 100 | consumed samples: 100 | elapsed time per iteration (ms): 4563.5 | throughput (token/sec/GPU): 1223.0 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.131279E+01 | loss scale: 1.0 | grad norm: 1.040 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 25 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 25 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 25 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1248.59, 1248.59) +Dataloader batch - tokens shape: torch.Size([1, 1395]) +Dataloader batch - labels shape: torch.Size([1, 1395]) +Dataloader batch - attn_mask shape: torch.Size([1, 1395]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 21 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1395) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1395) torch.int64 + loss_mask : 357 / 1395 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1395, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1395, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1395, 1, 576) torch.bfloat16 + out : (1, 1395) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3336 + top-1 accuracy : 19/357 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1553]) +Dataloader batch - labels shape: torch.Size([1, 1553]) +Dataloader batch - attn_mask shape: torch.Size([1, 1553]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 22 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1553) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1553) torch.int64 + loss_mask : 369 / 1553 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1553, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1553, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1553, 1, 576) torch.bfloat16 + out : (1, 1553) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2875 + top-1 accuracy : 27/369 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1177]) +Dataloader batch - labels shape: torch.Size([1, 1177]) +Dataloader batch - attn_mask shape: torch.Size([1, 1177]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 23 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1177) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1177) torch.int64 + loss_mask : 299 / 1177 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1177, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1177, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1177, 1, 576) torch.bfloat16 + out : (1, 1177) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 299 tokens) : 11.3431 + top-1 accuracy : 21/299 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1588]) +Dataloader batch - labels shape: torch.Size([1, 1588]) +Dataloader batch - attn_mask shape: torch.Size([1, 1588]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 24 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1588) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1588) torch.int64 + loss_mask : 455 / 1588 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1588, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1588, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1588, 1, 576) torch.bfloat16 + out : (1, 1588) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 455 tokens) : 11.3779 + top-1 accuracy : 16/455 = 3.5% + + [2026-04-29 12:23:12] iteration 26/ 100 | consumed samples: 104 | elapsed time per iteration (ms): 4397.4 | throughput (token/sec/GPU): 1299.2 | learning rate: 9.917726E-05 | global batch size: 4 | lm loss: 1.133764E+01 | loss scale: 1.0 | grad norm: 0.761 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 26 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 26 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 26 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1811.49, 1811.49) +Dataloader batch - tokens shape: torch.Size([1, 1498]) +Dataloader batch - labels shape: torch.Size([1, 1498]) +Dataloader batch - attn_mask shape: torch.Size([1, 1498]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 25 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1498) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1498) torch.int64 + loss_mask : 357 / 1498 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1498, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1498, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1498, 1, 576) torch.bfloat16 + out : (1, 1498) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3106 + top-1 accuracy : 16/357 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1553]) +Dataloader batch - labels shape: torch.Size([1, 1553]) +Dataloader batch - attn_mask shape: torch.Size([1, 1553]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 26 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1553) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1553) torch.int64 + loss_mask : 369 / 1553 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1553, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1553, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1553, 1, 576) torch.bfloat16 + out : (1, 1553) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.3102 + top-1 accuracy : 28/369 = 7.6% + +Dataloader batch - tokens shape: torch.Size([1, 1694]) +Dataloader batch - labels shape: torch.Size([1, 1694]) +Dataloader batch - attn_mask shape: torch.Size([1, 1694]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 27 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1694) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1694) torch.int64 + loss_mask : 348 / 1694 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1694, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1694, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1694, 1, 576) torch.bfloat16 + out : (1, 1694) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2493 + top-1 accuracy : 32/348 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1736]) +Dataloader batch - labels shape: torch.Size([1, 1736]) +Dataloader batch - attn_mask shape: torch.Size([1, 1736]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 28 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1736) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1736) torch.int64 + loss_mask : 365 / 1736 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1736, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1736, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1736, 1, 576) torch.bfloat16 + out : (1, 1736) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2502 + top-1 accuracy : 25/365 = 6.8% + + [2026-04-29 12:23:19] iteration 27/ 100 | consumed samples: 108 | elapsed time per iteration (ms): 5274.6 | throughput (token/sec/GPU): 1228.7 | learning rate: 9.887027E-05 | global batch size: 4 | lm loss: 1.128034E+01 | loss scale: 1.0 | grad norm: 0.887 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 27 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 27 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 27 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1079.35, 1079.35) +Dataloader batch - tokens shape: torch.Size([1, 1793]) +Dataloader batch - labels shape: torch.Size([1, 1793]) +Dataloader batch - attn_mask shape: torch.Size([1, 1793]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 29 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1793) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1793) torch.int64 + loss_mask : 435 / 1793 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1793, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1793, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1793, 1, 576) torch.bfloat16 + out : (1, 1793) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 435 tokens) : 11.3283 + top-1 accuracy : 28/435 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1457]) +Dataloader batch - labels shape: torch.Size([1, 1457]) +Dataloader batch - attn_mask shape: torch.Size([1, 1457]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 30 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1457) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1457) torch.int64 + loss_mask : 405 / 1457 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1457, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1457, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1457, 1, 576) torch.bfloat16 + out : (1, 1457) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.4073 + top-1 accuracy : 1/405 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1529]) +Dataloader batch - labels shape: torch.Size([1, 1529]) +Dataloader batch - attn_mask shape: torch.Size([1, 1529]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 31 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1529) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1529) torch.int64 + loss_mask : 390 / 1529 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1529, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1529, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1529, 1, 576) torch.bfloat16 + out : (1, 1529) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.2891 + top-1 accuracy : 2/390 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1381]) +Dataloader batch - labels shape: torch.Size([1, 1381]) +Dataloader batch - attn_mask shape: torch.Size([1, 1381]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 32 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1381) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1381) torch.int64 + loss_mask : 300 / 1381 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1381, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1381, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1381, 1, 576) torch.bfloat16 + out : (1, 1381) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.2697 + top-1 accuracy : 0/300 = 0.0% + + [2026-04-29 12:23:25] iteration 28/ 100 | consumed samples: 112 | elapsed time per iteration (ms): 4728.9 | throughput (token/sec/GPU): 1302.6 | learning rate: 9.851536E-05 | global batch size: 4 | lm loss: 1.132775E+01 | loss scale: 1.0 | grad norm: 0.749 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 28 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 28 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 28 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1587.02, 1587.02) +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 33 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 364 / 1542 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.3259 + top-1 accuracy : 21/364 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1399]) +Dataloader batch - labels shape: torch.Size([1, 1399]) +Dataloader batch - attn_mask shape: torch.Size([1, 1399]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 34 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1399) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1399) torch.int64 + loss_mask : 422 / 1399 tokens (30.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1399, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1399, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1399, 1, 576) torch.bfloat16 + out : (1, 1399) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.4425 + top-1 accuracy : 7/422 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1777]) +Dataloader batch - labels shape: torch.Size([1, 1777]) +Dataloader batch - attn_mask shape: torch.Size([1, 1777]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 35 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1777) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1777) torch.int64 + loss_mask : 409 / 1777 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1777, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1777, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1777, 1, 576) torch.bfloat16 + out : (1, 1777) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.2908 + top-1 accuracy : 22/409 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1731]) +Dataloader batch - labels shape: torch.Size([1, 1731]) +Dataloader batch - attn_mask shape: torch.Size([1, 1731]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 36 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1731) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1731) torch.int64 + loss_mask : 451 / 1731 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1731, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1731, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1731, 1, 576) torch.bfloat16 + out : (1, 1731) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 451 tokens) : 11.3920 + top-1 accuracy : 7/451 = 1.6% + + [2026-04-29 12:23:31] iteration 29/ 100 | consumed samples: 116 | elapsed time per iteration (ms): 4867.4 | throughput (token/sec/GPU): 1324.9 | learning rate: 9.811288E-05 | global batch size: 4 | lm loss: 1.136518E+01 | loss scale: 1.0 | grad norm: 0.752 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 29 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 29 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 29 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1150.08, 1150.08) +Dataloader batch - tokens shape: torch.Size([1, 1205]) +Dataloader batch - labels shape: torch.Size([1, 1205]) +Dataloader batch - attn_mask shape: torch.Size([1, 1205]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 37 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1205) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1205) torch.int64 + loss_mask : 300 / 1205 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1205, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1205, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1205, 1, 576) torch.bfloat16 + out : (1, 1205) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.3494 + top-1 accuracy : 10/300 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 38 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 377 / 1475 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.3295 + top-1 accuracy : 22/377 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1472]) +Dataloader batch - labels shape: torch.Size([1, 1472]) +Dataloader batch - attn_mask shape: torch.Size([1, 1472]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 39 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1472) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1472) torch.int64 + loss_mask : 312 / 1472 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1472, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1472, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1472, 1, 576) torch.bfloat16 + out : (1, 1472) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.2301 + top-1 accuracy : 23/312 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1040]) +Dataloader batch - labels shape: torch.Size([1, 1040]) +Dataloader batch - attn_mask shape: torch.Size([1, 1040]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 40 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1040) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1040) torch.int64 + loss_mask : 215 / 1040 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1040, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1040, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1040, 1, 576) torch.bfloat16 + out : (1, 1040) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 215 tokens) : 11.1874 + top-1 accuracy : 21/215 = 9.8% + + [2026-04-29 12:23:36] iteration 30/ 100 | consumed samples: 120 | elapsed time per iteration (ms): 4165.8 | throughput (token/sec/GPU): 1246.3 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.128331E+01 | loss scale: 1.0 | grad norm: 0.630 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 30 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 30 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 30 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1813.45, 1813.45) +Dataloader batch - tokens shape: torch.Size([1, 1432]) +Dataloader batch - labels shape: torch.Size([1, 1432]) +Dataloader batch - attn_mask shape: torch.Size([1, 1432]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 41 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1432) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1432) torch.int64 + loss_mask : 385 / 1432 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1432, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1432, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1432, 1, 576) torch.bfloat16 + out : (1, 1432) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.3653 + top-1 accuracy : 27/385 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1408]) +Dataloader batch - labels shape: torch.Size([1, 1408]) +Dataloader batch - attn_mask shape: torch.Size([1, 1408]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 42 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1408) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1408) torch.int64 + loss_mask : 348 / 1408 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1408, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1408, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1408, 1, 576) torch.bfloat16 + out : (1, 1408) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.3456 + top-1 accuracy : 15/348 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1087]) +Dataloader batch - labels shape: torch.Size([1, 1087]) +Dataloader batch - attn_mask shape: torch.Size([1, 1087]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 43 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1087) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1087) torch.int64 + loss_mask : 244 / 1087 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1087, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1087, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1087, 1, 576) torch.bfloat16 + out : (1, 1087) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 244 tokens) : 11.2286 + top-1 accuracy : 20/244 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1074]) +Dataloader batch - labels shape: torch.Size([1, 1074]) +Dataloader batch - attn_mask shape: torch.Size([1, 1074]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 44 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1074) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1074) torch.int64 + loss_mask : 310 / 1074 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1074, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1074, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1074, 1, 576) torch.bfloat16 + out : (1, 1074) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 310 tokens) : 11.3944 + top-1 accuracy : 14/310 = 4.5% + + [2026-04-29 12:23:42] iteration 31/ 100 | consumed samples: 124 | elapsed time per iteration (ms): 3981.7 | throughput (token/sec/GPU): 1256.0 | learning rate: 9.716682E-05 | global batch size: 4 | lm loss: 1.134109E+01 | loss scale: 1.0 | grad norm: 0.628 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 31 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 31 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 31 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1849.03, 1849.03) +Dataloader batch - tokens shape: torch.Size([1, 1383]) +Dataloader batch - labels shape: torch.Size([1, 1383]) +Dataloader batch - attn_mask shape: torch.Size([1, 1383]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 45 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1383) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1383) torch.int64 + loss_mask : 403 / 1383 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1383, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1383, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1383, 1, 576) torch.bfloat16 + out : (1, 1383) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.3868 + top-1 accuracy : 14/403 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1291]) +Dataloader batch - labels shape: torch.Size([1, 1291]) +Dataloader batch - attn_mask shape: torch.Size([1, 1291]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 46 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1291) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1291) torch.int64 + loss_mask : 396 / 1291 tokens (30.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1291, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1291, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1291, 1, 576) torch.bfloat16 + out : (1, 1291) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 396 tokens) : 11.3955 + top-1 accuracy : 0/396 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1228]) +Dataloader batch - labels shape: torch.Size([1, 1228]) +Dataloader batch - attn_mask shape: torch.Size([1, 1228]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 47 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1228) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1228) torch.int64 + loss_mask : 338 / 1228 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1228, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1228, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1228, 1, 576) torch.bfloat16 + out : (1, 1228) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.3653 + top-1 accuracy : 13/338 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1180]) +Dataloader batch - labels shape: torch.Size([1, 1180]) +Dataloader batch - attn_mask shape: torch.Size([1, 1180]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 48 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1180) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1180) torch.int64 + loss_mask : 260 / 1180 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1180, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1180, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1180, 1, 576) torch.bfloat16 + out : (1, 1180) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.3098 + top-1 accuracy : 5/260 = 1.9% + + [2026-04-29 12:23:48] iteration 32/ 100 | consumed samples: 128 | elapsed time per iteration (ms): 3917.0 | throughput (token/sec/GPU): 1297.4 | learning rate: 9.662419E-05 | global batch size: 4 | lm loss: 1.136973E+01 | loss scale: 1.0 | grad norm: 0.507 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 32 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 32 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 32 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2131.50, 2131.50) +Dataloader batch - tokens shape: torch.Size([1, 1219]) +Dataloader batch - labels shape: torch.Size([1, 1219]) +Dataloader batch - attn_mask shape: torch.Size([1, 1219]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 49 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1219) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1219) torch.int64 + loss_mask : 291 / 1219 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1219, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1219, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1219, 1, 576) torch.bfloat16 + out : (1, 1219) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 291 tokens) : 11.2999 + top-1 accuracy : 19/291 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1748]) +Dataloader batch - labels shape: torch.Size([1, 1748]) +Dataloader batch - attn_mask shape: torch.Size([1, 1748]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 50 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1748) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1748) torch.int64 + loss_mask : 380 / 1748 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1748, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1748, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1748, 1, 576) torch.bfloat16 + out : (1, 1748) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 380 tokens) : 11.2963 + top-1 accuracy : 24/380 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1873]) +Dataloader batch - labels shape: torch.Size([1, 1873]) +Dataloader batch - attn_mask shape: torch.Size([1, 1873]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 51 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1873) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1873) torch.int64 + loss_mask : 510 / 1873 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1873, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1873, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1873, 1, 576) torch.bfloat16 + out : (1, 1873) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 510 tokens) : 11.3447 + top-1 accuracy : 20/510 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1049]) +Dataloader batch - labels shape: torch.Size([1, 1049]) +Dataloader batch - attn_mask shape: torch.Size([1, 1049]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 52 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1049) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1049) torch.int64 + loss_mask : 265 / 1049 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1049, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1049, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1049, 1, 576) torch.bfloat16 + out : (1, 1049) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 265 tokens) : 11.3075 + top-1 accuracy : 1/265 = 0.4% + + [2026-04-29 12:23:55] iteration 33/ 100 | consumed samples: 132 | elapsed time per iteration (ms): 4549.3 | throughput (token/sec/GPU): 1294.5 | learning rate: 9.603585E-05 | global batch size: 4 | lm loss: 1.131618E+01 | loss scale: 1.0 | grad norm: 0.554 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 33 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 33 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 33 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2167.14, 2167.14) +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 53 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 392 / 1539 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3436 + top-1 accuracy : 18/392 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1454]) +Dataloader batch - labels shape: torch.Size([1, 1454]) +Dataloader batch - attn_mask shape: torch.Size([1, 1454]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 54 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1454) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1454) torch.int64 + loss_mask : 410 / 1454 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1454, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1454, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1454, 1, 576) torch.bfloat16 + out : (1, 1454) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.3232 + top-1 accuracy : 6/410 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1785]) +Dataloader batch - labels shape: torch.Size([1, 1785]) +Dataloader batch - attn_mask shape: torch.Size([1, 1785]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 55 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1785) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1785) torch.int64 + loss_mask : 426 / 1785 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1785, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1785, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1785, 1, 576) torch.bfloat16 + out : (1, 1785) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.3332 + top-1 accuracy : 21/426 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1565]) +Dataloader batch - labels shape: torch.Size([1, 1565]) +Dataloader batch - attn_mask shape: torch.Size([1, 1565]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 56 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1565) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1565) torch.int64 + loss_mask : 363 / 1565 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1565, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1565, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1565, 1, 576) torch.bfloat16 + out : (1, 1565) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2834 + top-1 accuracy : 10/363 = 2.8% + + [2026-04-29 12:24:02] iteration 34/ 100 | consumed samples: 136 | elapsed time per iteration (ms): 4818.4 | throughput (token/sec/GPU): 1316.4 | learning rate: 9.540239E-05 | global batch size: 4 | lm loss: 1.132181E+01 | loss scale: 1.0 | grad norm: 0.495 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 34 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 34 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 34 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1573.58, 1573.58) +Dataloader batch - tokens shape: torch.Size([1, 1025]) +Dataloader batch - labels shape: torch.Size([1, 1025]) +Dataloader batch - attn_mask shape: torch.Size([1, 1025]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 57 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1025) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1025) torch.int64 + loss_mask : 277 / 1025 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1025, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1025, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1025, 1, 576) torch.bfloat16 + out : (1, 1025) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 277 tokens) : 11.3252 + top-1 accuracy : 6/277 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1516]) +Dataloader batch - labels shape: torch.Size([1, 1516]) +Dataloader batch - attn_mask shape: torch.Size([1, 1516]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 58 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1516) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1516) torch.int64 + loss_mask : 325 / 1516 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1516, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1516, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1516, 1, 576) torch.bfloat16 + out : (1, 1516) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.2322 + top-1 accuracy : 20/325 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1899]) +Dataloader batch - labels shape: torch.Size([1, 1899]) +Dataloader batch - attn_mask shape: torch.Size([1, 1899]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 59 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1899) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1899) torch.int64 + loss_mask : 543 / 1899 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1899, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1899, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1899, 1, 576) torch.bfloat16 + out : (1, 1899) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 543 tokens) : 11.3966 + top-1 accuracy : 25/543 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1010]) +Dataloader batch - labels shape: torch.Size([1, 1010]) +Dataloader batch - attn_mask shape: torch.Size([1, 1010]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 60 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1010) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1010) torch.int64 + loss_mask : 225 / 1010 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1010, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1010, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1010, 1, 576) torch.bfloat16 + out : (1, 1010) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 225 tokens) : 11.2648 + top-1 accuracy : 14/225 = 6.2% + + [2026-04-29 12:24:08] iteration 35/ 100 | consumed samples: 140 | elapsed time per iteration (ms): 4297.1 | throughput (token/sec/GPU): 1268.3 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.132151E+01 | loss scale: 1.0 | grad norm: 0.625 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 35 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 35 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 35 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1235.81, 1235.81) +Dataloader batch - tokens shape: torch.Size([1, 1795]) +Dataloader batch - labels shape: torch.Size([1, 1795]) +Dataloader batch - attn_mask shape: torch.Size([1, 1795]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 61 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1795) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1795) torch.int64 + loss_mask : 428 / 1795 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1795, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1795, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1795, 1, 576) torch.bfloat16 + out : (1, 1795) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 428 tokens) : 11.2667 + top-1 accuracy : 21/428 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 62 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 389 / 1436 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.3462 + top-1 accuracy : 17/389 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1554]) +Dataloader batch - labels shape: torch.Size([1, 1554]) +Dataloader batch - attn_mask shape: torch.Size([1, 1554]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 63 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1554) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1554) torch.int64 + loss_mask : 387 / 1554 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1554, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1554, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1554, 1, 576) torch.bfloat16 + out : (1, 1554) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2994 + top-1 accuracy : 30/387 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 64 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 345 / 1434 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2860 + top-1 accuracy : 30/345 = 8.7% + + [2026-04-29 12:24:14] iteration 36/ 100 | consumed samples: 144 | elapsed time per iteration (ms): 4813.5 | throughput (token/sec/GPU): 1292.0 | learning rate: 9.400269E-05 | global batch size: 4 | lm loss: 1.129912E+01 | loss scale: 1.0 | grad norm: 0.441 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 36 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 36 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 36 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1601.60, 1601.60) +Dataloader batch - tokens shape: torch.Size([1, 1356]) +Dataloader batch - labels shape: torch.Size([1, 1356]) +Dataloader batch - attn_mask shape: torch.Size([1, 1356]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 65 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1356) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1356) torch.int64 + loss_mask : 280 / 1356 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1356, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1356, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1356, 1, 576) torch.bfloat16 + out : (1, 1356) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 280 tokens) : 11.1773 + top-1 accuracy : 25/280 = 8.9% + +Dataloader batch - tokens shape: torch.Size([1, 1244]) +Dataloader batch - labels shape: torch.Size([1, 1244]) +Dataloader batch - attn_mask shape: torch.Size([1, 1244]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 66 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1244) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1244) torch.int64 + loss_mask : 341 / 1244 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1244, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1244, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1244, 1, 576) torch.bfloat16 + out : (1, 1244) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.3379 + top-1 accuracy : 2/341 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 67 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 320 / 1400 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2470 + top-1 accuracy : 26/320 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1218]) +Dataloader batch - labels shape: torch.Size([1, 1218]) +Dataloader batch - attn_mask shape: torch.Size([1, 1218]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 68 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1218) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1218) torch.int64 + loss_mask : 319 / 1218 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1218, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1218, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1218, 1, 576) torch.bfloat16 + out : (1, 1218) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.3451 + top-1 accuracy : 0/319 = 0.0% + + [2026-04-29 12:24:19] iteration 37/ 100 | consumed samples: 148 | elapsed time per iteration (ms): 4129.5 | throughput (token/sec/GPU): 1263.6 | learning rate: 9.323782E-05 | global batch size: 4 | lm loss: 1.128093E+01 | loss scale: 1.0 | grad norm: 0.503 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 37 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 37 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 37 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1650.92, 1650.92) +Dataloader batch - tokens shape: torch.Size([1, 1614]) +Dataloader batch - labels shape: torch.Size([1, 1614]) +Dataloader batch - attn_mask shape: torch.Size([1, 1614]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 69 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1614) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1614) torch.int64 + loss_mask : 356 / 1614 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1614, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1614, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1614, 1, 576) torch.bfloat16 + out : (1, 1614) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2630 + top-1 accuracy : 25/356 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1556]) +Dataloader batch - labels shape: torch.Size([1, 1556]) +Dataloader batch - attn_mask shape: torch.Size([1, 1556]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 70 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1556) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1556) torch.int64 + loss_mask : 369 / 1556 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1556, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1556, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1556, 1, 576) torch.bfloat16 + out : (1, 1556) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2836 + top-1 accuracy : 31/369 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 1023]) +Dataloader batch - labels shape: torch.Size([1, 1023]) +Dataloader batch - attn_mask shape: torch.Size([1, 1023]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 71 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1023) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1023) torch.int64 + loss_mask : 258 / 1023 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1023, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1023, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1023, 1, 576) torch.bfloat16 + out : (1, 1023) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 258 tokens) : 11.2788 + top-1 accuracy : 17/258 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1560]) +Dataloader batch - labels shape: torch.Size([1, 1560]) +Dataloader batch - attn_mask shape: torch.Size([1, 1560]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 72 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1560) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1560) torch.int64 + loss_mask : 421 / 1560 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1560, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1560, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1560, 1, 576) torch.bfloat16 + out : (1, 1560) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 421 tokens) : 11.3482 + top-1 accuracy : 26/421 = 6.2% + + [2026-04-29 12:24:25] iteration 38/ 100 | consumed samples: 152 | elapsed time per iteration (ms): 4543.3 | throughput (token/sec/GPU): 1266.3 | learning rate: 9.243060E-05 | global batch size: 4 | lm loss: 1.129689E+01 | loss scale: 1.0 | grad norm: 0.513 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 38 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 38 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 38 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1617.58, 1617.58) +Dataloader batch - tokens shape: torch.Size([1, 1681]) +Dataloader batch - labels shape: torch.Size([1, 1681]) +Dataloader batch - attn_mask shape: torch.Size([1, 1681]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 73 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1681) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1681) torch.int64 + loss_mask : 408 / 1681 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1681, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1681, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1681, 1, 576) torch.bfloat16 + out : (1, 1681) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3177 + top-1 accuracy : 32/408 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1398]) +Dataloader batch - labels shape: torch.Size([1, 1398]) +Dataloader batch - attn_mask shape: torch.Size([1, 1398]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 74 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1398) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1398) torch.int64 + loss_mask : 328 / 1398 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1398, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1398, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1398, 1, 576) torch.bfloat16 + out : (1, 1398) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.2624 + top-1 accuracy : 15/328 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1853]) +Dataloader batch - labels shape: torch.Size([1, 1853]) +Dataloader batch - attn_mask shape: torch.Size([1, 1853]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 75 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1853) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1853) torch.int64 + loss_mask : 499 / 1853 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1853, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1853, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1853, 1, 576) torch.bfloat16 + out : (1, 1853) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 499 tokens) : 11.3689 + top-1 accuracy : 15/499 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 76 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 376 / 1539 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3072 + top-1 accuracy : 14/376 = 3.7% + + [2026-04-29 12:24:32] iteration 39/ 100 | consumed samples: 156 | elapsed time per iteration (ms): 4858.3 | throughput (token/sec/GPU): 1332.0 | learning rate: 9.158184E-05 | global batch size: 4 | lm loss: 1.131985E+01 | loss scale: 1.0 | grad norm: 0.429 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 39 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 39 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 39 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1698.07, 1698.07) +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 77 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 356 / 1434 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.3037 + top-1 accuracy : 28/356 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1562]) +Dataloader batch - labels shape: torch.Size([1, 1562]) +Dataloader batch - attn_mask shape: torch.Size([1, 1562]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 78 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1562) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1562) torch.int64 + loss_mask : 410 / 1562 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1562, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1562, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1562, 1, 576) torch.bfloat16 + out : (1, 1562) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.3960 + top-1 accuracy : 7/410 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1435]) +Dataloader batch - labels shape: torch.Size([1, 1435]) +Dataloader batch - attn_mask shape: torch.Size([1, 1435]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 79 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1435) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1435) torch.int64 + loss_mask : 376 / 1435 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1435, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1435, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1435, 1, 576) torch.bfloat16 + out : (1, 1435) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3384 + top-1 accuracy : 14/376 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1694]) +Dataloader batch - labels shape: torch.Size([1, 1694]) +Dataloader batch - attn_mask shape: torch.Size([1, 1694]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 80 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1694) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1694) torch.int64 + loss_mask : 344 / 1694 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1694, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1694, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1694, 1, 576) torch.bfloat16 + out : (1, 1694) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.1951 + top-1 accuracy : 34/344 = 9.9% + + [2026-04-29 12:24:38] iteration 40/ 100 | consumed samples: 160 | elapsed time per iteration (ms): 4832.3 | throughput (token/sec/GPU): 1267.5 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.131280E+01 | loss scale: 1.0 | grad norm: 0.450 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4821.77, 4821.77) + forward-compute ................................: (3607.96, 3607.96) + backward-compute ...............................: (1211.51, 1211.51) + batch-generator ................................: (463.04, 463.04) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.45, 0.45) + params-all-gather ..............................: (0.14, 0.14) + optimizer-copy-to-main-grad ....................: (0.13, 0.13) + optimizer-clip-main-grad .......................: (0.48, 0.48) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.28, 1.28) + optimizer-copy-main-to-model-params ............: (0.20, 0.20) + optimizer ......................................: (2.81, 2.81) +saving checkpoint at iteration 40 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 40 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 40 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1662.81, 1662.81) +Dataloader batch - tokens shape: torch.Size([1, 1581]) +Dataloader batch - labels shape: torch.Size([1, 1581]) +Dataloader batch - attn_mask shape: torch.Size([1, 1581]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 81 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1581) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1581) torch.int64 + loss_mask : 422 / 1581 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1581, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1581, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1581, 1, 576) torch.bfloat16 + out : (1, 1581) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.3586 + top-1 accuracy : 17/422 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1500]) +Dataloader batch - labels shape: torch.Size([1, 1500]) +Dataloader batch - attn_mask shape: torch.Size([1, 1500]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 82 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1500) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1500) torch.int64 + loss_mask : 391 / 1500 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1500, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1500, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1500, 1, 576) torch.bfloat16 + out : (1, 1500) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3426 + top-1 accuracy : 13/391 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1559]) +Dataloader batch - labels shape: torch.Size([1, 1559]) +Dataloader batch - attn_mask shape: torch.Size([1, 1559]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 83 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1559) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1559) torch.int64 + loss_mask : 340 / 1559 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1559, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1559, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1559, 1, 576) torch.bfloat16 + out : (1, 1559) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2193 + top-1 accuracy : 26/340 = 7.6% + +Dataloader batch - tokens shape: torch.Size([1, 1795]) +Dataloader batch - labels shape: torch.Size([1, 1795]) +Dataloader batch - attn_mask shape: torch.Size([1, 1795]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 84 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1795) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1795) torch.int64 + loss_mask : 432 / 1795 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1795, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1795, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1795, 1, 576) torch.bfloat16 + out : (1, 1795) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 432 tokens) : 11.3146 + top-1 accuracy : 7/432 = 1.6% + + [2026-04-29 12:24:45] iteration 41/ 100 | consumed samples: 164 | elapsed time per iteration (ms): 4887.2 | throughput (token/sec/GPU): 1316.7 | learning rate: 8.976308E-05 | global batch size: 4 | lm loss: 1.131278E+01 | loss scale: 1.0 | grad norm: 0.459 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 41 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 41 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 41 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1169.28, 1169.28) +Dataloader batch - tokens shape: torch.Size([1, 1732]) +Dataloader batch - labels shape: torch.Size([1, 1732]) +Dataloader batch - attn_mask shape: torch.Size([1, 1732]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 85 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1732) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1732) torch.int64 + loss_mask : 370 / 1732 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1732, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1732, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1732, 1, 576) torch.bfloat16 + out : (1, 1732) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.2719 + top-1 accuracy : 33/370 = 8.9% + +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 86 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1420) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1420) torch.int64 + loss_mask : 375 / 1420 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1420, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1420, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1420, 1, 576) torch.bfloat16 + out : (1, 1420) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.3466 + top-1 accuracy : 29/375 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1216]) +Dataloader batch - labels shape: torch.Size([1, 1216]) +Dataloader batch - attn_mask shape: torch.Size([1, 1216]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 87 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1216) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1216) torch.int64 + loss_mask : 298 / 1216 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1216, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1216, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1216, 1, 576) torch.bfloat16 + out : (1, 1216) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 298 tokens) : 11.3064 + top-1 accuracy : 9/298 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 88 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 356 / 1543 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2973 + top-1 accuracy : 27/356 = 7.6% + + [2026-04-29 12:24:51] iteration 42/ 100 | consumed samples: 168 | elapsed time per iteration (ms): 4640.8 | throughput (token/sec/GPU): 1273.7 | learning rate: 8.879489E-05 | global batch size: 4 | lm loss: 1.130576E+01 | loss scale: 1.0 | grad norm: 0.423 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 42 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 42 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 42 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1536.34, 1536.34) +Dataloader batch - tokens shape: torch.Size([1, 1072]) +Dataloader batch - labels shape: torch.Size([1, 1072]) +Dataloader batch - attn_mask shape: torch.Size([1, 1072]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 89 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1072) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1072) torch.int64 + loss_mask : 305 / 1072 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1072, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1072, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1072, 1, 576) torch.bfloat16 + out : (1, 1072) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.4321 + top-1 accuracy : 6/305 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 90 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 312 / 1400 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.2221 + top-1 accuracy : 30/312 = 9.6% + +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 91 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1696) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1696) torch.int64 + loss_mask : 385 / 1696 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1696, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1696, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1696, 1, 576) torch.bfloat16 + out : (1, 1696) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.2832 + top-1 accuracy : 29/385 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 848]) +Dataloader batch - labels shape: torch.Size([1, 848]) +Dataloader batch - attn_mask shape: torch.Size([1, 848]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 92 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 848) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 848) torch.int64 + loss_mask : 151 / 848 tokens (17.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (848, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (848, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (848, 1, 576) torch.bfloat16 + out : (1, 848) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 151 tokens) : 11.0210 + top-1 accuracy : 3/151 = 2.0% + + [2026-04-29 12:24:57] iteration 43/ 100 | consumed samples: 172 | elapsed time per iteration (ms): 4181.8 | throughput (token/sec/GPU): 1199.5 | learning rate: 8.778875E-05 | global batch size: 4 | lm loss: 1.127174E+01 | loss scale: 1.0 | grad norm: 0.462 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 43 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 43 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 43 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1901.25, 1901.25) +Dataloader batch - tokens shape: torch.Size([1, 995]) +Dataloader batch - labels shape: torch.Size([1, 995]) +Dataloader batch - attn_mask shape: torch.Size([1, 995]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 93 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 995) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 995) torch.int64 + loss_mask : 228 / 995 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (995, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (995, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (995, 1, 576) torch.bfloat16 + out : (1, 995) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 228 tokens) : 11.2677 + top-1 accuracy : 12/228 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1373]) +Dataloader batch - labels shape: torch.Size([1, 1373]) +Dataloader batch - attn_mask shape: torch.Size([1, 1373]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 94 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1373) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1373) torch.int64 + loss_mask : 327 / 1373 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1373, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1373, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1373, 1, 576) torch.bfloat16 + out : (1, 1373) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2624 + top-1 accuracy : 27/327 = 8.3% + +Dataloader batch - tokens shape: torch.Size([1, 1033]) +Dataloader batch - labels shape: torch.Size([1, 1033]) +Dataloader batch - attn_mask shape: torch.Size([1, 1033]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 95 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1033) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1033) torch.int64 + loss_mask : 201 / 1033 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1033, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1033, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1033, 1, 576) torch.bfloat16 + out : (1, 1033) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 201 tokens) : 11.1536 + top-1 accuracy : 23/201 = 11.4% + +Dataloader batch - tokens shape: torch.Size([1, 1707]) +Dataloader batch - labels shape: torch.Size([1, 1707]) +Dataloader batch - attn_mask shape: torch.Size([1, 1707]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 96 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1707) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1707) torch.int64 + loss_mask : 364 / 1707 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1707, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1707, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1707, 1, 576) torch.bfloat16 + out : (1, 1707) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.1961 + top-1 accuracy : 37/364 = 10.2% + + [2026-04-29 12:25:03] iteration 44/ 100 | consumed samples: 176 | elapsed time per iteration (ms): 4241.8 | throughput (token/sec/GPU): 1204.2 | learning rate: 8.674567E-05 | global batch size: 4 | lm loss: 1.122238E+01 | loss scale: 1.0 | grad norm: 0.528 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 44 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 44 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 44 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1371.31, 1371.31) +Dataloader batch - tokens shape: torch.Size([1, 1236]) +Dataloader batch - labels shape: torch.Size([1, 1236]) +Dataloader batch - attn_mask shape: torch.Size([1, 1236]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 97 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1236) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1236) torch.int64 + loss_mask : 340 / 1236 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1236, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1236, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1236, 1, 576) torch.bfloat16 + out : (1, 1236) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.3561 + top-1 accuracy : 14/340 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1481]) +Dataloader batch - labels shape: torch.Size([1, 1481]) +Dataloader batch - attn_mask shape: torch.Size([1, 1481]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 98 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1481) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1481) torch.int64 + loss_mask : 393 / 1481 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1481, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1481, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1481, 1, 576) torch.bfloat16 + out : (1, 1481) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 393 tokens) : 11.3337 + top-1 accuracy : 21/393 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1261]) +Dataloader batch - labels shape: torch.Size([1, 1261]) +Dataloader batch - attn_mask shape: torch.Size([1, 1261]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 99 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1261) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1261) torch.int64 + loss_mask : 366 / 1261 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1261, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1261, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1261, 1, 576) torch.bfloat16 + out : (1, 1261) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.4170 + top-1 accuracy : 1/366 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 100 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 360 / 1422 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.3636 + top-1 accuracy : 16/360 = 4.4% + + [2026-04-29 12:25:08] iteration 45/ 100 | consumed samples: 180 | elapsed time per iteration (ms): 4190.7 | throughput (token/sec/GPU): 1288.6 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.136718E+01 | loss scale: 1.0 | grad norm: 0.462 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 45 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 45 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 45 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1693.46, 1693.46) +Dataloader batch - tokens shape: torch.Size([1, 1706]) +Dataloader batch - labels shape: torch.Size([1, 1706]) +Dataloader batch - attn_mask shape: torch.Size([1, 1706]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 101 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1706) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1706) torch.int64 + loss_mask : 348 / 1706 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1706, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1706, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1706, 1, 576) torch.bfloat16 + out : (1, 1706) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2016 + top-1 accuracy : 31/348 = 8.9% + +Dataloader batch - tokens shape: torch.Size([1, 1323]) +Dataloader batch - labels shape: torch.Size([1, 1323]) +Dataloader batch - attn_mask shape: torch.Size([1, 1323]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 102 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1323) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1323) torch.int64 + loss_mask : 325 / 1323 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1323, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1323, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1323, 1, 576) torch.bfloat16 + out : (1, 1323) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.2947 + top-1 accuracy : 25/325 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1373]) +Dataloader batch - labels shape: torch.Size([1, 1373]) +Dataloader batch - attn_mask shape: torch.Size([1, 1373]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 103 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1373) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1373) torch.int64 + loss_mask : 293 / 1373 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1373, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1373, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1373, 1, 576) torch.bfloat16 + out : (1, 1373) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 293 tokens) : 11.2071 + top-1 accuracy : 28/293 = 9.6% + +Dataloader batch - tokens shape: torch.Size([1, 1869]) +Dataloader batch - labels shape: torch.Size([1, 1869]) +Dataloader batch - attn_mask shape: torch.Size([1, 1869]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 104 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1869) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1869) torch.int64 + loss_mask : 503 / 1869 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1869, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1869, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1869, 1, 576) torch.bfloat16 + out : (1, 1869) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 503 tokens) : 11.3337 + top-1 accuracy : 18/503 = 3.6% + + [2026-04-29 12:25:15] iteration 46/ 100 | consumed samples: 184 | elapsed time per iteration (ms): 5001.2 | throughput (token/sec/GPU): 1253.9 | learning rate: 8.455283E-05 | global batch size: 4 | lm loss: 1.126853E+01 | loss scale: 1.0 | grad norm: 0.415 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 46 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 46 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 46 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1153.95, 1153.95) +Dataloader batch - tokens shape: torch.Size([1, 1517]) +Dataloader batch - labels shape: torch.Size([1, 1517]) +Dataloader batch - attn_mask shape: torch.Size([1, 1517]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 105 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1517) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1517) torch.int64 + loss_mask : 349 / 1517 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1517, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1517, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1517, 1, 576) torch.bfloat16 + out : (1, 1517) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2658 + top-1 accuracy : 33/349 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1550]) +Dataloader batch - labels shape: torch.Size([1, 1550]) +Dataloader batch - attn_mask shape: torch.Size([1, 1550]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 106 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1550) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1550) torch.int64 + loss_mask : 424 / 1550 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1550, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1550, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1550, 1, 576) torch.bfloat16 + out : (1, 1550) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 424 tokens) : 11.3765 + top-1 accuracy : 19/424 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 107 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 365 / 1374 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.3379 + top-1 accuracy : 16/365 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 108 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 345 / 1722 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.1959 + top-1 accuracy : 23/345 = 6.7% + + [2026-04-29 12:25:21] iteration 47/ 100 | consumed samples: 188 | elapsed time per iteration (ms): 4868.2 | throughput (token/sec/GPU): 1266.0 | learning rate: 8.340524E-05 | global batch size: 4 | lm loss: 1.129895E+01 | loss scale: 1.0 | grad norm: 0.407 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 47 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 47 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 47 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1364.81, 1364.81) +Dataloader batch - tokens shape: torch.Size([1, 990]) +Dataloader batch - labels shape: torch.Size([1, 990]) +Dataloader batch - attn_mask shape: torch.Size([1, 990]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 109 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 990) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 990) torch.int64 + loss_mask : 218 / 990 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (990, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (990, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (990, 1, 576) torch.bfloat16 + out : (1, 990) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 218 tokens) : 11.2622 + top-1 accuracy : 12/218 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 110 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 344 / 1552 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.2738 + top-1 accuracy : 24/344 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1768]) +Dataloader batch - labels shape: torch.Size([1, 1768]) +Dataloader batch - attn_mask shape: torch.Size([1, 1768]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 111 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1768) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1768) torch.int64 + loss_mask : 427 / 1768 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1768, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1768, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1768, 1, 576) torch.bfloat16 + out : (1, 1768) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 427 tokens) : 11.2720 + top-1 accuracy : 33/427 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1553]) +Dataloader batch - labels shape: torch.Size([1, 1553]) +Dataloader batch - attn_mask shape: torch.Size([1, 1553]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 112 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1553) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1553) torch.int64 + loss_mask : 366 / 1553 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1553, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1553, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1553, 1, 576) torch.bfloat16 + out : (1, 1553) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.3104 + top-1 accuracy : 20/366 = 5.5% + + [2026-04-29 12:25:27] iteration 48/ 100 | consumed samples: 192 | elapsed time per iteration (ms): 4686.8 | throughput (token/sec/GPU): 1251.0 | learning rate: 8.222505E-05 | global batch size: 4 | lm loss: 1.128124E+01 | loss scale: 1.0 | grad norm: 0.385 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 48 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 48 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 48 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1597.34, 1597.34) +Dataloader batch - tokens shape: torch.Size([1, 1586]) +Dataloader batch - labels shape: torch.Size([1, 1586]) +Dataloader batch - attn_mask shape: torch.Size([1, 1586]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 113 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1586) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1586) torch.int64 + loss_mask : 450 / 1586 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1586, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1586, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1586, 1, 576) torch.bfloat16 + out : (1, 1586) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 450 tokens) : 11.3879 + top-1 accuracy : 16/450 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1051]) +Dataloader batch - labels shape: torch.Size([1, 1051]) +Dataloader batch - attn_mask shape: torch.Size([1, 1051]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 114 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1051) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1051) torch.int64 + loss_mask : 299 / 1051 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1051, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1051, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1051, 1, 576) torch.bfloat16 + out : (1, 1051) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 299 tokens) : 11.3156 + top-1 accuracy : 4/299 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1148]) +Dataloader batch - labels shape: torch.Size([1, 1148]) +Dataloader batch - attn_mask shape: torch.Size([1, 1148]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 115 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1148) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1148) torch.int64 + loss_mask : 315 / 1148 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1148, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1148, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1148, 1, 576) torch.bfloat16 + out : (1, 1148) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.3128 + top-1 accuracy : 11/315 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 116 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 397 / 1542 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.3414 + top-1 accuracy : 13/397 = 3.3% + + [2026-04-29 12:25:33] iteration 49/ 100 | consumed samples: 196 | elapsed time per iteration (ms): 3998.8 | throughput (token/sec/GPU): 1332.2 | learning rate: 8.101343E-05 | global batch size: 4 | lm loss: 1.134427E+01 | loss scale: 1.0 | grad norm: 0.400 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 49 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 49 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 49 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1734.44, 1734.44) +Dataloader batch - tokens shape: torch.Size([1, 1207]) +Dataloader batch - labels shape: torch.Size([1, 1207]) +Dataloader batch - attn_mask shape: torch.Size([1, 1207]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 117 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1207) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1207) torch.int64 + loss_mask : 302 / 1207 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1207, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1207, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1207, 1, 576) torch.bfloat16 + out : (1, 1207) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 302 tokens) : 11.3243 + top-1 accuracy : 1/302 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1769]) +Dataloader batch - labels shape: torch.Size([1, 1769]) +Dataloader batch - attn_mask shape: torch.Size([1, 1769]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 118 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1769) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1769) torch.int64 + loss_mask : 475 / 1769 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1769, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1769, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1769, 1, 576) torch.bfloat16 + out : (1, 1769) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 475 tokens) : 11.3283 + top-1 accuracy : 28/475 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1406]) +Dataloader batch - labels shape: torch.Size([1, 1406]) +Dataloader batch - attn_mask shape: torch.Size([1, 1406]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 119 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1406) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1406) torch.int64 + loss_mask : 374 / 1406 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1406, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1406, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1406, 1, 576) torch.bfloat16 + out : (1, 1406) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.3157 + top-1 accuracy : 16/374 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1735]) +Dataloader batch - labels shape: torch.Size([1, 1735]) +Dataloader batch - attn_mask shape: torch.Size([1, 1735]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 120 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1735) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1735) torch.int64 + loss_mask : 448 / 1735 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1735, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1735, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1735, 1, 576) torch.bfloat16 + out : (1, 1735) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 448 tokens) : 11.3200 + top-1 accuracy : 18/448 = 4.0% + + [2026-04-29 12:25:39] iteration 50/ 100 | consumed samples: 200 | elapsed time per iteration (ms): 4738.3 | throughput (token/sec/GPU): 1291.0 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.132229E+01 | loss scale: 1.0 | grad norm: 0.315 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 50 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 50 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 50 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1514.20, 1514.20) +Dataloader batch - tokens shape: torch.Size([1, 1904]) +Dataloader batch - labels shape: torch.Size([1, 1904]) +Dataloader batch - attn_mask shape: torch.Size([1, 1904]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 121 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1904) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1904) torch.int64 + loss_mask : 535 / 1904 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1904, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1904, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1904, 1, 576) torch.bfloat16 + out : (1, 1904) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 535 tokens) : 11.4000 + top-1 accuracy : 21/535 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1509]) +Dataloader batch - labels shape: torch.Size([1, 1509]) +Dataloader batch - attn_mask shape: torch.Size([1, 1509]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 122 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1509) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1509) torch.int64 + loss_mask : 335 / 1509 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1509, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1509, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1509, 1, 576) torch.bfloat16 + out : (1, 1509) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2216 + top-1 accuracy : 31/335 = 9.3% + +Dataloader batch - tokens shape: torch.Size([1, 1522]) +Dataloader batch - labels shape: torch.Size([1, 1522]) +Dataloader batch - attn_mask shape: torch.Size([1, 1522]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 123 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1522) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1522) torch.int64 + loss_mask : 371 / 1522 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1522, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1522, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1522, 1, 576) torch.bfloat16 + out : (1, 1522) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.3373 + top-1 accuracy : 19/371 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1786]) +Dataloader batch - labels shape: torch.Size([1, 1786]) +Dataloader batch - attn_mask shape: torch.Size([1, 1786]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 124 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1786) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1786) torch.int64 + loss_mask : 484 / 1786 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1786, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1786, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1786, 1, 576) torch.bfloat16 + out : (1, 1786) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 484 tokens) : 11.3710 + top-1 accuracy : 34/484 = 7.0% + + [2026-04-29 12:25:46] iteration 51/ 100 | consumed samples: 204 | elapsed time per iteration (ms): 5083.6 | throughput (token/sec/GPU): 1322.1 | learning rate: 7.850071E-05 | global batch size: 4 | lm loss: 1.134372E+01 | loss scale: 1.0 | grad norm: 0.341 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 51 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 51 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 51 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1047.42, 1047.42) +Dataloader batch - tokens shape: torch.Size([1, 1021]) +Dataloader batch - labels shape: torch.Size([1, 1021]) +Dataloader batch - attn_mask shape: torch.Size([1, 1021]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 125 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1021) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1021) torch.int64 + loss_mask : 262 / 1021 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1021, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1021, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1021, 1, 576) torch.bfloat16 + out : (1, 1021) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 262 tokens) : 11.2826 + top-1 accuracy : 24/262 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1516]) +Dataloader batch - labels shape: torch.Size([1, 1516]) +Dataloader batch - attn_mask shape: torch.Size([1, 1516]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 126 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1516) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1516) torch.int64 + loss_mask : 336 / 1516 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1516, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1516, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1516, 1, 576) torch.bfloat16 + out : (1, 1516) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.2628 + top-1 accuracy : 28/336 = 8.3% + +Dataloader batch - tokens shape: torch.Size([1, 1405]) +Dataloader batch - labels shape: torch.Size([1, 1405]) +Dataloader batch - attn_mask shape: torch.Size([1, 1405]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 127 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1405) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1405) torch.int64 + loss_mask : 348 / 1405 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1405, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1405, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1405, 1, 576) torch.bfloat16 + out : (1, 1405) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2902 + top-1 accuracy : 22/348 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 128 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 354 / 1539 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2948 + top-1 accuracy : 28/354 = 7.9% + + [2026-04-29 12:25:52] iteration 52/ 100 | consumed samples: 208 | elapsed time per iteration (ms): 4793.8 | throughput (token/sec/GPU): 1143.4 | learning rate: 7.720211E-05 | global batch size: 4 | lm loss: 1.128284E+01 | loss scale: 1.0 | grad norm: 0.434 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 52 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 52 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 52 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1556.26, 1556.26) +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 129 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 384 / 1386 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.3258 + top-1 accuracy : 5/384 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1621]) +Dataloader batch - labels shape: torch.Size([1, 1621]) +Dataloader batch - attn_mask shape: torch.Size([1, 1621]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 130 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1621) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1621) torch.int64 + loss_mask : 379 / 1621 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1621, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1621, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1621, 1, 576) torch.bfloat16 + out : (1, 1621) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.2974 + top-1 accuracy : 17/379 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1452]) +Dataloader batch - labels shape: torch.Size([1, 1452]) +Dataloader batch - attn_mask shape: torch.Size([1, 1452]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 131 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1452) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1452) torch.int64 + loss_mask : 397 / 1452 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1452, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1452, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1452, 1, 576) torch.bfloat16 + out : (1, 1452) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.3738 + top-1 accuracy : 11/397 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1575]) +Dataloader batch - labels shape: torch.Size([1, 1575]) +Dataloader batch - attn_mask shape: torch.Size([1, 1575]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 132 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1575) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1575) torch.int64 + loss_mask : 347 / 1575 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1575, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1575, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1575, 1, 576) torch.bfloat16 + out : (1, 1575) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2416 + top-1 accuracy : 7/347 = 2.0% + + [2026-04-29 12:25:58] iteration 53/ 100 | consumed samples: 212 | elapsed time per iteration (ms): 4636.4 | throughput (token/sec/GPU): 1301.4 | learning rate: 7.587705E-05 | global batch size: 4 | lm loss: 1.131192E+01 | loss scale: 1.0 | grad norm: 0.373 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 53 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 53 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 53 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1319.81, 1319.81) +Dataloader batch - tokens shape: torch.Size([1, 1721]) +Dataloader batch - labels shape: torch.Size([1, 1721]) +Dataloader batch - attn_mask shape: torch.Size([1, 1721]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 133 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1721) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1721) torch.int64 + loss_mask : 367 / 1721 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1721, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1721, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1721, 1, 576) torch.bfloat16 + out : (1, 1721) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2271 + top-1 accuracy : 21/367 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1895]) +Dataloader batch - labels shape: torch.Size([1, 1895]) +Dataloader batch - attn_mask shape: torch.Size([1, 1895]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 134 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1895) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1895) torch.int64 + loss_mask : 543 / 1895 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1895, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1895, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1895, 1, 576) torch.bfloat16 + out : (1, 1895) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 543 tokens) : 11.3798 + top-1 accuracy : 18/543 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1654]) +Dataloader batch - labels shape: torch.Size([1, 1654]) +Dataloader batch - attn_mask shape: torch.Size([1, 1654]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 135 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1654) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1654) torch.int64 + loss_mask : 374 / 1654 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1654, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1654, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1654, 1, 576) torch.bfloat16 + out : (1, 1654) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.2350 + top-1 accuracy : 32/374 = 8.6% + +Dataloader batch - tokens shape: torch.Size([1, 1206]) +Dataloader batch - labels shape: torch.Size([1, 1206]) +Dataloader batch - attn_mask shape: torch.Size([1, 1206]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 136 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1206) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1206) torch.int64 + loss_mask : 288 / 1206 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1206, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1206, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1206, 1, 576) torch.bfloat16 + out : (1, 1206) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 288 tokens) : 11.2934 + top-1 accuracy : 17/288 = 5.9% + + [2026-04-29 12:26:04] iteration 54/ 100 | consumed samples: 216 | elapsed time per iteration (ms): 4997.0 | throughput (token/sec/GPU): 1296.0 | learning rate: 7.452684E-05 | global batch size: 4 | lm loss: 1.129387E+01 | loss scale: 1.0 | grad norm: 0.407 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 54 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 54 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 54 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1801.55, 1801.55) +Dataloader batch - tokens shape: torch.Size([1, 1544]) +Dataloader batch - labels shape: torch.Size([1, 1544]) +Dataloader batch - attn_mask shape: torch.Size([1, 1544]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 137 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1544) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1544) torch.int64 + loss_mask : 399 / 1544 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1544, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1544, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1544, 1, 576) torch.bfloat16 + out : (1, 1544) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.3217 + top-1 accuracy : 10/399 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1551]) +Dataloader batch - labels shape: torch.Size([1, 1551]) +Dataloader batch - attn_mask shape: torch.Size([1, 1551]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 138 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1551) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1551) torch.int64 + loss_mask : 348 / 1551 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1551, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1551, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1551, 1, 576) torch.bfloat16 + out : (1, 1551) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2158 + top-1 accuracy : 4/348 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1693]) +Dataloader batch - labels shape: torch.Size([1, 1693]) +Dataloader batch - attn_mask shape: torch.Size([1, 1693]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 139 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1693) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1693) torch.int64 + loss_mask : 346 / 1693 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1693, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1693, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1693, 1, 576) torch.bfloat16 + out : (1, 1693) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.2066 + top-1 accuracy : 25/346 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 140 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 313 / 1496 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2078 + top-1 accuracy : 19/313 = 6.1% + + [2026-04-29 12:26:11] iteration 55/ 100 | consumed samples: 220 | elapsed time per iteration (ms): 5076.3 | throughput (token/sec/GPU): 1237.9 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.124179E+01 | loss scale: 1.0 | grad norm: 0.409 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 55 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 55 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 55 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1848.34, 1848.34) +Dataloader batch - tokens shape: torch.Size([1, 1553]) +Dataloader batch - labels shape: torch.Size([1, 1553]) +Dataloader batch - attn_mask shape: torch.Size([1, 1553]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 141 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1553) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1553) torch.int64 + loss_mask : 349 / 1553 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1553, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1553, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1553, 1, 576) torch.bfloat16 + out : (1, 1553) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.1922 + top-1 accuracy : 4/349 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 919]) +Dataloader batch - labels shape: torch.Size([1, 919]) +Dataloader batch - attn_mask shape: torch.Size([1, 919]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 142 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 919) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 919) torch.int64 + loss_mask : 176 / 919 tokens (19.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (919, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (919, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (919, 1, 576) torch.bfloat16 + out : (1, 919) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 176 tokens) : 11.2305 + top-1 accuracy : 0/176 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1714]) +Dataloader batch - labels shape: torch.Size([1, 1714]) +Dataloader batch - attn_mask shape: torch.Size([1, 1714]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 143 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1714) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1714) torch.int64 + loss_mask : 443 / 1714 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1714, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1714, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1714, 1, 576) torch.bfloat16 + out : (1, 1714) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 443 tokens) : 11.3247 + top-1 accuracy : 33/443 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1798]) +Dataloader batch - labels shape: torch.Size([1, 1798]) +Dataloader batch - attn_mask shape: torch.Size([1, 1798]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 144 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1798) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1798) torch.int64 + loss_mask : 450 / 1798 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1798, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1798, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1798, 1, 576) torch.bfloat16 + out : (1, 1798) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 450 tokens) : 11.2235 + top-1 accuracy : 10/450 = 2.2% + + [2026-04-29 12:26:18] iteration 56/ 100 | consumed samples: 224 | elapsed time per iteration (ms): 4880.6 | throughput (token/sec/GPU): 1226.1 | learning rate: 7.175637E-05 | global batch size: 4 | lm loss: 1.124832E+01 | loss scale: 1.0 | grad norm: 0.458 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 56 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 56 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 56 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1555.05, 1555.05) +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 145 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 313 / 1487 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.1997 + top-1 accuracy : 24/313 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 146 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1418) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1418) torch.int64 + loss_mask : 342 / 1418 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1418, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1418, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1418, 1, 576) torch.bfloat16 + out : (1, 1418) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2825 + top-1 accuracy : 28/342 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1585]) +Dataloader batch - labels shape: torch.Size([1, 1585]) +Dataloader batch - attn_mask shape: torch.Size([1, 1585]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 147 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1585) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1585) torch.int64 + loss_mask : 379 / 1585 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1585, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1585, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1585, 1, 576) torch.bfloat16 + out : (1, 1585) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.3026 + top-1 accuracy : 13/379 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1836]) +Dataloader batch - labels shape: torch.Size([1, 1836]) +Dataloader batch - attn_mask shape: torch.Size([1, 1836]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 148 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1836) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1836) torch.int64 + loss_mask : 458 / 1836 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1836, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1836, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1836, 1, 576) torch.bfloat16 + out : (1, 1836) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 458 tokens) : 11.3086 + top-1 accuracy : 3/458 = 0.7% + + [2026-04-29 12:26:24] iteration 57/ 100 | consumed samples: 228 | elapsed time per iteration (ms): 5054.9 | throughput (token/sec/GPU): 1251.5 | learning rate: 7.033885E-05 | global batch size: 4 | lm loss: 1.127826E+01 | loss scale: 1.0 | grad norm: 0.364 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 57 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 57 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 57 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1588.37, 1588.37) +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 149 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 369 / 1520 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2710 + top-1 accuracy : 15/369 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1480]) +Dataloader batch - labels shape: torch.Size([1, 1480]) +Dataloader batch - attn_mask shape: torch.Size([1, 1480]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 150 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1480) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1480) torch.int64 + loss_mask : 429 / 1480 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1480, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1480, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1480, 1, 576) torch.bfloat16 + out : (1, 1480) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 429 tokens) : 11.3819 + top-1 accuracy : 13/429 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 151 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1696) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1696) torch.int64 + loss_mask : 344 / 1696 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1696, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1696, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1696, 1, 576) torch.bfloat16 + out : (1, 1696) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.1824 + top-1 accuracy : 33/344 = 9.6% + +Dataloader batch - tokens shape: torch.Size([1, 1584]) +Dataloader batch - labels shape: torch.Size([1, 1584]) +Dataloader batch - attn_mask shape: torch.Size([1, 1584]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 152 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1584) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1584) torch.int64 + loss_mask : 386 / 1584 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1584, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1584, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1584, 1, 576) torch.bfloat16 + out : (1, 1584) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2020 + top-1 accuracy : 15/386 = 3.9% + + [2026-04-29 12:26:31] iteration 58/ 100 | consumed samples: 232 | elapsed time per iteration (ms): 4872.6 | throughput (token/sec/GPU): 1288.8 | learning rate: 6.890167E-05 | global batch size: 4 | lm loss: 1.126478E+01 | loss scale: 1.0 | grad norm: 0.358 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 58 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 58 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 58 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1330.42, 1330.42) +Dataloader batch - tokens shape: torch.Size([1, 1433]) +Dataloader batch - labels shape: torch.Size([1, 1433]) +Dataloader batch - attn_mask shape: torch.Size([1, 1433]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 153 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1433) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1433) torch.int64 + loss_mask : 375 / 1433 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1433, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1433, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1433, 1, 576) torch.bfloat16 + out : (1, 1433) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.3021 + top-1 accuracy : 28/375 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 154 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1470) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1470) torch.int64 + loss_mask : 372 / 1470 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1470, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1470, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1470, 1, 576) torch.bfloat16 + out : (1, 1470) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2992 + top-1 accuracy : 18/372 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1704]) +Dataloader batch - labels shape: torch.Size([1, 1704]) +Dataloader batch - attn_mask shape: torch.Size([1, 1704]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 155 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1704) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1704) torch.int64 + loss_mask : 442 / 1704 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1704, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1704, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1704, 1, 576) torch.bfloat16 + out : (1, 1704) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 442 tokens) : 11.2934 + top-1 accuracy : 17/442 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1005]) +Dataloader batch - labels shape: torch.Size([1, 1005]) +Dataloader batch - attn_mask shape: torch.Size([1, 1005]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 156 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1005) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1005) torch.int64 + loss_mask : 257 / 1005 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1005, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1005, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1005, 1, 576) torch.bfloat16 + out : (1, 1005) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 257 tokens) : 11.2970 + top-1 accuracy : 1/257 = 0.4% + + [2026-04-29 12:26:37] iteration 59/ 100 | consumed samples: 236 | elapsed time per iteration (ms): 4449.2 | throughput (token/sec/GPU): 1261.3 | learning rate: 6.744626E-05 | global batch size: 4 | lm loss: 1.129778E+01 | loss scale: 1.0 | grad norm: 0.387 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 59 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 59 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 59 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1377.92, 1377.92) +Dataloader batch - tokens shape: torch.Size([1, 1264]) +Dataloader batch - labels shape: torch.Size([1, 1264]) +Dataloader batch - attn_mask shape: torch.Size([1, 1264]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 157 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1264) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1264) torch.int64 + loss_mask : 327 / 1264 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1264, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1264, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1264, 1, 576) torch.bfloat16 + out : (1, 1264) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.3138 + top-1 accuracy : 3/327 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 158 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 363 / 1722 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2114 + top-1 accuracy : 24/363 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1623]) +Dataloader batch - labels shape: torch.Size([1, 1623]) +Dataloader batch - attn_mask shape: torch.Size([1, 1623]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 159 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1623) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1623) torch.int64 + loss_mask : 400 / 1623 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1623, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1623, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1623, 1, 576) torch.bfloat16 + out : (1, 1623) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.2533 + top-1 accuracy : 19/400 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1473]) +Dataloader batch - labels shape: torch.Size([1, 1473]) +Dataloader batch - attn_mask shape: torch.Size([1, 1473]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 160 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1473) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1473) torch.int64 + loss_mask : 332 / 1473 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1473, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1473, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1473, 1, 576) torch.bfloat16 + out : (1, 1473) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.2439 + top-1 accuracy : 24/332 = 7.2% + + [2026-04-29 12:26:43] iteration 60/ 100 | consumed samples: 240 | elapsed time per iteration (ms): 4941.9 | throughput (token/sec/GPU): 1230.7 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.125433E+01 | loss scale: 1.0 | grad norm: 0.345 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4929.49, 4929.49) + forward-compute ................................: (3649.73, 3649.73) + backward-compute ...............................: (1277.09, 1277.09) + batch-generator ................................: (398.04, 398.04) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.47, 0.47) + params-all-gather ..............................: (0.14, 0.14) + optimizer-copy-to-main-grad ....................: (0.17, 0.17) + optimizer-clip-main-grad .......................: (0.51, 0.51) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.29, 1.29) + optimizer-copy-main-to-model-params ............: (0.20, 0.20) + optimizer ......................................: (2.89, 2.89) +saving checkpoint at iteration 60 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 60 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 60 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1218.71, 1218.71) +Dataloader batch - tokens shape: torch.Size([1, 1589]) +Dataloader batch - labels shape: torch.Size([1, 1589]) +Dataloader batch - attn_mask shape: torch.Size([1, 1589]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 161 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1589) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1589) torch.int64 + loss_mask : 350 / 1589 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1589, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1589, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1589, 1, 576) torch.bfloat16 + out : (1, 1589) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.2381 + top-1 accuracy : 25/350 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 162 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 312 / 1444 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.1615 + top-1 accuracy : 13/312 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1029]) +Dataloader batch - labels shape: torch.Size([1, 1029]) +Dataloader batch - attn_mask shape: torch.Size([1, 1029]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 163 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1029) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1029) torch.int64 + loss_mask : 258 / 1029 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1029, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1029, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1029, 1, 576) torch.bfloat16 + out : (1, 1029) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 258 tokens) : 11.2989 + top-1 accuracy : 5/258 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1514]) +Dataloader batch - labels shape: torch.Size([1, 1514]) +Dataloader batch - attn_mask shape: torch.Size([1, 1514]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 164 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1514) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1514) torch.int64 + loss_mask : 404 / 1514 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1514, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1514, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1514, 1, 576) torch.bfloat16 + out : (1, 1514) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.2849 + top-1 accuracy : 29/404 = 7.2% + + [2026-04-29 12:26:49] iteration 61/ 100 | consumed samples: 244 | elapsed time per iteration (ms): 4487.5 | throughput (token/sec/GPU): 1242.6 | learning rate: 6.448653E-05 | global batch size: 4 | lm loss: 1.124620E+01 | loss scale: 1.0 | grad norm: 0.349 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 61 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 61 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 61 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1658.85, 1658.85) +Dataloader batch - tokens shape: torch.Size([1, 1613]) +Dataloader batch - labels shape: torch.Size([1, 1613]) +Dataloader batch - attn_mask shape: torch.Size([1, 1613]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 165 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1613) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1613) torch.int64 + loss_mask : 358 / 1613 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1613, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1613, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1613, 1, 576) torch.bfloat16 + out : (1, 1613) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.1937 + top-1 accuracy : 18/358 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1387]) +Dataloader batch - labels shape: torch.Size([1, 1387]) +Dataloader batch - attn_mask shape: torch.Size([1, 1387]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 166 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1387) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1387) torch.int64 + loss_mask : 313 / 1387 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1387, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1387, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1387, 1, 576) torch.bfloat16 + out : (1, 1387) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2465 + top-1 accuracy : 27/313 = 8.6% + +Dataloader batch - tokens shape: torch.Size([1, 1736]) +Dataloader batch - labels shape: torch.Size([1, 1736]) +Dataloader batch - attn_mask shape: torch.Size([1, 1736]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 167 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1736) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1736) torch.int64 + loss_mask : 375 / 1736 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1736, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1736, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1736, 1, 576) torch.bfloat16 + out : (1, 1736) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.2209 + top-1 accuracy : 37/375 = 9.9% + +Dataloader batch - tokens shape: torch.Size([1, 1384]) +Dataloader batch - labels shape: torch.Size([1, 1384]) +Dataloader batch - attn_mask shape: torch.Size([1, 1384]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 168 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1384) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1384) torch.int64 + loss_mask : 320 / 1384 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1384, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1384, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1384, 1, 576) torch.bfloat16 + out : (1, 1384) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.3126 + top-1 accuracy : 29/320 = 9.1% + + [2026-04-29 12:26:55] iteration 62/ 100 | consumed samples: 248 | elapsed time per iteration (ms): 4990.7 | throughput (token/sec/GPU): 1226.3 | learning rate: 6.298514E-05 | global batch size: 4 | lm loss: 1.124112E+01 | loss scale: 1.0 | grad norm: 0.384 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 62 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 62 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 62 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1475.24, 1475.24) +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 169 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 301 / 1364 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 301 tokens) : 11.2146 + top-1 accuracy : 17/301 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 170 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 392 / 1508 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3217 + top-1 accuracy : 10/392 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 171 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 312 / 1386 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.1921 + top-1 accuracy : 27/312 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1730]) +Dataloader batch - labels shape: torch.Size([1, 1730]) +Dataloader batch - attn_mask shape: torch.Size([1, 1730]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 172 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1730) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1730) torch.int64 + loss_mask : 372 / 1730 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1730, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1730, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1730, 1, 576) torch.bfloat16 + out : (1, 1730) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2114 + top-1 accuracy : 21/372 = 5.6% + + [2026-04-29 12:27:02] iteration 63/ 100 | consumed samples: 252 | elapsed time per iteration (ms): 4916.0 | throughput (token/sec/GPU): 1218.1 | learning rate: 6.147138E-05 | global batch size: 4 | lm loss: 1.123913E+01 | loss scale: 1.0 | grad norm: 0.344 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 63 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 63 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 63 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1527.72, 1527.72) +Dataloader batch - tokens shape: torch.Size([1, 1088]) +Dataloader batch - labels shape: torch.Size([1, 1088]) +Dataloader batch - attn_mask shape: torch.Size([1, 1088]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 173 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1088) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1088) torch.int64 + loss_mask : 248 / 1088 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1088, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1088, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1088, 1, 576) torch.bfloat16 + out : (1, 1088) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 248 tokens) : 11.2151 + top-1 accuracy : 10/248 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 174 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 355 / 1542 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2937 + top-1 accuracy : 20/355 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1851]) +Dataloader batch - labels shape: torch.Size([1, 1851]) +Dataloader batch - attn_mask shape: torch.Size([1, 1851]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 175 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1851) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1851) torch.int64 + loss_mask : 498 / 1851 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1851, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1851, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1851, 1, 576) torch.bfloat16 + out : (1, 1851) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 498 tokens) : 11.3649 + top-1 accuracy : 27/498 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1703]) +Dataloader batch - labels shape: torch.Size([1, 1703]) +Dataloader batch - attn_mask shape: torch.Size([1, 1703]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 176 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1703) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1703) torch.int64 + loss_mask : 337 / 1703 tokens (19.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1703, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1703, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1703, 1, 576) torch.bfloat16 + out : (1, 1703) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.1673 + top-1 accuracy : 25/337 = 7.4% + + [2026-04-29 12:27:08] iteration 64/ 100 | consumed samples: 256 | elapsed time per iteration (ms): 4962.2 | throughput (token/sec/GPU): 1246.2 | learning rate: 5.994675E-05 | global batch size: 4 | lm loss: 1.127519E+01 | loss scale: 1.0 | grad norm: 0.338 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 64 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 64 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 64 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1431.42, 1431.42) +Dataloader batch - tokens shape: torch.Size([1, 1367]) +Dataloader batch - labels shape: torch.Size([1, 1367]) +Dataloader batch - attn_mask shape: torch.Size([1, 1367]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 177 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1367) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1367) torch.int64 + loss_mask : 302 / 1367 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1367, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1367, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1367, 1, 576) torch.bfloat16 + out : (1, 1367) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 302 tokens) : 11.2701 + top-1 accuracy : 22/302 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1355]) +Dataloader batch - labels shape: torch.Size([1, 1355]) +Dataloader batch - attn_mask shape: torch.Size([1, 1355]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 178 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1355) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1355) torch.int64 + loss_mask : 291 / 1355 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1355, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1355, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1355, 1, 576) torch.bfloat16 + out : (1, 1355) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 291 tokens) : 11.1990 + top-1 accuracy : 20/291 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 179 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 289 / 1364 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 289 tokens) : 11.2091 + top-1 accuracy : 23/289 = 8.0% + +Dataloader batch - tokens shape: torch.Size([1, 1869]) +Dataloader batch - labels shape: torch.Size([1, 1869]) +Dataloader batch - attn_mask shape: torch.Size([1, 1869]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 180 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1869) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1869) torch.int64 + loss_mask : 503 / 1869 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1869, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1869, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1869, 1, 576) torch.bfloat16 + out : (1, 1869) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 503 tokens) : 11.3424 + top-1 accuracy : 11/503 = 2.2% + + [2026-04-29 12:27:14] iteration 65/ 100 | consumed samples: 260 | elapsed time per iteration (ms): 4773.4 | throughput (token/sec/GPU): 1247.5 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.126869E+01 | loss scale: 1.0 | grad norm: 0.296 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 65 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 65 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 65 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1088.13, 1088.13) +Dataloader batch - tokens shape: torch.Size([1, 1568]) +Dataloader batch - labels shape: torch.Size([1, 1568]) +Dataloader batch - attn_mask shape: torch.Size([1, 1568]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 181 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1568) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1568) torch.int64 + loss_mask : 342 / 1568 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1568, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1568, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1568, 1, 576) torch.bfloat16 + out : (1, 1568) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2536 + top-1 accuracy : 22/342 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1113]) +Dataloader batch - labels shape: torch.Size([1, 1113]) +Dataloader batch - attn_mask shape: torch.Size([1, 1113]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 182 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1113) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1113) torch.int64 + loss_mask : 280 / 1113 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1113, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1113, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1113, 1, 576) torch.bfloat16 + out : (1, 1113) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 280 tokens) : 11.3231 + top-1 accuracy : 13/280 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 183 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 339 / 1436 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2133 + top-1 accuracy : 21/339 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 184 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 335 / 1518 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2662 + top-1 accuracy : 23/335 = 6.9% + + [2026-04-29 12:27:20] iteration 66/ 100 | consumed samples: 264 | elapsed time per iteration (ms): 4496.5 | throughput (token/sec/GPU): 1253.2 | learning rate: 5.687092E-05 | global batch size: 4 | lm loss: 1.126131E+01 | loss scale: 1.0 | grad norm: 0.391 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 66 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 66 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 66 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1469.97, 1469.97) +Dataloader batch - tokens shape: torch.Size([1, 1642]) +Dataloader batch - labels shape: torch.Size([1, 1642]) +Dataloader batch - attn_mask shape: torch.Size([1, 1642]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 185 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1642) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1642) torch.int64 + loss_mask : 345 / 1642 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1642, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1642, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1642, 1, 576) torch.bfloat16 + out : (1, 1642) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2212 + top-1 accuracy : 35/345 = 10.1% + +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 186 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 311 / 1496 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.1933 + top-1 accuracy : 35/311 = 11.3% + +Dataloader batch - tokens shape: torch.Size([1, 1566]) +Dataloader batch - labels shape: torch.Size([1, 1566]) +Dataloader batch - attn_mask shape: torch.Size([1, 1566]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 187 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1566) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1566) torch.int64 + loss_mask : 431 / 1566 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1566, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1566, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1566, 1, 576) torch.bfloat16 + out : (1, 1566) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.3286 + top-1 accuracy : 0/431 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1086]) +Dataloader batch - labels shape: torch.Size([1, 1086]) +Dataloader batch - attn_mask shape: torch.Size([1, 1086]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 188 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1086) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1086) torch.int64 + loss_mask : 321 / 1086 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1086, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1086, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1086, 1, 576) torch.bfloat16 + out : (1, 1086) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.4415 + top-1 accuracy : 3/321 = 0.9% + + [2026-04-29 12:27:26] iteration 67/ 100 | consumed samples: 268 | elapsed time per iteration (ms): 4538.4 | throughput (token/sec/GPU): 1275.8 | learning rate: 5.532277E-05 | global batch size: 4 | lm loss: 1.129814E+01 | loss scale: 1.0 | grad norm: 0.351 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 67 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 67 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 67 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1547.58, 1547.58) +Dataloader batch - tokens shape: torch.Size([1, 1720]) +Dataloader batch - labels shape: torch.Size([1, 1720]) +Dataloader batch - attn_mask shape: torch.Size([1, 1720]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 189 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1720) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1720) torch.int64 + loss_mask : 349 / 1720 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1720, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1720, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1720, 1, 576) torch.bfloat16 + out : (1, 1720) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2152 + top-1 accuracy : 29/349 = 8.3% + +Dataloader batch - tokens shape: torch.Size([1, 1864]) +Dataloader batch - labels shape: torch.Size([1, 1864]) +Dataloader batch - attn_mask shape: torch.Size([1, 1864]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 190 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1864) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1864) torch.int64 + loss_mask : 491 / 1864 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1864, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1864, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1864, 1, 576) torch.bfloat16 + out : (1, 1864) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 491 tokens) : 11.3263 + top-1 accuracy : 25/491 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1404]) +Dataloader batch - labels shape: torch.Size([1, 1404]) +Dataloader batch - attn_mask shape: torch.Size([1, 1404]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 191 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1404) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1404) torch.int64 + loss_mask : 343 / 1404 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1404, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1404, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1404, 1, 576) torch.bfloat16 + out : (1, 1404) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.3536 + top-1 accuracy : 14/343 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1193]) +Dataloader batch - labels shape: torch.Size([1, 1193]) +Dataloader batch - attn_mask shape: torch.Size([1, 1193]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 192 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1193) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1193) torch.int64 + loss_mask : 260 / 1193 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1193, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1193, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1193, 1, 576) torch.bfloat16 + out : (1, 1193) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.1832 + top-1 accuracy : 30/260 = 11.5% + + [2026-04-29 12:27:32] iteration 68/ 100 | consumed samples: 272 | elapsed time per iteration (ms): 4821.2 | throughput (token/sec/GPU): 1282.0 | learning rate: 5.376985E-05 | global batch size: 4 | lm loss: 1.128015E+01 | loss scale: 1.0 | grad norm: 0.412 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 68 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 68 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 68 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1440.58, 1440.58) +Dataloader batch - tokens shape: torch.Size([1, 1894]) +Dataloader batch - labels shape: torch.Size([1, 1894]) +Dataloader batch - attn_mask shape: torch.Size([1, 1894]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 193 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1894) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1894) torch.int64 + loss_mask : 545 / 1894 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1894, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1894, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1894, 1, 576) torch.bfloat16 + out : (1, 1894) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 545 tokens) : 11.3696 + top-1 accuracy : 28/545 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1185]) +Dataloader batch - labels shape: torch.Size([1, 1185]) +Dataloader batch - attn_mask shape: torch.Size([1, 1185]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 194 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1185) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1185) torch.int64 + loss_mask : 284 / 1185 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1185, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1185, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1185, 1, 576) torch.bfloat16 + out : (1, 1185) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.2731 + top-1 accuracy : 8/284 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1829]) +Dataloader batch - labels shape: torch.Size([1, 1829]) +Dataloader batch - attn_mask shape: torch.Size([1, 1829]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 195 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1829) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1829) torch.int64 + loss_mask : 519 / 1829 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1829, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1829, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1829, 1, 576) torch.bfloat16 + out : (1, 1829) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 519 tokens) : 11.3598 + top-1 accuracy : 31/519 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1779]) +Dataloader batch - labels shape: torch.Size([1, 1779]) +Dataloader batch - attn_mask shape: torch.Size([1, 1779]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 196 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1779) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1779) torch.int64 + loss_mask : 406 / 1779 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1779, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1779, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1779, 1, 576) torch.bfloat16 + out : (1, 1779) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.2497 + top-1 accuracy : 15/406 = 3.7% + + [2026-04-29 12:27:39] iteration 69/ 100 | consumed samples: 276 | elapsed time per iteration (ms): 4982.9 | throughput (token/sec/GPU): 1342.0 | learning rate: 5.221368E-05 | global batch size: 4 | lm loss: 1.132334E+01 | loss scale: 1.0 | grad norm: 0.381 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 69 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 69 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 69 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1098.14, 1098.14) +Dataloader batch - tokens shape: torch.Size([1, 1485]) +Dataloader batch - labels shape: torch.Size([1, 1485]) +Dataloader batch - attn_mask shape: torch.Size([1, 1485]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 197 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1485) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1485) torch.int64 + loss_mask : 328 / 1485 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1485, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1485, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1485, 1, 576) torch.bfloat16 + out : (1, 1485) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.1742 + top-1 accuracy : 31/328 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1235]) +Dataloader batch - labels shape: torch.Size([1, 1235]) +Dataloader batch - attn_mask shape: torch.Size([1, 1235]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 198 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1235) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1235) torch.int64 + loss_mask : 295 / 1235 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1235, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1235, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1235, 1, 576) torch.bfloat16 + out : (1, 1235) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.2243 + top-1 accuracy : 17/295 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1448]) +Dataloader batch - labels shape: torch.Size([1, 1448]) +Dataloader batch - attn_mask shape: torch.Size([1, 1448]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 199 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1448) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1448) torch.int64 + loss_mask : 415 / 1448 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1448, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1448, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1448, 1, 576) torch.bfloat16 + out : (1, 1448) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.3514 + top-1 accuracy : 14/415 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1462]) +Dataloader batch - labels shape: torch.Size([1, 1462]) +Dataloader batch - attn_mask shape: torch.Size([1, 1462]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 200 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1462) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1462) torch.int64 + loss_mask : 314 / 1462 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1462, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1462, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1462, 1, 576) torch.bfloat16 + out : (1, 1462) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.2034 + top-1 accuracy : 22/314 = 7.0% + + [2026-04-29 12:27:44] iteration 70/ 100 | consumed samples: 280 | elapsed time per iteration (ms): 4601.4 | throughput (token/sec/GPU): 1223.5 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.124630E+01 | loss scale: 1.0 | grad norm: 0.378 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 70 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 70 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 70 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1313.10, 1313.10) +Dataloader batch - tokens shape: torch.Size([1, 1482]) +Dataloader batch - labels shape: torch.Size([1, 1482]) +Dataloader batch - attn_mask shape: torch.Size([1, 1482]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 201 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1482) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1482) torch.int64 + loss_mask : 322 / 1482 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1482, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1482, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1482, 1, 576) torch.bfloat16 + out : (1, 1482) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.2092 + top-1 accuracy : 14/322 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1589]) +Dataloader batch - labels shape: torch.Size([1, 1589]) +Dataloader batch - attn_mask shape: torch.Size([1, 1589]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 202 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1589) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1589) torch.int64 + loss_mask : 372 / 1589 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1589, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1589, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1589, 1, 576) torch.bfloat16 + out : (1, 1589) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2545 + top-1 accuracy : 30/372 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1706]) +Dataloader batch - labels shape: torch.Size([1, 1706]) +Dataloader batch - attn_mask shape: torch.Size([1, 1706]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 203 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1706) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1706) torch.int64 + loss_mask : 361 / 1706 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1706, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1706, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1706, 1, 576) torch.bfloat16 + out : (1, 1706) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.2415 + top-1 accuracy : 39/361 = 10.8% + +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 204 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 355 / 1520 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2564 + top-1 accuracy : 28/355 = 7.9% + + [2026-04-29 12:27:51] iteration 71/ 100 | consumed samples: 284 | elapsed time per iteration (ms): 4959.8 | throughput (token/sec/GPU): 1269.6 | learning rate: 4.909780E-05 | global batch size: 4 | lm loss: 1.124132E+01 | loss scale: 1.0 | grad norm: 0.359 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 71 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 71 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 71 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1108.27, 1108.27) +Dataloader batch - tokens shape: torch.Size([1, 972]) +Dataloader batch - labels shape: torch.Size([1, 972]) +Dataloader batch - attn_mask shape: torch.Size([1, 972]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 205 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 972) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 972) torch.int64 + loss_mask : 231 / 972 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (972, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (972, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (972, 1, 576) torch.bfloat16 + out : (1, 972) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 231 tokens) : 11.2373 + top-1 accuracy : 8/231 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1462]) +Dataloader batch - labels shape: torch.Size([1, 1462]) +Dataloader batch - attn_mask shape: torch.Size([1, 1462]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 206 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1462) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1462) torch.int64 + loss_mask : 392 / 1462 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1462, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1462, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1462, 1, 576) torch.bfloat16 + out : (1, 1462) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.2885 + top-1 accuracy : 42/392 = 10.7% + +Dataloader batch - tokens shape: torch.Size([1, 1658]) +Dataloader batch - labels shape: torch.Size([1, 1658]) +Dataloader batch - attn_mask shape: torch.Size([1, 1658]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 207 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1658) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1658) torch.int64 + loss_mask : 384 / 1658 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1658, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1658, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1658, 1, 576) torch.bfloat16 + out : (1, 1658) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.2922 + top-1 accuracy : 25/384 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 208 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1696) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1696) torch.int64 + loss_mask : 333 / 1696 tokens (19.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1696, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1696, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1696, 1, 576) torch.bfloat16 + out : (1, 1696) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.1868 + top-1 accuracy : 34/333 = 10.2% + + [2026-04-29 12:27:56] iteration 72/ 100 | consumed samples: 288 | elapsed time per iteration (ms): 4641.4 | throughput (token/sec/GPU): 1247.0 | learning rate: 4.754118E-05 | global batch size: 4 | lm loss: 1.125546E+01 | loss scale: 1.0 | grad norm: 0.422 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 72 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 72 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 72 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1136.22, 1136.22) +Dataloader batch - tokens shape: torch.Size([1, 1067]) +Dataloader batch - labels shape: torch.Size([1, 1067]) +Dataloader batch - attn_mask shape: torch.Size([1, 1067]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 209 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1067) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1067) torch.int64 + loss_mask : 282 / 1067 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1067, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1067, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1067, 1, 576) torch.bfloat16 + out : (1, 1067) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 282 tokens) : 11.3141 + top-1 accuracy : 7/282 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 210 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 398 / 1487 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.3548 + top-1 accuracy : 26/398 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1877]) +Dataloader batch - labels shape: torch.Size([1, 1877]) +Dataloader batch - attn_mask shape: torch.Size([1, 1877]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 211 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1877) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1877) torch.int64 + loss_mask : 519 / 1877 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1877, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1877, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1877, 1, 576) torch.bfloat16 + out : (1, 1877) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 519 tokens) : 11.3302 + top-1 accuracy : 17/519 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 798]) +Dataloader batch - labels shape: torch.Size([1, 798]) +Dataloader batch - attn_mask shape: torch.Size([1, 798]) +Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) + +==================================================================== + PIPELINE — STEP 212 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 798) torch.int64 + images : (16, 3, 1024, 1024) torch.bfloat16 + labels : (1, 798) torch.int64 + loss_mask : 139 / 798 tokens (17.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (16, 3, 1024, 1024) torch.bfloat16 + out : (16, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (16, 3072) + out : (16, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (798, 1, 576) torch.bfloat16 grad=False + image slots : 16 tokens replaced with vision embeddings + combined : (798, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (798, 1, 576) torch.bfloat16 + out : (1, 798) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 139 tokens) : 11.0157 + top-1 accuracy : 4/139 = 2.9% + + [2026-04-29 12:28:02] iteration 73/ 100 | consumed samples: 292 | elapsed time per iteration (ms): 4262.1 | throughput (token/sec/GPU): 1226.9 | learning rate: 4.598748E-05 | global batch size: 4 | lm loss: 1.130146E+01 | loss scale: 1.0 | grad norm: 0.397 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 73 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 73 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 73 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1702.11, 1702.11) +Dataloader batch - tokens shape: torch.Size([1, 1910]) +Dataloader batch - labels shape: torch.Size([1, 1910]) +Dataloader batch - attn_mask shape: torch.Size([1, 1910]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 213 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1910) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1910) torch.int64 + loss_mask : 552 / 1910 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1910, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1910, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1910, 1, 576) torch.bfloat16 + out : (1, 1910) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 552 tokens) : 11.3756 + top-1 accuracy : 20/552 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1461]) +Dataloader batch - labels shape: torch.Size([1, 1461]) +Dataloader batch - attn_mask shape: torch.Size([1, 1461]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 214 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1461) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1461) torch.int64 + loss_mask : 304 / 1461 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1461, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1461, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1461, 1, 576) torch.bfloat16 + out : (1, 1461) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 304 tokens) : 11.1184 + top-1 accuracy : 33/304 = 10.9% + +Dataloader batch - tokens shape: torch.Size([1, 1548]) +Dataloader batch - labels shape: torch.Size([1, 1548]) +Dataloader batch - attn_mask shape: torch.Size([1, 1548]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 215 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1548) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1548) torch.int64 + loss_mask : 406 / 1548 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1548, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1548, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1548, 1, 576) torch.bfloat16 + out : (1, 1548) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.3065 + top-1 accuracy : 19/406 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 216 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 426 / 1552 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.3406 + top-1 accuracy : 15/426 = 3.5% + + [2026-04-29 12:28:08] iteration 74/ 100 | consumed samples: 296 | elapsed time per iteration (ms): 4903.4 | throughput (token/sec/GPU): 1319.7 | learning rate: 4.443826E-05 | global batch size: 4 | lm loss: 1.130385E+01 | loss scale: 1.0 | grad norm: 0.273 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 74 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 74 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 74 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1279.16, 1279.16) +Dataloader batch - tokens shape: torch.Size([1, 1758]) +Dataloader batch - labels shape: torch.Size([1, 1758]) +Dataloader batch - attn_mask shape: torch.Size([1, 1758]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 217 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1758) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1758) torch.int64 + loss_mask : 409 / 1758 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1758, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1758, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1758, 1, 576) torch.bfloat16 + out : (1, 1758) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.2733 + top-1 accuracy : 33/409 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1129]) +Dataloader batch - labels shape: torch.Size([1, 1129]) +Dataloader batch - attn_mask shape: torch.Size([1, 1129]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 218 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1129) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1129) torch.int64 + loss_mask : 324 / 1129 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1129, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1129, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1129, 1, 576) torch.bfloat16 + out : (1, 1129) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.3479 + top-1 accuracy : 13/324 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 219 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 447 / 1475 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 447 tokens) : 11.4263 + top-1 accuracy : 5/447 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 220 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 420 / 1388 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 420 tokens) : 11.3947 + top-1 accuracy : 13/420 = 3.1% + + [2026-04-29 12:28:14] iteration 75/ 100 | consumed samples: 300 | elapsed time per iteration (ms): 4372.6 | throughput (token/sec/GPU): 1315.0 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.136302E+01 | loss scale: 1.0 | grad norm: 0.371 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 75 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 75 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 75 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1071.20, 1071.20) +Dataloader batch - tokens shape: torch.Size([1, 893]) +Dataloader batch - labels shape: torch.Size([1, 893]) +Dataloader batch - attn_mask shape: torch.Size([1, 893]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 221 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 893) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 893) torch.int64 + loss_mask : 200 / 893 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (893, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (893, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (893, 1, 576) torch.bfloat16 + out : (1, 893) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 200 tokens) : 11.2081 + top-1 accuracy : 8/200 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1165]) +Dataloader batch - labels shape: torch.Size([1, 1165]) +Dataloader batch - attn_mask shape: torch.Size([1, 1165]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 222 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1165) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1165) torch.int64 + loss_mask : 341 / 1165 tokens (29.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1165, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1165, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1165, 1, 576) torch.bfloat16 + out : (1, 1165) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.4365 + top-1 accuracy : 2/341 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 223 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1191) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1191) torch.int64 + loss_mask : 308 / 1191 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1191, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1191, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1191, 1, 576) torch.bfloat16 + out : (1, 1191) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.3200 + top-1 accuracy : 19/308 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1177]) +Dataloader batch - labels shape: torch.Size([1, 1177]) +Dataloader batch - attn_mask shape: torch.Size([1, 1177]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 224 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1177) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1177) torch.int64 + loss_mask : 254 / 1177 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1177, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1177, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1177, 1, 576) torch.bfloat16 + out : (1, 1177) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 254 tokens) : 11.1702 + top-1 accuracy : 18/254 = 7.1% + + [2026-04-29 12:28:19] iteration 76/ 100 | consumed samples: 304 | elapsed time per iteration (ms): 3588.8 | throughput (token/sec/GPU): 1233.3 | learning rate: 4.135936E-05 | global batch size: 4 | lm loss: 1.130125E+01 | loss scale: 1.0 | grad norm: 0.328 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 76 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 76 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 76 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1501.63, 1501.63) +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 225 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 366 / 1541 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2456 + top-1 accuracy : 32/366 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 226 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1420) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1420) torch.int64 + loss_mask : 334 / 1420 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1420, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1420, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1420, 1, 576) torch.bfloat16 + out : (1, 1420) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.2452 + top-1 accuracy : 27/334 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1054]) +Dataloader batch - labels shape: torch.Size([1, 1054]) +Dataloader batch - attn_mask shape: torch.Size([1, 1054]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 227 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1054) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1054) torch.int64 + loss_mask : 225 / 1054 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1054, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1054, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1054, 1, 576) torch.bfloat16 + out : (1, 1054) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 225 tokens) : 11.1657 + top-1 accuracy : 7/225 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1128]) +Dataloader batch - labels shape: torch.Size([1, 1128]) +Dataloader batch - attn_mask shape: torch.Size([1, 1128]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 228 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1128) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1128) torch.int64 + loss_mask : 327 / 1128 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1128, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1128, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1128, 1, 576) torch.bfloat16 + out : (1, 1128) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.3652 + top-1 accuracy : 11/327 = 3.4% + + [2026-04-29 12:28:24] iteration 77/ 100 | consumed samples: 308 | elapsed time per iteration (ms): 4186.2 | throughput (token/sec/GPU): 1228.6 | learning rate: 3.983273E-05 | global batch size: 4 | lm loss: 1.126238E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 77 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 77 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 77 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2038.45, 2038.45) +Dataloader batch - tokens shape: torch.Size([1, 957]) +Dataloader batch - labels shape: torch.Size([1, 957]) +Dataloader batch - attn_mask shape: torch.Size([1, 957]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 229 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 957) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 957) torch.int64 + loss_mask : 226 / 957 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (957, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (957, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (957, 1, 576) torch.bfloat16 + out : (1, 957) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 226 tokens) : 11.2447 + top-1 accuracy : 14/226 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1368]) +Dataloader batch - labels shape: torch.Size([1, 1368]) +Dataloader batch - attn_mask shape: torch.Size([1, 1368]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 230 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1368) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1368) torch.int64 + loss_mask : 319 / 1368 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1368, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1368, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1368, 1, 576) torch.bfloat16 + out : (1, 1368) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.2376 + top-1 accuracy : 7/319 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1735]) +Dataloader batch - labels shape: torch.Size([1, 1735]) +Dataloader batch - attn_mask shape: torch.Size([1, 1735]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 231 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1735) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1735) torch.int64 + loss_mask : 393 / 1735 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1735, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1735, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1735, 1, 576) torch.bfloat16 + out : (1, 1735) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 393 tokens) : 11.2205 + top-1 accuracy : 25/393 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1391]) +Dataloader batch - labels shape: torch.Size([1, 1391]) +Dataloader batch - attn_mask shape: torch.Size([1, 1391]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 232 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1391) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1391) torch.int64 + loss_mask : 334 / 1391 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1391, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1391, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1391, 1, 576) torch.bfloat16 + out : (1, 1391) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.2543 + top-1 accuracy : 9/334 = 2.7% + + [2026-04-29 12:28:31] iteration 78/ 100 | consumed samples: 312 | elapsed time per iteration (ms): 4391.0 | throughput (token/sec/GPU): 1241.4 | learning rate: 3.831667E-05 | global batch size: 4 | lm loss: 1.123792E+01 | loss scale: 1.0 | grad norm: 0.359 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 78 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 78 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 78 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1822.49, 1822.49) +Dataloader batch - tokens shape: torch.Size([1, 1650]) +Dataloader batch - labels shape: torch.Size([1, 1650]) +Dataloader batch - attn_mask shape: torch.Size([1, 1650]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 233 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1650) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1650) torch.int64 + loss_mask : 391 / 1650 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1650, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1650, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1650, 1, 576) torch.bfloat16 + out : (1, 1650) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2937 + top-1 accuracy : 26/391 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 234 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 348 / 1444 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2758 + top-1 accuracy : 18/348 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 235 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 290 / 1380 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.1802 + top-1 accuracy : 28/290 = 9.7% + +Dataloader batch - tokens shape: torch.Size([1, 1222]) +Dataloader batch - labels shape: torch.Size([1, 1222]) +Dataloader batch - attn_mask shape: torch.Size([1, 1222]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 236 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1222) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1222) torch.int64 + loss_mask : 328 / 1222 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1222, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1222, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1222, 1, 576) torch.bfloat16 + out : (1, 1222) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.3261 + top-1 accuracy : 11/328 = 3.4% + + [2026-04-29 12:28:37] iteration 79/ 100 | consumed samples: 316 | elapsed time per iteration (ms): 4710.6 | throughput (token/sec/GPU): 1209.2 | learning rate: 3.681269E-05 | global batch size: 4 | lm loss: 1.127269E+01 | loss scale: 1.0 | grad norm: 0.346 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 79 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 79 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 79 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1816.49, 1816.49) +Dataloader batch - tokens shape: torch.Size([1, 1161]) +Dataloader batch - labels shape: torch.Size([1, 1161]) +Dataloader batch - attn_mask shape: torch.Size([1, 1161]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 237 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1161) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1161) torch.int64 + loss_mask : 322 / 1161 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1161, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1161, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1161, 1, 576) torch.bfloat16 + out : (1, 1161) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.3296 + top-1 accuracy : 7/322 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 238 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 321 / 1374 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2287 + top-1 accuracy : 15/321 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 239 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 297 / 1422 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.1447 + top-1 accuracy : 8/297 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1133]) +Dataloader batch - labels shape: torch.Size([1, 1133]) +Dataloader batch - attn_mask shape: torch.Size([1, 1133]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 240 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1133) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1133) torch.int64 + loss_mask : 324 / 1133 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1133, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1133, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1133, 1, 576) torch.bfloat16 + out : (1, 1133) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.3818 + top-1 accuracy : 3/324 = 0.9% + + [2026-04-29 12:28:44] iteration 80/ 100 | consumed samples: 320 | elapsed time per iteration (ms): 4800.9 | throughput (token/sec/GPU): 1060.2 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.127392E+01 | loss scale: 1.0 | grad norm: 0.367 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4778.70, 4778.70) + forward-compute ................................: (3554.25, 3554.25) + backward-compute ...............................: (1220.09, 1220.09) + batch-generator ................................: (388.68, 388.68) + layernorm-grads-all-reduce .....................: (0.07, 0.07) + embedding-grads-all-reduce .....................: (0.11, 0.11) + all-grads-sync .................................: (1.30, 1.30) + params-all-gather ..............................: (0.42, 0.42) + optimizer-copy-to-main-grad ....................: (0.42, 0.42) + optimizer-clip-main-grad .......................: (1.46, 1.46) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.62, 2.62) + optimizer-copy-main-to-model-params ............: (0.63, 0.63) + optimizer ......................................: (7.71, 7.71) +saving checkpoint at iteration 80 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 80 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 80 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1855.26, 1855.26) +Dataloader batch - tokens shape: torch.Size([1, 957]) +Dataloader batch - labels shape: torch.Size([1, 957]) +Dataloader batch - attn_mask shape: torch.Size([1, 957]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 241 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 957) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 957) torch.int64 + loss_mask : 209 / 957 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (957, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (957, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (957, 1, 576) torch.bfloat16 + out : (1, 957) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 209 tokens) : 11.1600 + top-1 accuracy : 15/209 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1194]) +Dataloader batch - labels shape: torch.Size([1, 1194]) +Dataloader batch - attn_mask shape: torch.Size([1, 1194]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 242 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1194) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1194) torch.int64 + loss_mask : 288 / 1194 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1194, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1194, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1194, 1, 576) torch.bfloat16 + out : (1, 1194) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 288 tokens) : 11.2588 + top-1 accuracy : 17/288 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 985]) +Dataloader batch - labels shape: torch.Size([1, 985]) +Dataloader batch - attn_mask shape: torch.Size([1, 985]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 243 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 985) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 985) torch.int64 + loss_mask : 224 / 985 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (985, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (985, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (985, 1, 576) torch.bfloat16 + out : (1, 985) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 224 tokens) : 11.2230 + top-1 accuracy : 16/224 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1103]) +Dataloader batch - labels shape: torch.Size([1, 1103]) +Dataloader batch - attn_mask shape: torch.Size([1, 1103]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 244 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1103) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1103) torch.int64 + loss_mask : 296 / 1103 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1103, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1103, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1103, 1, 576) torch.bfloat16 + out : (1, 1103) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 296 tokens) : 11.3725 + top-1 accuracy : 14/296 = 4.7% + + [2026-04-29 12:28:50] iteration 81/ 100 | consumed samples: 324 | elapsed time per iteration (ms): 3694.8 | throughput (token/sec/GPU): 1147.3 | learning rate: 3.384688E-05 | global batch size: 4 | lm loss: 1.126372E+01 | loss scale: 1.0 | grad norm: 0.370 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 81 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 81 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 81 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1849.86, 1849.86) +Dataloader batch - tokens shape: torch.Size([1, 1645]) +Dataloader batch - labels shape: torch.Size([1, 1645]) +Dataloader batch - attn_mask shape: torch.Size([1, 1645]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 245 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1645) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1645) torch.int64 + loss_mask : 394 / 1645 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1645, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1645, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1645, 1, 576) torch.bfloat16 + out : (1, 1645) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.2224 + top-1 accuracy : 10/394 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1700]) +Dataloader batch - labels shape: torch.Size([1, 1700]) +Dataloader batch - attn_mask shape: torch.Size([1, 1700]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 246 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1700) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1700) torch.int64 + loss_mask : 351 / 1700 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1700, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1700, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1700, 1, 576) torch.bfloat16 + out : (1, 1700) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.1766 + top-1 accuracy : 30/351 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1366]) +Dataloader batch - labels shape: torch.Size([1, 1366]) +Dataloader batch - attn_mask shape: torch.Size([1, 1366]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 247 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1366) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1366) torch.int64 + loss_mask : 296 / 1366 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1366, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1366, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1366, 1, 576) torch.bfloat16 + out : (1, 1366) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 296 tokens) : 11.1789 + top-1 accuracy : 23/296 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 248 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 284 / 1364 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.1739 + top-1 accuracy : 27/284 = 9.5% + + [2026-04-29 12:28:57] iteration 82/ 100 | consumed samples: 328 | elapsed time per iteration (ms): 5080.6 | throughput (token/sec/GPU): 1195.7 | learning rate: 3.238799E-05 | global batch size: 4 | lm loss: 1.119016E+01 | loss scale: 1.0 | grad norm: 0.350 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 82 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 82 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 82 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1661.16, 1661.16) +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 249 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 343 / 1492 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.2033 + top-1 accuracy : 12/343 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1909]) +Dataloader batch - labels shape: torch.Size([1, 1909]) +Dataloader batch - attn_mask shape: torch.Size([1, 1909]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 250 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 551 / 1909 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 551 tokens) : 11.3749 + top-1 accuracy : 19/551 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1838]) +Dataloader batch - labels shape: torch.Size([1, 1838]) +Dataloader batch - attn_mask shape: torch.Size([1, 1838]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 251 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1838) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1838) torch.int64 + loss_mask : 468 / 1838 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1838, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1838, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1838, 1, 576) torch.bfloat16 + out : (1, 1838) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 468 tokens) : 11.3236 + top-1 accuracy : 5/468 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1126]) +Dataloader batch - labels shape: torch.Size([1, 1126]) +Dataloader batch - attn_mask shape: torch.Size([1, 1126]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 252 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1126) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1126) torch.int64 + loss_mask : 320 / 1126 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1126, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1126, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1126, 1, 576) torch.bfloat16 + out : (1, 1126) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.3924 + top-1 accuracy : 11/320 = 3.4% + + [2026-04-29 12:29:04] iteration 83/ 100 | consumed samples: 332 | elapsed time per iteration (ms): 6265.7 | throughput (token/sec/GPU): 1015.8 | learning rate: 3.094706E-05 | global batch size: 4 | lm loss: 1.132897E+01 | loss scale: 1.0 | grad norm: 0.308 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 83 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 83 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 83 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1423.14, 1423.14) +Dataloader batch - tokens shape: torch.Size([1, 1416]) +Dataloader batch - labels shape: torch.Size([1, 1416]) +Dataloader batch - attn_mask shape: torch.Size([1, 1416]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 253 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1416) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1416) torch.int64 + loss_mask : 353 / 1416 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1416, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1416, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1416, 1, 576) torch.bfloat16 + out : (1, 1416) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.3436 + top-1 accuracy : 18/353 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 254 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 337 / 1507 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.2060 + top-1 accuracy : 30/337 = 8.9% + +Dataloader batch - tokens shape: torch.Size([1, 1932]) +Dataloader batch - labels shape: torch.Size([1, 1932]) +Dataloader batch - attn_mask shape: torch.Size([1, 1932]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 255 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1932) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1932) torch.int64 + loss_mask : 561 / 1932 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1932, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1932, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1932, 1, 576) torch.bfloat16 + out : (1, 1932) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 561 tokens) : 11.3814 + top-1 accuracy : 4/561 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1512]) +Dataloader batch - labels shape: torch.Size([1, 1512]) +Dataloader batch - attn_mask shape: torch.Size([1, 1512]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 256 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1512) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1512) torch.int64 + loss_mask : 373 / 1512 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1512, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1512, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1512, 1, 576) torch.bfloat16 + out : (1, 1512) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.2547 + top-1 accuracy : 23/373 = 6.2% + + [2026-04-29 12:29:12] iteration 84/ 100 | consumed samples: 336 | elapsed time per iteration (ms): 5891.0 | throughput (token/sec/GPU): 1080.8 | learning rate: 2.952549E-05 | global batch size: 4 | lm loss: 1.130770E+01 | loss scale: 1.0 | grad norm: 0.353 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 84 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 84 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 84 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1859.02, 1859.02) +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 257 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 347 / 1542 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2385 + top-1 accuracy : 19/347 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1528]) +Dataloader batch - labels shape: torch.Size([1, 1528]) +Dataloader batch - attn_mask shape: torch.Size([1, 1528]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 258 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1528) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1528) torch.int64 + loss_mask : 370 / 1528 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1528, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1528, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1528, 1, 576) torch.bfloat16 + out : (1, 1528) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.2657 + top-1 accuracy : 15/370 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1655]) +Dataloader batch - labels shape: torch.Size([1, 1655]) +Dataloader batch - attn_mask shape: torch.Size([1, 1655]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 259 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1655) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1655) torch.int64 + loss_mask : 380 / 1655 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1655, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1655, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1655, 1, 576) torch.bfloat16 + out : (1, 1655) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 380 tokens) : 11.2787 + top-1 accuracy : 25/380 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1867]) +Dataloader batch - labels shape: torch.Size([1, 1867]) +Dataloader batch - attn_mask shape: torch.Size([1, 1867]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 260 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1867) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1867) torch.int64 + loss_mask : 548 / 1867 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1867, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1867, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1867, 1, 576) torch.bfloat16 + out : (1, 1867) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 548 tokens) : 11.4075 + top-1 accuracy : 5/548 = 0.9% + + [2026-04-29 12:29:19] iteration 85/ 100 | consumed samples: 340 | elapsed time per iteration (ms): 5600.4 | throughput (token/sec/GPU): 1177.1 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.131021E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 85 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 85 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 85 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1274.96, 1274.96) +Dataloader batch - tokens shape: torch.Size([1, 1458]) +Dataloader batch - labels shape: torch.Size([1, 1458]) +Dataloader batch - attn_mask shape: torch.Size([1, 1458]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 261 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1458) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1458) torch.int64 + loss_mask : 310 / 1458 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1458, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1458, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1458, 1, 576) torch.bfloat16 + out : (1, 1458) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 310 tokens) : 11.1994 + top-1 accuracy : 13/310 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 945]) +Dataloader batch - labels shape: torch.Size([1, 945]) +Dataloader batch - attn_mask shape: torch.Size([1, 945]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 262 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 945) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 945) torch.int64 + loss_mask : 211 / 945 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (945, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (945, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (945, 1, 576) torch.bfloat16 + out : (1, 945) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 211 tokens) : 11.1300 + top-1 accuracy : 3/211 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1654]) +Dataloader batch - labels shape: torch.Size([1, 1654]) +Dataloader batch - attn_mask shape: torch.Size([1, 1654]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 263 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1654) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1654) torch.int64 + loss_mask : 409 / 1654 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1654, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1654, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1654, 1, 576) torch.bfloat16 + out : (1, 1654) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.2682 + top-1 accuracy : 32/409 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1177]) +Dataloader batch - labels shape: torch.Size([1, 1177]) +Dataloader batch - attn_mask shape: torch.Size([1, 1177]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 264 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1177) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1177) torch.int64 + loss_mask : 263 / 1177 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1177, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1177, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1177, 1, 576) torch.bfloat16 + out : (1, 1177) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.2061 + top-1 accuracy : 16/263 = 6.1% + + [2026-04-29 12:29:25] iteration 86/ 100 | consumed samples: 344 | elapsed time per iteration (ms): 4379.3 | throughput (token/sec/GPU): 1195.2 | learning rate: 2.674610E-05 | global batch size: 4 | lm loss: 1.121220E+01 | loss scale: 1.0 | grad norm: 0.376 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 86 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 86 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 86 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1554.00, 1554.00) +Dataloader batch - tokens shape: torch.Size([1, 1417]) +Dataloader batch - labels shape: torch.Size([1, 1417]) +Dataloader batch - attn_mask shape: torch.Size([1, 1417]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 265 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1417) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1417) torch.int64 + loss_mask : 401 / 1417 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1417, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1417, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1417, 1, 576) torch.bfloat16 + out : (1, 1417) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 401 tokens) : 11.3728 + top-1 accuracy : 4/401 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 974]) +Dataloader batch - labels shape: torch.Size([1, 974]) +Dataloader batch - attn_mask shape: torch.Size([1, 974]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 266 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 974) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 974) torch.int64 + loss_mask : 229 / 974 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (974, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (974, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (974, 1, 576) torch.bfloat16 + out : (1, 974) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 229 tokens) : 11.2548 + top-1 accuracy : 3/229 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1371]) +Dataloader batch - labels shape: torch.Size([1, 1371]) +Dataloader batch - attn_mask shape: torch.Size([1, 1371]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 267 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1371) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1371) torch.int64 + loss_mask : 324 / 1371 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1371, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1371, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1371, 1, 576) torch.bfloat16 + out : (1, 1371) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.2560 + top-1 accuracy : 24/324 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1781]) +Dataloader batch - labels shape: torch.Size([1, 1781]) +Dataloader batch - attn_mask shape: torch.Size([1, 1781]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 268 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1781) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1781) torch.int64 + loss_mask : 401 / 1781 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1781, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1781, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1781, 1, 576) torch.bfloat16 + out : (1, 1781) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 401 tokens) : 11.2737 + top-1 accuracy : 23/401 = 5.7% + + [2026-04-29 12:29:31] iteration 87/ 100 | consumed samples: 348 | elapsed time per iteration (ms): 4389.7 | throughput (token/sec/GPU): 1262.7 | learning rate: 2.539102E-05 | global batch size: 4 | lm loss: 1.129560E+01 | loss scale: 1.0 | grad norm: 0.322 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 87 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 87 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 87 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1476.73, 1476.73) +Dataloader batch - tokens shape: torch.Size([1, 1333]) +Dataloader batch - labels shape: torch.Size([1, 1333]) +Dataloader batch - attn_mask shape: torch.Size([1, 1333]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 269 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1333) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1333) torch.int64 + loss_mask : 371 / 1333 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1333, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1333, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1333, 1, 576) torch.bfloat16 + out : (1, 1333) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.3141 + top-1 accuracy : 15/371 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1441]) +Dataloader batch - labels shape: torch.Size([1, 1441]) +Dataloader batch - attn_mask shape: torch.Size([1, 1441]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 270 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1441) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1441) torch.int64 + loss_mask : 389 / 1441 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1441, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1441, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1441, 1, 576) torch.bfloat16 + out : (1, 1441) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.2916 + top-1 accuracy : 25/389 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1172]) +Dataloader batch - labels shape: torch.Size([1, 1172]) +Dataloader batch - attn_mask shape: torch.Size([1, 1172]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 271 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1172) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1172) torch.int64 + loss_mask : 312 / 1172 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1172, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1172, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1172, 1, 576) torch.bfloat16 + out : (1, 1172) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.3251 + top-1 accuracy : 8/312 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1839]) +Dataloader batch - labels shape: torch.Size([1, 1839]) +Dataloader batch - attn_mask shape: torch.Size([1, 1839]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 272 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1839) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1839) torch.int64 + loss_mask : 487 / 1839 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1839, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1839, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1839, 1, 576) torch.bfloat16 + out : (1, 1839) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 487 tokens) : 11.3547 + top-1 accuracy : 31/487 = 6.4% + + [2026-04-29 12:29:38] iteration 88/ 100 | consumed samples: 352 | elapsed time per iteration (ms): 5240.6 | throughput (token/sec/GPU): 1103.9 | learning rate: 2.406083E-05 | global batch size: 4 | lm loss: 1.132336E+01 | loss scale: 1.0 | grad norm: 0.314 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 88 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 88 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 88 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1164.40, 1164.40) +Dataloader batch - tokens shape: torch.Size([1, 1427]) +Dataloader batch - labels shape: torch.Size([1, 1427]) +Dataloader batch - attn_mask shape: torch.Size([1, 1427]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 273 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1427) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1427) torch.int64 + loss_mask : 391 / 1427 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1427, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1427, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1427, 1, 576) torch.bfloat16 + out : (1, 1427) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3749 + top-1 accuracy : 3/391 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1389]) +Dataloader batch - labels shape: torch.Size([1, 1389]) +Dataloader batch - attn_mask shape: torch.Size([1, 1389]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 274 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1389) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1389) torch.int64 + loss_mask : 308 / 1389 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1389, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1389, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1389, 1, 576) torch.bfloat16 + out : (1, 1389) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.2319 + top-1 accuracy : 13/308 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1909]) +Dataloader batch - labels shape: torch.Size([1, 1909]) +Dataloader batch - attn_mask shape: torch.Size([1, 1909]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 275 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 550 / 1909 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 550 tokens) : 11.3745 + top-1 accuracy : 8/550 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 276 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 342 / 1518 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2667 + top-1 accuracy : 24/342 = 7.0% + + [2026-04-29 12:29:44] iteration 89/ 100 | consumed samples: 356 | elapsed time per iteration (ms): 5485.8 | throughput (token/sec/GPU): 1138.0 | learning rate: 2.275683E-05 | global batch size: 4 | lm loss: 1.132383E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 89 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 89 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 89 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1999.88, 1999.88) +Dataloader batch - tokens shape: torch.Size([1, 1589]) +Dataloader batch - labels shape: torch.Size([1, 1589]) +Dataloader batch - attn_mask shape: torch.Size([1, 1589]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 277 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1589) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1589) torch.int64 + loss_mask : 436 / 1589 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1589, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1589, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1589, 1, 576) torch.bfloat16 + out : (1, 1589) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.3796 + top-1 accuracy : 18/436 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1930]) +Dataloader batch - labels shape: torch.Size([1, 1930]) +Dataloader batch - attn_mask shape: torch.Size([1, 1930]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 278 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1930) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1930) torch.int64 + loss_mask : 565 / 1930 tokens (29.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1930, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1930, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1930, 1, 576) torch.bfloat16 + out : (1, 1930) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 565 tokens) : 11.4161 + top-1 accuracy : 4/565 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1721]) +Dataloader batch - labels shape: torch.Size([1, 1721]) +Dataloader batch - attn_mask shape: torch.Size([1, 1721]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 279 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1721) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1721) torch.int64 + loss_mask : 371 / 1721 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1721, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1721, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1721, 1, 576) torch.bfloat16 + out : (1, 1721) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.2466 + top-1 accuracy : 30/371 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1715]) +Dataloader batch - labels shape: torch.Size([1, 1715]) +Dataloader batch - attn_mask shape: torch.Size([1, 1715]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 280 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1715) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1715) torch.int64 + loss_mask : 395 / 1715 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1715, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1715, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1715, 1, 576) torch.bfloat16 + out : (1, 1715) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.2559 + top-1 accuracy : 25/395 = 6.3% + + [2026-04-29 12:29:52] iteration 90/ 100 | consumed samples: 360 | elapsed time per iteration (ms): 5365.9 | throughput (token/sec/GPU): 1296.1 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.133569E+01 | loss scale: 1.0 | grad norm: 0.313 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 90 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 90 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 90 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1063.59, 1063.59) +Dataloader batch - tokens shape: torch.Size([1, 1421]) +Dataloader batch - labels shape: torch.Size([1, 1421]) +Dataloader batch - attn_mask shape: torch.Size([1, 1421]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 281 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1421) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1421) torch.int64 + loss_mask : 320 / 1421 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1421, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1421, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1421, 1, 576) torch.bfloat16 + out : (1, 1421) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2452 + top-1 accuracy : 22/320 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 282 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 387 / 1504 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2935 + top-1 accuracy : 13/387 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1119]) +Dataloader batch - labels shape: torch.Size([1, 1119]) +Dataloader batch - attn_mask shape: torch.Size([1, 1119]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 283 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1119) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1119) torch.int64 + loss_mask : 269 / 1119 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1119, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1119, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1119, 1, 576) torch.bfloat16 + out : (1, 1119) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 269 tokens) : 11.2710 + top-1 accuracy : 12/269 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1180]) +Dataloader batch - labels shape: torch.Size([1, 1180]) +Dataloader batch - attn_mask shape: torch.Size([1, 1180]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 284 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1180) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1180) torch.int64 + loss_mask : 283 / 1180 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1180, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1180, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1180, 1, 576) torch.bfloat16 + out : (1, 1180) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 283 tokens) : 11.2364 + top-1 accuracy : 6/283 = 2.1% + + [2026-04-29 12:29:57] iteration 91/ 100 | consumed samples: 364 | elapsed time per iteration (ms): 4201.9 | throughput (token/sec/GPU): 1243.2 | learning rate: 2.023256E-05 | global batch size: 4 | lm loss: 1.126359E+01 | loss scale: 1.0 | grad norm: 0.317 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 91 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 91 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 91 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1592.82, 1592.82) +Dataloader batch - tokens shape: torch.Size([1, 1751]) +Dataloader batch - labels shape: torch.Size([1, 1751]) +Dataloader batch - attn_mask shape: torch.Size([1, 1751]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 285 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1751) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1751) torch.int64 + loss_mask : 381 / 1751 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1751, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1751, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1751, 1, 576) torch.bfloat16 + out : (1, 1751) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.1835 + top-1 accuracy : 26/381 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 286 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 386 / 1543 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2655 + top-1 accuracy : 9/386 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1497]) +Dataloader batch - labels shape: torch.Size([1, 1497]) +Dataloader batch - attn_mask shape: torch.Size([1, 1497]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 287 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1497) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1497) torch.int64 + loss_mask : 329 / 1497 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1497, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1497, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1497, 1, 576) torch.bfloat16 + out : (1, 1497) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2171 + top-1 accuracy : 28/329 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1903]) +Dataloader batch - labels shape: torch.Size([1, 1903]) +Dataloader batch - attn_mask shape: torch.Size([1, 1903]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 288 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1903) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1903) torch.int64 + loss_mask : 551 / 1903 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1903, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1903, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1903, 1, 576) torch.bfloat16 + out : (1, 1903) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 551 tokens) : 11.3362 + top-1 accuracy : 6/551 = 1.1% + + [2026-04-29 12:30:05] iteration 92/ 100 | consumed samples: 368 | elapsed time per iteration (ms): 6277.3 | throughput (token/sec/GPU): 1066.4 | learning rate: 1.901480E-05 | global batch size: 4 | lm loss: 1.126055E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 92 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 92 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 92 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1805.54, 1805.54) +Dataloader batch - tokens shape: torch.Size([1, 1703]) +Dataloader batch - labels shape: torch.Size([1, 1703]) +Dataloader batch - attn_mask shape: torch.Size([1, 1703]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 289 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1703) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1703) torch.int64 + loss_mask : 388 / 1703 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1703, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1703, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1703, 1, 576) torch.bfloat16 + out : (1, 1703) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2207 + top-1 accuracy : 27/388 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1491]) +Dataloader batch - labels shape: torch.Size([1, 1491]) +Dataloader batch - attn_mask shape: torch.Size([1, 1491]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 290 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1491) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1491) torch.int64 + loss_mask : 379 / 1491 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1491, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1491, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1491, 1, 576) torch.bfloat16 + out : (1, 1491) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.3128 + top-1 accuracy : 14/379 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 291 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 349 / 1431 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.3017 + top-1 accuracy : 22/349 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 292 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 314 / 1380 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.2128 + top-1 accuracy : 19/314 = 6.1% + + [2026-04-29 12:30:14] iteration 93/ 100 | consumed samples: 372 | elapsed time per iteration (ms): 7620.7 | throughput (token/sec/GPU): 788.0 | learning rate: 1.782823E-05 | global batch size: 4 | lm loss: 1.126315E+01 | loss scale: 1.0 | grad norm: 0.291 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 93 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 93 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 93 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1977.47, 1977.47) +Dataloader batch - tokens shape: torch.Size([1, 1549]) +Dataloader batch - labels shape: torch.Size([1, 1549]) +Dataloader batch - attn_mask shape: torch.Size([1, 1549]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 293 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1549) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1549) torch.int64 + loss_mask : 349 / 1549 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1549, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1549, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1549, 1, 576) torch.bfloat16 + out : (1, 1549) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2151 + top-1 accuracy : 17/349 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1531]) +Dataloader batch - labels shape: torch.Size([1, 1531]) +Dataloader batch - attn_mask shape: torch.Size([1, 1531]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 294 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1531) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1531) torch.int64 + loss_mask : 339 / 1531 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1531, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1531, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1531, 1, 576) torch.bfloat16 + out : (1, 1531) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2132 + top-1 accuracy : 20/339 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 295 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1519) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1519) torch.int64 + loss_mask : 378 / 1519 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1519, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1519, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1519, 1, 576) torch.bfloat16 + out : (1, 1519) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 378 tokens) : 11.2235 + top-1 accuracy : 21/378 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 296 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 360 / 1422 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.3049 + top-1 accuracy : 34/360 = 9.4% + + [2026-04-29 12:30:23] iteration 94/ 100 | consumed samples: 376 | elapsed time per iteration (ms): 7030.0 | throughput (token/sec/GPU): 856.5 | learning rate: 1.667403E-05 | global batch size: 4 | lm loss: 1.123955E+01 | loss scale: 1.0 | grad norm: 0.345 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 94 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 94 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 94 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1878.47, 1878.47) +Dataloader batch - tokens shape: torch.Size([1, 1417]) +Dataloader batch - labels shape: torch.Size([1, 1417]) +Dataloader batch - attn_mask shape: torch.Size([1, 1417]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 297 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1417) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1417) torch.int64 + loss_mask : 365 / 1417 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1417, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1417, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1417, 1, 576) torch.bfloat16 + out : (1, 1417) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2818 + top-1 accuracy : 19/365 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 298 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 409 / 1543 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.3387 + top-1 accuracy : 15/409 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1581]) +Dataloader batch - labels shape: torch.Size([1, 1581]) +Dataloader batch - attn_mask shape: torch.Size([1, 1581]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 299 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1581) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1581) torch.int64 + loss_mask : 433 / 1581 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1581, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1581, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1581, 1, 576) torch.bfloat16 + out : (1, 1581) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 433 tokens) : 11.3410 + top-1 accuracy : 29/433 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1554]) +Dataloader batch - labels shape: torch.Size([1, 1554]) +Dataloader batch - attn_mask shape: torch.Size([1, 1554]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 300 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1554) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1554) torch.int64 + loss_mask : 369 / 1554 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1554, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1554, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1554, 1, 576) torch.bfloat16 + out : (1, 1554) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2505 + top-1 accuracy : 24/369 = 6.5% + + [2026-04-29 12:30:31] iteration 95/ 100 | consumed samples: 380 | elapsed time per iteration (ms): 6064.4 | throughput (token/sec/GPU): 1005.0 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.130552E+01 | loss scale: 1.0 | grad norm: 0.291 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 95 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 95 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 95 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1836.51, 1836.51) +Dataloader batch - tokens shape: torch.Size([1, 1723]) +Dataloader batch - labels shape: torch.Size([1, 1723]) +Dataloader batch - attn_mask shape: torch.Size([1, 1723]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 301 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1723) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1723) torch.int64 + loss_mask : 367 / 1723 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1723, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1723, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1723, 1, 576) torch.bfloat16 + out : (1, 1723) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2070 + top-1 accuracy : 35/367 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1170]) +Dataloader batch - labels shape: torch.Size([1, 1170]) +Dataloader batch - attn_mask shape: torch.Size([1, 1170]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 302 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1170) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1170) torch.int64 + loss_mask : 313 / 1170 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1170, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1170, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1170, 1, 576) torch.bfloat16 + out : (1, 1170) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.3220 + top-1 accuracy : 20/313 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1556]) +Dataloader batch - labels shape: torch.Size([1, 1556]) +Dataloader batch - attn_mask shape: torch.Size([1, 1556]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 303 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1556) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1556) torch.int64 + loss_mask : 420 / 1556 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1556, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1556, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1556, 1, 576) torch.bfloat16 + out : (1, 1556) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 420 tokens) : 11.3500 + top-1 accuracy : 24/420 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1138]) +Dataloader batch - labels shape: torch.Size([1, 1138]) +Dataloader batch - attn_mask shape: torch.Size([1, 1138]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 304 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1138) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1138) torch.int64 + loss_mask : 290 / 1138 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1138, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1138, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1138, 1, 576) torch.bfloat16 + out : (1, 1138) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.3280 + top-1 accuracy : 11/290 = 3.8% + + [2026-04-29 12:30:39] iteration 96/ 100 | consumed samples: 384 | elapsed time per iteration (ms): 5940.6 | throughput (token/sec/GPU): 940.5 | learning rate: 1.446729E-05 | global batch size: 4 | lm loss: 1.130133E+01 | loss scale: 1.0 | grad norm: 0.298 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 96 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 96 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 96 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1852.10, 1852.10) +Dataloader batch - tokens shape: torch.Size([1, 1065]) +Dataloader batch - labels shape: torch.Size([1, 1065]) +Dataloader batch - attn_mask shape: torch.Size([1, 1065]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 305 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1065) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1065) torch.int64 + loss_mask : 291 / 1065 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1065, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1065, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1065, 1, 576) torch.bfloat16 + out : (1, 1065) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 291 tokens) : 11.3368 + top-1 accuracy : 12/291 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 306 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 385 / 1434 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.3011 + top-1 accuracy : 25/385 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 307 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 380 / 1507 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 380 tokens) : 11.3257 + top-1 accuracy : 11/380 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1085]) +Dataloader batch - labels shape: torch.Size([1, 1085]) +Dataloader batch - attn_mask shape: torch.Size([1, 1085]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 308 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1085) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1085) torch.int64 + loss_mask : 252 / 1085 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1085, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1085, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1085, 1, 576) torch.bfloat16 + out : (1, 1085) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 252 tokens) : 11.2164 + top-1 accuracy : 21/252 = 8.3% + + [2026-04-29 12:30:46] iteration 97/ 100 | consumed samples: 388 | elapsed time per iteration (ms): 5394.0 | throughput (token/sec/GPU): 943.8 | learning rate: 1.341694E-05 | global batch size: 4 | lm loss: 1.129986E+01 | loss scale: 1.0 | grad norm: 0.318 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 97 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 97 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 97 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2111.02, 2111.02) +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 309 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 327 / 1518 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2524 + top-1 accuracy : 30/327 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1833]) +Dataloader batch - labels shape: torch.Size([1, 1833]) +Dataloader batch - attn_mask shape: torch.Size([1, 1833]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 310 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1833) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1833) torch.int64 + loss_mask : 461 / 1833 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1833, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1833, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1833, 1, 576) torch.bfloat16 + out : (1, 1833) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 461 tokens) : 11.3209 + top-1 accuracy : 15/461 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1270]) +Dataloader batch - labels shape: torch.Size([1, 1270]) +Dataloader batch - attn_mask shape: torch.Size([1, 1270]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 311 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1270) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1270) torch.int64 + loss_mask : 373 / 1270 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1270, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1270, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1270, 1, 576) torch.bfloat16 + out : (1, 1270) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.3676 + top-1 accuracy : 0/373 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 987]) +Dataloader batch - labels shape: torch.Size([1, 987]) +Dataloader batch - attn_mask shape: torch.Size([1, 987]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 312 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 987) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 987) torch.int64 + loss_mask : 220 / 987 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (987, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (987, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (987, 1, 576) torch.bfloat16 + out : (1, 987) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 220 tokens) : 11.2477 + top-1 accuracy : 10/220 = 4.5% + + [2026-04-29 12:30:54] iteration 98/ 100 | consumed samples: 392 | elapsed time per iteration (ms): 6174.8 | throughput (token/sec/GPU): 908.2 | learning rate: 1.240333E-05 | global batch size: 4 | lm loss: 1.130563E+01 | loss scale: 1.0 | grad norm: 0.359 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 98 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 98 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 98 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1869.00, 1869.00) +Dataloader batch - tokens shape: torch.Size([1, 1144]) +Dataloader batch - labels shape: torch.Size([1, 1144]) +Dataloader batch - attn_mask shape: torch.Size([1, 1144]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 313 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1144) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1144) torch.int64 + loss_mask : 339 / 1144 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1144, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1144, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1144, 1, 576) torch.bfloat16 + out : (1, 1144) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.3535 + top-1 accuracy : 10/339 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1692]) +Dataloader batch - labels shape: torch.Size([1, 1692]) +Dataloader batch - attn_mask shape: torch.Size([1, 1692]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 314 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1692) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1692) torch.int64 + loss_mask : 349 / 1692 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1692, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1692, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1692, 1, 576) torch.bfloat16 + out : (1, 1692) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.1822 + top-1 accuracy : 22/349 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1070]) +Dataloader batch - labels shape: torch.Size([1, 1070]) +Dataloader batch - attn_mask shape: torch.Size([1, 1070]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 315 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1070) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1070) torch.int64 + loss_mask : 239 / 1070 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1070, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1070, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1070, 1, 576) torch.bfloat16 + out : (1, 1070) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 239 tokens) : 11.2533 + top-1 accuracy : 16/239 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1561]) +Dataloader batch - labels shape: torch.Size([1, 1561]) +Dataloader batch - attn_mask shape: torch.Size([1, 1561]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 316 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1561) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1561) torch.int64 + loss_mask : 337 / 1561 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1561, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1561, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1561, 1, 576) torch.bfloat16 + out : (1, 1561) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.1878 + top-1 accuracy : 21/337 = 6.2% + + [2026-04-29 12:31:03] iteration 99/ 100 | consumed samples: 396 | elapsed time per iteration (ms): 6639.8 | throughput (token/sec/GPU): 823.4 | learning rate: 1.142747E-05 | global batch size: 4 | lm loss: 1.124306E+01 | loss scale: 1.0 | grad norm: 0.338 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 99 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 99 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 99 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1897.21, 1897.21) +Dataloader batch - tokens shape: torch.Size([1, 1373]) +Dataloader batch - labels shape: torch.Size([1, 1373]) +Dataloader batch - attn_mask shape: torch.Size([1, 1373]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 317 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1373) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1373) torch.int64 + loss_mask : 415 / 1373 tokens (30.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1373, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1373, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1373, 1, 576) torch.bfloat16 + out : (1, 1373) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.4149 + top-1 accuracy : 17/415 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1755]) +Dataloader batch - labels shape: torch.Size([1, 1755]) +Dataloader batch - attn_mask shape: torch.Size([1, 1755]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 318 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1755) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1755) torch.int64 + loss_mask : 415 / 1755 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1755, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1755, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1755, 1, 576) torch.bfloat16 + out : (1, 1755) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.2686 + top-1 accuracy : 28/415 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1401]) +Dataloader batch - labels shape: torch.Size([1, 1401]) +Dataloader batch - attn_mask shape: torch.Size([1, 1401]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 319 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1401) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1401) torch.int64 + loss_mask : 321 / 1401 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1401, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1401, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1401, 1, 576) torch.bfloat16 + out : (1, 1401) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2353 + top-1 accuracy : 34/321 = 10.6% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 320 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 374 / 1431 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.3083 + top-1 accuracy : 3/374 = 0.8% + + [2026-04-29 12:31:10] iteration 100/ 100 | consumed samples: 400 | elapsed time per iteration (ms): 5435.8 | throughput (token/sec/GPU): 1096.4 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.131114E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (5415.53, 5415.53) + forward-compute ................................: (3924.16, 3924.16) + backward-compute ...............................: (1489.01, 1489.01) + batch-generator ................................: (468.12, 468.12) + layernorm-grads-all-reduce .....................: (0.03, 0.03) + embedding-grads-all-reduce .....................: (0.05, 0.05) + all-grads-sync .................................: (0.50, 0.50) + params-all-gather ..............................: (0.15, 0.15) + optimizer-copy-to-main-grad ....................: (0.15, 0.15) + optimizer-clip-main-grad .......................: (0.55, 0.55) + optimizer-count-zeros ..........................: (0.02, 0.02) + optimizer-inner-step ...........................: (1.32, 1.32) + optimizer-copy-main-to-model-params ............: (0.24, 0.24) + optimizer ......................................: (3.16, 3.16) +saving checkpoint at iteration 100 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 100 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 100 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1161.60, 1161.60) +[after training is done] datetime: 2026-04-29 12:31:11 +[rank0]:[W429 12:32:33.219356395 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log new file mode 100644 index 00000000..b86cc391 --- /dev/null +++ b/stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log @@ -0,0 +1,62359 @@ +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +False +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. + warnings.warn( +parse arguments function called +-------------- Configure model to llava-ov-mobilellm-140m -------------- +[DEBUG] Model config type: LlavaOnevision1_5Config +[DEBUG] Is MobileLLM: True + num_layers = 15 + hidden_size = 576 + ffn_hidden_size = 2048 + num_attention_heads = 9 + group_query_attention = True + num_query_groups = 3 + position_embedding_type = rope + add_position_embedding = False + rotary_interleaved = False + normalization = RMSNorm + swiglu = True + attention_dropout = 0 + hidden_dropout = 0 + add_bias_linear = False + add_qkv_bias = False + qk_layernorm = True + untie_embeddings_and_output_weights = False + vocab_size_in_config_file = 128256 + make_vocab_size_divisible_by = 128 + norm_epsilon = 1e-05 + rotary_base = 8000000 + kv_channels = None + num_experts = None + moe_ffn_hidden_size = None +---------------- End of configuration ---------------- +[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 +INFO: Set dataloader type to external since --training-phase=SFT +WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) +using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 + warnings.warn( +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.99 + adam_eps ........................................ 1e-05 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + add_question_in_pretrain ........................ False + additional_special_tokens ....................... None + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.local + attention_dropout ............................... 0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + caption_channels ................................ None + chat_template ................................... llama3 + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch + ckpt_fully_parallel_load ........................ True + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + combined_1f1b ................................... False + combined_1f1b_recipe ............................ ep_a2a + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + context_parallel_ulysses_degree ................. 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_loss_fusion ....................... False + cuda_graph_warmup_steps ......................... 3 + custom_pipeline_layers .......................... None + custom_pipeline_recompute_layers ................ None + d2d_max_data_volume ............................. 0 + d2d_optimizer ................................... disabled + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 1 + data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader + dataloader_type ................................. external + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + dense_mlp_activation_func_recompute ............. False + deprecated_use_mcore_models ..................... False + detail_log_interval ............................. 20 + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + ema_decay ....................................... 0.9999 + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_discard_sample ........................... False + enable_ema ...................................... False + enable_fa_within_mla ............................ False + enable_fp8_comm ................................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_mem_monitor .............................. False + enable_one_logger ............................... False + enable_turn_off_bucketing ....................... True + encoder_num_layers .............................. 15 + encoder_pipeline_model_parallel_size ............ 0 + encoder_seq_length .............................. 32768 + encoder_tensor_model_parallel_size .............. 0 + end_weight_decay ................................ 0.0 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000 + eval_iters ...................................... 100 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 1 + expert_tensor_parallel_size ..................... 1 + fastvit_image_size .............................. 1024 + ffn_hidden_size ................................. 2048 + finetune ........................................ False + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. None + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... delayed + fp8_wgrad ....................................... True + fps ............................................. 2.0 + fps_max_frames .................................. 768 + fps_min_frames .................................. 4 + frame_max_pixels ................................ 602112 + frame_min_pixels ................................ 100352 + global_batch_size ............................... 4 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... False + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M + hidden_dropout .................................. 0 + hidden_size ..................................... 576 + hierarchical_context_parallel_sizes ............. None + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + image_aspect_ratio .............................. pad + image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] + image_resolution ................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.02 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 65536.0 + is_tokenized_data ............................... False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kv_channels ..................................... 64 + kv_lora_rank .................................... 32 + language_model_type ............................. None + latent_frame_interval ........................... 1 + latent_in_channels .............................. None + latent_out_channels ............................. None + latent_patch_size ............................... (1, 1, 1) + latent_space_scale .............................. 1.0 + latent_time_scale ............................... 1.0 + layernorm_recompute ............................. False + lazy_mpu_init ................................... None + length_sort_desc ................................ False + length_sort_pool_size ........................... 0 + load ............................................ stage_1_alignment_mobilellm_140m + load_ema ........................................ None + local_rank ...................................... 0 + log_detail ...................................... True + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. False + log_timers_to_tensorboard ....................... True + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 500 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. 0.002 + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 0 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... True + max_latent_height ............................... None + max_latent_width ................................ None + max_pixels ...................................... 12845056 + max_position_embeddings ......................... 32768 + max_text_length ................................. None + max_tokens_to_oom ............................... 12000 + mem_monitor_force_print_token_threshold ......... 10000000 + mem_monitor_log ................................. None + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... None + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-06 + min_pixels ...................................... 3136 + mla_recompute ................................... False + mmap_bin_files .................................. True + mock_data ....................................... False + model_family .................................... llava_ov_1_5 + model_name ...................................... llava-ov-mobilellm-140m + moe_aux_loss_coeff .............................. 0.0 + moe_enable_deepep ............................... False + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. None + moe_grouped_gemm ................................ False + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_mlp_activation_func_recompute ............... False + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 2 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... allgather + moe_token_drop_policy ........................... probs + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mscale .......................................... 1.0 + mscale_all_dim .................................. 1.0 + mtp_loss_coef ................................... 0.1 + multi_latent_attention .......................... False + nccl_communicator_config_path ................... None + no_load_optim ................................... True + no_load_rng ..................................... True + no_persist_layer_norm ........................... False + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-05 + normalization ................................... RMSNorm + num_attention_heads ............................. 9 + num_bucket_build_workers ........................ 1 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... None + num_latent_frames ............................... None + num_layers ...................................... 15 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_nextn_predict_layers ........................ 0 + num_query_groups ................................ 3 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 16 + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_d2d_optimizer ........................... True + overlap_grad_reduce ............................. False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ False + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + packing_batch_size .............................. None + packing_pretrain_data ........................... False + packing_sft_data ................................ False + padded_vocab_size ............................... None + padding_side .................................... right + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_size .................... 1 + pipeline_model_parallel_split_rank .............. None + position_embedding_type ......................... rope + pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + print_mem_monitor_interval ...................... 100000 + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... full + recompute_method ................................ uniform + recompute_num_layers ............................ 3 + record_memory_history ........................... False + reduced_data_volume_from_pre_stage .............. 0 + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... disabled + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + rope_in_fp32 .................................... True + rope_scaling_factor ............................. 8.0 + rotary_base ..................................... 8000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + sample_rate ..................................... 1.0 + save ............................................ stage_1_alignment_mobilellm_140m + save_ema ........................................ None + save_interval ................................... 50 + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + separate_layernorm_and_collinear ................ False + seq_length ...................................... 32768 + sequence_parallel ............................... False + sft_data_mix_strategy ........................... concat + sft_data_streaming .............................. False + sft_dataset ..................................... None + sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json + sft_num_preprocess_workers ...................... None + sft_sort_batch .................................. False + sft_test_dataset ................................ None + sft_train_dataset ............................... None + sft_valid_dataset ............................... None + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + split_bw ........................................ False + split_special_tokens ............................ False + squared_relu .................................... False + start_weight_decay .............................. 0.0 + stdit_bucket_config ............................. None + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + streaming_buffer_size ........................... 16384 + suggested_communication_unit_size ............... 400000000 + swiglu .......................................... True + swin_backbone_type .............................. tiny + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. HFTokenizer + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 500 + train_on_prompt ................................. False + train_samples ................................... None + train_sync_interval ............................. None + trainable_modules ............................... ['adapter'] + training_phase .................................. sft + training_rice_vl_max_answer_length .............. 32768 + training_rice_vl_max_image_area ................. 1806336 + transformer_impl ................................ local + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. False + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... False + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_fast_tokenizer .............................. False + use_fastvit ..................................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_normhead .................................... False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ True + video_max_pixels ................................ 51380224 + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vision_tower_name ............................... mobileclip_l_1024 + vocab_extra_ids ................................. 0 + vocab_file ...................................... None + vocab_size ...................................... None + vocab_size_in_config_file ....................... 128256 + vpp_scheduler ................................... None + wandb_exp_name .................................. mobilellm_integration + wandb_project ................................... llava-ov-1_5 + wandb_save_dir .................................. + weight_decay .................................... 0.0 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 1 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +Building model trainer... +Model family: llava_ov_1_5 +Training phase: sft +Using unified model type for training. + +Starting training... +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 4 +> setting tensorboard ... +wandb: Currently logged in as: rana-zayed (rana-zayed-mbzuai) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin +wandb: creating run +wandb: Tracking run with wandb version 0.21.0 +wandb: Run data is saved locally in stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_133502-3bn4iqdv +wandb: Run `wandb offline` to turn off syncing. +wandb: Syncing run mobilellm_integration +wandb: ⭐️ View project at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 +wandb: 🚀 View run at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/3bn4iqdv +> AIAK building HFTokenizer tokenizer ... +INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 +WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. +WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. +WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). + > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +> compiling dataset index builder ... +make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so +make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.100 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[rank0]:[W429 13:35:09.652713489 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() +>>> done with compiling and loading fused kernels. Compilation time: 0.887 seconds +time to initialize megatron (seconds): 24.289 +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once +[after megatron is initialized] datetime: 2026-04-29 13:35:23 +building llava-ov-mobilellm-140m model ... +[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m +[DEBUG PROVIDER] Args num_layers: 15 +[DEBUG PROVIDER] Args hidden_size: 576 +[DEBUG PROVIDER] Args vocab_size: 128260 +[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 +[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 +[DEBUG PROVIDER] Model family: llava_ov_1_5 +[DEBUG PROVIDER] use_mobilellm flag: True +[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone +[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 +[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Loading vision config... +[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 +[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json +[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd +[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ================================================ +[DEBUG PROVIDER] Loading adapter config... +[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== +TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) +[DEBUG PROVIDER] ====================================== +[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 +[DEBUG PROVIDER] Building layer specs... +[DEBUG PROVIDER] ✓ Using MobileLLM layer specification +[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 +[DEBUG PROVIDER] MobileLLM layer spec created successfully +[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... +[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 +[INFO] Using MobileLLM as language backbone +[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb +[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 +Params for bucket 1 (11216448 elements, 11216512 padded size): + module.adapter.layernorm.weight + module.adapter.linear_fc2.linear.bias + module.adapter.linear_fc1.linear.weight + module.adapter.linear_fc2.linear.weight + module.adapter.layernorm.bias + module.adapter.linear_fc1.linear.bias +[DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +)] +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) +AdamW ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 0.0 + weight_decay: 0.0 + +Parameter Group 1 + amsgrad: False + betas: (0.9, 0.99) + capturable: False + decoupled_weight_decay: True + differentiable: False + eps: 1e-05 + foreach: None + fused: None + is_decoupled_lr: False + is_expert_parallel: False + lr: 0.0001 + lr_mult: 1.0 + max_lr: 0.0001 + maximize: False + min_lr: 1e-06 + wd_mult: 1.0 + weight_decay: 0.0 +) +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[DEBUG] FastViT enabled: use_fastvit=True +[DEBUG] Before handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 +FastViT enabled: Will resume/load checkpoint from: stage_1_alignment_mobilellm_140m +[DEBUG] After handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 + loading checkpoint from stage_1_alignment_mobilellm_140m at iteration 100 +Loading checkpoint with device: cuda:0, strict=False + checkpoint version 3.0 +WARNING:megatron.core.rerun_state_machine:RerunStateMachine disabled via CLI, ignoring machine state saved in checkpoint + successfully loaded checkpoint from stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] at iteration 100 +(min, max) time across ranks (ms): + load-checkpoint ................................: (5203.56, 5203.56) +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 13:35:31 +================================================================================ +FULL MODEL STRUCTURE: +================================================================================ + +--- Model 0 (pipeline rank 0) --- +DistributedDataParallel( + (module): Float16Module( + (module): LlavaOnevision1_5( + (vision_model): FastViTModel( + (vision_tower): MobileCLIPVisionTower( + (vision_tower): MCi( + (model): FastViT( + (patch_embed): Sequential( + (0): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) + ) + (2): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + (network): ModuleList( + (0): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) + (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (1): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (2): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) + (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (3): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (4): Sequential( + (0): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (4): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (5): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (6): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (7): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (8): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (9): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (10): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (11): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (12): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (13): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (14): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (15): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (16): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (17): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (18): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (19): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (20): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (21): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (22): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (23): RepMixerBlock( + (token_mixer): RepMixer( + (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) + (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (5): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (6): RepCPE( + (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) + ) + (7): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (2): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (3): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=768, out_features=2304, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=768, out_features=768, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) + (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + (8): PatchEmbed( + (proj): Sequential( + (0): ReparamLargeKernelConv( + (activation): GELU(approximate='none') + (se): Identity() + (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) + ) + (1): MobileOneBlock( + (se): Identity() + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) + ) + ) + ) + (9): RepCPE( + (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) + ) + (10): Sequential( + (0): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + (1): AttentionBlock( + (norm): LayerNormChannel() + (token_mixer): MHSA( + (qkv): Linear(in_features=1536, out_features=4608, bias=False) + (attn_drop): Dropout(p=0.0, inplace=False) + (proj): Linear(in_features=1536, out_features=1536, bias=True) + (proj_drop): Dropout(p=0.0, inplace=False) + ) + (convffn): ConvFFN( + (conv): Sequential( + (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) + (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + ) + (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) + (act): GELU(approximate='none') + (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) + (drop): Dropout(p=0.0, inplace=False) + ) + (drop_path): Identity() + ) + ) + ) + (conv_exp): MobileOneBlock( + (se): SEBlock( + (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) + (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) + ) + (activation): GELU(approximate='none') + (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) + ) + (head): GlobalPool2D() + ) + ) + ) + ) + (adapter): Adapter( + (layernorm): FusedLayerNorm() + (linear_fc1): TELinear( + (linear): Linear(in_features=3072, out_features=3072, bias=True) + ) + (linear_fc2): TELinear( + (linear): Linear(in_features=3072, out_features=576, bias=True) + ) + ) + (language_model): MobileLLMModel( + (embedding): LanguageModelEmbedding( + (word_embeddings): VocabParallelEmbedding() + (embedding_dropout): Dropout(p=0, inplace=False) + ) + (rotary_pos_emb): RotaryEmbedding() + (decoder): TransformerBlock( + (layers): ModuleList( + (0-14): 15 x TransformerLayer( + (input_layernorm): IdentityOp() + (self_attention): SelfAttention( + (core_attention): DotProductAttention( + (attention_dropout): Dropout(p=0, inplace=False) + ) + (linear_proj): TERowParallelLinear( + (linear): Linear(in_features=576, out_features=576, bias=False) + ) + (linear_qkv): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=960, bias=False) + ) + (q_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + (k_layernorm): TENorm( + (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) + ) + ) + (pre_cross_attn_layernorm): IdentityOp() + (cross_attention): IdentityOp() + (cross_attn_bda): IdentityFuncOp() + (pre_mlp_layernorm): IdentityOp() + (mlp): MLP( + (linear_fc1): TELayerNormColumnParallelLinear( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + (linear): Linear(in_features=576, out_features=4096, bias=False) + ) + (activation_func): ActivationFuncModule() + (linear_fc2): TERowParallelLinear( + (linear): Linear(in_features=2048, out_features=576, bias=False) + ) + ) + ) + ) + (final_layernorm): TENorm( + (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) + ) + ) + (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) + ) + ) + ) +) +================================================================================ + +================================================================================ +PARAMETER TRAINABILITY STATUS: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +================================================================================ +Total trainable parameters: 11,216,448 (11.22M) +Total frozen parameters: 265,417,440 (265.42M) +Total parameters: 276,633,888 (276.63M) +Trainable percentage: 4.05% +================================================================================ +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 2000 + validation: 400 + test: 400 +Using shared training tokenizer in FastVLM mode +Ensured multimodal special tokens for FastVLM tokenizer; added=0 +Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M +Initialized FastViT processor with image_size=1024 +Processor config: .TokenizerWrapper object at 0x7f2e946e7e20> +image_resolution: None +rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 1500), ] sum(count)=1500 +rank=0, worker=1: shard_range=[pretrain-000003.tar[1500, 2999), ] sum(count)=1499 +rank=0, worker=2: shard_range=[pretrain-000003.tar[2999, 4498), ] sum(count)=1499 +rank=0, worker=3: shard_range=[pretrain-000003.tar[4498, 5058), pretrain-000001.tar[0, 939), ] sum(count)=1499 +rank=0, worker=4: shard_range=[pretrain-000001.tar[939, 2439), ] sum(count)=1500 +rank=0, worker=5: shard_range=[pretrain-000001.tar[2439, 3938), ] sum(count)=1499 +rank=0, worker=6: shard_range=[pretrain-000001.tar[3938, 5036), pretrain-000002.tar[0, 401), ] sum(count)=1499 +rank=0, worker=7: shard_range=[pretrain-000002.tar[401, 1900), ] sum(count)=1499 +rank=0, worker=8: shard_range=[pretrain-000002.tar[1900, 3400), ] sum(count)=1500 +rank=0, worker=9: shard_range=[pretrain-000002.tar[3400, 4899), ] sum(count)=1499 +rank=0, worker=10: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 1359), ] sum(count)=1499 +rank=0, worker=11: shard_range=[pretrain-000004.tar[1359, 2858), ] sum(count)=1499 +rank=0, worker=12: shard_range=[pretrain-000004.tar[2858, 3821), pretrain-000000.tar[0, 537), ] sum(count)=1500 +rank=0, worker=13: shard_range=[pretrain-000000.tar[537, 2036), ] sum(count)=1499 +rank=0, worker=14: shard_range=[pretrain-000000.tar[2036, 3535), ] sum(count)=1499 +rank=0, worker=15: shard_range=[pretrain-000000.tar[3535, 5034), ] sum(count)=1499 +restored dataset state from stage_1_alignment_mobilellm_140m/dataloader/iter_0000100/mp_rank_00/train_dataloader_dprank000.pt +[after dataloaders are built] datetime: 2026-04-29 13:35:31 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (7978.23, 7978.23) + train/valid/test-data-iterators-setup ..........: (86.80, 86.80) +training ... + +================================================================================ +training with the following parameter status: +================================================================================ +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 +FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 +TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 +FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 +Setting rerun_state_machine.current_iteration to 100... +[before the start of training step] datetime: 2026-04-29 13:35:31 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 + +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 +You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. +Dataloader batch - tokens shape: torch.Size([1, 985]) +Dataloader batch - labels shape: torch.Size([1, 985]) +Dataloader batch - attn_mask shape: torch.Size([1, 985]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 985) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 985) torch.int64 + loss_mask : 213 / 985 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (985, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (985, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (985, 1, 576) torch.bfloat16 + out : (1, 985) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 213 tokens) : 11.2059 + top-1 accuracy : 16/213 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 404 / 1459 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.3175 + top-1 accuracy : 12/404 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 337 / 1400 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.2449 + top-1 accuracy : 5/337 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1872]) +Dataloader batch - labels shape: torch.Size([1, 1872]) +Dataloader batch - attn_mask shape: torch.Size([1, 1872]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1872) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1872) torch.int64 + loss_mask : 514 / 1872 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1872, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1872, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1872, 1, 576) torch.bfloat16 + out : (1, 1872) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 514 tokens) : 11.3168 + top-1 accuracy : 37/514 = 7.2% + +/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. + warnings.warn( # warn only once + [2026-04-29 13:36:34] iteration 101/ 500 | consumed samples: 404 | elapsed time per iteration (ms): 62508.2 | throughput (token/sec/GPU): 91.4 | learning rate: 1.000000E-04 | global batch size: 4 | lm loss: 1.128442E+01 | loss scale: 1.0 | grad norm: 0.352 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer layers in billions: 0.07 +Number of parameters in embedding layers in billions: 0.07 +Total number of parameters in billions: 0.14 +Number of parameters in most loaded shard in billions: 0.1403 +Theoretical memory footprints: weight and optimizer=2409.10 MB +[Rank 0] (after 101 iterations) memory (MB) | allocated: 793.81787109375 | max allocated: 5484.03466796875 | reserved: 9622.0 | max reserved: 9622.0 +Dataloader batch - tokens shape: torch.Size([1, 1449]) +Dataloader batch - labels shape: torch.Size([1, 1449]) +Dataloader batch - attn_mask shape: torch.Size([1, 1449]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1449) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1449) torch.int64 + loss_mask : 353 / 1449 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1449, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1449, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1449, 1, 576) torch.bfloat16 + out : (1, 1449) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.2924 + top-1 accuracy : 7/353 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1412) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1412) torch.int64 + loss_mask : 335 / 1412 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1412, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1412, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1412, 1, 576) torch.bfloat16 + out : (1, 1412) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.3050 + top-1 accuracy : 18/335 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1365]) +Dataloader batch - labels shape: torch.Size([1, 1365]) +Dataloader batch - attn_mask shape: torch.Size([1, 1365]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1365) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1365) torch.int64 + loss_mask : 315 / 1365 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1365, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1365, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1365, 1, 576) torch.bfloat16 + out : (1, 1365) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.2518 + top-1 accuracy : 19/315 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1582]) +Dataloader batch - labels shape: torch.Size([1, 1582]) +Dataloader batch - attn_mask shape: torch.Size([1, 1582]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1582) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1582) torch.int64 + loss_mask : 439 / 1582 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1582, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1582, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1582, 1, 576) torch.bfloat16 + out : (1, 1582) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 439 tokens) : 11.3241 + top-1 accuracy : 18/439 = 4.1% + + [2026-04-29 13:36:40] iteration 102/ 500 | consumed samples: 408 | elapsed time per iteration (ms): 6273.4 | throughput (token/sec/GPU): 925.8 | learning rate: 9.999902E-05 | global batch size: 4 | lm loss: 1.129612E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1895]) +Dataloader batch - labels shape: torch.Size([1, 1895]) +Dataloader batch - attn_mask shape: torch.Size([1, 1895]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1895) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1895) torch.int64 + loss_mask : 531 / 1895 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1895, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1895, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1895, 1, 576) torch.bfloat16 + out : (1, 1895) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 531 tokens) : 11.3597 + top-1 accuracy : 23/531 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1489]) +Dataloader batch - labels shape: torch.Size([1, 1489]) +Dataloader batch - attn_mask shape: torch.Size([1, 1489]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1489) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1489) torch.int64 + loss_mask : 313 / 1489 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1489, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1489, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1489, 1, 576) torch.bfloat16 + out : (1, 1489) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2138 + top-1 accuracy : 30/313 = 9.6% + +Dataloader batch - tokens shape: torch.Size([1, 1752]) +Dataloader batch - labels shape: torch.Size([1, 1752]) +Dataloader batch - attn_mask shape: torch.Size([1, 1752]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1752) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1752) torch.int64 + loss_mask : 394 / 1752 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1752, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1752, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1752, 1, 576) torch.bfloat16 + out : (1, 1752) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.2502 + top-1 accuracy : 26/394 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1495]) +Dataloader batch - labels shape: torch.Size([1, 1495]) +Dataloader batch - attn_mask shape: torch.Size([1, 1495]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1495) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1495) torch.int64 + loss_mask : 354 / 1495 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1495, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1495, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1495, 1, 576) torch.bfloat16 + out : (1, 1495) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2848 + top-1 accuracy : 20/354 = 5.6% + + [2026-04-29 13:36:47] iteration 103/ 500 | consumed samples: 412 | elapsed time per iteration (ms): 7095.3 | throughput (token/sec/GPU): 934.6 | learning rate: 9.999608E-05 | global batch size: 4 | lm loss: 1.128726E+01 | loss scale: 1.0 | grad norm: 0.549 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 999]) +Dataloader batch - labels shape: torch.Size([1, 999]) +Dataloader batch - attn_mask shape: torch.Size([1, 999]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 999) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 999) torch.int64 + loss_mask : 238 / 999 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (999, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (999, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (999, 1, 576) torch.bfloat16 + out : (1, 999) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 238 tokens) : 11.2851 + top-1 accuracy : 16/238 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 359 / 1444 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.2496 + top-1 accuracy : 22/359 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1387]) +Dataloader batch - labels shape: torch.Size([1, 1387]) +Dataloader batch - attn_mask shape: torch.Size([1, 1387]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1387) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1387) torch.int64 + loss_mask : 330 / 1387 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1387, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1387, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1387, 1, 576) torch.bfloat16 + out : (1, 1387) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.2850 + top-1 accuracy : 26/330 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1686]) +Dataloader batch - labels shape: torch.Size([1, 1686]) +Dataloader batch - attn_mask shape: torch.Size([1, 1686]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1686) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1686) torch.int64 + loss_mask : 410 / 1686 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1686, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1686, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1686, 1, 576) torch.bfloat16 + out : (1, 1686) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.2497 + top-1 accuracy : 13/410 = 3.2% + + [2026-04-29 13:36:53] iteration 104/ 500 | consumed samples: 416 | elapsed time per iteration (ms): 6029.8 | throughput (token/sec/GPU): 914.8 | learning rate: 9.999117E-05 | global batch size: 4 | lm loss: 1.126469E+01 | loss scale: 1.0 | grad norm: 0.473 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1534]) +Dataloader batch - labels shape: torch.Size([1, 1534]) +Dataloader batch - attn_mask shape: torch.Size([1, 1534]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1534) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1534) torch.int64 + loss_mask : 399 / 1534 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1534, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1534, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1534, 1, 576) torch.bfloat16 + out : (1, 1534) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.3398 + top-1 accuracy : 23/399 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1421]) +Dataloader batch - labels shape: torch.Size([1, 1421]) +Dataloader batch - attn_mask shape: torch.Size([1, 1421]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1421) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1421) torch.int64 + loss_mask : 358 / 1421 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1421, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1421, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1421, 1, 576) torch.bfloat16 + out : (1, 1421) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.3165 + top-1 accuracy : 20/358 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 398 / 1459 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.3363 + top-1 accuracy : 2/398 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1227]) +Dataloader batch - labels shape: torch.Size([1, 1227]) +Dataloader batch - attn_mask shape: torch.Size([1, 1227]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1227) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1227) torch.int64 + loss_mask : 325 / 1227 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1227, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1227, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1227, 1, 576) torch.bfloat16 + out : (1, 1227) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.3268 + top-1 accuracy : 10/325 = 3.1% + + [2026-04-29 13:36:59] iteration 105/ 500 | consumed samples: 420 | elapsed time per iteration (ms): 5807.2 | throughput (token/sec/GPU): 971.4 | learning rate: 9.998430E-05 | global batch size: 4 | lm loss: 1.133039E+01 | loss scale: 1.0 | grad norm: 0.470 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 1527]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 21 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1527) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1527) torch.int64 + loss_mask : 327 / 1527 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1527, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1527, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1527, 1, 576) torch.bfloat16 + out : (1, 1527) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.1968 + top-1 accuracy : 15/327 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1174]) +Dataloader batch - labels shape: torch.Size([1, 1174]) +Dataloader batch - attn_mask shape: torch.Size([1, 1174]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 22 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1174) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1174) torch.int64 + loss_mask : 326 / 1174 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1174, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1174, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1174, 1, 576) torch.bfloat16 + out : (1, 1174) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.3892 + top-1 accuracy : 6/326 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1781]) +Dataloader batch - labels shape: torch.Size([1, 1781]) +Dataloader batch - attn_mask shape: torch.Size([1, 1781]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 23 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1781) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1781) torch.int64 + loss_mask : 406 / 1781 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1781, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1781, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1781, 1, 576) torch.bfloat16 + out : (1, 1781) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.2330 + top-1 accuracy : 21/406 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1107]) +Dataloader batch - labels shape: torch.Size([1, 1107]) +Dataloader batch - attn_mask shape: torch.Size([1, 1107]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 24 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1107) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1107) torch.int64 + loss_mask : 299 / 1107 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1107, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1107, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1107, 1, 576) torch.bfloat16 + out : (1, 1107) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 299 tokens) : 11.3542 + top-1 accuracy : 9/299 = 3.0% + + [2026-04-29 13:37:05] iteration 106/ 500 | consumed samples: 424 | elapsed time per iteration (ms): 5897.8 | throughput (token/sec/GPU): 947.6 | learning rate: 9.997548E-05 | global batch size: 4 | lm loss: 1.128846E+01 | loss scale: 1.0 | grad norm: 0.379 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 25 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 364 / 1518 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2845 + top-1 accuracy : 5/364 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1453]) +Dataloader batch - labels shape: torch.Size([1, 1453]) +Dataloader batch - attn_mask shape: torch.Size([1, 1453]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 26 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1453) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1453) torch.int64 + loss_mask : 404 / 1453 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1453, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1453, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1453, 1, 576) torch.bfloat16 + out : (1, 1453) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.3466 + top-1 accuracy : 2/404 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1029]) +Dataloader batch - labels shape: torch.Size([1, 1029]) +Dataloader batch - attn_mask shape: torch.Size([1, 1029]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 27 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1029) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1029) torch.int64 + loss_mask : 263 / 1029 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1029, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1029, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1029, 1, 576) torch.bfloat16 + out : (1, 1029) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.3044 + top-1 accuracy : 6/263 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1081]) +Dataloader batch - labels shape: torch.Size([1, 1081]) +Dataloader batch - attn_mask shape: torch.Size([1, 1081]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 28 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1081) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1081) torch.int64 + loss_mask : 275 / 1081 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1081, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1081, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1081, 1, 576) torch.bfloat16 + out : (1, 1081) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 275 tokens) : 11.3373 + top-1 accuracy : 17/275 = 6.2% + + [2026-04-29 13:37:10] iteration 107/ 500 | consumed samples: 428 | elapsed time per iteration (ms): 5778.0 | throughput (token/sec/GPU): 879.4 | learning rate: 9.996469E-05 | global batch size: 4 | lm loss: 1.131884E+01 | loss scale: 1.0 | grad norm: 0.361 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1270]) +Dataloader batch - labels shape: torch.Size([1, 1270]) +Dataloader batch - attn_mask shape: torch.Size([1, 1270]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 29 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1270) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1270) torch.int64 + loss_mask : 343 / 1270 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1270, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1270, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1270, 1, 576) torch.bfloat16 + out : (1, 1270) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.3434 + top-1 accuracy : 11/343 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1047]) +Dataloader batch - labels shape: torch.Size([1, 1047]) +Dataloader batch - attn_mask shape: torch.Size([1, 1047]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 30 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1047) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1047) torch.int64 + loss_mask : 219 / 1047 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1047, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1047, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1047, 1, 576) torch.bfloat16 + out : (1, 1047) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 219 tokens) : 11.1684 + top-1 accuracy : 6/219 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1656]) +Dataloader batch - labels shape: torch.Size([1, 1656]) +Dataloader batch - attn_mask shape: torch.Size([1, 1656]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 31 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1656) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1656) torch.int64 + loss_mask : 413 / 1656 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1656, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1656, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1656, 1, 576) torch.bfloat16 + out : (1, 1656) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2663 + top-1 accuracy : 2/413 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 992]) +Dataloader batch - labels shape: torch.Size([1, 992]) +Dataloader batch - attn_mask shape: torch.Size([1, 992]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 32 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 992) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 992) torch.int64 + loss_mask : 222 / 992 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (992, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (992, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (992, 1, 576) torch.bfloat16 + out : (1, 992) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 222 tokens) : 11.2836 + top-1 accuracy : 11/222 = 5.0% + + [2026-04-29 13:37:16] iteration 108/ 500 | consumed samples: 432 | elapsed time per iteration (ms): 5692.2 | throughput (token/sec/GPU): 872.2 | learning rate: 9.995194E-05 | global batch size: 4 | lm loss: 1.127368E+01 | loss scale: 1.0 | grad norm: 0.422 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 33 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 413 / 1468 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.4118 + top-1 accuracy : 6/413 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1361]) +Dataloader batch - labels shape: torch.Size([1, 1361]) +Dataloader batch - attn_mask shape: torch.Size([1, 1361]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 34 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1361) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1361) torch.int64 + loss_mask : 286 / 1361 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1361, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1361, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1361, 1, 576) torch.bfloat16 + out : (1, 1361) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 286 tokens) : 11.1924 + top-1 accuracy : 15/286 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1084]) +Dataloader batch - labels shape: torch.Size([1, 1084]) +Dataloader batch - attn_mask shape: torch.Size([1, 1084]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 35 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1084) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1084) torch.int64 + loss_mask : 256 / 1084 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1084, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1084, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1084, 1, 576) torch.bfloat16 + out : (1, 1084) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 256 tokens) : 11.3106 + top-1 accuracy : 5/256 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1000]) +Dataloader batch - labels shape: torch.Size([1, 1000]) +Dataloader batch - attn_mask shape: torch.Size([1, 1000]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 36 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1000) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1000) torch.int64 + loss_mask : 248 / 1000 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1000, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1000, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1000, 1, 576) torch.bfloat16 + out : (1, 1000) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 248 tokens) : 11.2977 + top-1 accuracy : 7/248 = 2.8% + + [2026-04-29 13:37:22] iteration 109/ 500 | consumed samples: 436 | elapsed time per iteration (ms): 5420.4 | throughput (token/sec/GPU): 906.4 | learning rate: 9.993723E-05 | global batch size: 4 | lm loss: 1.131460E+01 | loss scale: 1.0 | grad norm: 0.482 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 37 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1696) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1696) torch.int64 + loss_mask : 337 / 1696 tokens (19.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1696, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1696, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1696, 1, 576) torch.bfloat16 + out : (1, 1696) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.2151 + top-1 accuracy : 37/337 = 11.0% + +Dataloader batch - tokens shape: torch.Size([1, 1522]) +Dataloader batch - labels shape: torch.Size([1, 1522]) +Dataloader batch - attn_mask shape: torch.Size([1, 1522]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 38 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1522) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1522) torch.int64 + loss_mask : 357 / 1522 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1522, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1522, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1522, 1, 576) torch.bfloat16 + out : (1, 1522) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2466 + top-1 accuracy : 27/357 = 7.6% + +Dataloader batch - tokens shape: torch.Size([1, 1692]) +Dataloader batch - labels shape: torch.Size([1, 1692]) +Dataloader batch - attn_mask shape: torch.Size([1, 1692]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 39 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1692) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1692) torch.int64 + loss_mask : 347 / 1692 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1692, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1692, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1692, 1, 576) torch.bfloat16 + out : (1, 1692) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.1795 + top-1 accuracy : 29/347 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 1564]) +Dataloader batch - labels shape: torch.Size([1, 1564]) +Dataloader batch - attn_mask shape: torch.Size([1, 1564]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 40 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1564) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1564) torch.int64 + loss_mask : 362 / 1564 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1564, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1564, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1564, 1, 576) torch.bfloat16 + out : (1, 1564) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2334 + top-1 accuracy : 21/362 = 5.8% + + [2026-04-29 13:37:29] iteration 110/ 500 | consumed samples: 440 | elapsed time per iteration (ms): 7161.7 | throughput (token/sec/GPU): 904.0 | learning rate: 9.992056E-05 | global batch size: 4 | lm loss: 1.121903E+01 | loss scale: 1.0 | grad norm: 0.422 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1564]) +Dataloader batch - labels shape: torch.Size([1, 1564]) +Dataloader batch - attn_mask shape: torch.Size([1, 1564]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 41 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1564) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1564) torch.int64 + loss_mask : 387 / 1564 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1564, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1564, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1564, 1, 576) torch.bfloat16 + out : (1, 1564) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2561 + top-1 accuracy : 10/387 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1457]) +Dataloader batch - labels shape: torch.Size([1, 1457]) +Dataloader batch - attn_mask shape: torch.Size([1, 1457]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 42 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1457) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1457) torch.int64 + loss_mask : 312 / 1457 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1457, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1457, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1457, 1, 576) torch.bfloat16 + out : (1, 1457) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.1624 + top-1 accuracy : 8/312 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1531]) +Dataloader batch - labels shape: torch.Size([1, 1531]) +Dataloader batch - attn_mask shape: torch.Size([1, 1531]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 43 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1531) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1531) torch.int64 + loss_mask : 399 / 1531 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1531, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1531, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1531, 1, 576) torch.bfloat16 + out : (1, 1531) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.2714 + top-1 accuracy : 21/399 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 44 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 317 / 1459 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.1703 + top-1 accuracy : 10/317 = 3.2% + + [2026-04-29 13:37:35] iteration 111/ 500 | consumed samples: 444 | elapsed time per iteration (ms): 6593.5 | throughput (token/sec/GPU): 911.7 | learning rate: 9.990193E-05 | global batch size: 4 | lm loss: 1.122050E+01 | loss scale: 1.0 | grad norm: 0.382 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1881]) +Dataloader batch - labels shape: torch.Size([1, 1881]) +Dataloader batch - attn_mask shape: torch.Size([1, 1881]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 45 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1881) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1881) torch.int64 + loss_mask : 506 / 1881 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1881, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1881, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1881, 1, 576) torch.bfloat16 + out : (1, 1881) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 506 tokens) : 11.3654 + top-1 accuracy : 13/506 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1453]) +Dataloader batch - labels shape: torch.Size([1, 1453]) +Dataloader batch - attn_mask shape: torch.Size([1, 1453]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 46 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1453) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1453) torch.int64 + loss_mask : 398 / 1453 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1453, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1453, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1453, 1, 576) torch.bfloat16 + out : (1, 1453) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.3884 + top-1 accuracy : 16/398 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1159]) +Dataloader batch - labels shape: torch.Size([1, 1159]) +Dataloader batch - attn_mask shape: torch.Size([1, 1159]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 47 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1159) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1159) torch.int64 + loss_mask : 247 / 1159 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1159, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1159, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1159, 1, 576) torch.bfloat16 + out : (1, 1159) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 247 tokens) : 11.1677 + top-1 accuracy : 19/247 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1837]) +Dataloader batch - labels shape: torch.Size([1, 1837]) +Dataloader batch - attn_mask shape: torch.Size([1, 1837]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 48 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1837) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1837) torch.int64 + loss_mask : 455 / 1837 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1837, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1837, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1837, 1, 576) torch.bfloat16 + out : (1, 1837) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 455 tokens) : 11.3281 + top-1 accuracy : 19/455 = 4.2% + + [2026-04-29 13:37:42] iteration 112/ 500 | consumed samples: 448 | elapsed time per iteration (ms): 6614.8 | throughput (token/sec/GPU): 956.9 | learning rate: 9.988135E-05 | global batch size: 4 | lm loss: 1.133011E+01 | loss scale: 1.0 | grad norm: 0.365 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1570]) +Dataloader batch - labels shape: torch.Size([1, 1570]) +Dataloader batch - attn_mask shape: torch.Size([1, 1570]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 49 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1570) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1570) torch.int64 + loss_mask : 335 / 1570 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1570, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1570, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1570, 1, 576) torch.bfloat16 + out : (1, 1570) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2413 + top-1 accuracy : 15/335 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1269]) +Dataloader batch - labels shape: torch.Size([1, 1269]) +Dataloader batch - attn_mask shape: torch.Size([1, 1269]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 50 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1269) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1269) torch.int64 + loss_mask : 345 / 1269 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1269, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1269, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1269, 1, 576) torch.bfloat16 + out : (1, 1269) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.3770 + top-1 accuracy : 9/345 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1545]) +Dataloader batch - labels shape: torch.Size([1, 1545]) +Dataloader batch - attn_mask shape: torch.Size([1, 1545]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 51 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1545) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1545) torch.int64 + loss_mask : 335 / 1545 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1545, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1545, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1545, 1, 576) torch.bfloat16 + out : (1, 1545) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.1921 + top-1 accuracy : 30/335 = 9.0% + +Dataloader batch - tokens shape: torch.Size([1, 1766]) +Dataloader batch - labels shape: torch.Size([1, 1766]) +Dataloader batch - attn_mask shape: torch.Size([1, 1766]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 52 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1766) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1766) torch.int64 + loss_mask : 387 / 1766 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1766, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1766, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1766, 1, 576) torch.bfloat16 + out : (1, 1766) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2228 + top-1 accuracy : 18/387 = 4.7% + + [2026-04-29 13:37:49] iteration 113/ 500 | consumed samples: 452 | elapsed time per iteration (ms): 6828.3 | throughput (token/sec/GPU): 900.7 | learning rate: 9.985880E-05 | global batch size: 4 | lm loss: 1.125784E+01 | loss scale: 1.0 | grad norm: 0.337 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1489]) +Dataloader batch - labels shape: torch.Size([1, 1489]) +Dataloader batch - attn_mask shape: torch.Size([1, 1489]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 53 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1489) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1489) torch.int64 + loss_mask : 441 / 1489 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1489, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1489, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1489, 1, 576) torch.bfloat16 + out : (1, 1489) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 441 tokens) : 11.3808 + top-1 accuracy : 18/441 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1214]) +Dataloader batch - labels shape: torch.Size([1, 1214]) +Dataloader batch - attn_mask shape: torch.Size([1, 1214]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 54 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1214) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1214) torch.int64 + loss_mask : 283 / 1214 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1214, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1214, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1214, 1, 576) torch.bfloat16 + out : (1, 1214) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 283 tokens) : 11.2809 + top-1 accuracy : 20/283 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1059]) +Dataloader batch - labels shape: torch.Size([1, 1059]) +Dataloader batch - attn_mask shape: torch.Size([1, 1059]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 55 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1059) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1059) torch.int64 + loss_mask : 220 / 1059 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1059, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1059, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1059, 1, 576) torch.bfloat16 + out : (1, 1059) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 220 tokens) : 11.2013 + top-1 accuracy : 21/220 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1402]) +Dataloader batch - labels shape: torch.Size([1, 1402]) +Dataloader batch - attn_mask shape: torch.Size([1, 1402]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 56 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1402) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1402) torch.int64 + loss_mask : 355 / 1402 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1402, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1402, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1402, 1, 576) torch.bfloat16 + out : (1, 1402) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2965 + top-1 accuracy : 2/355 = 0.6% + + [2026-04-29 13:37:54] iteration 114/ 500 | consumed samples: 456 | elapsed time per iteration (ms): 5735.0 | throughput (token/sec/GPU): 900.4 | learning rate: 9.983430E-05 | global batch size: 4 | lm loss: 1.130560E+01 | loss scale: 1.0 | grad norm: 0.436 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1080]) +Dataloader batch - labels shape: torch.Size([1, 1080]) +Dataloader batch - attn_mask shape: torch.Size([1, 1080]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 57 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1080) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1080) torch.int64 + loss_mask : 333 / 1080 tokens (30.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1080, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1080, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1080, 1, 576) torch.bfloat16 + out : (1, 1080) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.3696 + top-1 accuracy : 4/333 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 58 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1418) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1418) torch.int64 + loss_mask : 350 / 1418 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1418, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1418, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1418, 1, 576) torch.bfloat16 + out : (1, 1418) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.2830 + top-1 accuracy : 3/350 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1122]) +Dataloader batch - labels shape: torch.Size([1, 1122]) +Dataloader batch - attn_mask shape: torch.Size([1, 1122]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 59 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1122) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1122) torch.int64 + loss_mask : 271 / 1122 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1122, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1122, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1122, 1, 576) torch.bfloat16 + out : (1, 1122) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 271 tokens) : 11.3065 + top-1 accuracy : 18/271 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1413]) +Dataloader batch - labels shape: torch.Size([1, 1413]) +Dataloader batch - attn_mask shape: torch.Size([1, 1413]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 60 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1413) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1413) torch.int64 + loss_mask : 261 / 1413 tokens (18.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1413, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1413, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1413, 1, 576) torch.bfloat16 + out : (1, 1413) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 261 tokens) : 11.0933 + top-1 accuracy : 24/261 = 9.2% + + [2026-04-29 13:38:00] iteration 115/ 500 | consumed samples: 460 | elapsed time per iteration (ms): 5585.5 | throughput (token/sec/GPU): 901.1 | learning rate: 9.980785E-05 | global batch size: 4 | lm loss: 1.127125E+01 | loss scale: 1.0 | grad norm: 0.342 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 61 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 381 / 1543 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.3198 + top-1 accuracy : 7/381 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1859]) +Dataloader batch - labels shape: torch.Size([1, 1859]) +Dataloader batch - attn_mask shape: torch.Size([1, 1859]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 62 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1859) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1859) torch.int64 + loss_mask : 490 / 1859 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1859, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1859, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1859, 1, 576) torch.bfloat16 + out : (1, 1859) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 490 tokens) : 11.3163 + top-1 accuracy : 20/490 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1174]) +Dataloader batch - labels shape: torch.Size([1, 1174]) +Dataloader batch - attn_mask shape: torch.Size([1, 1174]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 63 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1174) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1174) torch.int64 + loss_mask : 315 / 1174 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1174, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1174, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1174, 1, 576) torch.bfloat16 + out : (1, 1174) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.3298 + top-1 accuracy : 4/315 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1147]) +Dataloader batch - labels shape: torch.Size([1, 1147]) +Dataloader batch - attn_mask shape: torch.Size([1, 1147]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 64 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1147) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1147) torch.int64 + loss_mask : 305 / 1147 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1147, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1147, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1147, 1, 576) torch.bfloat16 + out : (1, 1147) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.2815 + top-1 accuracy : 21/305 = 6.9% + + [2026-04-29 13:38:06] iteration 116/ 500 | consumed samples: 464 | elapsed time per iteration (ms): 6082.3 | throughput (token/sec/GPU): 940.9 | learning rate: 9.977943E-05 | global batch size: 4 | lm loss: 1.131290E+01 | loss scale: 1.0 | grad norm: 0.389 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1556]) +Dataloader batch - labels shape: torch.Size([1, 1556]) +Dataloader batch - attn_mask shape: torch.Size([1, 1556]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 65 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1556) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1556) torch.int64 + loss_mask : 424 / 1556 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1556, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1556, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1556, 1, 576) torch.bfloat16 + out : (1, 1556) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 424 tokens) : 11.3758 + top-1 accuracy : 17/424 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1472]) +Dataloader batch - labels shape: torch.Size([1, 1472]) +Dataloader batch - attn_mask shape: torch.Size([1, 1472]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 66 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1472) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1472) torch.int64 + loss_mask : 317 / 1472 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1472, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1472, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1472, 1, 576) torch.bfloat16 + out : (1, 1472) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.1791 + top-1 accuracy : 33/317 = 10.4% + +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 67 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1423) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1423) torch.int64 + loss_mask : 311 / 1423 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1423, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1423, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1423, 1, 576) torch.bfloat16 + out : (1, 1423) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.1934 + top-1 accuracy : 24/311 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1082]) +Dataloader batch - labels shape: torch.Size([1, 1082]) +Dataloader batch - attn_mask shape: torch.Size([1, 1082]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 68 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1082) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1082) torch.int64 + loss_mask : 326 / 1082 tokens (30.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1082, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1082, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1082, 1, 576) torch.bfloat16 + out : (1, 1082) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.3697 + top-1 accuracy : 4/326 = 1.2% + + [2026-04-29 13:38:12] iteration 117/ 500 | consumed samples: 468 | elapsed time per iteration (ms): 5870.1 | throughput (token/sec/GPU): 942.6 | learning rate: 9.974907E-05 | global batch size: 4 | lm loss: 1.128796E+01 | loss scale: 1.0 | grad norm: 0.356 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1327]) +Dataloader batch - labels shape: torch.Size([1, 1327]) +Dataloader batch - attn_mask shape: torch.Size([1, 1327]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 69 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1327) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1327) torch.int64 + loss_mask : 379 / 1327 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1327, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1327, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1327, 1, 576) torch.bfloat16 + out : (1, 1327) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.4011 + top-1 accuracy : 5/379 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 70 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 402 / 1518 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.3245 + top-1 accuracy : 13/402 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 795]) +Dataloader batch - labels shape: torch.Size([1, 795]) +Dataloader batch - attn_mask shape: torch.Size([1, 795]) +Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) + +==================================================================== + PIPELINE — STEP 71 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 795) torch.int64 + images : (16, 3, 1024, 1024) torch.bfloat16 + labels : (1, 795) torch.int64 + loss_mask : 150 / 795 tokens (18.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (16, 3, 1024, 1024) torch.bfloat16 + out : (16, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (16, 3072) + out : (16, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (795, 1, 576) torch.bfloat16 grad=False + image slots : 16 tokens replaced with vision embeddings + combined : (795, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (795, 1, 576) torch.bfloat16 + out : (1, 795) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 150 tokens) : 11.1070 + top-1 accuracy : 3/150 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1325]) +Dataloader batch - labels shape: torch.Size([1, 1325]) +Dataloader batch - attn_mask shape: torch.Size([1, 1325]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 72 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1325) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1325) torch.int64 + loss_mask : 385 / 1325 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1325, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1325, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1325, 1, 576) torch.bfloat16 + out : (1, 1325) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.3468 + top-1 accuracy : 0/385 = 0.0% + + [2026-04-29 13:38:17] iteration 118/ 500 | consumed samples: 472 | elapsed time per iteration (ms): 5425.7 | throughput (token/sec/GPU): 915.1 | learning rate: 9.971676E-05 | global batch size: 4 | lm loss: 1.132831E+01 | loss scale: 1.0 | grad norm: 0.406 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1405]) +Dataloader batch - labels shape: torch.Size([1, 1405]) +Dataloader batch - attn_mask shape: torch.Size([1, 1405]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 73 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1405) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1405) torch.int64 + loss_mask : 304 / 1405 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1405, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1405, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1405, 1, 576) torch.bfloat16 + out : (1, 1405) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 304 tokens) : 11.2126 + top-1 accuracy : 29/304 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 908]) +Dataloader batch - labels shape: torch.Size([1, 908]) +Dataloader batch - attn_mask shape: torch.Size([1, 908]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 74 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 908) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 908) torch.int64 + loss_mask : 173 / 908 tokens (19.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (908, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (908, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (908, 1, 576) torch.bfloat16 + out : (1, 908) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 173 tokens) : 11.1921 + top-1 accuracy : 12/173 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1419]) +Dataloader batch - labels shape: torch.Size([1, 1419]) +Dataloader batch - attn_mask shape: torch.Size([1, 1419]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 75 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1419) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1419) torch.int64 + loss_mask : 442 / 1419 tokens (31.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1419, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1419, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1419, 1, 576) torch.bfloat16 + out : (1, 1419) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 442 tokens) : 11.4428 + top-1 accuracy : 2/442 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1258]) +Dataloader batch - labels shape: torch.Size([1, 1258]) +Dataloader batch - attn_mask shape: torch.Size([1, 1258]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 76 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1258) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1258) torch.int64 + loss_mask : 364 / 1258 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1258, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1258, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1258, 1, 576) torch.bfloat16 + out : (1, 1258) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.4051 + top-1 accuracy : 1/364 = 0.3% + + [2026-04-29 13:38:23] iteration 119/ 500 | consumed samples: 476 | elapsed time per iteration (ms): 5695.3 | throughput (token/sec/GPU): 876.2 | learning rate: 9.968249E-05 | global batch size: 4 | lm loss: 1.134378E+01 | loss scale: 1.0 | grad norm: 0.374 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1863]) +Dataloader batch - labels shape: torch.Size([1, 1863]) +Dataloader batch - attn_mask shape: torch.Size([1, 1863]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 77 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1863) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1863) torch.int64 + loss_mask : 474 / 1863 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1863, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1863, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1863, 1, 576) torch.bfloat16 + out : (1, 1863) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 474 tokens) : 11.3282 + top-1 accuracy : 14/474 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1513]) +Dataloader batch - labels shape: torch.Size([1, 1513]) +Dataloader batch - attn_mask shape: torch.Size([1, 1513]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 78 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1513) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1513) torch.int64 + loss_mask : 374 / 1513 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1513, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1513, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1513, 1, 576) torch.bfloat16 + out : (1, 1513) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.2422 + top-1 accuracy : 11/374 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1854]) +Dataloader batch - labels shape: torch.Size([1, 1854]) +Dataloader batch - attn_mask shape: torch.Size([1, 1854]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 79 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1854) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1854) torch.int64 + loss_mask : 476 / 1854 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1854, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1854, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1854, 1, 576) torch.bfloat16 + out : (1, 1854) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 476 tokens) : 11.3314 + top-1 accuracy : 8/476 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 972]) +Dataloader batch - labels shape: torch.Size([1, 972]) +Dataloader batch - attn_mask shape: torch.Size([1, 972]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 80 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 972) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 972) torch.int64 + loss_mask : 190 / 972 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (972, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (972, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (972, 1, 576) torch.bfloat16 + out : (1, 972) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 190 tokens) : 11.2078 + top-1 accuracy : 11/190 = 5.8% + + [2026-04-29 13:38:30] iteration 120/ 500 | consumed samples: 480 | elapsed time per iteration (ms): 6451.8 | throughput (token/sec/GPU): 961.3 | learning rate: 9.964628E-05 | global batch size: 4 | lm loss: 1.129286E+01 | loss scale: 1.0 | grad norm: 0.396 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (6382.55, 6382.55) + forward-compute ................................: (5025.89, 5025.89) + backward-compute ...............................: (1335.75, 1335.75) + batch-generator ................................: (463.31, 463.31) + layernorm-grads-all-reduce .....................: (0.03, 0.03) + embedding-grads-all-reduce .....................: (0.04, 0.04) + all-grads-sync .................................: (5.11, 5.11) + params-all-gather ..............................: (0.20, 0.20) + optimizer-copy-to-main-grad ....................: (0.17, 0.17) + optimizer-clip-main-grad .......................: (0.94, 0.94) + optimizer-count-zeros ..........................: (0.02, 0.02) + optimizer-inner-step ...........................: (1.62, 1.62) + optimizer-copy-main-to-model-params ............: (0.34, 0.34) + optimizer ......................................: (5.49, 5.49) +Dataloader batch - tokens shape: torch.Size([1, 1005]) +Dataloader batch - labels shape: torch.Size([1, 1005]) +Dataloader batch - attn_mask shape: torch.Size([1, 1005]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 81 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1005) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1005) torch.int64 + loss_mask : 233 / 1005 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1005, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1005, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1005, 1, 576) torch.bfloat16 + out : (1, 1005) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 233 tokens) : 11.2586 + top-1 accuracy : 14/233 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 82 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 324 / 1424 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.2872 + top-1 accuracy : 21/324 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 955]) +Dataloader batch - labels shape: torch.Size([1, 955]) +Dataloader batch - attn_mask shape: torch.Size([1, 955]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 83 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 955) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 955) torch.int64 + loss_mask : 210 / 955 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (955, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (955, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (955, 1, 576) torch.bfloat16 + out : (1, 955) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 210 tokens) : 11.2135 + top-1 accuracy : 10/210 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1355]) +Dataloader batch - labels shape: torch.Size([1, 1355]) +Dataloader batch - attn_mask shape: torch.Size([1, 1355]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 84 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1355) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1355) torch.int64 + loss_mask : 307 / 1355 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1355, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1355, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1355, 1, 576) torch.bfloat16 + out : (1, 1355) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.2149 + top-1 accuracy : 26/307 = 8.5% + + [2026-04-29 13:38:35] iteration 121/ 500 | consumed samples: 484 | elapsed time per iteration (ms): 5278.0 | throughput (token/sec/GPU): 897.9 | learning rate: 9.960811E-05 | global batch size: 4 | lm loss: 1.124593E+01 | loss scale: 1.0 | grad norm: 0.416 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 966]) +Dataloader batch - labels shape: torch.Size([1, 966]) +Dataloader batch - attn_mask shape: torch.Size([1, 966]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 85 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 966) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 966) torch.int64 + loss_mask : 190 / 966 tokens (19.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (966, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (966, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (966, 1, 576) torch.bfloat16 + out : (1, 966) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 190 tokens) : 11.1053 + top-1 accuracy : 7/190 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1702]) +Dataloader batch - labels shape: torch.Size([1, 1702]) +Dataloader batch - attn_mask shape: torch.Size([1, 1702]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 86 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1702) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1702) torch.int64 + loss_mask : 375 / 1702 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1702, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1702, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1702, 1, 576) torch.bfloat16 + out : (1, 1702) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.2441 + top-1 accuracy : 17/375 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1523]) +Dataloader batch - labels shape: torch.Size([1, 1523]) +Dataloader batch - attn_mask shape: torch.Size([1, 1523]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 87 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1523) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1523) torch.int64 + loss_mask : 339 / 1523 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1523, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1523, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1523, 1, 576) torch.bfloat16 + out : (1, 1523) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2565 + top-1 accuracy : 7/339 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1871]) +Dataloader batch - labels shape: torch.Size([1, 1871]) +Dataloader batch - attn_mask shape: torch.Size([1, 1871]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 88 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1871) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1871) torch.int64 + loss_mask : 521 / 1871 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1871, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1871, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1871, 1, 576) torch.bfloat16 + out : (1, 1871) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 521 tokens) : 11.3472 + top-1 accuracy : 26/521 = 5.0% + + [2026-04-29 13:38:42] iteration 122/ 500 | consumed samples: 488 | elapsed time per iteration (ms): 6901.9 | throughput (token/sec/GPU): 878.3 | learning rate: 9.956800E-05 | global batch size: 4 | lm loss: 1.126623E+01 | loss scale: 1.0 | grad norm: 0.454 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1550]) +Dataloader batch - labels shape: torch.Size([1, 1550]) +Dataloader batch - attn_mask shape: torch.Size([1, 1550]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 89 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1550) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1550) torch.int64 + loss_mask : 391 / 1550 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1550, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1550, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1550, 1, 576) torch.bfloat16 + out : (1, 1550) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2893 + top-1 accuracy : 9/391 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1147]) +Dataloader batch - labels shape: torch.Size([1, 1147]) +Dataloader batch - attn_mask shape: torch.Size([1, 1147]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 90 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1147) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1147) torch.int64 + loss_mask : 295 / 1147 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1147, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1147, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1147, 1, 576) torch.bfloat16 + out : (1, 1147) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.3031 + top-1 accuracy : 1/295 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1472]) +Dataloader batch - labels shape: torch.Size([1, 1472]) +Dataloader batch - attn_mask shape: torch.Size([1, 1472]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 91 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1472) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1472) torch.int64 + loss_mask : 311 / 1472 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1472, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1472, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1472, 1, 576) torch.bfloat16 + out : (1, 1472) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.1260 + top-1 accuracy : 32/311 = 10.3% + +Dataloader batch - tokens shape: torch.Size([1, 1887]) +Dataloader batch - labels shape: torch.Size([1, 1887]) +Dataloader batch - attn_mask shape: torch.Size([1, 1887]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 92 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1887) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1887) torch.int64 + loss_mask : 529 / 1887 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1887, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1887, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1887, 1, 576) torch.bfloat16 + out : (1, 1887) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 529 tokens) : 11.4178 + top-1 accuracy : 27/529 = 5.1% + + [2026-04-29 13:38:48] iteration 123/ 500 | consumed samples: 492 | elapsed time per iteration (ms): 6489.9 | throughput (token/sec/GPU): 933.1 | learning rate: 9.952595E-05 | global batch size: 4 | lm loss: 1.130322E+01 | loss scale: 1.0 | grad norm: 0.356 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1479]) +Dataloader batch - labels shape: torch.Size([1, 1479]) +Dataloader batch - attn_mask shape: torch.Size([1, 1479]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 93 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1479) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1479) torch.int64 + loss_mask : 326 / 1479 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1479, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1479, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1479, 1, 576) torch.bfloat16 + out : (1, 1479) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.1881 + top-1 accuracy : 32/326 = 9.8% + +Dataloader batch - tokens shape: torch.Size([1, 1729]) +Dataloader batch - labels shape: torch.Size([1, 1729]) +Dataloader batch - attn_mask shape: torch.Size([1, 1729]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 94 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1729) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1729) torch.int64 + loss_mask : 354 / 1729 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1729, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1729, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1729, 1, 576) torch.bfloat16 + out : (1, 1729) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2429 + top-1 accuracy : 26/354 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1684]) +Dataloader batch - labels shape: torch.Size([1, 1684]) +Dataloader batch - attn_mask shape: torch.Size([1, 1684]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 95 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1684) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1684) torch.int64 + loss_mask : 336 / 1684 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1684, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1684, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1684, 1, 576) torch.bfloat16 + out : (1, 1684) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.1762 + top-1 accuracy : 35/336 = 10.4% + +Dataloader batch - tokens shape: torch.Size([1, 1390]) +Dataloader batch - labels shape: torch.Size([1, 1390]) +Dataloader batch - attn_mask shape: torch.Size([1, 1390]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 96 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1390) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1390) torch.int64 + loss_mask : 334 / 1390 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1390, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1390, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1390, 1, 576) torch.bfloat16 + out : (1, 1390) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.2615 + top-1 accuracy : 6/334 = 1.8% + + [2026-04-29 13:38:55] iteration 124/ 500 | consumed samples: 496 | elapsed time per iteration (ms): 6799.2 | throughput (token/sec/GPU): 923.9 | learning rate: 9.948195E-05 | global batch size: 4 | lm loss: 1.121768E+01 | loss scale: 1.0 | grad norm: 0.365 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1783]) +Dataloader batch - labels shape: torch.Size([1, 1783]) +Dataloader batch - attn_mask shape: torch.Size([1, 1783]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 97 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1783) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1783) torch.int64 + loss_mask : 413 / 1783 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1783, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1783, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1783, 1, 576) torch.bfloat16 + out : (1, 1783) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2926 + top-1 accuracy : 32/413 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1129]) +Dataloader batch - labels shape: torch.Size([1, 1129]) +Dataloader batch - attn_mask shape: torch.Size([1, 1129]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 98 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1129) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1129) torch.int64 + loss_mask : 261 / 1129 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1129, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1129, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1129, 1, 576) torch.bfloat16 + out : (1, 1129) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 261 tokens) : 11.2471 + top-1 accuracy : 6/261 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 993]) +Dataloader batch - labels shape: torch.Size([1, 993]) +Dataloader batch - attn_mask shape: torch.Size([1, 993]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 99 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 993) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 993) torch.int64 + loss_mask : 243 / 993 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (993, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (993, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (993, 1, 576) torch.bfloat16 + out : (1, 993) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 243 tokens) : 11.3329 + top-1 accuracy : 19/243 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1943]) +Dataloader batch - labels shape: torch.Size([1, 1943]) +Dataloader batch - attn_mask shape: torch.Size([1, 1943]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 100 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1943) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1943) torch.int64 + loss_mask : 585 / 1943 tokens (30.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1943, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1943, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1943, 1, 576) torch.bfloat16 + out : (1, 1943) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 585 tokens) : 11.4314 + top-1 accuracy : 25/585 = 4.3% + + [2026-04-29 13:39:01] iteration 125/ 500 | consumed samples: 500 | elapsed time per iteration (ms): 6161.9 | throughput (token/sec/GPU): 949.1 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.134525E+01 | loss scale: 1.0 | grad norm: 0.337 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1724]) +Dataloader batch - labels shape: torch.Size([1, 1724]) +Dataloader batch - attn_mask shape: torch.Size([1, 1724]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 101 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1724) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1724) torch.int64 + loss_mask : 376 / 1724 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1724, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1724, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1724, 1, 576) torch.bfloat16 + out : (1, 1724) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.2196 + top-1 accuracy : 4/376 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1793]) +Dataloader batch - labels shape: torch.Size([1, 1793]) +Dataloader batch - attn_mask shape: torch.Size([1, 1793]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 102 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1793) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1793) torch.int64 + loss_mask : 422 / 1793 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1793, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1793, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1793, 1, 576) torch.bfloat16 + out : (1, 1793) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.2956 + top-1 accuracy : 12/422 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1501]) +Dataloader batch - labels shape: torch.Size([1, 1501]) +Dataloader batch - attn_mask shape: torch.Size([1, 1501]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 103 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1501) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1501) torch.int64 + loss_mask : 376 / 1501 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1501, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1501, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1501, 1, 576) torch.bfloat16 + out : (1, 1501) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3195 + top-1 accuracy : 29/376 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1366]) +Dataloader batch - labels shape: torch.Size([1, 1366]) +Dataloader batch - attn_mask shape: torch.Size([1, 1366]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 104 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1366) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1366) torch.int64 + loss_mask : 307 / 1366 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1366, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1366, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1366, 1, 576) torch.bfloat16 + out : (1, 1366) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.1803 + top-1 accuracy : 21/307 = 6.8% + + [2026-04-29 13:39:08] iteration 126/ 500 | consumed samples: 504 | elapsed time per iteration (ms): 6732.4 | throughput (token/sec/GPU): 948.3 | learning rate: 9.938813E-05 | global batch size: 4 | lm loss: 1.125848E+01 | loss scale: 1.0 | grad norm: 0.296 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 105 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 359 / 1722 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.2120 + top-1 accuracy : 21/359 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1024]) +Dataloader batch - labels shape: torch.Size([1, 1024]) +Dataloader batch - attn_mask shape: torch.Size([1, 1024]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 106 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1024) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1024) torch.int64 + loss_mask : 253 / 1024 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1024, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1024, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1024, 1, 576) torch.bfloat16 + out : (1, 1024) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 253 tokens) : 11.2729 + top-1 accuracy : 4/253 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1071]) +Dataloader batch - labels shape: torch.Size([1, 1071]) +Dataloader batch - attn_mask shape: torch.Size([1, 1071]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 107 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1071) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1071) torch.int64 + loss_mask : 236 / 1071 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1071, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1071, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1071, 1, 576) torch.bfloat16 + out : (1, 1071) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 236 tokens) : 11.1930 + top-1 accuracy : 14/236 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1430]) +Dataloader batch - labels shape: torch.Size([1, 1430]) +Dataloader batch - attn_mask shape: torch.Size([1, 1430]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 108 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1430) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1430) torch.int64 + loss_mask : 378 / 1430 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1430, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1430, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1430, 1, 576) torch.bfloat16 + out : (1, 1430) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 378 tokens) : 11.3211 + top-1 accuracy : 16/378 = 4.2% + + [2026-04-29 13:39:14] iteration 127/ 500 | consumed samples: 508 | elapsed time per iteration (ms): 5641.1 | throughput (token/sec/GPU): 930.1 | learning rate: 9.933831E-05 | global batch size: 4 | lm loss: 1.125455E+01 | loss scale: 1.0 | grad norm: 0.306 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1192]) +Dataloader batch - labels shape: torch.Size([1, 1192]) +Dataloader batch - attn_mask shape: torch.Size([1, 1192]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 109 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1192) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1192) torch.int64 + loss_mask : 297 / 1192 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1192, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1192, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1192, 1, 576) torch.bfloat16 + out : (1, 1192) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.3082 + top-1 accuracy : 6/297 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 110 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 297 / 1374 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.1230 + top-1 accuracy : 16/297 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1417]) +Dataloader batch - labels shape: torch.Size([1, 1417]) +Dataloader batch - attn_mask shape: torch.Size([1, 1417]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 111 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1417) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1417) torch.int64 + loss_mask : 333 / 1417 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1417, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1417, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1417, 1, 576) torch.bfloat16 + out : (1, 1417) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.2373 + top-1 accuracy : 10/333 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1845]) +Dataloader batch - labels shape: torch.Size([1, 1845]) +Dataloader batch - attn_mask shape: torch.Size([1, 1845]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 112 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1845) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1845) torch.int64 + loss_mask : 477 / 1845 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1845, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1845, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1845, 1, 576) torch.bfloat16 + out : (1, 1845) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 477 tokens) : 11.2511 + top-1 accuracy : 15/477 = 3.1% + + [2026-04-29 13:39:20] iteration 128/ 500 | consumed samples: 512 | elapsed time per iteration (ms): 6345.3 | throughput (token/sec/GPU): 918.5 | learning rate: 9.928656E-05 | global batch size: 4 | lm loss: 1.123283E+01 | loss scale: 1.0 | grad norm: 0.373 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1405]) +Dataloader batch - labels shape: torch.Size([1, 1405]) +Dataloader batch - attn_mask shape: torch.Size([1, 1405]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 113 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1405) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1405) torch.int64 + loss_mask : 386 / 1405 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1405, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1405, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1405, 1, 576) torch.bfloat16 + out : (1, 1405) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3666 + top-1 accuracy : 1/386 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1031]) +Dataloader batch - labels shape: torch.Size([1, 1031]) +Dataloader batch - attn_mask shape: torch.Size([1, 1031]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 114 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1031) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1031) torch.int64 + loss_mask : 271 / 1031 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1031, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1031, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1031, 1, 576) torch.bfloat16 + out : (1, 1031) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 271 tokens) : 11.3204 + top-1 accuracy : 19/271 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1150]) +Dataloader batch - labels shape: torch.Size([1, 1150]) +Dataloader batch - attn_mask shape: torch.Size([1, 1150]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 115 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1150) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1150) torch.int64 + loss_mask : 343 / 1150 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1150, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1150, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1150, 1, 576) torch.bfloat16 + out : (1, 1150) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.4182 + top-1 accuracy : 4/343 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 116 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 306 / 1444 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.2358 + top-1 accuracy : 17/306 = 5.6% + + [2026-04-29 13:39:26] iteration 129/ 500 | consumed samples: 516 | elapsed time per iteration (ms): 5636.3 | throughput (token/sec/GPU): 892.4 | learning rate: 9.923288E-05 | global batch size: 4 | lm loss: 1.133993E+01 | loss scale: 1.0 | grad norm: 0.356 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1842]) +Dataloader batch - labels shape: torch.Size([1, 1842]) +Dataloader batch - attn_mask shape: torch.Size([1, 1842]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 117 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1842) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1842) torch.int64 + loss_mask : 493 / 1842 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1842, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1842, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1842, 1, 576) torch.bfloat16 + out : (1, 1842) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 493 tokens) : 11.3380 + top-1 accuracy : 11/493 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1432]) +Dataloader batch - labels shape: torch.Size([1, 1432]) +Dataloader batch - attn_mask shape: torch.Size([1, 1432]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 118 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1432) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1432) torch.int64 + loss_mask : 389 / 1432 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1432, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1432, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1432, 1, 576) torch.bfloat16 + out : (1, 1432) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.3105 + top-1 accuracy : 24/389 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1555]) +Dataloader batch - labels shape: torch.Size([1, 1555]) +Dataloader batch - attn_mask shape: torch.Size([1, 1555]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 119 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1555) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1555) torch.int64 + loss_mask : 395 / 1555 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1555, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1555, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1555, 1, 576) torch.bfloat16 + out : (1, 1555) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.3287 + top-1 accuracy : 3/395 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1463]) +Dataloader batch - labels shape: torch.Size([1, 1463]) +Dataloader batch - attn_mask shape: torch.Size([1, 1463]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 120 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1463) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1463) torch.int64 + loss_mask : 492 / 1463 tokens (33.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1463, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1463, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1463, 1, 576) torch.bfloat16 + out : (1, 1463) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 492 tokens) : 11.4790 + top-1 accuracy : 3/492 = 0.6% + + [2026-04-29 13:39:32] iteration 130/ 500 | consumed samples: 520 | elapsed time per iteration (ms): 6369.8 | throughput (token/sec/GPU): 987.8 | learning rate: 9.917726E-05 | global batch size: 4 | lm loss: 1.136906E+01 | loss scale: 1.0 | grad norm: 0.305 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1463]) +Dataloader batch - labels shape: torch.Size([1, 1463]) +Dataloader batch - attn_mask shape: torch.Size([1, 1463]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 121 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1463) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1463) torch.int64 + loss_mask : 354 / 1463 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1463, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1463, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1463, 1, 576) torch.bfloat16 + out : (1, 1463) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.3426 + top-1 accuracy : 20/354 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1955]) +Dataloader batch - labels shape: torch.Size([1, 1955]) +Dataloader batch - attn_mask shape: torch.Size([1, 1955]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 122 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1955) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1955) torch.int64 + loss_mask : 577 / 1955 tokens (29.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1955, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1955, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1955, 1, 576) torch.bfloat16 + out : (1, 1955) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 577 tokens) : 11.3841 + top-1 accuracy : 4/577 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 123 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 405 / 1543 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.3773 + top-1 accuracy : 2/405 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1656]) +Dataloader batch - labels shape: torch.Size([1, 1656]) +Dataloader batch - attn_mask shape: torch.Size([1, 1656]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 124 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1656) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1656) torch.int64 + loss_mask : 407 / 1656 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1656, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1656, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1656, 1, 576) torch.bfloat16 + out : (1, 1656) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.2852 + top-1 accuracy : 6/407 = 1.5% + + [2026-04-29 13:39:39] iteration 131/ 500 | consumed samples: 524 | elapsed time per iteration (ms): 6635.3 | throughput (token/sec/GPU): 997.2 | learning rate: 9.911971E-05 | global batch size: 4 | lm loss: 1.135101E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1014]) +Dataloader batch - labels shape: torch.Size([1, 1014]) +Dataloader batch - attn_mask shape: torch.Size([1, 1014]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 125 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1014) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1014) torch.int64 + loss_mask : 241 / 1014 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1014, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1014, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1014, 1, 576) torch.bfloat16 + out : (1, 1014) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 241 tokens) : 11.2600 + top-1 accuracy : 9/241 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1761]) +Dataloader batch - labels shape: torch.Size([1, 1761]) +Dataloader batch - attn_mask shape: torch.Size([1, 1761]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 126 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1761) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1761) torch.int64 + loss_mask : 391 / 1761 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1761, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1761, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1761, 1, 576) torch.bfloat16 + out : (1, 1761) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2597 + top-1 accuracy : 24/391 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1011]) +Dataloader batch - labels shape: torch.Size([1, 1011]) +Dataloader batch - attn_mask shape: torch.Size([1, 1011]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 127 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1011) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1011) torch.int64 + loss_mask : 237 / 1011 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1011, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1011, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1011, 1, 576) torch.bfloat16 + out : (1, 1011) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 237 tokens) : 11.2926 + top-1 accuracy : 4/237 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1867]) +Dataloader batch - labels shape: torch.Size([1, 1867]) +Dataloader batch - attn_mask shape: torch.Size([1, 1867]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 128 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1867) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1867) torch.int64 + loss_mask : 493 / 1867 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1867, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1867, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1867, 1, 576) torch.bfloat16 + out : (1, 1867) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 493 tokens) : 11.3229 + top-1 accuracy : 2/493 = 0.4% + + [2026-04-29 13:39:45] iteration 132/ 500 | consumed samples: 528 | elapsed time per iteration (ms): 5899.2 | throughput (token/sec/GPU): 958.3 | learning rate: 9.906023E-05 | global batch size: 4 | lm loss: 1.128837E+01 | loss scale: 1.0 | grad norm: 0.364 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1560]) +Dataloader batch - labels shape: torch.Size([1, 1560]) +Dataloader batch - attn_mask shape: torch.Size([1, 1560]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 129 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1560) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1560) torch.int64 + loss_mask : 377 / 1560 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1560, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1560, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1560, 1, 576) torch.bfloat16 + out : (1, 1560) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.2536 + top-1 accuracy : 22/377 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 130 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 311 / 1508 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.1884 + top-1 accuracy : 31/311 = 10.0% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 131 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 369 / 1507 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.1881 + top-1 accuracy : 12/369 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 132 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1412) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1412) torch.int64 + loss_mask : 315 / 1412 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1412, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1412, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1412, 1, 576) torch.bfloat16 + out : (1, 1412) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.2320 + top-1 accuracy : 16/315 = 5.1% + + [2026-04-29 13:39:51] iteration 133/ 500 | consumed samples: 532 | elapsed time per iteration (ms): 6505.5 | throughput (token/sec/GPU): 920.3 | learning rate: 9.899884E-05 | global batch size: 4 | lm loss: 1.121625E+01 | loss scale: 1.0 | grad norm: 0.369 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1089]) +Dataloader batch - labels shape: torch.Size([1, 1089]) +Dataloader batch - attn_mask shape: torch.Size([1, 1089]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 133 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1089) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1089) torch.int64 + loss_mask : 319 / 1089 tokens (29.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1089, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1089, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1089, 1, 576) torch.bfloat16 + out : (1, 1089) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.3691 + top-1 accuracy : 1/319 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1159]) +Dataloader batch - labels shape: torch.Size([1, 1159]) +Dataloader batch - attn_mask shape: torch.Size([1, 1159]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 134 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1159) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1159) torch.int64 + loss_mask : 282 / 1159 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1159, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1159, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1159, 1, 576) torch.bfloat16 + out : (1, 1159) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 282 tokens) : 11.3114 + top-1 accuracy : 15/282 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1196]) +Dataloader batch - labels shape: torch.Size([1, 1196]) +Dataloader batch - attn_mask shape: torch.Size([1, 1196]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 135 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1196) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1196) torch.int64 + loss_mask : 303 / 1196 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1196, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1196, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1196, 1, 576) torch.bfloat16 + out : (1, 1196) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.2786 + top-1 accuracy : 17/303 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1733]) +Dataloader batch - labels shape: torch.Size([1, 1733]) +Dataloader batch - attn_mask shape: torch.Size([1, 1733]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 136 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1733) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1733) torch.int64 + loss_mask : 379 / 1733 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1733, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1733, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1733, 1, 576) torch.bfloat16 + out : (1, 1733) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.1875 + top-1 accuracy : 33/379 = 8.7% + + [2026-04-29 13:39:57] iteration 134/ 500 | consumed samples: 536 | elapsed time per iteration (ms): 5562.2 | throughput (token/sec/GPU): 930.7 | learning rate: 9.893552E-05 | global batch size: 4 | lm loss: 1.128139E+01 | loss scale: 1.0 | grad norm: 0.311 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1718]) +Dataloader batch - labels shape: torch.Size([1, 1718]) +Dataloader batch - attn_mask shape: torch.Size([1, 1718]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 137 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1718) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1718) torch.int64 + loss_mask : 440 / 1718 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1718, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1718, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1718, 1, 576) torch.bfloat16 + out : (1, 1718) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 440 tokens) : 11.3084 + top-1 accuracy : 8/440 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1411]) +Dataloader batch - labels shape: torch.Size([1, 1411]) +Dataloader batch - attn_mask shape: torch.Size([1, 1411]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 138 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1411) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1411) torch.int64 + loss_mask : 363 / 1411 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1411, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1411, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1411, 1, 576) torch.bfloat16 + out : (1, 1411) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2617 + top-1 accuracy : 11/363 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1521]) +Dataloader batch - labels shape: torch.Size([1, 1521]) +Dataloader batch - attn_mask shape: torch.Size([1, 1521]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 139 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1521) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1521) torch.int64 + loss_mask : 346 / 1521 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1521, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1521, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1521, 1, 576) torch.bfloat16 + out : (1, 1521) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.2014 + top-1 accuracy : 25/346 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1800]) +Dataloader batch - labels shape: torch.Size([1, 1800]) +Dataloader batch - attn_mask shape: torch.Size([1, 1800]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 140 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1800) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1800) torch.int64 + loss_mask : 436 / 1800 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1800, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1800, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1800, 1, 576) torch.bfloat16 + out : (1, 1800) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.2668 + top-1 accuracy : 20/436 = 4.6% + + [2026-04-29 13:40:03] iteration 135/ 500 | consumed samples: 540 | elapsed time per iteration (ms): 6892.3 | throughput (token/sec/GPU): 935.8 | learning rate: 9.887027E-05 | global batch size: 4 | lm loss: 1.126288E+01 | loss scale: 1.0 | grad norm: 0.302 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 141 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 348 / 1496 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2142 + top-1 accuracy : 5/348 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1397]) +Dataloader batch - labels shape: torch.Size([1, 1397]) +Dataloader batch - attn_mask shape: torch.Size([1, 1397]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 142 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1397) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1397) torch.int64 + loss_mask : 333 / 1397 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1397, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1397, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1397, 1, 576) torch.bfloat16 + out : (1, 1397) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.2797 + top-1 accuracy : 13/333 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1748]) +Dataloader batch - labels shape: torch.Size([1, 1748]) +Dataloader batch - attn_mask shape: torch.Size([1, 1748]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 143 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1748) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1748) torch.int64 + loss_mask : 384 / 1748 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1748, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1748, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1748, 1, 576) torch.bfloat16 + out : (1, 1748) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.2823 + top-1 accuracy : 24/384 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 988]) +Dataloader batch - labels shape: torch.Size([1, 988]) +Dataloader batch - attn_mask shape: torch.Size([1, 988]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 144 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 988) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 988) torch.int64 + loss_mask : 223 / 988 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (988, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (988, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (988, 1, 576) torch.bfloat16 + out : (1, 988) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 223 tokens) : 11.2595 + top-1 accuracy : 20/223 = 9.0% + + [2026-04-29 13:40:10] iteration 136/ 500 | consumed samples: 544 | elapsed time per iteration (ms): 6420.5 | throughput (token/sec/GPU): 876.7 | learning rate: 9.880312E-05 | global batch size: 4 | lm loss: 1.125928E+01 | loss scale: 1.0 | grad norm: 0.323 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 145 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 287 / 1459 tokens (19.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 287 tokens) : 11.1366 + top-1 accuracy : 34/287 = 11.8% + +Dataloader batch - tokens shape: torch.Size([1, 1036]) +Dataloader batch - labels shape: torch.Size([1, 1036]) +Dataloader batch - attn_mask shape: torch.Size([1, 1036]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 146 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1036) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1036) torch.int64 + loss_mask : 220 / 1036 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1036, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1036, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1036, 1, 576) torch.bfloat16 + out : (1, 1036) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 220 tokens) : 11.1850 + top-1 accuracy : 4/220 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1356]) +Dataloader batch - labels shape: torch.Size([1, 1356]) +Dataloader batch - attn_mask shape: torch.Size([1, 1356]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 147 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1356) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1356) torch.int64 + loss_mask : 293 / 1356 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1356, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1356, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1356, 1, 576) torch.bfloat16 + out : (1, 1356) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 293 tokens) : 11.1797 + top-1 accuracy : 5/293 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1773]) +Dataloader batch - labels shape: torch.Size([1, 1773]) +Dataloader batch - attn_mask shape: torch.Size([1, 1773]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 148 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1773) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1773) torch.int64 + loss_mask : 413 / 1773 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1773, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1773, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1773, 1, 576) torch.bfloat16 + out : (1, 1773) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2286 + top-1 accuracy : 20/413 = 4.8% + + [2026-04-29 13:40:16] iteration 137/ 500 | consumed samples: 548 | elapsed time per iteration (ms): 6276.8 | throughput (token/sec/GPU): 896.0 | learning rate: 9.873405E-05 | global batch size: 4 | lm loss: 1.118712E+01 | loss scale: 1.0 | grad norm: 0.431 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1489]) +Dataloader batch - labels shape: torch.Size([1, 1489]) +Dataloader batch - attn_mask shape: torch.Size([1, 1489]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 149 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1489) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1489) torch.int64 + loss_mask : 386 / 1489 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1489, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1489, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1489, 1, 576) torch.bfloat16 + out : (1, 1489) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3305 + top-1 accuracy : 4/386 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1179]) +Dataloader batch - labels shape: torch.Size([1, 1179]) +Dataloader batch - attn_mask shape: torch.Size([1, 1179]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 150 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1179) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1179) torch.int64 + loss_mask : 315 / 1179 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1179, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1179, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1179, 1, 576) torch.bfloat16 + out : (1, 1179) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.2878 + top-1 accuracy : 4/315 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1906]) +Dataloader batch - labels shape: torch.Size([1, 1906]) +Dataloader batch - attn_mask shape: torch.Size([1, 1906]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 151 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1906) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1906) torch.int64 + loss_mask : 522 / 1906 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1906, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1906, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1906, 1, 576) torch.bfloat16 + out : (1, 1906) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 522 tokens) : 11.3656 + top-1 accuracy : 10/522 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1865]) +Dataloader batch - labels shape: torch.Size([1, 1865]) +Dataloader batch - attn_mask shape: torch.Size([1, 1865]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 152 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1865) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1865) torch.int64 + loss_mask : 500 / 1865 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1865, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1865, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1865, 1, 576) torch.bfloat16 + out : (1, 1865) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 500 tokens) : 11.3122 + top-1 accuracy : 16/500 = 3.2% + + [2026-04-29 13:40:23] iteration 138/ 500 | consumed samples: 552 | elapsed time per iteration (ms): 6658.9 | throughput (token/sec/GPU): 967.0 | learning rate: 9.866305E-05 | global batch size: 4 | lm loss: 1.132802E+01 | loss scale: 1.0 | grad norm: 0.325 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1197]) +Dataloader batch - labels shape: torch.Size([1, 1197]) +Dataloader batch - attn_mask shape: torch.Size([1, 1197]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 153 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1197) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1197) torch.int64 + loss_mask : 273 / 1197 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1197, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1197, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1197, 1, 576) torch.bfloat16 + out : (1, 1197) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 273 tokens) : 11.2877 + top-1 accuracy : 17/273 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1671]) +Dataloader batch - labels shape: torch.Size([1, 1671]) +Dataloader batch - attn_mask shape: torch.Size([1, 1671]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 154 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1671) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1671) torch.int64 + loss_mask : 385 / 1671 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1671, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1671, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1671, 1, 576) torch.bfloat16 + out : (1, 1671) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.2324 + top-1 accuracy : 27/385 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1860]) +Dataloader batch - labels shape: torch.Size([1, 1860]) +Dataloader batch - attn_mask shape: torch.Size([1, 1860]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 155 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1860) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1860) torch.int64 + loss_mask : 497 / 1860 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1860, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1860, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1860, 1, 576) torch.bfloat16 + out : (1, 1860) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 497 tokens) : 11.3333 + top-1 accuracy : 7/497 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 156 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 340 / 1518 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2361 + top-1 accuracy : 24/340 = 7.1% + + [2026-04-29 13:40:30] iteration 139/ 500 | consumed samples: 556 | elapsed time per iteration (ms): 7093.1 | throughput (token/sec/GPU): 880.6 | learning rate: 9.859016E-05 | global batch size: 4 | lm loss: 1.127688E+01 | loss scale: 1.0 | grad norm: 0.326 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 157 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 395 / 1552 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.3067 + top-1 accuracy : 25/395 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 158 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 419 / 1507 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.3629 + top-1 accuracy : 18/419 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1044]) +Dataloader batch - labels shape: torch.Size([1, 1044]) +Dataloader batch - attn_mask shape: torch.Size([1, 1044]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 159 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1044) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1044) torch.int64 + loss_mask : 284 / 1044 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1044, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1044, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1044, 1, 576) torch.bfloat16 + out : (1, 1044) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.3443 + top-1 accuracy : 3/284 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1879]) +Dataloader batch - labels shape: torch.Size([1, 1879]) +Dataloader batch - attn_mask shape: torch.Size([1, 1879]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 160 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1879) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1879) torch.int64 + loss_mask : 535 / 1879 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1879, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1879, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1879, 1, 576) torch.bfloat16 + out : (1, 1879) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 535 tokens) : 11.3024 + top-1 accuracy : 21/535 = 3.9% + + [2026-04-29 13:40:36] iteration 140/ 500 | consumed samples: 560 | elapsed time per iteration (ms): 6228.8 | throughput (token/sec/GPU): 960.4 | learning rate: 9.851536E-05 | global batch size: 4 | lm loss: 1.132625E+01 | loss scale: 1.0 | grad norm: 0.315 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (6168.38, 6168.38) + forward-compute ................................: (4876.52, 4876.52) + backward-compute ...............................: (1281.10, 1281.10) + batch-generator ................................: (448.33, 448.33) + layernorm-grads-all-reduce .....................: (0.03, 0.03) + embedding-grads-all-reduce .....................: (0.06, 0.06) + all-grads-sync .................................: (1.01, 1.01) + params-all-gather ..............................: (0.22, 0.22) + optimizer-copy-to-main-grad ....................: (0.19, 0.19) + optimizer-clip-main-grad .......................: (0.95, 0.95) + optimizer-count-zeros ..........................: (0.03, 0.03) + optimizer-inner-step ...........................: (1.55, 1.55) + optimizer-copy-main-to-model-params ............: (0.34, 0.34) + optimizer ......................................: (5.40, 5.40) +Dataloader batch - tokens shape: torch.Size([1, 1731]) +Dataloader batch - labels shape: torch.Size([1, 1731]) +Dataloader batch - attn_mask shape: torch.Size([1, 1731]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 161 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1731) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1731) torch.int64 + loss_mask : 366 / 1731 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1731, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1731, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1731, 1, 576) torch.bfloat16 + out : (1, 1731) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2368 + top-1 accuracy : 23/366 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1743]) +Dataloader batch - labels shape: torch.Size([1, 1743]) +Dataloader batch - attn_mask shape: torch.Size([1, 1743]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 162 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1743) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1743) torch.int64 + loss_mask : 388 / 1743 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1743, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1743, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1743, 1, 576) torch.bfloat16 + out : (1, 1743) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2276 + top-1 accuracy : 31/388 = 8.0% + +Dataloader batch - tokens shape: torch.Size([1, 1902]) +Dataloader batch - labels shape: torch.Size([1, 1902]) +Dataloader batch - attn_mask shape: torch.Size([1, 1902]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 163 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1902) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1902) torch.int64 + loss_mask : 550 / 1902 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1902, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1902, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1902, 1, 576) torch.bfloat16 + out : (1, 1902) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 550 tokens) : 11.3860 + top-1 accuracy : 12/550 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1708]) +Dataloader batch - labels shape: torch.Size([1, 1708]) +Dataloader batch - attn_mask shape: torch.Size([1, 1708]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 164 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1708) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1708) torch.int64 + loss_mask : 345 / 1708 tokens (20.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1708, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1708, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1708, 1, 576) torch.bfloat16 + out : (1, 1708) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.1546 + top-1 accuracy : 20/345 = 5.8% + + [2026-04-29 13:40:44] iteration 141/ 500 | consumed samples: 564 | elapsed time per iteration (ms): 7734.9 | throughput (token/sec/GPU): 915.8 | learning rate: 9.843866E-05 | global batch size: 4 | lm loss: 1.126720E+01 | loss scale: 1.0 | grad norm: 0.413 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1124]) +Dataloader batch - labels shape: torch.Size([1, 1124]) +Dataloader batch - attn_mask shape: torch.Size([1, 1124]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 165 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1124) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1124) torch.int64 + loss_mask : 278 / 1124 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1124, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1124, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1124, 1, 576) torch.bfloat16 + out : (1, 1124) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.3252 + top-1 accuracy : 0/278 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 166 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 392 / 1542 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3219 + top-1 accuracy : 27/392 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 167 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 278 / 1444 tokens (19.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.2068 + top-1 accuracy : 19/278 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1456]) +Dataloader batch - labels shape: torch.Size([1, 1456]) +Dataloader batch - attn_mask shape: torch.Size([1, 1456]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 168 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1456) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1456) torch.int64 + loss_mask : 306 / 1456 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1456, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1456, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1456, 1, 576) torch.bfloat16 + out : (1, 1456) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.2417 + top-1 accuracy : 16/306 = 5.2% + + [2026-04-29 13:40:50] iteration 142/ 500 | consumed samples: 568 | elapsed time per iteration (ms): 6053.5 | throughput (token/sec/GPU): 919.5 | learning rate: 9.836005E-05 | global batch size: 4 | lm loss: 1.127754E+01 | loss scale: 1.0 | grad norm: 0.369 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1147]) +Dataloader batch - labels shape: torch.Size([1, 1147]) +Dataloader batch - attn_mask shape: torch.Size([1, 1147]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 169 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1147) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1147) torch.int64 + loss_mask : 335 / 1147 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1147, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1147, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1147, 1, 576) torch.bfloat16 + out : (1, 1147) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.3285 + top-1 accuracy : 6/335 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1588]) +Dataloader batch - labels shape: torch.Size([1, 1588]) +Dataloader batch - attn_mask shape: torch.Size([1, 1588]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 170 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1588) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1588) torch.int64 + loss_mask : 332 / 1588 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1588, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1588, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1588, 1, 576) torch.bfloat16 + out : (1, 1588) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.1775 + top-1 accuracy : 34/332 = 10.2% + +Dataloader batch - tokens shape: torch.Size([1, 1866]) +Dataloader batch - labels shape: torch.Size([1, 1866]) +Dataloader batch - attn_mask shape: torch.Size([1, 1866]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 171 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1866) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1866) torch.int64 + loss_mask : 508 / 1866 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1866, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1866, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1866, 1, 576) torch.bfloat16 + out : (1, 1866) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 508 tokens) : 11.3412 + top-1 accuracy : 23/508 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1108]) +Dataloader batch - labels shape: torch.Size([1, 1108]) +Dataloader batch - attn_mask shape: torch.Size([1, 1108]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 172 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1108) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1108) torch.int64 + loss_mask : 299 / 1108 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1108, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1108, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1108, 1, 576) torch.bfloat16 + out : (1, 1108) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 299 tokens) : 11.3469 + top-1 accuracy : 2/299 = 0.7% + + [2026-04-29 13:40:56] iteration 143/ 500 | consumed samples: 572 | elapsed time per iteration (ms): 5952.9 | throughput (token/sec/GPU): 959.0 | learning rate: 9.827955E-05 | global batch size: 4 | lm loss: 1.130261E+01 | loss scale: 1.0 | grad norm: 0.289 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1404]) +Dataloader batch - labels shape: torch.Size([1, 1404]) +Dataloader batch - attn_mask shape: torch.Size([1, 1404]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 173 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1404) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1404) torch.int64 + loss_mask : 399 / 1404 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1404, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1404, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1404, 1, 576) torch.bfloat16 + out : (1, 1404) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.3410 + top-1 accuracy : 9/399 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 174 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 354 / 1438 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.3888 + top-1 accuracy : 10/354 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1669]) +Dataloader batch - labels shape: torch.Size([1, 1669]) +Dataloader batch - attn_mask shape: torch.Size([1, 1669]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 175 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1669) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1669) torch.int64 + loss_mask : 419 / 1669 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1669, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1669, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1669, 1, 576) torch.bfloat16 + out : (1, 1669) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.2859 + top-1 accuracy : 15/419 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1390]) +Dataloader batch - labels shape: torch.Size([1, 1390]) +Dataloader batch - attn_mask shape: torch.Size([1, 1390]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 176 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1390) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1390) torch.int64 + loss_mask : 307 / 1390 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1390, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1390, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1390, 1, 576) torch.bfloat16 + out : (1, 1390) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.2623 + top-1 accuracy : 18/307 = 5.9% + + [2026-04-29 13:41:02] iteration 144/ 500 | consumed samples: 576 | elapsed time per iteration (ms): 6147.8 | throughput (token/sec/GPU): 959.9 | learning rate: 9.819716E-05 | global batch size: 4 | lm loss: 1.132049E+01 | loss scale: 1.0 | grad norm: 0.281 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1557]) +Dataloader batch - labels shape: torch.Size([1, 1557]) +Dataloader batch - attn_mask shape: torch.Size([1, 1557]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 177 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1557) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1557) torch.int64 + loss_mask : 408 / 1557 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1557, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1557, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1557, 1, 576) torch.bfloat16 + out : (1, 1557) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3568 + top-1 accuracy : 21/408 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1464]) +Dataloader batch - labels shape: torch.Size([1, 1464]) +Dataloader batch - attn_mask shape: torch.Size([1, 1464]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 178 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1464) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1464) torch.int64 + loss_mask : 406 / 1464 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1464, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1464, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1464, 1, 576) torch.bfloat16 + out : (1, 1464) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.3400 + top-1 accuracy : 0/406 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1538]) +Dataloader batch - labels shape: torch.Size([1, 1538]) +Dataloader batch - attn_mask shape: torch.Size([1, 1538]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 179 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1538) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1538) torch.int64 + loss_mask : 384 / 1538 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1538, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1538, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1538, 1, 576) torch.bfloat16 + out : (1, 1538) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.2916 + top-1 accuracy : 16/384 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1165]) +Dataloader batch - labels shape: torch.Size([1, 1165]) +Dataloader batch - attn_mask shape: torch.Size([1, 1165]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 180 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1165) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1165) torch.int64 + loss_mask : 233 / 1165 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1165, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1165, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1165, 1, 576) torch.bfloat16 + out : (1, 1165) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 233 tokens) : 11.2029 + top-1 accuracy : 21/233 = 9.0% + + [2026-04-29 13:41:08] iteration 145/ 500 | consumed samples: 580 | elapsed time per iteration (ms): 6158.6 | throughput (token/sec/GPU): 929.4 | learning rate: 9.811288E-05 | global batch size: 4 | lm loss: 1.130945E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1007]) +Dataloader batch - labels shape: torch.Size([1, 1007]) +Dataloader batch - attn_mask shape: torch.Size([1, 1007]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 181 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1007) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1007) torch.int64 + loss_mask : 244 / 1007 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1007, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1007, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1007, 1, 576) torch.bfloat16 + out : (1, 1007) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 244 tokens) : 11.2514 + top-1 accuracy : 18/244 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1703]) +Dataloader batch - labels shape: torch.Size([1, 1703]) +Dataloader batch - attn_mask shape: torch.Size([1, 1703]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 182 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1703) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1703) torch.int64 + loss_mask : 375 / 1703 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1703, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1703, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1703, 1, 576) torch.bfloat16 + out : (1, 1703) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.1896 + top-1 accuracy : 42/375 = 11.2% + +Dataloader batch - tokens shape: torch.Size([1, 1032]) +Dataloader batch - labels shape: torch.Size([1, 1032]) +Dataloader batch - attn_mask shape: torch.Size([1, 1032]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 183 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1032) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1032) torch.int64 + loss_mask : 275 / 1032 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1032, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1032, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1032, 1, 576) torch.bfloat16 + out : (1, 1032) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 275 tokens) : 11.2489 + top-1 accuracy : 11/275 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1033]) +Dataloader batch - labels shape: torch.Size([1, 1033]) +Dataloader batch - attn_mask shape: torch.Size([1, 1033]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 184 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1033) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1033) torch.int64 + loss_mask : 228 / 1033 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1033, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1033, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1033, 1, 576) torch.bfloat16 + out : (1, 1033) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 228 tokens) : 11.1860 + top-1 accuracy : 7/228 = 3.1% + + [2026-04-29 13:41:14] iteration 146/ 500 | consumed samples: 584 | elapsed time per iteration (ms): 5455.1 | throughput (token/sec/GPU): 875.3 | learning rate: 9.802671E-05 | global batch size: 4 | lm loss: 1.121686E+01 | loss scale: 1.0 | grad norm: 0.343 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1579]) +Dataloader batch - labels shape: torch.Size([1, 1579]) +Dataloader batch - attn_mask shape: torch.Size([1, 1579]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 185 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1579) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1579) torch.int64 + loss_mask : 363 / 1579 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1579, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1579, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1579, 1, 576) torch.bfloat16 + out : (1, 1579) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2095 + top-1 accuracy : 24/363 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 186 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 295 / 1400 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.2497 + top-1 accuracy : 28/295 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 187 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 320 / 1380 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2186 + top-1 accuracy : 13/320 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1506]) +Dataloader batch - labels shape: torch.Size([1, 1506]) +Dataloader batch - attn_mask shape: torch.Size([1, 1506]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 188 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1506) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1506) torch.int64 + loss_mask : 386 / 1506 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1506, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1506, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1506, 1, 576) torch.bfloat16 + out : (1, 1506) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2727 + top-1 accuracy : 27/386 = 7.0% + + [2026-04-29 13:41:20] iteration 147/ 500 | consumed samples: 588 | elapsed time per iteration (ms): 6399.2 | throughput (token/sec/GPU): 916.5 | learning rate: 9.793865E-05 | global batch size: 4 | lm loss: 1.123823E+01 | loss scale: 1.0 | grad norm: 0.283 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1152]) +Dataloader batch - labels shape: torch.Size([1, 1152]) +Dataloader batch - attn_mask shape: torch.Size([1, 1152]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 189 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1152) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1152) torch.int64 + loss_mask : 278 / 1152 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1152, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1152, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1152, 1, 576) torch.bfloat16 + out : (1, 1152) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.3118 + top-1 accuracy : 4/278 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1476]) +Dataloader batch - labels shape: torch.Size([1, 1476]) +Dataloader batch - attn_mask shape: torch.Size([1, 1476]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 190 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1476) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1476) torch.int64 + loss_mask : 411 / 1476 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1476, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1476, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1476, 1, 576) torch.bfloat16 + out : (1, 1476) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.3982 + top-1 accuracy : 3/411 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1599]) +Dataloader batch - labels shape: torch.Size([1, 1599]) +Dataloader batch - attn_mask shape: torch.Size([1, 1599]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 191 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1599) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1599) torch.int64 + loss_mask : 467 / 1599 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1599, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1599, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1599, 1, 576) torch.bfloat16 + out : (1, 1599) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 467 tokens) : 11.3466 + top-1 accuracy : 18/467 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1485]) +Dataloader batch - labels shape: torch.Size([1, 1485]) +Dataloader batch - attn_mask shape: torch.Size([1, 1485]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 192 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1485) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1485) torch.int64 + loss_mask : 316 / 1485 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1485, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1485, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1485, 1, 576) torch.bfloat16 + out : (1, 1485) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.2275 + top-1 accuracy : 33/316 = 10.4% + + [2026-04-29 13:41:26] iteration 148/ 500 | consumed samples: 592 | elapsed time per iteration (ms): 5977.4 | throughput (token/sec/GPU): 955.6 | learning rate: 9.784872E-05 | global batch size: 4 | lm loss: 1.132890E+01 | loss scale: 1.0 | grad norm: 0.326 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1372]) +Dataloader batch - labels shape: torch.Size([1, 1372]) +Dataloader batch - attn_mask shape: torch.Size([1, 1372]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 193 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1372) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1372) torch.int64 + loss_mask : 312 / 1372 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1372, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1372, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1372, 1, 576) torch.bfloat16 + out : (1, 1372) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.3006 + top-1 accuracy : 8/312 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1891]) +Dataloader batch - labels shape: torch.Size([1, 1891]) +Dataloader batch - attn_mask shape: torch.Size([1, 1891]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 194 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1891) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1891) torch.int64 + loss_mask : 519 / 1891 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1891, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1891, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1891, 1, 576) torch.bfloat16 + out : (1, 1891) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 519 tokens) : 11.3463 + top-1 accuracy : 12/519 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1749]) +Dataloader batch - labels shape: torch.Size([1, 1749]) +Dataloader batch - attn_mask shape: torch.Size([1, 1749]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 195 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1749) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1749) torch.int64 + loss_mask : 388 / 1749 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1749, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1749, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1749, 1, 576) torch.bfloat16 + out : (1, 1749) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2287 + top-1 accuracy : 25/388 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1655]) +Dataloader batch - labels shape: torch.Size([1, 1655]) +Dataloader batch - attn_mask shape: torch.Size([1, 1655]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 196 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1655) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1655) torch.int64 + loss_mask : 378 / 1655 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1655, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1655, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1655, 1, 576) torch.bfloat16 + out : (1, 1655) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 378 tokens) : 11.3206 + top-1 accuracy : 24/378 = 6.3% + + [2026-04-29 13:41:33] iteration 149/ 500 | consumed samples: 596 | elapsed time per iteration (ms): 6897.5 | throughput (token/sec/GPU): 966.6 | learning rate: 9.775690E-05 | global batch size: 4 | lm loss: 1.130273E+01 | loss scale: 1.0 | grad norm: 0.296 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1531]) +Dataloader batch - labels shape: torch.Size([1, 1531]) +Dataloader batch - attn_mask shape: torch.Size([1, 1531]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 197 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1531) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1531) torch.int64 + loss_mask : 382 / 1531 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1531, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1531, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1531, 1, 576) torch.bfloat16 + out : (1, 1531) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.3077 + top-1 accuracy : 18/382 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1870]) +Dataloader batch - labels shape: torch.Size([1, 1870]) +Dataloader batch - attn_mask shape: torch.Size([1, 1870]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 198 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1870) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1870) torch.int64 + loss_mask : 520 / 1870 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1870, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1870, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1870, 1, 576) torch.bfloat16 + out : (1, 1870) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 520 tokens) : 11.3502 + top-1 accuracy : 3/520 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1138]) +Dataloader batch - labels shape: torch.Size([1, 1138]) +Dataloader batch - attn_mask shape: torch.Size([1, 1138]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 199 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1138) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1138) torch.int64 + loss_mask : 328 / 1138 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1138, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1138, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1138, 1, 576) torch.bfloat16 + out : (1, 1138) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.3422 + top-1 accuracy : 2/328 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1396]) +Dataloader batch - labels shape: torch.Size([1, 1396]) +Dataloader batch - attn_mask shape: torch.Size([1, 1396]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 200 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1396) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1396) torch.int64 + loss_mask : 316 / 1396 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1396, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1396, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1396, 1, 576) torch.bfloat16 + out : (1, 1396) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.2467 + top-1 accuracy : 27/316 = 8.5% + + [2026-04-29 13:41:39] iteration 150/ 500 | consumed samples: 600 | elapsed time per iteration (ms): 6135.0 | throughput (token/sec/GPU): 967.4 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.131686E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 150 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 150 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 150 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1894.41, 1894.41) +Dataloader batch - tokens shape: torch.Size([1, 1705]) +Dataloader batch - labels shape: torch.Size([1, 1705]) +Dataloader batch - attn_mask shape: torch.Size([1, 1705]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 201 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1705) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1705) torch.int64 + loss_mask : 357 / 1705 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1705, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1705, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1705, 1, 576) torch.bfloat16 + out : (1, 1705) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.1504 + top-1 accuracy : 34/357 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1012]) +Dataloader batch - labels shape: torch.Size([1, 1012]) +Dataloader batch - attn_mask shape: torch.Size([1, 1012]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 202 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1012) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1012) torch.int64 + loss_mask : 261 / 1012 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1012, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1012, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1012, 1, 576) torch.bfloat16 + out : (1, 1012) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 261 tokens) : 11.2761 + top-1 accuracy : 3/261 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1557]) +Dataloader batch - labels shape: torch.Size([1, 1557]) +Dataloader batch - attn_mask shape: torch.Size([1, 1557]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 203 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1557) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1557) torch.int64 + loss_mask : 436 / 1557 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1557, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1557, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1557, 1, 576) torch.bfloat16 + out : (1, 1557) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.3434 + top-1 accuracy : 29/436 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1373]) +Dataloader batch - labels shape: torch.Size([1, 1373]) +Dataloader batch - attn_mask shape: torch.Size([1, 1373]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 204 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1373) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1373) torch.int64 + loss_mask : 312 / 1373 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1373, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1373, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1373, 1, 576) torch.bfloat16 + out : (1, 1373) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.2506 + top-1 accuracy : 17/312 = 5.4% + + [2026-04-29 13:41:47] iteration 151/ 500 | consumed samples: 604 | elapsed time per iteration (ms): 5973.0 | throughput (token/sec/GPU): 945.4 | learning rate: 9.756766E-05 | global batch size: 4 | lm loss: 1.125890E+01 | loss scale: 1.0 | grad norm: 0.309 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1426]) +Dataloader batch - labels shape: torch.Size([1, 1426]) +Dataloader batch - attn_mask shape: torch.Size([1, 1426]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 205 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1426) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1426) torch.int64 + loss_mask : 369 / 1426 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1426, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1426, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1426, 1, 576) torch.bfloat16 + out : (1, 1426) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2960 + top-1 accuracy : 21/369 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1203]) +Dataloader batch - labels shape: torch.Size([1, 1203]) +Dataloader batch - attn_mask shape: torch.Size([1, 1203]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 206 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1203) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1203) torch.int64 + loss_mask : 307 / 1203 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1203, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1203, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1203, 1, 576) torch.bfloat16 + out : (1, 1203) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.2837 + top-1 accuracy : 7/307 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1707]) +Dataloader batch - labels shape: torch.Size([1, 1707]) +Dataloader batch - attn_mask shape: torch.Size([1, 1707]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 207 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1707) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1707) torch.int64 + loss_mask : 351 / 1707 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1707, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1707, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1707, 1, 576) torch.bfloat16 + out : (1, 1707) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.1582 + top-1 accuracy : 38/351 = 10.8% + +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 208 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1423) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1423) torch.int64 + loss_mask : 317 / 1423 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1423, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1423, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1423, 1, 576) torch.bfloat16 + out : (1, 1423) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2470 + top-1 accuracy : 10/317 = 3.2% + + [2026-04-29 13:41:52] iteration 152/ 500 | consumed samples: 608 | elapsed time per iteration (ms): 4608.9 | throughput (token/sec/GPU): 1249.5 | learning rate: 9.747023E-05 | global batch size: 4 | lm loss: 1.124564E+01 | loss scale: 1.0 | grad norm: 0.297 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 209 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 351 / 1532 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.2095 + top-1 accuracy : 19/351 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 210 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1385) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1385) torch.int64 + loss_mask : 311 / 1385 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1385, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1385, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1385, 1, 576) torch.bfloat16 + out : (1, 1385) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2142 + top-1 accuracy : 13/311 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1465]) +Dataloader batch - labels shape: torch.Size([1, 1465]) +Dataloader batch - attn_mask shape: torch.Size([1, 1465]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 211 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1465) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1465) torch.int64 + loss_mask : 412 / 1465 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1465, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1465, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1465, 1, 576) torch.bfloat16 + out : (1, 1465) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3653 + top-1 accuracy : 2/412 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1057]) +Dataloader batch - labels shape: torch.Size([1, 1057]) +Dataloader batch - attn_mask shape: torch.Size([1, 1057]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 212 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1057) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1057) torch.int64 + loss_mask : 288 / 1057 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1057, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1057, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1057, 1, 576) torch.bfloat16 + out : (1, 1057) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 288 tokens) : 11.3375 + top-1 accuracy : 9/288 = 3.1% + + [2026-04-29 13:41:56] iteration 153/ 500 | consumed samples: 612 | elapsed time per iteration (ms): 4375.9 | throughput (token/sec/GPU): 1242.9 | learning rate: 9.737095E-05 | global batch size: 4 | lm loss: 1.128477E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1464]) +Dataloader batch - labels shape: torch.Size([1, 1464]) +Dataloader batch - attn_mask shape: torch.Size([1, 1464]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 213 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1464) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1464) torch.int64 + loss_mask : 412 / 1464 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1464, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1464, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1464, 1, 576) torch.bfloat16 + out : (1, 1464) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3665 + top-1 accuracy : 12/412 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 214 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 370 / 1504 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.2783 + top-1 accuracy : 0/370 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1403]) +Dataloader batch - labels shape: torch.Size([1, 1403]) +Dataloader batch - attn_mask shape: torch.Size([1, 1403]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 215 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1403) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1403) torch.int64 + loss_mask : 354 / 1403 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1403, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1403, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1403, 1, 576) torch.bfloat16 + out : (1, 1403) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.3243 + top-1 accuracy : 17/354 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1571]) +Dataloader batch - labels shape: torch.Size([1, 1571]) +Dataloader batch - attn_mask shape: torch.Size([1, 1571]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 216 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1571) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1571) torch.int64 + loss_mask : 354 / 1571 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1571, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1571, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1571, 1, 576) torch.bfloat16 + out : (1, 1571) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.1740 + top-1 accuracy : 36/354 = 10.2% + + [2026-04-29 13:42:01] iteration 154/ 500 | consumed samples: 616 | elapsed time per iteration (ms): 4675.6 | throughput (token/sec/GPU): 1270.8 | learning rate: 9.726981E-05 | global batch size: 4 | lm loss: 1.128883E+01 | loss scale: 1.0 | grad norm: 0.315 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1465]) +Dataloader batch - labels shape: torch.Size([1, 1465]) +Dataloader batch - attn_mask shape: torch.Size([1, 1465]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 217 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1465) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1465) torch.int64 + loss_mask : 292 / 1465 tokens (19.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1465, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1465, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1465, 1, 576) torch.bfloat16 + out : (1, 1465) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 292 tokens) : 11.1569 + top-1 accuracy : 30/292 = 10.3% + +Dataloader batch - tokens shape: torch.Size([1, 1562]) +Dataloader batch - labels shape: torch.Size([1, 1562]) +Dataloader batch - attn_mask shape: torch.Size([1, 1562]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 218 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1562) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1562) torch.int64 + loss_mask : 413 / 1562 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1562, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1562, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1562, 1, 576) torch.bfloat16 + out : (1, 1562) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.3073 + top-1 accuracy : 25/413 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1821]) +Dataloader batch - labels shape: torch.Size([1, 1821]) +Dataloader batch - attn_mask shape: torch.Size([1, 1821]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 219 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1821) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1821) torch.int64 + loss_mask : 472 / 1821 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1821, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1821, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1821, 1, 576) torch.bfloat16 + out : (1, 1821) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 472 tokens) : 11.2717 + top-1 accuracy : 3/472 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1831]) +Dataloader batch - labels shape: torch.Size([1, 1831]) +Dataloader batch - attn_mask shape: torch.Size([1, 1831]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 220 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1831) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1831) torch.int64 + loss_mask : 460 / 1831 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1831, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1831, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1831, 1, 576) torch.bfloat16 + out : (1, 1831) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 460 tokens) : 11.3079 + top-1 accuracy : 10/460 = 2.2% + + [2026-04-29 13:42:06] iteration 155/ 500 | consumed samples: 620 | elapsed time per iteration (ms): 5185.7 | throughput (token/sec/GPU): 1288.0 | learning rate: 9.716682E-05 | global batch size: 4 | lm loss: 1.127035E+01 | loss scale: 1.0 | grad norm: 0.294 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 221 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 393 / 1750 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 393 tokens) : 11.2208 + top-1 accuracy : 27/393 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1737]) +Dataloader batch - labels shape: torch.Size([1, 1737]) +Dataloader batch - attn_mask shape: torch.Size([1, 1737]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 222 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1737) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1737) torch.int64 + loss_mask : 367 / 1737 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1737, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1737, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1737, 1, 576) torch.bfloat16 + out : (1, 1737) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2263 + top-1 accuracy : 35/367 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1398]) +Dataloader batch - labels shape: torch.Size([1, 1398]) +Dataloader batch - attn_mask shape: torch.Size([1, 1398]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 223 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1398) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1398) torch.int64 + loss_mask : 326 / 1398 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1398, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1398, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1398, 1, 576) torch.bfloat16 + out : (1, 1398) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2029 + top-1 accuracy : 7/326 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 224 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 359 / 1422 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.3056 + top-1 accuracy : 17/359 = 4.7% + + [2026-04-29 13:42:11] iteration 156/ 500 | consumed samples: 624 | elapsed time per iteration (ms): 4904.8 | throughput (token/sec/GPU): 1285.9 | learning rate: 9.706197E-05 | global batch size: 4 | lm loss: 1.123921E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1244]) +Dataloader batch - labels shape: torch.Size([1, 1244]) +Dataloader batch - attn_mask shape: torch.Size([1, 1244]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 225 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1244) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1244) torch.int64 + loss_mask : 361 / 1244 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1244, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1244, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1244, 1, 576) torch.bfloat16 + out : (1, 1244) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.3676 + top-1 accuracy : 1/361 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1408]) +Dataloader batch - labels shape: torch.Size([1, 1408]) +Dataloader batch - attn_mask shape: torch.Size([1, 1408]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 226 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1408) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1408) torch.int64 + loss_mask : 339 / 1408 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1408, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1408, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1408, 1, 576) torch.bfloat16 + out : (1, 1408) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2908 + top-1 accuracy : 25/339 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 227 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 357 / 1520 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2839 + top-1 accuracy : 4/357 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1583]) +Dataloader batch - labels shape: torch.Size([1, 1583]) +Dataloader batch - attn_mask shape: torch.Size([1, 1583]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 228 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1583) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1583) torch.int64 + loss_mask : 429 / 1583 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1583, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1583, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1583, 1, 576) torch.bfloat16 + out : (1, 1583) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 429 tokens) : 11.3458 + top-1 accuracy : 6/429 = 1.4% + + [2026-04-29 13:42:15] iteration 157/ 500 | consumed samples: 628 | elapsed time per iteration (ms): 4460.2 | throughput (token/sec/GPU): 1290.3 | learning rate: 9.695529E-05 | global batch size: 4 | lm loss: 1.132369E+01 | loss scale: 1.0 | grad norm: 0.334 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1164]) +Dataloader batch - labels shape: torch.Size([1, 1164]) +Dataloader batch - attn_mask shape: torch.Size([1, 1164]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 229 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1164) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1164) torch.int64 + loss_mask : 311 / 1164 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1164, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1164, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1164, 1, 576) torch.bfloat16 + out : (1, 1164) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.3596 + top-1 accuracy : 14/311 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 230 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1385) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1385) torch.int64 + loss_mask : 311 / 1385 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1385, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1385, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1385, 1, 576) torch.bfloat16 + out : (1, 1385) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2497 + top-1 accuracy : 10/311 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 231 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 440 / 1431 tokens (30.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 440 tokens) : 11.4447 + top-1 accuracy : 2/440 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1115]) +Dataloader batch - labels shape: torch.Size([1, 1115]) +Dataloader batch - attn_mask shape: torch.Size([1, 1115]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 232 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1115) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1115) torch.int64 + loss_mask : 275 / 1115 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1115, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1115, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1115, 1, 576) torch.bfloat16 + out : (1, 1115) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 275 tokens) : 11.2796 + top-1 accuracy : 21/275 = 7.6% + + [2026-04-29 13:42:19] iteration 158/ 500 | consumed samples: 632 | elapsed time per iteration (ms): 4117.9 | throughput (token/sec/GPU): 1237.3 | learning rate: 9.684676E-05 | global batch size: 4 | lm loss: 1.134556E+01 | loss scale: 1.0 | grad norm: 0.317 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1625]) +Dataloader batch - labels shape: torch.Size([1, 1625]) +Dataloader batch - attn_mask shape: torch.Size([1, 1625]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 233 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1625) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1625) torch.int64 + loss_mask : 353 / 1625 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1625, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1625, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1625, 1, 576) torch.bfloat16 + out : (1, 1625) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.2092 + top-1 accuracy : 30/353 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1221]) +Dataloader batch - labels shape: torch.Size([1, 1221]) +Dataloader batch - attn_mask shape: torch.Size([1, 1221]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 234 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1221) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1221) torch.int64 + loss_mask : 290 / 1221 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1221, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1221, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1221, 1, 576) torch.bfloat16 + out : (1, 1221) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.2792 + top-1 accuracy : 19/290 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1775]) +Dataloader batch - labels shape: torch.Size([1, 1775]) +Dataloader batch - attn_mask shape: torch.Size([1, 1775]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 235 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1775) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1775) torch.int64 + loss_mask : 412 / 1775 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1775, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1775, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1775, 1, 576) torch.bfloat16 + out : (1, 1775) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.2117 + top-1 accuracy : 19/412 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1259]) +Dataloader batch - labels shape: torch.Size([1, 1259]) +Dataloader batch - attn_mask shape: torch.Size([1, 1259]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 236 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1259) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1259) torch.int64 + loss_mask : 367 / 1259 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1259, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1259, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1259, 1, 576) torch.bfloat16 + out : (1, 1259) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.3836 + top-1 accuracy : 6/367 = 1.6% + + [2026-04-29 13:42:24] iteration 159/ 500 | consumed samples: 636 | elapsed time per iteration (ms): 4585.2 | throughput (token/sec/GPU): 1282.4 | learning rate: 9.673639E-05 | global batch size: 4 | lm loss: 1.126923E+01 | loss scale: 1.0 | grad norm: 0.266 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1447]) +Dataloader batch - labels shape: torch.Size([1, 1447]) +Dataloader batch - attn_mask shape: torch.Size([1, 1447]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 237 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1447) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1447) torch.int64 + loss_mask : 391 / 1447 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1447, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1447, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1447, 1, 576) torch.bfloat16 + out : (1, 1447) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3101 + top-1 accuracy : 7/391 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1530]) +Dataloader batch - labels shape: torch.Size([1, 1530]) +Dataloader batch - attn_mask shape: torch.Size([1, 1530]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 238 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1530) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1530) torch.int64 + loss_mask : 342 / 1530 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1530, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1530, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1530, 1, 576) torch.bfloat16 + out : (1, 1530) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2195 + top-1 accuracy : 32/342 = 9.4% + +Dataloader batch - tokens shape: torch.Size([1, 1408]) +Dataloader batch - labels shape: torch.Size([1, 1408]) +Dataloader batch - attn_mask shape: torch.Size([1, 1408]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 239 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1408) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1408) torch.int64 + loss_mask : 300 / 1408 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1408, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1408, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1408, 1, 576) torch.bfloat16 + out : (1, 1408) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.1858 + top-1 accuracy : 10/300 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1497]) +Dataloader batch - labels shape: torch.Size([1, 1497]) +Dataloader batch - attn_mask shape: torch.Size([1, 1497]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 240 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1497) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1497) torch.int64 + loss_mask : 448 / 1497 tokens (29.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1497, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1497, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1497, 1, 576) torch.bfloat16 + out : (1, 1497) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 448 tokens) : 11.3919 + top-1 accuracy : 7/448 = 1.6% + + [2026-04-29 13:42:28] iteration 160/ 500 | consumed samples: 640 | elapsed time per iteration (ms): 4524.2 | throughput (token/sec/GPU): 1300.1 | learning rate: 9.662419E-05 | global batch size: 4 | lm loss: 1.128874E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4508.89, 4508.89) + forward-compute ................................: (3366.43, 3366.43) + backward-compute ...............................: (1139.48, 1139.48) + batch-generator ................................: (415.21, 415.21) + layernorm-grads-all-reduce .....................: (0.04, 0.04) + embedding-grads-all-reduce .....................: (0.07, 0.07) + all-grads-sync .................................: (0.74, 0.74) + params-all-gather ..............................: (0.19, 0.19) + optimizer-copy-to-main-grad ....................: (0.26, 0.26) + optimizer-clip-main-grad .......................: (0.93, 0.93) + optimizer-count-zeros ..........................: (0.03, 0.03) + optimizer-inner-step ...........................: (1.51, 1.51) + optimizer-copy-main-to-model-params ............: (0.29, 0.29) + optimizer ......................................: (4.29, 4.29) +Dataloader batch - tokens shape: torch.Size([1, 914]) +Dataloader batch - labels shape: torch.Size([1, 914]) +Dataloader batch - attn_mask shape: torch.Size([1, 914]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 241 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 914) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 914) torch.int64 + loss_mask : 184 / 914 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (914, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (914, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (914, 1, 576) torch.bfloat16 + out : (1, 914) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 184 tokens) : 11.1975 + top-1 accuracy : 6/184 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1613]) +Dataloader batch - labels shape: torch.Size([1, 1613]) +Dataloader batch - attn_mask shape: torch.Size([1, 1613]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 242 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1613) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1613) torch.int64 + loss_mask : 387 / 1613 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1613, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1613, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1613, 1, 576) torch.bfloat16 + out : (1, 1613) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2886 + top-1 accuracy : 21/387 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1915]) +Dataloader batch - labels shape: torch.Size([1, 1915]) +Dataloader batch - attn_mask shape: torch.Size([1, 1915]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 243 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1915) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1915) torch.int64 + loss_mask : 565 / 1915 tokens (29.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1915, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1915, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1915, 1, 576) torch.bfloat16 + out : (1, 1915) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 565 tokens) : 11.3847 + top-1 accuracy : 10/565 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1564]) +Dataloader batch - labels shape: torch.Size([1, 1564]) +Dataloader batch - attn_mask shape: torch.Size([1, 1564]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 244 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1564) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1564) torch.int64 + loss_mask : 411 / 1564 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1564, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1564, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1564, 1, 576) torch.bfloat16 + out : (1, 1564) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.3499 + top-1 accuracy : 24/411 = 5.8% + + [2026-04-29 13:42:33] iteration 161/ 500 | consumed samples: 644 | elapsed time per iteration (ms): 4611.3 | throughput (token/sec/GPU): 1302.5 | learning rate: 9.651016E-05 | global batch size: 4 | lm loss: 1.132917E+01 | loss scale: 1.0 | grad norm: 0.413 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1517]) +Dataloader batch - labels shape: torch.Size([1, 1517]) +Dataloader batch - attn_mask shape: torch.Size([1, 1517]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 245 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1517) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1517) torch.int64 + loss_mask : 335 / 1517 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1517, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1517, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1517, 1, 576) torch.bfloat16 + out : (1, 1517) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2188 + top-1 accuracy : 30/335 = 9.0% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 246 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 307 / 1400 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.1862 + top-1 accuracy : 28/307 = 9.1% + +Dataloader batch - tokens shape: torch.Size([1, 1360]) +Dataloader batch - labels shape: torch.Size([1, 1360]) +Dataloader batch - attn_mask shape: torch.Size([1, 1360]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 247 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1360) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1360) torch.int64 + loss_mask : 277 / 1360 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1360, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1360, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1360, 1, 576) torch.bfloat16 + out : (1, 1360) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 277 tokens) : 11.2347 + top-1 accuracy : 32/277 = 11.6% + +Dataloader batch - tokens shape: torch.Size([1, 963]) +Dataloader batch - labels shape: torch.Size([1, 963]) +Dataloader batch - attn_mask shape: torch.Size([1, 963]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 248 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 963) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 963) torch.int64 + loss_mask : 210 / 963 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (963, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (963, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (963, 1, 576) torch.bfloat16 + out : (1, 963) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 210 tokens) : 11.2188 + top-1 accuracy : 25/210 = 11.9% + + [2026-04-29 13:42:37] iteration 162/ 500 | consumed samples: 648 | elapsed time per iteration (ms): 4426.6 | throughput (token/sec/GPU): 1183.7 | learning rate: 9.639430E-05 | global batch size: 4 | lm loss: 1.121383E+01 | loss scale: 1.0 | grad norm: 0.381 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1537]) +Dataloader batch - labels shape: torch.Size([1, 1537]) +Dataloader batch - attn_mask shape: torch.Size([1, 1537]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 249 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1537) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1537) torch.int64 + loss_mask : 317 / 1537 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1537, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1537, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1537, 1, 576) torch.bfloat16 + out : (1, 1537) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.1850 + top-1 accuracy : 24/317 = 7.6% + +Dataloader batch - tokens shape: torch.Size([1, 1368]) +Dataloader batch - labels shape: torch.Size([1, 1368]) +Dataloader batch - attn_mask shape: torch.Size([1, 1368]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 250 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1368) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1368) torch.int64 + loss_mask : 317 / 1368 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1368, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1368, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1368, 1, 576) torch.bfloat16 + out : (1, 1368) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2662 + top-1 accuracy : 21/317 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1232]) +Dataloader batch - labels shape: torch.Size([1, 1232]) +Dataloader batch - attn_mask shape: torch.Size([1, 1232]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 251 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1232) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1232) torch.int64 + loss_mask : 317 / 1232 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1232, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1232, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1232, 1, 576) torch.bfloat16 + out : (1, 1232) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.3233 + top-1 accuracy : 23/317 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1598]) +Dataloader batch - labels shape: torch.Size([1, 1598]) +Dataloader batch - attn_mask shape: torch.Size([1, 1598]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 252 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1598) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1598) torch.int64 + loss_mask : 353 / 1598 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1598, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1598, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1598, 1, 576) torch.bfloat16 + out : (1, 1598) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.1798 + top-1 accuracy : 30/353 = 8.5% + + [2026-04-29 13:42:42] iteration 163/ 500 | consumed samples: 652 | elapsed time per iteration (ms): 4674.9 | throughput (token/sec/GPU): 1226.8 | learning rate: 9.627664E-05 | global batch size: 4 | lm loss: 1.123696E+01 | loss scale: 1.0 | grad norm: 0.285 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 253 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1385) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1385) torch.int64 + loss_mask : 321 / 1385 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1385, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1385, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1385, 1, 576) torch.bfloat16 + out : (1, 1385) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2612 + top-1 accuracy : 7/321 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 254 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 364 / 1542 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2835 + top-1 accuracy : 18/364 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1578]) +Dataloader batch - labels shape: torch.Size([1, 1578]) +Dataloader batch - attn_mask shape: torch.Size([1, 1578]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 255 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1578) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1578) torch.int64 + loss_mask : 368 / 1578 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1578, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1578, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1578, 1, 576) torch.bfloat16 + out : (1, 1578) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2166 + top-1 accuracy : 32/368 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1107]) +Dataloader batch - labels shape: torch.Size([1, 1107]) +Dataloader batch - attn_mask shape: torch.Size([1, 1107]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 256 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1107) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1107) torch.int64 + loss_mask : 300 / 1107 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1107, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1107, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1107, 1, 576) torch.bfloat16 + out : (1, 1107) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.4149 + top-1 accuracy : 19/300 = 6.3% + + [2026-04-29 13:42:46] iteration 164/ 500 | consumed samples: 656 | elapsed time per iteration (ms): 4335.2 | throughput (token/sec/GPU): 1294.5 | learning rate: 9.615714E-05 | global batch size: 4 | lm loss: 1.128914E+01 | loss scale: 1.0 | grad norm: 0.300 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1483]) +Dataloader batch - labels shape: torch.Size([1, 1483]) +Dataloader batch - attn_mask shape: torch.Size([1, 1483]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 257 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1483) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1483) torch.int64 + loss_mask : 355 / 1483 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1483, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1483, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1483, 1, 576) torch.bfloat16 + out : (1, 1483) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2968 + top-1 accuracy : 17/355 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1578]) +Dataloader batch - labels shape: torch.Size([1, 1578]) +Dataloader batch - attn_mask shape: torch.Size([1, 1578]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 258 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1578) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1578) torch.int64 + loss_mask : 352 / 1578 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1578, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1578, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1578, 1, 576) torch.bfloat16 + out : (1, 1578) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.2530 + top-1 accuracy : 36/352 = 10.2% + +Dataloader batch - tokens shape: torch.Size([1, 1395]) +Dataloader batch - labels shape: torch.Size([1, 1395]) +Dataloader batch - attn_mask shape: torch.Size([1, 1395]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 259 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1395) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1395) torch.int64 + loss_mask : 318 / 1395 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1395, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1395, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1395, 1, 576) torch.bfloat16 + out : (1, 1395) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.1702 + top-1 accuracy : 29/318 = 9.1% + +Dataloader batch - tokens shape: torch.Size([1, 1131]) +Dataloader batch - labels shape: torch.Size([1, 1131]) +Dataloader batch - attn_mask shape: torch.Size([1, 1131]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 260 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1131) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1131) torch.int64 + loss_mask : 289 / 1131 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1131, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1131, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1131, 1, 576) torch.bfloat16 + out : (1, 1131) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 289 tokens) : 11.3129 + top-1 accuracy : 3/289 = 1.0% + + [2026-04-29 13:42:51] iteration 165/ 500 | consumed samples: 660 | elapsed time per iteration (ms): 4643.0 | throughput (token/sec/GPU): 1203.3 | learning rate: 9.603585E-05 | global batch size: 4 | lm loss: 1.125794E+01 | loss scale: 1.0 | grad norm: 0.319 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1799]) +Dataloader batch - labels shape: torch.Size([1, 1799]) +Dataloader batch - attn_mask shape: torch.Size([1, 1799]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 261 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1799) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1799) torch.int64 + loss_mask : 445 / 1799 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1799, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1799, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1799, 1, 576) torch.bfloat16 + out : (1, 1799) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 445 tokens) : 11.2735 + top-1 accuracy : 19/445 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1399]) +Dataloader batch - labels shape: torch.Size([1, 1399]) +Dataloader batch - attn_mask shape: torch.Size([1, 1399]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 262 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1399) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1399) torch.int64 + loss_mask : 338 / 1399 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1399, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1399, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1399, 1, 576) torch.bfloat16 + out : (1, 1399) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.2952 + top-1 accuracy : 9/338 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1540]) +Dataloader batch - labels shape: torch.Size([1, 1540]) +Dataloader batch - attn_mask shape: torch.Size([1, 1540]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 263 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1540) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1540) torch.int64 + loss_mask : 357 / 1540 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1540, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1540, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1540, 1, 576) torch.bfloat16 + out : (1, 1540) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2358 + top-1 accuracy : 15/357 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1146]) +Dataloader batch - labels shape: torch.Size([1, 1146]) +Dataloader batch - attn_mask shape: torch.Size([1, 1146]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 264 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1146) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1146) torch.int64 + loss_mask : 367 / 1146 tokens (32.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1146, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1146, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1146, 1, 576) torch.bfloat16 + out : (1, 1146) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.4037 + top-1 accuracy : 16/367 = 4.4% + + [2026-04-29 13:42:56] iteration 166/ 500 | consumed samples: 664 | elapsed time per iteration (ms): 4594.9 | throughput (token/sec/GPU): 1280.6 | learning rate: 9.591275E-05 | global batch size: 4 | lm loss: 1.130115E+01 | loss scale: 1.0 | grad norm: 0.308 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 949]) +Dataloader batch - labels shape: torch.Size([1, 949]) +Dataloader batch - attn_mask shape: torch.Size([1, 949]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 265 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 949) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 949) torch.int64 + loss_mask : 215 / 949 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (949, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (949, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (949, 1, 576) torch.bfloat16 + out : (1, 949) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 215 tokens) : 11.2074 + top-1 accuracy : 22/215 = 10.2% + +Dataloader batch - tokens shape: torch.Size([1, 1691]) +Dataloader batch - labels shape: torch.Size([1, 1691]) +Dataloader batch - attn_mask shape: torch.Size([1, 1691]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 266 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1691) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1691) torch.int64 + loss_mask : 350 / 1691 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1691, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1691, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1691, 1, 576) torch.bfloat16 + out : (1, 1691) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.1673 + top-1 accuracy : 38/350 = 10.9% + +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 267 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 371 / 1434 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.3048 + top-1 accuracy : 19/371 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1914]) +Dataloader batch - labels shape: torch.Size([1, 1914]) +Dataloader batch - attn_mask shape: torch.Size([1, 1914]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 268 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1914) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1914) torch.int64 + loss_mask : 535 / 1914 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1914, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1914, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1914, 1, 576) torch.bfloat16 + out : (1, 1914) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 535 tokens) : 11.3702 + top-1 accuracy : 4/535 = 0.7% + + [2026-04-29 13:43:00] iteration 167/ 500 | consumed samples: 668 | elapsed time per iteration (ms): 4645.7 | throughput (token/sec/GPU): 1288.9 | learning rate: 9.578785E-05 | global batch size: 4 | lm loss: 1.128162E+01 | loss scale: 1.0 | grad norm: 0.332 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1731]) +Dataloader batch - labels shape: torch.Size([1, 1731]) +Dataloader batch - attn_mask shape: torch.Size([1, 1731]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 269 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1731) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1731) torch.int64 + loss_mask : 386 / 1731 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1731, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1731, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1731, 1, 576) torch.bfloat16 + out : (1, 1731) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2664 + top-1 accuracy : 41/386 = 10.6% + +Dataloader batch - tokens shape: torch.Size([1, 1530]) +Dataloader batch - labels shape: torch.Size([1, 1530]) +Dataloader batch - attn_mask shape: torch.Size([1, 1530]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 270 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1530) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1530) torch.int64 + loss_mask : 325 / 1530 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1530, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1530, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1530, 1, 576) torch.bfloat16 + out : (1, 1530) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.1774 + top-1 accuracy : 17/325 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1077]) +Dataloader batch - labels shape: torch.Size([1, 1077]) +Dataloader batch - attn_mask shape: torch.Size([1, 1077]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 271 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1077) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1077) torch.int64 + loss_mask : 250 / 1077 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1077, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1077, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1077, 1, 576) torch.bfloat16 + out : (1, 1077) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 250 tokens) : 11.2262 + top-1 accuracy : 6/250 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1367]) +Dataloader batch - labels shape: torch.Size([1, 1367]) +Dataloader batch - attn_mask shape: torch.Size([1, 1367]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 272 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1367) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1367) torch.int64 + loss_mask : 306 / 1367 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1367, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1367, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1367, 1, 576) torch.bfloat16 + out : (1, 1367) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.2776 + top-1 accuracy : 12/306 = 3.9% + + [2026-04-29 13:43:05] iteration 168/ 500 | consumed samples: 672 | elapsed time per iteration (ms): 4702.6 | throughput (token/sec/GPU): 1213.2 | learning rate: 9.566115E-05 | global batch size: 4 | lm loss: 1.123833E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1878]) +Dataloader batch - labels shape: torch.Size([1, 1878]) +Dataloader batch - attn_mask shape: torch.Size([1, 1878]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 273 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1878) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1878) torch.int64 + loss_mask : 526 / 1878 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1878, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1878, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1878, 1, 576) torch.bfloat16 + out : (1, 1878) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 526 tokens) : 11.3863 + top-1 accuracy : 5/526 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1195]) +Dataloader batch - labels shape: torch.Size([1, 1195]) +Dataloader batch - attn_mask shape: torch.Size([1, 1195]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 274 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1195) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1195) torch.int64 + loss_mask : 298 / 1195 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1195, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1195, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1195, 1, 576) torch.bfloat16 + out : (1, 1195) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 298 tokens) : 11.3107 + top-1 accuracy : 12/298 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1212]) +Dataloader batch - labels shape: torch.Size([1, 1212]) +Dataloader batch - attn_mask shape: torch.Size([1, 1212]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 275 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1212) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1212) torch.int64 + loss_mask : 322 / 1212 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1212, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1212, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1212, 1, 576) torch.bfloat16 + out : (1, 1212) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.3386 + top-1 accuracy : 9/322 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 276 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 369 / 1518 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.3247 + top-1 accuracy : 23/369 = 6.2% + + [2026-04-29 13:43:10] iteration 169/ 500 | consumed samples: 676 | elapsed time per iteration (ms): 4537.1 | throughput (token/sec/GPU): 1279.0 | learning rate: 9.553267E-05 | global batch size: 4 | lm loss: 1.134629E+01 | loss scale: 1.0 | grad norm: 0.325 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1726]) +Dataloader batch - labels shape: torch.Size([1, 1726]) +Dataloader batch - attn_mask shape: torch.Size([1, 1726]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 277 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1726) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1726) torch.int64 + loss_mask : 373 / 1726 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1726, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1726, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1726, 1, 576) torch.bfloat16 + out : (1, 1726) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.2296 + top-1 accuracy : 40/373 = 10.7% + +Dataloader batch - tokens shape: torch.Size([1, 1440]) +Dataloader batch - labels shape: torch.Size([1, 1440]) +Dataloader batch - attn_mask shape: torch.Size([1, 1440]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 278 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1440) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1440) torch.int64 + loss_mask : 378 / 1440 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1440, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1440, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1440, 1, 576) torch.bfloat16 + out : (1, 1440) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 378 tokens) : 11.3160 + top-1 accuracy : 6/378 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1180]) +Dataloader batch - labels shape: torch.Size([1, 1180]) +Dataloader batch - attn_mask shape: torch.Size([1, 1180]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 279 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1180) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1180) torch.int64 + loss_mask : 262 / 1180 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1180, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1180, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1180, 1, 576) torch.bfloat16 + out : (1, 1180) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 262 tokens) : 11.1533 + top-1 accuracy : 8/262 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1550]) +Dataloader batch - labels shape: torch.Size([1, 1550]) +Dataloader batch - attn_mask shape: torch.Size([1, 1550]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 280 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1550) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1550) torch.int64 + loss_mask : 418 / 1550 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1550, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1550, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1550, 1, 576) torch.bfloat16 + out : (1, 1550) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 418 tokens) : 11.3402 + top-1 accuracy : 23/418 = 5.5% + + [2026-04-29 13:43:14] iteration 170/ 500 | consumed samples: 680 | elapsed time per iteration (ms): 4758.4 | throughput (token/sec/GPU): 1239.1 | learning rate: 9.540239E-05 | global batch size: 4 | lm loss: 1.127076E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1482]) +Dataloader batch - labels shape: torch.Size([1, 1482]) +Dataloader batch - attn_mask shape: torch.Size([1, 1482]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 281 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1482) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1482) torch.int64 + loss_mask : 345 / 1482 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1482, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1482, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1482, 1, 576) torch.bfloat16 + out : (1, 1482) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2699 + top-1 accuracy : 9/345 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 282 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1412) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1412) torch.int64 + loss_mask : 370 / 1412 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1412, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1412, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1412, 1, 576) torch.bfloat16 + out : (1, 1412) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.3071 + top-1 accuracy : 12/370 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1889]) +Dataloader batch - labels shape: torch.Size([1, 1889]) +Dataloader batch - attn_mask shape: torch.Size([1, 1889]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 283 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1889) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1889) torch.int64 + loss_mask : 533 / 1889 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1889, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1889, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1889, 1, 576) torch.bfloat16 + out : (1, 1889) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 533 tokens) : 11.4134 + top-1 accuracy : 8/533 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 284 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 353 / 1539 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.2026 + top-1 accuracy : 2/353 = 0.6% + + [2026-04-29 13:43:19] iteration 171/ 500 | consumed samples: 684 | elapsed time per iteration (ms): 4926.0 | throughput (token/sec/GPU): 1283.4 | learning rate: 9.527035E-05 | global batch size: 4 | lm loss: 1.131145E+01 | loss scale: 1.0 | grad norm: 0.252 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1266]) +Dataloader batch - labels shape: torch.Size([1, 1266]) +Dataloader batch - attn_mask shape: torch.Size([1, 1266]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 285 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1266) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1266) torch.int64 + loss_mask : 376 / 1266 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1266, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1266, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1266, 1, 576) torch.bfloat16 + out : (1, 1266) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3827 + top-1 accuracy : 1/376 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1945]) +Dataloader batch - labels shape: torch.Size([1, 1945]) +Dataloader batch - attn_mask shape: torch.Size([1, 1945]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 286 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1945) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1945) torch.int64 + loss_mask : 587 / 1945 tokens (30.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1945, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1945, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1945, 1, 576) torch.bfloat16 + out : (1, 1945) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 587 tokens) : 11.4183 + top-1 accuracy : 25/587 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1530]) +Dataloader batch - labels shape: torch.Size([1, 1530]) +Dataloader batch - attn_mask shape: torch.Size([1, 1530]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 287 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1530) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1530) torch.int64 + loss_mask : 346 / 1530 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1530, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1530, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1530, 1, 576) torch.bfloat16 + out : (1, 1530) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.2267 + top-1 accuracy : 26/346 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 288 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 372 / 1722 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2020 + top-1 accuracy : 15/372 = 4.0% + + [2026-04-29 13:43:24] iteration 172/ 500 | consumed samples: 688 | elapsed time per iteration (ms): 4828.1 | throughput (token/sec/GPU): 1338.6 | learning rate: 9.513652E-05 | global batch size: 4 | lm loss: 1.132302E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 289 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 316 / 1520 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.2010 + top-1 accuracy : 18/316 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1392]) +Dataloader batch - labels shape: torch.Size([1, 1392]) +Dataloader batch - attn_mask shape: torch.Size([1, 1392]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 290 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1392) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1392) torch.int64 + loss_mask : 349 / 1392 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1392, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1392, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1392, 1, 576) torch.bfloat16 + out : (1, 1392) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2803 + top-1 accuracy : 25/349 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1025]) +Dataloader batch - labels shape: torch.Size([1, 1025]) +Dataloader batch - attn_mask shape: torch.Size([1, 1025]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 291 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1025) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1025) torch.int64 + loss_mask : 250 / 1025 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1025, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1025, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1025, 1, 576) torch.bfloat16 + out : (1, 1025) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 250 tokens) : 11.2853 + top-1 accuracy : 2/250 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1759]) +Dataloader batch - labels shape: torch.Size([1, 1759]) +Dataloader batch - attn_mask shape: torch.Size([1, 1759]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 292 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1759) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1759) torch.int64 + loss_mask : 408 / 1759 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1759, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1759, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1759, 1, 576) torch.bfloat16 + out : (1, 1759) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.2754 + top-1 accuracy : 35/408 = 8.6% + + [2026-04-29 13:43:29] iteration 173/ 500 | consumed samples: 692 | elapsed time per iteration (ms): 4608.4 | throughput (token/sec/GPU): 1236.0 | learning rate: 9.500093E-05 | global batch size: 4 | lm loss: 1.126078E+01 | loss scale: 1.0 | grad norm: 0.305 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1291]) +Dataloader batch - labels shape: torch.Size([1, 1291]) +Dataloader batch - attn_mask shape: torch.Size([1, 1291]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 293 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1291) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1291) torch.int64 + loss_mask : 331 / 1291 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1291, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1291, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1291, 1, 576) torch.bfloat16 + out : (1, 1291) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.3046 + top-1 accuracy : 16/331 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 294 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 365 / 1422 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.3062 + top-1 accuracy : 4/365 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1226]) +Dataloader batch - labels shape: torch.Size([1, 1226]) +Dataloader batch - attn_mask shape: torch.Size([1, 1226]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 295 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1226) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1226) torch.int64 + loss_mask : 332 / 1226 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1226, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1226, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1226, 1, 576) torch.bfloat16 + out : (1, 1226) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.3724 + top-1 accuracy : 8/332 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1447]) +Dataloader batch - labels shape: torch.Size([1, 1447]) +Dataloader batch - attn_mask shape: torch.Size([1, 1447]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 296 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1447) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1447) torch.int64 + loss_mask : 413 / 1447 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1447, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1447, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1447, 1, 576) torch.bfloat16 + out : (1, 1447) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.3644 + top-1 accuracy : 4/413 = 1.0% + + [2026-04-29 13:43:33] iteration 174/ 500 | consumed samples: 696 | elapsed time per iteration (ms): 4200.0 | throughput (token/sec/GPU): 1282.4 | learning rate: 9.486356E-05 | global batch size: 4 | lm loss: 1.133775E+01 | loss scale: 1.0 | grad norm: 0.309 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 297 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1191) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1191) torch.int64 + loss_mask : 270 / 1191 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1191, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1191, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1191, 1, 576) torch.bfloat16 + out : (1, 1191) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 270 tokens) : 11.2172 + top-1 accuracy : 15/270 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1379]) +Dataloader batch - labels shape: torch.Size([1, 1379]) +Dataloader batch - attn_mask shape: torch.Size([1, 1379]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 298 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1379) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1379) torch.int64 + loss_mask : 366 / 1379 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1379, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1379, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1379, 1, 576) torch.bfloat16 + out : (1, 1379) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.3090 + top-1 accuracy : 5/366 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1251]) +Dataloader batch - labels shape: torch.Size([1, 1251]) +Dataloader batch - attn_mask shape: torch.Size([1, 1251]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 299 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1251) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1251) torch.int64 + loss_mask : 329 / 1251 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1251, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1251, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1251, 1, 576) torch.bfloat16 + out : (1, 1251) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.3762 + top-1 accuracy : 8/329 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1528]) +Dataloader batch - labels shape: torch.Size([1, 1528]) +Dataloader batch - attn_mask shape: torch.Size([1, 1528]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 300 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1528) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1528) torch.int64 + loss_mask : 345 / 1528 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1528, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1528, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1528, 1, 576) torch.bfloat16 + out : (1, 1528) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2602 + top-1 accuracy : 16/345 = 4.6% + + [2026-04-29 13:43:37] iteration 175/ 500 | consumed samples: 700 | elapsed time per iteration (ms): 4286.6 | throughput (token/sec/GPU): 1247.8 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.129409E+01 | loss scale: 1.0 | grad norm: 0.254 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1188]) +Dataloader batch - labels shape: torch.Size([1, 1188]) +Dataloader batch - attn_mask shape: torch.Size([1, 1188]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 301 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1188) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1188) torch.int64 + loss_mask : 268 / 1188 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1188, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1188, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1188, 1, 576) torch.bfloat16 + out : (1, 1188) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 268 tokens) : 11.2893 + top-1 accuracy : 11/268 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 302 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 357 / 1542 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2346 + top-1 accuracy : 14/357 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1809]) +Dataloader batch - labels shape: torch.Size([1, 1809]) +Dataloader batch - attn_mask shape: torch.Size([1, 1809]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 303 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1809) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1809) torch.int64 + loss_mask : 449 / 1809 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1809, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1809, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1809, 1, 576) torch.bfloat16 + out : (1, 1809) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 449 tokens) : 11.3085 + top-1 accuracy : 3/449 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1578]) +Dataloader batch - labels shape: torch.Size([1, 1578]) +Dataloader batch - attn_mask shape: torch.Size([1, 1578]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 304 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1578) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1578) torch.int64 + loss_mask : 388 / 1578 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1578, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1578, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1578, 1, 576) torch.bfloat16 + out : (1, 1578) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2770 + top-1 accuracy : 4/388 = 1.0% + + [2026-04-29 13:43:42] iteration 176/ 500 | consumed samples: 704 | elapsed time per iteration (ms): 4851.2 | throughput (token/sec/GPU): 1260.9 | learning rate: 9.458358E-05 | global batch size: 4 | lm loss: 1.127858E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1406]) +Dataloader batch - labels shape: torch.Size([1, 1406]) +Dataloader batch - attn_mask shape: torch.Size([1, 1406]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 305 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1406) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1406) torch.int64 + loss_mask : 314 / 1406 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1406, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1406, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1406, 1, 576) torch.bfloat16 + out : (1, 1406) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.2625 + top-1 accuracy : 21/314 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 306 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 390 / 1525 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3259 + top-1 accuracy : 5/390 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 961]) +Dataloader batch - labels shape: torch.Size([1, 961]) +Dataloader batch - attn_mask shape: torch.Size([1, 961]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 307 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 961) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 961) torch.int64 + loss_mask : 195 / 961 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (961, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (961, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (961, 1, 576) torch.bfloat16 + out : (1, 961) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 195 tokens) : 11.1604 + top-1 accuracy : 14/195 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1778]) +Dataloader batch - labels shape: torch.Size([1, 1778]) +Dataloader batch - attn_mask shape: torch.Size([1, 1778]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 308 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1778) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1778) torch.int64 + loss_mask : 404 / 1778 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1778, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1778, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1778, 1, 576) torch.bfloat16 + out : (1, 1778) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.2243 + top-1 accuracy : 7/404 = 1.7% + + [2026-04-29 13:43:47] iteration 177/ 500 | consumed samples: 708 | elapsed time per iteration (ms): 4614.2 | throughput (token/sec/GPU): 1228.8 | learning rate: 9.444096E-05 | global batch size: 4 | lm loss: 1.125436E+01 | loss scale: 1.0 | grad norm: 0.324 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1457]) +Dataloader batch - labels shape: torch.Size([1, 1457]) +Dataloader batch - attn_mask shape: torch.Size([1, 1457]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 309 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1457) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1457) torch.int64 + loss_mask : 355 / 1457 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1457, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1457, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1457, 1, 576) torch.bfloat16 + out : (1, 1457) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2392 + top-1 accuracy : 6/355 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1461]) +Dataloader batch - labels shape: torch.Size([1, 1461]) +Dataloader batch - attn_mask shape: torch.Size([1, 1461]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 310 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1461) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1461) torch.int64 + loss_mask : 399 / 1461 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1461, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1461, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1461, 1, 576) torch.bfloat16 + out : (1, 1461) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.3639 + top-1 accuracy : 9/399 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1574]) +Dataloader batch - labels shape: torch.Size([1, 1574]) +Dataloader batch - attn_mask shape: torch.Size([1, 1574]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 311 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1574) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1574) torch.int64 + loss_mask : 358 / 1574 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1574, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1574, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1574, 1, 576) torch.bfloat16 + out : (1, 1574) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2107 + top-1 accuracy : 23/358 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1498]) +Dataloader batch - labels shape: torch.Size([1, 1498]) +Dataloader batch - attn_mask shape: torch.Size([1, 1498]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 312 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1498) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1498) torch.int64 + loss_mask : 457 / 1498 tokens (30.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1498, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1498, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1498, 1, 576) torch.bfloat16 + out : (1, 1498) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 457 tokens) : 11.3999 + top-1 accuracy : 28/457 = 6.1% + + [2026-04-29 13:43:51] iteration 178/ 500 | consumed samples: 712 | elapsed time per iteration (ms): 4645.4 | throughput (token/sec/GPU): 1289.4 | learning rate: 9.429661E-05 | global batch size: 4 | lm loss: 1.131122E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 974]) +Dataloader batch - labels shape: torch.Size([1, 974]) +Dataloader batch - attn_mask shape: torch.Size([1, 974]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 313 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 974) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 974) torch.int64 + loss_mask : 227 / 974 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (974, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (974, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (974, 1, 576) torch.bfloat16 + out : (1, 974) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 227 tokens) : 11.2183 + top-1 accuracy : 6/227 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1862]) +Dataloader batch - labels shape: torch.Size([1, 1862]) +Dataloader batch - attn_mask shape: torch.Size([1, 1862]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 314 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1862) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1862) torch.int64 + loss_mask : 493 / 1862 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1862, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1862, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1862, 1, 576) torch.bfloat16 + out : (1, 1862) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 493 tokens) : 11.3464 + top-1 accuracy : 17/493 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1403]) +Dataloader batch - labels shape: torch.Size([1, 1403]) +Dataloader batch - attn_mask shape: torch.Size([1, 1403]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 315 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1403) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1403) torch.int64 + loss_mask : 346 / 1403 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1403, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1403, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1403, 1, 576) torch.bfloat16 + out : (1, 1403) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.2659 + top-1 accuracy : 14/346 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 316 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 395 / 1438 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.2918 + top-1 accuracy : 15/395 = 3.8% + + [2026-04-29 13:43:56] iteration 179/ 500 | consumed samples: 716 | elapsed time per iteration (ms): 4488.4 | throughput (token/sec/GPU): 1264.8 | learning rate: 9.415051E-05 | global batch size: 4 | lm loss: 1.129267E+01 | loss scale: 1.0 | grad norm: 0.215 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1429]) +Dataloader batch - labels shape: torch.Size([1, 1429]) +Dataloader batch - attn_mask shape: torch.Size([1, 1429]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 317 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1429) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1429) torch.int64 + loss_mask : 392 / 1429 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1429, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1429, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1429, 1, 576) torch.bfloat16 + out : (1, 1429) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.2772 + top-1 accuracy : 10/392 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1405]) +Dataloader batch - labels shape: torch.Size([1, 1405]) +Dataloader batch - attn_mask shape: torch.Size([1, 1405]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 318 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1405) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1405) torch.int64 + loss_mask : 376 / 1405 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1405, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1405, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1405, 1, 576) torch.bfloat16 + out : (1, 1405) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3091 + top-1 accuracy : 3/376 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1497]) +Dataloader batch - labels shape: torch.Size([1, 1497]) +Dataloader batch - attn_mask shape: torch.Size([1, 1497]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 319 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1497) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1497) torch.int64 + loss_mask : 331 / 1497 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1497, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1497, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1497, 1, 576) torch.bfloat16 + out : (1, 1497) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.2059 + top-1 accuracy : 23/331 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1382]) +Dataloader batch - labels shape: torch.Size([1, 1382]) +Dataloader batch - attn_mask shape: torch.Size([1, 1382]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 320 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1382) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1382) torch.int64 + loss_mask : 325 / 1382 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1382, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1382, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1382, 1, 576) torch.bfloat16 + out : (1, 1382) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.2728 + top-1 accuracy : 17/325 = 5.2% + + [2026-04-29 13:44:00] iteration 180/ 500 | consumed samples: 720 | elapsed time per iteration (ms): 4528.9 | throughput (token/sec/GPU): 1261.5 | learning rate: 9.400269E-05 | global batch size: 4 | lm loss: 1.126805E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4503.08, 4503.08) + forward-compute ................................: (3360.17, 3360.17) + backward-compute ...............................: (1138.82, 1138.82) + batch-generator ................................: (446.58, 446.58) + layernorm-grads-all-reduce .....................: (0.07, 0.07) + embedding-grads-all-reduce .....................: (0.10, 0.10) + all-grads-sync .................................: (1.03, 1.03) + params-all-gather ..............................: (0.36, 0.36) + optimizer-copy-to-main-grad ....................: (0.40, 0.40) + optimizer-clip-main-grad .......................: (1.30, 1.30) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.02, 2.02) + optimizer-copy-main-to-model-params ............: (0.59, 0.59) + optimizer ......................................: (6.70, 6.70) +Dataloader batch - tokens shape: torch.Size([1, 1430]) +Dataloader batch - labels shape: torch.Size([1, 1430]) +Dataloader batch - attn_mask shape: torch.Size([1, 1430]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 321 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1430) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1430) torch.int64 + loss_mask : 377 / 1430 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1430, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1430, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1430, 1, 576) torch.bfloat16 + out : (1, 1430) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.3293 + top-1 accuracy : 16/377 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1123]) +Dataloader batch - labels shape: torch.Size([1, 1123]) +Dataloader batch - attn_mask shape: torch.Size([1, 1123]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 322 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1123) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1123) torch.int64 + loss_mask : 264 / 1123 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1123, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1123, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1123, 1, 576) torch.bfloat16 + out : (1, 1123) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 264 tokens) : 11.2490 + top-1 accuracy : 15/264 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 994]) +Dataloader batch - labels shape: torch.Size([1, 994]) +Dataloader batch - attn_mask shape: torch.Size([1, 994]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 323 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 994) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 994) torch.int64 + loss_mask : 253 / 994 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (994, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (994, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (994, 1, 576) torch.bfloat16 + out : (1, 994) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 253 tokens) : 11.3054 + top-1 accuracy : 1/253 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 324 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 303 / 1386 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.2160 + top-1 accuracy : 24/303 = 7.9% + + [2026-04-29 13:44:04] iteration 181/ 500 | consumed samples: 724 | elapsed time per iteration (ms): 4109.7 | throughput (token/sec/GPU): 1200.3 | learning rate: 9.385314E-05 | global batch size: 4 | lm loss: 1.127788E+01 | loss scale: 1.0 | grad norm: 0.300 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 325 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 382 / 1422 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.2646 + top-1 accuracy : 11/382 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 991]) +Dataloader batch - labels shape: torch.Size([1, 991]) +Dataloader batch - attn_mask shape: torch.Size([1, 991]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 326 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 991) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 991) torch.int64 + loss_mask : 250 / 991 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (991, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (991, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (991, 1, 576) torch.bfloat16 + out : (1, 991) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 250 tokens) : 11.2223 + top-1 accuracy : 3/250 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1770]) +Dataloader batch - labels shape: torch.Size([1, 1770]) +Dataloader batch - attn_mask shape: torch.Size([1, 1770]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 327 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1770) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1770) torch.int64 + loss_mask : 408 / 1770 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1770, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1770, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1770, 1, 576) torch.bfloat16 + out : (1, 1770) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.2677 + top-1 accuracy : 24/408 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1381]) +Dataloader batch - labels shape: torch.Size([1, 1381]) +Dataloader batch - attn_mask shape: torch.Size([1, 1381]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 328 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1381) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1381) torch.int64 + loss_mask : 315 / 1381 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1381, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1381, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1381, 1, 576) torch.bfloat16 + out : (1, 1381) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.2868 + top-1 accuracy : 19/315 = 6.0% + + [2026-04-29 13:44:09] iteration 182/ 500 | consumed samples: 728 | elapsed time per iteration (ms): 4416.1 | throughput (token/sec/GPU): 1259.9 | learning rate: 9.370187E-05 | global batch size: 4 | lm loss: 1.126289E+01 | loss scale: 1.0 | grad norm: 0.254 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1806]) +Dataloader batch - labels shape: torch.Size([1, 1806]) +Dataloader batch - attn_mask shape: torch.Size([1, 1806]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 329 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1806) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1806) torch.int64 + loss_mask : 441 / 1806 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1806, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1806, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1806, 1, 576) torch.bfloat16 + out : (1, 1806) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 441 tokens) : 11.2953 + top-1 accuracy : 26/441 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 998]) +Dataloader batch - labels shape: torch.Size([1, 998]) +Dataloader batch - attn_mask shape: torch.Size([1, 998]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 330 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 998) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 998) torch.int64 + loss_mask : 230 / 998 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (998, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (998, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (998, 1, 576) torch.bfloat16 + out : (1, 998) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 230 tokens) : 11.2059 + top-1 accuracy : 15/230 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1486]) +Dataloader batch - labels shape: torch.Size([1, 1486]) +Dataloader batch - attn_mask shape: torch.Size([1, 1486]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 331 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1486) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1486) torch.int64 + loss_mask : 306 / 1486 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1486, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1486, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1486, 1, 576) torch.bfloat16 + out : (1, 1486) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.1441 + top-1 accuracy : 25/306 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1196]) +Dataloader batch - labels shape: torch.Size([1, 1196]) +Dataloader batch - attn_mask shape: torch.Size([1, 1196]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 332 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1196) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1196) torch.int64 + loss_mask : 274 / 1196 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1196, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1196, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1196, 1, 576) torch.bfloat16 + out : (1, 1196) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 274 tokens) : 11.2014 + top-1 accuracy : 15/274 = 5.5% + + [2026-04-29 13:44:13] iteration 183/ 500 | consumed samples: 732 | elapsed time per iteration (ms): 4574.3 | throughput (token/sec/GPU): 1199.3 | learning rate: 9.354889E-05 | global batch size: 4 | lm loss: 1.122132E+01 | loss scale: 1.0 | grad norm: 0.305 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1020]) +Dataloader batch - labels shape: torch.Size([1, 1020]) +Dataloader batch - attn_mask shape: torch.Size([1, 1020]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 333 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1020) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1020) torch.int64 + loss_mask : 246 / 1020 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1020, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1020, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1020, 1, 576) torch.bfloat16 + out : (1, 1020) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 246 tokens) : 11.2754 + top-1 accuracy : 5/246 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1911]) +Dataloader batch - labels shape: torch.Size([1, 1911]) +Dataloader batch - attn_mask shape: torch.Size([1, 1911]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 334 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1911) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1911) torch.int64 + loss_mask : 549 / 1911 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1911, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1911, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1911, 1, 576) torch.bfloat16 + out : (1, 1911) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 549 tokens) : 11.3770 + top-1 accuracy : 23/549 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1664]) +Dataloader batch - labels shape: torch.Size([1, 1664]) +Dataloader batch - attn_mask shape: torch.Size([1, 1664]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 335 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1664) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1664) torch.int64 + loss_mask : 388 / 1664 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1664, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1664, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1664, 1, 576) torch.bfloat16 + out : (1, 1664) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2644 + top-1 accuracy : 24/388 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1570]) +Dataloader batch - labels shape: torch.Size([1, 1570]) +Dataloader batch - attn_mask shape: torch.Size([1, 1570]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 336 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1570) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1570) torch.int64 + loss_mask : 379 / 1570 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1570, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1570, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1570, 1, 576) torch.bfloat16 + out : (1, 1570) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.2297 + top-1 accuracy : 13/379 = 3.4% + + [2026-04-29 13:44:19] iteration 184/ 500 | consumed samples: 736 | elapsed time per iteration (ms): 5134.2 | throughput (token/sec/GPU): 1200.8 | learning rate: 9.339421E-05 | global batch size: 4 | lm loss: 1.129728E+01 | loss scale: 1.0 | grad norm: 0.249 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1425]) +Dataloader batch - labels shape: torch.Size([1, 1425]) +Dataloader batch - attn_mask shape: torch.Size([1, 1425]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 337 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1425) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1425) torch.int64 + loss_mask : 352 / 1425 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1425, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1425, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1425, 1, 576) torch.bfloat16 + out : (1, 1425) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.2481 + top-1 accuracy : 17/352 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 338 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 284 / 1475 tokens (19.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.1190 + top-1 accuracy : 19/284 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1350]) +Dataloader batch - labels shape: torch.Size([1, 1350]) +Dataloader batch - attn_mask shape: torch.Size([1, 1350]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 339 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1350) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1350) torch.int64 + loss_mask : 280 / 1350 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1350, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1350, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1350, 1, 576) torch.bfloat16 + out : (1, 1350) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 280 tokens) : 11.1728 + top-1 accuracy : 23/280 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1026]) +Dataloader batch - labels shape: torch.Size([1, 1026]) +Dataloader batch - attn_mask shape: torch.Size([1, 1026]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 340 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1026) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1026) torch.int64 + loss_mask : 258 / 1026 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1026, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1026, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1026, 1, 576) torch.bfloat16 + out : (1, 1026) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 258 tokens) : 11.3299 + top-1 accuracy : 8/258 = 3.1% + + [2026-04-29 13:44:23] iteration 185/ 500 | consumed samples: 740 | elapsed time per iteration (ms): 4465.4 | throughput (token/sec/GPU): 1181.5 | learning rate: 9.323782E-05 | global batch size: 4 | lm loss: 1.121688E+01 | loss scale: 1.0 | grad norm: 0.303 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1427]) +Dataloader batch - labels shape: torch.Size([1, 1427]) +Dataloader batch - attn_mask shape: torch.Size([1, 1427]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 341 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1427) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1427) torch.int64 + loss_mask : 338 / 1427 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1427, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1427, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1427, 1, 576) torch.bfloat16 + out : (1, 1427) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.3074 + top-1 accuracy : 9/338 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1785]) +Dataloader batch - labels shape: torch.Size([1, 1785]) +Dataloader batch - attn_mask shape: torch.Size([1, 1785]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 342 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1785) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1785) torch.int64 + loss_mask : 404 / 1785 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1785, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1785, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1785, 1, 576) torch.bfloat16 + out : (1, 1785) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.2300 + top-1 accuracy : 15/404 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 343 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 326 / 1460 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2304 + top-1 accuracy : 7/326 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1216]) +Dataloader batch - labels shape: torch.Size([1, 1216]) +Dataloader batch - attn_mask shape: torch.Size([1, 1216]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 344 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1216) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1216) torch.int64 + loss_mask : 315 / 1216 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1216, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1216, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1216, 1, 576) torch.bfloat16 + out : (1, 1216) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.3364 + top-1 accuracy : 2/315 = 0.6% + + [2026-04-29 13:44:28] iteration 186/ 500 | consumed samples: 744 | elapsed time per iteration (ms): 4794.8 | throughput (token/sec/GPU): 1228.0 | learning rate: 9.307974E-05 | global batch size: 4 | lm loss: 1.127323E+01 | loss scale: 1.0 | grad norm: 0.367 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 345 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 349 / 1400 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2422 + top-1 accuracy : 6/349 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 346 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 386 / 1750 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2030 + top-1 accuracy : 9/386 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 347 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 349 / 1434 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2609 + top-1 accuracy : 13/349 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1367]) +Dataloader batch - labels shape: torch.Size([1, 1367]) +Dataloader batch - attn_mask shape: torch.Size([1, 1367]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 348 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1367) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1367) torch.int64 + loss_mask : 316 / 1367 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1367, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1367, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1367, 1, 576) torch.bfloat16 + out : (1, 1367) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.2279 + top-1 accuracy : 17/316 = 5.4% + + [2026-04-29 13:44:33] iteration 187/ 500 | consumed samples: 748 | elapsed time per iteration (ms): 4795.2 | throughput (token/sec/GPU): 1241.0 | learning rate: 9.291997E-05 | global batch size: 4 | lm loss: 1.123283E+01 | loss scale: 1.0 | grad norm: 0.273 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1315]) +Dataloader batch - labels shape: torch.Size([1, 1315]) +Dataloader batch - attn_mask shape: torch.Size([1, 1315]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 349 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1315) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1315) torch.int64 + loss_mask : 343 / 1315 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1315, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1315, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1315, 1, 576) torch.bfloat16 + out : (1, 1315) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.3417 + top-1 accuracy : 2/343 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1692]) +Dataloader batch - labels shape: torch.Size([1, 1692]) +Dataloader batch - attn_mask shape: torch.Size([1, 1692]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 350 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1692) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1692) torch.int64 + loss_mask : 366 / 1692 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1692, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1692, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1692, 1, 576) torch.bfloat16 + out : (1, 1692) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.1516 + top-1 accuracy : 37/366 = 10.1% + +Dataloader batch - tokens shape: torch.Size([1, 1372]) +Dataloader batch - labels shape: torch.Size([1, 1372]) +Dataloader batch - attn_mask shape: torch.Size([1, 1372]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 351 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1372) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1372) torch.int64 + loss_mask : 309 / 1372 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1372, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1372, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1372, 1, 576) torch.bfloat16 + out : (1, 1372) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.1674 + top-1 accuracy : 27/309 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1198]) +Dataloader batch - labels shape: torch.Size([1, 1198]) +Dataloader batch - attn_mask shape: torch.Size([1, 1198]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 352 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1198) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1198) torch.int64 + loss_mask : 305 / 1198 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1198, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1198, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1198, 1, 576) torch.bfloat16 + out : (1, 1198) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.3158 + top-1 accuracy : 21/305 = 6.9% + + [2026-04-29 13:44:37] iteration 188/ 500 | consumed samples: 752 | elapsed time per iteration (ms): 4530.1 | throughput (token/sec/GPU): 1231.1 | learning rate: 9.275852E-05 | global batch size: 4 | lm loss: 1.124243E+01 | loss scale: 1.0 | grad norm: 0.266 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1795]) +Dataloader batch - labels shape: torch.Size([1, 1795]) +Dataloader batch - attn_mask shape: torch.Size([1, 1795]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 353 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1795) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1795) torch.int64 + loss_mask : 427 / 1795 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1795, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1795, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1795, 1, 576) torch.bfloat16 + out : (1, 1795) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 427 tokens) : 11.2869 + top-1 accuracy : 24/427 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1019]) +Dataloader batch - labels shape: torch.Size([1, 1019]) +Dataloader batch - attn_mask shape: torch.Size([1, 1019]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 354 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1019) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1019) torch.int64 + loss_mask : 270 / 1019 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1019, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1019, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1019, 1, 576) torch.bfloat16 + out : (1, 1019) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 270 tokens) : 11.3109 + top-1 accuracy : 6/270 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1524]) +Dataloader batch - labels shape: torch.Size([1, 1524]) +Dataloader batch - attn_mask shape: torch.Size([1, 1524]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 355 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1524) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1524) torch.int64 + loss_mask : 358 / 1524 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1524, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1524, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1524, 1, 576) torch.bfloat16 + out : (1, 1524) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2273 + top-1 accuracy : 19/358 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1846]) +Dataloader batch - labels shape: torch.Size([1, 1846]) +Dataloader batch - attn_mask shape: torch.Size([1, 1846]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 356 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1846) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1846) torch.int64 + loss_mask : 469 / 1846 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1846, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1846, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1846, 1, 576) torch.bfloat16 + out : (1, 1846) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 469 tokens) : 11.3113 + top-1 accuracy : 10/469 = 2.1% + + [2026-04-29 13:44:42] iteration 189/ 500 | consumed samples: 756 | elapsed time per iteration (ms): 5006.1 | throughput (token/sec/GPU): 1235.3 | learning rate: 9.259539E-05 | global batch size: 4 | lm loss: 1.128466E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1495]) +Dataloader batch - labels shape: torch.Size([1, 1495]) +Dataloader batch - attn_mask shape: torch.Size([1, 1495]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 357 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1495) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1495) torch.int64 + loss_mask : 339 / 1495 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1495, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1495, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1495, 1, 576) torch.bfloat16 + out : (1, 1495) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2649 + top-1 accuracy : 6/339 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1103]) +Dataloader batch - labels shape: torch.Size([1, 1103]) +Dataloader batch - attn_mask shape: torch.Size([1, 1103]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 358 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1103) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1103) torch.int64 + loss_mask : 259 / 1103 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1103, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1103, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1103, 1, 576) torch.bfloat16 + out : (1, 1103) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 259 tokens) : 11.2382 + top-1 accuracy : 22/259 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1463]) +Dataloader batch - labels shape: torch.Size([1, 1463]) +Dataloader batch - attn_mask shape: torch.Size([1, 1463]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 359 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1463) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1463) torch.int64 + loss_mask : 307 / 1463 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1463, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1463, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1463, 1, 576) torch.bfloat16 + out : (1, 1463) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.1645 + top-1 accuracy : 17/307 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1584]) +Dataloader batch - labels shape: torch.Size([1, 1584]) +Dataloader batch - attn_mask shape: torch.Size([1, 1584]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 360 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1584) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1584) torch.int64 + loss_mask : 361 / 1584 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1584, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1584, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1584, 1, 576) torch.bfloat16 + out : (1, 1584) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.2761 + top-1 accuracy : 20/361 = 5.5% + + [2026-04-29 13:44:47] iteration 190/ 500 | consumed samples: 760 | elapsed time per iteration (ms): 4685.3 | throughput (token/sec/GPU): 1204.8 | learning rate: 9.243060E-05 | global batch size: 4 | lm loss: 1.123829E+01 | loss scale: 1.0 | grad norm: 0.324 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 361 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 409 / 1374 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.3890 + top-1 accuracy : 9/409 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1366]) +Dataloader batch - labels shape: torch.Size([1, 1366]) +Dataloader batch - attn_mask shape: torch.Size([1, 1366]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 362 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1366) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1366) torch.int64 + loss_mask : 325 / 1366 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1366, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1366, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1366, 1, 576) torch.bfloat16 + out : (1, 1366) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.2536 + top-1 accuracy : 18/325 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 363 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 350 / 1504 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.2290 + top-1 accuracy : 9/350 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1361]) +Dataloader batch - labels shape: torch.Size([1, 1361]) +Dataloader batch - attn_mask shape: torch.Size([1, 1361]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 364 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1361) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1361) torch.int64 + loss_mask : 311 / 1361 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1361, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1361, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1361, 1, 576) torch.bfloat16 + out : (1, 1361) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.1839 + top-1 accuracy : 10/311 = 3.2% + + [2026-04-29 13:44:51] iteration 191/ 500 | consumed samples: 764 | elapsed time per iteration (ms): 4610.3 | throughput (token/sec/GPU): 1215.7 | learning rate: 9.226415E-05 | global batch size: 4 | lm loss: 1.127158E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1538]) +Dataloader batch - labels shape: torch.Size([1, 1538]) +Dataloader batch - attn_mask shape: torch.Size([1, 1538]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 365 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1538) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1538) torch.int64 + loss_mask : 364 / 1538 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1538, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1538, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1538, 1, 576) torch.bfloat16 + out : (1, 1538) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2152 + top-1 accuracy : 18/364 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1036]) +Dataloader batch - labels shape: torch.Size([1, 1036]) +Dataloader batch - attn_mask shape: torch.Size([1, 1036]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 366 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1036) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1036) torch.int64 + loss_mask : 270 / 1036 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1036, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1036, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1036, 1, 576) torch.bfloat16 + out : (1, 1036) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 270 tokens) : 11.3402 + top-1 accuracy : 0/270 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1576]) +Dataloader batch - labels shape: torch.Size([1, 1576]) +Dataloader batch - attn_mask shape: torch.Size([1, 1576]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 367 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1576) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1576) torch.int64 + loss_mask : 364 / 1576 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1576, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1576, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1576, 1, 576) torch.bfloat16 + out : (1, 1576) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2500 + top-1 accuracy : 6/364 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 965]) +Dataloader batch - labels shape: torch.Size([1, 965]) +Dataloader batch - attn_mask shape: torch.Size([1, 965]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 368 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 965) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 965) torch.int64 + loss_mask : 213 / 965 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (965, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (965, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (965, 1, 576) torch.bfloat16 + out : (1, 965) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 213 tokens) : 11.2473 + top-1 accuracy : 14/213 = 6.6% + + [2026-04-29 13:44:56] iteration 192/ 500 | consumed samples: 768 | elapsed time per iteration (ms): 4306.1 | throughput (token/sec/GPU): 1187.9 | learning rate: 9.209604E-05 | global batch size: 4 | lm loss: 1.125918E+01 | loss scale: 1.0 | grad norm: 0.308 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1529]) +Dataloader batch - labels shape: torch.Size([1, 1529]) +Dataloader batch - attn_mask shape: torch.Size([1, 1529]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 369 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1529) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1529) torch.int64 + loss_mask : 341 / 1529 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1529, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1529, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1529, 1, 576) torch.bfloat16 + out : (1, 1529) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.2515 + top-1 accuracy : 17/341 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 370 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 335 / 1424 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2377 + top-1 accuracy : 9/335 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1566]) +Dataloader batch - labels shape: torch.Size([1, 1566]) +Dataloader batch - attn_mask shape: torch.Size([1, 1566]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 371 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1566) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1566) torch.int64 + loss_mask : 367 / 1566 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1566, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1566, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1566, 1, 576) torch.bfloat16 + out : (1, 1566) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2608 + top-1 accuracy : 20/367 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1604]) +Dataloader batch - labels shape: torch.Size([1, 1604]) +Dataloader batch - attn_mask shape: torch.Size([1, 1604]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 372 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1604) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1604) torch.int64 + loss_mask : 363 / 1604 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1604, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1604, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1604, 1, 576) torch.bfloat16 + out : (1, 1604) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2557 + top-1 accuracy : 25/363 = 6.9% + + [2026-04-29 13:45:01] iteration 193/ 500 | consumed samples: 772 | elapsed time per iteration (ms): 4877.1 | throughput (token/sec/GPU): 1255.5 | learning rate: 9.192628E-05 | global batch size: 4 | lm loss: 1.125173E+01 | loss scale: 1.0 | grad norm: 0.284 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1505]) +Dataloader batch - labels shape: torch.Size([1, 1505]) +Dataloader batch - attn_mask shape: torch.Size([1, 1505]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 373 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1505) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1505) torch.int64 + loss_mask : 336 / 1505 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1505, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1505, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1505, 1, 576) torch.bfloat16 + out : (1, 1505) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.2005 + top-1 accuracy : 29/336 = 8.6% + +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 374 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 308 / 1459 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.1818 + top-1 accuracy : 32/308 = 10.4% + +Dataloader batch - tokens shape: torch.Size([1, 1249]) +Dataloader batch - labels shape: torch.Size([1, 1249]) +Dataloader batch - attn_mask shape: torch.Size([1, 1249]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 375 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1249) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1249) torch.int64 + loss_mask : 330 / 1249 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1249, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1249, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1249, 1, 576) torch.bfloat16 + out : (1, 1249) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.3623 + top-1 accuracy : 11/330 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 376 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 391 / 1750 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2128 + top-1 accuracy : 1/391 = 0.3% + + [2026-04-29 13:45:05] iteration 194/ 500 | consumed samples: 776 | elapsed time per iteration (ms): 4838.6 | throughput (token/sec/GPU): 1232.4 | learning rate: 9.175488E-05 | global batch size: 4 | lm loss: 1.123893E+01 | loss scale: 1.0 | grad norm: 0.286 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1579]) +Dataloader batch - labels shape: torch.Size([1, 1579]) +Dataloader batch - attn_mask shape: torch.Size([1, 1579]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 377 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1579) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1579) torch.int64 + loss_mask : 379 / 1579 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1579, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1579, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1579, 1, 576) torch.bfloat16 + out : (1, 1579) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.2553 + top-1 accuracy : 9/379 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1449]) +Dataloader batch - labels shape: torch.Size([1, 1449]) +Dataloader batch - attn_mask shape: torch.Size([1, 1449]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 378 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1449) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1449) torch.int64 + loss_mask : 358 / 1449 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1449, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1449, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1449, 1, 576) torch.bfloat16 + out : (1, 1449) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2758 + top-1 accuracy : 11/358 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 824]) +Dataloader batch - labels shape: torch.Size([1, 824]) +Dataloader batch - attn_mask shape: torch.Size([1, 824]) +Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) + +==================================================================== + PIPELINE — STEP 379 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 824) torch.int64 + images : (16, 3, 1024, 1024) torch.bfloat16 + labels : (1, 824) torch.int64 + loss_mask : 157 / 824 tokens (19.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (16, 3, 1024, 1024) torch.bfloat16 + out : (16, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (16, 3072) + out : (16, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (824, 1, 576) torch.bfloat16 grad=False + image slots : 16 tokens replaced with vision embeddings + combined : (824, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (824, 1, 576) torch.bfloat16 + out : (1, 824) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 157 tokens) : 11.1482 + top-1 accuracy : 5/157 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1547]) +Dataloader batch - labels shape: torch.Size([1, 1547]) +Dataloader batch - attn_mask shape: torch.Size([1, 1547]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 380 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1547) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1547) torch.int64 + loss_mask : 393 / 1547 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1547, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1547, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1547, 1, 576) torch.bfloat16 + out : (1, 1547) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 393 tokens) : 11.2929 + top-1 accuracy : 5/393 = 1.3% + + [2026-04-29 13:45:10] iteration 195/ 500 | consumed samples: 780 | elapsed time per iteration (ms): 4363.0 | throughput (token/sec/GPU): 1237.5 | learning rate: 9.158184E-05 | global batch size: 4 | lm loss: 1.125941E+01 | loss scale: 1.0 | grad norm: 0.351 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1719]) +Dataloader batch - labels shape: torch.Size([1, 1719]) +Dataloader batch - attn_mask shape: torch.Size([1, 1719]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 381 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1719) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1719) torch.int64 + loss_mask : 435 / 1719 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1719, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1719, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1719, 1, 576) torch.bfloat16 + out : (1, 1719) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 435 tokens) : 11.3306 + top-1 accuracy : 12/435 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1023]) +Dataloader batch - labels shape: torch.Size([1, 1023]) +Dataloader batch - attn_mask shape: torch.Size([1, 1023]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 382 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1023) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1023) torch.int64 + loss_mask : 250 / 1023 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1023, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1023, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1023, 1, 576) torch.bfloat16 + out : (1, 1023) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 250 tokens) : 11.2997 + top-1 accuracy : 11/250 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1538]) +Dataloader batch - labels shape: torch.Size([1, 1538]) +Dataloader batch - attn_mask shape: torch.Size([1, 1538]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 383 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1538) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1538) torch.int64 + loss_mask : 400 / 1538 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1538, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1538, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1538, 1, 576) torch.bfloat16 + out : (1, 1538) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.3108 + top-1 accuracy : 14/400 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1557]) +Dataloader batch - labels shape: torch.Size([1, 1557]) +Dataloader batch - attn_mask shape: torch.Size([1, 1557]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 384 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1557) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1557) torch.int64 + loss_mask : 398 / 1557 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1557, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1557, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1557, 1, 576) torch.bfloat16 + out : (1, 1557) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.2873 + top-1 accuracy : 15/398 = 3.8% + + [2026-04-29 13:45:14] iteration 196/ 500 | consumed samples: 784 | elapsed time per iteration (ms): 4406.6 | throughput (token/sec/GPU): 1324.6 | learning rate: 9.140718E-05 | global batch size: 4 | lm loss: 1.130843E+01 | loss scale: 1.0 | grad norm: 0.311 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1635]) +Dataloader batch - labels shape: torch.Size([1, 1635]) +Dataloader batch - attn_mask shape: torch.Size([1, 1635]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 385 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1635) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1635) torch.int64 + loss_mask : 377 / 1635 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1635, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1635, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1635, 1, 576) torch.bfloat16 + out : (1, 1635) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.2690 + top-1 accuracy : 29/377 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1394]) +Dataloader batch - labels shape: torch.Size([1, 1394]) +Dataloader batch - attn_mask shape: torch.Size([1, 1394]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 386 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1394) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1394) torch.int64 + loss_mask : 381 / 1394 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1394, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1394, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1394, 1, 576) torch.bfloat16 + out : (1, 1394) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.2850 + top-1 accuracy : 16/381 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1537]) +Dataloader batch - labels shape: torch.Size([1, 1537]) +Dataloader batch - attn_mask shape: torch.Size([1, 1537]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 387 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1537) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1537) torch.int64 + loss_mask : 399 / 1537 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1537, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1537, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1537, 1, 576) torch.bfloat16 + out : (1, 1537) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.2806 + top-1 accuracy : 12/399 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1573]) +Dataloader batch - labels shape: torch.Size([1, 1573]) +Dataloader batch - attn_mask shape: torch.Size([1, 1573]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 388 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1573) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1573) torch.int64 + loss_mask : 386 / 1573 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1573, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1573, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1573, 1, 576) torch.bfloat16 + out : (1, 1573) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2661 + top-1 accuracy : 14/386 = 3.6% + + [2026-04-29 13:45:19] iteration 197/ 500 | consumed samples: 788 | elapsed time per iteration (ms): 4856.8 | throughput (token/sec/GPU): 1264.0 | learning rate: 9.123089E-05 | global batch size: 4 | lm loss: 1.127522E+01 | loss scale: 1.0 | grad norm: 0.269 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1321]) +Dataloader batch - labels shape: torch.Size([1, 1321]) +Dataloader batch - attn_mask shape: torch.Size([1, 1321]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 389 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1321) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1321) torch.int64 + loss_mask : 356 / 1321 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1321, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1321, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1321, 1, 576) torch.bfloat16 + out : (1, 1321) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.3477 + top-1 accuracy : 10/356 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1369]) +Dataloader batch - labels shape: torch.Size([1, 1369]) +Dataloader batch - attn_mask shape: torch.Size([1, 1369]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 390 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1369) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1369) torch.int64 + loss_mask : 390 / 1369 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1369, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1369, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1369, 1, 576) torch.bfloat16 + out : (1, 1369) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3565 + top-1 accuracy : 3/390 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1076]) +Dataloader batch - labels shape: torch.Size([1, 1076]) +Dataloader batch - attn_mask shape: torch.Size([1, 1076]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 391 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1076) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1076) torch.int64 + loss_mask : 313 / 1076 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1076, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1076, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1076, 1, 576) torch.bfloat16 + out : (1, 1076) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.3643 + top-1 accuracy : 5/313 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1760]) +Dataloader batch - labels shape: torch.Size([1, 1760]) +Dataloader batch - attn_mask shape: torch.Size([1, 1760]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 392 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1760) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1760) torch.int64 + loss_mask : 383 / 1760 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1760, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1760, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1760, 1, 576) torch.bfloat16 + out : (1, 1760) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.1964 + top-1 accuracy : 23/383 = 6.0% + + [2026-04-29 13:45:23] iteration 198/ 500 | consumed samples: 792 | elapsed time per iteration (ms): 4329.7 | throughput (token/sec/GPU): 1276.3 | learning rate: 9.105299E-05 | global batch size: 4 | lm loss: 1.131351E+01 | loss scale: 1.0 | grad norm: 0.248 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1451]) +Dataloader batch - labels shape: torch.Size([1, 1451]) +Dataloader batch - attn_mask shape: torch.Size([1, 1451]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 393 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1451) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1451) torch.int64 + loss_mask : 387 / 1451 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1451, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1451, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1451, 1, 576) torch.bfloat16 + out : (1, 1451) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.3138 + top-1 accuracy : 14/387 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1227]) +Dataloader batch - labels shape: torch.Size([1, 1227]) +Dataloader batch - attn_mask shape: torch.Size([1, 1227]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 394 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1227) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1227) torch.int64 + loss_mask : 355 / 1227 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1227, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1227, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1227, 1, 576) torch.bfloat16 + out : (1, 1227) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.3596 + top-1 accuracy : 2/355 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1219]) +Dataloader batch - labels shape: torch.Size([1, 1219]) +Dataloader batch - attn_mask shape: torch.Size([1, 1219]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 395 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1219) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1219) torch.int64 + loss_mask : 284 / 1219 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1219, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1219, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1219, 1, 576) torch.bfloat16 + out : (1, 1219) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.2002 + top-1 accuracy : 11/284 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1411]) +Dataloader batch - labels shape: torch.Size([1, 1411]) +Dataloader batch - attn_mask shape: torch.Size([1, 1411]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 396 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1411) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1411) torch.int64 + loss_mask : 323 / 1411 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1411, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1411, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1411, 1, 576) torch.bfloat16 + out : (1, 1411) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.2023 + top-1 accuracy : 13/323 = 4.0% + + [2026-04-29 13:45:28] iteration 199/ 500 | consumed samples: 796 | elapsed time per iteration (ms): 4240.9 | throughput (token/sec/GPU): 1251.6 | learning rate: 9.087348E-05 | global batch size: 4 | lm loss: 1.127524E+01 | loss scale: 1.0 | grad norm: 0.277 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1723]) +Dataloader batch - labels shape: torch.Size([1, 1723]) +Dataloader batch - attn_mask shape: torch.Size([1, 1723]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 397 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1723) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1723) torch.int64 + loss_mask : 367 / 1723 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1723, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1723, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1723, 1, 576) torch.bfloat16 + out : (1, 1723) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2190 + top-1 accuracy : 25/367 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1691]) +Dataloader batch - labels shape: torch.Size([1, 1691]) +Dataloader batch - attn_mask shape: torch.Size([1, 1691]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 398 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1691) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1691) torch.int64 + loss_mask : 343 / 1691 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1691, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1691, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1691, 1, 576) torch.bfloat16 + out : (1, 1691) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.1668 + top-1 accuracy : 32/343 = 9.3% + +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 399 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 315 / 1468 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.1500 + top-1 accuracy : 12/315 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1904]) +Dataloader batch - labels shape: torch.Size([1, 1904]) +Dataloader batch - attn_mask shape: torch.Size([1, 1904]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 400 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1904) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1904) torch.int64 + loss_mask : 538 / 1904 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1904, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1904, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1904, 1, 576) torch.bfloat16 + out : (1, 1904) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 538 tokens) : 11.3801 + top-1 accuracy : 2/538 = 0.4% + + [2026-04-29 13:45:33] iteration 200/ 500 | consumed samples: 800 | elapsed time per iteration (ms): 5267.3 | throughput (token/sec/GPU): 1288.3 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.124909E+01 | loss scale: 1.0 | grad norm: 0.350 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (5254.65, 5254.65) + forward-compute ................................: (3949.06, 3949.06) + backward-compute ...............................: (1303.69, 1303.69) + batch-generator ................................: (511.03, 511.03) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.35, 0.35) + params-all-gather ..............................: (0.11, 0.11) + optimizer-copy-to-main-grad ....................: (0.10, 0.10) + optimizer-clip-main-grad .......................: (0.40, 0.40) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.09, 1.09) + optimizer-copy-main-to-model-params ............: (0.18, 0.18) + optimizer ......................................: (2.43, 2.43) +saving checkpoint at iteration 200 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 200 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 200 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1978.33, 1978.33) +Dataloader batch - tokens shape: torch.Size([1, 1556]) +Dataloader batch - labels shape: torch.Size([1, 1556]) +Dataloader batch - attn_mask shape: torch.Size([1, 1556]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 401 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1556) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1556) torch.int64 + loss_mask : 359 / 1556 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1556, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1556, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1556, 1, 576) torch.bfloat16 + out : (1, 1556) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.3096 + top-1 accuracy : 19/359 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1410]) +Dataloader batch - labels shape: torch.Size([1, 1410]) +Dataloader batch - attn_mask shape: torch.Size([1, 1410]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 402 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1410) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1410) torch.int64 + loss_mask : 362 / 1410 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1410, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1410, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1410, 1, 576) torch.bfloat16 + out : (1, 1410) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2956 + top-1 accuracy : 21/362 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1049]) +Dataloader batch - labels shape: torch.Size([1, 1049]) +Dataloader batch - attn_mask shape: torch.Size([1, 1049]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 403 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1049) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1049) torch.int64 + loss_mask : 285 / 1049 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1049, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1049, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1049, 1, 576) torch.bfloat16 + out : (1, 1049) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 285 tokens) : 11.3920 + top-1 accuracy : 5/285 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1791]) +Dataloader batch - labels shape: torch.Size([1, 1791]) +Dataloader batch - attn_mask shape: torch.Size([1, 1791]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 404 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1791) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1791) torch.int64 + loss_mask : 406 / 1791 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1791, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1791, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1791, 1, 576) torch.bfloat16 + out : (1, 1791) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.2465 + top-1 accuracy : 4/406 = 1.0% + + [2026-04-29 13:45:40] iteration 201/ 500 | consumed samples: 804 | elapsed time per iteration (ms): 4697.2 | throughput (token/sec/GPU): 1236.1 | learning rate: 9.050967E-05 | global batch size: 4 | lm loss: 1.130451E+01 | loss scale: 1.0 | grad norm: 0.244 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1407]) +Dataloader batch - labels shape: torch.Size([1, 1407]) +Dataloader batch - attn_mask shape: torch.Size([1, 1407]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 405 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1407) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1407) torch.int64 + loss_mask : 349 / 1407 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1407, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1407, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1407, 1, 576) torch.bfloat16 + out : (1, 1407) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.3175 + top-1 accuracy : 8/349 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1836]) +Dataloader batch - labels shape: torch.Size([1, 1836]) +Dataloader batch - attn_mask shape: torch.Size([1, 1836]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 406 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1836) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1836) torch.int64 + loss_mask : 487 / 1836 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1836, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1836, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1836, 1, 576) torch.bfloat16 + out : (1, 1836) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 487 tokens) : 11.3269 + top-1 accuracy : 13/487 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 995]) +Dataloader batch - labels shape: torch.Size([1, 995]) +Dataloader batch - attn_mask shape: torch.Size([1, 995]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 407 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 995) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 995) torch.int64 + loss_mask : 241 / 995 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (995, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (995, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (995, 1, 576) torch.bfloat16 + out : (1, 995) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 241 tokens) : 11.2427 + top-1 accuracy : 0/241 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1928]) +Dataloader batch - labels shape: torch.Size([1, 1928]) +Dataloader batch - attn_mask shape: torch.Size([1, 1928]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 408 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1928) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1928) torch.int64 + loss_mask : 576 / 1928 tokens (29.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1928, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1928, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1928, 1, 576) torch.bfloat16 + out : (1, 1928) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 576 tokens) : 11.4123 + top-1 accuracy : 16/576 = 2.8% + + [2026-04-29 13:45:44] iteration 202/ 500 | consumed samples: 808 | elapsed time per iteration (ms): 4689.9 | throughput (token/sec/GPU): 1314.7 | learning rate: 9.032539E-05 | global batch size: 4 | lm loss: 1.134238E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 409 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1519) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1519) torch.int64 + loss_mask : 352 / 1519 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1519, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1519, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1519, 1, 576) torch.bfloat16 + out : (1, 1519) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.2670 + top-1 accuracy : 13/352 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1634]) +Dataloader batch - labels shape: torch.Size([1, 1634]) +Dataloader batch - attn_mask shape: torch.Size([1, 1634]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 410 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1634) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1634) torch.int64 + loss_mask : 368 / 1634 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1634, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1634, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1634, 1, 576) torch.bfloat16 + out : (1, 1634) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2171 + top-1 accuracy : 13/368 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1781]) +Dataloader batch - labels shape: torch.Size([1, 1781]) +Dataloader batch - attn_mask shape: torch.Size([1, 1781]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 411 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1781) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1781) torch.int64 + loss_mask : 416 / 1781 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1781, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1781, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1781, 1, 576) torch.bfloat16 + out : (1, 1781) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.2827 + top-1 accuracy : 20/416 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1399]) +Dataloader batch - labels shape: torch.Size([1, 1399]) +Dataloader batch - attn_mask shape: torch.Size([1, 1399]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 412 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1399) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1399) torch.int64 + loss_mask : 338 / 1399 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1399, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1399, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1399, 1, 576) torch.bfloat16 + out : (1, 1399) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.2969 + top-1 accuracy : 15/338 = 4.4% + + [2026-04-29 13:45:49] iteration 203/ 500 | consumed samples: 812 | elapsed time per iteration (ms): 4931.6 | throughput (token/sec/GPU): 1284.2 | learning rate: 9.013952E-05 | global batch size: 4 | lm loss: 1.126583E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1591]) +Dataloader batch - labels shape: torch.Size([1, 1591]) +Dataloader batch - attn_mask shape: torch.Size([1, 1591]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 413 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1591) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1591) torch.int64 + loss_mask : 367 / 1591 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1591, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1591, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1591, 1, 576) torch.bfloat16 + out : (1, 1591) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2681 + top-1 accuracy : 17/367 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1483]) +Dataloader batch - labels shape: torch.Size([1, 1483]) +Dataloader batch - attn_mask shape: torch.Size([1, 1483]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 414 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1483) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1483) torch.int64 + loss_mask : 396 / 1483 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1483, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1483, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1483, 1, 576) torch.bfloat16 + out : (1, 1483) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 396 tokens) : 11.3274 + top-1 accuracy : 7/396 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 415 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 286 / 1374 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 286 tokens) : 11.2492 + top-1 accuracy : 21/286 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1086]) +Dataloader batch - labels shape: torch.Size([1, 1086]) +Dataloader batch - attn_mask shape: torch.Size([1, 1086]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 416 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1086) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1086) torch.int64 + loss_mask : 329 / 1086 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1086, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1086, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1086, 1, 576) torch.bfloat16 + out : (1, 1086) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.4023 + top-1 accuracy : 1/329 = 0.3% + + [2026-04-29 13:45:54] iteration 204/ 500 | consumed samples: 816 | elapsed time per iteration (ms): 4457.6 | throughput (token/sec/GPU): 1241.5 | learning rate: 8.995208E-05 | global batch size: 4 | lm loss: 1.131327E+01 | loss scale: 1.0 | grad norm: 0.283 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1015]) +Dataloader batch - labels shape: torch.Size([1, 1015]) +Dataloader batch - attn_mask shape: torch.Size([1, 1015]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 417 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1015) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1015) torch.int64 + loss_mask : 265 / 1015 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1015, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1015, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1015, 1, 576) torch.bfloat16 + out : (1, 1015) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 265 tokens) : 11.3250 + top-1 accuracy : 13/265 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1497]) +Dataloader batch - labels shape: torch.Size([1, 1497]) +Dataloader batch - attn_mask shape: torch.Size([1, 1497]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 418 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1497) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1497) torch.int64 + loss_mask : 316 / 1497 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1497, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1497, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1497, 1, 576) torch.bfloat16 + out : (1, 1497) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.1814 + top-1 accuracy : 20/316 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1435]) +Dataloader batch - labels shape: torch.Size([1, 1435]) +Dataloader batch - attn_mask shape: torch.Size([1, 1435]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 419 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1435) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1435) torch.int64 + loss_mask : 358 / 1435 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1435, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1435, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1435, 1, 576) torch.bfloat16 + out : (1, 1435) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2913 + top-1 accuracy : 11/358 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1528]) +Dataloader batch - labels shape: torch.Size([1, 1528]) +Dataloader batch - attn_mask shape: torch.Size([1, 1528]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 420 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1528) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1528) torch.int64 + loss_mask : 336 / 1528 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1528, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1528, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1528, 1, 576) torch.bfloat16 + out : (1, 1528) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.2521 + top-1 accuracy : 9/336 = 2.7% + + [2026-04-29 13:45:58] iteration 205/ 500 | consumed samples: 820 | elapsed time per iteration (ms): 4566.8 | throughput (token/sec/GPU): 1198.9 | learning rate: 8.976308E-05 | global batch size: 4 | lm loss: 1.126074E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1110]) +Dataloader batch - labels shape: torch.Size([1, 1110]) +Dataloader batch - attn_mask shape: torch.Size([1, 1110]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 421 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1110) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1110) torch.int64 + loss_mask : 249 / 1110 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1110, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1110, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1110, 1, 576) torch.bfloat16 + out : (1, 1110) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 249 tokens) : 11.1566 + top-1 accuracy : 7/249 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1004]) +Dataloader batch - labels shape: torch.Size([1, 1004]) +Dataloader batch - attn_mask shape: torch.Size([1, 1004]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 422 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1004) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1004) torch.int64 + loss_mask : 251 / 1004 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1004, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1004, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1004, 1, 576) torch.bfloat16 + out : (1, 1004) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2837 + top-1 accuracy : 4/251 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1529]) +Dataloader batch - labels shape: torch.Size([1, 1529]) +Dataloader batch - attn_mask shape: torch.Size([1, 1529]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 423 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1529) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1529) torch.int64 + loss_mask : 330 / 1529 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1529, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1529, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1529, 1, 576) torch.bfloat16 + out : (1, 1529) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.1655 + top-1 accuracy : 22/330 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1419]) +Dataloader batch - labels shape: torch.Size([1, 1419]) +Dataloader batch - attn_mask shape: torch.Size([1, 1419]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 424 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1419) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1419) torch.int64 + loss_mask : 364 / 1419 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1419, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1419, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1419, 1, 576) torch.bfloat16 + out : (1, 1419) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2944 + top-1 accuracy : 15/364 = 4.1% + + [2026-04-29 13:46:02] iteration 206/ 500 | consumed samples: 824 | elapsed time per iteration (ms): 4146.9 | throughput (token/sec/GPU): 1220.7 | learning rate: 8.957252E-05 | global batch size: 4 | lm loss: 1.122781E+01 | loss scale: 1.0 | grad norm: 0.278 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1890]) +Dataloader batch - labels shape: torch.Size([1, 1890]) +Dataloader batch - attn_mask shape: torch.Size([1, 1890]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 425 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1890) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1890) torch.int64 + loss_mask : 524 / 1890 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1890, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1890, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1890, 1, 576) torch.bfloat16 + out : (1, 1890) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 524 tokens) : 11.3317 + top-1 accuracy : 23/524 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1578]) +Dataloader batch - labels shape: torch.Size([1, 1578]) +Dataloader batch - attn_mask shape: torch.Size([1, 1578]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 426 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1578) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1578) torch.int64 + loss_mask : 354 / 1578 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1578, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1578, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1578, 1, 576) torch.bfloat16 + out : (1, 1578) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2565 + top-1 accuracy : 6/354 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1152]) +Dataloader batch - labels shape: torch.Size([1, 1152]) +Dataloader batch - attn_mask shape: torch.Size([1, 1152]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 427 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1152) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1152) torch.int64 + loss_mask : 294 / 1152 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1152, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1152, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1152, 1, 576) torch.bfloat16 + out : (1, 1152) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.3737 + top-1 accuracy : 2/294 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1382]) +Dataloader batch - labels shape: torch.Size([1, 1382]) +Dataloader batch - attn_mask shape: torch.Size([1, 1382]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 428 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1382) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1382) torch.int64 + loss_mask : 328 / 1382 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1382, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1382, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1382, 1, 576) torch.bfloat16 + out : (1, 1382) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.2141 + top-1 accuracy : 9/328 = 2.7% + + [2026-04-29 13:46:07] iteration 207/ 500 | consumed samples: 828 | elapsed time per iteration (ms): 4705.6 | throughput (token/sec/GPU): 1275.5 | learning rate: 8.938042E-05 | global batch size: 4 | lm loss: 1.129644E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1377]) +Dataloader batch - labels shape: torch.Size([1, 1377]) +Dataloader batch - attn_mask shape: torch.Size([1, 1377]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 429 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1377) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1377) torch.int64 + loss_mask : 312 / 1377 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1377, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1377, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1377, 1, 576) torch.bfloat16 + out : (1, 1377) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.2704 + top-1 accuracy : 8/312 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1537]) +Dataloader batch - labels shape: torch.Size([1, 1537]) +Dataloader batch - attn_mask shape: torch.Size([1, 1537]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 430 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1537) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1537) torch.int64 + loss_mask : 358 / 1537 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1537, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1537, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1537, 1, 576) torch.bfloat16 + out : (1, 1537) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2511 + top-1 accuracy : 28/358 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1868]) +Dataloader batch - labels shape: torch.Size([1, 1868]) +Dataloader batch - attn_mask shape: torch.Size([1, 1868]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 431 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1868) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1868) torch.int64 + loss_mask : 508 / 1868 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1868, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1868, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1868, 1, 576) torch.bfloat16 + out : (1, 1868) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 508 tokens) : 11.3621 + top-1 accuracy : 7/508 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1000]) +Dataloader batch - labels shape: torch.Size([1, 1000]) +Dataloader batch - attn_mask shape: torch.Size([1, 1000]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 432 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1000) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1000) torch.int64 + loss_mask : 237 / 1000 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1000, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1000, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1000, 1, 576) torch.bfloat16 + out : (1, 1000) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 237 tokens) : 11.2796 + top-1 accuracy : 4/237 = 1.7% + + [2026-04-29 13:46:12] iteration 208/ 500 | consumed samples: 832 | elapsed time per iteration (ms): 4547.2 | throughput (token/sec/GPU): 1271.6 | learning rate: 8.918677E-05 | global batch size: 4 | lm loss: 1.129996E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1133]) +Dataloader batch - labels shape: torch.Size([1, 1133]) +Dataloader batch - attn_mask shape: torch.Size([1, 1133]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 433 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1133) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1133) torch.int64 + loss_mask : 336 / 1133 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1133, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1133, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1133, 1, 576) torch.bfloat16 + out : (1, 1133) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.4126 + top-1 accuracy : 4/336 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1001]) +Dataloader batch - labels shape: torch.Size([1, 1001]) +Dataloader batch - attn_mask shape: torch.Size([1, 1001]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 434 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1001) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1001) torch.int64 + loss_mask : 225 / 1001 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1001, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1001, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1001, 1, 576) torch.bfloat16 + out : (1, 1001) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 225 tokens) : 11.2407 + top-1 accuracy : 12/225 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1786]) +Dataloader batch - labels shape: torch.Size([1, 1786]) +Dataloader batch - attn_mask shape: torch.Size([1, 1786]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 435 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1786) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1786) torch.int64 + loss_mask : 415 / 1786 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1786, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1786, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1786, 1, 576) torch.bfloat16 + out : (1, 1786) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.2586 + top-1 accuracy : 27/415 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 436 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 407 / 1539 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.2975 + top-1 accuracy : 4/407 = 1.0% + + [2026-04-29 13:46:16] iteration 209/ 500 | consumed samples: 836 | elapsed time per iteration (ms): 4319.5 | throughput (token/sec/GPU): 1263.8 | learning rate: 8.899160E-05 | global batch size: 4 | lm loss: 1.130454E+01 | loss scale: 1.0 | grad norm: 0.264 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1485]) +Dataloader batch - labels shape: torch.Size([1, 1485]) +Dataloader batch - attn_mask shape: torch.Size([1, 1485]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 437 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1485) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1485) torch.int64 + loss_mask : 423 / 1485 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1485, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1485, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1485, 1, 576) torch.bfloat16 + out : (1, 1485) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 423 tokens) : 11.3531 + top-1 accuracy : 5/423 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1210]) +Dataloader batch - labels shape: torch.Size([1, 1210]) +Dataloader batch - attn_mask shape: torch.Size([1, 1210]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 438 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1210) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1210) torch.int64 + loss_mask : 323 / 1210 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1210, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1210, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1210, 1, 576) torch.bfloat16 + out : (1, 1210) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.3007 + top-1 accuracy : 3/323 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 439 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 341 / 1695 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.1731 + top-1 accuracy : 31/341 = 9.1% + +Dataloader batch - tokens shape: torch.Size([1, 1916]) +Dataloader batch - labels shape: torch.Size([1, 1916]) +Dataloader batch - attn_mask shape: torch.Size([1, 1916]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 440 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1916) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1916) torch.int64 + loss_mask : 562 / 1916 tokens (29.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1916, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1916, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1916, 1, 576) torch.bfloat16 + out : (1, 1916) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 562 tokens) : 11.3773 + top-1 accuracy : 20/562 = 3.6% + + [2026-04-29 13:46:21] iteration 210/ 500 | consumed samples: 840 | elapsed time per iteration (ms): 4846.4 | throughput (token/sec/GPU): 1301.2 | learning rate: 8.879489E-05 | global batch size: 4 | lm loss: 1.131386E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1410]) +Dataloader batch - labels shape: torch.Size([1, 1410]) +Dataloader batch - attn_mask shape: torch.Size([1, 1410]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 441 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1410) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1410) torch.int64 + loss_mask : 321 / 1410 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1410, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1410, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1410, 1, 576) torch.bfloat16 + out : (1, 1410) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2486 + top-1 accuracy : 14/321 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1164]) +Dataloader batch - labels shape: torch.Size([1, 1164]) +Dataloader batch - attn_mask shape: torch.Size([1, 1164]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 442 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1164) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1164) torch.int64 + loss_mask : 313 / 1164 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1164, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1164, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1164, 1, 576) torch.bfloat16 + out : (1, 1164) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.3809 + top-1 accuracy : 2/313 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1777]) +Dataloader batch - labels shape: torch.Size([1, 1777]) +Dataloader batch - attn_mask shape: torch.Size([1, 1777]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 443 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1777) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1777) torch.int64 + loss_mask : 406 / 1777 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1777, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1777, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1777, 1, 576) torch.bfloat16 + out : (1, 1777) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.2695 + top-1 accuracy : 24/406 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 1527]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 444 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1527) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1527) torch.int64 + loss_mask : 390 / 1527 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1527, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1527, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1527, 1, 576) torch.bfloat16 + out : (1, 1527) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.2838 + top-1 accuracy : 7/390 = 1.8% + + [2026-04-29 13:46:25] iteration 211/ 500 | consumed samples: 844 | elapsed time per iteration (ms): 4637.2 | throughput (token/sec/GPU): 1267.6 | learning rate: 8.859667E-05 | global batch size: 4 | lm loss: 1.129310E+01 | loss scale: 1.0 | grad norm: 0.244 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 445 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 348 / 1695 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.1641 + top-1 accuracy : 25/348 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1471]) +Dataloader batch - labels shape: torch.Size([1, 1471]) +Dataloader batch - attn_mask shape: torch.Size([1, 1471]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 446 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1471) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1471) torch.int64 + loss_mask : 303 / 1471 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1471, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1471, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1471, 1, 576) torch.bfloat16 + out : (1, 1471) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.1466 + top-1 accuracy : 28/303 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1773]) +Dataloader batch - labels shape: torch.Size([1, 1773]) +Dataloader batch - attn_mask shape: torch.Size([1, 1773]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 447 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1773) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1773) torch.int64 + loss_mask : 420 / 1773 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1773, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1773, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1773, 1, 576) torch.bfloat16 + out : (1, 1773) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 420 tokens) : 11.2642 + top-1 accuracy : 22/420 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1649]) +Dataloader batch - labels shape: torch.Size([1, 1649]) +Dataloader batch - attn_mask shape: torch.Size([1, 1649]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 448 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1649) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1649) torch.int64 + loss_mask : 431 / 1649 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1649, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1649, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1649, 1, 576) torch.bfloat16 + out : (1, 1649) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.3131 + top-1 accuracy : 4/431 = 0.9% + + [2026-04-29 13:46:31] iteration 212/ 500 | consumed samples: 848 | elapsed time per iteration (ms): 5195.4 | throughput (token/sec/GPU): 1268.0 | learning rate: 8.839693E-05 | global batch size: 4 | lm loss: 1.123132E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1514]) +Dataloader batch - labels shape: torch.Size([1, 1514]) +Dataloader batch - attn_mask shape: torch.Size([1, 1514]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 449 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1514) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1514) torch.int64 + loss_mask : 380 / 1514 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1514, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1514, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1514, 1, 576) torch.bfloat16 + out : (1, 1514) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 380 tokens) : 11.2525 + top-1 accuracy : 6/380 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 992]) +Dataloader batch - labels shape: torch.Size([1, 992]) +Dataloader batch - attn_mask shape: torch.Size([1, 992]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 450 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 992) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 992) torch.int64 + loss_mask : 242 / 992 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (992, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (992, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (992, 1, 576) torch.bfloat16 + out : (1, 992) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 242 tokens) : 11.3191 + top-1 accuracy : 21/242 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1522]) +Dataloader batch - labels shape: torch.Size([1, 1522]) +Dataloader batch - attn_mask shape: torch.Size([1, 1522]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 451 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1522) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1522) torch.int64 + loss_mask : 384 / 1522 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1522, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1522, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1522, 1, 576) torch.bfloat16 + out : (1, 1522) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.2507 + top-1 accuracy : 12/384 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1747]) +Dataloader batch - labels shape: torch.Size([1, 1747]) +Dataloader batch - attn_mask shape: torch.Size([1, 1747]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 452 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1747) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1747) torch.int64 + loss_mask : 382 / 1747 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1747, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1747, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1747, 1, 576) torch.bfloat16 + out : (1, 1747) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.1784 + top-1 accuracy : 25/382 = 6.5% + + [2026-04-29 13:46:35] iteration 213/ 500 | consumed samples: 852 | elapsed time per iteration (ms): 4559.4 | throughput (token/sec/GPU): 1266.6 | learning rate: 8.819569E-05 | global batch size: 4 | lm loss: 1.124325E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1752]) +Dataloader batch - labels shape: torch.Size([1, 1752]) +Dataloader batch - attn_mask shape: torch.Size([1, 1752]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 453 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1752) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1752) torch.int64 + loss_mask : 391 / 1752 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1752, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1752, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1752, 1, 576) torch.bfloat16 + out : (1, 1752) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2034 + top-1 accuracy : 23/391 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 454 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 295 / 1436 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.1666 + top-1 accuracy : 4/295 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 455 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 298 / 1380 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 298 tokens) : 11.2077 + top-1 accuracy : 23/298 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 456 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 357 / 1424 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3626 + top-1 accuracy : 17/357 = 4.8% + + [2026-04-29 13:46:40] iteration 214/ 500 | consumed samples: 856 | elapsed time per iteration (ms): 4825.5 | throughput (token/sec/GPU): 1241.7 | learning rate: 8.799296E-05 | global batch size: 4 | lm loss: 1.123865E+01 | loss scale: 1.0 | grad norm: 0.300 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 457 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 351 / 1695 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.1756 + top-1 accuracy : 21/351 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1220]) +Dataloader batch - labels shape: torch.Size([1, 1220]) +Dataloader batch - attn_mask shape: torch.Size([1, 1220]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 458 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1220) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1220) torch.int64 + loss_mask : 300 / 1220 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1220, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1220, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1220, 1, 576) torch.bfloat16 + out : (1, 1220) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.2240 + top-1 accuracy : 22/300 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 459 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 394 / 1539 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3416 + top-1 accuracy : 21/394 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1445]) +Dataloader batch - labels shape: torch.Size([1, 1445]) +Dataloader batch - attn_mask shape: torch.Size([1, 1445]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 460 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1445) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1445) torch.int64 + loss_mask : 368 / 1445 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1445, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1445, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1445, 1, 576) torch.bfloat16 + out : (1, 1445) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2998 + top-1 accuracy : 14/368 = 3.8% + + [2026-04-29 13:46:45] iteration 215/ 500 | consumed samples: 860 | elapsed time per iteration (ms): 4758.7 | throughput (token/sec/GPU): 1239.6 | learning rate: 8.778875E-05 | global batch size: 4 | lm loss: 1.126451E+01 | loss scale: 1.0 | grad norm: 0.255 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1355]) +Dataloader batch - labels shape: torch.Size([1, 1355]) +Dataloader batch - attn_mask shape: torch.Size([1, 1355]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 461 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1355) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1355) torch.int64 + loss_mask : 278 / 1355 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1355, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1355, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1355, 1, 576) torch.bfloat16 + out : (1, 1355) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.1681 + top-1 accuracy : 22/278 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1759]) +Dataloader batch - labels shape: torch.Size([1, 1759]) +Dataloader batch - attn_mask shape: torch.Size([1, 1759]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 462 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1759) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1759) torch.int64 + loss_mask : 434 / 1759 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1759, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1759, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1759, 1, 576) torch.bfloat16 + out : (1, 1759) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 434 tokens) : 11.3506 + top-1 accuracy : 23/434 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1494]) +Dataloader batch - labels shape: torch.Size([1, 1494]) +Dataloader batch - attn_mask shape: torch.Size([1, 1494]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 463 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1494) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1494) torch.int64 + loss_mask : 331 / 1494 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1494, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1494, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1494, 1, 576) torch.bfloat16 + out : (1, 1494) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.1546 + top-1 accuracy : 18/331 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 774]) +Dataloader batch - labels shape: torch.Size([1, 774]) +Dataloader batch - attn_mask shape: torch.Size([1, 774]) +Dataloader batch - image_grid_thw shape: torch.Size([15, 3]) + +==================================================================== + PIPELINE — STEP 464 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 774) torch.int64 + images : (15, 3, 1024, 1024) torch.bfloat16 + labels : (1, 774) torch.int64 + loss_mask : 160 / 774 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (15, 3, 1024, 1024) torch.bfloat16 + out : (15, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (15, 3072) + out : (15, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (774, 1, 576) torch.bfloat16 grad=False + image slots : 15 tokens replaced with vision embeddings + combined : (774, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (774, 1, 576) torch.bfloat16 + out : (1, 774) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 160 tokens) : 11.1209 + top-1 accuracy : 1/160 = 0.6% + + [2026-04-29 13:46:49] iteration 216/ 500 | consumed samples: 864 | elapsed time per iteration (ms): 4374.2 | throughput (token/sec/GPU): 1230.4 | learning rate: 8.758306E-05 | global batch size: 4 | lm loss: 1.122394E+01 | loss scale: 1.0 | grad norm: 0.310 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1204]) +Dataloader batch - labels shape: torch.Size([1, 1204]) +Dataloader batch - attn_mask shape: torch.Size([1, 1204]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 465 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1204) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1204) torch.int64 + loss_mask : 363 / 1204 tokens (30.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1204, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1204, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1204, 1, 576) torch.bfloat16 + out : (1, 1204) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.4012 + top-1 accuracy : 13/363 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1162]) +Dataloader batch - labels shape: torch.Size([1, 1162]) +Dataloader batch - attn_mask shape: torch.Size([1, 1162]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 466 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1162) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1162) torch.int64 + loss_mask : 260 / 1162 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1162, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1162, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1162, 1, 576) torch.bfloat16 + out : (1, 1162) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.1802 + top-1 accuracy : 10/260 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1506]) +Dataloader batch - labels shape: torch.Size([1, 1506]) +Dataloader batch - attn_mask shape: torch.Size([1, 1506]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 467 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1506) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1506) torch.int64 + loss_mask : 343 / 1506 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1506, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1506, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1506, 1, 576) torch.bfloat16 + out : (1, 1506) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.1910 + top-1 accuracy : 21/343 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1559]) +Dataloader batch - labels shape: torch.Size([1, 1559]) +Dataloader batch - attn_mask shape: torch.Size([1, 1559]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 468 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1559) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1559) torch.int64 + loss_mask : 359 / 1559 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1559, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1559, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1559, 1, 576) torch.bfloat16 + out : (1, 1559) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.2222 + top-1 accuracy : 20/359 = 5.6% + + [2026-04-29 13:46:54] iteration 217/ 500 | consumed samples: 868 | elapsed time per iteration (ms): 4547.7 | throughput (token/sec/GPU): 1194.2 | learning rate: 8.737589E-05 | global batch size: 4 | lm loss: 1.125494E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1432]) +Dataloader batch - labels shape: torch.Size([1, 1432]) +Dataloader batch - attn_mask shape: torch.Size([1, 1432]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 469 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1432) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1432) torch.int64 + loss_mask : 345 / 1432 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1432, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1432, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1432, 1, 576) torch.bfloat16 + out : (1, 1432) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2362 + top-1 accuracy : 3/345 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1531]) +Dataloader batch - labels shape: torch.Size([1, 1531]) +Dataloader batch - attn_mask shape: torch.Size([1, 1531]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 470 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1531) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1531) torch.int64 + loss_mask : 419 / 1531 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1531, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1531, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1531, 1, 576) torch.bfloat16 + out : (1, 1531) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.4014 + top-1 accuracy : 2/419 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 471 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 329 / 1508 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2274 + top-1 accuracy : 28/329 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1549]) +Dataloader batch - labels shape: torch.Size([1, 1549]) +Dataloader batch - attn_mask shape: torch.Size([1, 1549]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 472 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1549) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1549) torch.int64 + loss_mask : 403 / 1549 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1549, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1549, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1549, 1, 576) torch.bfloat16 + out : (1, 1549) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.3472 + top-1 accuracy : 3/403 = 0.7% + + [2026-04-29 13:46:58] iteration 218/ 500 | consumed samples: 872 | elapsed time per iteration (ms): 4676.2 | throughput (token/sec/GPU): 1287.4 | learning rate: 8.716727E-05 | global batch size: 4 | lm loss: 1.131041E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1872]) +Dataloader batch - labels shape: torch.Size([1, 1872]) +Dataloader batch - attn_mask shape: torch.Size([1, 1872]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 473 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1872) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1872) torch.int64 + loss_mask : 501 / 1872 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1872, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1872, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1872, 1, 576) torch.bfloat16 + out : (1, 1872) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 501 tokens) : 11.3389 + top-1 accuracy : 18/501 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1116]) +Dataloader batch - labels shape: torch.Size([1, 1116]) +Dataloader batch - attn_mask shape: torch.Size([1, 1116]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 474 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1116) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1116) torch.int64 + loss_mask : 263 / 1116 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1116, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1116, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1116, 1, 576) torch.bfloat16 + out : (1, 1116) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.2071 + top-1 accuracy : 1/263 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 475 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 344 / 1695 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.1295 + top-1 accuracy : 34/344 = 9.9% + +Dataloader batch - tokens shape: torch.Size([1, 1710]) +Dataloader batch - labels shape: torch.Size([1, 1710]) +Dataloader batch - attn_mask shape: torch.Size([1, 1710]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 476 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1710) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1710) torch.int64 + loss_mask : 369 / 1710 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1710, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1710, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1710, 1, 576) torch.bfloat16 + out : (1, 1710) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.1695 + top-1 accuracy : 38/369 = 10.3% + + [2026-04-29 13:47:03] iteration 219/ 500 | consumed samples: 876 | elapsed time per iteration (ms): 5107.9 | throughput (token/sec/GPU): 1251.6 | learning rate: 8.695719E-05 | global batch size: 4 | lm loss: 1.122432E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 477 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1420) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1420) torch.int64 + loss_mask : 352 / 1420 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1420, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1420, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1420, 1, 576) torch.bfloat16 + out : (1, 1420) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.3035 + top-1 accuracy : 15/352 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 478 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 370 / 1552 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.2539 + top-1 accuracy : 12/370 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1108]) +Dataloader batch - labels shape: torch.Size([1, 1108]) +Dataloader batch - attn_mask shape: torch.Size([1, 1108]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 479 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1108) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1108) torch.int64 + loss_mask : 298 / 1108 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1108, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1108, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1108, 1, 576) torch.bfloat16 + out : (1, 1108) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 298 tokens) : 11.3259 + top-1 accuracy : 3/298 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1540]) +Dataloader batch - labels shape: torch.Size([1, 1540]) +Dataloader batch - attn_mask shape: torch.Size([1, 1540]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 480 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1540) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1540) torch.int64 + loss_mask : 327 / 1540 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1540, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1540, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1540, 1, 576) torch.bfloat16 + out : (1, 1540) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2054 + top-1 accuracy : 33/327 = 10.1% + + [2026-04-29 13:47:08] iteration 220/ 500 | consumed samples: 880 | elapsed time per iteration (ms): 4417.1 | throughput (token/sec/GPU): 1272.3 | learning rate: 8.674567E-05 | global batch size: 4 | lm loss: 1.127101E+01 | loss scale: 1.0 | grad norm: 0.276 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4404.10, 4404.10) + forward-compute ................................: (3293.57, 3293.57) + backward-compute ...............................: (1108.59, 1108.59) + batch-generator ................................: (440.77, 440.77) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.41, 0.41) + params-all-gather ..............................: (0.12, 0.12) + optimizer-copy-to-main-grad ....................: (0.11, 0.11) + optimizer-clip-main-grad .......................: (0.43, 0.43) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.13, 1.13) + optimizer-copy-main-to-model-params ............: (0.19, 0.19) + optimizer ......................................: (2.55, 2.55) +Dataloader batch - tokens shape: torch.Size([1, 1862]) +Dataloader batch - labels shape: torch.Size([1, 1862]) +Dataloader batch - attn_mask shape: torch.Size([1, 1862]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 481 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1862) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1862) torch.int64 + loss_mask : 527 / 1862 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1862, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1862, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1862, 1, 576) torch.bfloat16 + out : (1, 1862) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 527 tokens) : 11.3271 + top-1 accuracy : 4/527 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 482 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 462 / 1436 tokens (32.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 462 tokens) : 11.4364 + top-1 accuracy : 2/462 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1766]) +Dataloader batch - labels shape: torch.Size([1, 1766]) +Dataloader batch - attn_mask shape: torch.Size([1, 1766]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 483 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1766) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1766) torch.int64 + loss_mask : 405 / 1766 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1766, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1766, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1766, 1, 576) torch.bfloat16 + out : (1, 1766) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.2233 + top-1 accuracy : 16/405 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 484 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 314 / 1388 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.2069 + top-1 accuracy : 15/314 = 4.8% + + [2026-04-29 13:47:13] iteration 221/ 500 | consumed samples: 884 | elapsed time per iteration (ms): 4984.5 | throughput (token/sec/GPU): 1294.4 | learning rate: 8.653271E-05 | global batch size: 4 | lm loss: 1.130995E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1085]) +Dataloader batch - labels shape: torch.Size([1, 1085]) +Dataloader batch - attn_mask shape: torch.Size([1, 1085]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 485 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1085) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1085) torch.int64 + loss_mask : 238 / 1085 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1085, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1085, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1085, 1, 576) torch.bfloat16 + out : (1, 1085) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 238 tokens) : 11.2145 + top-1 accuracy : 2/238 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1706]) +Dataloader batch - labels shape: torch.Size([1, 1706]) +Dataloader batch - attn_mask shape: torch.Size([1, 1706]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 486 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1706) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1706) torch.int64 + loss_mask : 347 / 1706 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1706, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1706, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1706, 1, 576) torch.bfloat16 + out : (1, 1706) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2175 + top-1 accuracy : 32/347 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 487 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 304 / 1388 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 304 tokens) : 11.2348 + top-1 accuracy : 18/304 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 918]) +Dataloader batch - labels shape: torch.Size([1, 918]) +Dataloader batch - attn_mask shape: torch.Size([1, 918]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 488 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 918) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 918) torch.int64 + loss_mask : 194 / 918 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (918, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (918, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (918, 1, 576) torch.bfloat16 + out : (1, 918) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 194 tokens) : 11.1938 + top-1 accuracy : 7/194 = 3.6% + + [2026-04-29 13:47:17] iteration 222/ 500 | consumed samples: 888 | elapsed time per iteration (ms): 4337.4 | throughput (token/sec/GPU): 1175.1 | learning rate: 8.631832E-05 | global batch size: 4 | lm loss: 1.121742E+01 | loss scale: 1.0 | grad norm: 0.314 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1274]) +Dataloader batch - labels shape: torch.Size([1, 1274]) +Dataloader batch - attn_mask shape: torch.Size([1, 1274]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 489 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1274) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1274) torch.int64 + loss_mask : 377 / 1274 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1274, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1274, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1274, 1, 576) torch.bfloat16 + out : (1, 1274) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.4184 + top-1 accuracy : 7/377 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1521]) +Dataloader batch - labels shape: torch.Size([1, 1521]) +Dataloader batch - attn_mask shape: torch.Size([1, 1521]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 490 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1521) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1521) torch.int64 + loss_mask : 365 / 1521 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1521, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1521, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1521, 1, 576) torch.bfloat16 + out : (1, 1521) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2738 + top-1 accuracy : 6/365 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1221]) +Dataloader batch - labels shape: torch.Size([1, 1221]) +Dataloader batch - attn_mask shape: torch.Size([1, 1221]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 491 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1221) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1221) torch.int64 + loss_mask : 293 / 1221 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1221, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1221, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1221, 1, 576) torch.bfloat16 + out : (1, 1221) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 293 tokens) : 11.2267 + top-1 accuracy : 6/293 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1233]) +Dataloader batch - labels shape: torch.Size([1, 1233]) +Dataloader batch - attn_mask shape: torch.Size([1, 1233]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 492 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1233) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1233) torch.int64 + loss_mask : 346 / 1233 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1233, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1233, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1233, 1, 576) torch.bfloat16 + out : (1, 1233) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.3293 + top-1 accuracy : 1/346 = 0.3% + + [2026-04-29 13:47:21] iteration 223/ 500 | consumed samples: 892 | elapsed time per iteration (ms): 4093.2 | throughput (token/sec/GPU): 1282.4 | learning rate: 8.610251E-05 | global batch size: 4 | lm loss: 1.131719E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1283]) +Dataloader batch - labels shape: torch.Size([1, 1283]) +Dataloader batch - attn_mask shape: torch.Size([1, 1283]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 493 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1283) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1283) torch.int64 + loss_mask : 392 / 1283 tokens (30.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1283, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1283, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1283, 1, 576) torch.bfloat16 + out : (1, 1283) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3674 + top-1 accuracy : 4/392 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 494 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 311 / 1492 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2056 + top-1 accuracy : 23/311 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1931]) +Dataloader batch - labels shape: torch.Size([1, 1931]) +Dataloader batch - attn_mask shape: torch.Size([1, 1931]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 495 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1931) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1931) torch.int64 + loss_mask : 561 / 1931 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1931, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1931, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1931, 1, 576) torch.bfloat16 + out : (1, 1931) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 561 tokens) : 11.4010 + top-1 accuracy : 14/561 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1285]) +Dataloader batch - labels shape: torch.Size([1, 1285]) +Dataloader batch - attn_mask shape: torch.Size([1, 1285]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 496 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1285) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1285) torch.int64 + loss_mask : 387 / 1285 tokens (30.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1285, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1285, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1285, 1, 576) torch.bfloat16 + out : (1, 1285) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.4330 + top-1 accuracy : 10/387 = 2.6% + + [2026-04-29 13:47:26] iteration 224/ 500 | consumed samples: 896 | elapsed time per iteration (ms): 4491.8 | throughput (token/sec/GPU): 1333.8 | learning rate: 8.588529E-05 | global batch size: 4 | lm loss: 1.136372E+01 | loss scale: 1.0 | grad norm: 0.311 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 497 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 363 / 1504 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.3138 + top-1 accuracy : 19/363 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1547]) +Dataloader batch - labels shape: torch.Size([1, 1547]) +Dataloader batch - attn_mask shape: torch.Size([1, 1547]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 498 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1547) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1547) torch.int64 + loss_mask : 426 / 1547 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1547, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1547, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1547, 1, 576) torch.bfloat16 + out : (1, 1547) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.3493 + top-1 accuracy : 19/426 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1491]) +Dataloader batch - labels shape: torch.Size([1, 1491]) +Dataloader batch - attn_mask shape: torch.Size([1, 1491]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 499 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1491) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1491) torch.int64 + loss_mask : 367 / 1491 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1491, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1491, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1491, 1, 576) torch.bfloat16 + out : (1, 1491) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2209 + top-1 accuracy : 9/367 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 500 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1423) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1423) torch.int64 + loss_mask : 273 / 1423 tokens (19.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1423, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1423, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1423, 1, 576) torch.bfloat16 + out : (1, 1423) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 273 tokens) : 11.0597 + top-1 accuracy : 10/273 = 3.7% + + [2026-04-29 13:47:31] iteration 225/ 500 | consumed samples: 900 | elapsed time per iteration (ms): 4760.3 | throughput (token/sec/GPU): 1253.1 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.125199E+01 | loss scale: 1.0 | grad norm: 0.291 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1794]) +Dataloader batch - labels shape: torch.Size([1, 1794]) +Dataloader batch - attn_mask shape: torch.Size([1, 1794]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 501 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1794) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1794) torch.int64 + loss_mask : 411 / 1794 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1794, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1794, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1794, 1, 576) torch.bfloat16 + out : (1, 1794) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.2474 + top-1 accuracy : 4/411 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1362]) +Dataloader batch - labels shape: torch.Size([1, 1362]) +Dataloader batch - attn_mask shape: torch.Size([1, 1362]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 502 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1362) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1362) torch.int64 + loss_mask : 404 / 1362 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1362, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1362, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1362, 1, 576) torch.bfloat16 + out : (1, 1362) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.3835 + top-1 accuracy : 10/404 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1151]) +Dataloader batch - labels shape: torch.Size([1, 1151]) +Dataloader batch - attn_mask shape: torch.Size([1, 1151]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 503 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1151) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1151) torch.int64 + loss_mask : 306 / 1151 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1151, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1151, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1151, 1, 576) torch.bfloat16 + out : (1, 1151) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.3540 + top-1 accuracy : 3/306 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1780]) +Dataloader batch - labels shape: torch.Size([1, 1780]) +Dataloader batch - attn_mask shape: torch.Size([1, 1780]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 504 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1780) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1780) torch.int64 + loss_mask : 397 / 1780 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1780, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1780, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1780, 1, 576) torch.bfloat16 + out : (1, 1780) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.2061 + top-1 accuracy : 19/397 = 4.8% + + [2026-04-29 13:47:35] iteration 226/ 500 | consumed samples: 904 | elapsed time per iteration (ms): 4715.1 | throughput (token/sec/GPU): 1291.0 | learning rate: 8.544665E-05 | global batch size: 4 | lm loss: 1.129431E+01 | loss scale: 1.0 | grad norm: 0.260 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 945]) +Dataloader batch - labels shape: torch.Size([1, 945]) +Dataloader batch - attn_mask shape: torch.Size([1, 945]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 505 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 945) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 945) torch.int64 + loss_mask : 190 / 945 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (945, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (945, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (945, 1, 576) torch.bfloat16 + out : (1, 945) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 190 tokens) : 11.1338 + top-1 accuracy : 8/190 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1524]) +Dataloader batch - labels shape: torch.Size([1, 1524]) +Dataloader batch - attn_mask shape: torch.Size([1, 1524]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 506 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1524) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1524) torch.int64 + loss_mask : 371 / 1524 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1524, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1524, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1524, 1, 576) torch.bfloat16 + out : (1, 1524) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.2563 + top-1 accuracy : 19/371 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1512]) +Dataloader batch - labels shape: torch.Size([1, 1512]) +Dataloader batch - attn_mask shape: torch.Size([1, 1512]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 507 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1512) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1512) torch.int64 + loss_mask : 383 / 1512 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1512, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1512, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1512, 1, 576) torch.bfloat16 + out : (1, 1512) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.3320 + top-1 accuracy : 32/383 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 1782]) +Dataloader batch - labels shape: torch.Size([1, 1782]) +Dataloader batch - attn_mask shape: torch.Size([1, 1782]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 508 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1782) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1782) torch.int64 + loss_mask : 420 / 1782 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1782, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1782, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1782, 1, 576) torch.bfloat16 + out : (1, 1782) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 420 tokens) : 11.2179 + top-1 accuracy : 0/420 = 0.0% + + [2026-04-29 13:47:40] iteration 227/ 500 | consumed samples: 908 | elapsed time per iteration (ms): 4628.4 | throughput (token/sec/GPU): 1245.1 | learning rate: 8.522525E-05 | global batch size: 4 | lm loss: 1.124868E+01 | loss scale: 1.0 | grad norm: 0.344 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1031]) +Dataloader batch - labels shape: torch.Size([1, 1031]) +Dataloader batch - attn_mask shape: torch.Size([1, 1031]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 509 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1031) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1031) torch.int64 + loss_mask : 264 / 1031 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1031, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1031, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1031, 1, 576) torch.bfloat16 + out : (1, 1031) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 264 tokens) : 11.2990 + top-1 accuracy : 1/264 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1766]) +Dataloader batch - labels shape: torch.Size([1, 1766]) +Dataloader batch - attn_mask shape: torch.Size([1, 1766]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 510 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1766) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1766) torch.int64 + loss_mask : 397 / 1766 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1766, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1766, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1766, 1, 576) torch.bfloat16 + out : (1, 1766) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.2319 + top-1 accuracy : 20/397 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1509]) +Dataloader batch - labels shape: torch.Size([1, 1509]) +Dataloader batch - attn_mask shape: torch.Size([1, 1509]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 511 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1509) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1509) torch.int64 + loss_mask : 363 / 1509 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1509, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1509, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1509, 1, 576) torch.bfloat16 + out : (1, 1509) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2742 + top-1 accuracy : 11/363 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1573]) +Dataloader batch - labels shape: torch.Size([1, 1573]) +Dataloader batch - attn_mask shape: torch.Size([1, 1573]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 512 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1573) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1573) torch.int64 + loss_mask : 392 / 1573 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1573, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1573, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1573, 1, 576) torch.bfloat16 + out : (1, 1573) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.2872 + top-1 accuracy : 17/392 = 4.3% + + [2026-04-29 13:47:45] iteration 228/ 500 | consumed samples: 912 | elapsed time per iteration (ms): 4600.3 | throughput (token/sec/GPU): 1277.9 | learning rate: 8.500247E-05 | global batch size: 4 | lm loss: 1.127060E+01 | loss scale: 1.0 | grad norm: 0.280 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1479]) +Dataloader batch - labels shape: torch.Size([1, 1479]) +Dataloader batch - attn_mask shape: torch.Size([1, 1479]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 513 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1479) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1479) torch.int64 + loss_mask : 441 / 1479 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1479, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1479, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1479, 1, 576) torch.bfloat16 + out : (1, 1479) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 441 tokens) : 11.3359 + top-1 accuracy : 5/441 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1680]) +Dataloader batch - labels shape: torch.Size([1, 1680]) +Dataloader batch - attn_mask shape: torch.Size([1, 1680]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 514 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1680) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1680) torch.int64 + loss_mask : 405 / 1680 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1680, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1680, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1680, 1, 576) torch.bfloat16 + out : (1, 1680) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.2861 + top-1 accuracy : 6/405 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1453]) +Dataloader batch - labels shape: torch.Size([1, 1453]) +Dataloader batch - attn_mask shape: torch.Size([1, 1453]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 515 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1453) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1453) torch.int64 + loss_mask : 393 / 1453 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1453, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1453, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1453, 1, 576) torch.bfloat16 + out : (1, 1453) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 393 tokens) : 11.3628 + top-1 accuracy : 6/393 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 946]) +Dataloader batch - labels shape: torch.Size([1, 946]) +Dataloader batch - attn_mask shape: torch.Size([1, 946]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 516 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 946) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 946) torch.int64 + loss_mask : 201 / 946 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (946, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (946, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (946, 1, 576) torch.bfloat16 + out : (1, 946) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 201 tokens) : 11.1687 + top-1 accuracy : 10/201 = 5.0% + + [2026-04-29 13:47:49] iteration 229/ 500 | consumed samples: 916 | elapsed time per iteration (ms): 4512.5 | throughput (token/sec/GPU): 1231.7 | learning rate: 8.477833E-05 | global batch size: 4 | lm loss: 1.130591E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1373]) +Dataloader batch - labels shape: torch.Size([1, 1373]) +Dataloader batch - attn_mask shape: torch.Size([1, 1373]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 517 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1373) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1373) torch.int64 + loss_mask : 331 / 1373 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1373, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1373, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1373, 1, 576) torch.bfloat16 + out : (1, 1373) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.2309 + top-1 accuracy : 9/331 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 518 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 341 / 1518 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.2506 + top-1 accuracy : 8/341 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1435]) +Dataloader batch - labels shape: torch.Size([1, 1435]) +Dataloader batch - attn_mask shape: torch.Size([1, 1435]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 519 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1435) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1435) torch.int64 + loss_mask : 344 / 1435 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1435, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1435, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1435, 1, 576) torch.bfloat16 + out : (1, 1435) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.2845 + top-1 accuracy : 17/344 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1855]) +Dataloader batch - labels shape: torch.Size([1, 1855]) +Dataloader batch - attn_mask shape: torch.Size([1, 1855]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 520 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1855) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1855) torch.int64 + loss_mask : 503 / 1855 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1855, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1855, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1855, 1, 576) torch.bfloat16 + out : (1, 1855) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 503 tokens) : 11.3650 + top-1 accuracy : 13/503 = 2.6% + + [2026-04-29 13:47:54] iteration 230/ 500 | consumed samples: 920 | elapsed time per iteration (ms): 4859.4 | throughput (token/sec/GPU): 1272.0 | learning rate: 8.455283E-05 | global batch size: 4 | lm loss: 1.129185E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1563]) +Dataloader batch - labels shape: torch.Size([1, 1563]) +Dataloader batch - attn_mask shape: torch.Size([1, 1563]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 521 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1563) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1563) torch.int64 + loss_mask : 368 / 1563 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1563, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1563, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1563, 1, 576) torch.bfloat16 + out : (1, 1563) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2350 + top-1 accuracy : 21/368 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 962]) +Dataloader batch - labels shape: torch.Size([1, 962]) +Dataloader batch - attn_mask shape: torch.Size([1, 962]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 522 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 962) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 962) torch.int64 + loss_mask : 205 / 962 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (962, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (962, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (962, 1, 576) torch.bfloat16 + out : (1, 962) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 205 tokens) : 11.1561 + top-1 accuracy : 5/205 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1138]) +Dataloader batch - labels shape: torch.Size([1, 1138]) +Dataloader batch - attn_mask shape: torch.Size([1, 1138]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 523 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1138) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1138) torch.int64 + loss_mask : 322 / 1138 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1138, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1138, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1138, 1, 576) torch.bfloat16 + out : (1, 1138) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.3748 + top-1 accuracy : 3/322 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1398]) +Dataloader batch - labels shape: torch.Size([1, 1398]) +Dataloader batch - attn_mask shape: torch.Size([1, 1398]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 524 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1398) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1398) torch.int64 + loss_mask : 335 / 1398 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1398, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1398, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1398, 1, 576) torch.bfloat16 + out : (1, 1398) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2448 + top-1 accuracy : 5/335 = 1.5% + + [2026-04-29 13:47:58] iteration 231/ 500 | consumed samples: 924 | elapsed time per iteration (ms): 4077.6 | throughput (token/sec/GPU): 1241.2 | learning rate: 8.432597E-05 | global batch size: 4 | lm loss: 1.126110E+01 | loss scale: 1.0 | grad norm: 0.261 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1931]) +Dataloader batch - labels shape: torch.Size([1, 1931]) +Dataloader batch - attn_mask shape: torch.Size([1, 1931]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 525 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1931) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1931) torch.int64 + loss_mask : 550 / 1931 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1931, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1931, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1931, 1, 576) torch.bfloat16 + out : (1, 1931) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 550 tokens) : 11.3169 + top-1 accuracy : 3/550 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 1527]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 526 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1527) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1527) torch.int64 + loss_mask : 421 / 1527 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1527, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1527, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1527, 1, 576) torch.bfloat16 + out : (1, 1527) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 421 tokens) : 11.4007 + top-1 accuracy : 1/421 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1092]) +Dataloader batch - labels shape: torch.Size([1, 1092]) +Dataloader batch - attn_mask shape: torch.Size([1, 1092]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 527 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1092) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1092) torch.int64 + loss_mask : 263 / 1092 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1092, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1092, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1092, 1, 576) torch.bfloat16 + out : (1, 1092) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.1970 + top-1 accuracy : 7/263 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1817]) +Dataloader batch - labels shape: torch.Size([1, 1817]) +Dataloader batch - attn_mask shape: torch.Size([1, 1817]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 528 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1817) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1817) torch.int64 + loss_mask : 451 / 1817 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1817, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1817, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1817, 1, 576) torch.bfloat16 + out : (1, 1817) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 451 tokens) : 11.2696 + top-1 accuracy : 6/451 = 1.3% + + [2026-04-29 13:48:03] iteration 232/ 500 | consumed samples: 928 | elapsed time per iteration (ms): 4713.3 | throughput (token/sec/GPU): 1350.8 | learning rate: 8.409777E-05 | global batch size: 4 | lm loss: 1.130646E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 529 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 391 / 1541 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3296 + top-1 accuracy : 26/391 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1488]) +Dataloader batch - labels shape: torch.Size([1, 1488]) +Dataloader batch - attn_mask shape: torch.Size([1, 1488]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 530 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1488) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1488) torch.int64 + loss_mask : 431 / 1488 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1488, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1488, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1488, 1, 576) torch.bfloat16 + out : (1, 1488) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.4230 + top-1 accuracy : 15/431 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1580]) +Dataloader batch - labels shape: torch.Size([1, 1580]) +Dataloader batch - attn_mask shape: torch.Size([1, 1580]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 531 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1580) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1580) torch.int64 + loss_mask : 365 / 1580 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1580, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1580, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1580, 1, 576) torch.bfloat16 + out : (1, 1580) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.1810 + top-1 accuracy : 5/365 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1742]) +Dataloader batch - labels shape: torch.Size([1, 1742]) +Dataloader batch - attn_mask shape: torch.Size([1, 1742]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 532 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1742) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1742) torch.int64 + loss_mask : 403 / 1742 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1742, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1742, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1742, 1, 576) torch.bfloat16 + out : (1, 1742) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.2238 + top-1 accuracy : 33/403 = 8.2% + + [2026-04-29 13:48:08] iteration 233/ 500 | consumed samples: 932 | elapsed time per iteration (ms): 5031.5 | throughput (token/sec/GPU): 1262.2 | learning rate: 8.386825E-05 | global batch size: 4 | lm loss: 1.129397E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1074]) +Dataloader batch - labels shape: torch.Size([1, 1074]) +Dataloader batch - attn_mask shape: torch.Size([1, 1074]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 533 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1074) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1074) torch.int64 + loss_mask : 253 / 1074 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1074, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1074, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1074, 1, 576) torch.bfloat16 + out : (1, 1074) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 253 tokens) : 11.2783 + top-1 accuracy : 3/253 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1808]) +Dataloader batch - labels shape: torch.Size([1, 1808]) +Dataloader batch - attn_mask shape: torch.Size([1, 1808]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 534 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1808) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1808) torch.int64 + loss_mask : 432 / 1808 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1808, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1808, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1808, 1, 576) torch.bfloat16 + out : (1, 1808) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 432 tokens) : 11.2805 + top-1 accuracy : 6/432 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1393]) +Dataloader batch - labels shape: torch.Size([1, 1393]) +Dataloader batch - attn_mask shape: torch.Size([1, 1393]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 535 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1393) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1393) torch.int64 + loss_mask : 329 / 1393 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1393, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1393, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1393, 1, 576) torch.bfloat16 + out : (1, 1393) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2249 + top-1 accuracy : 17/329 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1881]) +Dataloader batch - labels shape: torch.Size([1, 1881]) +Dataloader batch - attn_mask shape: torch.Size([1, 1881]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 536 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1881) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1881) torch.int64 + loss_mask : 542 / 1881 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1881, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1881, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1881, 1, 576) torch.bfloat16 + out : (1, 1881) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 542 tokens) : 11.3620 + top-1 accuracy : 18/542 = 3.3% + + [2026-04-29 13:48:12] iteration 234/ 500 | consumed samples: 936 | elapsed time per iteration (ms): 4799.4 | throughput (token/sec/GPU): 1282.7 | learning rate: 8.363740E-05 | global batch size: 4 | lm loss: 1.129676E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1408]) +Dataloader batch - labels shape: torch.Size([1, 1408]) +Dataloader batch - attn_mask shape: torch.Size([1, 1408]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 537 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1408) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1408) torch.int64 + loss_mask : 365 / 1408 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1408, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1408, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1408, 1, 576) torch.bfloat16 + out : (1, 1408) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2807 + top-1 accuracy : 9/365 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 538 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 356 / 1460 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2424 + top-1 accuracy : 15/356 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1669]) +Dataloader batch - labels shape: torch.Size([1, 1669]) +Dataloader batch - attn_mask shape: torch.Size([1, 1669]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 539 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1669) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1669) torch.int64 + loss_mask : 403 / 1669 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1669, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1669, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1669, 1, 576) torch.bfloat16 + out : (1, 1669) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.2617 + top-1 accuracy : 18/403 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1886]) +Dataloader batch - labels shape: torch.Size([1, 1886]) +Dataloader batch - attn_mask shape: torch.Size([1, 1886]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 540 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1886) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1886) torch.int64 + loss_mask : 532 / 1886 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1886, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1886, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1886, 1, 576) torch.bfloat16 + out : (1, 1886) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 532 tokens) : 11.3356 + top-1 accuracy : 24/532 = 4.5% + + [2026-04-29 13:48:17] iteration 235/ 500 | consumed samples: 940 | elapsed time per iteration (ms): 4841.4 | throughput (token/sec/GPU): 1326.7 | learning rate: 8.340524E-05 | global batch size: 4 | lm loss: 1.128547E+01 | loss scale: 1.0 | grad norm: 0.275 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1401]) +Dataloader batch - labels shape: torch.Size([1, 1401]) +Dataloader batch - attn_mask shape: torch.Size([1, 1401]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 541 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1401) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1401) torch.int64 + loss_mask : 349 / 1401 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1401, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1401, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1401, 1, 576) torch.bfloat16 + out : (1, 1401) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2817 + top-1 accuracy : 22/349 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1503]) +Dataloader batch - labels shape: torch.Size([1, 1503]) +Dataloader batch - attn_mask shape: torch.Size([1, 1503]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 542 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1503) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1503) torch.int64 + loss_mask : 358 / 1503 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1503, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1503, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1503, 1, 576) torch.bfloat16 + out : (1, 1503) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2394 + top-1 accuracy : 15/358 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1415]) +Dataloader batch - labels shape: torch.Size([1, 1415]) +Dataloader batch - attn_mask shape: torch.Size([1, 1415]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 543 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1415) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1415) torch.int64 + loss_mask : 354 / 1415 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1415, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1415, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1415, 1, 576) torch.bfloat16 + out : (1, 1415) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2997 + top-1 accuracy : 10/354 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1175]) +Dataloader batch - labels shape: torch.Size([1, 1175]) +Dataloader batch - attn_mask shape: torch.Size([1, 1175]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 544 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1175) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1175) torch.int64 + loss_mask : 271 / 1175 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1175, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1175, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1175, 1, 576) torch.bfloat16 + out : (1, 1175) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 271 tokens) : 11.2387 + top-1 accuracy : 2/271 = 0.7% + + [2026-04-29 13:48:22] iteration 236/ 500 | consumed samples: 944 | elapsed time per iteration (ms): 4469.5 | throughput (token/sec/GPU): 1229.2 | learning rate: 8.317177E-05 | global batch size: 4 | lm loss: 1.126637E+01 | loss scale: 1.0 | grad norm: 0.273 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1653]) +Dataloader batch - labels shape: torch.Size([1, 1653]) +Dataloader batch - attn_mask shape: torch.Size([1, 1653]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 545 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1653) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1653) torch.int64 + loss_mask : 381 / 1653 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1653, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1653, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1653, 1, 576) torch.bfloat16 + out : (1, 1653) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.2768 + top-1 accuracy : 12/381 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1067]) +Dataloader batch - labels shape: torch.Size([1, 1067]) +Dataloader batch - attn_mask shape: torch.Size([1, 1067]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 546 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1067) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1067) torch.int64 + loss_mask : 241 / 1067 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1067, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1067, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1067, 1, 576) torch.bfloat16 + out : (1, 1067) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 241 tokens) : 11.2334 + top-1 accuracy : 4/241 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 547 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 328 / 1487 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.1875 + top-1 accuracy : 36/328 = 11.0% + +Dataloader batch - tokens shape: torch.Size([1, 1413]) +Dataloader batch - labels shape: torch.Size([1, 1413]) +Dataloader batch - attn_mask shape: torch.Size([1, 1413]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 548 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1413) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1413) torch.int64 + loss_mask : 342 / 1413 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1413, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1413, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1413, 1, 576) torch.bfloat16 + out : (1, 1413) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2416 + top-1 accuracy : 2/342 = 0.6% + + [2026-04-29 13:48:26] iteration 237/ 500 | consumed samples: 948 | elapsed time per iteration (ms): 4479.3 | throughput (token/sec/GPU): 1254.7 | learning rate: 8.293701E-05 | global batch size: 4 | lm loss: 1.123670E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 549 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 331 / 1400 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.2436 + top-1 accuracy : 5/331 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1077]) +Dataloader batch - labels shape: torch.Size([1, 1077]) +Dataloader batch - attn_mask shape: torch.Size([1, 1077]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 550 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1077) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1077) torch.int64 + loss_mask : 233 / 1077 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1077, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1077, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1077, 1, 576) torch.bfloat16 + out : (1, 1077) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 233 tokens) : 11.2093 + top-1 accuracy : 24/233 = 10.3% + +Dataloader batch - tokens shape: torch.Size([1, 1512]) +Dataloader batch - labels shape: torch.Size([1, 1512]) +Dataloader batch - attn_mask shape: torch.Size([1, 1512]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 551 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1512) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1512) torch.int64 + loss_mask : 408 / 1512 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1512, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1512, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1512, 1, 576) torch.bfloat16 + out : (1, 1512) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3340 + top-1 accuracy : 4/408 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 969]) +Dataloader batch - labels shape: torch.Size([1, 969]) +Dataloader batch - attn_mask shape: torch.Size([1, 969]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 552 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 969) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 969) torch.int64 + loss_mask : 228 / 969 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (969, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (969, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (969, 1, 576) torch.bfloat16 + out : (1, 969) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 228 tokens) : 11.2370 + top-1 accuracy : 19/228 = 8.3% + + [2026-04-29 13:48:30] iteration 238/ 500 | consumed samples: 952 | elapsed time per iteration (ms): 4122.9 | throughput (token/sec/GPU): 1202.6 | learning rate: 8.270096E-05 | global batch size: 4 | lm loss: 1.126643E+01 | loss scale: 1.0 | grad norm: 0.276 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 553 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 328 / 1388 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.2827 + top-1 accuracy : 13/328 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1594]) +Dataloader batch - labels shape: torch.Size([1, 1594]) +Dataloader batch - attn_mask shape: torch.Size([1, 1594]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 554 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1594) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1594) torch.int64 + loss_mask : 360 / 1594 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1594, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1594, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1594, 1, 576) torch.bfloat16 + out : (1, 1594) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.1865 + top-1 accuracy : 18/360 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1707]) +Dataloader batch - labels shape: torch.Size([1, 1707]) +Dataloader batch - attn_mask shape: torch.Size([1, 1707]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 555 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1707) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1707) torch.int64 + loss_mask : 365 / 1707 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1707, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1707, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1707, 1, 576) torch.bfloat16 + out : (1, 1707) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.1523 + top-1 accuracy : 24/365 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1493]) +Dataloader batch - labels shape: torch.Size([1, 1493]) +Dataloader batch - attn_mask shape: torch.Size([1, 1493]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 556 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1493) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1493) torch.int64 + loss_mask : 341 / 1493 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1493, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1493, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1493, 1, 576) torch.bfloat16 + out : (1, 1493) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.2345 + top-1 accuracy : 21/341 = 6.2% + + [2026-04-29 13:48:35] iteration 239/ 500 | consumed samples: 956 | elapsed time per iteration (ms): 4922.0 | throughput (token/sec/GPU): 1256.0 | learning rate: 8.246364E-05 | global batch size: 4 | lm loss: 1.121191E+01 | loss scale: 1.0 | grad norm: 0.250 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1481]) +Dataloader batch - labels shape: torch.Size([1, 1481]) +Dataloader batch - attn_mask shape: torch.Size([1, 1481]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 557 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1481) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1481) torch.int64 + loss_mask : 296 / 1481 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1481, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1481, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1481, 1, 576) torch.bfloat16 + out : (1, 1481) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 296 tokens) : 11.1696 + top-1 accuracy : 30/296 = 10.1% + +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 558 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1385) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1385) torch.int64 + loss_mask : 327 / 1385 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1385, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1385, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1385, 1, 576) torch.bfloat16 + out : (1, 1385) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2907 + top-1 accuracy : 10/327 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1394]) +Dataloader batch - labels shape: torch.Size([1, 1394]) +Dataloader batch - attn_mask shape: torch.Size([1, 1394]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 559 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1394) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1394) torch.int64 + loss_mask : 332 / 1394 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1394, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1394, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1394, 1, 576) torch.bfloat16 + out : (1, 1394) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.2747 + top-1 accuracy : 14/332 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 980]) +Dataloader batch - labels shape: torch.Size([1, 980]) +Dataloader batch - attn_mask shape: torch.Size([1, 980]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 560 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 980) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 980) torch.int64 + loss_mask : 236 / 980 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (980, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (980, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (980, 1, 576) torch.bfloat16 + out : (1, 980) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 236 tokens) : 11.2374 + top-1 accuracy : 18/236 = 7.6% + + [2026-04-29 13:48:40] iteration 240/ 500 | consumed samples: 960 | elapsed time per iteration (ms): 4397.8 | throughput (token/sec/GPU): 1191.5 | learning rate: 8.222505E-05 | global batch size: 4 | lm loss: 1.124556E+01 | loss scale: 1.0 | grad norm: 0.286 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4370.82, 4370.82) + forward-compute ................................: (3238.27, 3238.27) + backward-compute ...............................: (1128.51, 1128.51) + batch-generator ................................: (419.38, 419.38) + layernorm-grads-all-reduce .....................: (0.07, 0.07) + embedding-grads-all-reduce .....................: (0.11, 0.11) + all-grads-sync .................................: (1.09, 1.09) + params-all-gather ..............................: (0.41, 0.41) + optimizer-copy-to-main-grad ....................: (0.34, 0.34) + optimizer-clip-main-grad .......................: (1.35, 1.35) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.21, 2.21) + optimizer-copy-main-to-model-params ............: (0.58, 0.58) + optimizer ......................................: (6.95, 6.95) +Dataloader batch - tokens shape: torch.Size([1, 1958]) +Dataloader batch - labels shape: torch.Size([1, 1958]) +Dataloader batch - attn_mask shape: torch.Size([1, 1958]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 561 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1958) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1958) torch.int64 + loss_mask : 576 / 1958 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1958, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1958, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1958, 1, 576) torch.bfloat16 + out : (1, 1958) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 576 tokens) : 11.3796 + top-1 accuracy : 7/576 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1545]) +Dataloader batch - labels shape: torch.Size([1, 1545]) +Dataloader batch - attn_mask shape: torch.Size([1, 1545]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 562 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1545) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1545) torch.int64 + loss_mask : 400 / 1545 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1545, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1545, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1545, 1, 576) torch.bfloat16 + out : (1, 1545) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.2796 + top-1 accuracy : 11/400 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1536]) +Dataloader batch - labels shape: torch.Size([1, 1536]) +Dataloader batch - attn_mask shape: torch.Size([1, 1536]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 563 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1536) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1536) torch.int64 + loss_mask : 386 / 1536 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1536, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1536, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1536, 1, 576) torch.bfloat16 + out : (1, 1536) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3054 + top-1 accuracy : 7/386 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1511]) +Dataloader batch - labels shape: torch.Size([1, 1511]) +Dataloader batch - attn_mask shape: torch.Size([1, 1511]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 564 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1511) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1511) torch.int64 + loss_mask : 347 / 1511 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1511, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1511, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1511, 1, 576) torch.bfloat16 + out : (1, 1511) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2341 + top-1 accuracy : 11/347 = 3.2% + + [2026-04-29 13:48:45] iteration 241/ 500 | consumed samples: 964 | elapsed time per iteration (ms): 5063.1 | throughput (token/sec/GPU): 1293.7 | learning rate: 8.198521E-05 | global batch size: 4 | lm loss: 1.130989E+01 | loss scale: 1.0 | grad norm: 0.259 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1047]) +Dataloader batch - labels shape: torch.Size([1, 1047]) +Dataloader batch - attn_mask shape: torch.Size([1, 1047]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 565 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1047) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1047) torch.int64 + loss_mask : 295 / 1047 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1047, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1047, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1047, 1, 576) torch.bfloat16 + out : (1, 1047) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.3607 + top-1 accuracy : 5/295 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1425]) +Dataloader batch - labels shape: torch.Size([1, 1425]) +Dataloader batch - attn_mask shape: torch.Size([1, 1425]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 566 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1425) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1425) torch.int64 + loss_mask : 365 / 1425 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1425, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1425, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1425, 1, 576) torch.bfloat16 + out : (1, 1425) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2794 + top-1 accuracy : 16/365 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1015]) +Dataloader batch - labels shape: torch.Size([1, 1015]) +Dataloader batch - attn_mask shape: torch.Size([1, 1015]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 567 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1015) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1015) torch.int64 + loss_mask : 234 / 1015 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1015, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1015, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1015, 1, 576) torch.bfloat16 + out : (1, 1015) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 234 tokens) : 11.2312 + top-1 accuracy : 6/234 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1495]) +Dataloader batch - labels shape: torch.Size([1, 1495]) +Dataloader batch - attn_mask shape: torch.Size([1, 1495]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 568 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1495) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1495) torch.int64 + loss_mask : 316 / 1495 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1495, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1495, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1495, 1, 576) torch.bfloat16 + out : (1, 1495) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.1997 + top-1 accuracy : 32/316 = 10.1% + + [2026-04-29 13:48:49] iteration 242/ 500 | consumed samples: 968 | elapsed time per iteration (ms): 4115.2 | throughput (token/sec/GPU): 1210.6 | learning rate: 8.174411E-05 | global batch size: 4 | lm loss: 1.126909E+01 | loss scale: 1.0 | grad norm: 0.258 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 970]) +Dataloader batch - labels shape: torch.Size([1, 970]) +Dataloader batch - attn_mask shape: torch.Size([1, 970]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 569 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 970) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 970) torch.int64 + loss_mask : 237 / 970 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (970, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (970, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (970, 1, 576) torch.bfloat16 + out : (1, 970) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 237 tokens) : 11.2576 + top-1 accuracy : 7/237 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1512]) +Dataloader batch - labels shape: torch.Size([1, 1512]) +Dataloader batch - attn_mask shape: torch.Size([1, 1512]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 570 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1512) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1512) torch.int64 + loss_mask : 373 / 1512 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1512, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1512, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1512, 1, 576) torch.bfloat16 + out : (1, 1512) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.3169 + top-1 accuracy : 6/373 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1478]) +Dataloader batch - labels shape: torch.Size([1, 1478]) +Dataloader batch - attn_mask shape: torch.Size([1, 1478]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 571 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1478) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1478) torch.int64 + loss_mask : 327 / 1478 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1478, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1478, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1478, 1, 576) torch.bfloat16 + out : (1, 1478) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2127 + top-1 accuracy : 11/327 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1829]) +Dataloader batch - labels shape: torch.Size([1, 1829]) +Dataloader batch - attn_mask shape: torch.Size([1, 1829]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 572 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1829) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1829) torch.int64 + loss_mask : 451 / 1829 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1829, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1829, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1829, 1, 576) torch.bfloat16 + out : (1, 1829) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 451 tokens) : 11.2620 + top-1 accuracy : 4/451 = 0.9% + + [2026-04-29 13:48:54] iteration 243/ 500 | consumed samples: 972 | elapsed time per iteration (ms): 4617.9 | throughput (token/sec/GPU): 1253.6 | learning rate: 8.150178E-05 | global batch size: 4 | lm loss: 1.126440E+01 | loss scale: 1.0 | grad norm: 0.261 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1851]) +Dataloader batch - labels shape: torch.Size([1, 1851]) +Dataloader batch - attn_mask shape: torch.Size([1, 1851]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 573 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1851) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1851) torch.int64 + loss_mask : 501 / 1851 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1851, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1851, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1851, 1, 576) torch.bfloat16 + out : (1, 1851) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 501 tokens) : 11.3152 + top-1 accuracy : 8/501 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1051]) +Dataloader batch - labels shape: torch.Size([1, 1051]) +Dataloader batch - attn_mask shape: torch.Size([1, 1051]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 574 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1051) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1051) torch.int64 + loss_mask : 307 / 1051 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1051, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1051, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1051, 1, 576) torch.bfloat16 + out : (1, 1051) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.3630 + top-1 accuracy : 4/307 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1821]) +Dataloader batch - labels shape: torch.Size([1, 1821]) +Dataloader batch - attn_mask shape: torch.Size([1, 1821]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 575 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1821) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1821) torch.int64 + loss_mask : 456 / 1821 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1821, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1821, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1821, 1, 576) torch.bfloat16 + out : (1, 1821) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 456 tokens) : 11.3164 + top-1 accuracy : 21/456 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1698]) +Dataloader batch - labels shape: torch.Size([1, 1698]) +Dataloader batch - attn_mask shape: torch.Size([1, 1698]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 576 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1698) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1698) torch.int64 + loss_mask : 348 / 1698 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1698, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1698, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1698, 1, 576) torch.bfloat16 + out : (1, 1698) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2076 + top-1 accuracy : 27/348 = 7.8% + + [2026-04-29 13:48:58] iteration 244/ 500 | consumed samples: 976 | elapsed time per iteration (ms): 4939.9 | throughput (token/sec/GPU): 1299.8 | learning rate: 8.125821E-05 | global batch size: 4 | lm loss: 1.130139E+01 | loss scale: 1.0 | grad norm: 0.274 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 577 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 381 / 1475 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.3554 + top-1 accuracy : 10/381 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 578 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 374 / 1496 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.2579 + top-1 accuracy : 28/374 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1429]) +Dataloader batch - labels shape: torch.Size([1, 1429]) +Dataloader batch - attn_mask shape: torch.Size([1, 1429]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 579 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1429) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1429) torch.int64 + loss_mask : 339 / 1429 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1429, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1429, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1429, 1, 576) torch.bfloat16 + out : (1, 1429) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2925 + top-1 accuracy : 5/339 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1472]) +Dataloader batch - labels shape: torch.Size([1, 1472]) +Dataloader batch - attn_mask shape: torch.Size([1, 1472]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 580 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1472) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1472) torch.int64 + loss_mask : 310 / 1472 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1472, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1472, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1472, 1, 576) torch.bfloat16 + out : (1, 1472) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 310 tokens) : 11.1635 + top-1 accuracy : 31/310 = 10.0% + + [2026-04-29 13:49:03] iteration 245/ 500 | consumed samples: 980 | elapsed time per iteration (ms): 4919.4 | throughput (token/sec/GPU): 1193.6 | learning rate: 8.101343E-05 | global batch size: 4 | lm loss: 1.127187E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1751]) +Dataloader batch - labels shape: torch.Size([1, 1751]) +Dataloader batch - attn_mask shape: torch.Size([1, 1751]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 581 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1751) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1751) torch.int64 + loss_mask : 399 / 1751 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1751, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1751, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1751, 1, 576) torch.bfloat16 + out : (1, 1751) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.2240 + top-1 accuracy : 9/399 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1515]) +Dataloader batch - labels shape: torch.Size([1, 1515]) +Dataloader batch - attn_mask shape: torch.Size([1, 1515]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 582 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1515) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1515) torch.int64 + loss_mask : 366 / 1515 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1515, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1515, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1515, 1, 576) torch.bfloat16 + out : (1, 1515) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2448 + top-1 accuracy : 29/366 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 583 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 318 / 1422 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.2878 + top-1 accuracy : 24/318 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 584 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1467) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1467) torch.int64 + loss_mask : 314 / 1467 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1467, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1467, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1467, 1, 576) torch.bfloat16 + out : (1, 1467) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.2162 + top-1 accuracy : 12/314 = 3.8% + + [2026-04-29 13:49:08] iteration 246/ 500 | consumed samples: 984 | elapsed time per iteration (ms): 4951.4 | throughput (token/sec/GPU): 1243.1 | learning rate: 8.076744E-05 | global batch size: 4 | lm loss: 1.124220E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1410]) +Dataloader batch - labels shape: torch.Size([1, 1410]) +Dataloader batch - attn_mask shape: torch.Size([1, 1410]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 585 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1410) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1410) torch.int64 + loss_mask : 340 / 1410 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1410, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1410, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1410, 1, 576) torch.bfloat16 + out : (1, 1410) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2743 + top-1 accuracy : 19/340 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1369]) +Dataloader batch - labels shape: torch.Size([1, 1369]) +Dataloader batch - attn_mask shape: torch.Size([1, 1369]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 586 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1369) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1369) torch.int64 + loss_mask : 293 / 1369 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1369, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1369, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1369, 1, 576) torch.bfloat16 + out : (1, 1369) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 293 tokens) : 11.1861 + top-1 accuracy : 9/293 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 587 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 297 / 1475 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.1892 + top-1 accuracy : 32/297 = 10.8% + +Dataloader batch - tokens shape: torch.Size([1, 1633]) +Dataloader batch - labels shape: torch.Size([1, 1633]) +Dataloader batch - attn_mask shape: torch.Size([1, 1633]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 588 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1633) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1633) torch.int64 + loss_mask : 389 / 1633 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1633, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1633, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1633, 1, 576) torch.bfloat16 + out : (1, 1633) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.2494 + top-1 accuracy : 10/389 = 2.6% + + [2026-04-29 13:49:13] iteration 247/ 500 | consumed samples: 988 | elapsed time per iteration (ms): 4856.3 | throughput (token/sec/GPU): 1212.2 | learning rate: 8.052024E-05 | global batch size: 4 | lm loss: 1.122821E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 589 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 343 / 1492 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.2952 + top-1 accuracy : 25/343 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1135]) +Dataloader batch - labels shape: torch.Size([1, 1135]) +Dataloader batch - attn_mask shape: torch.Size([1, 1135]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 590 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1135) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1135) torch.int64 + loss_mask : 339 / 1135 tokens (29.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1135, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1135, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1135, 1, 576) torch.bfloat16 + out : (1, 1135) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.4150 + top-1 accuracy : 15/339 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1884]) +Dataloader batch - labels shape: torch.Size([1, 1884]) +Dataloader batch - attn_mask shape: torch.Size([1, 1884]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 591 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1884) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1884) torch.int64 + loss_mask : 540 / 1884 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1884, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1884, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1884, 1, 576) torch.bfloat16 + out : (1, 1884) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 540 tokens) : 11.3763 + top-1 accuracy : 25/540 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 999]) +Dataloader batch - labels shape: torch.Size([1, 999]) +Dataloader batch - attn_mask shape: torch.Size([1, 999]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 592 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 999) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 999) torch.int64 + loss_mask : 234 / 999 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (999, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (999, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (999, 1, 576) torch.bfloat16 + out : (1, 999) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 234 tokens) : 11.2550 + top-1 accuracy : 18/234 = 7.7% + + [2026-04-29 13:49:18] iteration 248/ 500 | consumed samples: 992 | elapsed time per iteration (ms): 4306.4 | throughput (token/sec/GPU): 1279.5 | learning rate: 8.027186E-05 | global batch size: 4 | lm loss: 1.134673E+01 | loss scale: 1.0 | grad norm: 0.266 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 974]) +Dataloader batch - labels shape: torch.Size([1, 974]) +Dataloader batch - attn_mask shape: torch.Size([1, 974]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 593 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 974) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 974) torch.int64 + loss_mask : 224 / 974 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (974, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (974, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (974, 1, 576) torch.bfloat16 + out : (1, 974) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 224 tokens) : 11.2626 + top-1 accuracy : 5/224 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1562]) +Dataloader batch - labels shape: torch.Size([1, 1562]) +Dataloader batch - attn_mask shape: torch.Size([1, 1562]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 594 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1562) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1562) torch.int64 + loss_mask : 423 / 1562 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1562, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1562, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1562, 1, 576) torch.bfloat16 + out : (1, 1562) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 423 tokens) : 11.3397 + top-1 accuracy : 9/423 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 595 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 386 / 1438 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3639 + top-1 accuracy : 24/386 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1442]) +Dataloader batch - labels shape: torch.Size([1, 1442]) +Dataloader batch - attn_mask shape: torch.Size([1, 1442]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 596 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1442) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1442) torch.int64 + loss_mask : 345 / 1442 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1442, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1442, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1442, 1, 576) torch.bfloat16 + out : (1, 1442) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.3160 + top-1 accuracy : 6/345 = 1.7% + + [2026-04-29 13:49:22] iteration 249/ 500 | consumed samples: 996 | elapsed time per iteration (ms): 4457.5 | throughput (token/sec/GPU): 1215.0 | learning rate: 8.002231E-05 | global batch size: 4 | lm loss: 1.132802E+01 | loss scale: 1.0 | grad norm: 0.275 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1882]) +Dataloader batch - labels shape: torch.Size([1, 1882]) +Dataloader batch - attn_mask shape: torch.Size([1, 1882]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 597 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1882) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1882) torch.int64 + loss_mask : 510 / 1882 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1882, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1882, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1882, 1, 576) torch.bfloat16 + out : (1, 1882) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 510 tokens) : 11.4072 + top-1 accuracy : 12/510 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 598 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 300 / 1492 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.1580 + top-1 accuracy : 27/300 = 9.0% + +Dataloader batch - tokens shape: torch.Size([1, 1464]) +Dataloader batch - labels shape: torch.Size([1, 1464]) +Dataloader batch - attn_mask shape: torch.Size([1, 1464]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 599 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1464) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1464) torch.int64 + loss_mask : 413 / 1464 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1464, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1464, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1464, 1, 576) torch.bfloat16 + out : (1, 1464) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.3740 + top-1 accuracy : 6/413 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1471]) +Dataloader batch - labels shape: torch.Size([1, 1471]) +Dataloader batch - attn_mask shape: torch.Size([1, 1471]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 600 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1471) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1471) torch.int64 + loss_mask : 295 / 1471 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1471, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1471, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1471, 1, 576) torch.bfloat16 + out : (1, 1471) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.1726 + top-1 accuracy : 35/295 = 11.9% + + [2026-04-29 13:49:27] iteration 250/ 500 | consumed samples: 1000 | elapsed time per iteration (ms): 4899.7 | throughput (token/sec/GPU): 1287.6 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.130331E+01 | loss scale: 1.0 | grad norm: 0.231 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 250 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 250 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 250 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2086.61, 2086.61) +Dataloader batch - tokens shape: torch.Size([1, 1061]) +Dataloader batch - labels shape: torch.Size([1, 1061]) +Dataloader batch - attn_mask shape: torch.Size([1, 1061]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 601 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1061) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1061) torch.int64 + loss_mask : 207 / 1061 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1061, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1061, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1061, 1, 576) torch.bfloat16 + out : (1, 1061) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 207 tokens) : 11.1948 + top-1 accuracy : 16/207 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1870]) +Dataloader batch - labels shape: torch.Size([1, 1870]) +Dataloader batch - attn_mask shape: torch.Size([1, 1870]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 602 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1870) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1870) torch.int64 + loss_mask : 509 / 1870 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1870, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1870, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1870, 1, 576) torch.bfloat16 + out : (1, 1870) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 509 tokens) : 11.3227 + top-1 accuracy : 9/509 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1494]) +Dataloader batch - labels shape: torch.Size([1, 1494]) +Dataloader batch - attn_mask shape: torch.Size([1, 1494]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 603 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1494) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1494) torch.int64 + loss_mask : 317 / 1494 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1494, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1494, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1494, 1, 576) torch.bfloat16 + out : (1, 1494) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2014 + top-1 accuracy : 26/317 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1699]) +Dataloader batch - labels shape: torch.Size([1, 1699]) +Dataloader batch - attn_mask shape: torch.Size([1, 1699]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 604 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1699) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1699) torch.int64 + loss_mask : 353 / 1699 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1699, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1699, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1699, 1, 576) torch.bfloat16 + out : (1, 1699) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.1603 + top-1 accuracy : 16/353 = 4.5% + + [2026-04-29 13:49:34] iteration 251/ 500 | consumed samples: 1004 | elapsed time per iteration (ms): 4825.2 | throughput (token/sec/GPU): 1269.2 | learning rate: 7.951968E-05 | global batch size: 4 | lm loss: 1.123449E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 759]) +Dataloader batch - labels shape: torch.Size([1, 759]) +Dataloader batch - attn_mask shape: torch.Size([1, 759]) +Dataloader batch - image_grid_thw shape: torch.Size([15, 3]) + +==================================================================== + PIPELINE — STEP 605 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 759) torch.int64 + images : (15, 3, 1024, 1024) torch.bfloat16 + labels : (1, 759) torch.int64 + loss_mask : 156 / 759 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (15, 3, 1024, 1024) torch.bfloat16 + out : (15, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (15, 3072) + out : (15, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (759, 1, 576) torch.bfloat16 grad=False + image slots : 15 tokens replaced with vision embeddings + combined : (759, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (759, 1, 576) torch.bfloat16 + out : (1, 759) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 156 tokens) : 11.1549 + top-1 accuracy : 3/156 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1576]) +Dataloader batch - labels shape: torch.Size([1, 1576]) +Dataloader batch - attn_mask shape: torch.Size([1, 1576]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 606 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1576) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1576) torch.int64 + loss_mask : 353 / 1576 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1576, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1576, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1576, 1, 576) torch.bfloat16 + out : (1, 1576) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.2427 + top-1 accuracy : 12/353 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1530]) +Dataloader batch - labels shape: torch.Size([1, 1530]) +Dataloader batch - attn_mask shape: torch.Size([1, 1530]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 607 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1530) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1530) torch.int64 + loss_mask : 410 / 1530 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1530, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1530, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1530, 1, 576) torch.bfloat16 + out : (1, 1530) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.2824 + top-1 accuracy : 5/410 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 608 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1420) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1420) torch.int64 + loss_mask : 362 / 1420 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1420, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1420, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1420, 1, 576) torch.bfloat16 + out : (1, 1420) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.3190 + top-1 accuracy : 5/362 = 1.4% + + [2026-04-29 13:49:38] iteration 252/ 500 | consumed samples: 1008 | elapsed time per iteration (ms): 4322.0 | throughput (token/sec/GPU): 1222.8 | learning rate: 7.926664E-05 | global batch size: 4 | lm loss: 1.126627E+01 | loss scale: 1.0 | grad norm: 0.247 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1033]) +Dataloader batch - labels shape: torch.Size([1, 1033]) +Dataloader batch - attn_mask shape: torch.Size([1, 1033]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 609 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1033) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1033) torch.int64 + loss_mask : 250 / 1033 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1033, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1033, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1033, 1, 576) torch.bfloat16 + out : (1, 1033) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 250 tokens) : 11.3139 + top-1 accuracy : 8/250 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 610 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1191) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1191) torch.int64 + loss_mask : 259 / 1191 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1191, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1191, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1191, 1, 576) torch.bfloat16 + out : (1, 1191) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 259 tokens) : 11.2276 + top-1 accuracy : 17/259 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 611 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 386 / 1532 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3463 + top-1 accuracy : 14/386 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 612 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 457 / 1504 tokens (30.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 457 tokens) : 11.3903 + top-1 accuracy : 3/457 = 0.7% + + [2026-04-29 13:49:42] iteration 253/ 500 | consumed samples: 1012 | elapsed time per iteration (ms): 4285.0 | throughput (token/sec/GPU): 1227.5 | learning rate: 7.901245E-05 | global batch size: 4 | lm loss: 1.133243E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1510]) +Dataloader batch - labels shape: torch.Size([1, 1510]) +Dataloader batch - attn_mask shape: torch.Size([1, 1510]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 613 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1510) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1510) torch.int64 + loss_mask : 366 / 1510 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1510, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1510, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1510, 1, 576) torch.bfloat16 + out : (1, 1510) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2483 + top-1 accuracy : 22/366 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1080]) +Dataloader batch - labels shape: torch.Size([1, 1080]) +Dataloader batch - attn_mask shape: torch.Size([1, 1080]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 614 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1080) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1080) torch.int64 + loss_mask : 312 / 1080 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1080, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1080, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1080, 1, 576) torch.bfloat16 + out : (1, 1080) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.3477 + top-1 accuracy : 3/312 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1753]) +Dataloader batch - labels shape: torch.Size([1, 1753]) +Dataloader batch - attn_mask shape: torch.Size([1, 1753]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 615 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1753) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1753) torch.int64 + loss_mask : 397 / 1753 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1753, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1753, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1753, 1, 576) torch.bfloat16 + out : (1, 1753) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.2399 + top-1 accuracy : 31/397 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1659]) +Dataloader batch - labels shape: torch.Size([1, 1659]) +Dataloader batch - attn_mask shape: torch.Size([1, 1659]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 616 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1659) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1659) torch.int64 + loss_mask : 363 / 1659 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1659, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1659, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1659, 1, 576) torch.bfloat16 + out : (1, 1659) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.1810 + top-1 accuracy : 33/363 = 9.1% + + [2026-04-29 13:49:47] iteration 254/ 500 | consumed samples: 1016 | elapsed time per iteration (ms): 4821.8 | throughput (token/sec/GPU): 1244.8 | learning rate: 7.875715E-05 | global batch size: 4 | lm loss: 1.125052E+01 | loss scale: 1.0 | grad norm: 0.220 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1471]) +Dataloader batch - labels shape: torch.Size([1, 1471]) +Dataloader batch - attn_mask shape: torch.Size([1, 1471]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 617 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1471) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1471) torch.int64 + loss_mask : 323 / 1471 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1471, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1471, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1471, 1, 576) torch.bfloat16 + out : (1, 1471) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.1689 + top-1 accuracy : 9/323 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 618 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 326 / 1424 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2091 + top-1 accuracy : 21/326 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1365]) +Dataloader batch - labels shape: torch.Size([1, 1365]) +Dataloader batch - attn_mask shape: torch.Size([1, 1365]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 619 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1365) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1365) torch.int64 + loss_mask : 297 / 1365 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1365, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1365, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1365, 1, 576) torch.bfloat16 + out : (1, 1365) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.1847 + top-1 accuracy : 21/297 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 620 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 311 / 1386 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2080 + top-1 accuracy : 29/311 = 9.3% + + [2026-04-29 13:49:52] iteration 255/ 500 | consumed samples: 1020 | elapsed time per iteration (ms): 4630.2 | throughput (token/sec/GPU): 1219.4 | learning rate: 7.850071E-05 | global batch size: 4 | lm loss: 1.119271E+01 | loss scale: 1.0 | grad norm: 0.283 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1454]) +Dataloader batch - labels shape: torch.Size([1, 1454]) +Dataloader batch - attn_mask shape: torch.Size([1, 1454]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 621 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1454) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1454) torch.int64 + loss_mask : 309 / 1454 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1454, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1454, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1454, 1, 576) torch.bfloat16 + out : (1, 1454) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.2007 + top-1 accuracy : 34/309 = 11.0% + +Dataloader batch - tokens shape: torch.Size([1, 1361]) +Dataloader batch - labels shape: torch.Size([1, 1361]) +Dataloader batch - attn_mask shape: torch.Size([1, 1361]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 622 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1361) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1361) torch.int64 + loss_mask : 398 / 1361 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1361, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1361, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1361, 1, 576) torch.bfloat16 + out : (1, 1361) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.3750 + top-1 accuracy : 6/398 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 897]) +Dataloader batch - labels shape: torch.Size([1, 897]) +Dataloader batch - attn_mask shape: torch.Size([1, 897]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 623 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 897) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 897) torch.int64 + loss_mask : 195 / 897 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (897, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (897, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (897, 1, 576) torch.bfloat16 + out : (1, 897) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 195 tokens) : 11.1898 + top-1 accuracy : 8/195 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 624 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 376 / 1552 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.2855 + top-1 accuracy : 21/376 = 5.6% + + [2026-04-29 13:49:56] iteration 256/ 500 | consumed samples: 1024 | elapsed time per iteration (ms): 4441.7 | throughput (token/sec/GPU): 1185.1 | learning rate: 7.824317E-05 | global batch size: 4 | lm loss: 1.127826E+01 | loss scale: 1.0 | grad norm: 0.290 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1749]) +Dataloader batch - labels shape: torch.Size([1, 1749]) +Dataloader batch - attn_mask shape: torch.Size([1, 1749]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 625 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1749) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1749) torch.int64 + loss_mask : 448 / 1749 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1749, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1749, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1749, 1, 576) torch.bfloat16 + out : (1, 1749) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 448 tokens) : 11.2940 + top-1 accuracy : 41/448 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1815]) +Dataloader batch - labels shape: torch.Size([1, 1815]) +Dataloader batch - attn_mask shape: torch.Size([1, 1815]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 626 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1815) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1815) torch.int64 + loss_mask : 462 / 1815 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1815, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1815, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1815, 1, 576) torch.bfloat16 + out : (1, 1815) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 462 tokens) : 11.2445 + top-1 accuracy : 23/462 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1362]) +Dataloader batch - labels shape: torch.Size([1, 1362]) +Dataloader batch - attn_mask shape: torch.Size([1, 1362]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 627 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1362) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1362) torch.int64 + loss_mask : 303 / 1362 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1362, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1362, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1362, 1, 576) torch.bfloat16 + out : (1, 1362) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.2056 + top-1 accuracy : 14/303 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1178]) +Dataloader batch - labels shape: torch.Size([1, 1178]) +Dataloader batch - attn_mask shape: torch.Size([1, 1178]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 628 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1178) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1178) torch.int64 + loss_mask : 252 / 1178 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1178, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1178, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1178, 1, 576) torch.bfloat16 + out : (1, 1178) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 252 tokens) : 11.1717 + top-1 accuracy : 15/252 = 6.0% + + [2026-04-29 13:50:01] iteration 257/ 500 | consumed samples: 1028 | elapsed time per iteration (ms): 4953.3 | throughput (token/sec/GPU): 1232.3 | learning rate: 7.798453E-05 | global batch size: 4 | lm loss: 1.123906E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1503]) +Dataloader batch - labels shape: torch.Size([1, 1503]) +Dataloader batch - attn_mask shape: torch.Size([1, 1503]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 629 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1503) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1503) torch.int64 + loss_mask : 314 / 1503 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1503, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1503, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1503, 1, 576) torch.bfloat16 + out : (1, 1503) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.1723 + top-1 accuracy : 22/314 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1842]) +Dataloader batch - labels shape: torch.Size([1, 1842]) +Dataloader batch - attn_mask shape: torch.Size([1, 1842]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 630 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1842) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1842) torch.int64 + loss_mask : 464 / 1842 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1842, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1842, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1842, 1, 576) torch.bfloat16 + out : (1, 1842) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 464 tokens) : 11.2825 + top-1 accuracy : 20/464 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 631 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1467) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1467) torch.int64 + loss_mask : 412 / 1467 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1467, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1467, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1467, 1, 576) torch.bfloat16 + out : (1, 1467) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3147 + top-1 accuracy : 1/412 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1448]) +Dataloader batch - labels shape: torch.Size([1, 1448]) +Dataloader batch - attn_mask shape: torch.Size([1, 1448]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 632 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1448) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1448) torch.int64 + loss_mask : 401 / 1448 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1448, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1448, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1448, 1, 576) torch.bfloat16 + out : (1, 1448) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 401 tokens) : 11.3615 + top-1 accuracy : 21/401 = 5.2% + + [2026-04-29 13:50:06] iteration 258/ 500 | consumed samples: 1032 | elapsed time per iteration (ms): 4895.6 | throughput (token/sec/GPU): 1278.7 | learning rate: 7.772480E-05 | global batch size: 4 | lm loss: 1.128899E+01 | loss scale: 1.0 | grad norm: 0.261 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1246]) +Dataloader batch - labels shape: torch.Size([1, 1246]) +Dataloader batch - attn_mask shape: torch.Size([1, 1246]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 633 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1246) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1246) torch.int64 + loss_mask : 355 / 1246 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1246, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1246, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1246, 1, 576) torch.bfloat16 + out : (1, 1246) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.3667 + top-1 accuracy : 5/355 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1379]) +Dataloader batch - labels shape: torch.Size([1, 1379]) +Dataloader batch - attn_mask shape: torch.Size([1, 1379]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 634 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1379) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1379) torch.int64 + loss_mask : 327 / 1379 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1379, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1379, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1379, 1, 576) torch.bfloat16 + out : (1, 1379) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2559 + top-1 accuracy : 31/327 = 9.5% + +Dataloader batch - tokens shape: torch.Size([1, 1366]) +Dataloader batch - labels shape: torch.Size([1, 1366]) +Dataloader batch - attn_mask shape: torch.Size([1, 1366]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 635 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1366) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1366) torch.int64 + loss_mask : 305 / 1366 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1366, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1366, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1366, 1, 576) torch.bfloat16 + out : (1, 1366) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.1560 + top-1 accuracy : 21/305 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1086]) +Dataloader batch - labels shape: torch.Size([1, 1086]) +Dataloader batch - attn_mask shape: torch.Size([1, 1086]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 636 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1086) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1086) torch.int64 + loss_mask : 249 / 1086 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1086, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1086, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1086, 1, 576) torch.bfloat16 + out : (1, 1086) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 249 tokens) : 11.2462 + top-1 accuracy : 3/249 = 1.2% + + [2026-04-29 13:50:10] iteration 259/ 500 | consumed samples: 1036 | elapsed time per iteration (ms): 4272.9 | throughput (token/sec/GPU): 1188.2 | learning rate: 7.746398E-05 | global batch size: 4 | lm loss: 1.126113E+01 | loss scale: 1.0 | grad norm: 0.244 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1390]) +Dataloader batch - labels shape: torch.Size([1, 1390]) +Dataloader batch - attn_mask shape: torch.Size([1, 1390]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 637 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1390) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1390) torch.int64 + loss_mask : 328 / 1390 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1390, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1390, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1390, 1, 576) torch.bfloat16 + out : (1, 1390) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.2255 + top-1 accuracy : 24/328 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1265]) +Dataloader batch - labels shape: torch.Size([1, 1265]) +Dataloader batch - attn_mask shape: torch.Size([1, 1265]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 638 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1265) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1265) torch.int64 + loss_mask : 327 / 1265 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1265, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1265, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1265, 1, 576) torch.bfloat16 + out : (1, 1265) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.3300 + top-1 accuracy : 5/327 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1245]) +Dataloader batch - labels shape: torch.Size([1, 1245]) +Dataloader batch - attn_mask shape: torch.Size([1, 1245]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 639 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1245) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1245) torch.int64 + loss_mask : 322 / 1245 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1245, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1245, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1245, 1, 576) torch.bfloat16 + out : (1, 1245) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.3017 + top-1 accuracy : 13/322 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1366]) +Dataloader batch - labels shape: torch.Size([1, 1366]) +Dataloader batch - attn_mask shape: torch.Size([1, 1366]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 640 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1366) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1366) torch.int64 + loss_mask : 310 / 1366 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1366, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1366, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1366, 1, 576) torch.bfloat16 + out : (1, 1366) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 310 tokens) : 11.2402 + top-1 accuracy : 20/310 = 6.5% + + [2026-04-29 13:50:15] iteration 260/ 500 | consumed samples: 1040 | elapsed time per iteration (ms): 4232.0 | throughput (token/sec/GPU): 1244.3 | learning rate: 7.720211E-05 | global batch size: 4 | lm loss: 1.127465E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4210.14, 4210.14) + forward-compute ................................: (3110.32, 3110.32) + backward-compute ...............................: (1096.22, 1096.22) + batch-generator ................................: (405.77, 405.77) + layernorm-grads-all-reduce .....................: (0.07, 0.07) + embedding-grads-all-reduce .....................: (0.10, 0.10) + all-grads-sync .................................: (1.00, 1.00) + params-all-gather ..............................: (0.38, 0.38) + optimizer-copy-to-main-grad ....................: (0.32, 0.32) + optimizer-clip-main-grad .......................: (1.20, 1.20) + optimizer-count-zeros ..........................: (0.06, 0.06) + optimizer-inner-step ...........................: (2.07, 2.07) + optimizer-copy-main-to-model-params ............: (0.53, 0.53) + optimizer ......................................: (6.51, 6.51) +Dataloader batch - tokens shape: torch.Size([1, 1506]) +Dataloader batch - labels shape: torch.Size([1, 1506]) +Dataloader batch - attn_mask shape: torch.Size([1, 1506]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 641 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1506) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1506) torch.int64 + loss_mask : 372 / 1506 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1506, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1506, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1506, 1, 576) torch.bfloat16 + out : (1, 1506) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.3374 + top-1 accuracy : 15/372 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1391]) +Dataloader batch - labels shape: torch.Size([1, 1391]) +Dataloader batch - attn_mask shape: torch.Size([1, 1391]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 642 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1391) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1391) torch.int64 + loss_mask : 314 / 1391 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1391, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1391, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1391, 1, 576) torch.bfloat16 + out : (1, 1391) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.1935 + top-1 accuracy : 17/314 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1794]) +Dataloader batch - labels shape: torch.Size([1, 1794]) +Dataloader batch - attn_mask shape: torch.Size([1, 1794]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 643 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1794) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1794) torch.int64 + loss_mask : 415 / 1794 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1794, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1794, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1794, 1, 576) torch.bfloat16 + out : (1, 1794) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.2880 + top-1 accuracy : 34/415 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1163]) +Dataloader batch - labels shape: torch.Size([1, 1163]) +Dataloader batch - attn_mask shape: torch.Size([1, 1163]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 644 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1163) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1163) torch.int64 + loss_mask : 304 / 1163 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1163, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1163, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1163, 1, 576) torch.bfloat16 + out : (1, 1163) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 304 tokens) : 11.3571 + top-1 accuracy : 5/304 = 1.6% + + [2026-04-29 13:50:19] iteration 261/ 500 | consumed samples: 1044 | elapsed time per iteration (ms): 4687.3 | throughput (token/sec/GPU): 1248.9 | learning rate: 7.693917E-05 | global batch size: 4 | lm loss: 1.129491E+01 | loss scale: 1.0 | grad norm: 0.214 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1378]) +Dataloader batch - labels shape: torch.Size([1, 1378]) +Dataloader batch - attn_mask shape: torch.Size([1, 1378]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 645 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1378) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1378) torch.int64 + loss_mask : 311 / 1378 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1378, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1378, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1378, 1, 576) torch.bfloat16 + out : (1, 1378) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2390 + top-1 accuracy : 19/311 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1832]) +Dataloader batch - labels shape: torch.Size([1, 1832]) +Dataloader batch - attn_mask shape: torch.Size([1, 1832]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 646 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1832) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1832) torch.int64 + loss_mask : 461 / 1832 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1832, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1832, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1832, 1, 576) torch.bfloat16 + out : (1, 1832) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 461 tokens) : 11.3077 + top-1 accuracy : 16/461 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1196]) +Dataloader batch - labels shape: torch.Size([1, 1196]) +Dataloader batch - attn_mask shape: torch.Size([1, 1196]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 647 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1196) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1196) torch.int64 + loss_mask : 301 / 1196 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1196, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1196, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1196, 1, 576) torch.bfloat16 + out : (1, 1196) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 301 tokens) : 11.3275 + top-1 accuracy : 3/301 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1122]) +Dataloader batch - labels shape: torch.Size([1, 1122]) +Dataloader batch - attn_mask shape: torch.Size([1, 1122]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 648 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1122) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1122) torch.int64 + loss_mask : 294 / 1122 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1122, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1122, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1122, 1, 576) torch.bfloat16 + out : (1, 1122) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.3454 + top-1 accuracy : 17/294 = 5.8% + + [2026-04-29 13:50:24] iteration 262/ 500 | consumed samples: 1048 | elapsed time per iteration (ms): 4414.0 | throughput (token/sec/GPU): 1252.4 | learning rate: 7.667518E-05 | global batch size: 4 | lm loss: 1.130456E+01 | loss scale: 1.0 | grad norm: 0.255 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1054]) +Dataloader batch - labels shape: torch.Size([1, 1054]) +Dataloader batch - attn_mask shape: torch.Size([1, 1054]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 649 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1054) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1054) torch.int64 + loss_mask : 218 / 1054 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1054, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1054, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1054, 1, 576) torch.bfloat16 + out : (1, 1054) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 218 tokens) : 11.1912 + top-1 accuracy : 22/218 = 10.1% + +Dataloader batch - tokens shape: torch.Size([1, 1428]) +Dataloader batch - labels shape: torch.Size([1, 1428]) +Dataloader batch - attn_mask shape: torch.Size([1, 1428]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 650 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1428) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1428) torch.int64 + loss_mask : 389 / 1428 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1428, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1428, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1428, 1, 576) torch.bfloat16 + out : (1, 1428) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.3336 + top-1 accuracy : 8/389 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1086]) +Dataloader batch - labels shape: torch.Size([1, 1086]) +Dataloader batch - attn_mask shape: torch.Size([1, 1086]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 651 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1086) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1086) torch.int64 + loss_mask : 264 / 1086 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1086, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1086, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1086, 1, 576) torch.bfloat16 + out : (1, 1086) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 264 tokens) : 11.2869 + top-1 accuracy : 12/264 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1809]) +Dataloader batch - labels shape: torch.Size([1, 1809]) +Dataloader batch - attn_mask shape: torch.Size([1, 1809]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 652 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1809) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1809) torch.int64 + loss_mask : 451 / 1809 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1809, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1809, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1809, 1, 576) torch.bfloat16 + out : (1, 1809) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 451 tokens) : 11.2545 + top-1 accuracy : 12/451 = 2.7% + + [2026-04-29 13:50:29] iteration 263/ 500 | consumed samples: 1052 | elapsed time per iteration (ms): 4840.0 | throughput (token/sec/GPU): 1111.0 | learning rate: 7.641016E-05 | global batch size: 4 | lm loss: 1.127381E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1422]) +Dataloader batch - labels shape: torch.Size([1, 1422]) +Dataloader batch - attn_mask shape: torch.Size([1, 1422]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 653 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1422) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1422) torch.int64 + loss_mask : 421 / 1422 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1422, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1422, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1422, 1, 576) torch.bfloat16 + out : (1, 1422) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 421 tokens) : 11.4200 + top-1 accuracy : 5/421 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1748]) +Dataloader batch - labels shape: torch.Size([1, 1748]) +Dataloader batch - attn_mask shape: torch.Size([1, 1748]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 654 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1748) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1748) torch.int64 + loss_mask : 383 / 1748 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1748, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1748, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1748, 1, 576) torch.bfloat16 + out : (1, 1748) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.2389 + top-1 accuracy : 36/383 = 9.4% + +Dataloader batch - tokens shape: torch.Size([1, 1751]) +Dataloader batch - labels shape: torch.Size([1, 1751]) +Dataloader batch - attn_mask shape: torch.Size([1, 1751]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 655 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1751) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1751) torch.int64 + loss_mask : 392 / 1751 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1751, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1751, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1751, 1, 576) torch.bfloat16 + out : (1, 1751) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.2100 + top-1 accuracy : 33/392 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 963]) +Dataloader batch - labels shape: torch.Size([1, 963]) +Dataloader batch - attn_mask shape: torch.Size([1, 963]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 656 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 963) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 963) torch.int64 + loss_mask : 199 / 963 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (963, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (963, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (963, 1, 576) torch.bfloat16 + out : (1, 963) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 199 tokens) : 11.1615 + top-1 accuracy : 1/199 = 0.5% + + [2026-04-29 13:50:33] iteration 264/ 500 | consumed samples: 1056 | elapsed time per iteration (ms): 4773.7 | throughput (token/sec/GPU): 1232.6 | learning rate: 7.614411E-05 | global batch size: 4 | lm loss: 1.127439E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1158]) +Dataloader batch - labels shape: torch.Size([1, 1158]) +Dataloader batch - attn_mask shape: torch.Size([1, 1158]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 657 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1158) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1158) torch.int64 + loss_mask : 254 / 1158 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1158, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1158, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1158, 1, 576) torch.bfloat16 + out : (1, 1158) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 254 tokens) : 11.1873 + top-1 accuracy : 10/254 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1415]) +Dataloader batch - labels shape: torch.Size([1, 1415]) +Dataloader batch - attn_mask shape: torch.Size([1, 1415]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 658 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1415) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1415) torch.int64 + loss_mask : 372 / 1415 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1415, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1415, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1415, 1, 576) torch.bfloat16 + out : (1, 1415) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.3125 + top-1 accuracy : 19/372 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1214]) +Dataloader batch - labels shape: torch.Size([1, 1214]) +Dataloader batch - attn_mask shape: torch.Size([1, 1214]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 659 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1214) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1214) torch.int64 + loss_mask : 294 / 1214 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1214, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1214, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1214, 1, 576) torch.bfloat16 + out : (1, 1214) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.3079 + top-1 accuracy : 8/294 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1563]) +Dataloader batch - labels shape: torch.Size([1, 1563]) +Dataloader batch - attn_mask shape: torch.Size([1, 1563]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 660 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1563) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1563) torch.int64 + loss_mask : 355 / 1563 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1563, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1563, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1563, 1, 576) torch.bfloat16 + out : (1, 1563) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2101 + top-1 accuracy : 8/355 = 2.3% + + [2026-04-29 13:50:38] iteration 265/ 500 | consumed samples: 1060 | elapsed time per iteration (ms): 4510.7 | throughput (token/sec/GPU): 1186.1 | learning rate: 7.587705E-05 | global batch size: 4 | lm loss: 1.125798E+01 | loss scale: 1.0 | grad norm: 0.299 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1858]) +Dataloader batch - labels shape: torch.Size([1, 1858]) +Dataloader batch - attn_mask shape: torch.Size([1, 1858]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 661 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1858) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1858) torch.int64 + loss_mask : 492 / 1858 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1858, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1858, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1858, 1, 576) torch.bfloat16 + out : (1, 1858) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 492 tokens) : 11.3262 + top-1 accuracy : 3/492 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1362]) +Dataloader batch - labels shape: torch.Size([1, 1362]) +Dataloader batch - attn_mask shape: torch.Size([1, 1362]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 662 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1362) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1362) torch.int64 + loss_mask : 310 / 1362 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1362, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1362, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1362, 1, 576) torch.bfloat16 + out : (1, 1362) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 310 tokens) : 11.2466 + top-1 accuracy : 5/310 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1210]) +Dataloader batch - labels shape: torch.Size([1, 1210]) +Dataloader batch - attn_mask shape: torch.Size([1, 1210]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 663 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1210) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1210) torch.int64 + loss_mask : 292 / 1210 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1210, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1210, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1210, 1, 576) torch.bfloat16 + out : (1, 1210) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 292 tokens) : 11.2307 + top-1 accuracy : 11/292 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1879]) +Dataloader batch - labels shape: torch.Size([1, 1879]) +Dataloader batch - attn_mask shape: torch.Size([1, 1879]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 664 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1879) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1879) torch.int64 + loss_mask : 527 / 1879 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1879, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1879, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1879, 1, 576) torch.bfloat16 + out : (1, 1879) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 527 tokens) : 11.3276 + top-1 accuracy : 14/527 = 2.7% + + [2026-04-29 13:50:43] iteration 266/ 500 | consumed samples: 1064 | elapsed time per iteration (ms): 4909.4 | throughput (token/sec/GPU): 1285.1 | learning rate: 7.560897E-05 | global batch size: 4 | lm loss: 1.129424E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1426]) +Dataloader batch - labels shape: torch.Size([1, 1426]) +Dataloader batch - attn_mask shape: torch.Size([1, 1426]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 665 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1426) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1426) torch.int64 + loss_mask : 368 / 1426 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1426, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1426, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1426, 1, 576) torch.bfloat16 + out : (1, 1426) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.3675 + top-1 accuracy : 22/368 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1453]) +Dataloader batch - labels shape: torch.Size([1, 1453]) +Dataloader batch - attn_mask shape: torch.Size([1, 1453]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 666 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1453) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1453) torch.int64 + loss_mask : 356 / 1453 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1453, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1453, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1453, 1, 576) torch.bfloat16 + out : (1, 1453) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2471 + top-1 accuracy : 14/356 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1749]) +Dataloader batch - labels shape: torch.Size([1, 1749]) +Dataloader batch - attn_mask shape: torch.Size([1, 1749]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 667 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1749) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1749) torch.int64 + loss_mask : 391 / 1749 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1749, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1749, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1749, 1, 576) torch.bfloat16 + out : (1, 1749) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2469 + top-1 accuracy : 24/391 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1749]) +Dataloader batch - labels shape: torch.Size([1, 1749]) +Dataloader batch - attn_mask shape: torch.Size([1, 1749]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 668 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1749) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1749) torch.int64 + loss_mask : 404 / 1749 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1749, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1749, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1749, 1, 576) torch.bfloat16 + out : (1, 1749) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.2460 + top-1 accuracy : 17/404 = 4.2% + + [2026-04-29 13:50:48] iteration 267/ 500 | consumed samples: 1068 | elapsed time per iteration (ms): 5042.6 | throughput (token/sec/GPU): 1264.6 | learning rate: 7.533991E-05 | global batch size: 4 | lm loss: 1.127594E+01 | loss scale: 1.0 | grad norm: 0.250 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 669 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 322 / 1380 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.2771 + top-1 accuracy : 21/322 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1892]) +Dataloader batch - labels shape: torch.Size([1, 1892]) +Dataloader batch - attn_mask shape: torch.Size([1, 1892]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 670 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1892) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1892) torch.int64 + loss_mask : 534 / 1892 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1892, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1892, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1892, 1, 576) torch.bfloat16 + out : (1, 1892) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 534 tokens) : 11.3710 + top-1 accuracy : 10/534 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1846]) +Dataloader batch - labels shape: torch.Size([1, 1846]) +Dataloader batch - attn_mask shape: torch.Size([1, 1846]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 671 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1846) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1846) torch.int64 + loss_mask : 532 / 1846 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1846, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1846, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1846, 1, 576) torch.bfloat16 + out : (1, 1846) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 532 tokens) : 11.4057 + top-1 accuracy : 4/532 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1806]) +Dataloader batch - labels shape: torch.Size([1, 1806]) +Dataloader batch - attn_mask shape: torch.Size([1, 1806]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 672 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1806) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1806) torch.int64 + loss_mask : 426 / 1806 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1806, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1806, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1806, 1, 576) torch.bfloat16 + out : (1, 1806) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.2804 + top-1 accuracy : 10/426 = 2.3% + + [2026-04-29 13:50:53] iteration 268/ 500 | consumed samples: 1072 | elapsed time per iteration (ms): 5125.8 | throughput (token/sec/GPU): 1350.8 | learning rate: 7.506986E-05 | global batch size: 4 | lm loss: 1.134323E+01 | loss scale: 1.0 | grad norm: 0.254 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1639]) +Dataloader batch - labels shape: torch.Size([1, 1639]) +Dataloader batch - attn_mask shape: torch.Size([1, 1639]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 673 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1639) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1639) torch.int64 + loss_mask : 366 / 1639 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1639, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1639, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1639, 1, 576) torch.bfloat16 + out : (1, 1639) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2519 + top-1 accuracy : 19/366 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1531]) +Dataloader batch - labels shape: torch.Size([1, 1531]) +Dataloader batch - attn_mask shape: torch.Size([1, 1531]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 674 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1531) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1531) torch.int64 + loss_mask : 386 / 1531 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1531, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1531, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1531, 1, 576) torch.bfloat16 + out : (1, 1531) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2765 + top-1 accuracy : 23/386 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1046]) +Dataloader batch - labels shape: torch.Size([1, 1046]) +Dataloader batch - attn_mask shape: torch.Size([1, 1046]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 675 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1046) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1046) torch.int64 + loss_mask : 277 / 1046 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1046, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1046, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1046, 1, 576) torch.bfloat16 + out : (1, 1046) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 277 tokens) : 11.3179 + top-1 accuracy : 0/277 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 676 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 403 / 1695 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.2510 + top-1 accuracy : 18/403 = 4.5% + + [2026-04-29 13:50:58] iteration 269/ 500 | consumed samples: 1076 | elapsed time per iteration (ms): 4702.0 | throughput (token/sec/GPU): 1257.1 | learning rate: 7.479883E-05 | global batch size: 4 | lm loss: 1.127104E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1228]) +Dataloader batch - labels shape: torch.Size([1, 1228]) +Dataloader batch - attn_mask shape: torch.Size([1, 1228]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 677 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1228) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1228) torch.int64 + loss_mask : 306 / 1228 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1228, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1228, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1228, 1, 576) torch.bfloat16 + out : (1, 1228) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.2757 + top-1 accuracy : 1/306 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 678 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 286 / 1460 tokens (19.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 286 tokens) : 11.1679 + top-1 accuracy : 30/286 = 10.5% + +Dataloader batch - tokens shape: torch.Size([1, 1606]) +Dataloader batch - labels shape: torch.Size([1, 1606]) +Dataloader batch - attn_mask shape: torch.Size([1, 1606]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 679 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1606) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1606) torch.int64 + loss_mask : 386 / 1606 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1606, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1606, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1606, 1, 576) torch.bfloat16 + out : (1, 1606) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3314 + top-1 accuracy : 16/386 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1549]) +Dataloader batch - labels shape: torch.Size([1, 1549]) +Dataloader batch - attn_mask shape: torch.Size([1, 1549]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 680 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1549) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1549) torch.int64 + loss_mask : 334 / 1549 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1549, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1549, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1549, 1, 576) torch.bfloat16 + out : (1, 1549) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.2006 + top-1 accuracy : 26/334 = 7.8% + + [2026-04-29 13:51:02] iteration 270/ 500 | consumed samples: 1080 | elapsed time per iteration (ms): 4723.7 | throughput (token/sec/GPU): 1236.9 | learning rate: 7.452684E-05 | global batch size: 4 | lm loss: 1.124949E+01 | loss scale: 1.0 | grad norm: 0.274 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1736]) +Dataloader batch - labels shape: torch.Size([1, 1736]) +Dataloader batch - attn_mask shape: torch.Size([1, 1736]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 681 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1736) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1736) torch.int64 + loss_mask : 417 / 1736 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1736, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1736, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1736, 1, 576) torch.bfloat16 + out : (1, 1736) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 417 tokens) : 11.2740 + top-1 accuracy : 24/417 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 682 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 461 / 1520 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 461 tokens) : 11.4030 + top-1 accuracy : 1/461 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1703]) +Dataloader batch - labels shape: torch.Size([1, 1703]) +Dataloader batch - attn_mask shape: torch.Size([1, 1703]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 683 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1703) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1703) torch.int64 + loss_mask : 351 / 1703 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1703, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1703, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1703, 1, 576) torch.bfloat16 + out : (1, 1703) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.1612 + top-1 accuracy : 30/351 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 684 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 336 / 1507 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.2205 + top-1 accuracy : 34/336 = 10.1% + + [2026-04-29 13:51:07] iteration 271/ 500 | consumed samples: 1084 | elapsed time per iteration (ms): 5048.3 | throughput (token/sec/GPU): 1280.8 | learning rate: 7.425390E-05 | global batch size: 4 | lm loss: 1.127521E+01 | loss scale: 1.0 | grad norm: 0.280 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1559]) +Dataloader batch - labels shape: torch.Size([1, 1559]) +Dataloader batch - attn_mask shape: torch.Size([1, 1559]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 685 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1559) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1559) torch.int64 + loss_mask : 381 / 1559 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1559, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1559, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1559, 1, 576) torch.bfloat16 + out : (1, 1559) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.2352 + top-1 accuracy : 24/381 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1338]) +Dataloader batch - labels shape: torch.Size([1, 1338]) +Dataloader batch - attn_mask shape: torch.Size([1, 1338]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 686 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1338) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1338) torch.int64 + loss_mask : 389 / 1338 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1338, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1338, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1338, 1, 576) torch.bfloat16 + out : (1, 1338) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.3675 + top-1 accuracy : 4/389 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 687 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1467) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1467) torch.int64 + loss_mask : 408 / 1467 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1467, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1467, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1467, 1, 576) torch.bfloat16 + out : (1, 1467) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3507 + top-1 accuracy : 1/408 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 688 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 395 / 1443 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.3366 + top-1 accuracy : 5/395 = 1.3% + + [2026-04-29 13:51:12] iteration 272/ 500 | consumed samples: 1088 | elapsed time per iteration (ms): 4461.7 | throughput (token/sec/GPU): 1301.5 | learning rate: 7.398002E-05 | global batch size: 4 | lm loss: 1.132333E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1478]) +Dataloader batch - labels shape: torch.Size([1, 1478]) +Dataloader batch - attn_mask shape: torch.Size([1, 1478]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 689 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1478) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1478) torch.int64 + loss_mask : 309 / 1478 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1478, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1478, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1478, 1, 576) torch.bfloat16 + out : (1, 1478) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.1034 + top-1 accuracy : 30/309 = 9.7% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 690 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 369 / 1431 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.3652 + top-1 accuracy : 10/369 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 857]) +Dataloader batch - labels shape: torch.Size([1, 857]) +Dataloader batch - attn_mask shape: torch.Size([1, 857]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 691 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 857) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 857) torch.int64 + loss_mask : 169 / 857 tokens (19.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (857, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (857, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (857, 1, 576) torch.bfloat16 + out : (1, 857) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 169 tokens) : 11.0726 + top-1 accuracy : 4/169 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1580]) +Dataloader batch - labels shape: torch.Size([1, 1580]) +Dataloader batch - attn_mask shape: torch.Size([1, 1580]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 692 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1580) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1580) torch.int64 + loss_mask : 435 / 1580 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1580, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1580, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1580, 1, 576) torch.bfloat16 + out : (1, 1580) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 435 tokens) : 11.3118 + top-1 accuracy : 13/435 = 3.0% + + [2026-04-29 13:51:16] iteration 273/ 500 | consumed samples: 1092 | elapsed time per iteration (ms): 4289.2 | throughput (token/sec/GPU): 1246.4 | learning rate: 7.370520E-05 | global batch size: 4 | lm loss: 1.124543E+01 | loss scale: 1.0 | grad norm: 0.294 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1555]) +Dataloader batch - labels shape: torch.Size([1, 1555]) +Dataloader batch - attn_mask shape: torch.Size([1, 1555]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 693 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1555) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1555) torch.int64 + loss_mask : 368 / 1555 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1555, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1555, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1555, 1, 576) torch.bfloat16 + out : (1, 1555) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2541 + top-1 accuracy : 31/368 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 1548]) +Dataloader batch - labels shape: torch.Size([1, 1548]) +Dataloader batch - attn_mask shape: torch.Size([1, 1548]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 694 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1548) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1548) torch.int64 + loss_mask : 398 / 1548 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1548, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1548, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1548, 1, 576) torch.bfloat16 + out : (1, 1548) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.3251 + top-1 accuracy : 25/398 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1376]) +Dataloader batch - labels shape: torch.Size([1, 1376]) +Dataloader batch - attn_mask shape: torch.Size([1, 1376]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 695 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1376) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1376) torch.int64 + loss_mask : 295 / 1376 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1376, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1376, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1376, 1, 576) torch.bfloat16 + out : (1, 1376) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.1526 + top-1 accuracy : 18/295 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1150]) +Dataloader batch - labels shape: torch.Size([1, 1150]) +Dataloader batch - attn_mask shape: torch.Size([1, 1150]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 696 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1150) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1150) torch.int64 + loss_mask : 307 / 1150 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1150, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1150, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1150, 1, 576) torch.bfloat16 + out : (1, 1150) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.3657 + top-1 accuracy : 7/307 = 2.3% + + [2026-04-29 13:51:21] iteration 274/ 500 | consumed samples: 1096 | elapsed time per iteration (ms): 4478.0 | throughput (token/sec/GPU): 1257.0 | learning rate: 7.342947E-05 | global batch size: 4 | lm loss: 1.127790E+01 | loss scale: 1.0 | grad norm: 0.197 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1536]) +Dataloader batch - labels shape: torch.Size([1, 1536]) +Dataloader batch - attn_mask shape: torch.Size([1, 1536]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 697 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1536) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1536) torch.int64 + loss_mask : 394 / 1536 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1536, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1536, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1536, 1, 576) torch.bfloat16 + out : (1, 1536) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3483 + top-1 accuracy : 4/394 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1557]) +Dataloader batch - labels shape: torch.Size([1, 1557]) +Dataloader batch - attn_mask shape: torch.Size([1, 1557]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 698 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1557) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1557) torch.int64 + loss_mask : 410 / 1557 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1557, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1557, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1557, 1, 576) torch.bfloat16 + out : (1, 1557) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.3608 + top-1 accuracy : 6/410 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1035]) +Dataloader batch - labels shape: torch.Size([1, 1035]) +Dataloader batch - attn_mask shape: torch.Size([1, 1035]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 699 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1035) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1035) torch.int64 + loss_mask : 262 / 1035 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1035, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1035, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1035, 1, 576) torch.bfloat16 + out : (1, 1035) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 262 tokens) : 11.3067 + top-1 accuracy : 0/262 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1242]) +Dataloader batch - labels shape: torch.Size([1, 1242]) +Dataloader batch - attn_mask shape: torch.Size([1, 1242]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 700 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1242) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1242) torch.int64 + loss_mask : 348 / 1242 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1242, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1242, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1242, 1, 576) torch.bfloat16 + out : (1, 1242) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.3236 + top-1 accuracy : 2/348 = 0.6% + + [2026-04-29 13:51:25] iteration 275/ 500 | consumed samples: 1100 | elapsed time per iteration (ms): 4169.6 | throughput (token/sec/GPU): 1287.9 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.133816E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1517]) +Dataloader batch - labels shape: torch.Size([1, 1517]) +Dataloader batch - attn_mask shape: torch.Size([1, 1517]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 701 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1517) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1517) torch.int64 + loss_mask : 375 / 1517 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1517, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1517, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1517, 1, 576) torch.bfloat16 + out : (1, 1517) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.2995 + top-1 accuracy : 32/375 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1458]) +Dataloader batch - labels shape: torch.Size([1, 1458]) +Dataloader batch - attn_mask shape: torch.Size([1, 1458]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 702 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1458) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1458) torch.int64 + loss_mask : 368 / 1458 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1458, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1458, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1458, 1, 576) torch.bfloat16 + out : (1, 1458) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.3178 + top-1 accuracy : 18/368 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1441]) +Dataloader batch - labels shape: torch.Size([1, 1441]) +Dataloader batch - attn_mask shape: torch.Size([1, 1441]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 703 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1441) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1441) torch.int64 + loss_mask : 386 / 1441 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1441, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1441, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1441, 1, 576) torch.bfloat16 + out : (1, 1441) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2888 + top-1 accuracy : 3/386 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 704 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 277 / 1460 tokens (19.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 277 tokens) : 11.0981 + top-1 accuracy : 28/277 = 10.1% + + [2026-04-29 13:51:30] iteration 276/ 500 | consumed samples: 1104 | elapsed time per iteration (ms): 4741.8 | throughput (token/sec/GPU): 1239.2 | learning rate: 7.287529E-05 | global batch size: 4 | lm loss: 1.126169E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 705 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1423) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1423) torch.int64 + loss_mask : 349 / 1423 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1423, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1423, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1423, 1, 576) torch.bfloat16 + out : (1, 1423) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2960 + top-1 accuracy : 5/349 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1446]) +Dataloader batch - labels shape: torch.Size([1, 1446]) +Dataloader batch - attn_mask shape: torch.Size([1, 1446]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 706 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1446) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1446) torch.int64 + loss_mask : 397 / 1446 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1446, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1446, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1446, 1, 576) torch.bfloat16 + out : (1, 1446) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.3163 + top-1 accuracy : 4/397 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1894]) +Dataloader batch - labels shape: torch.Size([1, 1894]) +Dataloader batch - attn_mask shape: torch.Size([1, 1894]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 707 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1894) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1894) torch.int64 + loss_mask : 526 / 1894 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1894, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1894, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1894, 1, 576) torch.bfloat16 + out : (1, 1894) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 526 tokens) : 11.3506 + top-1 accuracy : 23/526 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1064]) +Dataloader batch - labels shape: torch.Size([1, 1064]) +Dataloader batch - attn_mask shape: torch.Size([1, 1064]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 708 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1064) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1064) torch.int64 + loss_mask : 242 / 1064 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1064, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1064, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1064, 1, 576) torch.bfloat16 + out : (1, 1064) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 242 tokens) : 11.1667 + top-1 accuracy : 17/242 = 7.0% + + [2026-04-29 13:51:34] iteration 277/ 500 | consumed samples: 1108 | elapsed time per iteration (ms): 4664.7 | throughput (token/sec/GPU): 1249.2 | learning rate: 7.259686E-05 | global batch size: 4 | lm loss: 1.129963E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1501]) +Dataloader batch - labels shape: torch.Size([1, 1501]) +Dataloader batch - attn_mask shape: torch.Size([1, 1501]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 709 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1501) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1501) torch.int64 + loss_mask : 352 / 1501 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1501, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1501, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1501, 1, 576) torch.bfloat16 + out : (1, 1501) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.2599 + top-1 accuracy : 4/352 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 710 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 281 / 1443 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 281 tokens) : 11.1571 + top-1 accuracy : 32/281 = 11.4% + +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 711 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 405 / 1543 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.3351 + top-1 accuracy : 9/405 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1346]) +Dataloader batch - labels shape: torch.Size([1, 1346]) +Dataloader batch - attn_mask shape: torch.Size([1, 1346]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 712 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1346) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1346) torch.int64 + loss_mask : 367 / 1346 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1346, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1346, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1346, 1, 576) torch.bfloat16 + out : (1, 1346) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.3757 + top-1 accuracy : 18/367 = 4.9% + + [2026-04-29 13:51:39] iteration 278/ 500 | consumed samples: 1112 | elapsed time per iteration (ms): 4617.6 | throughput (token/sec/GPU): 1263.2 | learning rate: 7.231756E-05 | global batch size: 4 | lm loss: 1.129127E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1943]) +Dataloader batch - labels shape: torch.Size([1, 1943]) +Dataloader batch - attn_mask shape: torch.Size([1, 1943]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 713 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1943) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1943) torch.int64 + loss_mask : 568 / 1943 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1943, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1943, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1943, 1, 576) torch.bfloat16 + out : (1, 1943) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 568 tokens) : 11.3889 + top-1 accuracy : 3/568 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 714 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 358 / 1434 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2795 + top-1 accuracy : 19/358 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1484]) +Dataloader batch - labels shape: torch.Size([1, 1484]) +Dataloader batch - attn_mask shape: torch.Size([1, 1484]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 715 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1484) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1484) torch.int64 + loss_mask : 308 / 1484 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1484, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1484, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1484, 1, 576) torch.bfloat16 + out : (1, 1484) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.1932 + top-1 accuracy : 32/308 = 10.4% + +Dataloader batch - tokens shape: torch.Size([1, 1409]) +Dataloader batch - labels shape: torch.Size([1, 1409]) +Dataloader batch - attn_mask shape: torch.Size([1, 1409]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 716 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1409) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1409) torch.int64 + loss_mask : 311 / 1409 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1409, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1409, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1409, 1, 576) torch.bfloat16 + out : (1, 1409) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2310 + top-1 accuracy : 12/311 = 3.9% + + [2026-04-29 13:51:44] iteration 279/ 500 | consumed samples: 1116 | elapsed time per iteration (ms): 4968.4 | throughput (token/sec/GPU): 1262.0 | learning rate: 7.203739E-05 | global batch size: 4 | lm loss: 1.129274E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 717 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 297 / 1459 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.1207 + top-1 accuracy : 10/297 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 718 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 346 / 1532 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.2485 + top-1 accuracy : 19/346 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1528]) +Dataloader batch - labels shape: torch.Size([1, 1528]) +Dataloader batch - attn_mask shape: torch.Size([1, 1528]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 719 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1528) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1528) torch.int64 + loss_mask : 350 / 1528 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1528, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1528, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1528, 1, 576) torch.bfloat16 + out : (1, 1528) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.2209 + top-1 accuracy : 16/350 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1071]) +Dataloader batch - labels shape: torch.Size([1, 1071]) +Dataloader batch - attn_mask shape: torch.Size([1, 1071]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 720 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1071) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1071) torch.int64 + loss_mask : 308 / 1071 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1071, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1071, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1071, 1, 576) torch.bfloat16 + out : (1, 1071) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.3766 + top-1 accuracy : 13/308 = 4.2% + + [2026-04-29 13:51:48] iteration 280/ 500 | consumed samples: 1120 | elapsed time per iteration (ms): 4601.4 | throughput (token/sec/GPU): 1214.9 | learning rate: 7.175637E-05 | global batch size: 4 | lm loss: 1.124224E+01 | loss scale: 1.0 | grad norm: 0.217 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4587.02, 4587.02) + forward-compute ................................: (3409.11, 3409.11) + backward-compute ...............................: (1175.09, 1175.09) + batch-generator ................................: (432.75, 432.75) + layernorm-grads-all-reduce .....................: (0.03, 0.03) + embedding-grads-all-reduce .....................: (0.04, 0.04) + all-grads-sync .................................: (0.71, 0.71) + params-all-gather ..............................: (0.20, 0.20) + optimizer-copy-to-main-grad ....................: (0.19, 0.19) + optimizer-clip-main-grad .......................: (0.72, 0.72) + optimizer-count-zeros ..........................: (0.02, 0.02) + optimizer-inner-step ...........................: (1.44, 1.44) + optimizer-copy-main-to-model-params ............: (0.27, 0.27) + optimizer ......................................: (3.62, 3.62) +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 721 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 329 / 1424 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2586 + top-1 accuracy : 13/329 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1039]) +Dataloader batch - labels shape: torch.Size([1, 1039]) +Dataloader batch - attn_mask shape: torch.Size([1, 1039]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 722 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1039) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1039) torch.int64 + loss_mask : 254 / 1039 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1039, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1039, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1039, 1, 576) torch.bfloat16 + out : (1, 1039) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 254 tokens) : 11.3268 + top-1 accuracy : 4/254 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1265]) +Dataloader batch - labels shape: torch.Size([1, 1265]) +Dataloader batch - attn_mask shape: torch.Size([1, 1265]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 723 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1265) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1265) torch.int64 + loss_mask : 388 / 1265 tokens (30.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1265, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1265, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1265, 1, 576) torch.bfloat16 + out : (1, 1265) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3968 + top-1 accuracy : 3/388 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1440]) +Dataloader batch - labels shape: torch.Size([1, 1440]) +Dataloader batch - attn_mask shape: torch.Size([1, 1440]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 724 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1440) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1440) torch.int64 + loss_mask : 455 / 1440 tokens (31.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1440, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1440, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1440, 1, 576) torch.bfloat16 + out : (1, 1440) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 455 tokens) : 11.4545 + top-1 accuracy : 2/455 = 0.4% + + [2026-04-29 13:51:52] iteration 281/ 500 | consumed samples: 1124 | elapsed time per iteration (ms): 4041.1 | throughput (token/sec/GPU): 1278.9 | learning rate: 7.147451E-05 | global batch size: 4 | lm loss: 1.137086E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1725]) +Dataloader batch - labels shape: torch.Size([1, 1725]) +Dataloader batch - attn_mask shape: torch.Size([1, 1725]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 725 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1725) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1725) torch.int64 + loss_mask : 416 / 1725 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1725, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1725, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1725, 1, 576) torch.bfloat16 + out : (1, 1725) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.2919 + top-1 accuracy : 27/416 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1717]) +Dataloader batch - labels shape: torch.Size([1, 1717]) +Dataloader batch - attn_mask shape: torch.Size([1, 1717]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 726 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1717) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1717) torch.int64 + loss_mask : 368 / 1717 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1717, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1717, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1717, 1, 576) torch.bfloat16 + out : (1, 1717) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2139 + top-1 accuracy : 27/368 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 727 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 324 / 1424 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.2316 + top-1 accuracy : 19/324 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1880]) +Dataloader batch - labels shape: torch.Size([1, 1880]) +Dataloader batch - attn_mask shape: torch.Size([1, 1880]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 728 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1880) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1880) torch.int64 + loss_mask : 519 / 1880 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1880, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1880, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1880, 1, 576) torch.bfloat16 + out : (1, 1880) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 519 tokens) : 11.3738 + top-1 accuracy : 6/519 = 1.2% + + [2026-04-29 13:51:58] iteration 282/ 500 | consumed samples: 1128 | elapsed time per iteration (ms): 5373.3 | throughput (token/sec/GPU): 1255.5 | learning rate: 7.119181E-05 | global batch size: 4 | lm loss: 1.128837E+01 | loss scale: 1.0 | grad norm: 0.226 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1389]) +Dataloader batch - labels shape: torch.Size([1, 1389]) +Dataloader batch - attn_mask shape: torch.Size([1, 1389]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 729 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1389) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1389) torch.int64 + loss_mask : 422 / 1389 tokens (30.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1389, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1389, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1389, 1, 576) torch.bfloat16 + out : (1, 1389) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.4389 + top-1 accuracy : 1/422 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1510]) +Dataloader batch - labels shape: torch.Size([1, 1510]) +Dataloader batch - attn_mask shape: torch.Size([1, 1510]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 730 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1510) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1510) torch.int64 + loss_mask : 377 / 1510 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1510, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1510, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1510, 1, 576) torch.bfloat16 + out : (1, 1510) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.2736 + top-1 accuracy : 21/377 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1482]) +Dataloader batch - labels shape: torch.Size([1, 1482]) +Dataloader batch - attn_mask shape: torch.Size([1, 1482]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 731 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1482) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1482) torch.int64 + loss_mask : 375 / 1482 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1482, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1482, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1482, 1, 576) torch.bfloat16 + out : (1, 1482) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.2996 + top-1 accuracy : 0/375 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1348]) +Dataloader batch - labels shape: torch.Size([1, 1348]) +Dataloader batch - attn_mask shape: torch.Size([1, 1348]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 732 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1348) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1348) torch.int64 + loss_mask : 373 / 1348 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1348, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1348, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1348, 1, 576) torch.bfloat16 + out : (1, 1348) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.3464 + top-1 accuracy : 14/373 = 3.8% + + [2026-04-29 13:52:02] iteration 283/ 500 | consumed samples: 1132 | elapsed time per iteration (ms): 4661.7 | throughput (token/sec/GPU): 1228.9 | learning rate: 7.090830E-05 | global batch size: 4 | lm loss: 1.134256E+01 | loss scale: 1.0 | grad norm: 0.226 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1543]) +Dataloader batch - labels shape: torch.Size([1, 1543]) +Dataloader batch - attn_mask shape: torch.Size([1, 1543]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 733 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1543) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1543) torch.int64 + loss_mask : 368 / 1543 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1543, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1543, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1543, 1, 576) torch.bfloat16 + out : (1, 1543) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2656 + top-1 accuracy : 3/368 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 734 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 413 / 1507 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.3163 + top-1 accuracy : 3/413 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1972]) +Dataloader batch - labels shape: torch.Size([1, 1972]) +Dataloader batch - attn_mask shape: torch.Size([1, 1972]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 735 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1972) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1972) torch.int64 + loss_mask : 618 / 1972 tokens (31.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1972, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1972, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1972, 1, 576) torch.bfloat16 + out : (1, 1972) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 618 tokens) : 11.4126 + top-1 accuracy : 12/618 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 736 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 392 / 1532 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.2278 + top-1 accuracy : 17/392 = 4.3% + + [2026-04-29 13:52:07] iteration 284/ 500 | consumed samples: 1136 | elapsed time per iteration (ms): 4960.7 | throughput (token/sec/GPU): 1321.2 | learning rate: 7.062397E-05 | global batch size: 4 | lm loss: 1.131973E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1202]) +Dataloader batch - labels shape: torch.Size([1, 1202]) +Dataloader batch - attn_mask shape: torch.Size([1, 1202]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 737 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1202) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1202) torch.int64 + loss_mask : 292 / 1202 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1202, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1202, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1202, 1, 576) torch.bfloat16 + out : (1, 1202) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 292 tokens) : 11.2533 + top-1 accuracy : 6/292 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 738 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 304 / 1364 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 304 tokens) : 11.2509 + top-1 accuracy : 24/304 = 7.9% + +Dataloader batch - tokens shape: torch.Size([1, 1413]) +Dataloader batch - labels shape: torch.Size([1, 1413]) +Dataloader batch - attn_mask shape: torch.Size([1, 1413]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 739 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1413) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1413) torch.int64 + loss_mask : 366 / 1413 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1413, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1413, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1413, 1, 576) torch.bfloat16 + out : (1, 1413) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2968 + top-1 accuracy : 9/366 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1015]) +Dataloader batch - labels shape: torch.Size([1, 1015]) +Dataloader batch - attn_mask shape: torch.Size([1, 1015]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 740 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1015) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1015) torch.int64 + loss_mask : 197 / 1015 tokens (19.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1015, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1015, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1015, 1, 576) torch.bfloat16 + out : (1, 1015) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 197 tokens) : 11.1506 + top-1 accuracy : 8/197 = 4.1% + + [2026-04-29 13:52:12] iteration 285/ 500 | consumed samples: 1140 | elapsed time per iteration (ms): 4391.8 | throughput (token/sec/GPU): 1137.1 | learning rate: 7.033885E-05 | global batch size: 4 | lm loss: 1.124896E+01 | loss scale: 1.0 | grad norm: 0.246 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1213]) +Dataloader batch - labels shape: torch.Size([1, 1213]) +Dataloader batch - attn_mask shape: torch.Size([1, 1213]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 741 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1213) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1213) torch.int64 + loss_mask : 326 / 1213 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1213, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1213, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1213, 1, 576) torch.bfloat16 + out : (1, 1213) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.4121 + top-1 accuracy : 19/326 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 742 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 372 / 1434 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.3330 + top-1 accuracy : 10/372 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1073]) +Dataloader batch - labels shape: torch.Size([1, 1073]) +Dataloader batch - attn_mask shape: torch.Size([1, 1073]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 743 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1073) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1073) torch.int64 + loss_mask : 237 / 1073 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1073, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1073, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1073, 1, 576) torch.bfloat16 + out : (1, 1073) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 237 tokens) : 11.1760 + top-1 accuracy : 17/237 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1544]) +Dataloader batch - labels shape: torch.Size([1, 1544]) +Dataloader batch - attn_mask shape: torch.Size([1, 1544]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 744 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1544) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1544) torch.int64 + loss_mask : 391 / 1544 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1544, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1544, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1544, 1, 576) torch.bfloat16 + out : (1, 1544) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3222 + top-1 accuracy : 16/391 = 4.1% + + [2026-04-29 13:52:16] iteration 286/ 500 | consumed samples: 1144 | elapsed time per iteration (ms): 4372.9 | throughput (token/sec/GPU): 1203.8 | learning rate: 7.005294E-05 | global batch size: 4 | lm loss: 1.132121E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1850]) +Dataloader batch - labels shape: torch.Size([1, 1850]) +Dataloader batch - attn_mask shape: torch.Size([1, 1850]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 745 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1850) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1850) torch.int64 + loss_mask : 508 / 1850 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1850, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1850, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1850, 1, 576) torch.bfloat16 + out : (1, 1850) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 508 tokens) : 11.3665 + top-1 accuracy : 25/508 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1561]) +Dataloader batch - labels shape: torch.Size([1, 1561]) +Dataloader batch - attn_mask shape: torch.Size([1, 1561]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 746 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1561) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1561) torch.int64 + loss_mask : 368 / 1561 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1561, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1561, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1561, 1, 576) torch.bfloat16 + out : (1, 1561) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2653 + top-1 accuracy : 26/368 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1847]) +Dataloader batch - labels shape: torch.Size([1, 1847]) +Dataloader batch - attn_mask shape: torch.Size([1, 1847]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 747 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1847) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1847) torch.int64 + loss_mask : 484 / 1847 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1847, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1847, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1847, 1, 576) torch.bfloat16 + out : (1, 1847) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 484 tokens) : 11.3058 + top-1 accuracy : 26/484 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1666]) +Dataloader batch - labels shape: torch.Size([1, 1666]) +Dataloader batch - attn_mask shape: torch.Size([1, 1666]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 748 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1666) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1666) torch.int64 + loss_mask : 431 / 1666 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1666, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1666, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1666, 1, 576) torch.bfloat16 + out : (1, 1666) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.1686 + top-1 accuracy : 23/431 = 5.3% + + [2026-04-29 13:52:22] iteration 287/ 500 | consumed samples: 1148 | elapsed time per iteration (ms): 5412.0 | throughput (token/sec/GPU): 1279.4 | learning rate: 6.976626E-05 | global batch size: 4 | lm loss: 1.128168E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1561]) +Dataloader batch - labels shape: torch.Size([1, 1561]) +Dataloader batch - attn_mask shape: torch.Size([1, 1561]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 749 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1561) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1561) torch.int64 + loss_mask : 407 / 1561 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1561, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1561, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1561, 1, 576) torch.bfloat16 + out : (1, 1561) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.2501 + top-1 accuracy : 11/407 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 750 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 300 / 1364 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.1984 + top-1 accuracy : 16/300 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1446]) +Dataloader batch - labels shape: torch.Size([1, 1446]) +Dataloader batch - attn_mask shape: torch.Size([1, 1446]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 751 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1446) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1446) torch.int64 + loss_mask : 390 / 1446 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1446, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1446, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1446, 1, 576) torch.bfloat16 + out : (1, 1446) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3699 + top-1 accuracy : 6/390 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1538]) +Dataloader batch - labels shape: torch.Size([1, 1538]) +Dataloader batch - attn_mask shape: torch.Size([1, 1538]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 752 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1538) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1538) torch.int64 + loss_mask : 371 / 1538 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1538, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1538, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1538, 1, 576) torch.bfloat16 + out : (1, 1538) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.2653 + top-1 accuracy : 10/371 = 2.7% + + [2026-04-29 13:52:26] iteration 288/ 500 | consumed samples: 1152 | elapsed time per iteration (ms): 4662.4 | throughput (token/sec/GPU): 1267.4 | learning rate: 6.947881E-05 | global batch size: 4 | lm loss: 1.127518E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 753 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 283 / 1443 tokens (19.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 283 tokens) : 11.1549 + top-1 accuracy : 7/283 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1169]) +Dataloader batch - labels shape: torch.Size([1, 1169]) +Dataloader batch - attn_mask shape: torch.Size([1, 1169]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 754 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1169) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1169) torch.int64 + loss_mask : 374 / 1169 tokens (32.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1169, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1169, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1169, 1, 576) torch.bfloat16 + out : (1, 1169) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.4394 + top-1 accuracy : 17/374 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1099]) +Dataloader batch - labels shape: torch.Size([1, 1099]) +Dataloader batch - attn_mask shape: torch.Size([1, 1099]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 755 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1099) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1099) torch.int64 + loss_mask : 297 / 1099 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1099, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1099, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1099, 1, 576) torch.bfloat16 + out : (1, 1099) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.3653 + top-1 accuracy : 2/297 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1700]) +Dataloader batch - labels shape: torch.Size([1, 1700]) +Dataloader batch - attn_mask shape: torch.Size([1, 1700]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 756 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1700) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1700) torch.int64 + loss_mask : 338 / 1700 tokens (19.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1700, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1700, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1700, 1, 576) torch.bfloat16 + out : (1, 1700) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.1767 + top-1 accuracy : 30/338 = 8.9% + + [2026-04-29 13:52:31] iteration 289/ 500 | consumed samples: 1156 | elapsed time per iteration (ms): 4672.6 | throughput (token/sec/GPU): 1158.0 | learning rate: 6.919062E-05 | global batch size: 4 | lm loss: 1.129132E+01 | loss scale: 1.0 | grad norm: 0.277 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1763]) +Dataloader batch - labels shape: torch.Size([1, 1763]) +Dataloader batch - attn_mask shape: torch.Size([1, 1763]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 757 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1763) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1763) torch.int64 + loss_mask : 409 / 1763 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1763, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1763, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1763, 1, 576) torch.bfloat16 + out : (1, 1763) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.2354 + top-1 accuracy : 38/409 = 9.3% + +Dataloader batch - tokens shape: torch.Size([1, 1846]) +Dataloader batch - labels shape: torch.Size([1, 1846]) +Dataloader batch - attn_mask shape: torch.Size([1, 1846]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 758 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1846) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1846) torch.int64 + loss_mask : 493 / 1846 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1846, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1846, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1846, 1, 576) torch.bfloat16 + out : (1, 1846) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 493 tokens) : 11.3386 + top-1 accuracy : 17/493 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1370]) +Dataloader batch - labels shape: torch.Size([1, 1370]) +Dataloader batch - attn_mask shape: torch.Size([1, 1370]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 759 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1370) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1370) torch.int64 + loss_mask : 273 / 1370 tokens (19.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1370, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1370, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1370, 1, 576) torch.bfloat16 + out : (1, 1370) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 273 tokens) : 11.1756 + top-1 accuracy : 22/273 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 760 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 331 / 1496 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.2429 + top-1 accuracy : 5/331 = 1.5% + + [2026-04-29 13:52:36] iteration 290/ 500 | consumed samples: 1160 | elapsed time per iteration (ms): 5188.0 | throughput (token/sec/GPU): 1248.1 | learning rate: 6.890167E-05 | global batch size: 4 | lm loss: 1.126001E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 986]) +Dataloader batch - labels shape: torch.Size([1, 986]) +Dataloader batch - attn_mask shape: torch.Size([1, 986]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 761 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 986) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 986) torch.int64 + loss_mask : 226 / 986 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (986, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (986, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (986, 1, 576) torch.bfloat16 + out : (1, 986) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 226 tokens) : 11.2126 + top-1 accuracy : 4/226 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1723]) +Dataloader batch - labels shape: torch.Size([1, 1723]) +Dataloader batch - attn_mask shape: torch.Size([1, 1723]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 762 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1723) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1723) torch.int64 + loss_mask : 376 / 1723 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1723, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1723, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1723, 1, 576) torch.bfloat16 + out : (1, 1723) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.2139 + top-1 accuracy : 19/376 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 763 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1519) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1519) torch.int64 + loss_mask : 323 / 1519 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1519, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1519, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1519, 1, 576) torch.bfloat16 + out : (1, 1519) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.2029 + top-1 accuracy : 28/323 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1215]) +Dataloader batch - labels shape: torch.Size([1, 1215]) +Dataloader batch - attn_mask shape: torch.Size([1, 1215]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 764 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1215) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1215) torch.int64 + loss_mask : 270 / 1215 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1215, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1215, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1215, 1, 576) torch.bfloat16 + out : (1, 1215) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 270 tokens) : 11.2507 + top-1 accuracy : 14/270 = 5.2% + + [2026-04-29 13:52:41] iteration 291/ 500 | consumed samples: 1164 | elapsed time per iteration (ms): 4584.8 | throughput (token/sec/GPU): 1187.2 | learning rate: 6.861200E-05 | global batch size: 4 | lm loss: 1.121901E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1135]) +Dataloader batch - labels shape: torch.Size([1, 1135]) +Dataloader batch - attn_mask shape: torch.Size([1, 1135]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 765 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1135) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1135) torch.int64 + loss_mask : 225 / 1135 tokens (19.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1135, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1135, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1135, 1, 576) torch.bfloat16 + out : (1, 1135) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 225 tokens) : 11.0769 + top-1 accuracy : 8/225 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1077]) +Dataloader batch - labels shape: torch.Size([1, 1077]) +Dataloader batch - attn_mask shape: torch.Size([1, 1077]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 766 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1077) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1077) torch.int64 + loss_mask : 266 / 1077 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1077, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1077, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1077, 1, 576) torch.bfloat16 + out : (1, 1077) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 266 tokens) : 11.2910 + top-1 accuracy : 10/266 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1707]) +Dataloader batch - labels shape: torch.Size([1, 1707]) +Dataloader batch - attn_mask shape: torch.Size([1, 1707]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 767 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1707) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1707) torch.int64 + loss_mask : 364 / 1707 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1707, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1707, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1707, 1, 576) torch.bfloat16 + out : (1, 1707) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2174 + top-1 accuracy : 38/364 = 10.4% + +Dataloader batch - tokens shape: torch.Size([1, 1751]) +Dataloader batch - labels shape: torch.Size([1, 1751]) +Dataloader batch - attn_mask shape: torch.Size([1, 1751]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 768 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1751) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1751) torch.int64 + loss_mask : 401 / 1751 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1751, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1751, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1751, 1, 576) torch.bfloat16 + out : (1, 1751) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 401 tokens) : 11.2243 + top-1 accuracy : 27/401 = 6.7% + + [2026-04-29 13:52:45] iteration 292/ 500 | consumed samples: 1168 | elapsed time per iteration (ms): 4654.5 | throughput (token/sec/GPU): 1218.2 | learning rate: 6.832162E-05 | global batch size: 4 | lm loss: 1.121000E+01 | loss scale: 1.0 | grad norm: 0.264 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1757]) +Dataloader batch - labels shape: torch.Size([1, 1757]) +Dataloader batch - attn_mask shape: torch.Size([1, 1757]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 769 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1757) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1757) torch.int64 + loss_mask : 409 / 1757 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1757, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1757, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1757, 1, 576) torch.bfloat16 + out : (1, 1757) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.2299 + top-1 accuracy : 28/409 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1693]) +Dataloader batch - labels shape: torch.Size([1, 1693]) +Dataloader batch - attn_mask shape: torch.Size([1, 1693]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 770 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1693) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1693) torch.int64 + loss_mask : 334 / 1693 tokens (19.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1693, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1693, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1693, 1, 576) torch.bfloat16 + out : (1, 1693) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.1727 + top-1 accuracy : 25/334 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 771 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 378 / 1525 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 378 tokens) : 11.2548 + top-1 accuracy : 9/378 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1859]) +Dataloader batch - labels shape: torch.Size([1, 1859]) +Dataloader batch - attn_mask shape: torch.Size([1, 1859]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 772 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1859) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1859) torch.int64 + loss_mask : 500 / 1859 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1859, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1859, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1859, 1, 576) torch.bfloat16 + out : (1, 1859) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 500 tokens) : 11.3416 + top-1 accuracy : 16/500 = 3.2% + + [2026-04-29 13:52:51] iteration 293/ 500 | consumed samples: 1172 | elapsed time per iteration (ms): 5563.9 | throughput (token/sec/GPU): 1228.3 | learning rate: 6.803052E-05 | global batch size: 4 | lm loss: 1.125836E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1602]) +Dataloader batch - labels shape: torch.Size([1, 1602]) +Dataloader batch - attn_mask shape: torch.Size([1, 1602]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 773 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1602) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1602) torch.int64 + loss_mask : 452 / 1602 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1602, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1602, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1602, 1, 576) torch.bfloat16 + out : (1, 1602) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 452 tokens) : 11.3449 + top-1 accuracy : 2/452 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1407]) +Dataloader batch - labels shape: torch.Size([1, 1407]) +Dataloader batch - attn_mask shape: torch.Size([1, 1407]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 774 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1407) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1407) torch.int64 + loss_mask : 327 / 1407 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1407, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1407, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1407, 1, 576) torch.bfloat16 + out : (1, 1407) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2324 + top-1 accuracy : 13/327 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 775 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 324 / 1386 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.2145 + top-1 accuracy : 21/324 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1702]) +Dataloader batch - labels shape: torch.Size([1, 1702]) +Dataloader batch - attn_mask shape: torch.Size([1, 1702]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 776 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1702) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1702) torch.int64 + loss_mask : 364 / 1702 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1702, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1702, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1702, 1, 576) torch.bfloat16 + out : (1, 1702) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2410 + top-1 accuracy : 28/364 = 7.7% + + [2026-04-29 13:52:56] iteration 294/ 500 | consumed samples: 1176 | elapsed time per iteration (ms): 5023.5 | throughput (token/sec/GPU): 1213.7 | learning rate: 6.773874E-05 | global batch size: 4 | lm loss: 1.126527E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1232]) +Dataloader batch - labels shape: torch.Size([1, 1232]) +Dataloader batch - attn_mask shape: torch.Size([1, 1232]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 777 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1232) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1232) torch.int64 + loss_mask : 322 / 1232 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1232, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1232, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1232, 1, 576) torch.bfloat16 + out : (1, 1232) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.2782 + top-1 accuracy : 18/322 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1484]) +Dataloader batch - labels shape: torch.Size([1, 1484]) +Dataloader batch - attn_mask shape: torch.Size([1, 1484]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 778 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1484) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1484) torch.int64 + loss_mask : 332 / 1484 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1484, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1484, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1484, 1, 576) torch.bfloat16 + out : (1, 1484) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.1786 + top-1 accuracy : 13/332 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1026]) +Dataloader batch - labels shape: torch.Size([1, 1026]) +Dataloader batch - attn_mask shape: torch.Size([1, 1026]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 779 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1026) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1026) torch.int64 + loss_mask : 252 / 1026 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1026, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1026, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1026, 1, 576) torch.bfloat16 + out : (1, 1026) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 252 tokens) : 11.2754 + top-1 accuracy : 8/252 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1879]) +Dataloader batch - labels shape: torch.Size([1, 1879]) +Dataloader batch - attn_mask shape: torch.Size([1, 1879]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 780 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1879) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1879) torch.int64 + loss_mask : 507 / 1879 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1879, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1879, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1879, 1, 576) torch.bfloat16 + out : (1, 1879) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 507 tokens) : 11.3323 + top-1 accuracy : 7/507 = 1.4% + + [2026-04-29 13:53:01] iteration 295/ 500 | consumed samples: 1180 | elapsed time per iteration (ms): 4545.6 | throughput (token/sec/GPU): 1236.6 | learning rate: 6.744626E-05 | global batch size: 4 | lm loss: 1.127371E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1483]) +Dataloader batch - labels shape: torch.Size([1, 1483]) +Dataloader batch - attn_mask shape: torch.Size([1, 1483]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 781 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1483) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1483) torch.int64 + loss_mask : 399 / 1483 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1483, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1483, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1483, 1, 576) torch.bfloat16 + out : (1, 1483) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.3535 + top-1 accuracy : 13/399 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1567]) +Dataloader batch - labels shape: torch.Size([1, 1567]) +Dataloader batch - attn_mask shape: torch.Size([1, 1567]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 782 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1567) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1567) torch.int64 + loss_mask : 350 / 1567 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1567, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1567, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1567, 1, 576) torch.bfloat16 + out : (1, 1567) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.2637 + top-1 accuracy : 28/350 = 8.0% + +Dataloader batch - tokens shape: torch.Size([1, 1425]) +Dataloader batch - labels shape: torch.Size([1, 1425]) +Dataloader batch - attn_mask shape: torch.Size([1, 1425]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 783 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1425) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1425) torch.int64 + loss_mask : 395 / 1425 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1425, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1425, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1425, 1, 576) torch.bfloat16 + out : (1, 1425) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.4067 + top-1 accuracy : 8/395 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1536]) +Dataloader batch - labels shape: torch.Size([1, 1536]) +Dataloader batch - attn_mask shape: torch.Size([1, 1536]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 784 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1536) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1536) torch.int64 + loss_mask : 391 / 1536 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1536, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1536, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1536, 1, 576) torch.bfloat16 + out : (1, 1536) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3068 + top-1 accuracy : 14/391 = 3.6% + + [2026-04-29 13:53:05] iteration 296/ 500 | consumed samples: 1184 | elapsed time per iteration (ms): 4749.4 | throughput (token/sec/GPU): 1265.6 | learning rate: 6.715312E-05 | global batch size: 4 | lm loss: 1.133482E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 785 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 316 / 1443 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.2380 + top-1 accuracy : 8/316 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1863]) +Dataloader batch - labels shape: torch.Size([1, 1863]) +Dataloader batch - attn_mask shape: torch.Size([1, 1863]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 786 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1863) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1863) torch.int64 + loss_mask : 498 / 1863 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1863, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1863, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1863, 1, 576) torch.bfloat16 + out : (1, 1863) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 498 tokens) : 11.3808 + top-1 accuracy : 24/498 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1083]) +Dataloader batch - labels shape: torch.Size([1, 1083]) +Dataloader batch - attn_mask shape: torch.Size([1, 1083]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 787 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1083) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1083) torch.int64 + loss_mask : 260 / 1083 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1083, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1083, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1083, 1, 576) torch.bfloat16 + out : (1, 1083) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.2421 + top-1 accuracy : 8/260 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 788 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 361 / 1542 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.2525 + top-1 accuracy : 40/361 = 11.1% + + [2026-04-29 13:53:10] iteration 297/ 500 | consumed samples: 1188 | elapsed time per iteration (ms): 4827.6 | throughput (token/sec/GPU): 1228.6 | learning rate: 6.685932E-05 | global batch size: 4 | lm loss: 1.129194E+01 | loss scale: 1.0 | grad norm: 0.230 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 789 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 397 / 1525 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.3282 + top-1 accuracy : 32/397 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1056]) +Dataloader batch - labels shape: torch.Size([1, 1056]) +Dataloader batch - attn_mask shape: torch.Size([1, 1056]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 790 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1056) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1056) torch.int64 + loss_mask : 214 / 1056 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1056, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1056, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1056, 1, 576) torch.bfloat16 + out : (1, 1056) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 214 tokens) : 11.1449 + top-1 accuracy : 16/214 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1036]) +Dataloader batch - labels shape: torch.Size([1, 1036]) +Dataloader batch - attn_mask shape: torch.Size([1, 1036]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 791 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1036) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1036) torch.int64 + loss_mask : 268 / 1036 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1036, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1036, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1036, 1, 576) torch.bfloat16 + out : (1, 1036) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 268 tokens) : 11.3434 + top-1 accuracy : 1/268 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1788]) +Dataloader batch - labels shape: torch.Size([1, 1788]) +Dataloader batch - attn_mask shape: torch.Size([1, 1788]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 792 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1788) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1788) torch.int64 + loss_mask : 499 / 1788 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1788, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1788, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1788, 1, 576) torch.bfloat16 + out : (1, 1788) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 499 tokens) : 11.3617 + top-1 accuracy : 8/499 = 1.6% + + [2026-04-29 13:53:14] iteration 298/ 500 | consumed samples: 1192 | elapsed time per iteration (ms): 4298.4 | throughput (token/sec/GPU): 1257.4 | learning rate: 6.656487E-05 | global batch size: 4 | lm loss: 1.131483E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1426]) +Dataloader batch - labels shape: torch.Size([1, 1426]) +Dataloader batch - attn_mask shape: torch.Size([1, 1426]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 793 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1426) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1426) torch.int64 + loss_mask : 394 / 1426 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1426, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1426, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1426, 1, 576) torch.bfloat16 + out : (1, 1426) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3528 + top-1 accuracy : 11/394 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1379]) +Dataloader batch - labels shape: torch.Size([1, 1379]) +Dataloader batch - attn_mask shape: torch.Size([1, 1379]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 794 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1379) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1379) torch.int64 + loss_mask : 330 / 1379 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1379, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1379, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1379, 1, 576) torch.bfloat16 + out : (1, 1379) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.1995 + top-1 accuracy : 17/330 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 795 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 326 / 1388 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2750 + top-1 accuracy : 18/326 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1791]) +Dataloader batch - labels shape: torch.Size([1, 1791]) +Dataloader batch - attn_mask shape: torch.Size([1, 1791]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 796 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1791) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1791) torch.int64 + loss_mask : 436 / 1791 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1791, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1791, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1791, 1, 576) torch.bfloat16 + out : (1, 1791) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.2931 + top-1 accuracy : 24/436 = 5.5% + + [2026-04-29 13:53:19] iteration 299/ 500 | consumed samples: 1196 | elapsed time per iteration (ms): 4828.4 | throughput (token/sec/GPU): 1239.3 | learning rate: 6.626978E-05 | global batch size: 4 | lm loss: 1.128416E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 797 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 339 / 1532 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.1874 + top-1 accuracy : 20/339 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1512]) +Dataloader batch - labels shape: torch.Size([1, 1512]) +Dataloader batch - attn_mask shape: torch.Size([1, 1512]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 798 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1512) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1512) torch.int64 + loss_mask : 330 / 1512 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1512, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1512, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1512, 1, 576) torch.bfloat16 + out : (1, 1512) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.2136 + top-1 accuracy : 19/330 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 799 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 330 / 1518 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.2983 + top-1 accuracy : 20/330 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 800 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 335 / 1492 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2358 + top-1 accuracy : 32/335 = 9.6% + + [2026-04-29 13:53:24] iteration 300/ 500 | consumed samples: 1200 | elapsed time per iteration (ms): 5048.5 | throughput (token/sec/GPU): 1199.2 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.123348E+01 | loss scale: 1.0 | grad norm: 0.318 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (5025.47, 5025.47) + forward-compute ................................: (3736.69, 3736.69) + backward-compute ...............................: (1284.79, 1284.79) + batch-generator ................................: (524.45, 524.45) + layernorm-grads-all-reduce .....................: (0.08, 0.08) + embedding-grads-all-reduce .....................: (0.10, 0.10) + all-grads-sync .................................: (0.99, 0.99) + params-all-gather ..............................: (0.38, 0.38) + optimizer-copy-to-main-grad ....................: (0.34, 0.34) + optimizer-clip-main-grad .......................: (1.30, 1.30) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.16, 2.16) + optimizer-copy-main-to-model-params ............: (0.55, 0.55) + optimizer ......................................: (6.87, 6.87) +saving checkpoint at iteration 300 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 300 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 300 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2518.93, 2518.93) +Dataloader batch - tokens shape: torch.Size([1, 1725]) +Dataloader batch - labels shape: torch.Size([1, 1725]) +Dataloader batch - attn_mask shape: torch.Size([1, 1725]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 801 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1725) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1725) torch.int64 + loss_mask : 347 / 1725 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1725, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1725, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1725, 1, 576) torch.bfloat16 + out : (1, 1725) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2150 + top-1 accuracy : 25/347 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 802 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 515 / 1492 tokens (34.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 515 tokens) : 11.4955 + top-1 accuracy : 2/515 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1378]) +Dataloader batch - labels shape: torch.Size([1, 1378]) +Dataloader batch - attn_mask shape: torch.Size([1, 1378]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 803 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1378) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1378) torch.int64 + loss_mask : 283 / 1378 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1378, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1378, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1378, 1, 576) torch.bfloat16 + out : (1, 1378) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 283 tokens) : 11.1375 + top-1 accuracy : 20/283 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1062]) +Dataloader batch - labels shape: torch.Size([1, 1062]) +Dataloader batch - attn_mask shape: torch.Size([1, 1062]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 804 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1062) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1062) torch.int64 + loss_mask : 290 / 1062 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1062, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1062, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1062, 1, 576) torch.bfloat16 + out : (1, 1062) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.3077 + top-1 accuracy : 5/290 = 1.7% + + [2026-04-29 13:53:31] iteration 301/ 500 | consumed samples: 1204 | elapsed time per iteration (ms): 4487.9 | throughput (token/sec/GPU): 1260.5 | learning rate: 6.567774E-05 | global batch size: 4 | lm loss: 1.131911E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1446]) +Dataloader batch - labels shape: torch.Size([1, 1446]) +Dataloader batch - attn_mask shape: torch.Size([1, 1446]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 805 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1446) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1446) torch.int64 + loss_mask : 408 / 1446 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1446, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1446, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1446, 1, 576) torch.bfloat16 + out : (1, 1446) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3660 + top-1 accuracy : 8/408 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1565]) +Dataloader batch - labels shape: torch.Size([1, 1565]) +Dataloader batch - attn_mask shape: torch.Size([1, 1565]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 806 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1565) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1565) torch.int64 + loss_mask : 385 / 1565 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1565, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1565, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1565, 1, 576) torch.bfloat16 + out : (1, 1565) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.3328 + top-1 accuracy : 21/385 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1477]) +Dataloader batch - labels shape: torch.Size([1, 1477]) +Dataloader batch - attn_mask shape: torch.Size([1, 1477]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 807 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1477) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1477) torch.int64 + loss_mask : 426 / 1477 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1477, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1477, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1477, 1, 576) torch.bfloat16 + out : (1, 1477) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.3927 + top-1 accuracy : 7/426 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1095]) +Dataloader batch - labels shape: torch.Size([1, 1095]) +Dataloader batch - attn_mask shape: torch.Size([1, 1095]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 808 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1095) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1095) torch.int64 + loss_mask : 274 / 1095 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1095, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1095, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1095, 1, 576) torch.bfloat16 + out : (1, 1095) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 274 tokens) : 11.3498 + top-1 accuracy : 0/274 = 0.0% + + [2026-04-29 13:53:36] iteration 302/ 500 | consumed samples: 1208 | elapsed time per iteration (ms): 4337.4 | throughput (token/sec/GPU): 1287.2 | learning rate: 6.538081E-05 | global batch size: 4 | lm loss: 1.136208E+01 | loss scale: 1.0 | grad norm: 0.290 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1232]) +Dataloader batch - labels shape: torch.Size([1, 1232]) +Dataloader batch - attn_mask shape: torch.Size([1, 1232]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 809 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1232) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1232) torch.int64 + loss_mask : 294 / 1232 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1232, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1232, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1232, 1, 576) torch.bfloat16 + out : (1, 1232) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.2672 + top-1 accuracy : 5/294 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1059]) +Dataloader batch - labels shape: torch.Size([1, 1059]) +Dataloader batch - attn_mask shape: torch.Size([1, 1059]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 810 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1059) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1059) torch.int64 + loss_mask : 287 / 1059 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1059, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1059, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1059, 1, 576) torch.bfloat16 + out : (1, 1059) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 287 tokens) : 11.4238 + top-1 accuracy : 1/287 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1500]) +Dataloader batch - labels shape: torch.Size([1, 1500]) +Dataloader batch - attn_mask shape: torch.Size([1, 1500]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 811 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1500) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1500) torch.int64 + loss_mask : 354 / 1500 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1500, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1500, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1500, 1, 576) torch.bfloat16 + out : (1, 1500) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2623 + top-1 accuracy : 31/354 = 8.8% + +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 812 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1420) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1420) torch.int64 + loss_mask : 361 / 1420 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1420, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1420, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1420, 1, 576) torch.bfloat16 + out : (1, 1420) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.2838 + top-1 accuracy : 3/361 = 0.8% + + [2026-04-29 13:53:40] iteration 303/ 500 | consumed samples: 1212 | elapsed time per iteration (ms): 4261.1 | throughput (token/sec/GPU): 1222.9 | learning rate: 6.508329E-05 | global batch size: 4 | lm loss: 1.130517E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1534]) +Dataloader batch - labels shape: torch.Size([1, 1534]) +Dataloader batch - attn_mask shape: torch.Size([1, 1534]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 813 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1534) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1534) torch.int64 + loss_mask : 384 / 1534 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1534, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1534, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1534, 1, 576) torch.bfloat16 + out : (1, 1534) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.2821 + top-1 accuracy : 4/384 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1087]) +Dataloader batch - labels shape: torch.Size([1, 1087]) +Dataloader batch - attn_mask shape: torch.Size([1, 1087]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 814 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1087) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1087) torch.int64 + loss_mask : 245 / 1087 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1087, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1087, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1087, 1, 576) torch.bfloat16 + out : (1, 1087) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 245 tokens) : 11.2385 + top-1 accuracy : 12/245 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1653]) +Dataloader batch - labels shape: torch.Size([1, 1653]) +Dataloader batch - attn_mask shape: torch.Size([1, 1653]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 815 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1653) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1653) torch.int64 + loss_mask : 426 / 1653 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1653, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1653, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1653, 1, 576) torch.bfloat16 + out : (1, 1653) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.3039 + top-1 accuracy : 7/426 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1414]) +Dataloader batch - labels shape: torch.Size([1, 1414]) +Dataloader batch - attn_mask shape: torch.Size([1, 1414]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 816 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1414) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1414) torch.int64 + loss_mask : 357 / 1414 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1414, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1414, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1414, 1, 576) torch.bfloat16 + out : (1, 1414) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3077 + top-1 accuracy : 6/357 = 1.7% + + [2026-04-29 13:53:44] iteration 304/ 500 | consumed samples: 1216 | elapsed time per iteration (ms): 4600.8 | throughput (token/sec/GPU): 1236.3 | learning rate: 6.478519E-05 | global batch size: 4 | lm loss: 1.128759E+01 | loss scale: 1.0 | grad norm: 0.259 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1555]) +Dataloader batch - labels shape: torch.Size([1, 1555]) +Dataloader batch - attn_mask shape: torch.Size([1, 1555]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 817 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1555) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1555) torch.int64 + loss_mask : 404 / 1555 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1555, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1555, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1555, 1, 576) torch.bfloat16 + out : (1, 1555) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.3053 + top-1 accuracy : 25/404 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1396]) +Dataloader batch - labels shape: torch.Size([1, 1396]) +Dataloader batch - attn_mask shape: torch.Size([1, 1396]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 818 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1396) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1396) torch.int64 + loss_mask : 349 / 1396 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1396, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1396, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1396, 1, 576) torch.bfloat16 + out : (1, 1396) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2456 + top-1 accuracy : 3/349 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1075]) +Dataloader batch - labels shape: torch.Size([1, 1075]) +Dataloader batch - attn_mask shape: torch.Size([1, 1075]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 819 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1075) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1075) torch.int64 + loss_mask : 238 / 1075 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1075, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1075, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1075, 1, 576) torch.bfloat16 + out : (1, 1075) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 238 tokens) : 11.1995 + top-1 accuracy : 16/238 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 820 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 338 / 1518 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.2233 + top-1 accuracy : 23/338 = 6.8% + + [2026-04-29 13:53:49] iteration 305/ 500 | consumed samples: 1220 | elapsed time per iteration (ms): 4549.8 | throughput (token/sec/GPU): 1218.5 | learning rate: 6.448653E-05 | global batch size: 4 | lm loss: 1.124986E+01 | loss scale: 1.0 | grad norm: 0.252 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 821 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 373 / 1424 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.3341 + top-1 accuracy : 7/373 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1590]) +Dataloader batch - labels shape: torch.Size([1, 1590]) +Dataloader batch - attn_mask shape: torch.Size([1, 1590]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 822 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1590) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1590) torch.int64 + loss_mask : 357 / 1590 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1590, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1590, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1590, 1, 576) torch.bfloat16 + out : (1, 1590) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2384 + top-1 accuracy : 6/357 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1249]) +Dataloader batch - labels shape: torch.Size([1, 1249]) +Dataloader batch - attn_mask shape: torch.Size([1, 1249]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 823 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1249) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1249) torch.int64 + loss_mask : 324 / 1249 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1249, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1249, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1249, 1, 576) torch.bfloat16 + out : (1, 1249) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.3390 + top-1 accuracy : 4/324 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1614]) +Dataloader batch - labels shape: torch.Size([1, 1614]) +Dataloader batch - attn_mask shape: torch.Size([1, 1614]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 824 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1614) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1614) torch.int64 + loss_mask : 467 / 1614 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1614, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1614, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1614, 1, 576) torch.bfloat16 + out : (1, 1614) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 467 tokens) : 11.3937 + top-1 accuracy : 10/467 = 2.1% + + [2026-04-29 13:53:54] iteration 306/ 500 | consumed samples: 1224 | elapsed time per iteration (ms): 4564.1 | throughput (token/sec/GPU): 1287.7 | learning rate: 6.418731E-05 | global batch size: 4 | lm loss: 1.133099E+01 | loss scale: 1.0 | grad norm: 0.255 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1561]) +Dataloader batch - labels shape: torch.Size([1, 1561]) +Dataloader batch - attn_mask shape: torch.Size([1, 1561]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 825 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1561) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1561) torch.int64 + loss_mask : 355 / 1561 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1561, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1561, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1561, 1, 576) torch.bfloat16 + out : (1, 1561) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2311 + top-1 accuracy : 15/355 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 826 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1470) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1470) torch.int64 + loss_mask : 376 / 1470 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1470, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1470, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1470, 1, 576) torch.bfloat16 + out : (1, 1470) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3464 + top-1 accuracy : 21/376 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1644]) +Dataloader batch - labels shape: torch.Size([1, 1644]) +Dataloader batch - attn_mask shape: torch.Size([1, 1644]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 827 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1644) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1644) torch.int64 + loss_mask : 425 / 1644 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1644, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1644, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1644, 1, 576) torch.bfloat16 + out : (1, 1644) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 425 tokens) : 11.3117 + top-1 accuracy : 9/425 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1762]) +Dataloader batch - labels shape: torch.Size([1, 1762]) +Dataloader batch - attn_mask shape: torch.Size([1, 1762]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 828 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1762) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1762) torch.int64 + loss_mask : 406 / 1762 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1762, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1762, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1762, 1, 576) torch.bfloat16 + out : (1, 1762) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.2442 + top-1 accuracy : 5/406 = 1.2% + + [2026-04-29 13:53:59] iteration 307/ 500 | consumed samples: 1228 | elapsed time per iteration (ms): 5056.2 | throughput (token/sec/GPU): 1273.1 | learning rate: 6.388756E-05 | global batch size: 4 | lm loss: 1.128419E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1792]) +Dataloader batch - labels shape: torch.Size([1, 1792]) +Dataloader batch - attn_mask shape: torch.Size([1, 1792]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 829 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1792) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1792) torch.int64 + loss_mask : 479 / 1792 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1792, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1792, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1792, 1, 576) torch.bfloat16 + out : (1, 1792) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 479 tokens) : 11.3722 + top-1 accuracy : 12/479 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1562]) +Dataloader batch - labels shape: torch.Size([1, 1562]) +Dataloader batch - attn_mask shape: torch.Size([1, 1562]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 830 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1562) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1562) torch.int64 + loss_mask : 414 / 1562 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1562, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1562, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1562, 1, 576) torch.bfloat16 + out : (1, 1562) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 414 tokens) : 11.3163 + top-1 accuracy : 3/414 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1114]) +Dataloader batch - labels shape: torch.Size([1, 1114]) +Dataloader batch - attn_mask shape: torch.Size([1, 1114]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 831 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1114) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1114) torch.int64 + loss_mask : 324 / 1114 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1114, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1114, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1114, 1, 576) torch.bfloat16 + out : (1, 1114) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.3721 + top-1 accuracy : 3/324 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1872]) +Dataloader batch - labels shape: torch.Size([1, 1872]) +Dataloader batch - attn_mask shape: torch.Size([1, 1872]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 832 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1872) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1872) torch.int64 + loss_mask : 506 / 1872 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1872, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1872, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1872, 1, 576) torch.bfloat16 + out : (1, 1872) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 506 tokens) : 11.3467 + top-1 accuracy : 7/506 = 1.4% + + [2026-04-29 13:54:03] iteration 308/ 500 | consumed samples: 1232 | elapsed time per iteration (ms): 4825.3 | throughput (token/sec/GPU): 1313.9 | learning rate: 6.358726E-05 | global batch size: 4 | lm loss: 1.135125E+01 | loss scale: 1.0 | grad norm: 0.269 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 934]) +Dataloader batch - labels shape: torch.Size([1, 934]) +Dataloader batch - attn_mask shape: torch.Size([1, 934]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 833 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 934) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 934) torch.int64 + loss_mask : 192 / 934 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (934, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (934, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (934, 1, 576) torch.bfloat16 + out : (1, 934) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 192 tokens) : 11.2180 + top-1 accuracy : 7/192 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1828]) +Dataloader batch - labels shape: torch.Size([1, 1828]) +Dataloader batch - attn_mask shape: torch.Size([1, 1828]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 834 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1828) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1828) torch.int64 + loss_mask : 470 / 1828 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1828, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1828, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1828, 1, 576) torch.bfloat16 + out : (1, 1828) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 470 tokens) : 11.2937 + top-1 accuracy : 16/470 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1506]) +Dataloader batch - labels shape: torch.Size([1, 1506]) +Dataloader batch - attn_mask shape: torch.Size([1, 1506]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 835 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1506) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1506) torch.int64 + loss_mask : 320 / 1506 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1506, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1506, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1506, 1, 576) torch.bfloat16 + out : (1, 1506) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2612 + top-1 accuracy : 27/320 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 1401]) +Dataloader batch - labels shape: torch.Size([1, 1401]) +Dataloader batch - attn_mask shape: torch.Size([1, 1401]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 836 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1401) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1401) torch.int64 + loss_mask : 319 / 1401 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1401, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1401, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1401, 1, 576) torch.bfloat16 + out : (1, 1401) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.2236 + top-1 accuracy : 5/319 = 1.6% + + [2026-04-29 13:54:08] iteration 309/ 500 | consumed samples: 1236 | elapsed time per iteration (ms): 4681.2 | throughput (token/sec/GPU): 1211.0 | learning rate: 6.328645E-05 | global batch size: 4 | lm loss: 1.125733E+01 | loss scale: 1.0 | grad norm: 0.319 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1573]) +Dataloader batch - labels shape: torch.Size([1, 1573]) +Dataloader batch - attn_mask shape: torch.Size([1, 1573]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 837 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1573) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1573) torch.int64 + loss_mask : 419 / 1573 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1573, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1573, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1573, 1, 576) torch.bfloat16 + out : (1, 1573) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.3685 + top-1 accuracy : 30/419 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1055]) +Dataloader batch - labels shape: torch.Size([1, 1055]) +Dataloader batch - attn_mask shape: torch.Size([1, 1055]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 838 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1055) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1055) torch.int64 + loss_mask : 284 / 1055 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1055, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1055, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1055, 1, 576) torch.bfloat16 + out : (1, 1055) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.2930 + top-1 accuracy : 6/284 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1031]) +Dataloader batch - labels shape: torch.Size([1, 1031]) +Dataloader batch - attn_mask shape: torch.Size([1, 1031]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 839 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1031) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1031) torch.int64 + loss_mask : 284 / 1031 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1031, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1031, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1031, 1, 576) torch.bfloat16 + out : (1, 1031) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 284 tokens) : 11.3438 + top-1 accuracy : 3/284 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1405]) +Dataloader batch - labels shape: torch.Size([1, 1405]) +Dataloader batch - attn_mask shape: torch.Size([1, 1405]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 840 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1405) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1405) torch.int64 + loss_mask : 340 / 1405 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1405, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1405, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1405, 1, 576) torch.bfloat16 + out : (1, 1405) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2146 + top-1 accuracy : 3/340 = 0.9% + + [2026-04-29 13:54:12] iteration 310/ 500 | consumed samples: 1240 | elapsed time per iteration (ms): 4028.3 | throughput (token/sec/GPU): 1257.1 | learning rate: 6.298514E-05 | global batch size: 4 | lm loss: 1.130762E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1584]) +Dataloader batch - labels shape: torch.Size([1, 1584]) +Dataloader batch - attn_mask shape: torch.Size([1, 1584]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 841 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1584) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1584) torch.int64 + loss_mask : 372 / 1584 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1584, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1584, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1584, 1, 576) torch.bfloat16 + out : (1, 1584) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2595 + top-1 accuracy : 20/372 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 842 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 371 / 1750 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.2345 + top-1 accuracy : 10/371 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 843 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 345 / 1438 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2397 + top-1 accuracy : 6/345 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1817]) +Dataloader batch - labels shape: torch.Size([1, 1817]) +Dataloader batch - attn_mask shape: torch.Size([1, 1817]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 844 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1817) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1817) torch.int64 + loss_mask : 476 / 1817 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1817, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1817, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1817, 1, 576) torch.bfloat16 + out : (1, 1817) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 476 tokens) : 11.3707 + top-1 accuracy : 22/476 = 4.6% + + [2026-04-29 13:54:17] iteration 311/ 500 | consumed samples: 1244 | elapsed time per iteration (ms): 5201.6 | throughput (token/sec/GPU): 1266.7 | learning rate: 6.268333E-05 | global batch size: 4 | lm loss: 1.128303E+01 | loss scale: 1.0 | grad norm: 0.278 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1706]) +Dataloader batch - labels shape: torch.Size([1, 1706]) +Dataloader batch - attn_mask shape: torch.Size([1, 1706]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 845 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1706) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1706) torch.int64 + loss_mask : 343 / 1706 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1706, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1706, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1706, 1, 576) torch.bfloat16 + out : (1, 1706) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.1543 + top-1 accuracy : 29/343 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1403]) +Dataloader batch - labels shape: torch.Size([1, 1403]) +Dataloader batch - attn_mask shape: torch.Size([1, 1403]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 846 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1403) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1403) torch.int64 + loss_mask : 370 / 1403 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1403, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1403, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1403, 1, 576) torch.bfloat16 + out : (1, 1403) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.3146 + top-1 accuracy : 22/370 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1399]) +Dataloader batch - labels shape: torch.Size([1, 1399]) +Dataloader batch - attn_mask shape: torch.Size([1, 1399]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 847 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1399) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1399) torch.int64 + loss_mask : 333 / 1399 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1399, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1399, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1399, 1, 576) torch.bfloat16 + out : (1, 1399) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.2866 + top-1 accuracy : 3/333 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 848 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 325 / 1380 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.2293 + top-1 accuracy : 20/325 = 6.2% + + [2026-04-29 13:54:22] iteration 312/ 500 | consumed samples: 1248 | elapsed time per iteration (ms): 4652.5 | throughput (token/sec/GPU): 1265.6 | learning rate: 6.238103E-05 | global batch size: 4 | lm loss: 1.124750E+01 | loss scale: 1.0 | grad norm: 0.309 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1891]) +Dataloader batch - labels shape: torch.Size([1, 1891]) +Dataloader batch - attn_mask shape: torch.Size([1, 1891]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 849 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1891) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1891) torch.int64 + loss_mask : 524 / 1891 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1891, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1891, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1891, 1, 576) torch.bfloat16 + out : (1, 1891) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 524 tokens) : 11.3559 + top-1 accuracy : 19/524 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1428]) +Dataloader batch - labels shape: torch.Size([1, 1428]) +Dataloader batch - attn_mask shape: torch.Size([1, 1428]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 850 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1428) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1428) torch.int64 + loss_mask : 357 / 1428 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1428, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1428, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1428, 1, 576) torch.bfloat16 + out : (1, 1428) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2707 + top-1 accuracy : 8/357 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1049]) +Dataloader batch - labels shape: torch.Size([1, 1049]) +Dataloader batch - attn_mask shape: torch.Size([1, 1049]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 851 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1049) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1049) torch.int64 + loss_mask : 279 / 1049 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1049, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1049, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1049, 1, 576) torch.bfloat16 + out : (1, 1049) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 279 tokens) : 11.3610 + top-1 accuracy : 1/279 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 852 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 335 / 1388 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2425 + top-1 accuracy : 13/335 = 3.9% + + [2026-04-29 13:54:27] iteration 313/ 500 | consumed samples: 1252 | elapsed time per iteration (ms): 4497.5 | throughput (token/sec/GPU): 1279.8 | learning rate: 6.207827E-05 | global batch size: 4 | lm loss: 1.131110E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1932]) +Dataloader batch - labels shape: torch.Size([1, 1932]) +Dataloader batch - attn_mask shape: torch.Size([1, 1932]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 853 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1932) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1932) torch.int64 + loss_mask : 558 / 1932 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1932, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1932, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1932, 1, 576) torch.bfloat16 + out : (1, 1932) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 558 tokens) : 11.4057 + top-1 accuracy : 0/558 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1046]) +Dataloader batch - labels shape: torch.Size([1, 1046]) +Dataloader batch - attn_mask shape: torch.Size([1, 1046]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 854 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1046) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1046) torch.int64 + loss_mask : 213 / 1046 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1046, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1046, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1046, 1, 576) torch.bfloat16 + out : (1, 1046) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 213 tokens) : 11.1960 + top-1 accuracy : 9/213 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 855 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 350 / 1532 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.1967 + top-1 accuracy : 20/350 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1408]) +Dataloader batch - labels shape: torch.Size([1, 1408]) +Dataloader batch - attn_mask shape: torch.Size([1, 1408]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 856 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1408) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1408) torch.int64 + loss_mask : 319 / 1408 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1408, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1408, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1408, 1, 576) torch.bfloat16 + out : (1, 1408) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.1899 + top-1 accuracy : 16/319 = 5.0% + + [2026-04-29 13:54:31] iteration 314/ 500 | consumed samples: 1256 | elapsed time per iteration (ms): 4670.2 | throughput (token/sec/GPU): 1267.2 | learning rate: 6.177505E-05 | global batch size: 4 | lm loss: 1.127609E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 996]) +Dataloader batch - labels shape: torch.Size([1, 996]) +Dataloader batch - attn_mask shape: torch.Size([1, 996]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 857 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 996) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 996) torch.int64 + loss_mask : 248 / 996 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (996, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (996, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (996, 1, 576) torch.bfloat16 + out : (1, 996) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 248 tokens) : 11.2687 + top-1 accuracy : 11/248 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1573]) +Dataloader batch - labels shape: torch.Size([1, 1573]) +Dataloader batch - attn_mask shape: torch.Size([1, 1573]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 858 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1573) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1573) torch.int64 + loss_mask : 373 / 1573 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1573, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1573, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1573, 1, 576) torch.bfloat16 + out : (1, 1573) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.2641 + top-1 accuracy : 11/373 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1720]) +Dataloader batch - labels shape: torch.Size([1, 1720]) +Dataloader batch - attn_mask shape: torch.Size([1, 1720]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 859 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1720) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1720) torch.int64 + loss_mask : 362 / 1720 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1720, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1720, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1720, 1, 576) torch.bfloat16 + out : (1, 1720) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2497 + top-1 accuracy : 8/362 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1114]) +Dataloader batch - labels shape: torch.Size([1, 1114]) +Dataloader batch - attn_mask shape: torch.Size([1, 1114]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 860 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1114) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1114) torch.int64 + loss_mask : 285 / 1114 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1114, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1114, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1114, 1, 576) torch.bfloat16 + out : (1, 1114) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 285 tokens) : 11.3027 + top-1 accuracy : 13/285 = 4.6% + + [2026-04-29 13:54:36] iteration 315/ 500 | consumed samples: 1260 | elapsed time per iteration (ms): 4494.3 | throughput (token/sec/GPU): 1202.2 | learning rate: 6.147138E-05 | global batch size: 4 | lm loss: 1.126956E+01 | loss scale: 1.0 | grad norm: 0.242 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1830]) +Dataloader batch - labels shape: torch.Size([1, 1830]) +Dataloader batch - attn_mask shape: torch.Size([1, 1830]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 861 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1830) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1830) torch.int64 + loss_mask : 465 / 1830 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1830, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1830, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1830, 1, 576) torch.bfloat16 + out : (1, 1830) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 465 tokens) : 11.3182 + top-1 accuracy : 15/465 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1882]) +Dataloader batch - labels shape: torch.Size([1, 1882]) +Dataloader batch - attn_mask shape: torch.Size([1, 1882]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 862 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1882) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1882) torch.int64 + loss_mask : 531 / 1882 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1882, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1882, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1882, 1, 576) torch.bfloat16 + out : (1, 1882) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 531 tokens) : 11.3638 + top-1 accuracy : 9/531 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1815]) +Dataloader batch - labels shape: torch.Size([1, 1815]) +Dataloader batch - attn_mask shape: torch.Size([1, 1815]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 863 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1815) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1815) torch.int64 + loss_mask : 444 / 1815 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1815, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1815, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1815, 1, 576) torch.bfloat16 + out : (1, 1815) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 444 tokens) : 11.3118 + top-1 accuracy : 4/444 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 864 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1467) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1467) torch.int64 + loss_mask : 365 / 1467 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1467, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1467, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1467, 1, 576) torch.bfloat16 + out : (1, 1467) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2638 + top-1 accuracy : 13/365 = 3.6% + + [2026-04-29 13:54:41] iteration 316/ 500 | consumed samples: 1264 | elapsed time per iteration (ms): 5312.7 | throughput (token/sec/GPU): 1316.5 | learning rate: 6.116727E-05 | global batch size: 4 | lm loss: 1.131905E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1179]) +Dataloader batch - labels shape: torch.Size([1, 1179]) +Dataloader batch - attn_mask shape: torch.Size([1, 1179]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 865 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1179) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1179) torch.int64 + loss_mask : 326 / 1179 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1179, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1179, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1179, 1, 576) torch.bfloat16 + out : (1, 1179) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.4158 + top-1 accuracy : 4/326 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1050]) +Dataloader batch - labels shape: torch.Size([1, 1050]) +Dataloader batch - attn_mask shape: torch.Size([1, 1050]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 866 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1050) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1050) torch.int64 + loss_mask : 223 / 1050 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1050, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1050, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1050, 1, 576) torch.bfloat16 + out : (1, 1050) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 223 tokens) : 11.1546 + top-1 accuracy : 4/223 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1876]) +Dataloader batch - labels shape: torch.Size([1, 1876]) +Dataloader batch - attn_mask shape: torch.Size([1, 1876]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 867 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1876) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1876) torch.int64 + loss_mask : 525 / 1876 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1876, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1876, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1876, 1, 576) torch.bfloat16 + out : (1, 1876) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 525 tokens) : 11.3396 + top-1 accuracy : 15/525 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1131]) +Dataloader batch - labels shape: torch.Size([1, 1131]) +Dataloader batch - attn_mask shape: torch.Size([1, 1131]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 868 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1131) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1131) torch.int64 + loss_mask : 308 / 1131 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1131, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1131, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1131, 1, 576) torch.bfloat16 + out : (1, 1131) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.3749 + top-1 accuracy : 0/308 = 0.0% + + [2026-04-29 13:54:45] iteration 317/ 500 | consumed samples: 1268 | elapsed time per iteration (ms): 4202.0 | throughput (token/sec/GPU): 1246.1 | learning rate: 6.086275E-05 | global batch size: 4 | lm loss: 1.133559E+01 | loss scale: 1.0 | grad norm: 0.231 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1458]) +Dataloader batch - labels shape: torch.Size([1, 1458]) +Dataloader batch - attn_mask shape: torch.Size([1, 1458]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 869 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1458) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1458) torch.int64 + loss_mask : 408 / 1458 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1458, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1458, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1458, 1, 576) torch.bfloat16 + out : (1, 1458) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3372 + top-1 accuracy : 3/408 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1376]) +Dataloader batch - labels shape: torch.Size([1, 1376]) +Dataloader batch - attn_mask shape: torch.Size([1, 1376]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 870 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1376) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1376) torch.int64 + loss_mask : 305 / 1376 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1376, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1376, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1376, 1, 576) torch.bfloat16 + out : (1, 1376) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.2448 + top-1 accuracy : 17/305 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1481]) +Dataloader batch - labels shape: torch.Size([1, 1481]) +Dataloader batch - attn_mask shape: torch.Size([1, 1481]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 871 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1481) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1481) torch.int64 + loss_mask : 340 / 1481 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1481, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1481, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1481, 1, 576) torch.bfloat16 + out : (1, 1481) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2106 + top-1 accuracy : 6/340 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1060]) +Dataloader batch - labels shape: torch.Size([1, 1060]) +Dataloader batch - attn_mask shape: torch.Size([1, 1060]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 872 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1060) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1060) torch.int64 + loss_mask : 287 / 1060 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1060, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1060, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1060, 1, 576) torch.bfloat16 + out : (1, 1060) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 287 tokens) : 11.3580 + top-1 accuracy : 0/287 = 0.0% + + [2026-04-29 13:54:50] iteration 318/ 500 | consumed samples: 1272 | elapsed time per iteration (ms): 4390.6 | throughput (token/sec/GPU): 1224.2 | learning rate: 6.055781E-05 | global batch size: 4 | lm loss: 1.128850E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1449]) +Dataloader batch - labels shape: torch.Size([1, 1449]) +Dataloader batch - attn_mask shape: torch.Size([1, 1449]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 873 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1449) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1449) torch.int64 + loss_mask : 431 / 1449 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1449, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1449, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1449, 1, 576) torch.bfloat16 + out : (1, 1449) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.3417 + top-1 accuracy : 5/431 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 874 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1412) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1412) torch.int64 + loss_mask : 354 / 1412 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1412, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1412, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1412, 1, 576) torch.bfloat16 + out : (1, 1412) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2390 + top-1 accuracy : 17/354 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1204]) +Dataloader batch - labels shape: torch.Size([1, 1204]) +Dataloader batch - attn_mask shape: torch.Size([1, 1204]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 875 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1204) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1204) torch.int64 + loss_mask : 290 / 1204 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1204, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1204, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1204, 1, 576) torch.bfloat16 + out : (1, 1204) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.2183 + top-1 accuracy : 20/290 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1495]) +Dataloader batch - labels shape: torch.Size([1, 1495]) +Dataloader batch - attn_mask shape: torch.Size([1, 1495]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 876 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1495) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1495) torch.int64 + loss_mask : 306 / 1495 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1495, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1495, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1495, 1, 576) torch.bfloat16 + out : (1, 1495) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.2252 + top-1 accuracy : 26/306 = 8.5% + + [2026-04-29 13:54:54] iteration 319/ 500 | consumed samples: 1276 | elapsed time per iteration (ms): 4403.3 | throughput (token/sec/GPU): 1262.7 | learning rate: 6.025247E-05 | global batch size: 4 | lm loss: 1.126364E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1442]) +Dataloader batch - labels shape: torch.Size([1, 1442]) +Dataloader batch - attn_mask shape: torch.Size([1, 1442]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 877 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1442) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1442) torch.int64 + loss_mask : 395 / 1442 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1442, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1442, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1442, 1, 576) torch.bfloat16 + out : (1, 1442) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.3309 + top-1 accuracy : 1/395 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1557]) +Dataloader batch - labels shape: torch.Size([1, 1557]) +Dataloader batch - attn_mask shape: torch.Size([1, 1557]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 878 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1557) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1557) torch.int64 + loss_mask : 414 / 1557 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1557, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1557, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1557, 1, 576) torch.bfloat16 + out : (1, 1557) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 414 tokens) : 11.2652 + top-1 accuracy : 14/414 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1872]) +Dataloader batch - labels shape: torch.Size([1, 1872]) +Dataloader batch - attn_mask shape: torch.Size([1, 1872]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 879 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1872) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1872) torch.int64 + loss_mask : 519 / 1872 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1872, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1872, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1872, 1, 576) torch.bfloat16 + out : (1, 1872) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 519 tokens) : 11.4007 + top-1 accuracy : 22/519 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1533]) +Dataloader batch - labels shape: torch.Size([1, 1533]) +Dataloader batch - attn_mask shape: torch.Size([1, 1533]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 880 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1533) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1533) torch.int64 + loss_mask : 347 / 1533 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1533, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1533, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1533, 1, 576) torch.bfloat16 + out : (1, 1533) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2151 + top-1 accuracy : 14/347 = 4.0% + + [2026-04-29 13:54:59] iteration 320/ 500 | consumed samples: 1280 | elapsed time per iteration (ms): 4991.9 | throughput (token/sec/GPU): 1282.9 | learning rate: 5.994675E-05 | global batch size: 4 | lm loss: 1.131231E+01 | loss scale: 1.0 | grad norm: 0.193 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4979.16, 4979.16) + forward-compute ................................: (3647.24, 3647.24) + backward-compute ...............................: (1329.65, 1329.65) + batch-generator ................................: (412.78, 412.78) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.04, 0.04) + all-grads-sync .................................: (0.56, 0.56) + params-all-gather ..............................: (0.15, 0.15) + optimizer-copy-to-main-grad ....................: (0.15, 0.15) + optimizer-clip-main-grad .......................: (0.57, 0.57) + optimizer-count-zeros ..........................: (0.02, 0.02) + optimizer-inner-step ...........................: (1.31, 1.31) + optimizer-copy-main-to-model-params ............: (0.24, 0.24) + optimizer ......................................: (3.14, 3.14) +Dataloader batch - tokens shape: torch.Size([1, 1819]) +Dataloader batch - labels shape: torch.Size([1, 1819]) +Dataloader batch - attn_mask shape: torch.Size([1, 1819]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 881 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1819) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1819) torch.int64 + loss_mask : 443 / 1819 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1819, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1819, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1819, 1, 576) torch.bfloat16 + out : (1, 1819) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 443 tokens) : 11.2853 + top-1 accuracy : 2/443 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1066]) +Dataloader batch - labels shape: torch.Size([1, 1066]) +Dataloader batch - attn_mask shape: torch.Size([1, 1066]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 882 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1066) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1066) torch.int64 + loss_mask : 246 / 1066 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1066, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1066, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1066, 1, 576) torch.bfloat16 + out : (1, 1066) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 246 tokens) : 11.2503 + top-1 accuracy : 10/246 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1497]) +Dataloader batch - labels shape: torch.Size([1, 1497]) +Dataloader batch - attn_mask shape: torch.Size([1, 1497]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 883 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1497) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1497) torch.int64 + loss_mask : 313 / 1497 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1497, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1497, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1497, 1, 576) torch.bfloat16 + out : (1, 1497) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.1832 + top-1 accuracy : 28/313 = 8.9% + +Dataloader batch - tokens shape: torch.Size([1, 1104]) +Dataloader batch - labels shape: torch.Size([1, 1104]) +Dataloader batch - attn_mask shape: torch.Size([1, 1104]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 884 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1104) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1104) torch.int64 + loss_mask : 255 / 1104 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1104, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1104, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1104, 1, 576) torch.bfloat16 + out : (1, 1104) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 255 tokens) : 11.2770 + top-1 accuracy : 17/255 = 6.7% + + [2026-04-29 13:55:04] iteration 321/ 500 | consumed samples: 1284 | elapsed time per iteration (ms): 4516.1 | throughput (token/sec/GPU): 1214.8 | learning rate: 5.964065E-05 | global batch size: 4 | lm loss: 1.125132E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1450]) +Dataloader batch - labels shape: torch.Size([1, 1450]) +Dataloader batch - attn_mask shape: torch.Size([1, 1450]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 885 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1450) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1450) torch.int64 + loss_mask : 342 / 1450 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1450, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1450, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1450, 1, 576) torch.bfloat16 + out : (1, 1450) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2942 + top-1 accuracy : 12/342 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1330]) +Dataloader batch - labels shape: torch.Size([1, 1330]) +Dataloader batch - attn_mask shape: torch.Size([1, 1330]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 886 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1330) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1330) torch.int64 + loss_mask : 428 / 1330 tokens (32.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1330, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1330, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1330, 1, 576) torch.bfloat16 + out : (1, 1330) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 428 tokens) : 11.4512 + top-1 accuracy : 0/428 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 887 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 295 / 1443 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.1824 + top-1 accuracy : 4/295 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 888 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 431 / 1487 tokens (29.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.3769 + top-1 accuracy : 3/431 = 0.7% + + [2026-04-29 13:55:08] iteration 322/ 500 | consumed samples: 1288 | elapsed time per iteration (ms): 4470.0 | throughput (token/sec/GPU): 1277.4 | learning rate: 5.933419E-05 | global batch size: 4 | lm loss: 1.134088E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1530]) +Dataloader batch - labels shape: torch.Size([1, 1530]) +Dataloader batch - attn_mask shape: torch.Size([1, 1530]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 889 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1530) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1530) torch.int64 + loss_mask : 329 / 1530 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1530, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1530, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1530, 1, 576) torch.bfloat16 + out : (1, 1530) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.1901 + top-1 accuracy : 19/329 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1901]) +Dataloader batch - labels shape: torch.Size([1, 1901]) +Dataloader batch - attn_mask shape: torch.Size([1, 1901]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 890 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1901) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1901) torch.int64 + loss_mask : 539 / 1901 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1901, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1901, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1901, 1, 576) torch.bfloat16 + out : (1, 1901) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 539 tokens) : 11.3533 + top-1 accuracy : 4/539 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1601]) +Dataloader batch - labels shape: torch.Size([1, 1601]) +Dataloader batch - attn_mask shape: torch.Size([1, 1601]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 891 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1601) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1601) torch.int64 + loss_mask : 390 / 1601 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1601, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1601, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1601, 1, 576) torch.bfloat16 + out : (1, 1601) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.2660 + top-1 accuracy : 19/390 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1398]) +Dataloader batch - labels shape: torch.Size([1, 1398]) +Dataloader batch - attn_mask shape: torch.Size([1, 1398]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 892 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1398) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1398) torch.int64 + loss_mask : 317 / 1398 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1398, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1398, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1398, 1, 576) torch.bfloat16 + out : (1, 1398) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2300 + top-1 accuracy : 16/317 = 5.0% + + [2026-04-29 13:55:13] iteration 323/ 500 | consumed samples: 1292 | elapsed time per iteration (ms): 5142.6 | throughput (token/sec/GPU): 1250.3 | learning rate: 5.902737E-05 | global batch size: 4 | lm loss: 1.127278E+01 | loss scale: 1.0 | grad norm: 0.268 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 952]) +Dataloader batch - labels shape: torch.Size([1, 952]) +Dataloader batch - attn_mask shape: torch.Size([1, 952]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 893 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 952) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 952) torch.int64 + loss_mask : 229 / 952 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (952, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (952, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (952, 1, 576) torch.bfloat16 + out : (1, 952) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 229 tokens) : 11.2717 + top-1 accuracy : 5/229 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1462]) +Dataloader batch - labels shape: torch.Size([1, 1462]) +Dataloader batch - attn_mask shape: torch.Size([1, 1462]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 894 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1462) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1462) torch.int64 + loss_mask : 407 / 1462 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1462, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1462, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1462, 1, 576) torch.bfloat16 + out : (1, 1462) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.3624 + top-1 accuracy : 8/407 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1826]) +Dataloader batch - labels shape: torch.Size([1, 1826]) +Dataloader batch - attn_mask shape: torch.Size([1, 1826]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 895 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1826) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1826) torch.int64 + loss_mask : 475 / 1826 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1826, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1826, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1826, 1, 576) torch.bfloat16 + out : (1, 1826) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 475 tokens) : 11.2921 + top-1 accuracy : 24/475 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 896 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 276 / 1436 tokens (19.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 276 tokens) : 11.1015 + top-1 accuracy : 22/276 = 8.0% + + [2026-04-29 13:55:18] iteration 324/ 500 | consumed samples: 1296 | elapsed time per iteration (ms): 4628.1 | throughput (token/sec/GPU): 1226.4 | learning rate: 5.872023E-05 | global batch size: 4 | lm loss: 1.127145E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1180]) +Dataloader batch - labels shape: torch.Size([1, 1180]) +Dataloader batch - attn_mask shape: torch.Size([1, 1180]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 897 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1180) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1180) torch.int64 + loss_mask : 278 / 1180 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1180, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1180, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1180, 1, 576) torch.bfloat16 + out : (1, 1180) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.2969 + top-1 accuracy : 5/278 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1817]) +Dataloader batch - labels shape: torch.Size([1, 1817]) +Dataloader batch - attn_mask shape: torch.Size([1, 1817]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 898 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1817) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1817) torch.int64 + loss_mask : 461 / 1817 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1817, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1817, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1817, 1, 576) torch.bfloat16 + out : (1, 1817) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 461 tokens) : 11.2819 + top-1 accuracy : 13/461 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1130]) +Dataloader batch - labels shape: torch.Size([1, 1130]) +Dataloader batch - attn_mask shape: torch.Size([1, 1130]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 899 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1130) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1130) torch.int64 + loss_mask : 323 / 1130 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1130, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1130, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1130, 1, 576) torch.bfloat16 + out : (1, 1130) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.3937 + top-1 accuracy : 1/323 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1137]) +Dataloader batch - labels shape: torch.Size([1, 1137]) +Dataloader batch - attn_mask shape: torch.Size([1, 1137]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 900 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1137) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1137) torch.int64 + loss_mask : 334 / 1137 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1137, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1137, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1137, 1, 576) torch.bfloat16 + out : (1, 1137) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.4002 + top-1 accuracy : 12/334 = 3.6% + + [2026-04-29 13:55:22] iteration 325/ 500 | consumed samples: 1300 | elapsed time per iteration (ms): 4279.8 | throughput (token/sec/GPU): 1230.0 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.133906E+01 | loss scale: 1.0 | grad norm: 0.206 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 969]) +Dataloader batch - labels shape: torch.Size([1, 969]) +Dataloader batch - attn_mask shape: torch.Size([1, 969]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 901 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 969) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 969) torch.int64 + loss_mask : 219 / 969 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (969, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (969, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (969, 1, 576) torch.bfloat16 + out : (1, 969) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 219 tokens) : 11.2580 + top-1 accuracy : 12/219 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1087]) +Dataloader batch - labels shape: torch.Size([1, 1087]) +Dataloader batch - attn_mask shape: torch.Size([1, 1087]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 902 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1087) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1087) torch.int64 + loss_mask : 313 / 1087 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1087, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1087, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1087, 1, 576) torch.bfloat16 + out : (1, 1087) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.3460 + top-1 accuracy : 6/313 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1522]) +Dataloader batch - labels shape: torch.Size([1, 1522]) +Dataloader batch - attn_mask shape: torch.Size([1, 1522]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 903 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1522) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1522) torch.int64 + loss_mask : 376 / 1522 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1522, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1522, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1522, 1, 576) torch.bfloat16 + out : (1, 1522) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.2661 + top-1 accuracy : 8/376 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1157]) +Dataloader batch - labels shape: torch.Size([1, 1157]) +Dataloader batch - attn_mask shape: torch.Size([1, 1157]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 904 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1157) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1157) torch.int64 + loss_mask : 346 / 1157 tokens (29.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1157, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1157, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1157, 1, 576) torch.bfloat16 + out : (1, 1157) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.3766 + top-1 accuracy : 0/346 = 0.0% + + [2026-04-29 13:55:26] iteration 326/ 500 | consumed samples: 1304 | elapsed time per iteration (ms): 3907.1 | throughput (token/sec/GPU): 1211.9 | learning rate: 5.810496E-05 | global batch size: 4 | lm loss: 1.131513E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1476]) +Dataloader batch - labels shape: torch.Size([1, 1476]) +Dataloader batch - attn_mask shape: torch.Size([1, 1476]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 905 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1476) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1476) torch.int64 + loss_mask : 313 / 1476 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1476, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1476, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1476, 1, 576) torch.bfloat16 + out : (1, 1476) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.1845 + top-1 accuracy : 19/313 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1837]) +Dataloader batch - labels shape: torch.Size([1, 1837]) +Dataloader batch - attn_mask shape: torch.Size([1, 1837]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 906 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1837) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1837) torch.int64 + loss_mask : 516 / 1837 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1837, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1837, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1837, 1, 576) torch.bfloat16 + out : (1, 1837) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 516 tokens) : 11.3626 + top-1 accuracy : 7/516 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1826]) +Dataloader batch - labels shape: torch.Size([1, 1826]) +Dataloader batch - attn_mask shape: torch.Size([1, 1826]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 907 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1826) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1826) torch.int64 + loss_mask : 465 / 1826 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1826, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1826, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1826, 1, 576) torch.bfloat16 + out : (1, 1826) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 465 tokens) : 11.3408 + top-1 accuracy : 9/465 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1153]) +Dataloader batch - labels shape: torch.Size([1, 1153]) +Dataloader batch - attn_mask shape: torch.Size([1, 1153]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 908 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1153) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1153) torch.int64 + loss_mask : 298 / 1153 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1153, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1153, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1153, 1, 576) torch.bfloat16 + out : (1, 1153) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 298 tokens) : 11.3195 + top-1 accuracy : 17/298 = 5.7% + + [2026-04-29 13:55:31] iteration 327/ 500 | consumed samples: 1308 | elapsed time per iteration (ms): 4971.4 | throughput (token/sec/GPU): 1265.6 | learning rate: 5.779687E-05 | global batch size: 4 | lm loss: 1.131317E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1779]) +Dataloader batch - labels shape: torch.Size([1, 1779]) +Dataloader batch - attn_mask shape: torch.Size([1, 1779]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 909 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1779) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1779) torch.int64 + loss_mask : 418 / 1779 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1779, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1779, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1779, 1, 576) torch.bfloat16 + out : (1, 1779) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 418 tokens) : 11.2545 + top-1 accuracy : 7/418 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1804]) +Dataloader batch - labels shape: torch.Size([1, 1804]) +Dataloader batch - attn_mask shape: torch.Size([1, 1804]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 910 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1804) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1804) torch.int64 + loss_mask : 438 / 1804 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1804, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1804, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1804, 1, 576) torch.bfloat16 + out : (1, 1804) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 438 tokens) : 11.2279 + top-1 accuracy : 4/438 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1338]) +Dataloader batch - labels shape: torch.Size([1, 1338]) +Dataloader batch - attn_mask shape: torch.Size([1, 1338]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 911 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1338) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1338) torch.int64 + loss_mask : 353 / 1338 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1338, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1338, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1338, 1, 576) torch.bfloat16 + out : (1, 1338) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.3521 + top-1 accuracy : 2/353 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1647]) +Dataloader batch - labels shape: torch.Size([1, 1647]) +Dataloader batch - attn_mask shape: torch.Size([1, 1647]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 912 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1647) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1647) torch.int64 + loss_mask : 375 / 1647 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1647, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1647, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1647, 1, 576) torch.bfloat16 + out : (1, 1647) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.2388 + top-1 accuracy : 17/375 = 4.5% + + [2026-04-29 13:55:36] iteration 328/ 500 | consumed samples: 1312 | elapsed time per iteration (ms): 5301.0 | throughput (token/sec/GPU): 1239.0 | learning rate: 5.748849E-05 | global batch size: 4 | lm loss: 1.126519E+01 | loss scale: 1.0 | grad norm: 0.201 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1880]) +Dataloader batch - labels shape: torch.Size([1, 1880]) +Dataloader batch - attn_mask shape: torch.Size([1, 1880]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 913 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1880) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1880) torch.int64 + loss_mask : 512 / 1880 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1880, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1880, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1880, 1, 576) torch.bfloat16 + out : (1, 1880) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 512 tokens) : 11.3286 + top-1 accuracy : 1/512 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 914 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 290 / 1364 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.2135 + top-1 accuracy : 31/290 = 10.7% + +Dataloader batch - tokens shape: torch.Size([1, 1731]) +Dataloader batch - labels shape: torch.Size([1, 1731]) +Dataloader batch - attn_mask shape: torch.Size([1, 1731]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 915 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1731) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1731) torch.int64 + loss_mask : 375 / 1731 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1731, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1731, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1731, 1, 576) torch.bfloat16 + out : (1, 1731) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 375 tokens) : 11.2214 + top-1 accuracy : 24/375 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 916 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 381 / 1438 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.3446 + top-1 accuracy : 23/381 = 6.0% + + [2026-04-29 13:55:41] iteration 329/ 500 | consumed samples: 1316 | elapsed time per iteration (ms): 4948.9 | throughput (token/sec/GPU): 1295.9 | learning rate: 5.717984E-05 | global batch size: 4 | lm loss: 1.128527E+01 | loss scale: 1.0 | grad norm: 0.231 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1523]) +Dataloader batch - labels shape: torch.Size([1, 1523]) +Dataloader batch - attn_mask shape: torch.Size([1, 1523]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 917 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1523) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1523) torch.int64 + loss_mask : 416 / 1523 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1523, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1523, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1523, 1, 576) torch.bfloat16 + out : (1, 1523) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.3269 + top-1 accuracy : 5/416 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 918 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 452 / 1520 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 452 tokens) : 11.4017 + top-1 accuracy : 3/452 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1387]) +Dataloader batch - labels shape: torch.Size([1, 1387]) +Dataloader batch - attn_mask shape: torch.Size([1, 1387]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 919 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1387) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1387) torch.int64 + loss_mask : 322 / 1387 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1387, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1387, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1387, 1, 576) torch.bfloat16 + out : (1, 1387) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.2627 + top-1 accuracy : 15/322 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 951]) +Dataloader batch - labels shape: torch.Size([1, 951]) +Dataloader batch - attn_mask shape: torch.Size([1, 951]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 920 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 951) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 951) torch.int64 + loss_mask : 213 / 951 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (951, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (951, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (951, 1, 576) torch.bfloat16 + out : (1, 951) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 213 tokens) : 11.2445 + top-1 accuracy : 4/213 = 1.9% + + [2026-04-29 13:55:46] iteration 330/ 500 | consumed samples: 1320 | elapsed time per iteration (ms): 4346.5 | throughput (token/sec/GPU): 1238.0 | learning rate: 5.687092E-05 | global batch size: 4 | lm loss: 1.132376E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1537]) +Dataloader batch - labels shape: torch.Size([1, 1537]) +Dataloader batch - attn_mask shape: torch.Size([1, 1537]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 921 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1537) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1537) torch.int64 + loss_mask : 385 / 1537 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1537, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1537, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1537, 1, 576) torch.bfloat16 + out : (1, 1537) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.3648 + top-1 accuracy : 7/385 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1442]) +Dataloader batch - labels shape: torch.Size([1, 1442]) +Dataloader batch - attn_mask shape: torch.Size([1, 1442]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 922 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1442) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1442) torch.int64 + loss_mask : 420 / 1442 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1442, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1442, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1442, 1, 576) torch.bfloat16 + out : (1, 1442) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 420 tokens) : 11.3774 + top-1 accuracy : 2/420 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1806]) +Dataloader batch - labels shape: torch.Size([1, 1806]) +Dataloader batch - attn_mask shape: torch.Size([1, 1806]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 923 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1806) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1806) torch.int64 + loss_mask : 444 / 1806 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1806, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1806, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1806, 1, 576) torch.bfloat16 + out : (1, 1806) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 444 tokens) : 11.2784 + top-1 accuracy : 7/444 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1958]) +Dataloader batch - labels shape: torch.Size([1, 1958]) +Dataloader batch - attn_mask shape: torch.Size([1, 1958]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 924 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1958) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1958) torch.int64 + loss_mask : 595 / 1958 tokens (30.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1958, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1958, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1958, 1, 576) torch.bfloat16 + out : (1, 1958) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 595 tokens) : 11.3636 + top-1 accuracy : 2/595 = 0.3% + + [2026-04-29 13:55:51] iteration 331/ 500 | consumed samples: 1324 | elapsed time per iteration (ms): 5033.0 | throughput (token/sec/GPU): 1339.8 | learning rate: 5.656174E-05 | global batch size: 4 | lm loss: 1.134648E+01 | loss scale: 1.0 | grad norm: 0.228 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1411]) +Dataloader batch - labels shape: torch.Size([1, 1411]) +Dataloader batch - attn_mask shape: torch.Size([1, 1411]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 925 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1411) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1411) torch.int64 + loss_mask : 405 / 1411 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1411, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1411, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1411, 1, 576) torch.bfloat16 + out : (1, 1411) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.3565 + top-1 accuracy : 3/405 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1454]) +Dataloader batch - labels shape: torch.Size([1, 1454]) +Dataloader batch - attn_mask shape: torch.Size([1, 1454]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 926 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1454) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1454) torch.int64 + loss_mask : 403 / 1454 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1454, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1454, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1454, 1, 576) torch.bfloat16 + out : (1, 1454) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.3530 + top-1 accuracy : 18/403 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1156]) +Dataloader batch - labels shape: torch.Size([1, 1156]) +Dataloader batch - attn_mask shape: torch.Size([1, 1156]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 927 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1156) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1156) torch.int64 + loss_mask : 217 / 1156 tokens (18.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1156, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1156, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1156, 1, 576) torch.bfloat16 + out : (1, 1156) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 217 tokens) : 11.1400 + top-1 accuracy : 16/217 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1437]) +Dataloader batch - labels shape: torch.Size([1, 1437]) +Dataloader batch - attn_mask shape: torch.Size([1, 1437]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 928 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1437) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1437) torch.int64 + loss_mask : 272 / 1437 tokens (18.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1437, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1437, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1437, 1, 576) torch.bfloat16 + out : (1, 1437) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 272 tokens) : 11.0737 + top-1 accuracy : 9/272 = 3.3% + + [2026-04-29 13:55:55] iteration 332/ 500 | consumed samples: 1328 | elapsed time per iteration (ms): 4513.3 | throughput (token/sec/GPU): 1209.3 | learning rate: 5.625233E-05 | global batch size: 4 | lm loss: 1.125987E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1567]) +Dataloader batch - labels shape: torch.Size([1, 1567]) +Dataloader batch - attn_mask shape: torch.Size([1, 1567]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 929 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1567) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1567) torch.int64 + loss_mask : 335 / 1567 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1567, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1567, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1567, 1, 576) torch.bfloat16 + out : (1, 1567) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.1742 + top-1 accuracy : 29/335 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1181]) +Dataloader batch - labels shape: torch.Size([1, 1181]) +Dataloader batch - attn_mask shape: torch.Size([1, 1181]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 930 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1181) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1181) torch.int64 + loss_mask : 263 / 1181 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1181, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1181, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1181, 1, 576) torch.bfloat16 + out : (1, 1181) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.2300 + top-1 accuracy : 18/263 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1401]) +Dataloader batch - labels shape: torch.Size([1, 1401]) +Dataloader batch - attn_mask shape: torch.Size([1, 1401]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 931 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1401) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1401) torch.int64 + loss_mask : 339 / 1401 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1401, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1401, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1401, 1, 576) torch.bfloat16 + out : (1, 1401) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.3157 + top-1 accuracy : 9/339 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 932 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 350 / 1546 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.1960 + top-1 accuracy : 23/350 = 6.6% + + [2026-04-29 13:56:00] iteration 333/ 500 | consumed samples: 1332 | elapsed time per iteration (ms): 4758.9 | throughput (token/sec/GPU): 1196.7 | learning rate: 5.594269E-05 | global batch size: 4 | lm loss: 1.122881E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1077]) +Dataloader batch - labels shape: torch.Size([1, 1077]) +Dataloader batch - attn_mask shape: torch.Size([1, 1077]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 933 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1077) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1077) torch.int64 + loss_mask : 306 / 1077 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1077, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1077, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1077, 1, 576) torch.bfloat16 + out : (1, 1077) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.3616 + top-1 accuracy : 0/306 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1462]) +Dataloader batch - labels shape: torch.Size([1, 1462]) +Dataloader batch - attn_mask shape: torch.Size([1, 1462]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 934 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1462) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1462) torch.int64 + loss_mask : 302 / 1462 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1462, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1462, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1462, 1, 576) torch.bfloat16 + out : (1, 1462) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 302 tokens) : 11.0565 + top-1 accuracy : 31/302 = 10.3% + +Dataloader batch - tokens shape: torch.Size([1, 1536]) +Dataloader batch - labels shape: torch.Size([1, 1536]) +Dataloader batch - attn_mask shape: torch.Size([1, 1536]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 935 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1536) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1536) torch.int64 + loss_mask : 380 / 1536 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1536, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1536, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1536, 1, 576) torch.bfloat16 + out : (1, 1536) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 380 tokens) : 11.2979 + top-1 accuracy : 1/380 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 936 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 313 / 1386 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2505 + top-1 accuracy : 24/313 = 7.7% + + [2026-04-29 13:56:04] iteration 334/ 500 | consumed samples: 1336 | elapsed time per iteration (ms): 4546.5 | throughput (token/sec/GPU): 1201.1 | learning rate: 5.563283E-05 | global batch size: 4 | lm loss: 1.124543E+01 | loss scale: 1.0 | grad norm: 0.214 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1640]) +Dataloader batch - labels shape: torch.Size([1, 1640]) +Dataloader batch - attn_mask shape: torch.Size([1, 1640]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 937 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1640) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1640) torch.int64 + loss_mask : 374 / 1640 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1640, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1640, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1640, 1, 576) torch.bfloat16 + out : (1, 1640) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.2883 + top-1 accuracy : 25/374 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1882]) +Dataloader batch - labels shape: torch.Size([1, 1882]) +Dataloader batch - attn_mask shape: torch.Size([1, 1882]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 938 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1882) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1882) torch.int64 + loss_mask : 531 / 1882 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1882, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1882, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1882, 1, 576) torch.bfloat16 + out : (1, 1882) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 531 tokens) : 11.3494 + top-1 accuracy : 20/531 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1595]) +Dataloader batch - labels shape: torch.Size([1, 1595]) +Dataloader batch - attn_mask shape: torch.Size([1, 1595]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 939 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1595) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1595) torch.int64 + loss_mask : 401 / 1595 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1595, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1595, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1595, 1, 576) torch.bfloat16 + out : (1, 1595) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 401 tokens) : 11.2730 + top-1 accuracy : 11/401 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 940 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 345 / 1436 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2492 + top-1 accuracy : 17/345 = 4.9% + + [2026-04-29 13:56:10] iteration 335/ 500 | consumed samples: 1340 | elapsed time per iteration (ms): 5187.8 | throughput (token/sec/GPU): 1263.2 | learning rate: 5.532277E-05 | global batch size: 4 | lm loss: 1.129604E+01 | loss scale: 1.0 | grad norm: 0.211 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1154]) +Dataloader batch - labels shape: torch.Size([1, 1154]) +Dataloader batch - attn_mask shape: torch.Size([1, 1154]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 941 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1154) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1154) torch.int64 + loss_mask : 321 / 1154 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1154, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1154, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1154, 1, 576) torch.bfloat16 + out : (1, 1154) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.3122 + top-1 accuracy : 4/321 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1735]) +Dataloader batch - labels shape: torch.Size([1, 1735]) +Dataloader batch - attn_mask shape: torch.Size([1, 1735]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 942 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1735) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1735) torch.int64 + loss_mask : 363 / 1735 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1735, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1735, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1735, 1, 576) torch.bfloat16 + out : (1, 1735) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2142 + top-1 accuracy : 10/363 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1521]) +Dataloader batch - labels shape: torch.Size([1, 1521]) +Dataloader batch - attn_mask shape: torch.Size([1, 1521]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 943 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1521) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1521) torch.int64 + loss_mask : 362 / 1521 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1521, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1521, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1521, 1, 576) torch.bfloat16 + out : (1, 1521) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2723 + top-1 accuracy : 12/362 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1033]) +Dataloader batch - labels shape: torch.Size([1, 1033]) +Dataloader batch - attn_mask shape: torch.Size([1, 1033]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 944 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1033) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1033) torch.int64 + loss_mask : 259 / 1033 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1033, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1033, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1033, 1, 576) torch.bfloat16 + out : (1, 1033) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 259 tokens) : 11.3653 + top-1 accuracy : 15/259 = 5.8% + + [2026-04-29 13:56:14] iteration 336/ 500 | consumed samples: 1344 | elapsed time per iteration (ms): 4379.4 | throughput (token/sec/GPU): 1242.9 | learning rate: 5.501252E-05 | global batch size: 4 | lm loss: 1.128441E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1821]) +Dataloader batch - labels shape: torch.Size([1, 1821]) +Dataloader batch - attn_mask shape: torch.Size([1, 1821]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 945 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1821) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1821) torch.int64 + loss_mask : 436 / 1821 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1821, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1821, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1821, 1, 576) torch.bfloat16 + out : (1, 1821) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.2873 + top-1 accuracy : 5/436 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1502]) +Dataloader batch - labels shape: torch.Size([1, 1502]) +Dataloader batch - attn_mask shape: torch.Size([1, 1502]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 946 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1502) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1502) torch.int64 + loss_mask : 386 / 1502 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1502, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1502, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1502, 1, 576) torch.bfloat16 + out : (1, 1502) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2078 + top-1 accuracy : 22/386 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 947 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 307 / 1487 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 307 tokens) : 11.1836 + top-1 accuracy : 25/307 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 979]) +Dataloader batch - labels shape: torch.Size([1, 979]) +Dataloader batch - attn_mask shape: torch.Size([1, 979]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 948 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 979) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 979) torch.int64 + loss_mask : 218 / 979 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (979, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (979, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (979, 1, 576) torch.bfloat16 + out : (1, 979) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 218 tokens) : 11.2000 + top-1 accuracy : 1/218 = 0.5% + + [2026-04-29 13:56:19] iteration 337/ 500 | consumed samples: 1348 | elapsed time per iteration (ms): 4568.8 | throughput (token/sec/GPU): 1267.1 | learning rate: 5.470209E-05 | global batch size: 4 | lm loss: 1.122676E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1477]) +Dataloader batch - labels shape: torch.Size([1, 1477]) +Dataloader batch - attn_mask shape: torch.Size([1, 1477]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 949 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1477) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1477) torch.int64 + loss_mask : 432 / 1477 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1477, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1477, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1477, 1, 576) torch.bfloat16 + out : (1, 1477) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 432 tokens) : 11.4038 + top-1 accuracy : 20/432 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 950 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 400 / 1532 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.3064 + top-1 accuracy : 1/400 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 951 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1519) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1519) torch.int64 + loss_mask : 394 / 1519 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1519, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1519, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1519, 1, 576) torch.bfloat16 + out : (1, 1519) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3194 + top-1 accuracy : 12/394 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 952 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 410 / 1468 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.2888 + top-1 accuracy : 3/410 = 0.7% + + [2026-04-29 13:56:23] iteration 338/ 500 | consumed samples: 1352 | elapsed time per iteration (ms): 4824.4 | throughput (token/sec/GPU): 1242.9 | learning rate: 5.439149E-05 | global batch size: 4 | lm loss: 1.133085E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 953 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 321 / 1468 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2213 + top-1 accuracy : 14/321 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1495]) +Dataloader batch - labels shape: torch.Size([1, 1495]) +Dataloader batch - attn_mask shape: torch.Size([1, 1495]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 954 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1495) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1495) torch.int64 + loss_mask : 362 / 1495 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1495, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1495, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1495, 1, 576) torch.bfloat16 + out : (1, 1495) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2927 + top-1 accuracy : 23/362 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 974]) +Dataloader batch - labels shape: torch.Size([1, 974]) +Dataloader batch - attn_mask shape: torch.Size([1, 974]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 955 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 974) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 974) torch.int64 + loss_mask : 203 / 974 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (974, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (974, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (974, 1, 576) torch.bfloat16 + out : (1, 974) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 203 tokens) : 11.1417 + top-1 accuracy : 9/203 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1720]) +Dataloader batch - labels shape: torch.Size([1, 1720]) +Dataloader batch - attn_mask shape: torch.Size([1, 1720]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 956 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1720) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1720) torch.int64 + loss_mask : 394 / 1720 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1720, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1720, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1720, 1, 576) torch.bfloat16 + out : (1, 1720) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.2494 + top-1 accuracy : 14/394 = 3.6% + + [2026-04-29 13:56:28] iteration 339/ 500 | consumed samples: 1356 | elapsed time per iteration (ms): 4645.0 | throughput (token/sec/GPU): 1217.9 | learning rate: 5.408074E-05 | global batch size: 4 | lm loss: 1.123755E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 957 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 392 / 1468 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3491 + top-1 accuracy : 16/392 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1762]) +Dataloader batch - labels shape: torch.Size([1, 1762]) +Dataloader batch - attn_mask shape: torch.Size([1, 1762]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 958 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1762) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1762) torch.int64 + loss_mask : 404 / 1762 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1762, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1762, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1762, 1, 576) torch.bfloat16 + out : (1, 1762) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.2998 + top-1 accuracy : 26/404 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1191]) +Dataloader batch - labels shape: torch.Size([1, 1191]) +Dataloader batch - attn_mask shape: torch.Size([1, 1191]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 959 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1191) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1191) torch.int64 + loss_mask : 297 / 1191 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1191, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1191, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1191, 1, 576) torch.bfloat16 + out : (1, 1191) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.3176 + top-1 accuracy : 3/297 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1705]) +Dataloader batch - labels shape: torch.Size([1, 1705]) +Dataloader batch - attn_mask shape: torch.Size([1, 1705]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 960 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1705) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1705) torch.int64 + loss_mask : 355 / 1705 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1705, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1705, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1705, 1, 576) torch.bfloat16 + out : (1, 1705) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.1939 + top-1 accuracy : 22/355 = 6.2% + + [2026-04-29 13:56:33] iteration 340/ 500 | consumed samples: 1360 | elapsed time per iteration (ms): 4995.5 | throughput (token/sec/GPU): 1226.3 | learning rate: 5.376985E-05 | global batch size: 4 | lm loss: 1.129081E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4979.61, 4979.61) + forward-compute ................................: (3702.41, 3702.41) + backward-compute ...............................: (1274.45, 1274.45) + batch-generator ................................: (439.64, 439.64) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.05, 0.05) + all-grads-sync .................................: (0.74, 0.74) + params-all-gather ..............................: (0.23, 0.23) + optimizer-copy-to-main-grad ....................: (0.15, 0.15) + optimizer-clip-main-grad .......................: (0.68, 0.68) + optimizer-count-zeros ..........................: (0.03, 0.03) + optimizer-inner-step ...........................: (1.45, 1.45) + optimizer-copy-main-to-model-params ............: (0.26, 0.26) + optimizer ......................................: (3.44, 3.44) +Dataloader batch - tokens shape: torch.Size([1, 1687]) +Dataloader batch - labels shape: torch.Size([1, 1687]) +Dataloader batch - attn_mask shape: torch.Size([1, 1687]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 961 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1687) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1687) torch.int64 + loss_mask : 387 / 1687 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1687, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1687, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1687, 1, 576) torch.bfloat16 + out : (1, 1687) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2173 + top-1 accuracy : 36/387 = 9.3% + +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 962 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 355 / 1722 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2223 + top-1 accuracy : 8/355 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 963 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 434 / 1436 tokens (30.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 434 tokens) : 11.3800 + top-1 accuracy : 13/434 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1465]) +Dataloader batch - labels shape: torch.Size([1, 1465]) +Dataloader batch - attn_mask shape: torch.Size([1, 1465]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 964 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1465) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1465) torch.int64 + loss_mask : 414 / 1465 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1465, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1465, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1465, 1, 576) torch.bfloat16 + out : (1, 1465) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 414 tokens) : 11.3845 + top-1 accuracy : 3/414 = 0.7% + + [2026-04-29 13:56:38] iteration 341/ 500 | consumed samples: 1364 | elapsed time per iteration (ms): 4972.6 | throughput (token/sec/GPU): 1269.0 | learning rate: 5.345883E-05 | global batch size: 4 | lm loss: 1.130637E+01 | loss scale: 1.0 | grad norm: 0.173 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1826]) +Dataloader batch - labels shape: torch.Size([1, 1826]) +Dataloader batch - attn_mask shape: torch.Size([1, 1826]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 965 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1826) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1826) torch.int64 + loss_mask : 465 / 1826 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1826, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1826, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1826, 1, 576) torch.bfloat16 + out : (1, 1826) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 465 tokens) : 11.3305 + top-1 accuracy : 23/465 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1413]) +Dataloader batch - labels shape: torch.Size([1, 1413]) +Dataloader batch - attn_mask shape: torch.Size([1, 1413]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 966 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1413) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1413) torch.int64 + loss_mask : 332 / 1413 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1413, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1413, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1413, 1, 576) torch.bfloat16 + out : (1, 1413) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.2784 + top-1 accuracy : 19/332 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1482]) +Dataloader batch - labels shape: torch.Size([1, 1482]) +Dataloader batch - attn_mask shape: torch.Size([1, 1482]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 967 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1482) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1482) torch.int64 + loss_mask : 456 / 1482 tokens (30.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1482, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1482, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1482, 1, 576) torch.bfloat16 + out : (1, 1482) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 456 tokens) : 11.4199 + top-1 accuracy : 8/456 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1871]) +Dataloader batch - labels shape: torch.Size([1, 1871]) +Dataloader batch - attn_mask shape: torch.Size([1, 1871]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 968 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1871) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1871) torch.int64 + loss_mask : 520 / 1871 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1871, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1871, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1871, 1, 576) torch.bfloat16 + out : (1, 1871) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 520 tokens) : 11.4320 + top-1 accuracy : 20/520 = 3.8% + + [2026-04-29 13:56:43] iteration 342/ 500 | consumed samples: 1368 | elapsed time per iteration (ms): 5140.9 | throughput (token/sec/GPU): 1282.3 | learning rate: 5.314768E-05 | global batch size: 4 | lm loss: 1.137350E+01 | loss scale: 1.0 | grad norm: 0.175 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1264]) +Dataloader batch - labels shape: torch.Size([1, 1264]) +Dataloader batch - attn_mask shape: torch.Size([1, 1264]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 969 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1264) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1264) torch.int64 + loss_mask : 342 / 1264 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1264, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1264, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1264, 1, 576) torch.bfloat16 + out : (1, 1264) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.3185 + top-1 accuracy : 0/342 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 970 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 436 / 1496 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.3827 + top-1 accuracy : 2/436 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1084]) +Dataloader batch - labels shape: torch.Size([1, 1084]) +Dataloader batch - attn_mask shape: torch.Size([1, 1084]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 971 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1084) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1084) torch.int64 + loss_mask : 275 / 1084 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1084, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1084, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1084, 1, 576) torch.bfloat16 + out : (1, 1084) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 275 tokens) : 11.2849 + top-1 accuracy : 16/275 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1859]) +Dataloader batch - labels shape: torch.Size([1, 1859]) +Dataloader batch - attn_mask shape: torch.Size([1, 1859]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 972 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1859) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1859) torch.int64 + loss_mask : 517 / 1859 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1859, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1859, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1859, 1, 576) torch.bfloat16 + out : (1, 1859) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 517 tokens) : 11.3247 + top-1 accuracy : 14/517 = 2.7% + + [2026-04-29 13:56:48] iteration 343/ 500 | consumed samples: 1372 | elapsed time per iteration (ms): 4422.8 | throughput (token/sec/GPU): 1289.4 | learning rate: 5.283644E-05 | global batch size: 4 | lm loss: 1.133250E+01 | loss scale: 1.0 | grad norm: 0.209 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1049]) +Dataloader batch - labels shape: torch.Size([1, 1049]) +Dataloader batch - attn_mask shape: torch.Size([1, 1049]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 973 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1049) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1049) torch.int64 + loss_mask : 270 / 1049 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1049, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1049, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1049, 1, 576) torch.bfloat16 + out : (1, 1049) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 270 tokens) : 11.3136 + top-1 accuracy : 2/270 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1022]) +Dataloader batch - labels shape: torch.Size([1, 1022]) +Dataloader batch - attn_mask shape: torch.Size([1, 1022]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 974 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1022) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1022) torch.int64 + loss_mask : 254 / 1022 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1022, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1022, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1022, 1, 576) torch.bfloat16 + out : (1, 1022) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 254 tokens) : 11.3475 + top-1 accuracy : 15/254 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1507]) +Dataloader batch - labels shape: torch.Size([1, 1507]) +Dataloader batch - attn_mask shape: torch.Size([1, 1507]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 975 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1507) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1507) torch.int64 + loss_mask : 320 / 1507 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1507, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1507, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1507, 1, 576) torch.bfloat16 + out : (1, 1507) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.1346 + top-1 accuracy : 26/320 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1545]) +Dataloader batch - labels shape: torch.Size([1, 1545]) +Dataloader batch - attn_mask shape: torch.Size([1, 1545]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 976 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1545) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1545) torch.int64 + loss_mask : 389 / 1545 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1545, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1545, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1545, 1, 576) torch.bfloat16 + out : (1, 1545) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.2404 + top-1 accuracy : 3/389 = 0.8% + + [2026-04-29 13:56:52] iteration 344/ 500 | consumed samples: 1376 | elapsed time per iteration (ms): 4451.2 | throughput (token/sec/GPU): 1150.9 | learning rate: 5.252510E-05 | global batch size: 4 | lm loss: 1.125102E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 999]) +Dataloader batch - labels shape: torch.Size([1, 999]) +Dataloader batch - attn_mask shape: torch.Size([1, 999]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 977 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 999) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 999) torch.int64 + loss_mask : 227 / 999 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (999, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (999, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (999, 1, 576) torch.bfloat16 + out : (1, 999) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 227 tokens) : 11.1883 + top-1 accuracy : 1/227 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1623]) +Dataloader batch - labels shape: torch.Size([1, 1623]) +Dataloader batch - attn_mask shape: torch.Size([1, 1623]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 978 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1623) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1623) torch.int64 + loss_mask : 383 / 1623 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1623, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1623, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1623, 1, 576) torch.bfloat16 + out : (1, 1623) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.2411 + top-1 accuracy : 16/383 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1495]) +Dataloader batch - labels shape: torch.Size([1, 1495]) +Dataloader batch - attn_mask shape: torch.Size([1, 1495]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 979 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1495) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1495) torch.int64 + loss_mask : 437 / 1495 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1495, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1495, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1495, 1, 576) torch.bfloat16 + out : (1, 1495) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 437 tokens) : 11.4063 + top-1 accuracy : 4/437 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1789]) +Dataloader batch - labels shape: torch.Size([1, 1789]) +Dataloader batch - attn_mask shape: torch.Size([1, 1789]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 980 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1789) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1789) torch.int64 + loss_mask : 426 / 1789 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1789, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1789, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1789, 1, 576) torch.bfloat16 + out : (1, 1789) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.2884 + top-1 accuracy : 11/426 = 2.6% + + [2026-04-29 13:56:57] iteration 345/ 500 | consumed samples: 1380 | elapsed time per iteration (ms): 5079.6 | throughput (token/sec/GPU): 1162.7 | learning rate: 5.221368E-05 | global batch size: 4 | lm loss: 1.129565E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1080]) +Dataloader batch - labels shape: torch.Size([1, 1080]) +Dataloader batch - attn_mask shape: torch.Size([1, 1080]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 981 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1080) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1080) torch.int64 + loss_mask : 251 / 1080 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1080, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1080, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1080, 1, 576) torch.bfloat16 + out : (1, 1080) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2192 + top-1 accuracy : 23/251 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1512]) +Dataloader batch - labels shape: torch.Size([1, 1512]) +Dataloader batch - attn_mask shape: torch.Size([1, 1512]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 982 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1512) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1512) torch.int64 + loss_mask : 332 / 1512 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1512, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1512, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1512, 1, 576) torch.bfloat16 + out : (1, 1512) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.1931 + top-1 accuracy : 20/332 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1900]) +Dataloader batch - labels shape: torch.Size([1, 1900]) +Dataloader batch - attn_mask shape: torch.Size([1, 1900]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 983 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1900) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1900) torch.int64 + loss_mask : 534 / 1900 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1900, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1900, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1900, 1, 576) torch.bfloat16 + out : (1, 1900) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 534 tokens) : 11.3531 + top-1 accuracy : 6/534 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1204]) +Dataloader batch - labels shape: torch.Size([1, 1204]) +Dataloader batch - attn_mask shape: torch.Size([1, 1204]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 984 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1204) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1204) torch.int64 + loss_mask : 267 / 1204 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1204, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1204, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1204, 1, 576) torch.bfloat16 + out : (1, 1204) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 267 tokens) : 11.2612 + top-1 accuracy : 9/267 = 3.4% + + [2026-04-29 13:57:02] iteration 346/ 500 | consumed samples: 1384 | elapsed time per iteration (ms): 4824.0 | throughput (token/sec/GPU): 1180.8 | learning rate: 5.190220E-05 | global batch size: 4 | lm loss: 1.127270E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1582]) +Dataloader batch - labels shape: torch.Size([1, 1582]) +Dataloader batch - attn_mask shape: torch.Size([1, 1582]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 985 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1582) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1582) torch.int64 + loss_mask : 366 / 1582 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1582, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1582, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1582, 1, 576) torch.bfloat16 + out : (1, 1582) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2671 + top-1 accuracy : 10/366 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1372]) +Dataloader batch - labels shape: torch.Size([1, 1372]) +Dataloader batch - attn_mask shape: torch.Size([1, 1372]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 986 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1372) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1372) torch.int64 + loss_mask : 391 / 1372 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1372, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1372, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1372, 1, 576) torch.bfloat16 + out : (1, 1372) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3973 + top-1 accuracy : 5/391 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 987 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 311 / 1400 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2031 + top-1 accuracy : 15/311 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 944]) +Dataloader batch - labels shape: torch.Size([1, 944]) +Dataloader batch - attn_mask shape: torch.Size([1, 944]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 988 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 944) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 944) torch.int64 + loss_mask : 215 / 944 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (944, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (944, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (944, 1, 576) torch.bfloat16 + out : (1, 944) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 215 tokens) : 11.1965 + top-1 accuracy : 15/215 = 7.0% + + [2026-04-29 13:57:06] iteration 347/ 500 | consumed samples: 1388 | elapsed time per iteration (ms): 4380.7 | throughput (token/sec/GPU): 1209.4 | learning rate: 5.159065E-05 | global batch size: 4 | lm loss: 1.127941E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1893]) +Dataloader batch - labels shape: torch.Size([1, 1893]) +Dataloader batch - attn_mask shape: torch.Size([1, 1893]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 989 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1893) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1893) torch.int64 + loss_mask : 543 / 1893 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1893, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1893, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1893, 1, 576) torch.bfloat16 + out : (1, 1893) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 543 tokens) : 11.3796 + top-1 accuracy : 2/543 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1180]) +Dataloader batch - labels shape: torch.Size([1, 1180]) +Dataloader batch - attn_mask shape: torch.Size([1, 1180]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 990 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1180) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1180) torch.int64 + loss_mask : 331 / 1180 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1180, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1180, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1180, 1, 576) torch.bfloat16 + out : (1, 1180) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.3708 + top-1 accuracy : 6/331 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1662]) +Dataloader batch - labels shape: torch.Size([1, 1662]) +Dataloader batch - attn_mask shape: torch.Size([1, 1662]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 991 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1662) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1662) torch.int64 + loss_mask : 391 / 1662 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1662, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1662, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1662, 1, 576) torch.bfloat16 + out : (1, 1662) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2758 + top-1 accuracy : 10/391 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1960]) +Dataloader batch - labels shape: torch.Size([1, 1960]) +Dataloader batch - attn_mask shape: torch.Size([1, 1960]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 992 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1960) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1960) torch.int64 + loss_mask : 583 / 1960 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1960, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1960, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1960, 1, 576) torch.bfloat16 + out : (1, 1960) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 583 tokens) : 11.3441 + top-1 accuracy : 2/583 = 0.3% + + [2026-04-29 13:57:11] iteration 348/ 500 | consumed samples: 1392 | elapsed time per iteration (ms): 5141.9 | throughput (token/sec/GPU): 1302.1 | learning rate: 5.127907E-05 | global batch size: 4 | lm loss: 1.134486E+01 | loss scale: 1.0 | grad norm: 0.198 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 993 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 352 / 1722 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.1740 + top-1 accuracy : 34/352 = 9.7% + +Dataloader batch - tokens shape: torch.Size([1, 1054]) +Dataloader batch - labels shape: torch.Size([1, 1054]) +Dataloader batch - attn_mask shape: torch.Size([1, 1054]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 994 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1054) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1054) torch.int64 + loss_mask : 268 / 1054 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1054, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1054, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1054, 1, 576) torch.bfloat16 + out : (1, 1054) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 268 tokens) : 11.2669 + top-1 accuracy : 7/268 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1358]) +Dataloader batch - labels shape: torch.Size([1, 1358]) +Dataloader batch - attn_mask shape: torch.Size([1, 1358]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 995 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1358) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1358) torch.int64 + loss_mask : 387 / 1358 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1358, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1358, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1358, 1, 576) torch.bfloat16 + out : (1, 1358) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.3668 + top-1 accuracy : 1/387 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1636]) +Dataloader batch - labels shape: torch.Size([1, 1636]) +Dataloader batch - attn_mask shape: torch.Size([1, 1636]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 996 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1636) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1636) torch.int64 + loss_mask : 400 / 1636 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1636, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1636, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1636, 1, 576) torch.bfloat16 + out : (1, 1636) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.2860 + top-1 accuracy : 10/400 = 2.5% + + [2026-04-29 13:57:17] iteration 349/ 500 | consumed samples: 1396 | elapsed time per iteration (ms): 5237.8 | throughput (token/sec/GPU): 1101.6 | learning rate: 5.096745E-05 | global batch size: 4 | lm loss: 1.127656E+01 | loss scale: 1.0 | grad norm: 0.217 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1163]) +Dataloader batch - labels shape: torch.Size([1, 1163]) +Dataloader batch - attn_mask shape: torch.Size([1, 1163]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 997 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1163) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1163) torch.int64 + loss_mask : 320 / 1163 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1163, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1163, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1163, 1, 576) torch.bfloat16 + out : (1, 1163) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.3239 + top-1 accuracy : 3/320 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 998 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 465 / 1431 tokens (32.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 465 tokens) : 11.4584 + top-1 accuracy : 5/465 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1767]) +Dataloader batch - labels shape: torch.Size([1, 1767]) +Dataloader batch - attn_mask shape: torch.Size([1, 1767]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 999 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1767) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1767) torch.int64 + loss_mask : 409 / 1767 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1767, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1767, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1767, 1, 576) torch.bfloat16 + out : (1, 1767) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 409 tokens) : 11.2964 + top-1 accuracy : 9/409 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1046]) +Dataloader batch - labels shape: torch.Size([1, 1046]) +Dataloader batch - attn_mask shape: torch.Size([1, 1046]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1000 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1046) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1046) torch.int64 + loss_mask : 251 / 1046 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1046, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1046, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1046, 1, 576) torch.bfloat16 + out : (1, 1046) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2705 + top-1 accuracy : 13/251 = 5.2% + + [2026-04-29 13:57:21] iteration 350/ 500 | consumed samples: 1400 | elapsed time per iteration (ms): 4217.0 | throughput (token/sec/GPU): 1282.2 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.135012E+01 | loss scale: 1.0 | grad norm: 0.173 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 350 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 350 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 350 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2414.00, 2414.00) +Dataloader batch - tokens shape: torch.Size([1, 1432]) +Dataloader batch - labels shape: torch.Size([1, 1432]) +Dataloader batch - attn_mask shape: torch.Size([1, 1432]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1001 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1432) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1432) torch.int64 + loss_mask : 361 / 1432 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1432, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1432, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1432, 1, 576) torch.bfloat16 + out : (1, 1432) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.2519 + top-1 accuracy : 19/361 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1002 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 321 / 1374 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2630 + top-1 accuracy : 14/321 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1070]) +Dataloader batch - labels shape: torch.Size([1, 1070]) +Dataloader batch - attn_mask shape: torch.Size([1, 1070]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1003 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1070) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1070) torch.int64 + loss_mask : 305 / 1070 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1070, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1070, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1070, 1, 576) torch.bfloat16 + out : (1, 1070) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.3676 + top-1 accuracy : 0/305 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1836]) +Dataloader batch - labels shape: torch.Size([1, 1836]) +Dataloader batch - attn_mask shape: torch.Size([1, 1836]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1004 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1836) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1836) torch.int64 + loss_mask : 471 / 1836 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1836, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1836, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1836, 1, 576) torch.bfloat16 + out : (1, 1836) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 471 tokens) : 11.2949 + top-1 accuracy : 8/471 = 1.7% + + [2026-04-29 13:57:28] iteration 351/ 500 | consumed samples: 1404 | elapsed time per iteration (ms): 4616.5 | throughput (token/sec/GPU): 1237.3 | learning rate: 5.034418E-05 | global batch size: 4 | lm loss: 1.129241E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1397]) +Dataloader batch - labels shape: torch.Size([1, 1397]) +Dataloader batch - attn_mask shape: torch.Size([1, 1397]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1005 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1397) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1397) torch.int64 + loss_mask : 423 / 1397 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1397, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1397, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1397, 1, 576) torch.bfloat16 + out : (1, 1397) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 423 tokens) : 11.4415 + top-1 accuracy : 1/423 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1756]) +Dataloader batch - labels shape: torch.Size([1, 1756]) +Dataloader batch - attn_mask shape: torch.Size([1, 1756]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1006 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1756) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1756) torch.int64 + loss_mask : 411 / 1756 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1756, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1756, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1756, 1, 576) torch.bfloat16 + out : (1, 1756) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.2645 + top-1 accuracy : 32/411 = 7.8% + +Dataloader batch - tokens shape: torch.Size([1, 1806]) +Dataloader batch - labels shape: torch.Size([1, 1806]) +Dataloader batch - attn_mask shape: torch.Size([1, 1806]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1007 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1806) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1806) torch.int64 + loss_mask : 442 / 1806 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1806, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1806, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1806, 1, 576) torch.bfloat16 + out : (1, 1806) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 442 tokens) : 11.2491 + top-1 accuracy : 20/442 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1608]) +Dataloader batch - labels shape: torch.Size([1, 1608]) +Dataloader batch - attn_mask shape: torch.Size([1, 1608]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1008 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1608) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1608) torch.int64 + loss_mask : 400 / 1608 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1608, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1608, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1608, 1, 576) torch.bfloat16 + out : (1, 1608) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.3232 + top-1 accuracy : 11/400 = 2.8% + + [2026-04-29 13:57:33] iteration 352/ 500 | consumed samples: 1408 | elapsed time per iteration (ms): 5114.8 | throughput (token/sec/GPU): 1283.9 | learning rate: 5.003254E-05 | global batch size: 4 | lm loss: 1.131911E+01 | loss scale: 1.0 | grad norm: 0.193 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1553]) +Dataloader batch - labels shape: torch.Size([1, 1553]) +Dataloader batch - attn_mask shape: torch.Size([1, 1553]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1009 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1553) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1553) torch.int64 + loss_mask : 418 / 1553 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1553, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1553, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1553, 1, 576) torch.bfloat16 + out : (1, 1553) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 418 tokens) : 11.2991 + top-1 accuracy : 7/418 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1788]) +Dataloader batch - labels shape: torch.Size([1, 1788]) +Dataloader batch - attn_mask shape: torch.Size([1, 1788]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1010 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1788) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1788) torch.int64 + loss_mask : 427 / 1788 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1788, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1788, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1788, 1, 576) torch.bfloat16 + out : (1, 1788) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 427 tokens) : 11.2867 + top-1 accuracy : 24/427 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1111]) +Dataloader batch - labels shape: torch.Size([1, 1111]) +Dataloader batch - attn_mask shape: torch.Size([1, 1111]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1011 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1111) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1111) torch.int64 + loss_mask : 288 / 1111 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1111, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1111, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1111, 1, 576) torch.bfloat16 + out : (1, 1111) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 288 tokens) : 11.3084 + top-1 accuracy : 19/288 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1012 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 484 / 1468 tokens (33.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 484 tokens) : 11.4655 + top-1 accuracy : 2/484 = 0.4% + + [2026-04-29 13:57:38] iteration 353/ 500 | consumed samples: 1412 | elapsed time per iteration (ms): 4557.1 | throughput (token/sec/GPU): 1299.1 | learning rate: 4.972093E-05 | global batch size: 4 | lm loss: 1.134729E+01 | loss scale: 1.0 | grad norm: 0.214 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1724]) +Dataloader batch - labels shape: torch.Size([1, 1724]) +Dataloader batch - attn_mask shape: torch.Size([1, 1724]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1013 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1724) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1724) torch.int64 + loss_mask : 351 / 1724 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1724, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1724, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1724, 1, 576) torch.bfloat16 + out : (1, 1724) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.1848 + top-1 accuracy : 21/351 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1401]) +Dataloader batch - labels shape: torch.Size([1, 1401]) +Dataloader batch - attn_mask shape: torch.Size([1, 1401]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1014 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1401) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1401) torch.int64 + loss_mask : 334 / 1401 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1401, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1401, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1401, 1, 576) torch.bfloat16 + out : (1, 1401) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.2648 + top-1 accuracy : 16/334 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1882]) +Dataloader batch - labels shape: torch.Size([1, 1882]) +Dataloader batch - attn_mask shape: torch.Size([1, 1882]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1015 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1882) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1882) torch.int64 + loss_mask : 523 / 1882 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1882, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1882, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1882, 1, 576) torch.bfloat16 + out : (1, 1882) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 523 tokens) : 11.3413 + top-1 accuracy : 11/523 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1560]) +Dataloader batch - labels shape: torch.Size([1, 1560]) +Dataloader batch - attn_mask shape: torch.Size([1, 1560]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1016 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1560) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1560) torch.int64 + loss_mask : 350 / 1560 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1560, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1560, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1560, 1, 576) torch.bfloat16 + out : (1, 1560) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.2323 + top-1 accuracy : 15/350 = 4.3% + + [2026-04-29 13:57:43] iteration 354/ 500 | consumed samples: 1416 | elapsed time per iteration (ms): 5134.4 | throughput (token/sec/GPU): 1279.0 | learning rate: 4.940934E-05 | global batch size: 4 | lm loss: 1.126514E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1017 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 397 / 1542 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.3078 + top-1 accuracy : 12/397 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1156]) +Dataloader batch - labels shape: torch.Size([1, 1156]) +Dataloader batch - attn_mask shape: torch.Size([1, 1156]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1018 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1156) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1156) torch.int64 + loss_mask : 311 / 1156 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1156, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1156, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1156, 1, 576) torch.bfloat16 + out : (1, 1156) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.3059 + top-1 accuracy : 9/311 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1837]) +Dataloader batch - labels shape: torch.Size([1, 1837]) +Dataloader batch - attn_mask shape: torch.Size([1, 1837]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1019 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1837) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1837) torch.int64 + loss_mask : 481 / 1837 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1837, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1837, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1837, 1, 576) torch.bfloat16 + out : (1, 1837) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 481 tokens) : 11.3186 + top-1 accuracy : 23/481 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1544]) +Dataloader batch - labels shape: torch.Size([1, 1544]) +Dataloader batch - attn_mask shape: torch.Size([1, 1544]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1020 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1544) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1544) torch.int64 + loss_mask : 386 / 1544 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1544, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1544, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1544, 1, 576) torch.bfloat16 + out : (1, 1544) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2710 + top-1 accuracy : 20/386 = 5.2% + + [2026-04-29 13:57:47] iteration 355/ 500 | consumed samples: 1420 | elapsed time per iteration (ms): 4775.2 | throughput (token/sec/GPU): 1273.0 | learning rate: 4.909780E-05 | global batch size: 4 | lm loss: 1.130169E+01 | loss scale: 1.0 | grad norm: 0.248 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 1021 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 381 / 1388 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.3582 + top-1 accuracy : 1/381 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1806]) +Dataloader batch - labels shape: torch.Size([1, 1806]) +Dataloader batch - attn_mask shape: torch.Size([1, 1806]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1022 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1806) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1806) torch.int64 + loss_mask : 454 / 1806 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1806, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1806, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1806, 1, 576) torch.bfloat16 + out : (1, 1806) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 454 tokens) : 11.2647 + top-1 accuracy : 11/454 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1081]) +Dataloader batch - labels shape: torch.Size([1, 1081]) +Dataloader batch - attn_mask shape: torch.Size([1, 1081]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1023 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1081) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1081) torch.int64 + loss_mask : 274 / 1081 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1081, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1081, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1081, 1, 576) torch.bfloat16 + out : (1, 1081) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 274 tokens) : 11.3217 + top-1 accuracy : 4/274 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1368]) +Dataloader batch - labels shape: torch.Size([1, 1368]) +Dataloader batch - attn_mask shape: torch.Size([1, 1368]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1024 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1368) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1368) torch.int64 + loss_mask : 283 / 1368 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1368, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1368, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1368, 1, 576) torch.bfloat16 + out : (1, 1368) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 283 tokens) : 11.1985 + top-1 accuracy : 29/283 = 10.2% + + [2026-04-29 13:57:52] iteration 356/ 500 | consumed samples: 1424 | elapsed time per iteration (ms): 4375.1 | throughput (token/sec/GPU): 1289.8 | learning rate: 4.878632E-05 | global batch size: 4 | lm loss: 1.128806E+01 | loss scale: 1.0 | grad norm: 0.185 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1782]) +Dataloader batch - labels shape: torch.Size([1, 1782]) +Dataloader batch - attn_mask shape: torch.Size([1, 1782]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1025 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1782) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1782) torch.int64 + loss_mask : 424 / 1782 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1782, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1782, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1782, 1, 576) torch.bfloat16 + out : (1, 1782) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 424 tokens) : 11.2310 + top-1 accuracy : 29/424 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1788]) +Dataloader batch - labels shape: torch.Size([1, 1788]) +Dataloader batch - attn_mask shape: torch.Size([1, 1788]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1026 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1788) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1788) torch.int64 + loss_mask : 411 / 1788 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1788, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1788, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1788, 1, 576) torch.bfloat16 + out : (1, 1788) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.2802 + top-1 accuracy : 20/411 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1416]) +Dataloader batch - labels shape: torch.Size([1, 1416]) +Dataloader batch - attn_mask shape: torch.Size([1, 1416]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1027 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1416) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1416) torch.int64 + loss_mask : 353 / 1416 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1416, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1416, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1416, 1, 576) torch.bfloat16 + out : (1, 1416) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.2966 + top-1 accuracy : 20/353 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1360]) +Dataloader batch - labels shape: torch.Size([1, 1360]) +Dataloader batch - attn_mask shape: torch.Size([1, 1360]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1028 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1360) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1360) torch.int64 + loss_mask : 302 / 1360 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1360, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1360, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1360, 1, 576) torch.bfloat16 + out : (1, 1360) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 302 tokens) : 11.2117 + top-1 accuracy : 2/302 = 0.7% + + [2026-04-29 13:57:57] iteration 357/ 500 | consumed samples: 1428 | elapsed time per iteration (ms): 5012.9 | throughput (token/sec/GPU): 1265.9 | learning rate: 4.847490E-05 | global batch size: 4 | lm loss: 1.125619E+01 | loss scale: 1.0 | grad norm: 0.211 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1838]) +Dataloader batch - labels shape: torch.Size([1, 1838]) +Dataloader batch - attn_mask shape: torch.Size([1, 1838]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1029 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1838) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1838) torch.int64 + loss_mask : 476 / 1838 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1838, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1838, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1838, 1, 576) torch.bfloat16 + out : (1, 1838) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 476 tokens) : 11.3329 + top-1 accuracy : 9/476 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1030 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 364 / 1504 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2674 + top-1 accuracy : 19/364 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 906]) +Dataloader batch - labels shape: torch.Size([1, 906]) +Dataloader batch - attn_mask shape: torch.Size([1, 906]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1031 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 906) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 906) torch.int64 + loss_mask : 194 / 906 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (906, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (906, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (906, 1, 576) torch.bfloat16 + out : (1, 906) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 194 tokens) : 11.1805 + top-1 accuracy : 4/194 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1425]) +Dataloader batch - labels shape: torch.Size([1, 1425]) +Dataloader batch - attn_mask shape: torch.Size([1, 1425]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1032 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1425) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1425) torch.int64 + loss_mask : 366 / 1425 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1425, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1425, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1425, 1, 576) torch.bfloat16 + out : (1, 1425) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.3322 + top-1 accuracy : 8/366 = 2.2% + + [2026-04-29 13:58:01] iteration 358/ 500 | consumed samples: 1432 | elapsed time per iteration (ms): 4647.5 | throughput (token/sec/GPU): 1220.6 | learning rate: 4.816356E-05 | global batch size: 4 | lm loss: 1.129460E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1072]) +Dataloader batch - labels shape: torch.Size([1, 1072]) +Dataloader batch - attn_mask shape: torch.Size([1, 1072]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1033 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1072) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1072) torch.int64 + loss_mask : 223 / 1072 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1072, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1072, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1072, 1, 576) torch.bfloat16 + out : (1, 1072) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 223 tokens) : 11.2327 + top-1 accuracy : 18/223 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 817]) +Dataloader batch - labels shape: torch.Size([1, 817]) +Dataloader batch - attn_mask shape: torch.Size([1, 817]) +Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) + +==================================================================== + PIPELINE — STEP 1034 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 817) torch.int64 + images : (16, 3, 1024, 1024) torch.bfloat16 + labels : (1, 817) torch.int64 + loss_mask : 169 / 817 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (16, 3, 1024, 1024) torch.bfloat16 + out : (16, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (16, 3072) + out : (16, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (817, 1, 576) torch.bfloat16 grad=False + image slots : 16 tokens replaced with vision embeddings + combined : (817, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (817, 1, 576) torch.bfloat16 + out : (1, 817) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 169 tokens) : 11.1595 + top-1 accuracy : 5/169 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1035 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 361 / 1508 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.2628 + top-1 accuracy : 20/361 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1036 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 402 / 1444 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.3792 + top-1 accuracy : 6/402 = 1.5% + + [2026-04-29 13:58:06] iteration 359/ 500 | consumed samples: 1436 | elapsed time per iteration (ms): 4037.3 | throughput (token/sec/GPU): 1199.1 | learning rate: 4.785231E-05 | global batch size: 4 | lm loss: 1.128237E+01 | loss scale: 1.0 | grad norm: 0.293 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1556]) +Dataloader batch - labels shape: torch.Size([1, 1556]) +Dataloader batch - attn_mask shape: torch.Size([1, 1556]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1037 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1556) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1556) torch.int64 + loss_mask : 422 / 1556 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1556, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1556, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1556, 1, 576) torch.bfloat16 + out : (1, 1556) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.3195 + top-1 accuracy : 10/422 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1038 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 347 / 1695 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.1955 + top-1 accuracy : 28/347 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1857]) +Dataloader batch - labels shape: torch.Size([1, 1857]) +Dataloader batch - attn_mask shape: torch.Size([1, 1857]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1039 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1857) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1857) torch.int64 + loss_mask : 494 / 1857 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1857, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1857, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1857, 1, 576) torch.bfloat16 + out : (1, 1857) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 494 tokens) : 11.3447 + top-1 accuracy : 6/494 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 881]) +Dataloader batch - labels shape: torch.Size([1, 881]) +Dataloader batch - attn_mask shape: torch.Size([1, 881]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 1040 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 881) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 881) torch.int64 + loss_mask : 181 / 881 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (881, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (881, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (881, 1, 576) torch.bfloat16 + out : (1, 881) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 181 tokens) : 11.1350 + top-1 accuracy : 8/181 = 4.4% + + [2026-04-29 13:58:10] iteration 360/ 500 | consumed samples: 1440 | elapsed time per iteration (ms): 4741.0 | throughput (token/sec/GPU): 1263.2 | learning rate: 4.754118E-05 | global batch size: 4 | lm loss: 1.127518E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4727.33, 4727.33) + forward-compute ................................: (3551.50, 3551.50) + backward-compute ...............................: (1173.85, 1173.85) + batch-generator ................................: (482.54, 482.54) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.40, 0.40) + params-all-gather ..............................: (0.14, 0.14) + optimizer-copy-to-main-grad ....................: (0.11, 0.11) + optimizer-clip-main-grad .......................: (0.45, 0.45) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.18, 1.18) + optimizer-copy-main-to-model-params ............: (0.20, 0.20) + optimizer ......................................: (2.66, 2.66) +Dataloader batch - tokens shape: torch.Size([1, 1474]) +Dataloader batch - labels shape: torch.Size([1, 1474]) +Dataloader batch - attn_mask shape: torch.Size([1, 1474]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1041 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1474) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1474) torch.int64 + loss_mask : 294 / 1474 tokens (19.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1474, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1474, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1474, 1, 576) torch.bfloat16 + out : (1, 1474) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.1084 + top-1 accuracy : 32/294 = 10.9% + +Dataloader batch - tokens shape: torch.Size([1, 871]) +Dataloader batch - labels shape: torch.Size([1, 871]) +Dataloader batch - attn_mask shape: torch.Size([1, 871]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 1042 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 871) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 871) torch.int64 + loss_mask : 182 / 871 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (871, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (871, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (871, 1, 576) torch.bfloat16 + out : (1, 871) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 182 tokens) : 11.1721 + top-1 accuracy : 3/182 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 1527]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1043 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1527) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1527) torch.int64 + loss_mask : 351 / 1527 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1527, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1527, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1527, 1, 576) torch.bfloat16 + out : (1, 1527) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.2551 + top-1 accuracy : 17/351 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1286]) +Dataloader batch - labels shape: torch.Size([1, 1286]) +Dataloader batch - attn_mask shape: torch.Size([1, 1286]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1044 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1286) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1286) torch.int64 + loss_mask : 344 / 1286 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1286, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1286, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1286, 1, 576) torch.bfloat16 + out : (1, 1286) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.3522 + top-1 accuracy : 4/344 = 1.2% + + [2026-04-29 13:58:15] iteration 361/ 500 | consumed samples: 1444 | elapsed time per iteration (ms): 4255.5 | throughput (token/sec/GPU): 1212.1 | learning rate: 4.723015E-05 | global batch size: 4 | lm loss: 1.123388E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1588]) +Dataloader batch - labels shape: torch.Size([1, 1588]) +Dataloader batch - attn_mask shape: torch.Size([1, 1588]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1045 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1588) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1588) torch.int64 + loss_mask : 339 / 1588 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1588, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1588, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1588, 1, 576) torch.bfloat16 + out : (1, 1588) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2255 + top-1 accuracy : 15/339 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1427]) +Dataloader batch - labels shape: torch.Size([1, 1427]) +Dataloader batch - attn_mask shape: torch.Size([1, 1427]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1046 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1427) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1427) torch.int64 + loss_mask : 373 / 1427 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1427, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1427, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1427, 1, 576) torch.bfloat16 + out : (1, 1427) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.3047 + top-1 accuracy : 17/373 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1367]) +Dataloader batch - labels shape: torch.Size([1, 1367]) +Dataloader batch - attn_mask shape: torch.Size([1, 1367]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1047 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1367) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1367) torch.int64 + loss_mask : 321 / 1367 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1367, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1367, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1367, 1, 576) torch.bfloat16 + out : (1, 1367) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2430 + top-1 accuracy : 7/321 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1458]) +Dataloader batch - labels shape: torch.Size([1, 1458]) +Dataloader batch - attn_mask shape: torch.Size([1, 1458]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1048 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1458) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1458) torch.int64 + loss_mask : 293 / 1458 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1458, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1458, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1458, 1, 576) torch.bfloat16 + out : (1, 1458) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 293 tokens) : 11.1044 + top-1 accuracy : 22/293 = 7.5% + + [2026-04-29 13:58:19] iteration 362/ 500 | consumed samples: 1448 | elapsed time per iteration (ms): 4663.9 | throughput (token/sec/GPU): 1252.2 | learning rate: 4.691926E-05 | global batch size: 4 | lm loss: 1.122524E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1049 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 336 / 1431 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 336 tokens) : 11.1879 + top-1 accuracy : 17/336 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1669]) +Dataloader batch - labels shape: torch.Size([1, 1669]) +Dataloader batch - attn_mask shape: torch.Size([1, 1669]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1050 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1669) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1669) torch.int64 + loss_mask : 368 / 1669 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1669, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1669, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1669, 1, 576) torch.bfloat16 + out : (1, 1669) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2006 + top-1 accuracy : 16/368 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1011]) +Dataloader batch - labels shape: torch.Size([1, 1011]) +Dataloader batch - attn_mask shape: torch.Size([1, 1011]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1051 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1011) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1011) torch.int64 + loss_mask : 246 / 1011 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1011, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1011, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1011, 1, 576) torch.bfloat16 + out : (1, 1011) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 246 tokens) : 11.3239 + top-1 accuracy : 1/246 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1762]) +Dataloader batch - labels shape: torch.Size([1, 1762]) +Dataloader batch - attn_mask shape: torch.Size([1, 1762]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1052 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1762) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1762) torch.int64 + loss_mask : 402 / 1762 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1762, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1762, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1762, 1, 576) torch.bfloat16 + out : (1, 1762) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.2913 + top-1 accuracy : 26/402 = 6.5% + + [2026-04-29 13:58:24] iteration 363/ 500 | consumed samples: 1452 | elapsed time per iteration (ms): 4941.5 | throughput (token/sec/GPU): 1188.5 | learning rate: 4.660851E-05 | global batch size: 4 | lm loss: 1.124685E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1049]) +Dataloader batch - labels shape: torch.Size([1, 1049]) +Dataloader batch - attn_mask shape: torch.Size([1, 1049]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1053 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1049) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1049) torch.int64 + loss_mask : 289 / 1049 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1049, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1049, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1049, 1, 576) torch.bfloat16 + out : (1, 1049) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 289 tokens) : 11.3170 + top-1 accuracy : 2/289 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1055]) +Dataloader batch - labels shape: torch.Size([1, 1055]) +Dataloader batch - attn_mask shape: torch.Size([1, 1055]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1054 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1055) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1055) torch.int64 + loss_mask : 292 / 1055 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1055, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1055, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1055, 1, 576) torch.bfloat16 + out : (1, 1055) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 292 tokens) : 11.3656 + top-1 accuracy : 4/292 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1383]) +Dataloader batch - labels shape: torch.Size([1, 1383]) +Dataloader batch - attn_mask shape: torch.Size([1, 1383]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1055 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1383) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1383) torch.int64 + loss_mask : 303 / 1383 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1383, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1383, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1383, 1, 576) torch.bfloat16 + out : (1, 1383) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.1863 + top-1 accuracy : 15/303 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1069]) +Dataloader batch - labels shape: torch.Size([1, 1069]) +Dataloader batch - attn_mask shape: torch.Size([1, 1069]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1056 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1069) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1069) torch.int64 + loss_mask : 234 / 1069 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1069, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1069, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1069, 1, 576) torch.bfloat16 + out : (1, 1069) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 234 tokens) : 11.1886 + top-1 accuracy : 6/234 = 2.6% + + [2026-04-29 13:58:28] iteration 364/ 500 | consumed samples: 1456 | elapsed time per iteration (ms): 3750.4 | throughput (token/sec/GPU): 1214.8 | learning rate: 4.629791E-05 | global batch size: 4 | lm loss: 1.126741E+01 | loss scale: 1.0 | grad norm: 0.229 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1551]) +Dataloader batch - labels shape: torch.Size([1, 1551]) +Dataloader batch - attn_mask shape: torch.Size([1, 1551]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1057 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1551) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1551) torch.int64 + loss_mask : 354 / 1551 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1551, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1551, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1551, 1, 576) torch.bfloat16 + out : (1, 1551) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2948 + top-1 accuracy : 19/354 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1074]) +Dataloader batch - labels shape: torch.Size([1, 1074]) +Dataloader batch - attn_mask shape: torch.Size([1, 1074]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1058 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1074) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1074) torch.int64 + loss_mask : 248 / 1074 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1074, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1074, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1074, 1, 576) torch.bfloat16 + out : (1, 1074) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 248 tokens) : 11.2453 + top-1 accuracy : 16/248 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1513]) +Dataloader batch - labels shape: torch.Size([1, 1513]) +Dataloader batch - attn_mask shape: torch.Size([1, 1513]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1059 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1513) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1513) torch.int64 + loss_mask : 364 / 1513 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1513, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1513, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1513, 1, 576) torch.bfloat16 + out : (1, 1513) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2880 + top-1 accuracy : 5/364 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1060 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 381 / 1518 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.2840 + top-1 accuracy : 24/381 = 6.3% + + [2026-04-29 13:58:32] iteration 365/ 500 | consumed samples: 1460 | elapsed time per iteration (ms): 4607.3 | throughput (token/sec/GPU): 1227.6 | learning rate: 4.598748E-05 | global batch size: 4 | lm loss: 1.128077E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1123]) +Dataloader batch - labels shape: torch.Size([1, 1123]) +Dataloader batch - attn_mask shape: torch.Size([1, 1123]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1061 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1123) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1123) torch.int64 + loss_mask : 260 / 1123 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1123, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1123, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1123, 1, 576) torch.bfloat16 + out : (1, 1123) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.2309 + top-1 accuracy : 4/260 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1453]) +Dataloader batch - labels shape: torch.Size([1, 1453]) +Dataloader batch - attn_mask shape: torch.Size([1, 1453]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1062 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1453) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1453) torch.int64 + loss_mask : 301 / 1453 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1453, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1453, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1453, 1, 576) torch.bfloat16 + out : (1, 1453) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 301 tokens) : 11.1620 + top-1 accuracy : 15/301 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1472]) +Dataloader batch - labels shape: torch.Size([1, 1472]) +Dataloader batch - attn_mask shape: torch.Size([1, 1472]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1063 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1472) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1472) torch.int64 + loss_mask : 325 / 1472 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1472, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1472, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1472, 1, 576) torch.bfloat16 + out : (1, 1472) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.1569 + top-1 accuracy : 8/325 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1972]) +Dataloader batch - labels shape: torch.Size([1, 1972]) +Dataloader batch - attn_mask shape: torch.Size([1, 1972]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1064 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1972) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1972) torch.int64 + loss_mask : 585 / 1972 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1972, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1972, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1972, 1, 576) torch.bfloat16 + out : (1, 1972) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 585 tokens) : 11.3934 + top-1 accuracy : 4/585 = 0.7% + + [2026-04-29 13:58:37] iteration 366/ 500 | consumed samples: 1464 | elapsed time per iteration (ms): 4884.9 | throughput (token/sec/GPU): 1232.4 | learning rate: 4.567723E-05 | global batch size: 4 | lm loss: 1.126509E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1065 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 358 / 1525 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2351 + top-1 accuracy : 2/358 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1799]) +Dataloader batch - labels shape: torch.Size([1, 1799]) +Dataloader batch - attn_mask shape: torch.Size([1, 1799]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1066 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1799) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1799) torch.int64 + loss_mask : 421 / 1799 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1799, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1799, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1799, 1, 576) torch.bfloat16 + out : (1, 1799) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 421 tokens) : 11.2524 + top-1 accuracy : 5/421 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1613]) +Dataloader batch - labels shape: torch.Size([1, 1613]) +Dataloader batch - attn_mask shape: torch.Size([1, 1613]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1067 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1613) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1613) torch.int64 + loss_mask : 377 / 1613 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1613, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1613, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1613, 1, 576) torch.bfloat16 + out : (1, 1613) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 377 tokens) : 11.2111 + top-1 accuracy : 22/377 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1095]) +Dataloader batch - labels shape: torch.Size([1, 1095]) +Dataloader batch - attn_mask shape: torch.Size([1, 1095]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1068 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1095) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1095) torch.int64 + loss_mask : 290 / 1095 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1095, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1095, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1095, 1, 576) torch.bfloat16 + out : (1, 1095) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.2850 + top-1 accuracy : 14/290 = 4.8% + + [2026-04-29 13:58:42] iteration 367/ 500 | consumed samples: 1468 | elapsed time per iteration (ms): 4782.1 | throughput (token/sec/GPU): 1261.4 | learning rate: 4.536717E-05 | global batch size: 4 | lm loss: 1.124389E+01 | loss scale: 1.0 | grad norm: 0.257 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1095]) +Dataloader batch - labels shape: torch.Size([1, 1095]) +Dataloader batch - attn_mask shape: torch.Size([1, 1095]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1069 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1095) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1095) torch.int64 + loss_mask : 246 / 1095 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1095, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1095, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1095, 1, 576) torch.bfloat16 + out : (1, 1095) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 246 tokens) : 11.1865 + top-1 accuracy : 20/246 = 8.1% + +Dataloader batch - tokens shape: torch.Size([1, 1488]) +Dataloader batch - labels shape: torch.Size([1, 1488]) +Dataloader batch - attn_mask shape: torch.Size([1, 1488]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1070 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1488) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1488) torch.int64 + loss_mask : 444 / 1488 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1488, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1488, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1488, 1, 576) torch.bfloat16 + out : (1, 1488) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 444 tokens) : 11.3704 + top-1 accuracy : 9/444 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1071 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 300 / 1388 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.2449 + top-1 accuracy : 11/300 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1793]) +Dataloader batch - labels shape: torch.Size([1, 1793]) +Dataloader batch - attn_mask shape: torch.Size([1, 1793]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1072 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1793) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1793) torch.int64 + loss_mask : 442 / 1793 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1793, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1793, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1793, 1, 576) torch.bfloat16 + out : (1, 1793) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 442 tokens) : 11.2679 + top-1 accuracy : 13/442 = 2.9% + + [2026-04-29 13:58:47] iteration 368/ 500 | consumed samples: 1472 | elapsed time per iteration (ms): 4547.7 | throughput (token/sec/GPU): 1267.5 | learning rate: 4.505731E-05 | global batch size: 4 | lm loss: 1.128090E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1581]) +Dataloader batch - labels shape: torch.Size([1, 1581]) +Dataloader batch - attn_mask shape: torch.Size([1, 1581]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1073 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1581) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1581) torch.int64 + loss_mask : 354 / 1581 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1581, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1581, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1581, 1, 576) torch.bfloat16 + out : (1, 1581) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 354 tokens) : 11.2056 + top-1 accuracy : 13/354 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1274]) +Dataloader batch - labels shape: torch.Size([1, 1274]) +Dataloader batch - attn_mask shape: torch.Size([1, 1274]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1074 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1274) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1274) torch.int64 + loss_mask : 385 / 1274 tokens (30.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1274, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1274, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1274, 1, 576) torch.bfloat16 + out : (1, 1274) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.3444 + top-1 accuracy : 2/385 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1075 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 290 / 1364 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.1767 + top-1 accuracy : 17/290 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1842]) +Dataloader batch - labels shape: torch.Size([1, 1842]) +Dataloader batch - attn_mask shape: torch.Size([1, 1842]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1076 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1842) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1842) torch.int64 + loss_mask : 483 / 1842 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1842, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1842, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1842, 1, 576) torch.bfloat16 + out : (1, 1842) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 483 tokens) : 11.3102 + top-1 accuracy : 9/483 = 1.9% + + [2026-04-29 13:58:52] iteration 369/ 500 | consumed samples: 1476 | elapsed time per iteration (ms): 4821.7 | throughput (token/sec/GPU): 1257.0 | learning rate: 4.474767E-05 | global batch size: 4 | lm loss: 1.126881E+01 | loss scale: 1.0 | grad norm: 0.219 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1434]) +Dataloader batch - labels shape: torch.Size([1, 1434]) +Dataloader batch - attn_mask shape: torch.Size([1, 1434]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1077 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1434) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1434) torch.int64 + loss_mask : 368 / 1434 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1434, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1434, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1434, 1, 576) torch.bfloat16 + out : (1, 1434) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2993 + top-1 accuracy : 15/368 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 962]) +Dataloader batch - labels shape: torch.Size([1, 962]) +Dataloader batch - attn_mask shape: torch.Size([1, 962]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1078 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 962) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 962) torch.int64 + loss_mask : 207 / 962 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (962, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (962, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (962, 1, 576) torch.bfloat16 + out : (1, 962) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 207 tokens) : 11.2307 + top-1 accuracy : 5/207 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 991]) +Dataloader batch - labels shape: torch.Size([1, 991]) +Dataloader batch - attn_mask shape: torch.Size([1, 991]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1079 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 991) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 991) torch.int64 + loss_mask : 214 / 991 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (991, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (991, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (991, 1, 576) torch.bfloat16 + out : (1, 991) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 214 tokens) : 11.2075 + top-1 accuracy : 1/214 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1413]) +Dataloader batch - labels shape: torch.Size([1, 1413]) +Dataloader batch - attn_mask shape: torch.Size([1, 1413]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1080 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1413) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1413) torch.int64 + loss_mask : 352 / 1413 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1413, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1413, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1413, 1, 576) torch.bfloat16 + out : (1, 1413) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 352 tokens) : 11.2869 + top-1 accuracy : 7/352 = 2.0% + + [2026-04-29 13:58:55] iteration 370/ 500 | consumed samples: 1480 | elapsed time per iteration (ms): 3954.5 | throughput (token/sec/GPU): 1213.8 | learning rate: 4.443826E-05 | global batch size: 4 | lm loss: 1.126581E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1081 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 325 / 1518 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.1727 + top-1 accuracy : 4/325 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1393]) +Dataloader batch - labels shape: torch.Size([1, 1393]) +Dataloader batch - attn_mask shape: torch.Size([1, 1393]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1082 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1393) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1393) torch.int64 + loss_mask : 342 / 1393 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1393, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1393, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1393, 1, 576) torch.bfloat16 + out : (1, 1393) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.3199 + top-1 accuracy : 8/342 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1398]) +Dataloader batch - labels shape: torch.Size([1, 1398]) +Dataloader batch - attn_mask shape: torch.Size([1, 1398]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1083 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1398) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1398) torch.int64 + loss_mask : 330 / 1398 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1398, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1398, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1398, 1, 576) torch.bfloat16 + out : (1, 1398) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.2642 + top-1 accuracy : 14/330 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1215]) +Dataloader batch - labels shape: torch.Size([1, 1215]) +Dataloader batch - attn_mask shape: torch.Size([1, 1215]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1084 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1215) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1215) torch.int64 + loss_mask : 294 / 1215 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1215, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1215, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1215, 1, 576) torch.bfloat16 + out : (1, 1215) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.2217 + top-1 accuracy : 16/294 = 5.4% + + [2026-04-29 13:59:00] iteration 371/ 500 | consumed samples: 1484 | elapsed time per iteration (ms): 4506.2 | throughput (token/sec/GPU): 1225.9 | learning rate: 4.412908E-05 | global batch size: 4 | lm loss: 1.124625E+01 | loss scale: 1.0 | grad norm: 0.200 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1215]) +Dataloader batch - labels shape: torch.Size([1, 1215]) +Dataloader batch - attn_mask shape: torch.Size([1, 1215]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1085 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1215) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1215) torch.int64 + loss_mask : 332 / 1215 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1215, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1215, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1215, 1, 576) torch.bfloat16 + out : (1, 1215) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.3406 + top-1 accuracy : 4/332 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1753]) +Dataloader batch - labels shape: torch.Size([1, 1753]) +Dataloader batch - attn_mask shape: torch.Size([1, 1753]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1086 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1753) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1753) torch.int64 + loss_mask : 369 / 1753 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1753, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1753, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1753, 1, 576) torch.bfloat16 + out : (1, 1753) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2784 + top-1 accuracy : 22/369 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1138]) +Dataloader batch - labels shape: torch.Size([1, 1138]) +Dataloader batch - attn_mask shape: torch.Size([1, 1138]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1087 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1138) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1138) torch.int64 + loss_mask : 345 / 1138 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1138, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1138, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1138, 1, 576) torch.bfloat16 + out : (1, 1138) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.4305 + top-1 accuracy : 4/345 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1581]) +Dataloader batch - labels shape: torch.Size([1, 1581]) +Dataloader batch - attn_mask shape: torch.Size([1, 1581]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1088 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1581) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1581) torch.int64 + loss_mask : 357 / 1581 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1581, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1581, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1581, 1, 576) torch.bfloat16 + out : (1, 1581) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2334 + top-1 accuracy : 27/357 = 7.6% + + [2026-04-29 13:59:05] iteration 372/ 500 | consumed samples: 1488 | elapsed time per iteration (ms): 4651.7 | throughput (token/sec/GPU): 1222.6 | learning rate: 4.382016E-05 | global batch size: 4 | lm loss: 1.131907E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1972]) +Dataloader batch - labels shape: torch.Size([1, 1972]) +Dataloader batch - attn_mask shape: torch.Size([1, 1972]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1089 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1972) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1972) torch.int64 + loss_mask : 622 / 1972 tokens (31.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1972, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1972, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1972, 1, 576) torch.bfloat16 + out : (1, 1972) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 622 tokens) : 11.4017 + top-1 accuracy : 7/622 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1721]) +Dataloader batch - labels shape: torch.Size([1, 1721]) +Dataloader batch - attn_mask shape: torch.Size([1, 1721]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1090 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1721) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1721) torch.int64 + loss_mask : 367 / 1721 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1721, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1721, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1721, 1, 576) torch.bfloat16 + out : (1, 1721) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.1900 + top-1 accuracy : 30/367 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1091 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1385) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1385) torch.int64 + loss_mask : 286 / 1385 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1385, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1385, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1385, 1, 576) torch.bfloat16 + out : (1, 1385) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 286 tokens) : 11.1631 + top-1 accuracy : 18/286 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1092 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1423) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1423) torch.int64 + loss_mask : 344 / 1423 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1423, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1423, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1423, 1, 576) torch.bfloat16 + out : (1, 1423) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.2725 + top-1 accuracy : 6/344 = 1.7% + + [2026-04-29 13:59:10] iteration 373/ 500 | consumed samples: 1492 | elapsed time per iteration (ms): 5142.8 | throughput (token/sec/GPU): 1264.1 | learning rate: 4.351151E-05 | global batch size: 4 | lm loss: 1.128413E+01 | loss scale: 1.0 | grad norm: 0.213 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1568]) +Dataloader batch - labels shape: torch.Size([1, 1568]) +Dataloader batch - attn_mask shape: torch.Size([1, 1568]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1093 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1568) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1568) torch.int64 + loss_mask : 372 / 1568 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1568, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1568, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1568, 1, 576) torch.bfloat16 + out : (1, 1568) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2502 + top-1 accuracy : 13/372 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1098]) +Dataloader batch - labels shape: torch.Size([1, 1098]) +Dataloader batch - attn_mask shape: torch.Size([1, 1098]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1094 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1098) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1098) torch.int64 + loss_mask : 238 / 1098 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1098, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1098, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1098, 1, 576) torch.bfloat16 + out : (1, 1098) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 238 tokens) : 11.1905 + top-1 accuracy : 3/238 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1926]) +Dataloader batch - labels shape: torch.Size([1, 1926]) +Dataloader batch - attn_mask shape: torch.Size([1, 1926]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1095 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1926) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1926) torch.int64 + loss_mask : 572 / 1926 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1926, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1926, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1926, 1, 576) torch.bfloat16 + out : (1, 1926) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 572 tokens) : 11.3856 + top-1 accuracy : 5/572 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1401]) +Dataloader batch - labels shape: torch.Size([1, 1401]) +Dataloader batch - attn_mask shape: torch.Size([1, 1401]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1096 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1401) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1401) torch.int64 + loss_mask : 333 / 1401 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1401, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1401, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1401, 1, 576) torch.bfloat16 + out : (1, 1401) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 333 tokens) : 11.2494 + top-1 accuracy : 11/333 = 3.3% + + [2026-04-29 13:59:14] iteration 374/ 500 | consumed samples: 1496 | elapsed time per iteration (ms): 4648.0 | throughput (token/sec/GPU): 1289.4 | learning rate: 4.320313E-05 | global batch size: 4 | lm loss: 1.129174E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1540]) +Dataloader batch - labels shape: torch.Size([1, 1540]) +Dataloader batch - attn_mask shape: torch.Size([1, 1540]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1097 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1540) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1540) torch.int64 + loss_mask : 334 / 1540 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1540, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1540, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1540, 1, 576) torch.bfloat16 + out : (1, 1540) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.2512 + top-1 accuracy : 20/334 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1098 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 386 / 1431 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3472 + top-1 accuracy : 9/386 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1490]) +Dataloader batch - labels shape: torch.Size([1, 1490]) +Dataloader batch - attn_mask shape: torch.Size([1, 1490]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1099 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1490) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1490) torch.int64 + loss_mask : 313 / 1490 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1490, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1490, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1490, 1, 576) torch.bfloat16 + out : (1, 1490) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2076 + top-1 accuracy : 17/313 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1100 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 387 / 1750 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2667 + top-1 accuracy : 6/387 = 1.6% + + [2026-04-29 13:59:19] iteration 375/ 500 | consumed samples: 1500 | elapsed time per iteration (ms): 4961.0 | throughput (token/sec/GPU): 1252.0 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.127192E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1556]) +Dataloader batch - labels shape: torch.Size([1, 1556]) +Dataloader batch - attn_mask shape: torch.Size([1, 1556]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1101 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1556) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1556) torch.int64 + loss_mask : 343 / 1556 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1556, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1556, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1556, 1, 576) torch.bfloat16 + out : (1, 1556) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.2294 + top-1 accuracy : 13/343 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 983]) +Dataloader batch - labels shape: torch.Size([1, 983]) +Dataloader batch - attn_mask shape: torch.Size([1, 983]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1102 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 983) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 983) torch.int64 + loss_mask : 198 / 983 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (983, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (983, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (983, 1, 576) torch.bfloat16 + out : (1, 983) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 198 tokens) : 11.0921 + top-1 accuracy : 10/198 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1103 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1470) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1470) torch.int64 + loss_mask : 317 / 1470 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1470, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1470, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1470, 1, 576) torch.bfloat16 + out : (1, 1470) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.1450 + top-1 accuracy : 4/317 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1017]) +Dataloader batch - labels shape: torch.Size([1, 1017]) +Dataloader batch - attn_mask shape: torch.Size([1, 1017]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1104 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1017) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1017) torch.int64 + loss_mask : 250 / 1017 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1017, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1017, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1017, 1, 576) torch.bfloat16 + out : (1, 1017) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 250 tokens) : 11.2293 + top-1 accuracy : 7/250 = 2.8% + + [2026-04-29 13:59:24] iteration 376/ 500 | consumed samples: 1504 | elapsed time per iteration (ms): 4305.0 | throughput (token/sec/GPU): 1167.5 | learning rate: 4.258725E-05 | global batch size: 4 | lm loss: 1.118069E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1036]) +Dataloader batch - labels shape: torch.Size([1, 1036]) +Dataloader batch - attn_mask shape: torch.Size([1, 1036]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1105 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1036) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1036) torch.int64 + loss_mask : 219 / 1036 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1036, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1036, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1036, 1, 576) torch.bfloat16 + out : (1, 1036) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 219 tokens) : 11.2017 + top-1 accuracy : 7/219 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1523]) +Dataloader batch - labels shape: torch.Size([1, 1523]) +Dataloader batch - attn_mask shape: torch.Size([1, 1523]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1106 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1523) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1523) torch.int64 + loss_mask : 335 / 1523 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1523, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1523, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1523, 1, 576) torch.bfloat16 + out : (1, 1523) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2483 + top-1 accuracy : 28/335 = 8.4% + +Dataloader batch - tokens shape: torch.Size([1, 1514]) +Dataloader batch - labels shape: torch.Size([1, 1514]) +Dataloader batch - attn_mask shape: torch.Size([1, 1514]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1107 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1514) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1514) torch.int64 + loss_mask : 320 / 1514 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1514, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1514, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1514, 1, 576) torch.bfloat16 + out : (1, 1514) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.1660 + top-1 accuracy : 24/320 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1393]) +Dataloader batch - labels shape: torch.Size([1, 1393]) +Dataloader batch - attn_mask shape: torch.Size([1, 1393]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1108 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1393) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1393) torch.int64 + loss_mask : 326 / 1393 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1393, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1393, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1393, 1, 576) torch.bfloat16 + out : (1, 1393) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2349 + top-1 accuracy : 14/326 = 4.3% + + [2026-04-29 13:59:28] iteration 377/ 500 | consumed samples: 1508 | elapsed time per iteration (ms): 4560.3 | throughput (token/sec/GPU): 1198.6 | learning rate: 4.227977E-05 | global batch size: 4 | lm loss: 1.121422E+01 | loss scale: 1.0 | grad norm: 0.228 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1074]) +Dataloader batch - labels shape: torch.Size([1, 1074]) +Dataloader batch - attn_mask shape: torch.Size([1, 1074]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1109 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1074) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1074) torch.int64 + loss_mask : 316 / 1074 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1074, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1074, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1074, 1, 576) torch.bfloat16 + out : (1, 1074) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.4098 + top-1 accuracy : 10/316 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1875]) +Dataloader batch - labels shape: torch.Size([1, 1875]) +Dataloader batch - attn_mask shape: torch.Size([1, 1875]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1110 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1875) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1875) torch.int64 + loss_mask : 517 / 1875 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1875, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1875, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1875, 1, 576) torch.bfloat16 + out : (1, 1875) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 517 tokens) : 11.3414 + top-1 accuracy : 6/517 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1296]) +Dataloader batch - labels shape: torch.Size([1, 1296]) +Dataloader batch - attn_mask shape: torch.Size([1, 1296]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1111 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1296) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1296) torch.int64 + loss_mask : 356 / 1296 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1296, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1296, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1296, 1, 576) torch.bfloat16 + out : (1, 1296) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.3303 + top-1 accuracy : 5/356 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1514]) +Dataloader batch - labels shape: torch.Size([1, 1514]) +Dataloader batch - attn_mask shape: torch.Size([1, 1514]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1112 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1514) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1514) torch.int64 + loss_mask : 392 / 1514 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1514, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1514, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1514, 1, 576) torch.bfloat16 + out : (1, 1514) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3133 + top-1 accuracy : 17/392 = 4.3% + + [2026-04-29 13:59:33] iteration 378/ 500 | consumed samples: 1512 | elapsed time per iteration (ms): 4420.0 | throughput (token/sec/GPU): 1302.9 | learning rate: 4.197262E-05 | global batch size: 4 | lm loss: 1.134558E+01 | loss scale: 1.0 | grad norm: 0.246 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1523]) +Dataloader batch - labels shape: torch.Size([1, 1523]) +Dataloader batch - attn_mask shape: torch.Size([1, 1523]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1113 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1523) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1523) torch.int64 + loss_mask : 382 / 1523 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1523, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1523, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1523, 1, 576) torch.bfloat16 + out : (1, 1523) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.2803 + top-1 accuracy : 5/382 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1761]) +Dataloader batch - labels shape: torch.Size([1, 1761]) +Dataloader batch - attn_mask shape: torch.Size([1, 1761]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1114 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1761) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1761) torch.int64 + loss_mask : 406 / 1761 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1761, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1761, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1761, 1, 576) torch.bfloat16 + out : (1, 1761) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.2842 + top-1 accuracy : 14/406 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 937]) +Dataloader batch - labels shape: torch.Size([1, 937]) +Dataloader batch - attn_mask shape: torch.Size([1, 937]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1115 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 937) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 937) torch.int64 + loss_mask : 204 / 937 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (937, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (937, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (937, 1, 576) torch.bfloat16 + out : (1, 937) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 204 tokens) : 11.1731 + top-1 accuracy : 2/204 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1562]) +Dataloader batch - labels shape: torch.Size([1, 1562]) +Dataloader batch - attn_mask shape: torch.Size([1, 1562]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1116 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1562) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1562) torch.int64 + loss_mask : 415 / 1562 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1562, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1562, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1562, 1, 576) torch.bfloat16 + out : (1, 1562) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.3281 + top-1 accuracy : 19/415 = 4.6% + + [2026-04-29 13:59:37] iteration 379/ 500 | consumed samples: 1516 | elapsed time per iteration (ms): 4658.6 | throughput (token/sec/GPU): 1241.4 | learning rate: 4.166581E-05 | global batch size: 4 | lm loss: 1.127998E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1522]) +Dataloader batch - labels shape: torch.Size([1, 1522]) +Dataloader batch - attn_mask shape: torch.Size([1, 1522]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1117 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1522) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1522) torch.int64 + loss_mask : 366 / 1522 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1522, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1522, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1522, 1, 576) torch.bfloat16 + out : (1, 1522) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2778 + top-1 accuracy : 18/366 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1599]) +Dataloader batch - labels shape: torch.Size([1, 1599]) +Dataloader batch - attn_mask shape: torch.Size([1, 1599]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1118 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1599) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1599) torch.int64 + loss_mask : 458 / 1599 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1599, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1599, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1599, 1, 576) torch.bfloat16 + out : (1, 1599) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 458 tokens) : 11.3888 + top-1 accuracy : 3/458 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1874]) +Dataloader batch - labels shape: torch.Size([1, 1874]) +Dataloader batch - attn_mask shape: torch.Size([1, 1874]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1119 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1874) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1874) torch.int64 + loss_mask : 505 / 1874 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1874, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1874, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1874, 1, 576) torch.bfloat16 + out : (1, 1874) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 505 tokens) : 11.3527 + top-1 accuracy : 6/505 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1696]) +Dataloader batch - labels shape: torch.Size([1, 1696]) +Dataloader batch - attn_mask shape: torch.Size([1, 1696]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1120 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1696) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1696) torch.int64 + loss_mask : 363 / 1696 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1696, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1696, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1696, 1, 576) torch.bfloat16 + out : (1, 1696) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2030 + top-1 accuracy : 16/363 = 4.4% + + [2026-04-29 13:59:42] iteration 380/ 500 | consumed samples: 1520 | elapsed time per iteration (ms): 5087.9 | throughput (token/sec/GPU): 1315.1 | learning rate: 4.135936E-05 | global batch size: 4 | lm loss: 1.131415E+01 | loss scale: 1.0 | grad norm: 0.201 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (5077.75, 5077.75) + forward-compute ................................: (3824.77, 3824.77) + backward-compute ...............................: (1251.10, 1251.10) + batch-generator ................................: (502.07, 502.07) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.38, 0.38) + params-all-gather ..............................: (0.11, 0.11) + optimizer-copy-to-main-grad ....................: (0.10, 0.10) + optimizer-clip-main-grad .......................: (0.41, 0.41) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.13, 1.13) + optimizer-copy-main-to-model-params ............: (0.18, 0.18) + optimizer ......................................: (2.48, 2.48) +Dataloader batch - tokens shape: torch.Size([1, 1397]) +Dataloader batch - labels shape: torch.Size([1, 1397]) +Dataloader batch - attn_mask shape: torch.Size([1, 1397]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1121 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1397) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1397) torch.int64 + loss_mask : 378 / 1397 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1397, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1397, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1397, 1, 576) torch.bfloat16 + out : (1, 1397) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 378 tokens) : 11.3612 + top-1 accuracy : 9/378 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1165]) +Dataloader batch - labels shape: torch.Size([1, 1165]) +Dataloader batch - attn_mask shape: torch.Size([1, 1165]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1122 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1165) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1165) torch.int64 + loss_mask : 311 / 1165 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1165, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1165, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1165, 1, 576) torch.bfloat16 + out : (1, 1165) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.3554 + top-1 accuracy : 5/311 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1871]) +Dataloader batch - labels shape: torch.Size([1, 1871]) +Dataloader batch - attn_mask shape: torch.Size([1, 1871]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1123 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1871) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1871) torch.int64 + loss_mask : 504 / 1871 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1871, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1871, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1871, 1, 576) torch.bfloat16 + out : (1, 1871) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 504 tokens) : 11.3120 + top-1 accuracy : 6/504 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1377]) +Dataloader batch - labels shape: torch.Size([1, 1377]) +Dataloader batch - attn_mask shape: torch.Size([1, 1377]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1124 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1377) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1377) torch.int64 + loss_mask : 316 / 1377 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1377, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1377, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1377, 1, 576) torch.bfloat16 + out : (1, 1377) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 316 tokens) : 11.2109 + top-1 accuracy : 16/316 = 5.1% + + [2026-04-29 13:59:47] iteration 381/ 500 | consumed samples: 1524 | elapsed time per iteration (ms): 4593.1 | throughput (token/sec/GPU): 1264.9 | learning rate: 4.105326E-05 | global batch size: 4 | lm loss: 1.131208E+01 | loss scale: 1.0 | grad norm: 0.181 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1125 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 314 / 1460 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.1359 + top-1 accuracy : 15/314 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1739]) +Dataloader batch - labels shape: torch.Size([1, 1739]) +Dataloader batch - attn_mask shape: torch.Size([1, 1739]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1126 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1739) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1739) torch.int64 + loss_mask : 385 / 1739 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1739, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1739, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1739, 1, 576) torch.bfloat16 + out : (1, 1739) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 385 tokens) : 11.2371 + top-1 accuracy : 29/385 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1017]) +Dataloader batch - labels shape: torch.Size([1, 1017]) +Dataloader batch - attn_mask shape: torch.Size([1, 1017]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1127 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1017) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1017) torch.int64 + loss_mask : 261 / 1017 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1017, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1017, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1017, 1, 576) torch.bfloat16 + out : (1, 1017) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 261 tokens) : 11.2826 + top-1 accuracy : 6/261 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1497]) +Dataloader batch - labels shape: torch.Size([1, 1497]) +Dataloader batch - attn_mask shape: torch.Size([1, 1497]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1128 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1497) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1497) torch.int64 + loss_mask : 350 / 1497 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1497, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1497, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1497, 1, 576) torch.bfloat16 + out : (1, 1497) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.1904 + top-1 accuracy : 17/350 = 4.9% + + [2026-04-29 13:59:52] iteration 382/ 500 | consumed samples: 1528 | elapsed time per iteration (ms): 4740.0 | throughput (token/sec/GPU): 1205.3 | learning rate: 4.074753E-05 | global batch size: 4 | lm loss: 1.120944E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1545]) +Dataloader batch - labels shape: torch.Size([1, 1545]) +Dataloader batch - attn_mask shape: torch.Size([1, 1545]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1129 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1545) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1545) torch.int64 + loss_mask : 400 / 1545 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1545, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1545, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1545, 1, 576) torch.bfloat16 + out : (1, 1545) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.2921 + top-1 accuracy : 11/400 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1310]) +Dataloader batch - labels shape: torch.Size([1, 1310]) +Dataloader batch - attn_mask shape: torch.Size([1, 1310]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1130 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1310) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1310) torch.int64 + loss_mask : 335 / 1310 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1310, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1310, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1310, 1, 576) torch.bfloat16 + out : (1, 1310) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.3307 + top-1 accuracy : 9/335 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1739]) +Dataloader batch - labels shape: torch.Size([1, 1739]) +Dataloader batch - attn_mask shape: torch.Size([1, 1739]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1131 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1739) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1739) torch.int64 + loss_mask : 376 / 1739 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1739, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1739, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1739, 1, 576) torch.bfloat16 + out : (1, 1739) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.2285 + top-1 accuracy : 22/376 = 5.9% + +Dataloader batch - tokens shape: torch.Size([1, 1388]) +Dataloader batch - labels shape: torch.Size([1, 1388]) +Dataloader batch - attn_mask shape: torch.Size([1, 1388]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1132 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1388) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1388) torch.int64 + loss_mask : 303 / 1388 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1388, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1388, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1388, 1, 576) torch.bfloat16 + out : (1, 1388) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.2026 + top-1 accuracy : 27/303 = 8.9% + + [2026-04-29 13:59:57] iteration 383/ 500 | consumed samples: 1532 | elapsed time per iteration (ms): 4797.6 | throughput (token/sec/GPU): 1246.9 | learning rate: 4.044220E-05 | global batch size: 4 | lm loss: 1.126517E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1537]) +Dataloader batch - labels shape: torch.Size([1, 1537]) +Dataloader batch - attn_mask shape: torch.Size([1, 1537]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1133 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1537) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1537) torch.int64 + loss_mask : 350 / 1537 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1537, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1537, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1537, 1, 576) torch.bfloat16 + out : (1, 1537) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 350 tokens) : 11.3042 + top-1 accuracy : 23/350 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1591]) +Dataloader batch - labels shape: torch.Size([1, 1591]) +Dataloader batch - attn_mask shape: torch.Size([1, 1591]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1134 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1591) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1591) torch.int64 + loss_mask : 362 / 1591 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1591, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1591, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1591, 1, 576) torch.bfloat16 + out : (1, 1591) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2101 + top-1 accuracy : 12/362 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 980]) +Dataloader batch - labels shape: torch.Size([1, 980]) +Dataloader batch - attn_mask shape: torch.Size([1, 980]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1135 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 980) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 980) torch.int64 + loss_mask : 190 / 980 tokens (19.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (980, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (980, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (980, 1, 576) torch.bfloat16 + out : (1, 980) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 190 tokens) : 11.0600 + top-1 accuracy : 6/190 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1378]) +Dataloader batch - labels shape: torch.Size([1, 1378]) +Dataloader batch - attn_mask shape: torch.Size([1, 1378]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1136 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1378) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1378) torch.int64 + loss_mask : 338 / 1378 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1378, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1378, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1378, 1, 576) torch.bfloat16 + out : (1, 1378) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.2121 + top-1 accuracy : 9/338 = 2.7% + + [2026-04-29 14:00:01] iteration 384/ 500 | consumed samples: 1536 | elapsed time per iteration (ms): 4470.1 | throughput (token/sec/GPU): 1227.3 | learning rate: 4.013726E-05 | global batch size: 4 | lm loss: 1.121420E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1815]) +Dataloader batch - labels shape: torch.Size([1, 1815]) +Dataloader batch - attn_mask shape: torch.Size([1, 1815]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1137 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1815) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1815) torch.int64 + loss_mask : 432 / 1815 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1815, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1815, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1815, 1, 576) torch.bfloat16 + out : (1, 1815) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 432 tokens) : 11.3159 + top-1 accuracy : 10/432 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1719]) +Dataloader batch - labels shape: torch.Size([1, 1719]) +Dataloader batch - attn_mask shape: torch.Size([1, 1719]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1138 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1719) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1719) torch.int64 + loss_mask : 435 / 1719 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1719, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1719, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1719, 1, 576) torch.bfloat16 + out : (1, 1719) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 435 tokens) : 11.2442 + top-1 accuracy : 16/435 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1501]) +Dataloader batch - labels shape: torch.Size([1, 1501]) +Dataloader batch - attn_mask shape: torch.Size([1, 1501]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1139 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1501) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1501) torch.int64 + loss_mask : 388 / 1501 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1501, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1501, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1501, 1, 576) torch.bfloat16 + out : (1, 1501) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2912 + top-1 accuracy : 21/388 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1948]) +Dataloader batch - labels shape: torch.Size([1, 1948]) +Dataloader batch - attn_mask shape: torch.Size([1, 1948]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1140 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1948) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1948) torch.int64 + loss_mask : 583 / 1948 tokens (29.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1948, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1948, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1948, 1, 576) torch.bfloat16 + out : (1, 1948) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 583 tokens) : 11.3713 + top-1 accuracy : 9/583 = 1.5% + + [2026-04-29 14:00:06] iteration 385/ 500 | consumed samples: 1540 | elapsed time per iteration (ms): 5199.3 | throughput (token/sec/GPU): 1343.1 | learning rate: 3.983273E-05 | global batch size: 4 | lm loss: 1.131126E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1298]) +Dataloader batch - labels shape: torch.Size([1, 1298]) +Dataloader batch - attn_mask shape: torch.Size([1, 1298]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1141 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1298) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1298) torch.int64 + loss_mask : 406 / 1298 tokens (31.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1298, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1298, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1298, 1, 576) torch.bfloat16 + out : (1, 1298) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.4351 + top-1 accuracy : 1/406 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1816]) +Dataloader batch - labels shape: torch.Size([1, 1816]) +Dataloader batch - attn_mask shape: torch.Size([1, 1816]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1142 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1816) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1816) torch.int64 + loss_mask : 440 / 1816 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1816, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1816, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1816, 1, 576) torch.bfloat16 + out : (1, 1816) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 440 tokens) : 11.2555 + top-1 accuracy : 8/440 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1574]) +Dataloader batch - labels shape: torch.Size([1, 1574]) +Dataloader batch - attn_mask shape: torch.Size([1, 1574]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1143 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1574) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1574) torch.int64 + loss_mask : 362 / 1574 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1574, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1574, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1574, 1, 576) torch.bfloat16 + out : (1, 1574) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2103 + top-1 accuracy : 23/362 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1446]) +Dataloader batch - labels shape: torch.Size([1, 1446]) +Dataloader batch - attn_mask shape: torch.Size([1, 1446]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1144 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1446) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1446) torch.int64 + loss_mask : 394 / 1446 tokens (27.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1446, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1446, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1446, 1, 576) torch.bfloat16 + out : (1, 1446) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3445 + top-1 accuracy : 5/394 = 1.3% + + [2026-04-29 14:00:11] iteration 386/ 500 | consumed samples: 1544 | elapsed time per iteration (ms): 4750.8 | throughput (token/sec/GPU): 1291.2 | learning rate: 3.952862E-05 | global batch size: 4 | lm loss: 1.131269E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1145 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 387 / 1539 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.3459 + top-1 accuracy : 7/387 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1710]) +Dataloader batch - labels shape: torch.Size([1, 1710]) +Dataloader batch - attn_mask shape: torch.Size([1, 1710]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1146 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1710) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1710) torch.int64 + loss_mask : 358 / 1710 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1710, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1710, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1710, 1, 576) torch.bfloat16 + out : (1, 1710) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.2073 + top-1 accuracy : 40/358 = 11.2% + +Dataloader batch - tokens shape: torch.Size([1, 1521]) +Dataloader batch - labels shape: torch.Size([1, 1521]) +Dataloader batch - attn_mask shape: torch.Size([1, 1521]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1147 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1521) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1521) torch.int64 + loss_mask : 392 / 1521 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1521, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1521, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1521, 1, 576) torch.bfloat16 + out : (1, 1521) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 392 tokens) : 11.3162 + top-1 accuracy : 15/392 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1148 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 315 / 1424 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.2375 + top-1 accuracy : 2/315 = 0.6% + + [2026-04-29 14:00:16] iteration 387/ 500 | consumed samples: 1548 | elapsed time per iteration (ms): 4858.3 | throughput (token/sec/GPU): 1274.9 | learning rate: 3.922495E-05 | global batch size: 4 | lm loss: 1.128019E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1163]) +Dataloader batch - labels shape: torch.Size([1, 1163]) +Dataloader batch - attn_mask shape: torch.Size([1, 1163]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1149 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1163) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1163) torch.int64 + loss_mask : 290 / 1163 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1163, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1163, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1163, 1, 576) torch.bfloat16 + out : (1, 1163) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.2816 + top-1 accuracy : 9/290 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1979]) +Dataloader batch - labels shape: torch.Size([1, 1979]) +Dataloader batch - attn_mask shape: torch.Size([1, 1979]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1150 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1979) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1979) torch.int64 + loss_mask : 626 / 1979 tokens (31.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1979, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1979, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1979, 1, 576) torch.bfloat16 + out : (1, 1979) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 626 tokens) : 11.4268 + top-1 accuracy : 4/626 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 993]) +Dataloader batch - labels shape: torch.Size([1, 993]) +Dataloader batch - attn_mask shape: torch.Size([1, 993]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1151 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 993) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 993) torch.int64 + loss_mask : 232 / 993 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (993, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (993, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (993, 1, 576) torch.bfloat16 + out : (1, 993) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 232 tokens) : 11.2456 + top-1 accuracy : 4/232 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1702]) +Dataloader batch - labels shape: torch.Size([1, 1702]) +Dataloader batch - attn_mask shape: torch.Size([1, 1702]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1152 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1702) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1702) torch.int64 + loss_mask : 364 / 1702 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1702, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1702, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1702, 1, 576) torch.bfloat16 + out : (1, 1702) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2188 + top-1 accuracy : 20/364 = 5.5% + + [2026-04-29 14:00:20] iteration 388/ 500 | consumed samples: 1552 | elapsed time per iteration (ms): 4522.4 | throughput (token/sec/GPU): 1290.7 | learning rate: 3.892173E-05 | global batch size: 4 | lm loss: 1.132107E+01 | loss scale: 1.0 | grad norm: 0.194 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1227]) +Dataloader batch - labels shape: torch.Size([1, 1227]) +Dataloader batch - attn_mask shape: torch.Size([1, 1227]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1153 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1227) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1227) torch.int64 + loss_mask : 327 / 1227 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1227, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1227, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1227, 1, 576) torch.bfloat16 + out : (1, 1227) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.3514 + top-1 accuracy : 1/327 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1853]) +Dataloader batch - labels shape: torch.Size([1, 1853]) +Dataloader batch - attn_mask shape: torch.Size([1, 1853]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1154 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1853) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1853) torch.int64 + loss_mask : 481 / 1853 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1853, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1853, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1853, 1, 576) torch.bfloat16 + out : (1, 1853) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 481 tokens) : 11.3050 + top-1 accuracy : 7/481 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 1155 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 416 / 1431 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.3830 + top-1 accuracy : 1/416 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1510]) +Dataloader batch - labels shape: torch.Size([1, 1510]) +Dataloader batch - attn_mask shape: torch.Size([1, 1510]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1156 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1510) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1510) torch.int64 + loss_mask : 359 / 1510 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1510, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1510, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1510, 1, 576) torch.bfloat16 + out : (1, 1510) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.2270 + top-1 accuracy : 26/359 = 7.2% + + [2026-04-29 14:00:25] iteration 389/ 500 | consumed samples: 1556 | elapsed time per iteration (ms): 4646.8 | throughput (token/sec/GPU): 1295.7 | learning rate: 3.861896E-05 | global batch size: 4 | lm loss: 1.131741E+01 | loss scale: 1.0 | grad norm: 0.192 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1807]) +Dataloader batch - labels shape: torch.Size([1, 1807]) +Dataloader batch - attn_mask shape: torch.Size([1, 1807]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1157 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1807) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1807) torch.int64 + loss_mask : 446 / 1807 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1807, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1807, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1807, 1, 576) torch.bfloat16 + out : (1, 1807) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 446 tokens) : 11.2735 + top-1 accuracy : 26/446 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1171]) +Dataloader batch - labels shape: torch.Size([1, 1171]) +Dataloader batch - attn_mask shape: torch.Size([1, 1171]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1158 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1171) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1171) torch.int64 + loss_mask : 254 / 1171 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1171, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1171, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1171, 1, 576) torch.bfloat16 + out : (1, 1171) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 254 tokens) : 11.2428 + top-1 accuracy : 24/254 = 9.4% + +Dataloader batch - tokens shape: torch.Size([1, 1439]) +Dataloader batch - labels shape: torch.Size([1, 1439]) +Dataloader batch - attn_mask shape: torch.Size([1, 1439]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1159 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1439) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1439) torch.int64 + loss_mask : 389 / 1439 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1439, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1439, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1439, 1, 576) torch.bfloat16 + out : (1, 1439) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.3415 + top-1 accuracy : 8/389 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1821]) +Dataloader batch - labels shape: torch.Size([1, 1821]) +Dataloader batch - attn_mask shape: torch.Size([1, 1821]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1160 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1821) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1821) torch.int64 + loss_mask : 470 / 1821 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1821, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1821, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1821, 1, 576) torch.bfloat16 + out : (1, 1821) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 470 tokens) : 11.3145 + top-1 accuracy : 20/470 = 4.3% + + [2026-04-29 14:00:30] iteration 390/ 500 | consumed samples: 1560 | elapsed time per iteration (ms): 4943.9 | throughput (token/sec/GPU): 1261.8 | learning rate: 3.831667E-05 | global batch size: 4 | lm loss: 1.129785E+01 | loss scale: 1.0 | grad norm: 0.170 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1548]) +Dataloader batch - labels shape: torch.Size([1, 1548]) +Dataloader batch - attn_mask shape: torch.Size([1, 1548]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1161 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1548) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1548) torch.int64 + loss_mask : 346 / 1548 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1548, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1548, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1548, 1, 576) torch.bfloat16 + out : (1, 1548) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 346 tokens) : 11.2306 + top-1 accuracy : 17/346 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1112]) +Dataloader batch - labels shape: torch.Size([1, 1112]) +Dataloader batch - attn_mask shape: torch.Size([1, 1112]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1162 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1112) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1112) torch.int64 + loss_mask : 318 / 1112 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1112, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1112, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1112, 1, 576) torch.bfloat16 + out : (1, 1112) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.3615 + top-1 accuracy : 15/318 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1468]) +Dataloader batch - labels shape: torch.Size([1, 1468]) +Dataloader batch - attn_mask shape: torch.Size([1, 1468]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1163 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1468) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1468) torch.int64 + loss_mask : 318 / 1468 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1468, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1468, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1468, 1, 576) torch.bfloat16 + out : (1, 1468) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.1505 + top-1 accuracy : 15/318 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1416]) +Dataloader batch - labels shape: torch.Size([1, 1416]) +Dataloader batch - attn_mask shape: torch.Size([1, 1416]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1164 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1416) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1416) torch.int64 + loss_mask : 416 / 1416 tokens (29.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1416, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1416, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1416, 1, 576) torch.bfloat16 + out : (1, 1416) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.4183 + top-1 accuracy : 1/416 = 0.2% + + [2026-04-29 14:00:34] iteration 391/ 500 | consumed samples: 1564 | elapsed time per iteration (ms): 4394.2 | throughput (token/sec/GPU): 1261.7 | learning rate: 3.801486E-05 | global batch size: 4 | lm loss: 1.129798E+01 | loss scale: 1.0 | grad norm: 0.215 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1041]) +Dataloader batch - labels shape: torch.Size([1, 1041]) +Dataloader batch - attn_mask shape: torch.Size([1, 1041]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1165 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1041) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1041) torch.int64 + loss_mask : 210 / 1041 tokens (20.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1041, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1041, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1041, 1, 576) torch.bfloat16 + out : (1, 1041) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 210 tokens) : 11.1707 + top-1 accuracy : 5/210 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1166 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 330 / 1424 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.2284 + top-1 accuracy : 22/330 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1381]) +Dataloader batch - labels shape: torch.Size([1, 1381]) +Dataloader batch - attn_mask shape: torch.Size([1, 1381]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1167 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1381) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1381) torch.int64 + loss_mask : 321 / 1381 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1381, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1381, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1381, 1, 576) torch.bfloat16 + out : (1, 1381) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 321 tokens) : 11.2442 + top-1 accuracy : 28/321 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1558]) +Dataloader batch - labels shape: torch.Size([1, 1558]) +Dataloader batch - attn_mask shape: torch.Size([1, 1558]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1168 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1558) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1558) torch.int64 + loss_mask : 434 / 1558 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1558, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1558, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1558, 1, 576) torch.bfloat16 + out : (1, 1558) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 434 tokens) : 11.3536 + top-1 accuracy : 16/434 = 3.7% + + [2026-04-29 14:00:39] iteration 392/ 500 | consumed samples: 1568 | elapsed time per iteration (ms): 4522.8 | throughput (token/sec/GPU): 1194.8 | learning rate: 3.771354E-05 | global batch size: 4 | lm loss: 1.126492E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1013]) +Dataloader batch - labels shape: torch.Size([1, 1013]) +Dataloader batch - attn_mask shape: torch.Size([1, 1013]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1169 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1013) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1013) torch.int64 + loss_mask : 252 / 1013 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1013, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1013, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1013, 1, 576) torch.bfloat16 + out : (1, 1013) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 252 tokens) : 11.2740 + top-1 accuracy : 5/252 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1170 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 412 / 1546 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3338 + top-1 accuracy : 26/412 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1764]) +Dataloader batch - labels shape: torch.Size([1, 1764]) +Dataloader batch - attn_mask shape: torch.Size([1, 1764]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1171 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1764) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1764) torch.int64 + loss_mask : 399 / 1764 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1764, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1764, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1764, 1, 576) torch.bfloat16 + out : (1, 1764) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 399 tokens) : 11.2619 + top-1 accuracy : 8/399 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1918]) +Dataloader batch - labels shape: torch.Size([1, 1918]) +Dataloader batch - attn_mask shape: torch.Size([1, 1918]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1172 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1918) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1918) torch.int64 + loss_mask : 552 / 1918 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1918, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1918, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1918, 1, 576) torch.bfloat16 + out : (1, 1918) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 552 tokens) : 11.3886 + top-1 accuracy : 9/552 = 1.6% + + [2026-04-29 14:00:44] iteration 393/ 500 | consumed samples: 1572 | elapsed time per iteration (ms): 4883.9 | throughput (token/sec/GPU): 1277.9 | learning rate: 3.741273E-05 | global batch size: 4 | lm loss: 1.132547E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1879]) +Dataloader batch - labels shape: torch.Size([1, 1879]) +Dataloader batch - attn_mask shape: torch.Size([1, 1879]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1173 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1879) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1879) torch.int64 + loss_mask : 507 / 1879 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1879, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1879, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1879, 1, 576) torch.bfloat16 + out : (1, 1879) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 507 tokens) : 11.3436 + top-1 accuracy : 3/507 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1419]) +Dataloader batch - labels shape: torch.Size([1, 1419]) +Dataloader batch - attn_mask shape: torch.Size([1, 1419]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1174 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1419) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1419) torch.int64 + loss_mask : 326 / 1419 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1419, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1419, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1419, 1, 576) torch.bfloat16 + out : (1, 1419) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2601 + top-1 accuracy : 10/326 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1124]) +Dataloader batch - labels shape: torch.Size([1, 1124]) +Dataloader batch - attn_mask shape: torch.Size([1, 1124]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1175 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1124) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1124) torch.int64 + loss_mask : 264 / 1124 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1124, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1124, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1124, 1, 576) torch.bfloat16 + out : (1, 1124) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 264 tokens) : 11.2673 + top-1 accuracy : 2/264 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1593]) +Dataloader batch - labels shape: torch.Size([1, 1593]) +Dataloader batch - attn_mask shape: torch.Size([1, 1593]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1176 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1593) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1593) torch.int64 + loss_mask : 357 / 1593 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1593, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1593, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1593, 1, 576) torch.bfloat16 + out : (1, 1593) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2187 + top-1 accuracy : 23/357 = 6.4% + + [2026-04-29 14:00:49] iteration 394/ 500 | consumed samples: 1576 | elapsed time per iteration (ms): 4764.8 | throughput (token/sec/GPU): 1262.4 | learning rate: 3.711244E-05 | global batch size: 4 | lm loss: 1.128037E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1476]) +Dataloader batch - labels shape: torch.Size([1, 1476]) +Dataloader batch - attn_mask shape: torch.Size([1, 1476]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1177 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1476) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1476) torch.int64 + loss_mask : 426 / 1476 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1476, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1476, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1476, 1, 576) torch.bfloat16 + out : (1, 1476) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 426 tokens) : 11.3191 + top-1 accuracy : 1/426 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1531]) +Dataloader batch - labels shape: torch.Size([1, 1531]) +Dataloader batch - attn_mask shape: torch.Size([1, 1531]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1178 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1531) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1531) torch.int64 + loss_mask : 360 / 1531 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1531, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1531, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1531, 1, 576) torch.bfloat16 + out : (1, 1531) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.2152 + top-1 accuracy : 12/360 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1393]) +Dataloader batch - labels shape: torch.Size([1, 1393]) +Dataloader batch - attn_mask shape: torch.Size([1, 1393]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1179 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1393) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1393) torch.int64 + loss_mask : 332 / 1393 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1393, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1393, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1393, 1, 576) torch.bfloat16 + out : (1, 1393) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.2688 + top-1 accuracy : 21/332 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1180 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 405 / 1518 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.2613 + top-1 accuracy : 4/405 = 1.0% + + [2026-04-29 14:00:53] iteration 395/ 500 | consumed samples: 1580 | elapsed time per iteration (ms): 4648.5 | throughput (token/sec/GPU): 1273.1 | learning rate: 3.681269E-05 | global batch size: 4 | lm loss: 1.126823E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1020]) +Dataloader batch - labels shape: torch.Size([1, 1020]) +Dataloader batch - attn_mask shape: torch.Size([1, 1020]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1181 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1020) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1020) torch.int64 + loss_mask : 263 / 1020 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1020, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1020, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1020, 1, 576) torch.bfloat16 + out : (1, 1020) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.3733 + top-1 accuracy : 21/263 = 8.0% + +Dataloader batch - tokens shape: torch.Size([1, 1069]) +Dataloader batch - labels shape: torch.Size([1, 1069]) +Dataloader batch - attn_mask shape: torch.Size([1, 1069]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1182 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1069) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1069) torch.int64 + loss_mask : 260 / 1069 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1069, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1069, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1069, 1, 576) torch.bfloat16 + out : (1, 1069) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.2347 + top-1 accuracy : 4/260 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 995]) +Dataloader batch - labels shape: torch.Size([1, 995]) +Dataloader batch - attn_mask shape: torch.Size([1, 995]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1183 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 995) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 995) torch.int64 + loss_mask : 226 / 995 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (995, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (995, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (995, 1, 576) torch.bfloat16 + out : (1, 995) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 226 tokens) : 11.2231 + top-1 accuracy : 20/226 = 8.8% + +Dataloader batch - tokens shape: torch.Size([1, 1516]) +Dataloader batch - labels shape: torch.Size([1, 1516]) +Dataloader batch - attn_mask shape: torch.Size([1, 1516]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1184 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1516) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1516) torch.int64 + loss_mask : 320 / 1516 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1516, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1516, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1516, 1, 576) torch.bfloat16 + out : (1, 1516) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2158 + top-1 accuracy : 29/320 = 9.1% + + [2026-04-29 14:00:57] iteration 396/ 500 | consumed samples: 1584 | elapsed time per iteration (ms): 3930.8 | throughput (token/sec/GPU): 1170.2 | learning rate: 3.651347E-05 | global batch size: 4 | lm loss: 1.126070E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1563]) +Dataloader batch - labels shape: torch.Size([1, 1563]) +Dataloader batch - attn_mask shape: torch.Size([1, 1563]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1185 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1563) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1563) torch.int64 + loss_mask : 379 / 1563 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1563, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1563, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1563, 1, 576) torch.bfloat16 + out : (1, 1563) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.2773 + top-1 accuracy : 11/379 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1509]) +Dataloader batch - labels shape: torch.Size([1, 1509]) +Dataloader batch - attn_mask shape: torch.Size([1, 1509]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1186 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1509) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1509) torch.int64 + loss_mask : 345 / 1509 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1509, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1509, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1509, 1, 576) torch.bfloat16 + out : (1, 1509) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2334 + top-1 accuracy : 10/345 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 953]) +Dataloader batch - labels shape: torch.Size([1, 953]) +Dataloader batch - attn_mask shape: torch.Size([1, 953]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1187 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 953) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 953) torch.int64 + loss_mask : 203 / 953 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (953, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (953, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (953, 1, 576) torch.bfloat16 + out : (1, 953) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 203 tokens) : 11.2030 + top-1 accuracy : 10/203 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1490]) +Dataloader batch - labels shape: torch.Size([1, 1490]) +Dataloader batch - attn_mask shape: torch.Size([1, 1490]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1188 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1490) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1490) torch.int64 + loss_mask : 447 / 1490 tokens (30.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1490, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1490, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1490, 1, 576) torch.bfloat16 + out : (1, 1490) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 447 tokens) : 11.4015 + top-1 accuracy : 6/447 = 1.3% + + [2026-04-29 14:01:02] iteration 397/ 500 | consumed samples: 1588 | elapsed time per iteration (ms): 4490.1 | throughput (token/sec/GPU): 1228.3 | learning rate: 3.621481E-05 | global batch size: 4 | lm loss: 1.129572E+01 | loss scale: 1.0 | grad norm: 0.260 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1866]) +Dataloader batch - labels shape: torch.Size([1, 1866]) +Dataloader batch - attn_mask shape: torch.Size([1, 1866]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1189 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1866) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1866) torch.int64 + loss_mask : 504 / 1866 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1866, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1866, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1866, 1, 576) torch.bfloat16 + out : (1, 1866) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 504 tokens) : 11.3768 + top-1 accuracy : 16/504 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1752]) +Dataloader batch - labels shape: torch.Size([1, 1752]) +Dataloader batch - attn_mask shape: torch.Size([1, 1752]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1190 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1752) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1752) torch.int64 + loss_mask : 383 / 1752 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1752, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1752, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1752, 1, 576) torch.bfloat16 + out : (1, 1752) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.2400 + top-1 accuracy : 20/383 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1661]) +Dataloader batch - labels shape: torch.Size([1, 1661]) +Dataloader batch - attn_mask shape: torch.Size([1, 1661]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1191 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1661) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1661) torch.int64 + loss_mask : 398 / 1661 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1661, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1661, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1661, 1, 576) torch.bfloat16 + out : (1, 1661) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.2818 + top-1 accuracy : 18/398 = 4.5% + +Dataloader batch - tokens shape: torch.Size([1, 1402]) +Dataloader batch - labels shape: torch.Size([1, 1402]) +Dataloader batch - attn_mask shape: torch.Size([1, 1402]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1192 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1402) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1402) torch.int64 + loss_mask : 344 / 1402 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1402, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1402, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1402, 1, 576) torch.bfloat16 + out : (1, 1402) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.2378 + top-1 accuracy : 33/344 = 9.6% + + [2026-04-29 14:01:07] iteration 398/ 500 | consumed samples: 1592 | elapsed time per iteration (ms): 5213.3 | throughput (token/sec/GPU): 1281.5 | learning rate: 3.591671E-05 | global batch size: 4 | lm loss: 1.129205E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1193 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 367 / 1504 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2124 + top-1 accuracy : 6/367 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1118]) +Dataloader batch - labels shape: torch.Size([1, 1118]) +Dataloader batch - attn_mask shape: torch.Size([1, 1118]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1194 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1118) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1118) torch.int64 + loss_mask : 283 / 1118 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1118, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1118, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1118, 1, 576) torch.bfloat16 + out : (1, 1118) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 283 tokens) : 11.3015 + top-1 accuracy : 0/283 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1282]) +Dataloader batch - labels shape: torch.Size([1, 1282]) +Dataloader batch - attn_mask shape: torch.Size([1, 1282]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1195 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1282) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1282) torch.int64 + loss_mask : 379 / 1282 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1282, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1282, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1282, 1, 576) torch.bfloat16 + out : (1, 1282) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.4125 + top-1 accuracy : 1/379 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1387]) +Dataloader batch - labels shape: torch.Size([1, 1387]) +Dataloader batch - attn_mask shape: torch.Size([1, 1387]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1196 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1387) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1387) torch.int64 + loss_mask : 411 / 1387 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1387, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1387, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1387, 1, 576) torch.bfloat16 + out : (1, 1387) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.4101 + top-1 accuracy : 1/411 = 0.2% + + [2026-04-29 14:01:11] iteration 399/ 500 | consumed samples: 1596 | elapsed time per iteration (ms): 4208.1 | throughput (token/sec/GPU): 1257.3 | learning rate: 3.561919E-05 | global batch size: 4 | lm loss: 1.133900E+01 | loss scale: 1.0 | grad norm: 0.202 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1711]) +Dataloader batch - labels shape: torch.Size([1, 1711]) +Dataloader batch - attn_mask shape: torch.Size([1, 1711]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1197 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1711) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1711) torch.int64 + loss_mask : 397 / 1711 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1711, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1711, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1711, 1, 576) torch.bfloat16 + out : (1, 1711) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 397 tokens) : 11.2305 + top-1 accuracy : 40/397 = 10.1% + +Dataloader batch - tokens shape: torch.Size([1, 1096]) +Dataloader batch - labels shape: torch.Size([1, 1096]) +Dataloader batch - attn_mask shape: torch.Size([1, 1096]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1198 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1096) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1096) torch.int64 + loss_mask : 258 / 1096 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1096, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1096, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1096, 1, 576) torch.bfloat16 + out : (1, 1096) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 258 tokens) : 11.2161 + top-1 accuracy : 6/258 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1877]) +Dataloader batch - labels shape: torch.Size([1, 1877]) +Dataloader batch - attn_mask shape: torch.Size([1, 1877]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1199 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1877) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1877) torch.int64 + loss_mask : 528 / 1877 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1877, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1877, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1877, 1, 576) torch.bfloat16 + out : (1, 1877) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 528 tokens) : 11.3805 + top-1 accuracy : 17/528 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1341]) +Dataloader batch - labels shape: torch.Size([1, 1341]) +Dataloader batch - attn_mask shape: torch.Size([1, 1341]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1200 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1341) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1341) torch.int64 + loss_mask : 268 / 1341 tokens (20.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1341, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1341, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1341, 1, 576) torch.bfloat16 + out : (1, 1341) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 268 tokens) : 11.1666 + top-1 accuracy : 21/268 = 7.8% + + [2026-04-29 14:01:16] iteration 400/ 500 | consumed samples: 1600 | elapsed time per iteration (ms): 4801.9 | throughput (token/sec/GPU): 1254.7 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.127072E+01 | loss scale: 1.0 | grad norm: 0.253 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4776.89, 4776.89) + forward-compute ................................: (3549.11, 3549.11) + backward-compute ...............................: (1223.86, 1223.86) + batch-generator ................................: (434.91, 434.91) + layernorm-grads-all-reduce .....................: (0.07, 0.07) + embedding-grads-all-reduce .....................: (0.10, 0.10) + all-grads-sync .................................: (1.08, 1.08) + params-all-gather ..............................: (0.38, 0.38) + optimizer-copy-to-main-grad ....................: (0.33, 0.33) + optimizer-clip-main-grad .......................: (1.27, 1.27) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.15, 2.15) + optimizer-copy-main-to-model-params ............: (0.55, 0.55) + optimizer ......................................: (6.72, 6.72) +saving checkpoint at iteration 400 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 400 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 400 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (1907.50, 1907.50) +Dataloader batch - tokens shape: torch.Size([1, 1492]) +Dataloader batch - labels shape: torch.Size([1, 1492]) +Dataloader batch - attn_mask shape: torch.Size([1, 1492]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1201 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1492) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1492) torch.int64 + loss_mask : 445 / 1492 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1492, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1492, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1492, 1, 576) torch.bfloat16 + out : (1, 1492) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 445 tokens) : 11.3833 + top-1 accuracy : 3/445 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1603]) +Dataloader batch - labels shape: torch.Size([1, 1603]) +Dataloader batch - attn_mask shape: torch.Size([1, 1603]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1202 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1603) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1603) torch.int64 + loss_mask : 402 / 1603 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1603, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1603, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1603, 1, 576) torch.bfloat16 + out : (1, 1603) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.2703 + top-1 accuracy : 16/402 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1440]) +Dataloader batch - labels shape: torch.Size([1, 1440]) +Dataloader batch - attn_mask shape: torch.Size([1, 1440]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1203 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1440) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1440) torch.int64 + loss_mask : 290 / 1440 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1440, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1440, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1440, 1, 576) torch.bfloat16 + out : (1, 1440) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 290 tokens) : 11.1299 + top-1 accuracy : 27/290 = 9.3% + +Dataloader batch - tokens shape: torch.Size([1, 1409]) +Dataloader batch - labels shape: torch.Size([1, 1409]) +Dataloader batch - attn_mask shape: torch.Size([1, 1409]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1204 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1409) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1409) torch.int64 + loss_mask : 348 / 1409 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1409, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1409, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1409, 1, 576) torch.bfloat16 + out : (1, 1409) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2683 + top-1 accuracy : 3/348 = 0.9% + + [2026-04-29 14:01:22] iteration 401/ 500 | consumed samples: 1604 | elapsed time per iteration (ms): 4765.8 | throughput (token/sec/GPU): 1247.2 | learning rate: 3.502594E-05 | global batch size: 4 | lm loss: 1.127627E+01 | loss scale: 1.0 | grad norm: 0.187 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1121]) +Dataloader batch - labels shape: torch.Size([1, 1121]) +Dataloader batch - attn_mask shape: torch.Size([1, 1121]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1205 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1121) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1121) torch.int64 + loss_mask : 312 / 1121 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1121, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1121, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1121, 1, 576) torch.bfloat16 + out : (1, 1121) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.3860 + top-1 accuracy : 11/312 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1620]) +Dataloader batch - labels shape: torch.Size([1, 1620]) +Dataloader batch - attn_mask shape: torch.Size([1, 1620]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1206 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1620) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1620) torch.int64 + loss_mask : 386 / 1620 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1620, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1620, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1620, 1, 576) torch.bfloat16 + out : (1, 1620) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2270 + top-1 accuracy : 22/386 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1207 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 419 / 1475 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.3917 + top-1 accuracy : 8/419 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1588]) +Dataloader batch - labels shape: torch.Size([1, 1588]) +Dataloader batch - attn_mask shape: torch.Size([1, 1588]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1208 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1588) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1588) torch.int64 + loss_mask : 364 / 1588 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1588, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1588, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1588, 1, 576) torch.bfloat16 + out : (1, 1588) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.2164 + top-1 accuracy : 18/364 = 4.9% + + [2026-04-29 14:01:27] iteration 402/ 500 | consumed samples: 1608 | elapsed time per iteration (ms): 4594.6 | throughput (token/sec/GPU): 1263.2 | learning rate: 3.473022E-05 | global batch size: 4 | lm loss: 1.130450E+01 | loss scale: 1.0 | grad norm: 0.176 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1505]) +Dataloader batch - labels shape: torch.Size([1, 1505]) +Dataloader batch - attn_mask shape: torch.Size([1, 1505]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1209 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1505) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1505) torch.int64 + loss_mask : 355 / 1505 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1505, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1505, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1505, 1, 576) torch.bfloat16 + out : (1, 1505) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2402 + top-1 accuracy : 2/355 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1210 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 412 / 1508 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3420 + top-1 accuracy : 2/412 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1126]) +Dataloader batch - labels shape: torch.Size([1, 1126]) +Dataloader batch - attn_mask shape: torch.Size([1, 1126]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1211 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1126) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1126) torch.int64 + loss_mask : 325 / 1126 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1126, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1126, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1126, 1, 576) torch.bfloat16 + out : (1, 1126) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.3555 + top-1 accuracy : 2/325 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1863]) +Dataloader batch - labels shape: torch.Size([1, 1863]) +Dataloader batch - attn_mask shape: torch.Size([1, 1863]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1212 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1863) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1863) torch.int64 + loss_mask : 517 / 1863 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1863, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1863, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1863, 1, 576) torch.bfloat16 + out : (1, 1863) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 517 tokens) : 11.3232 + top-1 accuracy : 15/517 = 2.9% + + [2026-04-29 14:01:32] iteration 403/ 500 | consumed samples: 1612 | elapsed time per iteration (ms): 4763.8 | throughput (token/sec/GPU): 1259.9 | learning rate: 3.443513E-05 | global batch size: 4 | lm loss: 1.131622E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1821]) +Dataloader batch - labels shape: torch.Size([1, 1821]) +Dataloader batch - attn_mask shape: torch.Size([1, 1821]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1213 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1821) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1821) torch.int64 + loss_mask : 449 / 1821 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1821, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1821, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1821, 1, 576) torch.bfloat16 + out : (1, 1821) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 449 tokens) : 11.3024 + top-1 accuracy : 16/449 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1619]) +Dataloader batch - labels shape: torch.Size([1, 1619]) +Dataloader batch - attn_mask shape: torch.Size([1, 1619]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1214 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1619) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1619) torch.int64 + loss_mask : 418 / 1619 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1619, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1619, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1619, 1, 576) torch.bfloat16 + out : (1, 1619) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 418 tokens) : 11.3018 + top-1 accuracy : 31/418 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1914]) +Dataloader batch - labels shape: torch.Size([1, 1914]) +Dataloader batch - attn_mask shape: torch.Size([1, 1914]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1215 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1914) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1914) torch.int64 + loss_mask : 549 / 1914 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1914, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1914, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1914, 1, 576) torch.bfloat16 + out : (1, 1914) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 549 tokens) : 11.3539 + top-1 accuracy : 4/549 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1753]) +Dataloader batch - labels shape: torch.Size([1, 1753]) +Dataloader batch - attn_mask shape: torch.Size([1, 1753]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1216 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1753) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1753) torch.int64 + loss_mask : 408 / 1753 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1753, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1753, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1753, 1, 576) torch.bfloat16 + out : (1, 1753) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.2917 + top-1 accuracy : 22/408 = 5.4% + + [2026-04-29 14:01:37] iteration 404/ 500 | consumed samples: 1616 | elapsed time per iteration (ms): 5375.2 | throughput (token/sec/GPU): 1322.2 | learning rate: 3.414068E-05 | global batch size: 4 | lm loss: 1.131541E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1770]) +Dataloader batch - labels shape: torch.Size([1, 1770]) +Dataloader batch - attn_mask shape: torch.Size([1, 1770]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1217 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1770) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1770) torch.int64 + loss_mask : 408 / 1770 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1770, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1770, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1770, 1, 576) torch.bfloat16 + out : (1, 1770) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.2544 + top-1 accuracy : 29/408 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1218 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 309 / 1460 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.1679 + top-1 accuracy : 11/309 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1768]) +Dataloader batch - labels shape: torch.Size([1, 1768]) +Dataloader batch - attn_mask shape: torch.Size([1, 1768]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1219 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1768) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1768) torch.int64 + loss_mask : 477 / 1768 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1768, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1768, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1768, 1, 576) torch.bfloat16 + out : (1, 1768) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 477 tokens) : 11.3765 + top-1 accuracy : 6/477 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1542]) +Dataloader batch - labels shape: torch.Size([1, 1542]) +Dataloader batch - attn_mask shape: torch.Size([1, 1542]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1220 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1542) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1542) torch.int64 + loss_mask : 387 / 1542 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1542, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1542, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1542, 1, 576) torch.bfloat16 + out : (1, 1542) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2857 + top-1 accuracy : 3/387 = 0.8% + + [2026-04-29 14:01:42] iteration 405/ 500 | consumed samples: 1620 | elapsed time per iteration (ms): 5129.9 | throughput (token/sec/GPU): 1274.9 | learning rate: 3.384688E-05 | global batch size: 4 | lm loss: 1.128200E+01 | loss scale: 1.0 | grad norm: 0.188 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1063]) +Dataloader batch - labels shape: torch.Size([1, 1063]) +Dataloader batch - attn_mask shape: torch.Size([1, 1063]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1221 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1063) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1063) torch.int64 + loss_mask : 242 / 1063 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1063, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1063, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1063, 1, 576) torch.bfloat16 + out : (1, 1063) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 242 tokens) : 11.2476 + top-1 accuracy : 16/242 = 6.6% + +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1222 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 403 / 1552 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.2866 + top-1 accuracy : 17/403 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1432]) +Dataloader batch - labels shape: torch.Size([1, 1432]) +Dataloader batch - attn_mask shape: torch.Size([1, 1432]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1223 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1432) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1432) torch.int64 + loss_mask : 388 / 1432 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1432, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1432, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1432, 1, 576) torch.bfloat16 + out : (1, 1432) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3555 + top-1 accuracy : 25/388 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1572]) +Dataloader batch - labels shape: torch.Size([1, 1572]) +Dataloader batch - attn_mask shape: torch.Size([1, 1572]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1224 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1572) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1572) torch.int64 + loss_mask : 407 / 1572 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1572, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1572, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1572, 1, 576) torch.bfloat16 + out : (1, 1572) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.3272 + top-1 accuracy : 7/407 = 1.7% + + [2026-04-29 14:01:47] iteration 406/ 500 | consumed samples: 1624 | elapsed time per iteration (ms): 4450.9 | throughput (token/sec/GPU): 1262.5 | learning rate: 3.355373E-05 | global batch size: 4 | lm loss: 1.131008E+01 | loss scale: 1.0 | grad norm: 0.193 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1365]) +Dataloader batch - labels shape: torch.Size([1, 1365]) +Dataloader batch - attn_mask shape: torch.Size([1, 1365]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1225 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1365) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1365) torch.int64 + loss_mask : 297 / 1365 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1365, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1365, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1365, 1, 576) torch.bfloat16 + out : (1, 1365) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 297 tokens) : 11.2132 + top-1 accuracy : 29/297 = 9.8% + +Dataloader batch - tokens shape: torch.Size([1, 1267]) +Dataloader batch - labels shape: torch.Size([1, 1267]) +Dataloader batch - attn_mask shape: torch.Size([1, 1267]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1226 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1267) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1267) torch.int64 + loss_mask : 339 / 1267 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1267, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1267, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1267, 1, 576) torch.bfloat16 + out : (1, 1267) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.3834 + top-1 accuracy : 17/339 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1476]) +Dataloader batch - labels shape: torch.Size([1, 1476]) +Dataloader batch - attn_mask shape: torch.Size([1, 1476]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1227 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1476) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1476) torch.int64 + loss_mask : 411 / 1476 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1476, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1476, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1476, 1, 576) torch.bfloat16 + out : (1, 1476) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.3384 + top-1 accuracy : 2/411 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1534]) +Dataloader batch - labels shape: torch.Size([1, 1534]) +Dataloader batch - attn_mask shape: torch.Size([1, 1534]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1228 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1534) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1534) torch.int64 + loss_mask : 398 / 1534 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1534, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1534, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1534, 1, 576) torch.bfloat16 + out : (1, 1534) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.2857 + top-1 accuracy : 8/398 = 2.0% + + [2026-04-29 14:01:51] iteration 407/ 500 | consumed samples: 1628 | elapsed time per iteration (ms): 4500.8 | throughput (token/sec/GPU): 1253.6 | learning rate: 3.326126E-05 | global batch size: 4 | lm loss: 1.130873E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1735]) +Dataloader batch - labels shape: torch.Size([1, 1735]) +Dataloader batch - attn_mask shape: torch.Size([1, 1735]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1229 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1735) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1735) torch.int64 + loss_mask : 353 / 1735 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1735, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1735, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1735, 1, 576) torch.bfloat16 + out : (1, 1735) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.1959 + top-1 accuracy : 2/353 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1702]) +Dataloader batch - labels shape: torch.Size([1, 1702]) +Dataloader batch - attn_mask shape: torch.Size([1, 1702]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1230 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1702) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1702) torch.int64 + loss_mask : 358 / 1702 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1702, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1702, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1702, 1, 576) torch.bfloat16 + out : (1, 1702) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.1944 + top-1 accuracy : 22/358 = 6.1% + +Dataloader batch - tokens shape: torch.Size([1, 1136]) +Dataloader batch - labels shape: torch.Size([1, 1136]) +Dataloader batch - attn_mask shape: torch.Size([1, 1136]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1231 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1136) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1136) torch.int64 + loss_mask : 292 / 1136 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1136, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1136, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1136, 1, 576) torch.bfloat16 + out : (1, 1136) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 292 tokens) : 11.2673 + top-1 accuracy : 16/292 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1060]) +Dataloader batch - labels shape: torch.Size([1, 1060]) +Dataloader batch - attn_mask shape: torch.Size([1, 1060]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1232 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1060) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1060) torch.int64 + loss_mask : 306 / 1060 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1060, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1060, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1060, 1, 576) torch.bfloat16 + out : (1, 1060) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.3093 + top-1 accuracy : 4/306 = 1.3% + + [2026-04-29 14:01:56] iteration 408/ 500 | consumed samples: 1632 | elapsed time per iteration (ms): 4512.5 | throughput (token/sec/GPU): 1248.3 | learning rate: 3.296947E-05 | global batch size: 4 | lm loss: 1.123795E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1477]) +Dataloader batch - labels shape: torch.Size([1, 1477]) +Dataloader batch - attn_mask shape: torch.Size([1, 1477]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1233 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1477) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1477) torch.int64 + loss_mask : 288 / 1477 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1477, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1477, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1477, 1, 576) torch.bfloat16 + out : (1, 1477) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 288 tokens) : 11.1248 + top-1 accuracy : 22/288 = 7.6% + +Dataloader batch - tokens shape: torch.Size([1, 1045]) +Dataloader batch - labels shape: torch.Size([1, 1045]) +Dataloader batch - attn_mask shape: torch.Size([1, 1045]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1234 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1045) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1045) torch.int64 + loss_mask : 261 / 1045 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1045, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1045, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1045, 1, 576) torch.bfloat16 + out : (1, 1045) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 261 tokens) : 11.1997 + top-1 accuracy : 7/261 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1720]) +Dataloader batch - labels shape: torch.Size([1, 1720]) +Dataloader batch - attn_mask shape: torch.Size([1, 1720]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1235 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1720) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1720) torch.int64 + loss_mask : 363 / 1720 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1720, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1720, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1720, 1, 576) torch.bfloat16 + out : (1, 1720) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.2431 + top-1 accuracy : 36/363 = 9.9% + +Dataloader batch - tokens shape: torch.Size([1, 1839]) +Dataloader batch - labels shape: torch.Size([1, 1839]) +Dataloader batch - attn_mask shape: torch.Size([1, 1839]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1236 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1839) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1839) torch.int64 + loss_mask : 472 / 1839 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1839, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1839, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1839, 1, 576) torch.bfloat16 + out : (1, 1839) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 472 tokens) : 11.3457 + top-1 accuracy : 20/472 = 4.2% + + [2026-04-29 14:02:01] iteration 409/ 500 | consumed samples: 1636 | elapsed time per iteration (ms): 4842.0 | throughput (token/sec/GPU): 1255.9 | learning rate: 3.267838E-05 | global batch size: 4 | lm loss: 1.124527E+01 | loss scale: 1.0 | grad norm: 0.218 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1700]) +Dataloader batch - labels shape: torch.Size([1, 1700]) +Dataloader batch - attn_mask shape: torch.Size([1, 1700]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1237 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1700) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1700) torch.int64 + loss_mask : 422 / 1700 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1700, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1700, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1700, 1, 576) torch.bfloat16 + out : (1, 1700) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.3265 + top-1 accuracy : 22/422 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1417]) +Dataloader batch - labels shape: torch.Size([1, 1417]) +Dataloader batch - attn_mask shape: torch.Size([1, 1417]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1238 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1417) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1417) torch.int64 + loss_mask : 390 / 1417 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1417, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1417, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1417, 1, 576) torch.bfloat16 + out : (1, 1417) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3048 + top-1 accuracy : 3/390 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1133]) +Dataloader batch - labels shape: torch.Size([1, 1133]) +Dataloader batch - attn_mask shape: torch.Size([1, 1133]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1239 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1133) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1133) torch.int64 + loss_mask : 294 / 1133 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1133, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1133, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1133, 1, 576) torch.bfloat16 + out : (1, 1133) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.3167 + top-1 accuracy : 24/294 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 920]) +Dataloader batch - labels shape: torch.Size([1, 920]) +Dataloader batch - attn_mask shape: torch.Size([1, 920]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1240 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 920) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 920) torch.int64 + loss_mask : 189 / 920 tokens (20.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (920, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (920, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (920, 1, 576) torch.bfloat16 + out : (1, 920) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 189 tokens) : 11.1755 + top-1 accuracy : 8/189 = 4.2% + + [2026-04-29 14:02:05] iteration 410/ 500 | consumed samples: 1640 | elapsed time per iteration (ms): 4296.8 | throughput (token/sec/GPU): 1203.2 | learning rate: 3.238799E-05 | global batch size: 4 | lm loss: 1.129568E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1741]) +Dataloader batch - labels shape: torch.Size([1, 1741]) +Dataloader batch - attn_mask shape: torch.Size([1, 1741]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1241 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1741) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1741) torch.int64 + loss_mask : 388 / 1741 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1741, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1741, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1741, 1, 576) torch.bfloat16 + out : (1, 1741) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2382 + top-1 accuracy : 34/388 = 8.8% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1242 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 286 / 1364 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 286 tokens) : 11.1978 + top-1 accuracy : 21/286 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1243 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 328 / 1487 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 328 tokens) : 11.2542 + top-1 accuracy : 16/328 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1908]) +Dataloader batch - labels shape: torch.Size([1, 1908]) +Dataloader batch - attn_mask shape: torch.Size([1, 1908]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1244 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1908) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1908) torch.int64 + loss_mask : 545 / 1908 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1908, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1908, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1908, 1, 576) torch.bfloat16 + out : (1, 1908) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 545 tokens) : 11.3752 + top-1 accuracy : 23/545 = 4.2% + + [2026-04-29 14:02:10] iteration 411/ 500 | consumed samples: 1644 | elapsed time per iteration (ms): 5264.0 | throughput (token/sec/GPU): 1234.8 | learning rate: 3.209832E-05 | global batch size: 4 | lm loss: 1.128238E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1109]) +Dataloader batch - labels shape: torch.Size([1, 1109]) +Dataloader batch - attn_mask shape: torch.Size([1, 1109]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1245 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1109) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1109) torch.int64 + loss_mask : 265 / 1109 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1109, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1109, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1109, 1, 576) torch.bfloat16 + out : (1, 1109) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 265 tokens) : 11.2484 + top-1 accuracy : 13/265 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1107]) +Dataloader batch - labels shape: torch.Size([1, 1107]) +Dataloader batch - attn_mask shape: torch.Size([1, 1107]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1246 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1107) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1107) torch.int64 + loss_mask : 262 / 1107 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1107, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1107, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1107, 1, 576) torch.bfloat16 + out : (1, 1107) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 262 tokens) : 11.2530 + top-1 accuracy : 4/262 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1768]) +Dataloader batch - labels shape: torch.Size([1, 1768]) +Dataloader batch - attn_mask shape: torch.Size([1, 1768]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1247 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1768) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1768) torch.int64 + loss_mask : 390 / 1768 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1768, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1768, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1768, 1, 576) torch.bfloat16 + out : (1, 1768) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.2268 + top-1 accuracy : 27/390 = 6.9% + +Dataloader batch - tokens shape: torch.Size([1, 1467]) +Dataloader batch - labels shape: torch.Size([1, 1467]) +Dataloader batch - attn_mask shape: torch.Size([1, 1467]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1248 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1467) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1467) torch.int64 + loss_mask : 411 / 1467 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1467, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1467, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1467, 1, 576) torch.bfloat16 + out : (1, 1467) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.3707 + top-1 accuracy : 3/411 = 0.7% + + [2026-04-29 14:02:15] iteration 412/ 500 | consumed samples: 1648 | elapsed time per iteration (ms): 4472.1 | throughput (token/sec/GPU): 1218.9 | learning rate: 3.180938E-05 | global batch size: 4 | lm loss: 1.128082E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1482]) +Dataloader batch - labels shape: torch.Size([1, 1482]) +Dataloader batch - attn_mask shape: torch.Size([1, 1482]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1249 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1482) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1482) torch.int64 + loss_mask : 406 / 1482 tokens (27.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1482, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1482, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1482, 1, 576) torch.bfloat16 + out : (1, 1482) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.3618 + top-1 accuracy : 5/406 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 966]) +Dataloader batch - labels shape: torch.Size([1, 966]) +Dataloader batch - attn_mask shape: torch.Size([1, 966]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1250 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 966) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 966) torch.int64 + loss_mask : 196 / 966 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (966, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (966, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (966, 1, 576) torch.bfloat16 + out : (1, 966) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 196 tokens) : 11.1398 + top-1 accuracy : 5/196 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1251 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1418) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1418) torch.int64 + loss_mask : 376 / 1418 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1418, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1418, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1418, 1, 576) torch.bfloat16 + out : (1, 1418) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.3385 + top-1 accuracy : 4/376 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1527]) +Dataloader batch - labels shape: torch.Size([1, 1527]) +Dataloader batch - attn_mask shape: torch.Size([1, 1527]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1252 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1527) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1527) torch.int64 + loss_mask : 406 / 1527 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1527, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1527, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1527, 1, 576) torch.bfloat16 + out : (1, 1527) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 406 tokens) : 11.3139 + top-1 accuracy : 20/406 = 4.9% + + [2026-04-29 14:02:19] iteration 413/ 500 | consumed samples: 1652 | elapsed time per iteration (ms): 4343.5 | throughput (token/sec/GPU): 1241.6 | learning rate: 3.152119E-05 | global batch size: 4 | lm loss: 1.130999E+01 | loss scale: 1.0 | grad norm: 0.177 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1820]) +Dataloader batch - labels shape: torch.Size([1, 1820]) +Dataloader batch - attn_mask shape: torch.Size([1, 1820]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1253 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1820) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1820) torch.int64 + loss_mask : 459 / 1820 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1820, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1820, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1820, 1, 576) torch.bfloat16 + out : (1, 1820) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 459 tokens) : 11.2826 + top-1 accuracy : 4/459 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1711]) +Dataloader batch - labels shape: torch.Size([1, 1711]) +Dataloader batch - attn_mask shape: torch.Size([1, 1711]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1254 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1711) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1711) torch.int64 + loss_mask : 355 / 1711 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1711, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1711, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1711, 1, 576) torch.bfloat16 + out : (1, 1711) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2469 + top-1 accuracy : 39/355 = 11.0% + +Dataloader batch - tokens shape: torch.Size([1, 1909]) +Dataloader batch - labels shape: torch.Size([1, 1909]) +Dataloader batch - attn_mask shape: torch.Size([1, 1909]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1255 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1909) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1909) torch.int64 + loss_mask : 543 / 1909 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1909, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1909, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1909, 1, 576) torch.bfloat16 + out : (1, 1909) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 543 tokens) : 11.3489 + top-1 accuracy : 6/543 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1550]) +Dataloader batch - labels shape: torch.Size([1, 1550]) +Dataloader batch - attn_mask shape: torch.Size([1, 1550]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1256 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1550) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1550) torch.int64 + loss_mask : 351 / 1550 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1550, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1550, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1550, 1, 576) torch.bfloat16 + out : (1, 1550) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.2211 + top-1 accuracy : 3/351 = 0.9% + + [2026-04-29 14:02:24] iteration 414/ 500 | consumed samples: 1656 | elapsed time per iteration (ms): 5388.3 | throughput (token/sec/GPU): 1297.2 | learning rate: 3.123374E-05 | global batch size: 4 | lm loss: 1.128361E+01 | loss scale: 1.0 | grad norm: 0.174 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1257 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 398 / 1546 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.3097 + top-1 accuracy : 8/398 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1122]) +Dataloader batch - labels shape: torch.Size([1, 1122]) +Dataloader batch - attn_mask shape: torch.Size([1, 1122]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1258 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1122) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1122) torch.int64 + loss_mask : 261 / 1122 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1122, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1122, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1122, 1, 576) torch.bfloat16 + out : (1, 1122) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 261 tokens) : 11.2499 + top-1 accuracy : 6/261 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1072]) +Dataloader batch - labels shape: torch.Size([1, 1072]) +Dataloader batch - attn_mask shape: torch.Size([1, 1072]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1259 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1072) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1072) torch.int64 + loss_mask : 260 / 1072 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1072, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1072, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1072, 1, 576) torch.bfloat16 + out : (1, 1072) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.3033 + top-1 accuracy : 15/260 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1663]) +Dataloader batch - labels shape: torch.Size([1, 1663]) +Dataloader batch - attn_mask shape: torch.Size([1, 1663]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1260 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1663) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1663) torch.int64 + loss_mask : 387 / 1663 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1663, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1663, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1663, 1, 576) torch.bfloat16 + out : (1, 1663) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2333 + top-1 accuracy : 26/387 = 6.7% + + [2026-04-29 14:02:29] iteration 415/ 500 | consumed samples: 1660 | elapsed time per iteration (ms): 4457.5 | throughput (token/sec/GPU): 1212.1 | learning rate: 3.094706E-05 | global batch size: 4 | lm loss: 1.127384E+01 | loss scale: 1.0 | grad norm: 0.226 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1397]) +Dataloader batch - labels shape: torch.Size([1, 1397]) +Dataloader batch - attn_mask shape: torch.Size([1, 1397]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1261 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1397) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1397) torch.int64 + loss_mask : 320 / 1397 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1397, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1397, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1397, 1, 576) torch.bfloat16 + out : (1, 1397) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2501 + top-1 accuracy : 29/320 = 9.1% + +Dataloader batch - tokens shape: torch.Size([1, 1281]) +Dataloader batch - labels shape: torch.Size([1, 1281]) +Dataloader batch - attn_mask shape: torch.Size([1, 1281]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1262 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1281) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1281) torch.int64 + loss_mask : 373 / 1281 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1281, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1281, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1281, 1, 576) torch.bfloat16 + out : (1, 1281) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.4225 + top-1 accuracy : 2/373 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1463]) +Dataloader batch - labels shape: torch.Size([1, 1463]) +Dataloader batch - attn_mask shape: torch.Size([1, 1463]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1263 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1463) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1463) torch.int64 + loss_mask : 312 / 1463 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1463, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1463, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1463, 1, 576) torch.bfloat16 + out : (1, 1463) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.1611 + top-1 accuracy : 16/312 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1694]) +Dataloader batch - labels shape: torch.Size([1, 1694]) +Dataloader batch - attn_mask shape: torch.Size([1, 1694]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1264 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1694) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1694) torch.int64 + loss_mask : 344 / 1694 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1694, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1694, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1694, 1, 576) torch.bfloat16 + out : (1, 1694) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.2334 + top-1 accuracy : 35/344 = 10.2% + + [2026-04-29 14:02:34] iteration 416/ 500 | consumed samples: 1664 | elapsed time per iteration (ms): 4721.0 | throughput (token/sec/GPU): 1236.0 | learning rate: 3.066115E-05 | global batch size: 4 | lm loss: 1.127291E+01 | loss scale: 1.0 | grad norm: 0.184 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1719]) +Dataloader batch - labels shape: torch.Size([1, 1719]) +Dataloader batch - attn_mask shape: torch.Size([1, 1719]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1265 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1719) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1719) torch.int64 + loss_mask : 362 / 1719 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1719, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1719, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1719, 1, 576) torch.bfloat16 + out : (1, 1719) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2075 + top-1 accuracy : 29/362 = 8.0% + +Dataloader batch - tokens shape: torch.Size([1, 1849]) +Dataloader batch - labels shape: torch.Size([1, 1849]) +Dataloader batch - attn_mask shape: torch.Size([1, 1849]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1266 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1849) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1849) torch.int64 + loss_mask : 488 / 1849 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1849, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1849, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1849, 1, 576) torch.bfloat16 + out : (1, 1849) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 488 tokens) : 11.2985 + top-1 accuracy : 15/488 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1302]) +Dataloader batch - labels shape: torch.Size([1, 1302]) +Dataloader batch - attn_mask shape: torch.Size([1, 1302]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1267 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1302) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1302) torch.int64 + loss_mask : 386 / 1302 tokens (29.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1302, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1302, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1302, 1, 576) torch.bfloat16 + out : (1, 1302) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3688 + top-1 accuracy : 8/386 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1550]) +Dataloader batch - labels shape: torch.Size([1, 1550]) +Dataloader batch - attn_mask shape: torch.Size([1, 1550]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1268 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1550) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1550) torch.int64 + loss_mask : 345 / 1550 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1550, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1550, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1550, 1, 576) torch.bfloat16 + out : (1, 1550) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.2416 + top-1 accuracy : 28/345 = 8.1% + + [2026-04-29 14:02:39] iteration 417/ 500 | consumed samples: 1668 | elapsed time per iteration (ms): 5063.0 | throughput (token/sec/GPU): 1268.0 | learning rate: 3.037603E-05 | global batch size: 4 | lm loss: 1.128241E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1540]) +Dataloader batch - labels shape: torch.Size([1, 1540]) +Dataloader batch - attn_mask shape: torch.Size([1, 1540]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1269 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1540) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1540) torch.int64 + loss_mask : 390 / 1540 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1540, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1540, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1540, 1, 576) torch.bfloat16 + out : (1, 1540) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.3332 + top-1 accuracy : 14/390 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1471]) +Dataloader batch - labels shape: torch.Size([1, 1471]) +Dataloader batch - attn_mask shape: torch.Size([1, 1471]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1270 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1471) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1471) torch.int64 + loss_mask : 279 / 1471 tokens (19.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1471, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1471, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1471, 1, 576) torch.bfloat16 + out : (1, 1471) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 279 tokens) : 11.1431 + top-1 accuracy : 32/279 = 11.5% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1271 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 322 / 1400 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 322 tokens) : 11.2330 + top-1 accuracy : 18/322 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1608]) +Dataloader batch - labels shape: torch.Size([1, 1608]) +Dataloader batch - attn_mask shape: torch.Size([1, 1608]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1272 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1608) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1608) torch.int64 + loss_mask : 347 / 1608 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1608, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1608, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1608, 1, 576) torch.bfloat16 + out : (1, 1608) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.1695 + top-1 accuracy : 23/347 = 6.6% + + [2026-04-29 14:02:44] iteration 418/ 500 | consumed samples: 1672 | elapsed time per iteration (ms): 4914.5 | throughput (token/sec/GPU): 1224.8 | learning rate: 3.009170E-05 | global batch size: 4 | lm loss: 1.122698E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1373]) +Dataloader batch - labels shape: torch.Size([1, 1373]) +Dataloader batch - attn_mask shape: torch.Size([1, 1373]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1273 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1373) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1373) torch.int64 + loss_mask : 402 / 1373 tokens (29.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1373, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1373, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1373, 1, 576) torch.bfloat16 + out : (1, 1373) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.3491 + top-1 accuracy : 7/402 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1710]) +Dataloader batch - labels shape: torch.Size([1, 1710]) +Dataloader batch - attn_mask shape: torch.Size([1, 1710]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1274 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1710) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1710) torch.int64 + loss_mask : 388 / 1710 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1710, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1710, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1710, 1, 576) torch.bfloat16 + out : (1, 1710) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2383 + top-1 accuracy : 31/388 = 8.0% + +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1275 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 337 / 1541 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.2122 + top-1 accuracy : 25/337 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1792]) +Dataloader batch - labels shape: torch.Size([1, 1792]) +Dataloader batch - attn_mask shape: torch.Size([1, 1792]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1276 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1792) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1792) torch.int64 + loss_mask : 419 / 1792 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1792, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1792, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1792, 1, 576) torch.bfloat16 + out : (1, 1792) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.2710 + top-1 accuracy : 20/419 = 4.8% + + [2026-04-29 14:02:49] iteration 419/ 500 | consumed samples: 1676 | elapsed time per iteration (ms): 5005.8 | throughput (token/sec/GPU): 1281.7 | learning rate: 2.980819E-05 | global batch size: 4 | lm loss: 1.127028E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1528]) +Dataloader batch - labels shape: torch.Size([1, 1528]) +Dataloader batch - attn_mask shape: torch.Size([1, 1528]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1277 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1528) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1528) torch.int64 + loss_mask : 374 / 1528 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1528, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1528, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1528, 1, 576) torch.bfloat16 + out : (1, 1528) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.2238 + top-1 accuracy : 8/374 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1864]) +Dataloader batch - labels shape: torch.Size([1, 1864]) +Dataloader batch - attn_mask shape: torch.Size([1, 1864]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1278 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1864) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1864) torch.int64 + loss_mask : 495 / 1864 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1864, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1864, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1864, 1, 576) torch.bfloat16 + out : (1, 1864) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 495 tokens) : 11.2980 + top-1 accuracy : 2/495 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1551]) +Dataloader batch - labels shape: torch.Size([1, 1551]) +Dataloader batch - attn_mask shape: torch.Size([1, 1551]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1279 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1551) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1551) torch.int64 + loss_mask : 491 / 1551 tokens (31.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1551, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1551, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1551, 1, 576) torch.bfloat16 + out : (1, 1551) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 491 tokens) : 11.4403 + top-1 accuracy : 4/491 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1764]) +Dataloader batch - labels shape: torch.Size([1, 1764]) +Dataloader batch - attn_mask shape: torch.Size([1, 1764]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1280 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1764) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1764) torch.int64 + loss_mask : 402 / 1764 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1764, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1764, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1764, 1, 576) torch.bfloat16 + out : (1, 1764) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.2704 + top-1 accuracy : 29/402 = 7.2% + + [2026-04-29 14:02:54] iteration 420/ 500 | consumed samples: 1680 | elapsed time per iteration (ms): 5045.4 | throughput (token/sec/GPU): 1329.3 | learning rate: 2.952549E-05 | global batch size: 4 | lm loss: 1.131562E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (5033.36, 5033.36) + forward-compute ................................: (3757.96, 3757.96) + backward-compute ...............................: (1273.46, 1273.46) + batch-generator ................................: (453.96, 453.96) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.45, 0.45) + params-all-gather ..............................: (0.14, 0.14) + optimizer-copy-to-main-grad ....................: (0.11, 0.11) + optimizer-clip-main-grad .......................: (0.46, 0.46) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.22, 1.22) + optimizer-copy-main-to-model-params ............: (0.19, 0.19) + optimizer ......................................: (2.69, 2.69) +Dataloader batch - tokens shape: torch.Size([1, 1242]) +Dataloader batch - labels shape: torch.Size([1, 1242]) +Dataloader batch - attn_mask shape: torch.Size([1, 1242]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1281 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1242) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1242) torch.int64 + loss_mask : 313 / 1242 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1242, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1242, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1242, 1, 576) torch.bfloat16 + out : (1, 1242) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2924 + top-1 accuracy : 11/313 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1233]) +Dataloader batch - labels shape: torch.Size([1, 1233]) +Dataloader batch - attn_mask shape: torch.Size([1, 1233]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1282 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1233) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1233) torch.int64 + loss_mask : 343 / 1233 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1233, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1233, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1233, 1, 576) torch.bfloat16 + out : (1, 1233) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.3454 + top-1 accuracy : 1/343 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1334]) +Dataloader batch - labels shape: torch.Size([1, 1334]) +Dataloader batch - attn_mask shape: torch.Size([1, 1334]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1283 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1334) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1334) torch.int64 + loss_mask : 372 / 1334 tokens (27.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1334, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1334, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1334, 1, 576) torch.bfloat16 + out : (1, 1334) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.3717 + top-1 accuracy : 3/372 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1284 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 343 / 1695 tokens (20.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.1660 + top-1 accuracy : 22/343 = 6.4% + + [2026-04-29 14:02:58] iteration 421/ 500 | consumed samples: 1684 | elapsed time per iteration (ms): 4387.8 | throughput (token/sec/GPU): 1254.4 | learning rate: 2.924363E-05 | global batch size: 4 | lm loss: 1.129555E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1435]) +Dataloader batch - labels shape: torch.Size([1, 1435]) +Dataloader batch - attn_mask shape: torch.Size([1, 1435]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1285 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1435) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1435) torch.int64 + loss_mask : 281 / 1435 tokens (19.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1435, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1435, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1435, 1, 576) torch.bfloat16 + out : (1, 1435) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 281 tokens) : 11.0995 + top-1 accuracy : 15/281 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1853]) +Dataloader batch - labels shape: torch.Size([1, 1853]) +Dataloader batch - attn_mask shape: torch.Size([1, 1853]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1286 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1853) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1853) torch.int64 + loss_mask : 485 / 1853 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1853, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1853, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1853, 1, 576) torch.bfloat16 + out : (1, 1853) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 485 tokens) : 11.3382 + top-1 accuracy : 12/485 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1357]) +Dataloader batch - labels shape: torch.Size([1, 1357]) +Dataloader batch - attn_mask shape: torch.Size([1, 1357]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1287 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1357) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1357) torch.int64 + loss_mask : 317 / 1357 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1357, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1357, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1357, 1, 576) torch.bfloat16 + out : (1, 1357) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2054 + top-1 accuracy : 10/317 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1103]) +Dataloader batch - labels shape: torch.Size([1, 1103]) +Dataloader batch - attn_mask shape: torch.Size([1, 1103]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1288 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1103) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1103) torch.int64 + loss_mask : 294 / 1103 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1103, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1103, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1103, 1, 576) torch.bfloat16 + out : (1, 1103) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.3303 + top-1 accuracy : 16/294 = 5.4% + + [2026-04-29 14:03:03] iteration 422/ 500 | consumed samples: 1688 | elapsed time per iteration (ms): 4663.6 | throughput (token/sec/GPU): 1232.5 | learning rate: 2.896261E-05 | global batch size: 4 | lm loss: 1.125723E+01 | loss scale: 1.0 | grad norm: 0.211 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 937]) +Dataloader batch - labels shape: torch.Size([1, 937]) +Dataloader batch - attn_mask shape: torch.Size([1, 937]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1289 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 937) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 937) torch.int64 + loss_mask : 207 / 937 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (937, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (937, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (937, 1, 576) torch.bfloat16 + out : (1, 937) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 207 tokens) : 11.2145 + top-1 accuracy : 6/207 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1152]) +Dataloader batch - labels shape: torch.Size([1, 1152]) +Dataloader batch - attn_mask shape: torch.Size([1, 1152]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1290 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1152) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1152) torch.int64 + loss_mask : 224 / 1152 tokens (19.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1152, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1152, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1152, 1, 576) torch.bfloat16 + out : (1, 1152) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 224 tokens) : 11.1899 + top-1 accuracy : 26/224 = 11.6% + +Dataloader batch - tokens shape: torch.Size([1, 1370]) +Dataloader batch - labels shape: torch.Size([1, 1370]) +Dataloader batch - attn_mask shape: torch.Size([1, 1370]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1291 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1370) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1370) torch.int64 + loss_mask : 329 / 1370 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1370, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1370, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1370, 1, 576) torch.bfloat16 + out : (1, 1370) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2509 + top-1 accuracy : 14/329 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1808]) +Dataloader batch - labels shape: torch.Size([1, 1808]) +Dataloader batch - attn_mask shape: torch.Size([1, 1808]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1292 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1808) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1808) torch.int64 + loss_mask : 450 / 1808 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1808, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1808, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1808, 1, 576) torch.bfloat16 + out : (1, 1808) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 450 tokens) : 11.2888 + top-1 accuracy : 26/450 = 5.8% + + [2026-04-29 14:03:07] iteration 423/ 500 | consumed samples: 1692 | elapsed time per iteration (ms): 4434.2 | throughput (token/sec/GPU): 1187.8 | learning rate: 2.868244E-05 | global batch size: 4 | lm loss: 1.124749E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1135]) +Dataloader batch - labels shape: torch.Size([1, 1135]) +Dataloader batch - attn_mask shape: torch.Size([1, 1135]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1293 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1135) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1135) torch.int64 + loss_mask : 218 / 1135 tokens (19.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1135, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1135, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1135, 1, 576) torch.bfloat16 + out : (1, 1135) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 218 tokens) : 11.1186 + top-1 accuracy : 14/218 = 6.4% + +Dataloader batch - tokens shape: torch.Size([1, 1630]) +Dataloader batch - labels shape: torch.Size([1, 1630]) +Dataloader batch - attn_mask shape: torch.Size([1, 1630]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1294 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1630) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1630) torch.int64 + loss_mask : 386 / 1630 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1630, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1630, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1630, 1, 576) torch.bfloat16 + out : (1, 1630) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2335 + top-1 accuracy : 37/386 = 9.6% + +Dataloader batch - tokens shape: torch.Size([1, 1439]) +Dataloader batch - labels shape: torch.Size([1, 1439]) +Dataloader batch - attn_mask shape: torch.Size([1, 1439]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1295 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1439) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1439) torch.int64 + loss_mask : 382 / 1439 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1439, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1439, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1439, 1, 576) torch.bfloat16 + out : (1, 1439) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.3062 + top-1 accuracy : 4/382 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 881]) +Dataloader batch - labels shape: torch.Size([1, 881]) +Dataloader batch - attn_mask shape: torch.Size([1, 881]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 1296 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 881) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 881) torch.int64 + loss_mask : 184 / 881 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (881, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (881, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (881, 1, 576) torch.bfloat16 + out : (1, 881) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 184 tokens) : 11.1598 + top-1 accuracy : 5/184 = 2.7% + + [2026-04-29 14:03:11] iteration 424/ 500 | consumed samples: 1696 | elapsed time per iteration (ms): 4266.0 | throughput (token/sec/GPU): 1192.0 | learning rate: 2.840314E-05 | global batch size: 4 | lm loss: 1.122425E+01 | loss scale: 1.0 | grad norm: 0.242 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1110]) +Dataloader batch - labels shape: torch.Size([1, 1110]) +Dataloader batch - attn_mask shape: torch.Size([1, 1110]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1297 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1110) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1110) torch.int64 + loss_mask : 267 / 1110 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1110, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1110, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1110, 1, 576) torch.bfloat16 + out : (1, 1110) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 267 tokens) : 11.2992 + top-1 accuracy : 22/267 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1356]) +Dataloader batch - labels shape: torch.Size([1, 1356]) +Dataloader batch - attn_mask shape: torch.Size([1, 1356]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1298 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1356) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1356) torch.int64 + loss_mask : 293 / 1356 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1356, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1356, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1356, 1, 576) torch.bfloat16 + out : (1, 1356) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 293 tokens) : 11.1909 + top-1 accuracy : 13/293 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1916]) +Dataloader batch - labels shape: torch.Size([1, 1916]) +Dataloader batch - attn_mask shape: torch.Size([1, 1916]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1299 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1916) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1916) torch.int64 + loss_mask : 552 / 1916 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1916, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1916, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1916, 1, 576) torch.bfloat16 + out : (1, 1916) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 552 tokens) : 11.4160 + top-1 accuracy : 16/552 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1650]) +Dataloader batch - labels shape: torch.Size([1, 1650]) +Dataloader batch - attn_mask shape: torch.Size([1, 1650]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1300 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1650) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1650) torch.int64 + loss_mask : 398 / 1650 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1650, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1650, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1650, 1, 576) torch.bfloat16 + out : (1, 1650) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.2504 + top-1 accuracy : 24/398 = 6.0% + + [2026-04-29 14:03:16] iteration 425/ 500 | consumed samples: 1700 | elapsed time per iteration (ms): 4809.6 | throughput (token/sec/GPU): 1254.2 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.130803E+01 | loss scale: 1.0 | grad norm: 0.202 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1417]) +Dataloader batch - labels shape: torch.Size([1, 1417]) +Dataloader batch - attn_mask shape: torch.Size([1, 1417]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1301 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1417) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1417) torch.int64 + loss_mask : 319 / 1417 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1417, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1417, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1417, 1, 576) torch.bfloat16 + out : (1, 1417) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.2741 + top-1 accuracy : 27/319 = 8.5% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1302 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 408 / 1532 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.3175 + top-1 accuracy : 5/408 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1827]) +Dataloader batch - labels shape: torch.Size([1, 1827]) +Dataloader batch - attn_mask shape: torch.Size([1, 1827]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1303 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1827) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1827) torch.int64 + loss_mask : 464 / 1827 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1827, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1827, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1827, 1, 576) torch.bfloat16 + out : (1, 1827) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 464 tokens) : 11.2728 + top-1 accuracy : 13/464 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1684]) +Dataloader batch - labels shape: torch.Size([1, 1684]) +Dataloader batch - attn_mask shape: torch.Size([1, 1684]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1304 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1684) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1684) torch.int64 + loss_mask : 366 / 1684 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1684, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1684, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1684, 1, 576) torch.bfloat16 + out : (1, 1684) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2749 + top-1 accuracy : 24/366 = 6.6% + + [2026-04-29 14:03:21] iteration 426/ 500 | consumed samples: 1704 | elapsed time per iteration (ms): 5227.7 | throughput (token/sec/GPU): 1235.7 | learning rate: 2.784717E-05 | global batch size: 4 | lm loss: 1.128528E+01 | loss scale: 1.0 | grad norm: 0.182 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1781]) +Dataloader batch - labels shape: torch.Size([1, 1781]) +Dataloader batch - attn_mask shape: torch.Size([1, 1781]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1305 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1781) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1781) torch.int64 + loss_mask : 413 / 1781 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1781, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1781, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1781, 1, 576) torch.bfloat16 + out : (1, 1781) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2344 + top-1 accuracy : 17/413 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1504]) +Dataloader batch - labels shape: torch.Size([1, 1504]) +Dataloader batch - attn_mask shape: torch.Size([1, 1504]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1306 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1504) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1504) torch.int64 + loss_mask : 368 / 1504 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1504, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1504, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1504, 1, 576) torch.bfloat16 + out : (1, 1504) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.3033 + top-1 accuracy : 21/368 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1111]) +Dataloader batch - labels shape: torch.Size([1, 1111]) +Dataloader batch - attn_mask shape: torch.Size([1, 1111]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1307 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1111) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1111) torch.int64 + loss_mask : 312 / 1111 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1111, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1111, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1111, 1, 576) torch.bfloat16 + out : (1, 1111) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 312 tokens) : 11.3480 + top-1 accuracy : 7/312 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1540]) +Dataloader batch - labels shape: torch.Size([1, 1540]) +Dataloader batch - attn_mask shape: torch.Size([1, 1540]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1308 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1540) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1540) torch.int64 + loss_mask : 339 / 1540 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1540, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1540, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1540, 1, 576) torch.bfloat16 + out : (1, 1540) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2064 + top-1 accuracy : 21/339 = 6.2% + + [2026-04-29 14:03:26] iteration 427/ 500 | consumed samples: 1708 | elapsed time per iteration (ms): 5039.6 | throughput (token/sec/GPU): 1177.9 | learning rate: 2.757053E-05 | global batch size: 4 | lm loss: 1.127025E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1924]) +Dataloader batch - labels shape: torch.Size([1, 1924]) +Dataloader batch - attn_mask shape: torch.Size([1, 1924]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1309 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1924) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1924) torch.int64 + loss_mask : 562 / 1924 tokens (29.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1924, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1924, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1924, 1, 576) torch.bfloat16 + out : (1, 1924) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 562 tokens) : 11.3833 + top-1 accuracy : 8/562 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1613]) +Dataloader batch - labels shape: torch.Size([1, 1613]) +Dataloader batch - attn_mask shape: torch.Size([1, 1613]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1310 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1613) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1613) torch.int64 + loss_mask : 395 / 1613 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1613, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1613, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1613, 1, 576) torch.bfloat16 + out : (1, 1613) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.2695 + top-1 accuracy : 13/395 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1044]) +Dataloader batch - labels shape: torch.Size([1, 1044]) +Dataloader batch - attn_mask shape: torch.Size([1, 1044]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1311 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1044) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1044) torch.int64 + loss_mask : 232 / 1044 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1044, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1044, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1044, 1, 576) torch.bfloat16 + out : (1, 1044) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 232 tokens) : 11.1593 + top-1 accuracy : 5/232 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1312 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 388 / 1444 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3242 + top-1 accuracy : 4/388 = 1.0% + + [2026-04-29 14:03:31] iteration 428/ 500 | consumed samples: 1712 | elapsed time per iteration (ms): 4740.4 | throughput (token/sec/GPU): 1271.0 | learning rate: 2.729480E-05 | global batch size: 4 | lm loss: 1.130728E+01 | loss scale: 1.0 | grad norm: 0.198 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1917]) +Dataloader batch - labels shape: torch.Size([1, 1917]) +Dataloader batch - attn_mask shape: torch.Size([1, 1917]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1313 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1917) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1917) torch.int64 + loss_mask : 537 / 1917 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1917, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1917, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1917, 1, 576) torch.bfloat16 + out : (1, 1917) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 537 tokens) : 11.3971 + top-1 accuracy : 5/537 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1435]) +Dataloader batch - labels shape: torch.Size([1, 1435]) +Dataloader batch - attn_mask shape: torch.Size([1, 1435]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1314 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1435) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1435) torch.int64 + loss_mask : 298 / 1435 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1435, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1435, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1435, 1, 576) torch.bfloat16 + out : (1, 1435) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 298 tokens) : 11.1441 + top-1 accuracy : 5/298 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1200]) +Dataloader batch - labels shape: torch.Size([1, 1200]) +Dataloader batch - attn_mask shape: torch.Size([1, 1200]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1315 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1200) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1200) torch.int64 + loss_mask : 268 / 1200 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1200, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1200, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1200, 1, 576) torch.bfloat16 + out : (1, 1200) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 268 tokens) : 11.2260 + top-1 accuracy : 15/268 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1217]) +Dataloader batch - labels shape: torch.Size([1, 1217]) +Dataloader batch - attn_mask shape: torch.Size([1, 1217]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1316 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1217) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1217) torch.int64 + loss_mask : 305 / 1217 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1217, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1217, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1217, 1, 576) torch.bfloat16 + out : (1, 1217) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 305 tokens) : 11.2873 + top-1 accuracy : 10/305 = 3.3% + + [2026-04-29 14:03:36] iteration 429/ 500 | consumed samples: 1716 | elapsed time per iteration (ms): 4680.2 | throughput (token/sec/GPU): 1232.6 | learning rate: 2.701998E-05 | global batch size: 4 | lm loss: 1.128722E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1905]) +Dataloader batch - labels shape: torch.Size([1, 1905]) +Dataloader batch - attn_mask shape: torch.Size([1, 1905]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1317 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1905) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1905) torch.int64 + loss_mask : 549 / 1905 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1905, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1905, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1905, 1, 576) torch.bfloat16 + out : (1, 1905) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 549 tokens) : 11.3607 + top-1 accuracy : 14/549 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1586]) +Dataloader batch - labels shape: torch.Size([1, 1586]) +Dataloader batch - attn_mask shape: torch.Size([1, 1586]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1318 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1586) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1586) torch.int64 + loss_mask : 448 / 1586 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1586, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1586, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1586, 1, 576) torch.bfloat16 + out : (1, 1586) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 448 tokens) : 11.3437 + top-1 accuracy : 3/448 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1847]) +Dataloader batch - labels shape: torch.Size([1, 1847]) +Dataloader batch - attn_mask shape: torch.Size([1, 1847]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1319 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1847) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1847) torch.int64 + loss_mask : 494 / 1847 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1847, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1847, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1847, 1, 576) torch.bfloat16 + out : (1, 1847) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 494 tokens) : 11.3236 + top-1 accuracy : 3/494 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1766]) +Dataloader batch - labels shape: torch.Size([1, 1766]) +Dataloader batch - attn_mask shape: torch.Size([1, 1766]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1320 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1766) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1766) torch.int64 + loss_mask : 413 / 1766 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1766, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1766, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1766, 1, 576) torch.bfloat16 + out : (1, 1766) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2481 + top-1 accuracy : 12/413 = 2.9% + + [2026-04-29 14:03:41] iteration 430/ 500 | consumed samples: 1720 | elapsed time per iteration (ms): 5546.9 | throughput (token/sec/GPU): 1280.7 | learning rate: 2.674610E-05 | global batch size: 4 | lm loss: 1.132265E+01 | loss scale: 1.0 | grad norm: 0.184 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1805]) +Dataloader batch - labels shape: torch.Size([1, 1805]) +Dataloader batch - attn_mask shape: torch.Size([1, 1805]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1321 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1805) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1805) torch.int64 + loss_mask : 435 / 1805 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1805, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1805, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1805, 1, 576) torch.bfloat16 + out : (1, 1805) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 435 tokens) : 11.2751 + top-1 accuracy : 7/435 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1776]) +Dataloader batch - labels shape: torch.Size([1, 1776]) +Dataloader batch - attn_mask shape: torch.Size([1, 1776]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1322 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1776) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1776) torch.int64 + loss_mask : 443 / 1776 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1776, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1776, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1776, 1, 576) torch.bfloat16 + out : (1, 1776) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 443 tokens) : 11.3107 + top-1 accuracy : 7/443 = 1.6% + +Dataloader batch - tokens shape: torch.Size([1, 1002]) +Dataloader batch - labels shape: torch.Size([1, 1002]) +Dataloader batch - attn_mask shape: torch.Size([1, 1002]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1323 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1002) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1002) torch.int64 + loss_mask : 233 / 1002 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1002, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1002, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1002, 1, 576) torch.bfloat16 + out : (1, 1002) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 233 tokens) : 11.3011 + top-1 accuracy : 12/233 = 5.2% + +Dataloader batch - tokens shape: torch.Size([1, 1103]) +Dataloader batch - labels shape: torch.Size([1, 1103]) +Dataloader batch - attn_mask shape: torch.Size([1, 1103]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1324 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1103) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1103) torch.int64 + loss_mask : 334 / 1103 tokens (30.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1103, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1103, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1103, 1, 576) torch.bfloat16 + out : (1, 1103) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.3979 + top-1 accuracy : 0/334 = 0.0% + + [2026-04-29 14:03:46] iteration 431/ 500 | consumed samples: 1724 | elapsed time per iteration (ms): 4424.3 | throughput (token/sec/GPU): 1285.2 | learning rate: 2.647316E-05 | global batch size: 4 | lm loss: 1.131859E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1514]) +Dataloader batch - labels shape: torch.Size([1, 1514]) +Dataloader batch - attn_mask shape: torch.Size([1, 1514]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1325 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1514) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1514) torch.int64 + loss_mask : 356 / 1514 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1514, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1514, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1514, 1, 576) torch.bfloat16 + out : (1, 1514) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2141 + top-1 accuracy : 4/356 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1412]) +Dataloader batch - labels shape: torch.Size([1, 1412]) +Dataloader batch - attn_mask shape: torch.Size([1, 1412]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1326 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1412) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1412) torch.int64 + loss_mask : 347 / 1412 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1412, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1412, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1412, 1, 576) torch.bfloat16 + out : (1, 1412) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2687 + top-1 accuracy : 9/347 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1066]) +Dataloader batch - labels shape: torch.Size([1, 1066]) +Dataloader batch - attn_mask shape: torch.Size([1, 1066]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1327 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1066) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1066) torch.int64 + loss_mask : 262 / 1066 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1066, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1066, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1066, 1, 576) torch.bfloat16 + out : (1, 1066) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 262 tokens) : 11.3247 + top-1 accuracy : 1/262 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1222]) +Dataloader batch - labels shape: torch.Size([1, 1222]) +Dataloader batch - attn_mask shape: torch.Size([1, 1222]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1328 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1222) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1222) torch.int64 + loss_mask : 279 / 1222 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1222, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1222, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1222, 1, 576) torch.bfloat16 + out : (1, 1222) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 279 tokens) : 11.2352 + top-1 accuracy : 8/279 = 2.9% + + [2026-04-29 14:03:50] iteration 432/ 500 | consumed samples: 1728 | elapsed time per iteration (ms): 4430.1 | throughput (token/sec/GPU): 1176.9 | learning rate: 2.620117E-05 | global batch size: 4 | lm loss: 1.125733E+01 | loss scale: 1.0 | grad norm: 0.175 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1583]) +Dataloader batch - labels shape: torch.Size([1, 1583]) +Dataloader batch - attn_mask shape: torch.Size([1, 1583]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1329 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1583) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1583) torch.int64 + loss_mask : 360 / 1583 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1583, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1583, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1583, 1, 576) torch.bfloat16 + out : (1, 1583) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.1905 + top-1 accuracy : 27/360 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1785]) +Dataloader batch - labels shape: torch.Size([1, 1785]) +Dataloader batch - attn_mask shape: torch.Size([1, 1785]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1330 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1785) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1785) torch.int64 + loss_mask : 473 / 1785 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1785, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1785, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1785, 1, 576) torch.bfloat16 + out : (1, 1785) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 473 tokens) : 11.3478 + top-1 accuracy : 7/473 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1233]) +Dataloader batch - labels shape: torch.Size([1, 1233]) +Dataloader batch - attn_mask shape: torch.Size([1, 1233]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1331 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1233) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1233) torch.int64 + loss_mask : 295 / 1233 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1233, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1233, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1233, 1, 576) torch.bfloat16 + out : (1, 1233) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.2625 + top-1 accuracy : 14/295 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1006]) +Dataloader batch - labels shape: torch.Size([1, 1006]) +Dataloader batch - attn_mask shape: torch.Size([1, 1006]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1332 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1006) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1006) torch.int64 + loss_mask : 251 / 1006 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1006, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1006, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1006, 1, 576) torch.bfloat16 + out : (1, 1006) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2345 + top-1 accuracy : 6/251 = 2.4% + + [2026-04-29 14:03:55] iteration 433/ 500 | consumed samples: 1732 | elapsed time per iteration (ms): 4589.8 | throughput (token/sec/GPU): 1221.6 | learning rate: 2.593014E-05 | global batch size: 4 | lm loss: 1.126782E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1001]) +Dataloader batch - labels shape: torch.Size([1, 1001]) +Dataloader batch - attn_mask shape: torch.Size([1, 1001]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1333 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1001) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1001) torch.int64 + loss_mask : 251 / 1001 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1001, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1001, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1001, 1, 576) torch.bfloat16 + out : (1, 1001) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2821 + top-1 accuracy : 1/251 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1694]) +Dataloader batch - labels shape: torch.Size([1, 1694]) +Dataloader batch - attn_mask shape: torch.Size([1, 1694]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1334 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1694) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1694) torch.int64 + loss_mask : 356 / 1694 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1694, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1694, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1694, 1, 576) torch.bfloat16 + out : (1, 1694) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2059 + top-1 accuracy : 8/356 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1499]) +Dataloader batch - labels shape: torch.Size([1, 1499]) +Dataloader batch - attn_mask shape: torch.Size([1, 1499]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1335 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1499) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1499) torch.int64 + loss_mask : 342 / 1499 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1499, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1499, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1499, 1, 576) torch.bfloat16 + out : (1, 1499) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.2006 + top-1 accuracy : 5/342 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1514]) +Dataloader batch - labels shape: torch.Size([1, 1514]) +Dataloader batch - attn_mask shape: torch.Size([1, 1514]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1336 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1514) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1514) torch.int64 + loss_mask : 367 / 1514 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1514, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1514, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1514, 1, 576) torch.bfloat16 + out : (1, 1514) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.2589 + top-1 accuracy : 8/367 = 2.2% + + [2026-04-29 14:03:59] iteration 434/ 500 | consumed samples: 1736 | elapsed time per iteration (ms): 4563.2 | throughput (token/sec/GPU): 1250.9 | learning rate: 2.566009E-05 | global batch size: 4 | lm loss: 1.123382E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1395]) +Dataloader batch - labels shape: torch.Size([1, 1395]) +Dataloader batch - attn_mask shape: torch.Size([1, 1395]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1337 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1395) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1395) torch.int64 + loss_mask : 415 / 1395 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1395, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1395, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1395, 1, 576) torch.bfloat16 + out : (1, 1395) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.3703 + top-1 accuracy : 0/415 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1472]) +Dataloader batch - labels shape: torch.Size([1, 1472]) +Dataloader batch - attn_mask shape: torch.Size([1, 1472]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1338 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1472) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1472) torch.int64 + loss_mask : 412 / 1472 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1472, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1472, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1472, 1, 576) torch.bfloat16 + out : (1, 1472) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 412 tokens) : 11.3361 + top-1 accuracy : 6/412 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1725]) +Dataloader batch - labels shape: torch.Size([1, 1725]) +Dataloader batch - attn_mask shape: torch.Size([1, 1725]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1339 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1725) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1725) torch.int64 + loss_mask : 372 / 1725 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1725, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1725, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1725, 1, 576) torch.bfloat16 + out : (1, 1725) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.1828 + top-1 accuracy : 25/372 = 6.7% + +Dataloader batch - tokens shape: torch.Size([1, 1480]) +Dataloader batch - labels shape: torch.Size([1, 1480]) +Dataloader batch - attn_mask shape: torch.Size([1, 1480]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1340 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1480) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1480) torch.int64 + loss_mask : 335 / 1480 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1480, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1480, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1480, 1, 576) torch.bfloat16 + out : (1, 1480) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2012 + top-1 accuracy : 5/335 = 1.5% + + [2026-04-29 14:04:04] iteration 435/ 500 | consumed samples: 1740 | elapsed time per iteration (ms): 4710.4 | throughput (token/sec/GPU): 1289.1 | learning rate: 2.539102E-05 | global batch size: 4 | lm loss: 1.127870E+01 | loss scale: 1.0 | grad norm: 0.179 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1188]) +Dataloader batch - labels shape: torch.Size([1, 1188]) +Dataloader batch - attn_mask shape: torch.Size([1, 1188]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1341 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1188) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1188) torch.int64 + loss_mask : 279 / 1188 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1188, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1188, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1188, 1, 576) torch.bfloat16 + out : (1, 1188) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 279 tokens) : 11.2429 + top-1 accuracy : 10/279 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1433]) +Dataloader batch - labels shape: torch.Size([1, 1433]) +Dataloader batch - attn_mask shape: torch.Size([1, 1433]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1342 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1433) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1433) torch.int64 + loss_mask : 327 / 1433 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1433, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1433, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1433, 1, 576) torch.bfloat16 + out : (1, 1433) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.2387 + top-1 accuracy : 6/327 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1423]) +Dataloader batch - labels shape: torch.Size([1, 1423]) +Dataloader batch - attn_mask shape: torch.Size([1, 1423]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1343 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1423) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1423) torch.int64 + loss_mask : 370 / 1423 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1423, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1423, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1423, 1, 576) torch.bfloat16 + out : (1, 1423) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.2671 + top-1 accuracy : 2/370 = 0.5% + +Dataloader batch - tokens shape: torch.Size([1, 1469]) +Dataloader batch - labels shape: torch.Size([1, 1469]) +Dataloader batch - attn_mask shape: torch.Size([1, 1469]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1344 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1469) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1469) torch.int64 + loss_mask : 314 / 1469 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1469, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1469, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1469, 1, 576) torch.bfloat16 + out : (1, 1469) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.1466 + top-1 accuracy : 5/314 = 1.6% + + [2026-04-29 14:04:09] iteration 436/ 500 | consumed samples: 1744 | elapsed time per iteration (ms): 4608.7 | throughput (token/sec/GPU): 1196.2 | learning rate: 2.512295E-05 | global batch size: 4 | lm loss: 1.122534E+01 | loss scale: 1.0 | grad norm: 0.209 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1207]) +Dataloader batch - labels shape: torch.Size([1, 1207]) +Dataloader batch - attn_mask shape: torch.Size([1, 1207]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1345 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1207) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1207) torch.int64 + loss_mask : 268 / 1207 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1207, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1207, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1207, 1, 576) torch.bfloat16 + out : (1, 1207) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 268 tokens) : 11.2522 + top-1 accuracy : 4/268 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1346 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 351 / 1508 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 351 tokens) : 11.2228 + top-1 accuracy : 18/351 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1846]) +Dataloader batch - labels shape: torch.Size([1, 1846]) +Dataloader batch - attn_mask shape: torch.Size([1, 1846]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1347 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1846) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1846) torch.int64 + loss_mask : 520 / 1846 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1846, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1846, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1846, 1, 576) torch.bfloat16 + out : (1, 1846) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 520 tokens) : 11.3516 + top-1 accuracy : 12/520 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 997]) +Dataloader batch - labels shape: torch.Size([1, 997]) +Dataloader batch - attn_mask shape: torch.Size([1, 997]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1348 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 997) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 997) torch.int64 + loss_mask : 239 / 997 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (997, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (997, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (997, 1, 576) torch.bfloat16 + out : (1, 997) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 239 tokens) : 11.2699 + top-1 accuracy : 6/239 = 2.5% + + [2026-04-29 14:04:13] iteration 437/ 500 | consumed samples: 1748 | elapsed time per iteration (ms): 4445.3 | throughput (token/sec/GPU): 1250.3 | learning rate: 2.485589E-05 | global batch size: 4 | lm loss: 1.128531E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1428]) +Dataloader batch - labels shape: torch.Size([1, 1428]) +Dataloader batch - attn_mask shape: torch.Size([1, 1428]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1349 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1428) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1428) torch.int64 + loss_mask : 370 / 1428 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1428, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1428, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1428, 1, 576) torch.bfloat16 + out : (1, 1428) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 370 tokens) : 11.2992 + top-1 accuracy : 5/370 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1484]) +Dataloader batch - labels shape: torch.Size([1, 1484]) +Dataloader batch - attn_mask shape: torch.Size([1, 1484]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1350 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1484) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1484) torch.int64 + loss_mask : 338 / 1484 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1484, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1484, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1484, 1, 576) torch.bfloat16 + out : (1, 1484) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.1783 + top-1 accuracy : 13/338 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1868]) +Dataloader batch - labels shape: torch.Size([1, 1868]) +Dataloader batch - attn_mask shape: torch.Size([1, 1868]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1351 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1868) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1868) torch.int64 + loss_mask : 510 / 1868 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1868, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1868, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1868, 1, 576) torch.bfloat16 + out : (1, 1868) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 510 tokens) : 11.3413 + top-1 accuracy : 12/510 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1457]) +Dataloader batch - labels shape: torch.Size([1, 1457]) +Dataloader batch - attn_mask shape: torch.Size([1, 1457]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1352 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1457) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1457) torch.int64 + loss_mask : 313 / 1457 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1457, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1457, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1457, 1, 576) torch.bfloat16 + out : (1, 1457) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.1786 + top-1 accuracy : 9/313 = 2.9% + + [2026-04-29 14:04:18] iteration 438/ 500 | consumed samples: 1752 | elapsed time per iteration (ms): 4936.9 | throughput (token/sec/GPU): 1263.4 | learning rate: 2.458984E-05 | global batch size: 4 | lm loss: 1.126189E+01 | loss scale: 1.0 | grad norm: 0.198 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1435]) +Dataloader batch - labels shape: torch.Size([1, 1435]) +Dataloader batch - attn_mask shape: torch.Size([1, 1435]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1353 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1435) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1435) torch.int64 + loss_mask : 404 / 1435 tokens (28.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1435, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1435, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1435, 1, 576) torch.bfloat16 + out : (1, 1435) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 404 tokens) : 11.3252 + top-1 accuracy : 8/404 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1425]) +Dataloader batch - labels shape: torch.Size([1, 1425]) +Dataloader batch - attn_mask shape: torch.Size([1, 1425]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1354 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1425) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1425) torch.int64 + loss_mask : 331 / 1425 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1425, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1425, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1425, 1, 576) torch.bfloat16 + out : (1, 1425) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.2847 + top-1 accuracy : 19/331 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1387]) +Dataloader batch - labels shape: torch.Size([1, 1387]) +Dataloader batch - attn_mask shape: torch.Size([1, 1387]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1355 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1387) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1387) torch.int64 + loss_mask : 291 / 1387 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1387, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1387, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1387, 1, 576) torch.bfloat16 + out : (1, 1387) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 291 tokens) : 11.1565 + top-1 accuracy : 25/291 = 8.6% + +Dataloader batch - tokens shape: torch.Size([1, 1400]) +Dataloader batch - labels shape: torch.Size([1, 1400]) +Dataloader batch - attn_mask shape: torch.Size([1, 1400]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1356 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1400) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1400) torch.int64 + loss_mask : 295 / 1400 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1400, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1400, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1400, 1, 576) torch.bfloat16 + out : (1, 1400) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.1666 + top-1 accuracy : 24/295 = 8.1% + + [2026-04-29 14:04:23] iteration 439/ 500 | consumed samples: 1756 | elapsed time per iteration (ms): 4616.0 | throughput (token/sec/GPU): 1223.3 | learning rate: 2.432481E-05 | global batch size: 4 | lm loss: 1.124248E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1403]) +Dataloader batch - labels shape: torch.Size([1, 1403]) +Dataloader batch - attn_mask shape: torch.Size([1, 1403]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1357 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1403) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1403) torch.int64 + loss_mask : 348 / 1403 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1403, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1403, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1403, 1, 576) torch.bfloat16 + out : (1, 1403) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2533 + top-1 accuracy : 12/348 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 958]) +Dataloader batch - labels shape: torch.Size([1, 958]) +Dataloader batch - attn_mask shape: torch.Size([1, 958]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1358 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 958) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 958) torch.int64 + loss_mask : 207 / 958 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (958, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (958, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (958, 1, 576) torch.bfloat16 + out : (1, 958) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 207 tokens) : 11.1995 + top-1 accuracy : 8/207 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1359 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1519) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1519) torch.int64 + loss_mask : 317 / 1519 tokens (20.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1519, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1519, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1519, 1, 576) torch.bfloat16 + out : (1, 1519) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.1548 + top-1 accuracy : 24/317 = 7.6% + +Dataloader batch - tokens shape: torch.Size([1, 1505]) +Dataloader batch - labels shape: torch.Size([1, 1505]) +Dataloader batch - attn_mask shape: torch.Size([1, 1505]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1360 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1505) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1505) torch.int64 + loss_mask : 349 / 1505 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1505, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1505, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1505, 1, 576) torch.bfloat16 + out : (1, 1505) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2082 + top-1 accuracy : 20/349 = 5.7% + + [2026-04-29 14:04:27] iteration 440/ 500 | consumed samples: 1760 | elapsed time per iteration (ms): 4528.2 | throughput (token/sec/GPU): 1189.2 | learning rate: 2.406083E-05 | global batch size: 4 | lm loss: 1.120570E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4504.86, 4504.86) + forward-compute ................................: (3331.60, 3331.60) + backward-compute ...............................: (1169.11, 1169.11) + batch-generator ................................: (445.92, 445.92) + layernorm-grads-all-reduce .....................: (0.06, 0.06) + embedding-grads-all-reduce .....................: (0.10, 0.10) + all-grads-sync .................................: (1.17, 1.17) + params-all-gather ..............................: (0.42, 0.42) + optimizer-copy-to-main-grad ....................: (0.36, 0.36) + optimizer-clip-main-grad .......................: (1.39, 1.39) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.35, 2.35) + optimizer-copy-main-to-model-params ............: (0.59, 0.59) + optimizer ......................................: (7.19, 7.19) +Dataloader batch - tokens shape: torch.Size([1, 1877]) +Dataloader batch - labels shape: torch.Size([1, 1877]) +Dataloader batch - attn_mask shape: torch.Size([1, 1877]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1361 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1877) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1877) torch.int64 + loss_mask : 522 / 1877 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1877, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1877, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1877, 1, 576) torch.bfloat16 + out : (1, 1877) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 522 tokens) : 11.3629 + top-1 accuracy : 4/522 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1403]) +Dataloader batch - labels shape: torch.Size([1, 1403]) +Dataloader batch - attn_mask shape: torch.Size([1, 1403]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1362 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1403) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1403) torch.int64 + loss_mask : 329 / 1403 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1403, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1403, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1403, 1, 576) torch.bfloat16 + out : (1, 1403) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2216 + top-1 accuracy : 19/329 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1519]) +Dataloader batch - labels shape: torch.Size([1, 1519]) +Dataloader batch - attn_mask shape: torch.Size([1, 1519]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1363 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1519) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1519) torch.int64 + loss_mask : 380 / 1519 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1519, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1519, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1519, 1, 576) torch.bfloat16 + out : (1, 1519) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 380 tokens) : 11.2969 + top-1 accuracy : 9/380 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1569]) +Dataloader batch - labels shape: torch.Size([1, 1569]) +Dataloader batch - attn_mask shape: torch.Size([1, 1569]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1364 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1569) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1569) torch.int64 + loss_mask : 431 / 1569 tokens (27.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1569, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1569, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1569, 1, 576) torch.bfloat16 + out : (1, 1569) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 431 tokens) : 11.3617 + top-1 accuracy : 21/431 = 4.9% + + [2026-04-29 14:04:32] iteration 441/ 500 | consumed samples: 1764 | elapsed time per iteration (ms): 5054.3 | throughput (token/sec/GPU): 1259.9 | learning rate: 2.379789E-05 | global batch size: 4 | lm loss: 1.131950E+01 | loss scale: 1.0 | grad norm: 0.186 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1365 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 313 / 1475 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2278 + top-1 accuracy : 15/313 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1366 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 441 / 1487 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 441 tokens) : 11.3917 + top-1 accuracy : 8/441 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1098]) +Dataloader batch - labels shape: torch.Size([1, 1098]) +Dataloader batch - attn_mask shape: torch.Size([1, 1098]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1367 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1098) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1098) torch.int64 + loss_mask : 234 / 1098 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1098, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1098, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1098, 1, 576) torch.bfloat16 + out : (1, 1098) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 234 tokens) : 11.2319 + top-1 accuracy : 10/234 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1454]) +Dataloader batch - labels shape: torch.Size([1, 1454]) +Dataloader batch - attn_mask shape: torch.Size([1, 1454]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1368 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1454) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1454) torch.int64 + loss_mask : 309 / 1454 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1454, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1454, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1454, 1, 576) torch.bfloat16 + out : (1, 1454) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.1270 + top-1 accuracy : 18/309 = 5.8% + + [2026-04-29 14:04:37] iteration 442/ 500 | consumed samples: 1768 | elapsed time per iteration (ms): 4553.5 | throughput (token/sec/GPU): 1210.9 | learning rate: 2.353601E-05 | global batch size: 4 | lm loss: 1.126023E+01 | loss scale: 1.0 | grad norm: 0.180 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1402]) +Dataloader batch - labels shape: torch.Size([1, 1402]) +Dataloader batch - attn_mask shape: torch.Size([1, 1402]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1369 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1402) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1402) torch.int64 + loss_mask : 422 / 1402 tokens (30.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1402, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1402, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1402, 1, 576) torch.bfloat16 + out : (1, 1402) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 422 tokens) : 11.4165 + top-1 accuracy : 8/422 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1266]) +Dataloader batch - labels shape: torch.Size([1, 1266]) +Dataloader batch - attn_mask shape: torch.Size([1, 1266]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1370 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1266) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1266) torch.int64 + loss_mask : 327 / 1266 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1266, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1266, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1266, 1, 576) torch.bfloat16 + out : (1, 1266) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 327 tokens) : 11.3463 + top-1 accuracy : 1/327 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1466]) +Dataloader batch - labels shape: torch.Size([1, 1466]) +Dataloader batch - attn_mask shape: torch.Size([1, 1466]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1371 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1466) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1466) torch.int64 + loss_mask : 309 / 1466 tokens (21.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1466, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1466, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1466, 1, 576) torch.bfloat16 + out : (1, 1466) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.2182 + top-1 accuracy : 29/309 = 9.4% + +Dataloader batch - tokens shape: torch.Size([1, 1606]) +Dataloader batch - labels shape: torch.Size([1, 1606]) +Dataloader batch - attn_mask shape: torch.Size([1, 1606]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1372 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1606) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1606) torch.int64 + loss_mask : 382 / 1606 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1606, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1606, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1606, 1, 576) torch.bfloat16 + out : (1, 1606) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.2767 + top-1 accuracy : 12/382 = 3.1% + + [2026-04-29 14:04:41] iteration 443/ 500 | consumed samples: 1772 | elapsed time per iteration (ms): 4578.1 | throughput (token/sec/GPU): 1253.8 | learning rate: 2.327520E-05 | global batch size: 4 | lm loss: 1.132092E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1841]) +Dataloader batch - labels shape: torch.Size([1, 1841]) +Dataloader batch - attn_mask shape: torch.Size([1, 1841]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1373 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1841) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1841) torch.int64 + loss_mask : 488 / 1841 tokens (26.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1841, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1841, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1841, 1, 576) torch.bfloat16 + out : (1, 1841) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 488 tokens) : 11.3362 + top-1 accuracy : 6/488 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1182]) +Dataloader batch - labels shape: torch.Size([1, 1182]) +Dataloader batch - attn_mask shape: torch.Size([1, 1182]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1374 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1182) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1182) torch.int64 + loss_mask : 292 / 1182 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1182, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1182, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1182, 1, 576) torch.bfloat16 + out : (1, 1182) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 292 tokens) : 11.2717 + top-1 accuracy : 4/292 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1038]) +Dataloader batch - labels shape: torch.Size([1, 1038]) +Dataloader batch - attn_mask shape: torch.Size([1, 1038]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1375 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1038) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1038) torch.int64 + loss_mask : 266 / 1038 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1038, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1038, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1038, 1, 576) torch.bfloat16 + out : (1, 1038) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 266 tokens) : 11.3302 + top-1 accuracy : 2/266 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1509]) +Dataloader batch - labels shape: torch.Size([1, 1509]) +Dataloader batch - attn_mask shape: torch.Size([1, 1509]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1376 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1509) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1509) torch.int64 + loss_mask : 388 / 1509 tokens (25.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1509, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1509, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1509, 1, 576) torch.bfloat16 + out : (1, 1509) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3189 + top-1 accuracy : 19/388 = 4.9% + + [2026-04-29 14:04:46] iteration 444/ 500 | consumed samples: 1776 | elapsed time per iteration (ms): 4381.2 | throughput (token/sec/GPU): 1271.3 | learning rate: 2.301547E-05 | global batch size: 4 | lm loss: 1.131724E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1617]) +Dataloader batch - labels shape: torch.Size([1, 1617]) +Dataloader batch - attn_mask shape: torch.Size([1, 1617]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1377 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1617) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1617) torch.int64 + loss_mask : 381 / 1617 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1617, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1617, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1617, 1, 576) torch.bfloat16 + out : (1, 1617) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 381 tokens) : 11.3039 + top-1 accuracy : 21/381 = 5.5% + +Dataloader batch - tokens shape: torch.Size([1, 1848]) +Dataloader batch - labels shape: torch.Size([1, 1848]) +Dataloader batch - attn_mask shape: torch.Size([1, 1848]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1378 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1848) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1848) torch.int64 + loss_mask : 492 / 1848 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1848, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1848, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1848, 1, 576) torch.bfloat16 + out : (1, 1848) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 492 tokens) : 11.3397 + top-1 accuracy : 14/492 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1726]) +Dataloader batch - labels shape: torch.Size([1, 1726]) +Dataloader batch - attn_mask shape: torch.Size([1, 1726]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1379 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1726) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1726) torch.int64 + loss_mask : 418 / 1726 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1726, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1726, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1726, 1, 576) torch.bfloat16 + out : (1, 1726) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 418 tokens) : 11.3025 + top-1 accuracy : 8/418 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1502]) +Dataloader batch - labels shape: torch.Size([1, 1502]) +Dataloader batch - attn_mask shape: torch.Size([1, 1502]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1380 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1502) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1502) torch.int64 + loss_mask : 402 / 1502 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1502, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1502, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1502, 1, 576) torch.bfloat16 + out : (1, 1502) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.3124 + top-1 accuracy : 2/402 = 0.5% + + [2026-04-29 14:04:51] iteration 445/ 500 | consumed samples: 1780 | elapsed time per iteration (ms): 5260.9 | throughput (token/sec/GPU): 1272.2 | learning rate: 2.275683E-05 | global batch size: 4 | lm loss: 1.131599E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1454]) +Dataloader batch - labels shape: torch.Size([1, 1454]) +Dataloader batch - attn_mask shape: torch.Size([1, 1454]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1381 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1454) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1454) torch.int64 + loss_mask : 402 / 1454 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1454, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1454, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1454, 1, 576) torch.bfloat16 + out : (1, 1454) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.3463 + top-1 accuracy : 0/402 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1052]) +Dataloader batch - labels shape: torch.Size([1, 1052]) +Dataloader batch - attn_mask shape: torch.Size([1, 1052]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1382 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1052) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1052) torch.int64 + loss_mask : 227 / 1052 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1052, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1052, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1052, 1, 576) torch.bfloat16 + out : (1, 1052) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 227 tokens) : 11.1334 + top-1 accuracy : 6/227 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 997]) +Dataloader batch - labels shape: torch.Size([1, 997]) +Dataloader batch - attn_mask shape: torch.Size([1, 997]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1383 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 997) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 997) torch.int64 + loss_mask : 222 / 997 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (997, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (997, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (997, 1, 576) torch.bfloat16 + out : (1, 997) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 222 tokens) : 11.2523 + top-1 accuracy : 7/222 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1103]) +Dataloader batch - labels shape: torch.Size([1, 1103]) +Dataloader batch - attn_mask shape: torch.Size([1, 1103]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1384 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1103) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1103) torch.int64 + loss_mask : 253 / 1103 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1103, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1103, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1103, 1, 576) torch.bfloat16 + out : (1, 1103) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 253 tokens) : 11.2546 + top-1 accuracy : 11/253 = 4.3% + + [2026-04-29 14:04:55] iteration 446/ 500 | consumed samples: 1784 | elapsed time per iteration (ms): 4046.3 | throughput (token/sec/GPU): 1138.3 | learning rate: 2.249929E-05 | global batch size: 4 | lm loss: 1.126260E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1214]) +Dataloader batch - labels shape: torch.Size([1, 1214]) +Dataloader batch - attn_mask shape: torch.Size([1, 1214]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1385 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1214) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1214) torch.int64 + loss_mask : 300 / 1214 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1214, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1214, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1214, 1, 576) torch.bfloat16 + out : (1, 1214) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 300 tokens) : 11.3009 + top-1 accuracy : 2/300 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1360]) +Dataloader batch - labels shape: torch.Size([1, 1360]) +Dataloader batch - attn_mask shape: torch.Size([1, 1360]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1386 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1360) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1360) torch.int64 + loss_mask : 388 / 1360 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1360, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1360, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1360, 1, 576) torch.bfloat16 + out : (1, 1360) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3155 + top-1 accuracy : 6/388 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1006]) +Dataloader batch - labels shape: torch.Size([1, 1006]) +Dataloader batch - attn_mask shape: torch.Size([1, 1006]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1387 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1006) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1006) torch.int64 + loss_mask : 239 / 1006 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1006, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1006, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1006, 1, 576) torch.bfloat16 + out : (1, 1006) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 239 tokens) : 11.2749 + top-1 accuracy : 1/239 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1823]) +Dataloader batch - labels shape: torch.Size([1, 1823]) +Dataloader batch - attn_mask shape: torch.Size([1, 1823]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1388 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1823) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1823) torch.int64 + loss_mask : 471 / 1823 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1823, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1823, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1823, 1, 576) torch.bfloat16 + out : (1, 1823) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 471 tokens) : 11.3572 + top-1 accuracy : 8/471 = 1.7% + + [2026-04-29 14:05:00] iteration 447/ 500 | consumed samples: 1788 | elapsed time per iteration (ms): 4337.4 | throughput (token/sec/GPU): 1245.7 | learning rate: 2.224286E-05 | global batch size: 4 | lm loss: 1.131952E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1260]) +Dataloader batch - labels shape: torch.Size([1, 1260]) +Dataloader batch - attn_mask shape: torch.Size([1, 1260]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1389 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1260) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1260) torch.int64 + loss_mask : 330 / 1260 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1260, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1260, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1260, 1, 576) torch.bfloat16 + out : (1, 1260) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.3008 + top-1 accuracy : 6/330 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1386]) +Dataloader batch - labels shape: torch.Size([1, 1386]) +Dataloader batch - attn_mask shape: torch.Size([1, 1386]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1390 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1386) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1386) torch.int64 + loss_mask : 314 / 1386 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1386, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1386, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1386, 1, 576) torch.bfloat16 + out : (1, 1386) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 314 tokens) : 11.2254 + top-1 accuracy : 9/314 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1223]) +Dataloader batch - labels shape: torch.Size([1, 1223]) +Dataloader batch - attn_mask shape: torch.Size([1, 1223]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1391 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1223) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1223) torch.int64 + loss_mask : 299 / 1223 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1223, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1223, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1223, 1, 576) torch.bfloat16 + out : (1, 1223) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 299 tokens) : 11.3086 + top-1 accuracy : 6/299 = 2.0% + +Dataloader batch - tokens shape: torch.Size([1, 1379]) +Dataloader batch - labels shape: torch.Size([1, 1379]) +Dataloader batch - attn_mask shape: torch.Size([1, 1379]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1392 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1379) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1379) torch.int64 + loss_mask : 331 / 1379 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1379, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1379, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1379, 1, 576) torch.bfloat16 + out : (1, 1379) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 331 tokens) : 11.2472 + top-1 accuracy : 15/331 = 4.5% + + [2026-04-29 14:05:04] iteration 448/ 500 | consumed samples: 1792 | elapsed time per iteration (ms): 4323.2 | throughput (token/sec/GPU): 1213.9 | learning rate: 2.198754E-05 | global batch size: 4 | lm loss: 1.127010E+01 | loss scale: 1.0 | grad norm: 0.234 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1545]) +Dataloader batch - labels shape: torch.Size([1, 1545]) +Dataloader batch - attn_mask shape: torch.Size([1, 1545]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1393 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1545) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1545) torch.int64 + loss_mask : 384 / 1545 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1545, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1545, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1545, 1, 576) torch.bfloat16 + out : (1, 1545) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 384 tokens) : 11.3188 + top-1 accuracy : 7/384 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1279]) +Dataloader batch - labels shape: torch.Size([1, 1279]) +Dataloader batch - attn_mask shape: torch.Size([1, 1279]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1394 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1279) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1279) torch.int64 + loss_mask : 342 / 1279 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1279, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1279, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1279, 1, 576) torch.bfloat16 + out : (1, 1279) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 342 tokens) : 11.3014 + top-1 accuracy : 2/342 = 0.6% + +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1395 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1418) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1418) torch.int64 + loss_mask : 317 / 1418 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1418, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1418, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1418, 1, 576) torch.bfloat16 + out : (1, 1418) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 317 tokens) : 11.2717 + top-1 accuracy : 20/317 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1418]) +Dataloader batch - labels shape: torch.Size([1, 1418]) +Dataloader batch - attn_mask shape: torch.Size([1, 1418]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1396 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1418) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1418) torch.int64 + loss_mask : 276 / 1418 tokens (19.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1418, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1418, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1418, 1, 576) torch.bfloat16 + out : (1, 1418) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 276 tokens) : 11.1475 + top-1 accuracy : 12/276 = 4.3% + + [2026-04-29 14:05:09] iteration 449/ 500 | consumed samples: 1796 | elapsed time per iteration (ms): 4682.8 | throughput (token/sec/GPU): 1208.7 | learning rate: 2.173336E-05 | global batch size: 4 | lm loss: 1.126712E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1378]) +Dataloader batch - labels shape: torch.Size([1, 1378]) +Dataloader batch - attn_mask shape: torch.Size([1, 1378]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1397 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1378) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1378) torch.int64 + loss_mask : 411 / 1378 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1378, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1378, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1378, 1, 576) torch.bfloat16 + out : (1, 1378) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.3679 + top-1 accuracy : 5/411 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1479]) +Dataloader batch - labels shape: torch.Size([1, 1479]) +Dataloader batch - attn_mask shape: torch.Size([1, 1479]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1398 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1479) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1479) torch.int64 + loss_mask : 500 / 1479 tokens (33.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1479, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1479, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1479, 1, 576) torch.bfloat16 + out : (1, 1479) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 500 tokens) : 11.4631 + top-1 accuracy : 22/500 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1442]) +Dataloader batch - labels shape: torch.Size([1, 1442]) +Dataloader batch - attn_mask shape: torch.Size([1, 1442]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1399 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1442) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1442) torch.int64 + loss_mask : 339 / 1442 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1442, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1442, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1442, 1, 576) torch.bfloat16 + out : (1, 1442) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.2227 + top-1 accuracy : 17/339 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1212]) +Dataloader batch - labels shape: torch.Size([1, 1212]) +Dataloader batch - attn_mask shape: torch.Size([1, 1212]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1400 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1212) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1212) torch.int64 + loss_mask : 310 / 1212 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1212, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1212, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1212, 1, 576) torch.bfloat16 + out : (1, 1212) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 310 tokens) : 11.2830 + top-1 accuracy : 14/310 = 4.5% + + [2026-04-29 14:05:13] iteration 450/ 500 | consumed samples: 1800 | elapsed time per iteration (ms): 4205.3 | throughput (token/sec/GPU): 1310.5 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.134999E+01 | loss scale: 1.0 | grad norm: 0.213 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +saving checkpoint at iteration 450 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 450 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 450 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (2923.40, 2923.40) +Dataloader batch - tokens shape: torch.Size([1, 1395]) +Dataloader batch - labels shape: torch.Size([1, 1395]) +Dataloader batch - attn_mask shape: torch.Size([1, 1395]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1401 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1395) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1395) torch.int64 + loss_mask : 329 / 1395 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1395, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1395, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1395, 1, 576) torch.bfloat16 + out : (1, 1395) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2820 + top-1 accuracy : 14/329 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1777]) +Dataloader batch - labels shape: torch.Size([1, 1777]) +Dataloader batch - attn_mask shape: torch.Size([1, 1777]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1402 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1777) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1777) torch.int64 + loss_mask : 415 / 1777 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1777, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1777, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1777, 1, 576) torch.bfloat16 + out : (1, 1777) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.2417 + top-1 accuracy : 20/415 = 4.8% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1403 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 394 / 1475 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 394 tokens) : 11.3048 + top-1 accuracy : 10/394 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1410]) +Dataloader batch - labels shape: torch.Size([1, 1410]) +Dataloader batch - attn_mask shape: torch.Size([1, 1410]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1404 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1410) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1410) torch.int64 + loss_mask : 356 / 1410 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1410, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1410, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1410, 1, 576) torch.bfloat16 + out : (1, 1410) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2741 + top-1 accuracy : 6/356 = 1.7% + + [2026-04-29 14:05:21] iteration 451/ 500 | consumed samples: 1804 | elapsed time per iteration (ms): 4861.1 | throughput (token/sec/GPU): 1246.0 | learning rate: 2.122843E-05 | global batch size: 4 | lm loss: 1.127494E+01 | loss scale: 1.0 | grad norm: 0.185 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1212]) +Dataloader batch - labels shape: torch.Size([1, 1212]) +Dataloader batch - attn_mask shape: torch.Size([1, 1212]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1405 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1212) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1212) torch.int64 + loss_mask : 318 / 1212 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1212, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1212, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1212, 1, 576) torch.bfloat16 + out : (1, 1212) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.2481 + top-1 accuracy : 9/318 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1406 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 382 / 1438 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.3329 + top-1 accuracy : 7/382 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1385]) +Dataloader batch - labels shape: torch.Size([1, 1385]) +Dataloader batch - attn_mask shape: torch.Size([1, 1385]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 1407 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1385) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1385) torch.int64 + loss_mask : 357 / 1385 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1385, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1385, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1385, 1, 576) torch.bfloat16 + out : (1, 1385) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.3205 + top-1 accuracy : 4/357 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1397]) +Dataloader batch - labels shape: torch.Size([1, 1397]) +Dataloader batch - attn_mask shape: torch.Size([1, 1397]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1408 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1397) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1397) torch.int64 + loss_mask : 319 / 1397 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1397, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1397, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1397, 1, 576) torch.bfloat16 + out : (1, 1397) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.1698 + top-1 accuracy : 18/319 = 5.6% + + [2026-04-29 14:05:25] iteration 452/ 500 | consumed samples: 1808 | elapsed time per iteration (ms): 4411.7 | throughput (token/sec/GPU): 1231.3 | learning rate: 2.097770E-05 | global batch size: 4 | lm loss: 1.127229E+01 | loss scale: 1.0 | grad norm: 0.202 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1380]) +Dataloader batch - labels shape: torch.Size([1, 1380]) +Dataloader batch - attn_mask shape: torch.Size([1, 1380]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1409 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1380) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1380) torch.int64 + loss_mask : 306 / 1380 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1380, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1380, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1380, 1, 576) torch.bfloat16 + out : (1, 1380) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 306 tokens) : 11.1615 + top-1 accuracy : 11/306 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1424]) +Dataloader batch - labels shape: torch.Size([1, 1424]) +Dataloader batch - attn_mask shape: torch.Size([1, 1424]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1410 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1424) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1424) torch.int64 + loss_mask : 337 / 1424 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1424, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1424, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1424, 1, 576) torch.bfloat16 + out : (1, 1424) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.3101 + top-1 accuracy : 22/337 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1690]) +Dataloader batch - labels shape: torch.Size([1, 1690]) +Dataloader batch - attn_mask shape: torch.Size([1, 1690]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1411 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1690) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1690) torch.int64 + loss_mask : 413 / 1690 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1690, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1690, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1690, 1, 576) torch.bfloat16 + out : (1, 1690) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2815 + top-1 accuracy : 11/413 = 2.7% + +Dataloader batch - tokens shape: torch.Size([1, 1470]) +Dataloader batch - labels shape: torch.Size([1, 1470]) +Dataloader batch - attn_mask shape: torch.Size([1, 1470]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1412 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1470) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1470) torch.int64 + loss_mask : 324 / 1470 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1470, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1470, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1470, 1, 576) torch.bfloat16 + out : (1, 1470) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 324 tokens) : 11.1912 + top-1 accuracy : 5/324 = 1.5% + + [2026-04-29 14:05:30] iteration 453/ 500 | consumed samples: 1812 | elapsed time per iteration (ms): 4905.5 | throughput (token/sec/GPU): 1215.8 | learning rate: 2.072814E-05 | global batch size: 4 | lm loss: 1.124064E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1558]) +Dataloader batch - labels shape: torch.Size([1, 1558]) +Dataloader batch - attn_mask shape: torch.Size([1, 1558]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1413 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1558) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1558) torch.int64 + loss_mask : 415 / 1558 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1558, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1558, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1558, 1, 576) torch.bfloat16 + out : (1, 1558) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 415 tokens) : 11.3581 + top-1 accuracy : 8/415 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1375]) +Dataloader batch - labels shape: torch.Size([1, 1375]) +Dataloader batch - attn_mask shape: torch.Size([1, 1375]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1414 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1375) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1375) torch.int64 + loss_mask : 313 / 1375 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1375, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1375, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1375, 1, 576) torch.bfloat16 + out : (1, 1375) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 313 tokens) : 11.2814 + top-1 accuracy : 7/313 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1415 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 366 / 1541 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2509 + top-1 accuracy : 27/366 = 7.4% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1416 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 367 / 1695 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 367 tokens) : 11.1585 + top-1 accuracy : 29/367 = 7.9% + + [2026-04-29 14:05:35] iteration 454/ 500 | consumed samples: 1816 | elapsed time per iteration (ms): 4942.3 | throughput (token/sec/GPU): 1248.2 | learning rate: 2.047975E-05 | global batch size: 4 | lm loss: 1.126469E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1752]) +Dataloader batch - labels shape: torch.Size([1, 1752]) +Dataloader batch - attn_mask shape: torch.Size([1, 1752]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1417 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1752) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1752) torch.int64 + loss_mask : 398 / 1752 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1752, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1752, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1752, 1, 576) torch.bfloat16 + out : (1, 1752) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 398 tokens) : 11.2752 + top-1 accuracy : 21/398 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1563]) +Dataloader batch - labels shape: torch.Size([1, 1563]) +Dataloader batch - attn_mask shape: torch.Size([1, 1563]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1418 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1563) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1563) torch.int64 + loss_mask : 427 / 1563 tokens (27.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1563, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1563, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1563, 1, 576) torch.bfloat16 + out : (1, 1563) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 427 tokens) : 11.3071 + top-1 accuracy : 6/427 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1419]) +Dataloader batch - labels shape: torch.Size([1, 1419]) +Dataloader batch - attn_mask shape: torch.Size([1, 1419]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1419 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1419) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1419) torch.int64 + loss_mask : 335 / 1419 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1419, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1419, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1419, 1, 576) torch.bfloat16 + out : (1, 1419) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.2640 + top-1 accuracy : 25/335 = 7.5% + +Dataloader batch - tokens shape: torch.Size([1, 1043]) +Dataloader batch - labels shape: torch.Size([1, 1043]) +Dataloader batch - attn_mask shape: torch.Size([1, 1043]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1420 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1043) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1043) torch.int64 + loss_mask : 216 / 1043 tokens (20.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1043, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1043, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1043, 1, 576) torch.bfloat16 + out : (1, 1043) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 216 tokens) : 11.2259 + top-1 accuracy : 20/216 = 9.3% + + [2026-04-29 14:05:39] iteration 455/ 500 | consumed samples: 1820 | elapsed time per iteration (ms): 4625.4 | throughput (token/sec/GPU): 1249.0 | learning rate: 2.023256E-05 | global batch size: 4 | lm loss: 1.127461E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1722]) +Dataloader batch - labels shape: torch.Size([1, 1722]) +Dataloader batch - attn_mask shape: torch.Size([1, 1722]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1421 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1722) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1722) torch.int64 + loss_mask : 362 / 1722 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1722, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1722, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1722, 1, 576) torch.bfloat16 + out : (1, 1722) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2272 + top-1 accuracy : 26/362 = 7.2% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1422 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 402 / 1532 tokens (26.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 402 tokens) : 11.2804 + top-1 accuracy : 16/402 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1376]) +Dataloader batch - labels shape: torch.Size([1, 1376]) +Dataloader batch - attn_mask shape: torch.Size([1, 1376]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1423 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1376) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1376) torch.int64 + loss_mask : 408 / 1376 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1376, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1376, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1376, 1, 576) torch.bfloat16 + out : (1, 1376) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 408 tokens) : 11.4339 + top-1 accuracy : 1/408 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1061]) +Dataloader batch - labels shape: torch.Size([1, 1061]) +Dataloader batch - attn_mask shape: torch.Size([1, 1061]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1424 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1061) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1061) torch.int64 + loss_mask : 303 / 1061 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1061, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1061, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1061, 1, 576) torch.bfloat16 + out : (1, 1061) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.3361 + top-1 accuracy : 5/303 = 1.7% + + [2026-04-29 14:05:44] iteration 456/ 500 | consumed samples: 1824 | elapsed time per iteration (ms): 4495.4 | throughput (token/sec/GPU): 1266.0 | learning rate: 1.998657E-05 | global batch size: 4 | lm loss: 1.132124E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1829]) +Dataloader batch - labels shape: torch.Size([1, 1829]) +Dataloader batch - attn_mask shape: torch.Size([1, 1829]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1425 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1829) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1829) torch.int64 + loss_mask : 478 / 1829 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1829, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1829, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1829, 1, 576) torch.bfloat16 + out : (1, 1829) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 478 tokens) : 11.3082 + top-1 accuracy : 9/478 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1552]) +Dataloader batch - labels shape: torch.Size([1, 1552]) +Dataloader batch - attn_mask shape: torch.Size([1, 1552]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1426 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1552) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1552) torch.int64 + loss_mask : 360 / 1552 tokens (23.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1552, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1552, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1552, 1, 576) torch.bfloat16 + out : (1, 1552) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.1742 + top-1 accuracy : 11/360 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1761]) +Dataloader batch - labels shape: torch.Size([1, 1761]) +Dataloader batch - attn_mask shape: torch.Size([1, 1761]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1427 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1761) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1761) torch.int64 + loss_mask : 400 / 1761 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1761, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1761, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1761, 1, 576) torch.bfloat16 + out : (1, 1761) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.2193 + top-1 accuracy : 37/400 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1079]) +Dataloader batch - labels shape: torch.Size([1, 1079]) +Dataloader batch - attn_mask shape: torch.Size([1, 1079]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1428 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1079) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1079) torch.int64 + loss_mask : 323 / 1079 tokens (29.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1079, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1079, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1079, 1, 576) torch.bfloat16 + out : (1, 1079) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.4161 + top-1 accuracy : 14/323 = 4.3% + + [2026-04-29 14:05:49] iteration 457/ 500 | consumed samples: 1828 | elapsed time per iteration (ms): 4906.4 | throughput (token/sec/GPU): 1267.9 | learning rate: 1.974179E-05 | global batch size: 4 | lm loss: 1.127684E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1160]) +Dataloader batch - labels shape: torch.Size([1, 1160]) +Dataloader batch - attn_mask shape: torch.Size([1, 1160]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1429 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1160) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1160) torch.int64 + loss_mask : 275 / 1160 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1160, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1160, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1160, 1, 576) torch.bfloat16 + out : (1, 1160) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 275 tokens) : 11.2349 + top-1 accuracy : 4/275 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1359]) +Dataloader batch - labels shape: torch.Size([1, 1359]) +Dataloader batch - attn_mask shape: torch.Size([1, 1359]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1430 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1359) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1359) torch.int64 + loss_mask : 285 / 1359 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1359, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1359, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1359, 1, 576) torch.bfloat16 + out : (1, 1359) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 285 tokens) : 11.1878 + top-1 accuracy : 16/285 = 5.6% + +Dataloader batch - tokens shape: torch.Size([1, 1532]) +Dataloader batch - labels shape: torch.Size([1, 1532]) +Dataloader batch - attn_mask shape: torch.Size([1, 1532]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1431 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1532) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1532) torch.int64 + loss_mask : 353 / 1532 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1532, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1532, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1532, 1, 576) torch.bfloat16 + out : (1, 1532) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 353 tokens) : 11.2486 + top-1 accuracy : 18/353 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1541]) +Dataloader batch - labels shape: torch.Size([1, 1541]) +Dataloader batch - attn_mask shape: torch.Size([1, 1541]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1432 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1541) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1541) torch.int64 + loss_mask : 335 / 1541 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1541, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1541, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1541, 1, 576) torch.bfloat16 + out : (1, 1541) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 335 tokens) : 11.1550 + top-1 accuracy : 30/335 = 9.0% + + [2026-04-29 14:05:53] iteration 458/ 500 | consumed samples: 1832 | elapsed time per iteration (ms): 4639.3 | throughput (token/sec/GPU): 1205.4 | learning rate: 1.949823E-05 | global batch size: 4 | lm loss: 1.120657E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1570]) +Dataloader batch - labels shape: torch.Size([1, 1570]) +Dataloader batch - attn_mask shape: torch.Size([1, 1570]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1433 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1570) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1570) torch.int64 + loss_mask : 360 / 1570 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1570, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1570, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1570, 1, 576) torch.bfloat16 + out : (1, 1570) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.2276 + top-1 accuracy : 30/360 = 8.3% + +Dataloader batch - tokens shape: torch.Size([1, 1438]) +Dataloader batch - labels shape: torch.Size([1, 1438]) +Dataloader batch - attn_mask shape: torch.Size([1, 1438]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1434 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1438) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1438) torch.int64 + loss_mask : 363 / 1438 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1438, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1438, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1438, 1, 576) torch.bfloat16 + out : (1, 1438) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 363 tokens) : 11.3109 + top-1 accuracy : 12/363 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1563]) +Dataloader batch - labels shape: torch.Size([1, 1563]) +Dataloader batch - attn_mask shape: torch.Size([1, 1563]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1435 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1563) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1563) torch.int64 + loss_mask : 443 / 1563 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1563, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1563, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1563, 1, 576) torch.bfloat16 + out : (1, 1563) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 443 tokens) : 11.3312 + top-1 accuracy : 34/443 = 7.7% + +Dataloader batch - tokens shape: torch.Size([1, 1459]) +Dataloader batch - labels shape: torch.Size([1, 1459]) +Dataloader batch - attn_mask shape: torch.Size([1, 1459]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1436 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1459) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1459) torch.int64 + loss_mask : 309 / 1459 tokens (21.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1459, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1459, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1459, 1, 576) torch.bfloat16 + out : (1, 1459) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.1497 + top-1 accuracy : 21/309 = 6.8% + + [2026-04-29 14:05:58] iteration 459/ 500 | consumed samples: 1836 | elapsed time per iteration (ms): 4893.4 | throughput (token/sec/GPU): 1232.3 | learning rate: 1.925589E-05 | global batch size: 4 | lm loss: 1.126288E+01 | loss scale: 1.0 | grad norm: 0.174 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1431]) +Dataloader batch - labels shape: torch.Size([1, 1431]) +Dataloader batch - attn_mask shape: torch.Size([1, 1431]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1437 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1431) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1431) torch.int64 + loss_mask : 364 / 1431 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1431, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1431, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1431, 1, 576) torch.bfloat16 + out : (1, 1431) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 364 tokens) : 11.3455 + top-1 accuracy : 9/364 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1606]) +Dataloader batch - labels shape: torch.Size([1, 1606]) +Dataloader batch - attn_mask shape: torch.Size([1, 1606]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1438 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1606) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1606) torch.int64 + loss_mask : 388 / 1606 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1606, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1606, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1606, 1, 576) torch.bfloat16 + out : (1, 1606) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.2296 + top-1 accuracy : 32/388 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1439 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 417 / 1546 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 417 tokens) : 11.3086 + top-1 accuracy : 5/417 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1754]) +Dataloader batch - labels shape: torch.Size([1, 1754]) +Dataloader batch - attn_mask shape: torch.Size([1, 1754]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1440 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1754) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1754) torch.int64 + loss_mask : 391 / 1754 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1754, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1754, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1754, 1, 576) torch.bfloat16 + out : (1, 1754) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.2600 + top-1 accuracy : 31/391 = 7.9% + + [2026-04-29 14:06:03] iteration 460/ 500 | consumed samples: 1840 | elapsed time per iteration (ms): 5005.8 | throughput (token/sec/GPU): 1265.9 | learning rate: 1.901480E-05 | global batch size: 4 | lm loss: 1.128537E+01 | loss scale: 1.0 | grad norm: 0.181 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4978.19, 4978.19) + forward-compute ................................: (3731.37, 3731.37) + backward-compute ...............................: (1242.84, 1242.84) + batch-generator ................................: (516.23, 516.23) + layernorm-grads-all-reduce .....................: (0.07, 0.07) + embedding-grads-all-reduce .....................: (0.10, 0.10) + all-grads-sync .................................: (1.09, 1.09) + params-all-gather ..............................: (0.38, 0.38) + optimizer-copy-to-main-grad ....................: (0.35, 0.35) + optimizer-clip-main-grad .......................: (1.29, 1.29) + optimizer-count-zeros ..........................: (0.05, 0.05) + optimizer-inner-step ...........................: (2.13, 2.13) + optimizer-copy-main-to-model-params ............: (0.55, 0.55) + optimizer ......................................: (6.80, 6.80) +Dataloader batch - tokens shape: torch.Size([1, 1516]) +Dataloader batch - labels shape: torch.Size([1, 1516]) +Dataloader batch - attn_mask shape: torch.Size([1, 1516]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1441 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1516) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1516) torch.int64 + loss_mask : 374 / 1516 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1516, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1516, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1516, 1, 576) torch.bfloat16 + out : (1, 1516) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.2991 + top-1 accuracy : 16/374 = 4.3% + +Dataloader batch - tokens shape: torch.Size([1, 1443]) +Dataloader batch - labels shape: torch.Size([1, 1443]) +Dataloader batch - attn_mask shape: torch.Size([1, 1443]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1442 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1443) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1443) torch.int64 + loss_mask : 355 / 1443 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1443, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1443, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1443, 1, 576) torch.bfloat16 + out : (1, 1443) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 355 tokens) : 11.2737 + top-1 accuracy : 29/355 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1397]) +Dataloader batch - labels shape: torch.Size([1, 1397]) +Dataloader batch - attn_mask shape: torch.Size([1, 1397]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1443 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1397) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1397) torch.int64 + loss_mask : 334 / 1397 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1397, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1397, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1397, 1, 576) torch.bfloat16 + out : (1, 1397) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 334 tokens) : 11.3039 + top-1 accuracy : 10/334 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1594]) +Dataloader batch - labels shape: torch.Size([1, 1594]) +Dataloader batch - attn_mask shape: torch.Size([1, 1594]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1444 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1594) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1594) torch.int64 + loss_mask : 366 / 1594 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1594, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1594, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1594, 1, 576) torch.bfloat16 + out : (1, 1594) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.2056 + top-1 accuracy : 19/366 = 5.2% + + [2026-04-29 14:06:08] iteration 461/ 500 | consumed samples: 1844 | elapsed time per iteration (ms): 4712.5 | throughput (token/sec/GPU): 1262.6 | learning rate: 1.877495E-05 | global batch size: 4 | lm loss: 1.126996E+01 | loss scale: 1.0 | grad norm: 0.219 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1444]) +Dataloader batch - labels shape: torch.Size([1, 1444]) +Dataloader batch - attn_mask shape: torch.Size([1, 1444]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1445 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1444) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1444) torch.int64 + loss_mask : 360 / 1444 tokens (24.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1444, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1444, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1444, 1, 576) torch.bfloat16 + out : (1, 1444) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.3267 + top-1 accuracy : 11/360 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1013]) +Dataloader batch - labels shape: torch.Size([1, 1013]) +Dataloader batch - attn_mask shape: torch.Size([1, 1013]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1446 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1013) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1013) torch.int64 + loss_mask : 251 / 1013 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1013, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1013, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1013, 1, 576) torch.bfloat16 + out : (1, 1013) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2099 + top-1 accuracy : 11/251 = 4.4% + +Dataloader batch - tokens shape: torch.Size([1, 1390]) +Dataloader batch - labels shape: torch.Size([1, 1390]) +Dataloader batch - attn_mask shape: torch.Size([1, 1390]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1447 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1390) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1390) torch.int64 + loss_mask : 320 / 1390 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1390, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1390, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1390, 1, 576) torch.bfloat16 + out : (1, 1390) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 320 tokens) : 11.2529 + top-1 accuracy : 17/320 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1448 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 362 / 1520 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 362 tokens) : 11.2678 + top-1 accuracy : 16/362 = 4.4% + + [2026-04-29 14:06:12] iteration 462/ 500 | consumed samples: 1848 | elapsed time per iteration (ms): 4344.4 | throughput (token/sec/GPU): 1235.4 | learning rate: 1.853636E-05 | global batch size: 4 | lm loss: 1.126927E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1255]) +Dataloader batch - labels shape: torch.Size([1, 1255]) +Dataloader batch - attn_mask shape: torch.Size([1, 1255]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1449 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1255) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1255) torch.int64 + loss_mask : 318 / 1255 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1255, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1255, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1255, 1, 576) torch.bfloat16 + out : (1, 1255) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.3104 + top-1 accuracy : 3/318 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1394]) +Dataloader batch - labels shape: torch.Size([1, 1394]) +Dataloader batch - attn_mask shape: torch.Size([1, 1394]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1450 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1394) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1394) torch.int64 + loss_mask : 339 / 1394 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1394, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1394, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1394, 1, 576) torch.bfloat16 + out : (1, 1394) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.3122 + top-1 accuracy : 1/339 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1802]) +Dataloader batch - labels shape: torch.Size([1, 1802]) +Dataloader batch - attn_mask shape: torch.Size([1, 1802]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1451 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1802) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1802) torch.int64 + loss_mask : 466 / 1802 tokens (25.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1802, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1802, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1802, 1, 576) torch.bfloat16 + out : (1, 1802) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 466 tokens) : 11.3059 + top-1 accuracy : 25/466 = 5.4% + +Dataloader batch - tokens shape: torch.Size([1, 1537]) +Dataloader batch - labels shape: torch.Size([1, 1537]) +Dataloader batch - attn_mask shape: torch.Size([1, 1537]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1452 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1537) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1537) torch.int64 + loss_mask : 344 / 1537 tokens (22.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1537, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1537, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1537, 1, 576) torch.bfloat16 + out : (1, 1537) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.1924 + top-1 accuracy : 8/344 = 2.3% + + [2026-04-29 14:06:17] iteration 463/ 500 | consumed samples: 1852 | elapsed time per iteration (ms): 4774.4 | throughput (token/sec/GPU): 1254.2 | learning rate: 1.829904E-05 | global batch size: 4 | lm loss: 1.128170E+01 | loss scale: 1.0 | grad norm: 0.243 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1155]) +Dataloader batch - labels shape: torch.Size([1, 1155]) +Dataloader batch - attn_mask shape: torch.Size([1, 1155]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1453 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1155) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1155) torch.int64 + loss_mask : 309 / 1155 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1155, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1155, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1155, 1, 576) torch.bfloat16 + out : (1, 1155) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 309 tokens) : 11.3191 + top-1 accuracy : 11/309 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1102]) +Dataloader batch - labels shape: torch.Size([1, 1102]) +Dataloader batch - attn_mask shape: torch.Size([1, 1102]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1454 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1102) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1102) torch.int64 + loss_mask : 251 / 1102 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1102, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1102, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1102, 1, 576) torch.bfloat16 + out : (1, 1102) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 251 tokens) : 11.2367 + top-1 accuracy : 7/251 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1205]) +Dataloader batch - labels shape: torch.Size([1, 1205]) +Dataloader batch - attn_mask shape: torch.Size([1, 1205]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1455 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1205) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1205) torch.int64 + loss_mask : 271 / 1205 tokens (22.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1205, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1205, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1205, 1, 576) torch.bfloat16 + out : (1, 1205) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 271 tokens) : 11.1992 + top-1 accuracy : 5/271 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 782]) +Dataloader batch - labels shape: torch.Size([1, 782]) +Dataloader batch - attn_mask shape: torch.Size([1, 782]) +Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) + +==================================================================== + PIPELINE — STEP 1456 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 782) torch.int64 + images : (16, 3, 1024, 1024) torch.bfloat16 + labels : (1, 782) torch.int64 + loss_mask : 136 / 782 tokens (17.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (16, 3, 1024, 1024) torch.bfloat16 + out : (16, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (16, 3072) + out : (16, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (782, 1, 576) torch.bfloat16 grad=False + image slots : 16 tokens replaced with vision embeddings + combined : (782, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (782, 1, 576) torch.bfloat16 + out : (1, 782) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 136 tokens) : 11.1125 + top-1 accuracy : 16/136 = 11.8% + + [2026-04-29 14:06:21] iteration 464/ 500 | consumed samples: 1856 | elapsed time per iteration (ms): 3686.5 | throughput (token/sec/GPU): 1151.2 | learning rate: 1.806299E-05 | global batch size: 4 | lm loss: 1.123506E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1402]) +Dataloader batch - labels shape: torch.Size([1, 1402]) +Dataloader batch - attn_mask shape: torch.Size([1, 1402]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1457 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1402) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1402) torch.int64 + loss_mask : 340 / 1402 tokens (24.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1402, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1402, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1402, 1, 576) torch.bfloat16 + out : (1, 1402) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2275 + top-1 accuracy : 7/340 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1137]) +Dataloader batch - labels shape: torch.Size([1, 1137]) +Dataloader batch - attn_mask shape: torch.Size([1, 1137]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1458 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1137) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1137) torch.int64 + loss_mask : 323 / 1137 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1137, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1137, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1137, 1, 576) torch.bfloat16 + out : (1, 1137) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.3680 + top-1 accuracy : 1/323 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1927]) +Dataloader batch - labels shape: torch.Size([1, 1927]) +Dataloader batch - attn_mask shape: torch.Size([1, 1927]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1459 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1927) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1927) torch.int64 + loss_mask : 550 / 1927 tokens (28.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1927, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1927, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1927, 1, 576) torch.bfloat16 + out : (1, 1927) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 550 tokens) : 11.3882 + top-1 accuracy : 2/550 = 0.4% + +Dataloader batch - tokens shape: torch.Size([1, 1384]) +Dataloader batch - labels shape: torch.Size([1, 1384]) +Dataloader batch - attn_mask shape: torch.Size([1, 1384]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1460 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1384) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1384) torch.int64 + loss_mask : 338 / 1384 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1384, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1384, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1384, 1, 576) torch.bfloat16 + out : (1, 1384) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.2644 + top-1 accuracy : 21/338 = 6.2% + + [2026-04-29 14:06:25] iteration 465/ 500 | consumed samples: 1860 | elapsed time per iteration (ms): 4503.6 | throughput (token/sec/GPU): 1298.9 | learning rate: 1.782823E-05 | global batch size: 4 | lm loss: 1.132176E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1069]) +Dataloader batch - labels shape: torch.Size([1, 1069]) +Dataloader batch - attn_mask shape: torch.Size([1, 1069]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1461 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1069) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1069) torch.int64 + loss_mask : 299 / 1069 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1069, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1069, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1069, 1, 576) torch.bfloat16 + out : (1, 1069) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 299 tokens) : 11.3651 + top-1 accuracy : 0/299 = 0.0% + +Dataloader batch - tokens shape: torch.Size([1, 1242]) +Dataloader batch - labels shape: torch.Size([1, 1242]) +Dataloader batch - attn_mask shape: torch.Size([1, 1242]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1462 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1242) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1242) torch.int64 + loss_mask : 303 / 1242 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1242, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1242, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1242, 1, 576) torch.bfloat16 + out : (1, 1242) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.2800 + top-1 accuracy : 4/303 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1808]) +Dataloader batch - labels shape: torch.Size([1, 1808]) +Dataloader batch - attn_mask shape: torch.Size([1, 1808]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1463 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1808) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1808) torch.int64 + loss_mask : 454 / 1808 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1808, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1808, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1808, 1, 576) torch.bfloat16 + out : (1, 1808) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 454 tokens) : 11.2695 + top-1 accuracy : 18/454 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1545]) +Dataloader batch - labels shape: torch.Size([1, 1545]) +Dataloader batch - attn_mask shape: torch.Size([1, 1545]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1464 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1545) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1545) torch.int64 + loss_mask : 407 / 1545 tokens (26.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1545, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1545, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1545, 1, 576) torch.bfloat16 + out : (1, 1545) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.3094 + top-1 accuracy : 10/407 = 2.5% + + [2026-04-29 14:06:30] iteration 466/ 500 | consumed samples: 1864 | elapsed time per iteration (ms): 4539.1 | throughput (token/sec/GPU): 1247.8 | learning rate: 1.759476E-05 | global batch size: 4 | lm loss: 1.130231E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1733]) +Dataloader batch - labels shape: torch.Size([1, 1733]) +Dataloader batch - attn_mask shape: torch.Size([1, 1733]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1465 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1733) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1733) torch.int64 + loss_mask : 373 / 1733 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1733, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1733, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1733, 1, 576) torch.bfloat16 + out : (1, 1733) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 373 tokens) : 11.2156 + top-1 accuracy : 13/373 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1268]) +Dataloader batch - labels shape: torch.Size([1, 1268]) +Dataloader batch - attn_mask shape: torch.Size([1, 1268]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1466 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1268) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1268) torch.int64 + loss_mask : 344 / 1268 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1268, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1268, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1268, 1, 576) torch.bfloat16 + out : (1, 1268) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.3077 + top-1 accuracy : 5/344 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1103]) +Dataloader batch - labels shape: torch.Size([1, 1103]) +Dataloader batch - attn_mask shape: torch.Size([1, 1103]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1467 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1103) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1103) torch.int64 + loss_mask : 278 / 1103 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1103, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1103, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1103, 1, 576) torch.bfloat16 + out : (1, 1103) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.2637 + top-1 accuracy : 3/278 = 1.1% + +Dataloader batch - tokens shape: torch.Size([1, 1437]) +Dataloader batch - labels shape: torch.Size([1, 1437]) +Dataloader batch - attn_mask shape: torch.Size([1, 1437]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1468 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1437) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1437) torch.int64 + loss_mask : 388 / 1437 tokens (27.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1437, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1437, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1437, 1, 576) torch.bfloat16 + out : (1, 1437) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3361 + top-1 accuracy : 16/388 = 4.1% + + [2026-04-29 14:06:34] iteration 467/ 500 | consumed samples: 1868 | elapsed time per iteration (ms): 4506.9 | throughput (token/sec/GPU): 1229.4 | learning rate: 1.736260E-05 | global batch size: 4 | lm loss: 1.128199E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1060]) +Dataloader batch - labels shape: torch.Size([1, 1060]) +Dataloader batch - attn_mask shape: torch.Size([1, 1060]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1469 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1060) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1060) torch.int64 + loss_mask : 230 / 1060 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1060, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1060, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1060, 1, 576) torch.bfloat16 + out : (1, 1060) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 230 tokens) : 11.1820 + top-1 accuracy : 9/230 = 3.9% + +Dataloader batch - tokens shape: torch.Size([1, 1547]) +Dataloader batch - labels shape: torch.Size([1, 1547]) +Dataloader batch - attn_mask shape: torch.Size([1, 1547]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1470 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1547) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1547) torch.int64 + loss_mask : 391 / 1547 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1547, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1547, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1547, 1, 576) torch.bfloat16 + out : (1, 1547) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3049 + top-1 accuracy : 12/391 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1530]) +Dataloader batch - labels shape: torch.Size([1, 1530]) +Dataloader batch - attn_mask shape: torch.Size([1, 1530]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1471 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1530) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1530) torch.int64 + loss_mask : 326 / 1530 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1530, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1530, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1530, 1, 576) torch.bfloat16 + out : (1, 1530) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.1850 + top-1 accuracy : 15/326 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1374]) +Dataloader batch - labels shape: torch.Size([1, 1374]) +Dataloader batch - attn_mask shape: torch.Size([1, 1374]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1472 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1374) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1374) torch.int64 + loss_mask : 296 / 1374 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1374, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1374, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1374, 1, 576) torch.bfloat16 + out : (1, 1374) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 296 tokens) : 11.1673 + top-1 accuracy : 17/296 = 5.7% + + [2026-04-29 14:06:39] iteration 468/ 500 | consumed samples: 1872 | elapsed time per iteration (ms): 4544.6 | throughput (token/sec/GPU): 1212.6 | learning rate: 1.713175E-05 | global batch size: 4 | lm loss: 1.121796E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1563]) +Dataloader batch - labels shape: torch.Size([1, 1563]) +Dataloader batch - attn_mask shape: torch.Size([1, 1563]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1473 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1563) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1563) torch.int64 + loss_mask : 347 / 1563 tokens (22.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1563, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1563, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1563, 1, 576) torch.bfloat16 + out : (1, 1563) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.1905 + top-1 accuracy : 12/347 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1121]) +Dataloader batch - labels shape: torch.Size([1, 1121]) +Dataloader batch - attn_mask shape: torch.Size([1, 1121]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1474 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1121) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1121) torch.int64 + loss_mask : 318 / 1121 tokens (28.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1121, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1121, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1121, 1, 576) torch.bfloat16 + out : (1, 1121) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 318 tokens) : 11.4040 + top-1 accuracy : 11/318 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1600]) +Dataloader batch - labels shape: torch.Size([1, 1600]) +Dataloader batch - attn_mask shape: torch.Size([1, 1600]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1475 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1600) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1600) torch.int64 + loss_mask : 445 / 1600 tokens (27.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1600, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1600, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1600, 1, 576) torch.bfloat16 + out : (1, 1600) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 445 tokens) : 11.4147 + top-1 accuracy : 4/445 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1413]) +Dataloader batch - labels shape: torch.Size([1, 1413]) +Dataloader batch - attn_mask shape: torch.Size([1, 1413]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1476 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1413) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1413) torch.int64 + loss_mask : 349 / 1413 tokens (24.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1413, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1413, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1413, 1, 576) torch.bfloat16 + out : (1, 1413) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 349 tokens) : 11.2610 + top-1 accuracy : 12/349 = 3.4% + + [2026-04-29 14:06:44] iteration 469/ 500 | consumed samples: 1876 | elapsed time per iteration (ms): 4702.7 | throughput (token/sec/GPU): 1211.4 | learning rate: 1.690222E-05 | global batch size: 4 | lm loss: 1.132229E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1548]) +Dataloader batch - labels shape: torch.Size([1, 1548]) +Dataloader batch - attn_mask shape: torch.Size([1, 1548]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1477 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1548) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1548) torch.int64 + loss_mask : 416 / 1548 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1548, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1548, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1548, 1, 576) torch.bfloat16 + out : (1, 1548) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.2973 + top-1 accuracy : 4/416 = 1.0% + +Dataloader batch - tokens shape: torch.Size([1, 1463]) +Dataloader batch - labels shape: torch.Size([1, 1463]) +Dataloader batch - attn_mask shape: torch.Size([1, 1463]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1478 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1463) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1463) torch.int64 + loss_mask : 405 / 1463 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1463, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1463, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1463, 1, 576) torch.bfloat16 + out : (1, 1463) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.3143 + top-1 accuracy : 10/405 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1572]) +Dataloader batch - labels shape: torch.Size([1, 1572]) +Dataloader batch - attn_mask shape: torch.Size([1, 1572]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1479 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1572) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1572) torch.int64 + loss_mask : 436 / 1572 tokens (27.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1572, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1572, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1572, 1, 576) torch.bfloat16 + out : (1, 1572) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 436 tokens) : 11.2831 + top-1 accuracy : 10/436 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1179]) +Dataloader batch - labels shape: torch.Size([1, 1179]) +Dataloader batch - attn_mask shape: torch.Size([1, 1179]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1480 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1179) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1179) torch.int64 + loss_mask : 325 / 1179 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1179, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1179, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1179, 1, 576) torch.bfloat16 + out : (1, 1179) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 325 tokens) : 11.3439 + top-1 accuracy : 16/325 = 4.9% + + [2026-04-29 14:06:48] iteration 470/ 500 | consumed samples: 1880 | elapsed time per iteration (ms): 4480.1 | throughput (token/sec/GPU): 1286.1 | learning rate: 1.667403E-05 | global batch size: 4 | lm loss: 1.130732E+01 | loss scale: 1.0 | grad norm: 0.230 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1612]) +Dataloader batch - labels shape: torch.Size([1, 1612]) +Dataloader batch - attn_mask shape: torch.Size([1, 1612]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1481 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1612) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1612) torch.int64 + loss_mask : 390 / 1612 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1612, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1612, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1612, 1, 576) torch.bfloat16 + out : (1, 1612) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 390 tokens) : 11.2683 + top-1 accuracy : 3/390 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1068]) +Dataloader batch - labels shape: torch.Size([1, 1068]) +Dataloader batch - attn_mask shape: torch.Size([1, 1068]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1482 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1068) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1068) torch.int64 + loss_mask : 289 / 1068 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1068, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1068, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1068, 1, 576) torch.bfloat16 + out : (1, 1068) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 289 tokens) : 11.3525 + top-1 accuracy : 7/289 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1839]) +Dataloader batch - labels shape: torch.Size([1, 1839]) +Dataloader batch - attn_mask shape: torch.Size([1, 1839]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1483 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1839) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1839) torch.int64 + loss_mask : 468 / 1839 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1839, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1839, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1839, 1, 576) torch.bfloat16 + out : (1, 1839) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 468 tokens) : 11.2712 + top-1 accuracy : 11/468 = 2.4% + +Dataloader batch - tokens shape: torch.Size([1, 1875]) +Dataloader batch - labels shape: torch.Size([1, 1875]) +Dataloader batch - attn_mask shape: torch.Size([1, 1875]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1484 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1875) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1875) torch.int64 + loss_mask : 537 / 1875 tokens (28.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1875, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1875, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1875, 1, 576) torch.bfloat16 + out : (1, 1875) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 537 tokens) : 11.3658 + top-1 accuracy : 5/537 = 0.9% + + [2026-04-29 14:06:53] iteration 471/ 500 | consumed samples: 1884 | elapsed time per iteration (ms): 5096.1 | throughput (token/sec/GPU): 1254.7 | learning rate: 1.644718E-05 | global batch size: 4 | lm loss: 1.131465E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 987]) +Dataloader batch - labels shape: torch.Size([1, 987]) +Dataloader batch - attn_mask shape: torch.Size([1, 987]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1485 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 987) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 987) torch.int64 + loss_mask : 236 / 987 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (987, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (987, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (987, 1, 576) torch.bfloat16 + out : (1, 987) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 236 tokens) : 11.2684 + top-1 accuracy : 21/236 = 8.9% + +Dataloader batch - tokens shape: torch.Size([1, 1841]) +Dataloader batch - labels shape: torch.Size([1, 1841]) +Dataloader batch - attn_mask shape: torch.Size([1, 1841]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1486 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1841) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1841) torch.int64 + loss_mask : 480 / 1841 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1841, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1841, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1841, 1, 576) torch.bfloat16 + out : (1, 1841) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 480 tokens) : 11.3357 + top-1 accuracy : 34/480 = 7.1% + +Dataloader batch - tokens shape: torch.Size([1, 1372]) +Dataloader batch - labels shape: torch.Size([1, 1372]) +Dataloader batch - attn_mask shape: torch.Size([1, 1372]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1487 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1372) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1372) torch.int64 + loss_mask : 295 / 1372 tokens (21.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1372, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1372, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1372, 1, 576) torch.bfloat16 + out : (1, 1372) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 295 tokens) : 11.1611 + top-1 accuracy : 15/295 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1884]) +Dataloader batch - labels shape: torch.Size([1, 1884]) +Dataloader batch - attn_mask shape: torch.Size([1, 1884]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1488 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1884) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1884) torch.int64 + loss_mask : 549 / 1884 tokens (29.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1884, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1884, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1884, 1, 576) torch.bfloat16 + out : (1, 1884) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 549 tokens) : 11.3687 + top-1 accuracy : 20/549 = 3.6% + + [2026-04-29 14:06:58] iteration 472/ 500 | consumed samples: 1888 | elapsed time per iteration (ms): 4723.0 | throughput (token/sec/GPU): 1288.2 | learning rate: 1.622167E-05 | global batch size: 4 | lm loss: 1.130412E+01 | loss scale: 1.0 | grad norm: 0.180 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1749]) +Dataloader batch - labels shape: torch.Size([1, 1749]) +Dataloader batch - attn_mask shape: torch.Size([1, 1749]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1489 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1749) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1749) torch.int64 + loss_mask : 400 / 1749 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1749, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1749, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1749, 1, 576) torch.bfloat16 + out : (1, 1749) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 400 tokens) : 11.2769 + top-1 accuracy : 37/400 = 9.2% + +Dataloader batch - tokens shape: torch.Size([1, 1159]) +Dataloader batch - labels shape: torch.Size([1, 1159]) +Dataloader batch - attn_mask shape: torch.Size([1, 1159]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1490 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1159) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1159) torch.int64 + loss_mask : 311 / 1159 tokens (26.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1159, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1159, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1159, 1, 576) torch.bfloat16 + out : (1, 1159) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 311 tokens) : 11.2982 + top-1 accuracy : 8/311 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1353]) +Dataloader batch - labels shape: torch.Size([1, 1353]) +Dataloader batch - attn_mask shape: torch.Size([1, 1353]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1491 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1353) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1353) torch.int64 + loss_mask : 366 / 1353 tokens (27.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1353, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1353, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1353, 1, 576) torch.bfloat16 + out : (1, 1353) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.4099 + top-1 accuracy : 14/366 = 3.8% + +Dataloader batch - tokens shape: torch.Size([1, 1150]) +Dataloader batch - labels shape: torch.Size([1, 1150]) +Dataloader batch - attn_mask shape: torch.Size([1, 1150]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1492 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1150) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1150) torch.int64 + loss_mask : 294 / 1150 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1150, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1150, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1150, 1, 576) torch.bfloat16 + out : (1, 1150) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.2888 + top-1 accuracy : 1/294 = 0.3% + + [2026-04-29 14:07:02] iteration 473/ 500 | consumed samples: 1892 | elapsed time per iteration (ms): 4232.9 | throughput (token/sec/GPU): 1278.3 | learning rate: 1.599753E-05 | global batch size: 4 | lm loss: 1.131980E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1426]) +Dataloader batch - labels shape: torch.Size([1, 1426]) +Dataloader batch - attn_mask shape: torch.Size([1, 1426]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1493 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1426) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1426) torch.int64 + loss_mask : 358 / 1426 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1426, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1426, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1426, 1, 576) torch.bfloat16 + out : (1, 1426) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 358 tokens) : 11.3051 + top-1 accuracy : 8/358 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1074]) +Dataloader batch - labels shape: torch.Size([1, 1074]) +Dataloader batch - attn_mask shape: torch.Size([1, 1074]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1494 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1074) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1074) torch.int64 + loss_mask : 259 / 1074 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1074, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1074, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1074, 1, 576) torch.bfloat16 + out : (1, 1074) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 259 tokens) : 11.2641 + top-1 accuracy : 8/259 = 3.1% + +Dataloader batch - tokens shape: torch.Size([1, 1695]) +Dataloader batch - labels shape: torch.Size([1, 1695]) +Dataloader batch - attn_mask shape: torch.Size([1, 1695]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1495 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1695) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1695) torch.int64 + loss_mask : 345 / 1695 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1695, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1695, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1695, 1, 576) torch.bfloat16 + out : (1, 1695) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 345 tokens) : 11.1758 + top-1 accuracy : 10/345 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1402]) +Dataloader batch - labels shape: torch.Size([1, 1402]) +Dataloader batch - attn_mask shape: torch.Size([1, 1402]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1496 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1402) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1402) torch.int64 + loss_mask : 330 / 1402 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1402, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1402, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1402, 1, 576) torch.bfloat16 + out : (1, 1402) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 330 tokens) : 11.2267 + top-1 accuracy : 4/330 = 1.2% + + [2026-04-29 14:07:07] iteration 474/ 500 | consumed samples: 1896 | elapsed time per iteration (ms): 4568.8 | throughput (token/sec/GPU): 1225.0 | learning rate: 1.577475E-05 | global batch size: 4 | lm loss: 1.124232E+01 | loss scale: 1.0 | grad norm: 0.165 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1518]) +Dataloader batch - labels shape: torch.Size([1, 1518]) +Dataloader batch - attn_mask shape: torch.Size([1, 1518]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1497 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1518) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1518) torch.int64 + loss_mask : 360 / 1518 tokens (23.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1518, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1518, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1518, 1, 576) torch.bfloat16 + out : (1, 1518) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 360 tokens) : 11.2463 + top-1 accuracy : 18/360 = 5.0% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1498 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 343 / 1508 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 343 tokens) : 11.2196 + top-1 accuracy : 24/343 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1816]) +Dataloader batch - labels shape: torch.Size([1, 1816]) +Dataloader batch - attn_mask shape: torch.Size([1, 1816]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1499 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1816) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1816) torch.int64 + loss_mask : 459 / 1816 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1816, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1816, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1816, 1, 576) torch.bfloat16 + out : (1, 1816) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 459 tokens) : 11.2777 + top-1 accuracy : 7/459 = 1.5% + +Dataloader batch - tokens shape: torch.Size([1, 1011]) +Dataloader batch - labels shape: torch.Size([1, 1011]) +Dataloader batch - attn_mask shape: torch.Size([1, 1011]) +Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) + +==================================================================== + PIPELINE — STEP 1500 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1011) torch.int64 + images : (19, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1011) torch.int64 + loss_mask : 238 / 1011 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (19, 3, 1024, 1024) torch.bfloat16 + out : (19, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (19, 3072) + out : (19, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1011, 1, 576) torch.bfloat16 grad=False + image slots : 19 tokens replaced with vision embeddings + combined : (1011, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1011, 1, 576) torch.bfloat16 + out : (1, 1011) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 238 tokens) : 11.2286 + top-1 accuracy : 1/238 = 0.4% + + [2026-04-29 14:07:12] iteration 475/ 500 | consumed samples: 1900 | elapsed time per iteration (ms): 4954.2 | throughput (token/sec/GPU): 1181.4 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.124705E+01 | loss scale: 1.0 | grad norm: 0.169 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1496]) +Dataloader batch - labels shape: torch.Size([1, 1496]) +Dataloader batch - attn_mask shape: torch.Size([1, 1496]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1501 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1496) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1496) torch.int64 + loss_mask : 326 / 1496 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1496, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1496, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1496, 1, 576) torch.bfloat16 + out : (1, 1496) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.2161 + top-1 accuracy : 27/326 = 8.3% + +Dataloader batch - tokens shape: torch.Size([1, 1252]) +Dataloader batch - labels shape: torch.Size([1, 1252]) +Dataloader batch - attn_mask shape: torch.Size([1, 1252]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1502 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1252) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1252) torch.int64 + loss_mask : 337 / 1252 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1252, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1252, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1252, 1, 576) torch.bfloat16 + out : (1, 1252) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 337 tokens) : 11.3371 + top-1 accuracy : 10/337 = 3.0% + +Dataloader batch - tokens shape: torch.Size([1, 1634]) +Dataloader batch - labels shape: torch.Size([1, 1634]) +Dataloader batch - attn_mask shape: torch.Size([1, 1634]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1503 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1634) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1634) torch.int64 + loss_mask : 389 / 1634 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1634, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1634, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1634, 1, 576) torch.bfloat16 + out : (1, 1634) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 389 tokens) : 11.2691 + top-1 accuracy : 19/389 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1634]) +Dataloader batch - labels shape: torch.Size([1, 1634]) +Dataloader batch - attn_mask shape: torch.Size([1, 1634]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1504 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1634) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1634) torch.int64 + loss_mask : 405 / 1634 tokens (24.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1634, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1634, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1634, 1, 576) torch.bfloat16 + out : (1, 1634) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 405 tokens) : 11.2708 + top-1 accuracy : 8/405 = 2.0% + + [2026-04-29 14:07:17] iteration 476/ 500 | consumed samples: 1904 | elapsed time per iteration (ms): 4848.5 | throughput (token/sec/GPU): 1240.8 | learning rate: 1.533333E-05 | global batch size: 4 | lm loss: 1.127345E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1446]) +Dataloader batch - labels shape: torch.Size([1, 1446]) +Dataloader batch - attn_mask shape: torch.Size([1, 1446]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1505 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1446) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1446) torch.int64 + loss_mask : 376 / 1446 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1446, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1446, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1446, 1, 576) torch.bfloat16 + out : (1, 1446) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 376 tokens) : 11.2840 + top-1 accuracy : 1/376 = 0.3% + +Dataloader batch - tokens shape: torch.Size([1, 1339]) +Dataloader batch - labels shape: torch.Size([1, 1339]) +Dataloader batch - attn_mask shape: torch.Size([1, 1339]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1506 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1339) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1339) torch.int64 + loss_mask : 286 / 1339 tokens (21.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1339, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1339, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1339, 1, 576) torch.bfloat16 + out : (1, 1339) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 286 tokens) : 11.2181 + top-1 accuracy : 8/286 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1659]) +Dataloader batch - labels shape: torch.Size([1, 1659]) +Dataloader batch - attn_mask shape: torch.Size([1, 1659]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1507 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1659) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1659) torch.int64 + loss_mask : 407 / 1659 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1659, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1659, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1659, 1, 576) torch.bfloat16 + out : (1, 1659) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 407 tokens) : 11.2514 + top-1 accuracy : 19/407 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1562]) +Dataloader batch - labels shape: torch.Size([1, 1562]) +Dataloader batch - attn_mask shape: torch.Size([1, 1562]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1508 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1562) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1562) torch.int64 + loss_mask : 348 / 1562 tokens (22.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1562, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1562, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1562, 1, 576) torch.bfloat16 + out : (1, 1562) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2049 + top-1 accuracy : 9/348 = 2.6% + + [2026-04-29 14:07:21] iteration 477/ 500 | consumed samples: 1908 | elapsed time per iteration (ms): 4883.3 | throughput (token/sec/GPU): 1229.9 | learning rate: 1.511471E-05 | global batch size: 4 | lm loss: 1.124190E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1539]) +Dataloader batch - labels shape: torch.Size([1, 1539]) +Dataloader batch - attn_mask shape: torch.Size([1, 1539]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1509 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1539) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1539) torch.int64 + loss_mask : 359 / 1539 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1539, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1539, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1539, 1, 576) torch.bfloat16 + out : (1, 1539) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.1771 + top-1 accuracy : 12/359 = 3.3% + +Dataloader batch - tokens shape: torch.Size([1, 1714]) +Dataloader batch - labels shape: torch.Size([1, 1714]) +Dataloader batch - attn_mask shape: torch.Size([1, 1714]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1510 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1714) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1714) torch.int64 + loss_mask : 356 / 1714 tokens (20.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1714, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1714, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1714, 1, 576) torch.bfloat16 + out : (1, 1714) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 356 tokens) : 11.2154 + top-1 accuracy : 25/356 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1511 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 416 / 1546 tokens (26.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 416 tokens) : 11.3206 + top-1 accuracy : 12/416 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1204]) +Dataloader batch - labels shape: torch.Size([1, 1204]) +Dataloader batch - attn_mask shape: torch.Size([1, 1204]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1512 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1204) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1204) torch.int64 + loss_mask : 308 / 1204 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1204, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1204, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1204, 1, 576) torch.bfloat16 + out : (1, 1204) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 308 tokens) : 11.3168 + top-1 accuracy : 8/308 = 2.6% + + [2026-04-29 14:07:26] iteration 478/ 500 | consumed samples: 1912 | elapsed time per iteration (ms): 4906.6 | throughput (token/sec/GPU): 1223.4 | learning rate: 1.489749E-05 | global batch size: 4 | lm loss: 1.125796E+01 | loss scale: 1.0 | grad norm: 0.201 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1437]) +Dataloader batch - labels shape: torch.Size([1, 1437]) +Dataloader batch - attn_mask shape: torch.Size([1, 1437]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1513 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1437) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1437) torch.int64 + loss_mask : 382 / 1437 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1437, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1437, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1437, 1, 576) torch.bfloat16 + out : (1, 1437) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.3255 + top-1 accuracy : 5/382 = 1.3% + +Dataloader batch - tokens shape: torch.Size([1, 1417]) +Dataloader batch - labels shape: torch.Size([1, 1417]) +Dataloader batch - attn_mask shape: torch.Size([1, 1417]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1514 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1417) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1417) torch.int64 + loss_mask : 359 / 1417 tokens (25.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1417, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1417, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1417, 1, 576) torch.bfloat16 + out : (1, 1417) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 359 tokens) : 11.2890 + top-1 accuracy : 7/359 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1480]) +Dataloader batch - labels shape: torch.Size([1, 1480]) +Dataloader batch - attn_mask shape: torch.Size([1, 1480]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1515 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1480) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1480) torch.int64 + loss_mask : 326 / 1480 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1480, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1480, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1480, 1, 576) torch.bfloat16 + out : (1, 1480) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 326 tokens) : 11.1895 + top-1 accuracy : 13/326 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1017]) +Dataloader batch - labels shape: torch.Size([1, 1017]) +Dataloader batch - attn_mask shape: torch.Size([1, 1017]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1516 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1017) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1017) torch.int64 + loss_mask : 207 / 1017 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1017, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1017, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1017, 1, 576) torch.bfloat16 + out : (1, 1017) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 207 tokens) : 11.1882 + top-1 accuracy : 8/207 = 3.9% + + [2026-04-29 14:07:31] iteration 479/ 500 | consumed samples: 1916 | elapsed time per iteration (ms): 4669.5 | throughput (token/sec/GPU): 1146.0 | learning rate: 1.468168E-05 | global batch size: 4 | lm loss: 1.125812E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1554]) +Dataloader batch - labels shape: torch.Size([1, 1554]) +Dataloader batch - attn_mask shape: torch.Size([1, 1554]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1517 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1554) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1554) torch.int64 + loss_mask : 396 / 1554 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1554, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1554, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1554, 1, 576) torch.bfloat16 + out : (1, 1554) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 396 tokens) : 11.2889 + top-1 accuracy : 25/396 = 6.3% + +Dataloader batch - tokens shape: torch.Size([1, 1096]) +Dataloader batch - labels shape: torch.Size([1, 1096]) +Dataloader batch - attn_mask shape: torch.Size([1, 1096]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1518 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1096) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1096) torch.int64 + loss_mask : 262 / 1096 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1096, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1096, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1096, 1, 576) torch.bfloat16 + out : (1, 1096) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 262 tokens) : 11.3491 + top-1 accuracy : 14/262 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1421]) +Dataloader batch - labels shape: torch.Size([1, 1421]) +Dataloader batch - attn_mask shape: torch.Size([1, 1421]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1519 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1421) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1421) torch.int64 + loss_mask : 344 / 1421 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1421, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1421, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1421, 1, 576) torch.bfloat16 + out : (1, 1421) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 344 tokens) : 11.2638 + top-1 accuracy : 6/344 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1778]) +Dataloader batch - labels shape: torch.Size([1, 1778]) +Dataloader batch - attn_mask shape: torch.Size([1, 1778]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1520 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1778) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1778) torch.int64 + loss_mask : 410 / 1778 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1778, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1778, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1778, 1, 576) torch.bfloat16 + out : (1, 1778) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 410 tokens) : 11.2502 + top-1 accuracy : 23/410 = 5.6% + + [2026-04-29 14:07:36] iteration 480/ 500 | consumed samples: 1920 | elapsed time per iteration (ms): 4647.2 | throughput (token/sec/GPU): 1258.6 | learning rate: 1.446729E-05 | global batch size: 4 | lm loss: 1.128271E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (4633.17, 4633.17) + forward-compute ................................: (3437.56, 3437.56) + backward-compute ...............................: (1193.56, 1193.56) + batch-generator ................................: (414.07, 414.07) + layernorm-grads-all-reduce .....................: (0.02, 0.02) + embedding-grads-all-reduce .....................: (0.03, 0.03) + all-grads-sync .................................: (0.44, 0.44) + params-all-gather ..............................: (0.14, 0.14) + optimizer-copy-to-main-grad ....................: (0.11, 0.11) + optimizer-clip-main-grad .......................: (0.50, 0.50) + optimizer-count-zeros ..........................: (0.01, 0.01) + optimizer-inner-step ...........................: (1.27, 1.27) + optimizer-copy-main-to-model-params ............: (0.20, 0.20) + optimizer ......................................: (2.80, 2.80) +Dataloader batch - tokens shape: torch.Size([1, 1687]) +Dataloader batch - labels shape: torch.Size([1, 1687]) +Dataloader batch - attn_mask shape: torch.Size([1, 1687]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1521 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1687) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1687) torch.int64 + loss_mask : 435 / 1687 tokens (25.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1687, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1687, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1687, 1, 576) torch.bfloat16 + out : (1, 1687) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 435 tokens) : 11.2843 + top-1 accuracy : 16/435 = 3.7% + +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1522 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 413 / 1750 tokens (23.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2534 + top-1 accuracy : 21/413 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1090]) +Dataloader batch - labels shape: torch.Size([1, 1090]) +Dataloader batch - attn_mask shape: torch.Size([1, 1090]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1523 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1090) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1090) torch.int64 + loss_mask : 246 / 1090 tokens (22.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1090, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1090, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1090, 1, 576) torch.bfloat16 + out : (1, 1090) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 246 tokens) : 11.2771 + top-1 accuracy : 14/246 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1067]) +Dataloader batch - labels shape: torch.Size([1, 1067]) +Dataloader batch - attn_mask shape: torch.Size([1, 1067]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1524 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1067) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1067) torch.int64 + loss_mask : 236 / 1067 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1067, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1067, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1067, 1, 576) torch.bfloat16 + out : (1, 1067) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 236 tokens) : 11.2189 + top-1 accuracy : 6/236 = 2.5% + + [2026-04-29 14:07:40] iteration 481/ 500 | consumed samples: 1924 | elapsed time per iteration (ms): 4789.9 | throughput (token/sec/GPU): 1167.9 | learning rate: 1.425433E-05 | global batch size: 4 | lm loss: 1.126175E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1462]) +Dataloader batch - labels shape: torch.Size([1, 1462]) +Dataloader batch - attn_mask shape: torch.Size([1, 1462]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1525 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1462) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1462) torch.int64 + loss_mask : 332 / 1462 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1462, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1462, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1462, 1, 576) torch.bfloat16 + out : (1, 1462) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 332 tokens) : 11.2026 + top-1 accuracy : 7/332 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1043]) +Dataloader batch - labels shape: torch.Size([1, 1043]) +Dataloader batch - attn_mask shape: torch.Size([1, 1043]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1526 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1043) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1043) torch.int64 + loss_mask : 278 / 1043 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1043, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1043, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1043, 1, 576) torch.bfloat16 + out : (1, 1043) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 278 tokens) : 11.3088 + top-1 accuracy : 6/278 = 2.2% + +Dataloader batch - tokens shape: torch.Size([1, 1710]) +Dataloader batch - labels shape: torch.Size([1, 1710]) +Dataloader batch - attn_mask shape: torch.Size([1, 1710]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1527 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1710) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1710) torch.int64 + loss_mask : 347 / 1710 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1710, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1710, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1710, 1, 576) torch.bfloat16 + out : (1, 1710) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.2217 + top-1 accuracy : 17/347 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1288]) +Dataloader batch - labels shape: torch.Size([1, 1288]) +Dataloader batch - attn_mask shape: torch.Size([1, 1288]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1528 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1288) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1288) torch.int64 + loss_mask : 383 / 1288 tokens (29.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1288, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1288, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1288, 1, 576) torch.bfloat16 + out : (1, 1288) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.4065 + top-1 accuracy : 3/383 = 0.8% + + [2026-04-29 14:07:45] iteration 482/ 500 | consumed samples: 1928 | elapsed time per iteration (ms): 4553.9 | throughput (token/sec/GPU): 1208.4 | learning rate: 1.404281E-05 | global batch size: 4 | lm loss: 1.128785E+01 | loss scale: 1.0 | grad norm: 0.213 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1778]) +Dataloader batch - labels shape: torch.Size([1, 1778]) +Dataloader batch - attn_mask shape: torch.Size([1, 1778]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1529 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1778) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1778) torch.int64 + loss_mask : 411 / 1778 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1778, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1778, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1778, 1, 576) torch.bfloat16 + out : (1, 1778) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 411 tokens) : 11.2581 + top-1 accuracy : 21/411 = 5.1% + +Dataloader batch - tokens shape: torch.Size([1, 1828]) +Dataloader batch - labels shape: torch.Size([1, 1828]) +Dataloader batch - attn_mask shape: torch.Size([1, 1828]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1530 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1828) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1828) torch.int64 + loss_mask : 475 / 1828 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1828, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1828, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1828, 1, 576) torch.bfloat16 + out : (1, 1828) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 475 tokens) : 11.3228 + top-1 accuracy : 12/475 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1730]) +Dataloader batch - labels shape: torch.Size([1, 1730]) +Dataloader batch - attn_mask shape: torch.Size([1, 1730]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1531 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1730) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1730) torch.int64 + loss_mask : 383 / 1730 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1730, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1730, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1730, 1, 576) torch.bfloat16 + out : (1, 1730) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.2425 + top-1 accuracy : 26/383 = 6.8% + +Dataloader batch - tokens shape: torch.Size([1, 1678]) +Dataloader batch - labels shape: torch.Size([1, 1678]) +Dataloader batch - attn_mask shape: torch.Size([1, 1678]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1532 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1678) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1678) torch.int64 + loss_mask : 403 / 1678 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1678, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1678, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1678, 1, 576) torch.bfloat16 + out : (1, 1678) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 403 tokens) : 11.2870 + top-1 accuracy : 7/403 = 1.7% + + [2026-04-29 14:07:50] iteration 483/ 500 | consumed samples: 1932 | elapsed time per iteration (ms): 5407.2 | throughput (token/sec/GPU): 1297.2 | learning rate: 1.383273E-05 | global batch size: 4 | lm loss: 1.127985E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1899]) +Dataloader batch - labels shape: torch.Size([1, 1899]) +Dataloader batch - attn_mask shape: torch.Size([1, 1899]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1533 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1899) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1899) torch.int64 + loss_mask : 533 / 1899 tokens (28.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1899, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1899, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1899, 1, 576) torch.bfloat16 + out : (1, 1899) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 533 tokens) : 11.3547 + top-1 accuracy : 9/533 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1812]) +Dataloader batch - labels shape: torch.Size([1, 1812]) +Dataloader batch - attn_mask shape: torch.Size([1, 1812]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1534 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1812) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1812) torch.int64 + loss_mask : 520 / 1812 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1812, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1812, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1812, 1, 576) torch.bfloat16 + out : (1, 1812) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 520 tokens) : 11.3370 + top-1 accuracy : 9/520 = 1.7% + +Dataloader batch - tokens shape: torch.Size([1, 1211]) +Dataloader batch - labels shape: torch.Size([1, 1211]) +Dataloader batch - attn_mask shape: torch.Size([1, 1211]) +Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) + +==================================================================== + PIPELINE — STEP 1535 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1211) torch.int64 + images : (21, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1211) torch.int64 + loss_mask : 296 / 1211 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (21, 3, 1024, 1024) torch.bfloat16 + out : (21, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (21, 3072) + out : (21, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1211, 1, 576) torch.bfloat16 grad=False + image slots : 21 tokens replaced with vision embeddings + combined : (1211, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1211, 1, 576) torch.bfloat16 + out : (1, 1211) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 296 tokens) : 11.3540 + top-1 accuracy : 2/296 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1464]) +Dataloader batch - labels shape: torch.Size([1, 1464]) +Dataloader batch - attn_mask shape: torch.Size([1, 1464]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1536 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1464) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1464) torch.int64 + loss_mask : 368 / 1464 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1464, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1464, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1464, 1, 576) torch.bfloat16 + out : (1, 1464) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2662 + top-1 accuracy : 10/368 = 2.7% + + [2026-04-29 14:07:55] iteration 484/ 500 | consumed samples: 1936 | elapsed time per iteration (ms): 4977.2 | throughput (token/sec/GPU): 1283.0 | learning rate: 1.362411E-05 | global batch size: 4 | lm loss: 1.133028E+01 | loss scale: 1.0 | grad norm: 0.184 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1381]) +Dataloader batch - labels shape: torch.Size([1, 1381]) +Dataloader batch - attn_mask shape: torch.Size([1, 1381]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1537 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1381) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1381) torch.int64 + loss_mask : 329 / 1381 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1381, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1381, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1381, 1, 576) torch.bfloat16 + out : (1, 1381) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 329 tokens) : 11.2298 + top-1 accuracy : 4/329 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 1391]) +Dataloader batch - labels shape: torch.Size([1, 1391]) +Dataloader batch - attn_mask shape: torch.Size([1, 1391]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1538 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1391) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1391) torch.int64 + loss_mask : 341 / 1391 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1391, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1391, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1391, 1, 576) torch.bfloat16 + out : (1, 1391) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 341 tokens) : 11.2992 + top-1 accuracy : 22/341 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1420]) +Dataloader batch - labels shape: torch.Size([1, 1420]) +Dataloader batch - attn_mask shape: torch.Size([1, 1420]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1539 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1420) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1420) torch.int64 + loss_mask : 323 / 1420 tokens (22.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1420, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1420, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1420, 1, 576) torch.bfloat16 + out : (1, 1420) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.2580 + top-1 accuracy : 17/323 = 5.3% + +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1540 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 274 / 1364 tokens (20.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 274 tokens) : 11.1123 + top-1 accuracy : 17/274 = 6.2% + + [2026-04-29 14:08:00] iteration 485/ 500 | consumed samples: 1940 | elapsed time per iteration (ms): 4650.3 | throughput (token/sec/GPU): 1194.8 | learning rate: 1.341694E-05 | global batch size: 4 | lm loss: 1.123029E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1590]) +Dataloader batch - labels shape: torch.Size([1, 1590]) +Dataloader batch - attn_mask shape: torch.Size([1, 1590]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1541 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1590) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1590) torch.int64 + loss_mask : 339 / 1590 tokens (21.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1590, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1590, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1590, 1, 576) torch.bfloat16 + out : (1, 1590) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.1850 + top-1 accuracy : 30/339 = 8.8% + +Dataloader batch - tokens shape: torch.Size([1, 959]) +Dataloader batch - labels shape: torch.Size([1, 959]) +Dataloader batch - attn_mask shape: torch.Size([1, 959]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1542 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 959) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 959) torch.int64 + loss_mask : 208 / 959 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (959, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (959, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (959, 1, 576) torch.bfloat16 + out : (1, 959) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 208 tokens) : 11.1609 + top-1 accuracy : 18/208 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1863]) +Dataloader batch - labels shape: torch.Size([1, 1863]) +Dataloader batch - attn_mask shape: torch.Size([1, 1863]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1543 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1863) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1863) torch.int64 + loss_mask : 515 / 1863 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1863, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1863, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1863, 1, 576) torch.bfloat16 + out : (1, 1863) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 515 tokens) : 11.3316 + top-1 accuracy : 13/515 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1885]) +Dataloader batch - labels shape: torch.Size([1, 1885]) +Dataloader batch - attn_mask shape: torch.Size([1, 1885]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1544 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1885) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1885) torch.int64 + loss_mask : 542 / 1885 tokens (28.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1885, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1885, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1885, 1, 576) torch.bfloat16 + out : (1, 1885) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 542 tokens) : 11.3700 + top-1 accuracy : 17/542 = 3.1% + + [2026-04-29 14:08:05] iteration 486/ 500 | consumed samples: 1944 | elapsed time per iteration (ms): 4974.0 | throughput (token/sec/GPU): 1266.0 | learning rate: 1.321125E-05 | global batch size: 4 | lm loss: 1.129146E+01 | loss scale: 1.0 | grad norm: 0.217 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1545 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 357 / 1546 tokens (23.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 357 tokens) : 11.2731 + top-1 accuracy : 5/357 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1242]) +Dataloader batch - labels shape: torch.Size([1, 1242]) +Dataloader batch - attn_mask shape: torch.Size([1, 1242]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1546 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1242) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1242) torch.int64 + loss_mask : 315 / 1242 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1242, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1242, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1242, 1, 576) torch.bfloat16 + out : (1, 1242) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 315 tokens) : 11.3054 + top-1 accuracy : 10/315 = 3.2% + +Dataloader batch - tokens shape: torch.Size([1, 1533]) +Dataloader batch - labels shape: torch.Size([1, 1533]) +Dataloader batch - attn_mask shape: torch.Size([1, 1533]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1547 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1533) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1533) torch.int64 + loss_mask : 386 / 1533 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1533, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1533, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1533, 1, 576) torch.bfloat16 + out : (1, 1533) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.3142 + top-1 accuracy : 18/386 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1471]) +Dataloader batch - labels shape: torch.Size([1, 1471]) +Dataloader batch - attn_mask shape: torch.Size([1, 1471]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1548 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1471) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1471) torch.int64 + loss_mask : 319 / 1471 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1471, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1471, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1471, 1, 576) torch.bfloat16 + out : (1, 1471) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 319 tokens) : 11.1810 + top-1 accuracy : 32/319 = 10.0% + + [2026-04-29 14:08:10] iteration 487/ 500 | consumed samples: 1948 | elapsed time per iteration (ms): 4861.2 | throughput (token/sec/GPU): 1191.5 | learning rate: 1.300703E-05 | global batch size: 4 | lm loss: 1.127067E+01 | loss scale: 1.0 | grad norm: 0.188 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1364]) +Dataloader batch - labels shape: torch.Size([1, 1364]) +Dataloader batch - attn_mask shape: torch.Size([1, 1364]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1549 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1364) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1364) torch.int64 + loss_mask : 287 / 1364 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1364, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1364, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1364, 1, 576) torch.bfloat16 + out : (1, 1364) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 287 tokens) : 11.1446 + top-1 accuracy : 14/287 = 4.9% + +Dataloader batch - tokens shape: torch.Size([1, 1213]) +Dataloader batch - labels shape: torch.Size([1, 1213]) +Dataloader batch - attn_mask shape: torch.Size([1, 1213]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1550 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1213) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1213) torch.int64 + loss_mask : 294 / 1213 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1213, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1213, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1213, 1, 576) torch.bfloat16 + out : (1, 1213) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 294 tokens) : 11.2555 + top-1 accuracy : 4/294 = 1.4% + +Dataloader batch - tokens shape: torch.Size([1, 1119]) +Dataloader batch - labels shape: torch.Size([1, 1119]) +Dataloader batch - attn_mask shape: torch.Size([1, 1119]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1551 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1119) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1119) torch.int64 + loss_mask : 263 / 1119 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1119, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1119, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1119, 1, 576) torch.bfloat16 + out : (1, 1119) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 263 tokens) : 11.2451 + top-1 accuracy : 23/263 = 8.7% + +Dataloader batch - tokens shape: torch.Size([1, 1508]) +Dataloader batch - labels shape: torch.Size([1, 1508]) +Dataloader batch - attn_mask shape: torch.Size([1, 1508]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1552 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1508) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1508) torch.int64 + loss_mask : 379 / 1508 tokens (25.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1508, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1508, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1508, 1, 576) torch.bfloat16 + out : (1, 1508) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 379 tokens) : 11.2847 + top-1 accuracy : 28/379 = 7.4% + + [2026-04-29 14:08:14] iteration 488/ 500 | consumed samples: 1952 | elapsed time per iteration (ms): 4250.1 | throughput (token/sec/GPU): 1224.4 | learning rate: 1.280430E-05 | global batch size: 4 | lm loss: 1.123629E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1336]) +Dataloader batch - labels shape: torch.Size([1, 1336]) +Dataloader batch - attn_mask shape: torch.Size([1, 1336]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1553 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1336) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1336) torch.int64 + loss_mask : 374 / 1336 tokens (28.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1336, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1336, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1336, 1, 576) torch.bfloat16 + out : (1, 1336) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 374 tokens) : 11.3531 + top-1 accuracy : 13/374 = 3.5% + +Dataloader batch - tokens shape: torch.Size([1, 1441]) +Dataloader batch - labels shape: torch.Size([1, 1441]) +Dataloader batch - attn_mask shape: torch.Size([1, 1441]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 1554 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1441) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1441) torch.int64 + loss_mask : 417 / 1441 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1441, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1441, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1441, 1, 576) torch.bfloat16 + out : (1, 1441) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 417 tokens) : 11.3869 + top-1 accuracy : 1/417 = 0.2% + +Dataloader batch - tokens shape: torch.Size([1, 1299]) +Dataloader batch - labels shape: torch.Size([1, 1299]) +Dataloader batch - attn_mask shape: torch.Size([1, 1299]) +Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) + +==================================================================== + PIPELINE — STEP 1555 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1299) torch.int64 + images : (23, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1299) torch.int64 + loss_mask : 339 / 1299 tokens (26.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (23, 3, 1024, 1024) torch.bfloat16 + out : (23, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (23, 3072) + out : (23, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1299, 1, 576) torch.bfloat16 grad=False + image slots : 23 tokens replaced with vision embeddings + combined : (1299, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1299, 1, 576) torch.bfloat16 + out : (1, 1299) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 339 tokens) : 11.3072 + top-1 accuracy : 6/339 = 1.8% + +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1556 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 347 / 1525 tokens (22.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 347 tokens) : 11.1929 + top-1 accuracy : 20/347 = 5.8% + + [2026-04-29 14:08:19] iteration 489/ 500 | consumed samples: 1956 | elapsed time per iteration (ms): 4387.2 | throughput (token/sec/GPU): 1276.7 | learning rate: 1.260307E-05 | global batch size: 4 | lm loss: 1.131447E+01 | loss scale: 1.0 | grad norm: 0.257 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1759]) +Dataloader batch - labels shape: torch.Size([1, 1759]) +Dataloader batch - attn_mask shape: torch.Size([1, 1759]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1557 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1759) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1759) torch.int64 + loss_mask : 413 / 1759 tokens (23.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1759, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1759, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1759, 1, 576) torch.bfloat16 + out : (1, 1759) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 413 tokens) : 11.2797 + top-1 accuracy : 34/413 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1580]) +Dataloader batch - labels shape: torch.Size([1, 1580]) +Dataloader batch - attn_mask shape: torch.Size([1, 1580]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1558 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1580) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1580) torch.int64 + loss_mask : 420 / 1580 tokens (26.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1580, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1580, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1580, 1, 576) torch.bfloat16 + out : (1, 1580) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 420 tokens) : 11.3710 + top-1 accuracy : 8/420 = 1.9% + +Dataloader batch - tokens shape: torch.Size([1, 1952]) +Dataloader batch - labels shape: torch.Size([1, 1952]) +Dataloader batch - attn_mask shape: torch.Size([1, 1952]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1559 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1952) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1952) torch.int64 + loss_mask : 582 / 1952 tokens (29.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1952, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1952, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1952, 1, 576) torch.bfloat16 + out : (1, 1952) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 582 tokens) : 11.3992 + top-1 accuracy : 5/582 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1520]) +Dataloader batch - labels shape: torch.Size([1, 1520]) +Dataloader batch - attn_mask shape: torch.Size([1, 1520]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1560 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1520) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1520) torch.int64 + loss_mask : 388 / 1520 tokens (25.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1520, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1520, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1520, 1, 576) torch.bfloat16 + out : (1, 1520) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 388 tokens) : 11.3180 + top-1 accuracy : 16/388 = 4.1% + + [2026-04-29 14:08:24] iteration 490/ 500 | consumed samples: 1960 | elapsed time per iteration (ms): 5158.4 | throughput (token/sec/GPU): 1320.4 | learning rate: 1.240333E-05 | global batch size: 4 | lm loss: 1.134782E+01 | loss scale: 1.0 | grad norm: 0.167 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1180]) +Dataloader batch - labels shape: torch.Size([1, 1180]) +Dataloader batch - attn_mask shape: torch.Size([1, 1180]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1561 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1180) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1180) torch.int64 + loss_mask : 260 / 1180 tokens (22.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1180, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1180, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1180, 1, 576) torch.bfloat16 + out : (1, 1180) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 260 tokens) : 11.2341 + top-1 accuracy : 15/260 = 5.8% + +Dataloader batch - tokens shape: torch.Size([1, 1440]) +Dataloader batch - labels shape: torch.Size([1, 1440]) +Dataloader batch - attn_mask shape: torch.Size([1, 1440]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1562 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1440) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1440) torch.int64 + loss_mask : 366 / 1440 tokens (25.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1440, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1440, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1440, 1, 576) torch.bfloat16 + out : (1, 1440) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 366 tokens) : 11.3004 + top-1 accuracy : 30/366 = 8.2% + +Dataloader batch - tokens shape: torch.Size([1, 1476]) +Dataloader batch - labels shape: torch.Size([1, 1476]) +Dataloader batch - attn_mask shape: torch.Size([1, 1476]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1563 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1476) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1476) torch.int64 + loss_mask : 427 / 1476 tokens (28.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1476, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1476, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1476, 1, 576) torch.bfloat16 + out : (1, 1476) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 427 tokens) : 11.3783 + top-1 accuracy : 5/427 = 1.2% + +Dataloader batch - tokens shape: torch.Size([1, 976]) +Dataloader batch - labels shape: torch.Size([1, 976]) +Dataloader batch - attn_mask shape: torch.Size([1, 976]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1564 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 976) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 976) torch.int64 + loss_mask : 224 / 976 tokens (23.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (976, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (976, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (976, 1, 576) torch.bfloat16 + out : (1, 976) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 224 tokens) : 11.2594 + top-1 accuracy : 5/224 = 2.2% + + [2026-04-29 14:08:28] iteration 491/ 500 | consumed samples: 1964 | elapsed time per iteration (ms): 4220.9 | throughput (token/sec/GPU): 1201.6 | learning rate: 1.220511E-05 | global batch size: 4 | lm loss: 1.130575E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1729]) +Dataloader batch - labels shape: torch.Size([1, 1729]) +Dataloader batch - attn_mask shape: torch.Size([1, 1729]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1565 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1729) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1729) torch.int64 + loss_mask : 419 / 1729 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1729, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1729, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1729, 1, 576) torch.bfloat16 + out : (1, 1729) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 419 tokens) : 11.3419 + top-1 accuracy : 9/419 = 2.1% + +Dataloader batch - tokens shape: torch.Size([1, 1567]) +Dataloader batch - labels shape: torch.Size([1, 1567]) +Dataloader batch - attn_mask shape: torch.Size([1, 1567]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1566 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1567) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1567) torch.int64 + loss_mask : 386 / 1567 tokens (24.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1567, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1567, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1567, 1, 576) torch.bfloat16 + out : (1, 1567) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2434 + top-1 accuracy : 11/386 = 2.8% + +Dataloader batch - tokens shape: torch.Size([1, 1639]) +Dataloader batch - labels shape: torch.Size([1, 1639]) +Dataloader batch - attn_mask shape: torch.Size([1, 1639]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1567 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1639) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1639) torch.int64 + loss_mask : 383 / 1639 tokens (23.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1639, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1639, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1639, 1, 576) torch.bfloat16 + out : (1, 1639) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 383 tokens) : 11.2370 + top-1 accuracy : 16/383 = 4.2% + +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1568 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 348 / 1436 tokens (24.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 348 tokens) : 11.2805 + top-1 accuracy : 19/348 = 5.5% + + [2026-04-29 14:08:33] iteration 492/ 500 | consumed samples: 1968 | elapsed time per iteration (ms): 5127.4 | throughput (token/sec/GPU): 1242.5 | learning rate: 1.200840E-05 | global batch size: 4 | lm loss: 1.127707E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1750]) +Dataloader batch - labels shape: torch.Size([1, 1750]) +Dataloader batch - attn_mask shape: torch.Size([1, 1750]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1569 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1750) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1750) torch.int64 + loss_mask : 386 / 1750 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1750, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1750, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1750, 1, 576) torch.bfloat16 + out : (1, 1750) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 386 tokens) : 11.2310 + top-1 accuracy : 27/386 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1570 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 382 / 1525 tokens (25.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 382 tokens) : 11.2868 + top-1 accuracy : 13/382 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1203]) +Dataloader batch - labels shape: torch.Size([1, 1203]) +Dataloader batch - attn_mask shape: torch.Size([1, 1203]) +Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) + +==================================================================== + PIPELINE — STEP 1571 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1203) torch.int64 + images : (22, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1203) torch.int64 + loss_mask : 275 / 1203 tokens (22.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (22, 3, 1024, 1024) torch.bfloat16 + out : (22, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (22, 3072) + out : (22, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1203, 1, 576) torch.bfloat16 grad=False + image slots : 22 tokens replaced with vision embeddings + combined : (1203, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1203, 1, 576) torch.bfloat16 + out : (1, 1203) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 275 tokens) : 11.2741 + top-1 accuracy : 2/275 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1475]) +Dataloader batch - labels shape: torch.Size([1, 1475]) +Dataloader batch - attn_mask shape: torch.Size([1, 1475]) +Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) + +==================================================================== + PIPELINE — STEP 1572 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1475) torch.int64 + images : (24, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1475) torch.int64 + loss_mask : 465 / 1475 tokens (31.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (24, 3, 1024, 1024) torch.bfloat16 + out : (24, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (24, 3072) + out : (24, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1475, 1, 576) torch.bfloat16 grad=False + image slots : 24 tokens replaced with vision embeddings + combined : (1475, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1475, 1, 576) torch.bfloat16 + out : (1, 1475) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 465 tokens) : 11.4555 + top-1 accuracy : 15/465 = 3.2% + + [2026-04-29 14:08:38] iteration 493/ 500 | consumed samples: 1972 | elapsed time per iteration (ms): 4720.6 | throughput (token/sec/GPU): 1261.1 | learning rate: 1.181322E-05 | global batch size: 4 | lm loss: 1.132224E+01 | loss scale: 1.0 | grad norm: 0.192 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1721]) +Dataloader batch - labels shape: torch.Size([1, 1721]) +Dataloader batch - attn_mask shape: torch.Size([1, 1721]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1573 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1721) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1721) torch.int64 + loss_mask : 361 / 1721 tokens (21.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1721, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1721, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1721, 1, 576) torch.bfloat16 + out : (1, 1721) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 361 tokens) : 11.1916 + top-1 accuracy : 33/361 = 9.1% + +Dataloader batch - tokens shape: torch.Size([1, 1749]) +Dataloader batch - labels shape: torch.Size([1, 1749]) +Dataloader batch - attn_mask shape: torch.Size([1, 1749]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1574 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1749) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1749) torch.int64 + loss_mask : 387 / 1749 tokens (22.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1749, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1749, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1749, 1, 576) torch.bfloat16 + out : (1, 1749) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 387 tokens) : 11.2710 + top-1 accuracy : 25/387 = 6.5% + +Dataloader batch - tokens shape: torch.Size([1, 1846]) +Dataloader batch - labels shape: torch.Size([1, 1846]) +Dataloader batch - attn_mask shape: torch.Size([1, 1846]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1575 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1846) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1846) torch.int64 + loss_mask : 493 / 1846 tokens (26.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1846, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1846, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1846, 1, 576) torch.bfloat16 + out : (1, 1846) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 493 tokens) : 11.3375 + top-1 accuracy : 13/493 = 2.6% + +Dataloader batch - tokens shape: torch.Size([1, 1916]) +Dataloader batch - labels shape: torch.Size([1, 1916]) +Dataloader batch - attn_mask shape: torch.Size([1, 1916]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1576 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1916) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1916) torch.int64 + loss_mask : 549 / 1916 tokens (28.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1916, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1916, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1916, 1, 576) torch.bfloat16 + out : (1, 1916) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 549 tokens) : 11.3890 + top-1 accuracy : 8/549 = 1.5% + + [2026-04-29 14:08:43] iteration 494/ 500 | consumed samples: 1976 | elapsed time per iteration (ms): 5454.9 | throughput (token/sec/GPU): 1325.8 | learning rate: 1.161958E-05 | global batch size: 4 | lm loss: 1.130949E+01 | loss scale: 1.0 | grad norm: 0.187 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1664]) +Dataloader batch - labels shape: torch.Size([1, 1664]) +Dataloader batch - attn_mask shape: torch.Size([1, 1664]) +Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) + +==================================================================== + PIPELINE — STEP 1577 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1664) torch.int64 + images : (30, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1664) torch.int64 + loss_mask : 396 / 1664 tokens (23.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (30, 3, 1024, 1024) torch.bfloat16 + out : (30, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (30, 3072) + out : (30, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1664, 1, 576) torch.bfloat16 grad=False + image slots : 30 tokens replaced with vision embeddings + combined : (1664, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1664, 1, 576) torch.bfloat16 + out : (1, 1664) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 396 tokens) : 11.1905 + top-1 accuracy : 29/396 = 7.3% + +Dataloader batch - tokens shape: torch.Size([1, 1778]) +Dataloader batch - labels shape: torch.Size([1, 1778]) +Dataloader batch - attn_mask shape: torch.Size([1, 1778]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1578 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1778) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1778) torch.int64 + loss_mask : 470 / 1778 tokens (26.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1778, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1778, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1778, 1, 576) torch.bfloat16 + out : (1, 1778) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 470 tokens) : 11.3403 + top-1 accuracy : 22/470 = 4.7% + +Dataloader batch - tokens shape: torch.Size([1, 1851]) +Dataloader batch - labels shape: torch.Size([1, 1851]) +Dataloader batch - attn_mask shape: torch.Size([1, 1851]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1579 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1851) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1851) torch.int64 + loss_mask : 481 / 1851 tokens (26.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1851, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1851, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1851, 1, 576) torch.bfloat16 + out : (1, 1851) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 481 tokens) : 11.3379 + top-1 accuracy : 12/481 = 2.5% + +Dataloader batch - tokens shape: torch.Size([1, 1409]) +Dataloader batch - labels shape: torch.Size([1, 1409]) +Dataloader batch - attn_mask shape: torch.Size([1, 1409]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1580 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1409) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1409) torch.int64 + loss_mask : 338 / 1409 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1409, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1409, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1409, 1, 576) torch.bfloat16 + out : (1, 1409) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 338 tokens) : 11.2766 + top-1 accuracy : 14/338 = 4.1% + + [2026-04-29 14:08:49] iteration 495/ 500 | consumed samples: 1980 | elapsed time per iteration (ms): 5377.0 | throughput (token/sec/GPU): 1246.4 | learning rate: 1.142747E-05 | global batch size: 4 | lm loss: 1.129167E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1525]) +Dataloader batch - labels shape: torch.Size([1, 1525]) +Dataloader batch - attn_mask shape: torch.Size([1, 1525]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1581 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1525) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1525) torch.int64 + loss_mask : 391 / 1525 tokens (25.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1525, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1525, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1525, 1, 576) torch.bfloat16 + out : (1, 1525) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 391 tokens) : 11.3064 + top-1 accuracy : 16/391 = 4.1% + +Dataloader batch - tokens shape: torch.Size([1, 1429]) +Dataloader batch - labels shape: torch.Size([1, 1429]) +Dataloader batch - attn_mask shape: torch.Size([1, 1429]) +Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) + +==================================================================== + PIPELINE — STEP 1582 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1429) torch.int64 + images : (25, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1429) torch.int64 + loss_mask : 395 / 1429 tokens (27.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (25, 3, 1024, 1024) torch.bfloat16 + out : (25, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (25, 3072) + out : (25, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1429, 1, 576) torch.bfloat16 grad=False + image slots : 25 tokens replaced with vision embeddings + combined : (1429, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1429, 1, 576) torch.bfloat16 + out : (1, 1429) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 395 tokens) : 11.3260 + top-1 accuracy : 9/395 = 2.3% + +Dataloader batch - tokens shape: torch.Size([1, 1010]) +Dataloader batch - labels shape: torch.Size([1, 1010]) +Dataloader batch - attn_mask shape: torch.Size([1, 1010]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1583 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1010) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1010) torch.int64 + loss_mask : 242 / 1010 tokens (24.0%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1010, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1010, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1010, 1, 576) torch.bfloat16 + out : (1, 1010) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 242 tokens) : 11.2908 + top-1 accuracy : 2/242 = 0.8% + +Dataloader batch - tokens shape: torch.Size([1, 1460]) +Dataloader batch - labels shape: torch.Size([1, 1460]) +Dataloader batch - attn_mask shape: torch.Size([1, 1460]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1584 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1460) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1460) torch.int64 + loss_mask : 271 / 1460 tokens (18.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1460, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1460, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1460, 1, 576) torch.bfloat16 + out : (1, 1460) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 271 tokens) : 11.1549 + top-1 accuracy : 33/271 = 12.2% + + [2026-04-29 14:08:53] iteration 496/ 500 | consumed samples: 1984 | elapsed time per iteration (ms): 4434.6 | throughput (token/sec/GPU): 1223.1 | learning rate: 1.123692E-05 | global batch size: 4 | lm loss: 1.127784E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1735]) +Dataloader batch - labels shape: torch.Size([1, 1735]) +Dataloader batch - attn_mask shape: torch.Size([1, 1735]) +Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) + +==================================================================== + PIPELINE — STEP 1585 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1735) torch.int64 + images : (31, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1735) torch.int64 + loss_mask : 438 / 1735 tokens (25.2%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (31, 3, 1024, 1024) torch.bfloat16 + out : (31, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (31, 3072) + out : (31, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1735, 1, 576) torch.bfloat16 grad=False + image slots : 31 tokens replaced with vision embeddings + combined : (1735, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1735, 1, 576) torch.bfloat16 + out : (1, 1735) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 438 tokens) : 11.2687 + top-1 accuracy : 4/438 = 0.9% + +Dataloader batch - tokens shape: torch.Size([1, 1688]) +Dataloader batch - labels shape: torch.Size([1, 1688]) +Dataloader batch - attn_mask shape: torch.Size([1, 1688]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1586 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1688) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1688) torch.int64 + loss_mask : 365 / 1688 tokens (21.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1688, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1688, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1688, 1, 576) torch.bfloat16 + out : (1, 1688) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 365 tokens) : 11.2427 + top-1 accuracy : 13/365 = 3.6% + +Dataloader batch - tokens shape: torch.Size([1, 1408]) +Dataloader batch - labels shape: torch.Size([1, 1408]) +Dataloader batch - attn_mask shape: torch.Size([1, 1408]) +Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) + +==================================================================== + PIPELINE — STEP 1587 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1408) torch.int64 + images : (26, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1408) torch.int64 + loss_mask : 340 / 1408 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (26, 3, 1024, 1024) torch.bfloat16 + out : (26, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (26, 3072) + out : (26, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1408, 1, 576) torch.bfloat16 grad=False + image slots : 26 tokens replaced with vision embeddings + combined : (1408, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1408, 1, 576) torch.bfloat16 + out : (1, 1408) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 340 tokens) : 11.2070 + top-1 accuracy : 10/340 = 2.9% + +Dataloader batch - tokens shape: torch.Size([1, 1469]) +Dataloader batch - labels shape: torch.Size([1, 1469]) +Dataloader batch - attn_mask shape: torch.Size([1, 1469]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1588 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1469) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1469) torch.int64 + loss_mask : 303 / 1469 tokens (20.6%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1469, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1469, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1469, 1, 576) torch.bfloat16 + out : (1, 1469) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 303 tokens) : 11.1586 + top-1 accuracy : 31/303 = 10.2% + + [2026-04-29 14:08:58] iteration 497/ 500 | consumed samples: 1988 | elapsed time per iteration (ms): 5180.5 | throughput (token/sec/GPU): 1216.1 | learning rate: 1.104791E-05 | global batch size: 4 | lm loss: 1.122456E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1487]) +Dataloader batch - labels shape: torch.Size([1, 1487]) +Dataloader batch - attn_mask shape: torch.Size([1, 1487]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1589 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1487) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1487) torch.int64 + loss_mask : 323 / 1487 tokens (21.7%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1487, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1487, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1487, 1, 576) torch.bfloat16 + out : (1, 1487) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 323 tokens) : 11.2317 + top-1 accuracy : 15/323 = 4.6% + +Dataloader batch - tokens shape: torch.Size([1, 1050]) +Dataloader batch - labels shape: torch.Size([1, 1050]) +Dataloader batch - attn_mask shape: torch.Size([1, 1050]) +Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) + +==================================================================== + PIPELINE — STEP 1590 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1050) torch.int64 + images : (20, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1050) torch.int64 + loss_mask : 229 / 1050 tokens (21.8%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (20, 3, 1024, 1024) torch.bfloat16 + out : (20, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (20, 3072) + out : (20, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1050, 1, 576) torch.bfloat16 grad=False + image slots : 20 tokens replaced with vision embeddings + combined : (1050, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1050, 1, 576) torch.bfloat16 + out : (1, 1050) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 229 tokens) : 11.2055 + top-1 accuracy : 22/229 = 9.6% + +Dataloader batch - tokens shape: torch.Size([1, 956]) +Dataloader batch - labels shape: torch.Size([1, 956]) +Dataloader batch - attn_mask shape: torch.Size([1, 956]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1591 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 956) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 956) torch.int64 + loss_mask : 209 / 956 tokens (21.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (956, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (956, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (956, 1, 576) torch.bfloat16 + out : (1, 956) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 209 tokens) : 11.1843 + top-1 accuracy : 12/209 = 5.7% + +Dataloader batch - tokens shape: torch.Size([1, 1011]) +Dataloader batch - labels shape: torch.Size([1, 1011]) +Dataloader batch - attn_mask shape: torch.Size([1, 1011]) +Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) + +==================================================================== + PIPELINE — STEP 1592 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1011) torch.int64 + images : (18, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1011) torch.int64 + loss_mask : 244 / 1011 tokens (24.1%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (18, 3, 1024, 1024) torch.bfloat16 + out : (18, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (18, 3072) + out : (18, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1011, 1, 576) torch.bfloat16 grad=False + image slots : 18 tokens replaced with vision embeddings + combined : (1011, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1011, 1, 576) torch.bfloat16 + out : (1, 1011) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 244 tokens) : 11.2646 + top-1 accuracy : 1/244 = 0.4% + + [2026-04-29 14:09:02] iteration 498/ 500 | consumed samples: 1992 | elapsed time per iteration (ms): 3952.8 | throughput (token/sec/GPU): 1139.5 | learning rate: 1.086048E-05 | global batch size: 4 | lm loss: 1.122386E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1546]) +Dataloader batch - labels shape: torch.Size([1, 1546]) +Dataloader batch - attn_mask shape: torch.Size([1, 1546]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1593 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1546) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1546) torch.int64 + loss_mask : 369 / 1546 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1546, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1546, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1546, 1, 576) torch.bfloat16 + out : (1, 1546) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2567 + top-1 accuracy : 22/369 = 6.0% + +Dataloader batch - tokens shape: torch.Size([1, 854]) +Dataloader batch - labels shape: torch.Size([1, 854]) +Dataloader batch - attn_mask shape: torch.Size([1, 854]) +Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) + +==================================================================== + PIPELINE — STEP 1594 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 854) torch.int64 + images : (17, 3, 1024, 1024) torch.bfloat16 + labels : (1, 854) torch.int64 + loss_mask : 174 / 854 tokens (20.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (17, 3, 1024, 1024) torch.bfloat16 + out : (17, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (17, 3072) + out : (17, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (854, 1, 576) torch.bfloat16 grad=False + image slots : 17 tokens replaced with vision embeddings + combined : (854, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (854, 1, 576) torch.bfloat16 + out : (1, 854) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 174 tokens) : 11.1691 + top-1 accuracy : 7/174 = 4.0% + +Dataloader batch - tokens shape: torch.Size([1, 1505]) +Dataloader batch - labels shape: torch.Size([1, 1505]) +Dataloader batch - attn_mask shape: torch.Size([1, 1505]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1595 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1505) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1505) torch.int64 + loss_mask : 368 / 1505 tokens (24.5%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1505, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1505, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1505, 1, 576) torch.bfloat16 + out : (1, 1505) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 368 tokens) : 11.2868 + top-1 accuracy : 23/368 = 6.2% + +Dataloader batch - tokens shape: torch.Size([1, 1522]) +Dataloader batch - labels shape: torch.Size([1, 1522]) +Dataloader batch - attn_mask shape: torch.Size([1, 1522]) +Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) + +==================================================================== + PIPELINE — STEP 1596 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1522) torch.int64 + images : (27, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1522) torch.int64 + loss_mask : 372 / 1522 tokens (24.4%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (27, 3, 1024, 1024) torch.bfloat16 + out : (27, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (27, 3072) + out : (27, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1522, 1, 576) torch.bfloat16 grad=False + image slots : 27 tokens replaced with vision embeddings + combined : (1522, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1522, 1, 576) torch.bfloat16 + out : (1, 1522) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 372 tokens) : 11.2793 + top-1 accuracy : 25/372 = 6.7% + + [2026-04-29 14:09:07] iteration 499/ 500 | consumed samples: 1996 | elapsed time per iteration (ms): 4621.9 | throughput (token/sec/GPU): 1174.2 | learning rate: 1.067461E-05 | global batch size: 4 | lm loss: 1.125999E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +Dataloader batch - tokens shape: torch.Size([1, 1436]) +Dataloader batch - labels shape: torch.Size([1, 1436]) +Dataloader batch - attn_mask shape: torch.Size([1, 1436]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1597 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1436) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1436) torch.int64 + loss_mask : 291 / 1436 tokens (20.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1436, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1436, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1436, 1, 576) torch.bfloat16 + out : (1, 1436) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 291 tokens) : 11.1969 + top-1 accuracy : 10/291 = 3.4% + +Dataloader batch - tokens shape: torch.Size([1, 1917]) +Dataloader batch - labels shape: torch.Size([1, 1917]) +Dataloader batch - attn_mask shape: torch.Size([1, 1917]) +Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) + +==================================================================== + PIPELINE — STEP 1598 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1917) torch.int64 + images : (32, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1917) torch.int64 + loss_mask : 543 / 1917 tokens (28.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (32, 3, 1024, 1024) torch.bfloat16 + out : (32, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (32, 3072) + out : (32, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1917, 1, 576) torch.bfloat16 grad=False + image slots : 32 tokens replaced with vision embeddings + combined : (1917, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1917, 1, 576) torch.bfloat16 + out : (1, 1917) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 543 tokens) : 11.3933 + top-1 accuracy : 4/543 = 0.7% + +Dataloader batch - tokens shape: torch.Size([1, 1582]) +Dataloader batch - labels shape: torch.Size([1, 1582]) +Dataloader batch - attn_mask shape: torch.Size([1, 1582]) +Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) + +==================================================================== + PIPELINE — STEP 1599 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1582) torch.int64 + images : (29, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1582) torch.int64 + loss_mask : 369 / 1582 tokens (23.3%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (29, 3, 1024, 1024) torch.bfloat16 + out : (29, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (29, 3072) + out : (29, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1582, 1, 576) torch.bfloat16 grad=False + image slots : 29 tokens replaced with vision embeddings + combined : (1582, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1582, 1, 576) torch.bfloat16 + out : (1, 1582) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 369 tokens) : 11.2707 + top-1 accuracy : 26/369 = 7.0% + +Dataloader batch - tokens shape: torch.Size([1, 1555]) +Dataloader batch - labels shape: torch.Size([1, 1555]) +Dataloader batch - attn_mask shape: torch.Size([1, 1555]) +Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) + +==================================================================== + PIPELINE — STEP 1600 (Stage 1 Alignment: adapter only) +==================================================================== + +[1/6] BATCH + input_ids : (1, 1555) torch.int64 + images : (28, 3, 1024, 1024) torch.bfloat16 + labels : (1, 1555) torch.int64 + loss_mask : 371 / 1555 tokens (23.9%) contribute to loss + +[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] + in : (28, 3, 1024, 1024) torch.bfloat16 + out : (28, 3072) torch.bfloat16 grad=False + +[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] + in : (28, 3072) + out : (28, 576) grad=True + +[4/6] TOKEN FUSION + text embeddings : (1555, 1, 576) torch.bfloat16 grad=False + image slots : 28 tokens replaced with vision embeddings + combined : (1555, 1, 576) grad=True + +[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] + in : (1555, 1, 576) torch.bfloat16 + out : (1, 1555) torch.float32 grad=True + +[6/6] LOSS / LOGITS + loss (mean over 371 tokens) : 11.2579 + top-1 accuracy : 3/371 = 0.8% + + [2026-04-29 14:09:12] iteration 500/ 500 | consumed samples: 2000 | elapsed time per iteration (ms): 5423.1 | throughput (token/sec/GPU): 1196.7 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.129635E+01 | loss scale: 1.0 | grad norm: 0.169 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | +(min, max) time across ranks (ms): + forward-backward ...............................: (5409.72, 5409.72) + forward-compute ................................: (3952.95, 3952.95) + backward-compute ...............................: (1454.42, 1454.42) + batch-generator ................................: (531.21, 531.21) + layernorm-grads-all-reduce .....................: (0.03, 0.03) + embedding-grads-all-reduce .....................: (0.04, 0.04) + all-grads-sync .................................: (0.55, 0.55) + params-all-gather ..............................: (0.17, 0.17) + optimizer-copy-to-main-grad ....................: (0.15, 0.15) + optimizer-clip-main-grad .......................: (0.64, 0.64) + optimizer-count-zeros ..........................: (0.02, 0.02) + optimizer-inner-step ...........................: (1.38, 1.38) + optimizer-copy-main-to-model-params ............: (0.26, 0.26) + optimizer ......................................: (3.39, 3.39) +saving checkpoint at iteration 500 to stage_1_alignment_mobilellm_140m in torch format +saving dataloader checkpoint at iteration 500 to stage_1_alignment_mobilellm_140m/dataloader + successfully saved checkpoint from iteration 500 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] +(min, max) time across ranks (ms): + save-checkpoint ................................: (3241.43, 3241.43) +[after training is done] datetime: 2026-04-29 14:09:15 +wandb: uploading artifact stage_1_alignment_mobilellm_140m; uploading output.log +wandb: uploading artifact stage_1_alignment_mobilellm_140m +wandb: uploading artifact stage_1_alignment_mobilellm_140m; uploading history steps 397-399, summary, console lines 61532-61700 +wandb: uploading history steps 397-399, summary, console lines 61532-61700 +wandb: uploading data +wandb: +wandb: +wandb: Run history: +wandb: Token throughput (per-sec-per-GPU) ▁▂▂▁▂▂▇██▇▇█▇▇▇▇▇▇▆▇▇▇▇▆▆▇█▅▇▇▇▇▇▆▆▅▇▇▇█ +wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: grad-norm ▇▇▅▇▆▆▆█▂▃▅▅▄▄▅▂▃▃▃▃▃▂▃▂▂▃▃▂▃▁▁▁▂▂▄▁▁▃▂▃ +wandb: iteration-time ▅▅▅█▇▆▃▃▂▃▃▃▂▃▃▃▄▂▁▁▂▂▂▃▂▄▃▄▄▃▃▃▂▂▂▂▃▃▃▃ +wandb: learning-rate ███████████▇▇▇▇▇▇▇▆▆▅▅▅▅▄▄▃▃▃▃▂▂▂▂▂▂▁▁▁▁ +wandb: lm loss ▅▄▄▃▃▃▂▆▆▆▅▅▄▁▆▇▅▃▇▄▅▅▇▅▅▄▅█▅▂▄▆▆▄▆▂▄█▅▂ +wandb: loss-scale ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: num-zeros ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ +wandb: samples vs steps ▁▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▃▃▃▄▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▇▇▇██ +wandb: +wandb: Run summary: +wandb: Token throughput (per-sec-per-GPU) 1196.74249 +wandb: batch-size 4 +wandb: grad-norm 0.16943 +wandb: iteration-time 5.42305 +wandb: learning-rate 1e-05 +wandb: lm loss 11.29635 +wandb: loss-scale 1 +wandb: num-zeros 0 +wandb: samples vs steps 2000 +wandb: +wandb: 🚀 View run mobilellm_integration at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/3bn4iqdv +wandb: ⭐️ View project at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 +wandb: Synced 5 W&B file(s), 0 media file(s), 16 artifact file(s) and 0 other file(s) +wandb: Find logs at: stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_133502-3bn4iqdv/logs +[rank0]:[W429 14:10:46.038762046 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) From ce62b3ceb593fd2cc3e3ecf183289e4b167b257b Mon Sep 17 00:00:00 2001 From: RanaZay Date: Wed, 6 May 2026 19:40:55 +0400 Subject: [PATCH 28/39] Add MobileLLM FastViT midtraining handoff --- .gitignore | 14 + .../data/multimodal/qwen2vl_task_encoder.py | 37 +- .../models/fastvit/fastvit_vision_model.py | 12 +- .../models/fastvit/mobileclip_encoder.py | 8 +- .../llavaov_1_5/llavaov_1_5_provider.py | 6 + .../models/mobilellm/mobilellm_config.py | 10 +- .../models/mobilellm/mobilellm_layer_spec.py | 18 +- docs/mobilellm_fastvit_amd_midtraining.md | 119 +++++ .../eval/eval_mmmu_val_mobilellm_fastvit.py | 483 ++++++++++++++++++ .../eval_realworldqa_mobilellm_fastvit.py | 376 ++++++++++++++ ..._mobilellm_fastvit_english_branch_chain.sh | 154 ++++++ ..._training_mobilellm_fastvit_imagenet_en.sh | 215 ++++++++ .../stage_1_alignment_mobilellm_140m.sh | 16 +- ...e_faithful_mobilellm_fastvit_checkpoint.py | 194 +++++++ ...epare_hf_caption_parquet_to_energon_wds.py | 213 ++++++++ tools/preview_caption_wds_sample.py | 92 ++++ 16 files changed, 1926 insertions(+), 41 deletions(-) create mode 100644 docs/mobilellm_fastvit_amd_midtraining.md create mode 100644 examples/llava_ov_1_5/eval/eval_mmmu_val_mobilellm_fastvit.py create mode 100644 examples/llava_ov_1_5/eval/eval_realworldqa_mobilellm_fastvit.py create mode 100755 examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh create mode 100755 examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh create mode 100644 tools/create_faithful_mobilellm_fastvit_checkpoint.py create mode 100644 tools/prepare_hf_caption_parquet_to_energon_wds.py create mode 100644 tools/preview_caption_wds_sample.py diff --git a/.gitignore b/.gitignore index 11a185b6..2220df92 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,17 @@ checkpoints/ stage_1_alignment_llava_ov_4b/ tmp/ Stage1/training_output.log + +# Local training/evaluation artifacts +*.log +*.out +wandb/ +tensorboard/ +eval_outputs/ +data/hf_cache/ +data/hf_cache_midtraining_stream/ +data/midtraining_*_webdataset/ +stage_1_5_midtraining_*/ +stage_1_alignment_mobilellm_140m/ +stage_1_alignment_mobilellm_140m_fastvlm_faithful_smoke/ +stage_1_alignment_mobilellm_140m_fastvlm_faithful/ diff --git a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py index 22736dfd..39684eaa 100644 --- a/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py +++ b/aiak_training_llm/data/multimodal/qwen2vl_task_encoder.py @@ -184,7 +184,14 @@ def __getattr__(self, name): # Initialize FastViT image processor fastvit_image_size = getattr(args, 'fastvit_image_size', 1024) self.fastvit_processor = FastViTImageProcessor(image_size=fastvit_image_size) - _rank0_print(f"Initialized FastViT processor with image_size={fastvit_image_size}") + fastvit_patch_size = getattr(args, 'fastvit_patch_size', 64) + self.fastvit_tokens_per_side = fastvit_image_size // fastvit_patch_size + self.fastvit_num_image_tokens = self.fastvit_tokens_per_side ** 2 + self.image_token_block = IMAGE_TOKEN * self.fastvit_num_image_tokens + _rank0_print( + f"Initialized FastViT processor with image_size={fastvit_image_size}, " + f"tokens_per_image={self.fastvit_num_image_tokens}" + ) else: # For Qwen2-VL: Load full processor (tokenizer + image/video processor) self.processor = AutoProcessor.from_pretrained( @@ -193,6 +200,7 @@ def __getattr__(self, name): local_files_only=False ) _rank0_print(f"Loaded Qwen2-VL processor from {self.args.hf_tokenizer_path}") + self.image_token_block = IMAGE_TOKEN_WITH_TAGS _rank0_print(f"Processor config: {self.processor}") @@ -347,15 +355,14 @@ def _process(self, image, text): pixel = [pixel_values] - # FastViT: Set grid_thw to represent 1 tile (no dynamic grid) - # Format: tensor([[num_tiles, height, width]]) - image_grid_thw = torch.tensor([[1, 1, 1]]) + # FastViT: one square tile with a spatial token grid. + image_grid_thw = torch.tensor([[1, self.fastvit_tokens_per_side, self.fastvit_tokens_per_side]]) # Create target tensor (same as Qwen2-VL path) target = input_ids.clone() - target[target == vision_start_id] = IGNORE_INDEX - target[target == img_pad_id] = IGNORE_INDEX - target[target == vision_end_id] = IGNORE_INDEX + for token_id in (vision_start_id, img_pad_id, vision_end_id): + if token_id is not None: + target[target == token_id] = IGNORE_INDEX return input_ids, target, pixel, image_grid_thw, attn_mask @@ -408,7 +415,7 @@ def process_sft_vqa(self, context, answer, image): }], tokenize=False ).replace( - "", IMAGE_TOKEN_WITH_TAGS + "", self.image_token_block ) if text[-1] == '\n': text = text[:-1] @@ -555,8 +562,8 @@ def encode_captioning(self, sample: CaptioningSample) -> ImageTaskSample: # assert self.args.training_phase == constants.TrainingPhase.PRETRAIN, "Only support PRETRAIN phase" - # text = IMAGE_TOKEN_WITH_TAGS + sample.caption + self.tokenizer.tokenizer.eos_token - text = IMAGE_TOKEN_WITH_TAGS + sample.caption + self.tokenizer.eos_token + # text = self.image_token_block + sample.caption + self.tokenizer.tokenizer.eos_token + text = self.image_token_block + sample.caption + self.tokenizer.eos_token input_ids, target, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) num_tiles = [len(image_grid_thw)] if image_grid_thw is not None else [1] @@ -594,7 +601,7 @@ def encode_vqa4packing(self, sample: VQASample) -> ImageTaskSample: 'content': sample.answers }], tokenize=False - ).replace("", IMAGE_TOKEN_WITH_TAGS) + ).replace("", self.image_token_block) if text[-1] == '\n': text = text[:-1] @@ -710,10 +717,10 @@ def encode_vaq(self, sample: VQASample) -> ImageTaskSample: if self.args.add_question_in_pretrain: text = (sample.context + sample.answers).replace( "", - IMAGE_TOKEN_WITH_TAGS + self.image_token_block ) else: - text = IMAGE_TOKEN_WITH_TAGS + sample.answers + text = self.image_token_block + sample.answers # text = text + self.tokenizer.tokenizer.eos_token text = text + self.tokenizer.eos_token input_ids, target, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) @@ -771,7 +778,7 @@ def encode_vaq(self, sample: VQASample) -> ImageTaskSample: 'content': sample.answers }], tokenize=False - ).replace("", IMAGE_TOKEN_WITH_TAGS) + ).replace("", self.image_token_block) if text[-1] == '\n': text = text[:-1] input_ids, _, imgs, image_grid_thw, attn_mask = self._process(sample.image, text) @@ -983,4 +990,4 @@ def build_train_datasets( ) if worker_config.should_log(level=1): dataset = LogSampleDataset(dataset, mode="train", worker_config=worker_config) - return dataset \ No newline at end of file + return dataset diff --git a/aiak_training_llm/models/fastvit/fastvit_vision_model.py b/aiak_training_llm/models/fastvit/fastvit_vision_model.py index c468b819..85f63ee2 100644 --- a/aiak_training_llm/models/fastvit/fastvit_vision_model.py +++ b/aiak_training_llm/models/fastvit/fastvit_vision_model.py @@ -28,10 +28,10 @@ def __init__(self, config, layer_spec): # Create a simple args object for MobileCLIPVisionTower class Args: - def __init__(self): - self.unfreeze_mm_vision_tower = False + def __init__(self, unfreeze_mm_vision_tower): + self.unfreeze_mm_vision_tower = unfreeze_mm_vision_tower - args = Args() + args = Args(getattr(config, 'unfreeze_mm_vision_tower', False)) # FastViTHD model name with resolution # Format: mobileclip_l_{resolution} @@ -64,7 +64,7 @@ def forward(self, images, grid_thw=None): grid_thw: Grid dimensions (not used by FastViT, kept for API compatibility) Returns: - Vision features [batch, num_tokens, hidden_size] + Vision features [total_image_tokens, hidden_size] window_index: None (FastViT doesn't use windowing) """ # MobileCLIPVisionTower expects single images or list of images @@ -74,6 +74,10 @@ def forward(self, images, grid_thw=None): image_features = self.vision_tower.forward_images(images) else: raise ValueError(f"Unexpected image tensor shape: {images.shape}") + + if image_features.dim() == 3: + # [num_images, tokens_per_image, hidden] -> [all_image_tokens, hidden] + image_features = image_features.flatten(0, 1).contiguous() # Return features and None for window_index (compatibility with Qwen2-VL API) return image_features, None diff --git a/aiak_training_llm/models/fastvit/mobileclip_encoder.py b/aiak_training_llm/models/fastvit/mobileclip_encoder.py index 40f66cda..63d01051 100644 --- a/aiak_training_llm/models/fastvit/mobileclip_encoder.py +++ b/aiak_training_llm/models/fastvit/mobileclip_encoder.py @@ -60,10 +60,10 @@ def feature_select(self, image_forward_outs): # Features from penultimate layer (conv_exp output): (B, C, H, W) image_features = image_forward_outs["image_embeddings"] - # Global average pool over spatial dims: (B, C, H, W) -> (B, C) - # FastVLM uses 1 visual token per image (not per patch) for efficiency. + # Match FastVLM: keep the spatial FastViTHD grid as visual tokens. + # (B, C, H, W) -> (B, H * W, C) B, C, H, W = image_features.shape - image_features = image_features.mean(dim=[-2, -1]) # (B, C) + image_features = image_features.reshape(B, C, H * W).transpose(1, 2).contiguous() return image_features def forward(self, images): @@ -112,4 +112,4 @@ def num_patches_per_side(self): @property def num_patches(self): - return (self.config["image_cfg"]["image_size"] // self.config["image_cfg"]["patch_size"]) ** 2 \ No newline at end of file + return (self.config["image_cfg"]["image_size"] // self.config["image_cfg"]["patch_size"]) ** 2 diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index c440bc92..991b3f94 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -113,8 +113,14 @@ def rice_vl_model_provider( setattr(vision_config, 'patch_size', image_cfg['patch_size']) # 64 setattr(vision_config, 'image_size', resolution) # from vision_tower_name (1024) setattr(vision_config, 'num_attention_heads', image_cfg['embed_dim'] // 64) # 48 + train_vision_model = ( + args.trainable_modules == ['all'] + or "vision_model" in args.trainable_modules + ) + setattr(vision_config, 'unfreeze_mm_vision_tower', train_vision_model) print_rank_0(f'[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: {vision_tower_name}') + print_rank_0(f'[DEBUG PROVIDER] FastViT train vision tower: {train_vision_model}') print_rank_0(f'[DEBUG PROVIDER] FastViT config loaded from {json_config_path}') print_rank_0(f'[DEBUG PROVIDER] image_cfg: embed_dim={image_cfg["embed_dim"]}, patch_size={image_cfg["patch_size"]}, model={image_cfg["model_name"]}') print_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ==========') diff --git a/aiak_training_llm/models/mobilellm/mobilellm_config.py b/aiak_training_llm/models/mobilellm/mobilellm_config.py index ec7aa0d2..f6b93fc3 100644 --- a/aiak_training_llm/models/mobilellm/mobilellm_config.py +++ b/aiak_training_llm/models/mobilellm/mobilellm_config.py @@ -75,13 +75,9 @@ def get_mobilellm_config(args): untie_embeddings_and_output_weights=not tie_word_embeddings, # Position embeddings: - # NOTE: The original MobileLLM-R1 HF config sets no_rope_layers=[1,1,...,1] - # (all 1s in a 15-element list), meaning all attention layers use NoPE - # (no positional encoding) per the Llama4/MobileLLM-R1 architecture. - # rope_theta=8000000 is present in config.json but not applied in HF transformers. - # This Megatron integration applies RoPE with that theta value, which is an - # intentional divergence to provide positional encoding for the multimodal - # adaptation. If you want strict NoPE fidelity, set position_embedding_type="none". + # HF Llama4TextAttention treats no_rope_layers[layer_idx] as the + # use_rope flag. MobileLLM-R1-140M sets it to 1 for all layers, so this + # Megatron integration should use RoPE with rope_theta=8000000. position_embedding_type="rope", rotary_base=rope_theta, rotary_percent=1.0, diff --git a/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py b/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py index 6ddb5fc3..b76d32fb 100644 --- a/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py +++ b/aiak_training_llm/models/mobilellm/mobilellm_layer_spec.py @@ -1,5 +1,8 @@ """MobileLLM layer specification - Standard LLaMA architecture""" +import torch +import torch.nn as nn + from megatron.core.transformer.enums import AttnMaskType from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.spec_utils import ModuleSpec @@ -11,6 +14,17 @@ from aiak_training_llm.models.dispatch import multiacc_modules +class MobileLLML2QKNorm(nn.Module): + """Parameter-free L2 Q/K norm used by HF Llama4/MobileLLM.""" + + def __init__(self, hidden_size=None, config=None, eps=1e-5): + super().__init__() + self.eps = eps + + def forward(self, x): + return (x.float() * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + self.eps)).to(x.dtype) + + def _is_te_min_version(version: str) -> bool: """Check if Transformer Engine version is at least the specified version.""" try: @@ -38,9 +52,7 @@ def get_mobilellm_layer_with_te_spec(config: TransformerConfig) -> ModuleSpec: Returns: ModuleSpec for MobileLLM transformer layer """ - # TENorm significantly harms convergence when used for QKLayerNorm if TE Version < 1.9; - # we instead use the Apex implementation. - qk_norm = multiacc_modules.TENorm if _is_te_min_version("1.9.0") else multiacc_modules.LocalNorm + qk_norm = MobileLLML2QKNorm # Standard dense MLP with SwiGLU mlp = ModuleSpec( diff --git a/docs/mobilellm_fastvit_amd_midtraining.md b/docs/mobilellm_fastvit_amd_midtraining.md new file mode 100644 index 00000000..4b485434 --- /dev/null +++ b/docs/mobilellm_fastvit_amd_midtraining.md @@ -0,0 +1,119 @@ +# AMD Terminal Midtraining Handoff + +This branch contains the MobileLLM-R1 + FastViT/FastVLM integration and the +scripts needed to start Stage 1.5. Keep model checkpoints outside Git and copy +them between machines directly. + +## 1. Get the Code + +```bash +git clone git@github.com:RanaZay/LLaVA-OneVision-1.5.git +cd LLaVA-OneVision-1.5 +git checkout mobile-llm-integration +git pull +git lfs pull +``` + +Copy the Stage 1 alignment checkpoint from the GPU machine to the AMD machine. +From the GPU machine, run: + +```bash +rsync -avP \ + stage_1_alignment_mobilellm_140m_fastvlm_faithful/latest_checkpointed_iteration.txt \ + stage_1_alignment_mobilellm_140m_fastvlm_faithful/iter_0002500 \ + user@AMD_HOST:/path/to/LLaVA-OneVision-1.5/stage_1_alignment_mobilellm_140m_fastvlm_faithful/ +``` + +On the AMD machine, verify the Stage 1 checkpoint is present: + +```bash +cat stage_1_alignment_mobilellm_140m_fastvlm_faithful/latest_checkpointed_iteration.txt +ls -lh stage_1_alignment_mobilellm_140m_fastvlm_faithful/iter_0002500/mp_rank_00/model_optim_rng.pt +``` + +Expected iteration: `2500`. + +## 2. Start a Terminal Session + +Use `tmux` so training survives SSH disconnects: + +```bash +tmux new -s mobile_vlm_midtrain +``` + +Detach without stopping training: + +```text +Ctrl-b then d +``` + +Reattach: + +```bash +tmux attach -t mobile_vlm_midtrain +``` + +## 3. Set Environment + +Adjust paths for the AMD machine: + +```bash +export REPO_ROOT="$PWD" +export PYTHON_BIN=/path/to/your/env/bin/python +export TORCHRUN=/path/to/your/env/bin/torchrun + +export CUDA_VISIBLE_DEVICES=0,1,2,3 +export GPUS_PER_NODE=4 + +export WANDB_API_KEY="your_wandb_key" +export WANDB_PROJECT="llava-ov-1_5" +``` + +If the AMD machine has a different Conda env, replace `PYTHON_BIN` and +`TORCHRUN` accordingly. + +## 4. Run Full English Branch Midtraining + +This prepares the full HF English branch into local Energon/WebDataset shards, +then runs 1000 Stage 1.5 iterations. `MAX_SAMPLES=0` means no artificial cap. + +ImageNet EN: + +```bash +MAX_SAMPLES=0 \ +NSTEP=1000 \ +GBS=4 \ +MIN_FREE_GB=100 \ +bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet +``` + +Multiple English branches, chained one after another: + +```bash +MAX_SAMPLES=0 \ +NSTEP=1000 \ +GBS=4 \ +MIN_FREE_GB=100 \ +bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet datacomp1b +``` + +The second branch automatically loads from the checkpoint produced by the first. + +## 5. Useful Options + +```bash +START_CKPT=/path/to/checkpoint_dir +DATA_ROOT=/large/disk/midtraining_full_en +CACHE_DIR=/large/disk/hf_cache_midtraining_stream +KEEP_PREPARED_DATA=1 +PREPARE_ONLY=1 +MIDTRAIN_TRAINABLE_MODULES="language_model adapter vision_model" +``` + +For a quick smoke test only: + +```bash +MAX_SAMPLES=4000 NSTEP=1000 bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet +``` + +For real full-data preparation, keep `MAX_SAMPLES=0`. diff --git a/examples/llava_ov_1_5/eval/eval_mmmu_val_mobilellm_fastvit.py b/examples/llava_ov_1_5/eval/eval_mmmu_val_mobilellm_fastvit.py new file mode 100644 index 00000000..028306a1 --- /dev/null +++ b/examples/llava_ov_1_5/eval/eval_mmmu_val_mobilellm_fastvit.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +""" +Evaluate the FastViT + MobileLLM-R1 checkpoint on MMMU validation. + +This script is intentionally independent of lmms-eval because the current +checkpoint is Megatron/MCore format, not HuggingFace format. + +Smoke test: + CUDA_VISIBLE_DEVICES=4 /home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin/python \ + examples/llava_ov_1_5/eval/eval_mmmu_val_mobilellm_fastvit.py \ + --checkpoint stage_1_alignment_mobilellm_140m/iter_0000100 \ + --subjects Math Physics \ + --max-samples 10 \ + --output-path eval_outputs/mmmu_val_mobilellm_fastvit_10.jsonl + +Full validation: + CUDA_VISIBLE_DEVICES=4 /home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin/python \ + examples/llava_ov_1_5/eval/eval_mmmu_val_mobilellm_fastvit.py \ + --checkpoint stage_1_alignment_mobilellm_140m/iter_0000100 \ + --subjects all \ + --output-path eval_outputs/mmmu_val_mobilellm_fastvit_full.jsonl +""" + +from __future__ import annotations + +import argparse +import ast +import contextlib +import io +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def parse_our_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="MMMU-val evaluator for FastViT + MobileLLM") + parser.add_argument("--checkpoint", default="stage_1_alignment_mobilellm_140m/iter_0000100") + parser.add_argument("--output-path", default="eval_outputs/mmmu_val_mobilellm_fastvit.jsonl") + parser.add_argument("--subjects", nargs="+", default=["all"]) + parser.add_argument("--max-samples", type=int, default=0, help="0 means all selected samples.") + parser.add_argument("--max-new-tokens", type=int, default=16) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--hf-cache", default=str(REPO_ROOT / "data" / "hf_cache")) + parser.add_argument("--local-mmmu-path", default="", help="Optional local MMMU dataset root.") + parser.add_argument("--image-size", type=int, default=1024) + parser.add_argument("--master-port", default="29610") + parser.add_argument("--verbose-forward", action="store_true", help="Print per-token Megatron forward diagnostics.") + return parser.parse_args() + + +OUR_ARGS = parse_our_args() + + +def resolve_path(path: str) -> str: + p = Path(path) + if p.is_absolute(): + return str(p) + return str(REPO_ROOT / p) + + +LOAD_CKPT = resolve_path(OUR_ARGS.checkpoint) +DATA_PATH = str(REPO_ROOT / "data" / "LLaVA-558K-Webdataset") + + +def configure_megatron_argv() -> None: + sys.argv = [ + sys.argv[0], + "--model-name", "llava-ov-mobilellm-140m", + "--tokenizer-type", "HFTokenizer", + "--hf-tokenizer-path", "facebook/MobileLLM-R1-140M", + "--use-fastvit", + "--fastvit-image-size", str(OUR_ARGS.image_size), + "--vision-tower-name", f"mobileclip_l_{OUR_ARGS.image_size}", + "--image-aspect-ratio", "pad", + "--training-phase", "sft", + "--chat-template", "llama3", + "--trainable-modules", "adapter", + "--micro-batch-size", "1", + "--global-batch-size", "1", + "--train-iters", "1", + "--seq-length", "4096", + "--max-position-embeddings", "32768", + "--attention-backend", "local", + "--transformer-impl", "local", + "--no-gradient-accumulation-fusion", + "--no-rope-fusion", + "--norm-epsilon", "1e-05", + "--init-method-std", "0.02", + "--training-rice-vl-max-answer-length", "32768", + "--bf16", + "--tensor-model-parallel-size", "1", + "--pipeline-model-parallel-size", "1", + "--use-distributed-optimizer", + "--distributed-backend", "nccl", + "--pretrained-checkpoint", LOAD_CKPT, + "--no-load-optim", + "--no-load-rng", + "--data-path", DATA_PATH, + "--split", "100,0,0", + "--num-workers", "0", + "--dataloader-type", "external", + "--log-interval", "1", + ] + + +configure_megatron_argv() + +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "aiak_megatron")) + +os.environ.setdefault("RANK", "0") +os.environ.setdefault("LOCAL_RANK", "0") +os.environ.setdefault("WORLD_SIZE", "1") +os.environ.setdefault("MASTER_ADDR", "127.0.0.1") +os.environ.setdefault("MASTER_PORT", OUR_ARGS.master_port) +os.environ.setdefault("HF_DATASETS_CACHE", OUR_ARGS.hf_cache) + +import numpy as np +import torch +from PIL import Image + + +SUBJECTS = [ + "Accounting", + "Agriculture", + "Architecture_and_Engineering", + "Art", + "Art_Theory", + "Basic_Medical_Science", + "Biology", + "Chemistry", + "Clinical_Medicine", + "Computer_Science", + "Design", + "Diagnostics_and_Laboratory_Medicine", + "Economics", + "Electronics", + "Energy_and_Power", + "Finance", + "Geography", + "History", + "Literature", + "Manage", + "Marketing", + "Materials", + "Math", + "Mechanical_Engineering", + "Music", + "Pharmacy", + "Physics", + "Psychology", + "Public_Health", + "Sociology", +] + + +def megatron_init(): + from aiak_training_llm.train.arguments import ( + aiak_extra_train_args_provider, + validate_aiak_extra_args, + ) + from aiak_training_llm.utils.initialize import initialize_aiak_megatron, parse_arguments + + args = parse_arguments( + extra_args_provider=aiak_extra_train_args_provider, + validate_extra_args_provider=validate_aiak_extra_args, + ) + initialize_aiak_megatron(args) + return args + + +def build_model(): + from megatron.training.training import get_model + from aiak_training_llm.models import get_model_family, get_model_provider + from aiak_training_llm.utils import get_args + + args = get_args() + model_family = get_model_family(args.model_name) + provider = get_model_provider(model_family) + models = get_model(provider, model_type=None, wrap_with_ddp=False) + return models[0] if isinstance(models, (list, tuple)) else models + + +def load_weights(model, ckpt_dir: str) -> None: + ckpt_file = Path(ckpt_dir) / "mp_rank_00" / "model_optim_rng.pt" + if not ckpt_file.exists(): + raise FileNotFoundError(f"Checkpoint file not found: {ckpt_file}") + + print(f"[ckpt] Loading {ckpt_file}") + raw = torch.load(str(ckpt_file), map_location="cpu", weights_only=False) + saved = raw["model"] + + has_lm_prefix = any(k.startswith("language_model.") for k in saved) + + def remap(k: str) -> str: + if has_lm_prefix: + return k + if k.startswith(("embedding.", "decoder.", "output_layer.")): + return "language_model." + k + return k + + remapped = {remap(k): v for k, v in saved.items()} + model_sd = model.state_dict() + loaded = 0 + for name, param in model_sd.items(): + src = remapped.get(name) + if src is not None and src.shape == param.shape: + model_sd[name].copy_(src.to(dtype=param.dtype)) + loaded += 1 + model.load_state_dict(model_sd, strict=False) + print(f"[ckpt] Loaded {loaded}/{len(model_sd)} tensors") + + +def parse_options(options: Any) -> list[str]: + if options is None: + return [] + if isinstance(options, list): + return [str(x) for x in options] + if isinstance(options, str): + try: + parsed = ast.literal_eval(options) + if isinstance(parsed, list): + return [str(x) for x in parsed] + except Exception: + pass + return [x.strip() for x in options.split("\n") if x.strip()] + return [] + + +def get_images(sample: dict[str, Any]) -> list[Image.Image]: + images = [] + for i in range(1, 8): + img = sample.get(f"image_{i}") + if img is not None: + images.append(img.convert("RGB")) + if not images and sample.get("image") is not None: + images.append(sample["image"].convert("RGB")) + return images + + +def expand2square(image: Image.Image, background: tuple[int, int, int] = (0, 0, 0)) -> Image.Image: + width, height = image.size + if width == height: + return image + side = max(width, height) + square = Image.new(image.mode, (side, side), background) + square.paste(image, ((side - width) // 2, (side - height) // 2)) + return square + + +def preprocess_images(images: list[Image.Image], size: int, device, dtype) -> torch.Tensor: + tensors = [] + for img in images: + img = expand2square(img) + resized = img.resize((size, size), Image.BICUBIC) + arr = np.array(resized, dtype=np.float32) / 255.0 + tensors.append(torch.from_numpy(arr).permute(2, 0, 1)) + return torch.stack(tensors).to(device=device, dtype=dtype) + + +def build_prompt(sample: dict[str, Any], num_images: int) -> str: + question = sample["question"].strip() + options = parse_options(sample.get("options")) + tokens_per_image = (OUR_ARGS.image_size // 64) ** 2 + image_tokens = "\n".join(["<|image_pad|>" * tokens_per_image] * max(1, num_images)) + + if sample.get("question_type") == "multiple-choice" and options: + option_letters = [chr(ord("A") + i) for i in range(len(options))] + options_text = "\n".join(f"{letter}. {option}" for letter, option in zip(option_letters, options)) + instruction = "Answer with only the option letter." + return f"{image_tokens}\n{question}\n{options_text}\n{instruction}" + + return f"{image_tokens}\n{question}\nAnswer the question directly." + + +def tokenise_prompt(prompt: str, tokenizer, image_token_id: int, device) -> torch.Tensor: + ids = tokenizer.tokenizer.encode(prompt, add_special_tokens=True) + input_ids = torch.tensor(ids, dtype=torch.long, device=device).unsqueeze(0) + count = (input_ids == image_token_id).sum().item() + if count == 0: + raise ValueError("Prompt does not contain an image token after tokenization.") + return input_ids + + +@torch.no_grad() +def generate(model, images, input_ids, tokenizer, max_new_tokens: int, temperature: float) -> str: + eos_ids = {tokenizer.eos_token_id} + generated = [] + for _ in range(max_new_tokens): + seq_len = input_ids.shape[1] + position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0) + quiet_context = contextlib.nullcontext() + if not OUR_ARGS.verbose_forward: + quiet_context = contextlib.redirect_stdout(io.StringIO()) + with quiet_context: + logits = model( + images=images, + image_grid_thw=None, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=None, + attn_mask_type=None, + labels=None, + packed_seq_params=None, + ) + if logits.dim() == 3 and logits.shape[0] != 1: + logits = logits.transpose(0, 1).contiguous() + next_logits = logits[0, -1, :] + if temperature and temperature > 0: + probs = torch.softmax(next_logits / temperature, dim=-1) + next_id = int(torch.multinomial(probs, num_samples=1)) + else: + next_id = int(next_logits.argmax()) + generated.append(next_id) + if next_id in eos_ids: + break + input_ids = torch.cat( + [input_ids, torch.tensor([[next_id]], dtype=torch.long, device=input_ids.device)], + dim=1, + ) + return tokenizer.decode(generated, skip_special_tokens=True).strip() + + +def normalize_answer(text: str) -> str: + return re.sub(r"\s+", " ", text.strip().lower()) + + +def extract_choice(prediction: str, num_options: int) -> str: + letters = [chr(ord("A") + i) for i in range(num_options)] + pred = prediction.strip() + match = re.search(r"\b([A-Z])\b", pred.upper()) + if match and match.group(1) in letters: + return match.group(1) + if pred[:1].upper() in letters: + return pred[:1].upper() + return pred + + +def score_prediction(sample: dict[str, Any], prediction: str) -> tuple[bool, str]: + answer = str(sample["answer"]).strip() + options = parse_options(sample.get("options")) + if sample.get("question_type") == "multiple-choice" and options: + pred = extract_choice(prediction, len(options)) + return pred == answer, pred + return normalize_answer(prediction) == normalize_answer(answer), prediction + + +def load_mmmu_samples(subjects: list[str], max_samples: int) -> list[dict[str, Any]]: + import datasets + + Path(OUR_ARGS.hf_cache).mkdir(parents=True, exist_ok=True) + selected_subjects = SUBJECTS if subjects == ["all"] else subjects + samples: list[dict[str, Any]] = [] + for subject in selected_subjects: + dataset_name = "MMMU/MMMU" + if OUR_ARGS.local_mmmu_path: + dataset_name = str(Path(OUR_ARGS.local_mmmu_path) / subject) + ds = datasets.load_dataset( + dataset_name, + split=datasets.Split.VALIDATION, + cache_dir=OUR_ARGS.hf_cache, + verification_mode="no_checks", + ) + else: + ds = datasets.load_dataset( + dataset_name, + subject, + split=datasets.Split.VALIDATION, + cache_dir=OUR_ARGS.hf_cache, + ) + for sample in ds: + if str(sample.get("id", "")).startswith("val"): + sample = dict(sample) + sample["subject"] = subject + samples.append(sample) + if max_samples and len(samples) >= max_samples: + return samples + return samples + + +def main() -> None: + print("[init] Initialising Megatron") + megatron_init() + device = torch.device("cuda:0") + dtype = torch.bfloat16 + + from aiak_training_llm.utils import get_tokenizer + + tokenizer = get_tokenizer() + inner_tokenizer = tokenizer.tokenizer + image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>") + + print("[model] Building model") + model = build_model().to(device=device, dtype=dtype) + model.eval() + for p in model.parameters(): + p.requires_grad_(False) + + load_weights(model, LOAD_CKPT) + + print("[data] Loading MMMU validation samples") + samples = load_mmmu_samples(OUR_ARGS.subjects, OUR_ARGS.max_samples) + print(f"[data] Loaded {len(samples)} samples") + + output_path = Path(resolve_path(OUR_ARGS.output_path)) + output_path.parent.mkdir(parents=True, exist_ok=True) + + correct = 0 + total = 0 + mc_correct = 0 + mc_total = 0 + open_correct = 0 + open_total = 0 + + with output_path.open("w", encoding="utf-8") as f: + for idx, sample in enumerate(samples, start=1): + images = get_images(sample) + if not images: + print(f"[warn] Skipping {sample.get('id')} with no image") + continue + + prompt = build_prompt(sample, len(images)) + image_tensor = preprocess_images(images, OUR_ARGS.image_size, device, dtype) + input_ids = tokenise_prompt(prompt, tokenizer, image_token_id, device) + prediction = generate( + model, + image_tensor, + input_ids, + inner_tokenizer, + OUR_ARGS.max_new_tokens, + OUR_ARGS.temperature, + ) + is_correct, parsed_prediction = score_prediction(sample, prediction) + + total += 1 + correct += int(is_correct) + if sample.get("question_type") == "multiple-choice": + mc_total += 1 + mc_correct += int(is_correct) + else: + open_total += 1 + open_correct += int(is_correct) + + row = { + "id": sample.get("id"), + "subject": sample.get("subject"), + "question_type": sample.get("question_type"), + "answer": sample.get("answer"), + "prediction": prediction, + "parsed_prediction": parsed_prediction, + "correct": is_correct, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + f.flush() + + running = 100.0 * correct / total + print(f"[{idx}/{len(samples)}] {sample.get('id')} correct={is_correct} acc={running:.2f}% pred={parsed_prediction!r} ans={sample.get('answer')!r}") + + metrics = { + "total": total, + "accuracy": 100.0 * correct / total if total else 0.0, + "multiple_choice_total": mc_total, + "multiple_choice_accuracy": 100.0 * mc_correct / mc_total if mc_total else 0.0, + "open_total": open_total, + "open_accuracy": 100.0 * open_correct / open_total if open_total else 0.0, + "checkpoint": LOAD_CKPT, + "output_path": str(output_path), + } + metrics_path = output_path.with_suffix(".metrics.json") + metrics_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") + + print("\n===== MMMU validation summary =====") + print(json.dumps(metrics, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/examples/llava_ov_1_5/eval/eval_realworldqa_mobilellm_fastvit.py b/examples/llava_ov_1_5/eval/eval_realworldqa_mobilellm_fastvit.py new file mode 100644 index 00000000..23f70507 --- /dev/null +++ b/examples/llava_ov_1_5/eval/eval_realworldqa_mobilellm_fastvit.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Evaluate a FastViT + MobileLLM Megatron checkpoint on RealWorldQA.""" + +from __future__ import annotations + +import argparse +import contextlib +import io +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def parse_our_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="RealWorldQA evaluator for FastViT + MobileLLM") + parser.add_argument("--checkpoint", required=True) + parser.add_argument("--output-path", default="eval_outputs/realworldqa_mobilellm_fastvit.jsonl") + parser.add_argument("--dataset-path", default="lmms-lab/RealWorldQA") + parser.add_argument("--split", default="test") + parser.add_argument("--max-samples", type=int, default=0, help="0 means all samples.") + parser.add_argument("--max-new-tokens", type=int, default=16) + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--hf-cache", default=str(REPO_ROOT / "data" / "hf_cache")) + parser.add_argument("--image-size", type=int, default=1024) + parser.add_argument("--master-port", default="29620") + parser.add_argument("--no-streaming", action="store_true", help="Download/cache the dataset instead of streaming it.") + parser.add_argument("--verbose-forward", action="store_true", help="Print per-token Megatron forward diagnostics.") + return parser.parse_args() + + +OUR_ARGS = parse_our_args() + + +def resolve_path(path: str) -> str: + p = Path(path) + if p.is_absolute(): + return str(p) + return str(REPO_ROOT / p) + + +LOAD_CKPT = resolve_path(OUR_ARGS.checkpoint) +DATA_PATH = str(REPO_ROOT / "data" / "LLaVA-558K-Webdataset") + + +def configure_megatron_argv() -> None: + sys.argv = [ + sys.argv[0], + "--model-name", "llava-ov-mobilellm-140m", + "--tokenizer-type", "HFTokenizer", + "--hf-tokenizer-path", "facebook/MobileLLM-R1-140M", + "--use-fastvit", + "--fastvit-image-size", str(OUR_ARGS.image_size), + "--vision-tower-name", f"mobileclip_l_{OUR_ARGS.image_size}", + "--image-aspect-ratio", "pad", + "--training-phase", "sft", + "--chat-template", "llama3", + "--trainable-modules", "adapter", + "--micro-batch-size", "1", + "--global-batch-size", "1", + "--train-iters", "1", + "--seq-length", "4096", + "--max-position-embeddings", "32768", + "--attention-backend", "local", + "--transformer-impl", "local", + "--no-gradient-accumulation-fusion", + "--no-rope-fusion", + "--norm-epsilon", "1e-05", + "--init-method-std", "0.02", + "--training-rice-vl-max-answer-length", "32768", + "--bf16", + "--tensor-model-parallel-size", "1", + "--pipeline-model-parallel-size", "1", + "--use-distributed-optimizer", + "--distributed-backend", "nccl", + "--pretrained-checkpoint", LOAD_CKPT, + "--no-load-optim", + "--no-load-rng", + "--data-path", DATA_PATH, + "--split", "100,0,0", + "--num-workers", "0", + "--dataloader-type", "external", + "--log-interval", "1", + ] + + +configure_megatron_argv() + +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "aiak_megatron")) + +os.environ.setdefault("RANK", "0") +os.environ.setdefault("LOCAL_RANK", "0") +os.environ.setdefault("WORLD_SIZE", "1") +os.environ.setdefault("MASTER_ADDR", "127.0.0.1") +os.environ.setdefault("MASTER_PORT", OUR_ARGS.master_port) +os.environ.setdefault("HF_DATASETS_CACHE", OUR_ARGS.hf_cache) + +import numpy as np +import torch +from PIL import Image + + +def megatron_init(): + from aiak_training_llm.train.arguments import ( + aiak_extra_train_args_provider, + validate_aiak_extra_args, + ) + from aiak_training_llm.utils.initialize import initialize_aiak_megatron, parse_arguments + + args = parse_arguments( + extra_args_provider=aiak_extra_train_args_provider, + validate_extra_args_provider=validate_aiak_extra_args, + ) + initialize_aiak_megatron(args) + return args + + +def build_model(): + from megatron.training.training import get_model + from aiak_training_llm.models import get_model_family, get_model_provider + from aiak_training_llm.utils import get_args + + args = get_args() + model_family = get_model_family(args.model_name) + provider = get_model_provider(model_family) + models = get_model(provider, model_type=None, wrap_with_ddp=False) + return models[0] if isinstance(models, (list, tuple)) else models + + +def load_weights(model, ckpt_dir: str) -> None: + ckpt_file = Path(ckpt_dir) / "mp_rank_00" / "model_optim_rng.pt" + if not ckpt_file.exists(): + raise FileNotFoundError(f"Checkpoint file not found: {ckpt_file}") + + print(f"[ckpt] Loading {ckpt_file}", flush=True) + raw = torch.load(str(ckpt_file), map_location="cpu", weights_only=False) + saved = raw["model"] + + has_lm_prefix = any(k.startswith("language_model.") for k in saved) + + def remap(k: str) -> str: + if has_lm_prefix: + return k + if k.startswith(("embedding.", "decoder.", "output_layer.")): + return "language_model." + k + return k + + remapped = {remap(k): v for k, v in saved.items()} + model_sd = model.state_dict() + loaded = 0 + for name, param in model_sd.items(): + src = remapped.get(name) + if src is not None and src.shape == param.shape: + model_sd[name].copy_(src.to(dtype=param.dtype)) + loaded += 1 + model.load_state_dict(model_sd, strict=False) + print(f"[ckpt] Loaded {loaded}/{len(model_sd)} tensors", flush=True) + + +def expand2square(image: Image.Image, background: tuple[int, int, int] = (0, 0, 0)) -> Image.Image: + width, height = image.size + if width == height: + return image + side = max(width, height) + square = Image.new(image.mode, (side, side), background) + square.paste(image, ((side - width) // 2, (side - height) // 2)) + return square + + +def preprocess_image(image: Image.Image, size: int, device, dtype) -> torch.Tensor: + image = expand2square(image.convert("RGB")) + resized = image.resize((size, size), Image.BICUBIC) + arr = np.array(resized, dtype=np.float32) / 255.0 + tensor = torch.from_numpy(arr).permute(2, 0, 1) + return tensor.unsqueeze(0).to(device=device, dtype=dtype) + + +def build_prompt(question: str) -> str: + tokens_per_image = (OUR_ARGS.image_size // 64) ** 2 + image_tokens = "<|image_pad|>" * tokens_per_image + question = question.strip() + if re.search(r"\bplease answer\b", question, flags=re.IGNORECASE): + return f"{image_tokens}\n{question}\nAnswer:" + return f"{image_tokens}\n{question}\nAnswer with a short phrase.\nAnswer:" + + +def tokenise_prompt(prompt: str, tokenizer, image_token_id: int, device) -> torch.Tensor: + ids = tokenizer.tokenizer.encode(prompt, add_special_tokens=True) + input_ids = torch.tensor(ids, dtype=torch.long, device=device).unsqueeze(0) + count = (input_ids == image_token_id).sum().item() + if count == 0: + raise ValueError("Prompt does not contain an image token after tokenization.") + return input_ids + + +@torch.no_grad() +def generate(model, image, input_ids, tokenizer, max_new_tokens: int, temperature: float) -> str: + eos_ids = {tokenizer.eos_token_id} + generated = [] + for _ in range(max_new_tokens): + seq_len = input_ids.shape[1] + position_ids = torch.arange(seq_len, device=input_ids.device).unsqueeze(0) + quiet_context = contextlib.nullcontext() + if not OUR_ARGS.verbose_forward: + quiet_context = contextlib.redirect_stdout(io.StringIO()) + with quiet_context: + logits = model( + images=image, + image_grid_thw=None, + input_ids=input_ids, + position_ids=position_ids, + attention_mask=None, + attn_mask_type=None, + labels=None, + packed_seq_params=None, + ) + if logits.dim() == 3 and logits.shape[0] != 1: + logits = logits.transpose(0, 1).contiguous() + next_logits = logits[0, -1, :] + if temperature and temperature > 0: + probs = torch.softmax(next_logits / temperature, dim=-1) + next_id = int(torch.multinomial(probs, num_samples=1)) + else: + next_id = int(next_logits.argmax()) + generated.append(next_id) + if next_id in eos_ids: + break + input_ids = torch.cat( + [input_ids, torch.tensor([[next_id]], dtype=torch.long, device=input_ids.device)], + dim=1, + ) + return tokenizer.decode(generated, skip_special_tokens=True).strip() + + +def normalize_answer(text: Any) -> str: + text = str(text).strip().lower() + text = re.sub(r"[^a-z0-9]+", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def parse_letter(prediction: str, valid_letters: set[str]) -> str: + pred = prediction.strip().upper() + match = re.search(r"\b([A-Z])\b", pred) + if match and match.group(1) in valid_letters: + return match.group(1) + if pred[:1] in valid_letters: + return pred[:1] + return prediction.strip() + + +def score_prediction(answer: Any, prediction: str) -> tuple[bool, str]: + gold = str(answer).strip() + if re.fullmatch(r"[A-Z]", gold.upper()): + parsed = parse_letter(prediction, set("ABCDEFGHIJ")) + return parsed.upper() == gold.upper(), parsed + parsed = prediction.strip() + return normalize_answer(parsed) == normalize_answer(gold), parsed + + +def load_realworldqa_samples() -> list[dict[str, Any]]: + import datasets + + Path(OUR_ARGS.hf_cache).mkdir(parents=True, exist_ok=True) + ds = datasets.load_dataset( + OUR_ARGS.dataset_path, + split=OUR_ARGS.split, + cache_dir=OUR_ARGS.hf_cache, + streaming=not OUR_ARGS.no_streaming, + ) + + samples: list[dict[str, Any]] = [] + for idx, sample in enumerate(ds): + item = dict(sample) + item.setdefault("id", item.get("image_path") or str(idx)) + samples.append(item) + if OUR_ARGS.max_samples and len(samples) >= OUR_ARGS.max_samples: + break + return samples + + +def main() -> None: + print("[init] Initialising Megatron", flush=True) + megatron_init() + device = torch.device("cuda:0") + dtype = torch.bfloat16 + + from aiak_training_llm.utils import get_tokenizer + + tokenizer = get_tokenizer() + inner_tokenizer = tokenizer.tokenizer + image_token_id = tokenizer.convert_tokens_to_ids("<|image_pad|>") + + print("[model] Building model", flush=True) + model = build_model().to(device=device, dtype=dtype) + model.eval() + for p in model.parameters(): + p.requires_grad_(False) + + load_weights(model, LOAD_CKPT) + + print("[data] Loading RealWorldQA samples", flush=True) + samples = load_realworldqa_samples() + print(f"[data] Loaded {len(samples)} samples", flush=True) + + output_path = Path(resolve_path(OUR_ARGS.output_path)) + output_path.parent.mkdir(parents=True, exist_ok=True) + + correct = 0 + total = 0 + with output_path.open("w", encoding="utf-8") as f: + for idx, sample in enumerate(samples, start=1): + image = sample.get("image") + if image is None: + print(f"[warn] Skipping {sample.get('id')} with no image", flush=True) + continue + + question = str(sample.get("question", "")) + answer = sample.get("answer") + prompt = build_prompt(question) + image_tensor = preprocess_image(image, OUR_ARGS.image_size, device, dtype) + input_ids = tokenise_prompt(prompt, tokenizer, image_token_id, device) + prediction = generate( + model, + image_tensor, + input_ids, + inner_tokenizer, + OUR_ARGS.max_new_tokens, + OUR_ARGS.temperature, + ) + is_correct, parsed_prediction = score_prediction(answer, prediction) + + total += 1 + correct += int(is_correct) + row = { + "id": sample.get("id"), + "image_path": sample.get("image_path"), + "question": question, + "answer": answer, + "prediction": prediction, + "parsed_prediction": parsed_prediction, + "correct": is_correct, + } + f.write(json.dumps(row, ensure_ascii=False) + "\n") + f.flush() + + running = 100.0 * correct / total + print( + f"[{idx}/{len(samples)}] {sample.get('id')} correct={is_correct} " + f"acc={running:.2f}% pred={parsed_prediction!r} ans={answer!r}", + flush=True, + ) + + metrics = { + "total": total, + "accuracy": 100.0 * correct / total if total else 0.0, + "checkpoint": LOAD_CKPT, + "dataset_path": OUR_ARGS.dataset_path, + "split": OUR_ARGS.split, + "streaming": not OUR_ARGS.no_streaming, + "output_path": str(output_path), + } + metrics_path = output_path.with_suffix(".metrics.json") + metrics_path.write_text(json.dumps(metrics, indent=2), encoding="utf-8") + + print("\n===== RealWorldQA summary =====", flush=True) + print(json.dumps(metrics, indent=2), flush=True) + + +if __name__ == "__main__": + main() diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh new file mode 100755 index 00000000..a917f778 --- /dev/null +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# Terminal-friendly Stage 1.5 chain runner for MobileLLM-R1 + FastViT. +# +# Usage: +# bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet +# bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet datacomp1b +# +# Defaults prepare the full English branch from HF into local Energon/WebDataset +# shards, then run 1000 iterations. Set MAX_SAMPLES to a positive number only +# for smoke tests. + +set -euo pipefail + +if [[ "$#" -lt 1 ]]; then + echo "Usage: $0 [ ...]" + echo "Example: $0 imagenet datacomp1b" + exit 1 +fi + +REPO_ROOT="${REPO_ROOT:-/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5}" +cd "$REPO_ROOT" + +HF_REPO_ID="${HF_REPO_ID:-mvp-lab/LLaVA-OneVision-1.5-Mid-Training-85M}" +LANG_CODE="${LANG_CODE:-EN}" +LANG_LOWER="$(echo "$LANG_CODE" | tr '[:upper:]' '[:lower:]')" + +PYTHON_BIN="${PYTHON_BIN:-/home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin/python}" +TORCHRUN="${TORCHRUN:-/home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin/torchrun}" +export PYTHON_BIN TORCHRUN + +START_CKPT="${START_CKPT:-$REPO_ROOT/stage_1_alignment_mobilellm_140m_fastvlm_faithful}" +CHECKPOINT_PATH="$START_CKPT" + +DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_${LANG_LOWER}}" +CACHE_DIR="${CACHE_DIR:-$REPO_ROOT/data/hf_cache_midtraining_stream}" +mkdir -p "$DATA_ROOT" "$CACHE_DIR" + +TP="${TP:-1}" +PP="${PP:-1}" +SEQ_LEN="${SEQ_LEN:-4096}" +MBS="${MBS:-1}" +GBS="${GBS:-4}" +NSTEP="${NSTEP:-1000}" + +GPUS_PER_NODE="${GPUS_PER_NODE:-4}" +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" +MASTER_PORT_BASE="${MASTER_PORT_BASE:-26035}" +export GPUS_PER_NODE CUDA_VISIBLE_DEVICES + +MAX_SAMPLES="${MAX_SAMPLES:-0}" +SHARD_MAXCOUNT="${SHARD_MAXCOUNT:-1000}" +SHARD_MAXSIZE="${SHARD_MAXSIZE:-2000000000}" +INDEX_WORKERS="${INDEX_WORKERS:-8}" +MIN_FREE_GB="${MIN_FREE_GB:-20}" +KEEP_PREPARED_DATA="${KEEP_PREPARED_DATA:-1}" +PREPARE_ONLY="${PREPARE_ONLY:-0}" + +export MIDTRAIN_TRAINABLE_MODULES="${MIDTRAIN_TRAINABLE_MODULES:-language_model adapter vision_model}" +export NO_SAVE_OPTIM_RNG="${NO_SAVE_OPTIM_RNG:-1}" +export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-1}" + +export WANDB_ENABLE="${WANDB_ENABLE:-1}" +export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" + +if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then + echo "Missing START_CKPT/latest_checkpointed_iteration.txt: $START_CKPT" + exit 1 +fi + +sanitize_name() { + echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_]/_/g' +} + +prepare_branch() { + local branch="$1" + local branch_safe="$2" + local data_path="$3" + + if [[ -f "$data_path/.nv-meta/dataset.yaml" ]]; then + echo "[$branch] prepared dataset already exists: $data_path" + return + fi + + local data_glob="${DATA_FILES_GLOB:-$branch/$LANG_CODE/*/*.parquet}" + echo "[$branch] preparing HF files: $data_glob" + "$PYTHON_BIN" tools/prepare_hf_caption_parquet_to_energon_wds.py \ + --repo-id "$HF_REPO_ID" \ + --data-files "$data_glob" \ + --output-dir "$data_path" \ + --max-samples "$MAX_SAMPLES" \ + --maxcount "$SHARD_MAXCOUNT" \ + --maxsize "$SHARD_MAXSIZE" \ + --shard-prefix "${branch_safe}-${LANG_LOWER}" \ + --cache-dir "$CACHE_DIR" \ + --index-workers "$INDEX_WORKERS" \ + --min-free-gb "$MIN_FREE_GB" +} + +run_branch() { + local branch="$1" + local index="$2" + local branch_safe + branch_safe="$(sanitize_name "$branch")" + + local data_path="$DATA_ROOT/${branch_safe}_${LANG_LOWER}_webdataset" + local save_path="$REPO_ROOT/stage_1_5_midtraining_mobilellm_fastvit_${branch_safe}_${LANG_LOWER}_full_wandb" + + echo "====================================================================" + echo "Stage 1.5 branch: $branch/$LANG_CODE" + echo "Load checkpoint : $CHECKPOINT_PATH" + echo "Data path : $data_path" + echo "Save checkpoint : $save_path" + echo "Steps : $NSTEP" + echo "Global batch : $GBS" + echo "Max samples prep: $MAX_SAMPLES (0 means full branch)" + echo "====================================================================" + + prepare_branch "$branch" "$branch_safe" "$data_path" + + if [[ "$PREPARE_ONLY" == "1" ]]; then + echo "[$branch] PREPARE_ONLY=1, skipping training." + return + fi + + export DATA_PATH="$data_path" + export CHECKPOINT_PATH + export SAVE_CKPT_PATH="$save_path" + export TENSORBOARD_PATH="$save_path/tensorboard" + export MASTER_PORT="$((MASTER_PORT_BASE + index))" + export WANDB_NAME="${WANDB_NAME_PREFIX:-stage1_5_mobilellm_fastvit}_${branch_safe}_${LANG_LOWER}_full_${NSTEP}steps_${GPUS_PER_NODE}gpu" + + bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh \ + "$TP" "$PP" "$SEQ_LEN" "$MBS" "$GBS" "$NSTEP" + + if [[ ! -f "$save_path/latest_checkpointed_iteration.txt" ]]; then + echo "[$branch] missing checkpoint tracker after training: $save_path" + exit 1 + fi + + CHECKPOINT_PATH="$save_path" + + if [[ "$KEEP_PREPARED_DATA" == "0" ]]; then + echo "[$branch] removing prepared data: $data_path" + rm -rf "$data_path" + fi +} + +index=0 +for branch in "$@"; do + run_branch "$branch" "$index" + index=$((index + 1)) +done + +echo "All requested English branches completed." diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh new file mode 100755 index 00000000..0ce5dc1f --- /dev/null +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh @@ -0,0 +1,215 @@ +#!/bin/bash +# Stage 1.5 midtraining for the MobileLLM-R1-140M + FastViT/FastVLM pipeline. +# +# This launcher expects a small Energon/WebDataset slice prepared from: +# mvp-lab/LLaVA-OneVision-1.5-Mid-Training-85M / imagenet / EN +# See tools/prepare_hf_caption_parquet_to_energon_wds.py. + +set -euo pipefail + +REPO_ROOT="${REPO_ROOT:-/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5}" +AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" +AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" +TORCHRUN="${TORCHRUN:-torchrun}" +PYTHON_BIN="${PYTHON_BIN:-python}" + +TP="${1:-1}" +PP="${2:-1}" +SEQ_LEN="${3:-4096}" +MBS="${4:-1}" +GBS="${5:-4}" +NSTEP="${6:-1000}" + +DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/midtraining_imagenet_en_part00_webdataset"}" +TOKENIZER_PATH="${TOKENIZER_PATH:-facebook/MobileLLM-R1-140M}" +CHECKPOINT_PATH="${CHECKPOINT_PATH:-"$REPO_ROOT/stage_1_alignment_mobilellm_140m_fastvlm_faithful"}" +SAVE_CKPT_PATH="${SAVE_CKPT_PATH:-"$REPO_ROOT/stage_1_5_midtraining_mobilellm_fastvit_imagenet_en"}" +TENSORBOARD_PATH="${TENSORBOARD_PATH:-"$SAVE_CKPT_PATH/tensorboard"}" + +GPUS_PER_NODE="${GPUS_PER_NODE:-4}" +MASTER_PORT="${MASTER_PORT:-26015}" +SAVE_INTERVAL="${SAVE_INTERVAL:-250}" +NUM_WORKERS="${NUM_WORKERS:-8}" +MIDTRAIN_TRAINABLE_MODULES="${MIDTRAIN_TRAINABLE_MODULES:-language_model adapter vision_model}" + +if [ ! -f "$DATA_PATH/.nv-meta/dataset.yaml" ]; then + echo "Missing prepared midtraining dataset: $DATA_PATH" + echo "Create a small ImageNet-EN slice first, for example:" + echo " python tools/prepare_hf_caption_parquet_to_energon_wds.py \\" + echo " --data-files 'imagenet/EN/part00/*.parquet' \\" + echo " --max-samples 10000 \\" + echo " --output-dir '$DATA_PATH'" + exit 1 +fi + +if [ ! -f "$CHECKPOINT_PATH/latest_checkpointed_iteration.txt" ]; then + echo "Missing Stage 1 checkpoint tracker: $CHECKPOINT_PATH/latest_checkpointed_iteration.txt" + echo "Set CHECKPOINT_PATH to the completed 2500-step Stage 1 alignment checkpoint directory." + exit 1 +fi + +mkdir -p "$SAVE_CKPT_PATH" "$TENSORBOARD_PATH" "$SAVE_CKPT_PATH/dataloader" + +declare -a list_ip=("localhost") +CURRENT_IP=$(hostname -I | awk '{print $1}') +if [ -z "$CURRENT_IP" ]; then + CURRENT_IP=$(hostname -i 2>/dev/null | awk '{print $1}') +fi + +SINGLE_NODE=0 +if [[ ${#list_ip[@]} -eq 1 && ( "${list_ip[0]}" == "localhost" || "${list_ip[0]}" == "127.0.0.1" ) ]]; then + SINGLE_NODE=1 +fi + +NNODES=${#list_ip[@]} +MASTER_ADDR=${MASTER_ADDR:-${list_ip[0]}} + +if [[ $SINGLE_NODE -eq 1 ]]; then + NNODES=1 + MASTER_ADDR=127.0.0.1 + NODE_RANK=0 + DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --master_port "$MASTER_PORT" + ) +else + NODE_RANK=-1 + for i in "${!list_ip[@]}"; do + if [[ "${list_ip[$i]}" == "${CURRENT_IP}" ]]; then + NODE_RANK=$i + break + fi + done + if [ "$NODE_RANK" -eq -1 ]; then + echo "Error: Current IP ($CURRENT_IP) not found in list_ip." + exit 1 + fi + DISTRIBUTED_ARGS=( + --nproc_per_node "$GPUS_PER_NODE" + --nnodes "$NNODES" + --node_rank "$NODE_RANK" + --master_addr "$MASTER_ADDR" + --master_port "$MASTER_PORT" + ) +fi + +MODEL_ARGS=( + --model-name llava-ov-mobilellm-140m +) + +DATA_ARGS=( + --tokenizer-type HFTokenizer + --hf-tokenizer-path "$TOKENIZER_PATH" + --data-path "$DATA_PATH" + --dataloader-type external + --split 100,0,0 + --num-workers "$NUM_WORKERS" + --chat-template llama3 + + --use-fastvit + --fastvit-image-size 1024 + --vision-tower-name mobileclip_l_1024 + --image-aspect-ratio pad +) + +read -r -a TRAINABLE_MODULES_ARRAY <<< "$MIDTRAIN_TRAINABLE_MODULES" + +TRAINING_ARGS=( + --training-phase sft + --trainable-modules "${TRAINABLE_MODULES_ARRAY[@]}" + --no-gradient-accumulation-fusion + --seq-length "$SEQ_LEN" + --no-rope-fusion + --training-rice-vl-max-answer-length "$SEQ_LEN" + --transformer-impl local + --max-position-embeddings 32768 + --init-method-std 0.02 + --micro-batch-size "$MBS" + --global-batch-size "$GBS" + --lr "${LR:-1.0e-5}" + --min-lr "${MIN_LR:-1.0e-6}" + --clip-grad 1.0 + --weight-decay "${WEIGHT_DECAY:-0.01}" + --optimizer adam + --adam-beta1 0.9 + --adam-beta2 0.95 + --adam-eps 1e-05 + --norm-epsilon 1e-05 + --train-iters "$NSTEP" + --lr-decay-iters "$NSTEP" + --lr-decay-style cosine + --lr-warmup-fraction "${LR_WARMUP_FRACTION:-0.002}" + --initial-loss-scale 65536 + --bf16 + --finetune + --load "$CHECKPOINT_PATH" + --save "$SAVE_CKPT_PATH" + --save-interval "$SAVE_INTERVAL" + --ckpt-format torch + --dataloader-save "$SAVE_CKPT_PATH/dataloader" + --ckpt-fully-parallel-load + --recompute-granularity full + --recompute-method uniform + --recompute-num-layers 3 +) + +if [[ "${NO_SAVE_OPTIM_RNG:-0}" == "1" ]]; then + TRAINING_ARGS+=( + --no-save-optim + --no-save-rng + ) +fi + +MODEL_PARALLEL_ARGS=( + --attention-backend local + --pipeline-model-parallel-size "$PP" + --tensor-model-parallel-size "$TP" + --use-distributed-optimizer + --distributed-backend nccl +) + +LOGGING_ARGS=( + --log-interval 1 + --tensorboard-dir "$TENSORBOARD_PATH" + --log-timers-to-tensorboard +) + +if [[ "${WANDB_ENABLE:-0}" == "1" || -n "${WANDB_API_KEY:-}" ]]; then + LOGGING_ARGS+=( + --wandb-project "${WANDB_PROJECT:-llava-ov-mobilellm}" + --wandb-exp-name "${WANDB_NAME:-stage1_5-mobilellm-fastvit-imagenet-en}" + ) +fi + +TM=$(date "+%Y-%m-%d_%H:%M:%S") +logfile="$SAVE_CKPT_PATH/run_${TM}_tp${TP}_pp${PP}_seqlen${SEQ_LEN}_mbs${MBS}_gbs${GBS}_${NSTEP}steps.log" + +export OFFLINE_PACKED_DATA=1 +export OFFLINE_PACKING_VQA=1 +export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-garbage_collection_threshold:0.72,max_split_size_mb:128}" + +echo "=========================================" +echo "Starting Stage 1.5 Midtraining" +echo "Model: MobileLLM-R1-140M + FastViT/FastVLM" +echo "Data: $DATA_PATH" +echo "Load Stage 1: $CHECKPOINT_PATH" +echo "Save Stage 1.5: $SAVE_CKPT_PATH" +echo "Trainable modules: ${TRAINABLE_MODULES_ARRAY[*]}" +echo "GPUs: $GPUS_PER_NODE | SEQ_LEN: $SEQ_LEN | MBS: $MBS | GBS: $GBS | ITERS: $NSTEP" +echo "=========================================" + +if [[ "${PRINT_DATA_SAMPLE:-1}" == "1" ]]; then + "$PYTHON_BIN" "$REPO_ROOT/tools/preview_caption_wds_sample.py" "$DATA_PATH" \ + --sample-index "${DATA_SAMPLE_INDEX:-0}" \ + --max-chars "${DATA_SAMPLE_MAX_CHARS:-700}" || true +fi + +PYTHONPATH="$AIAK_MAGATRON_PATH:$AIAK_TRAINING_PATH:${PYTHONPATH:-}" \ +"$TORCHRUN" "${DISTRIBUTED_ARGS[@]}" \ + "$AIAK_TRAINING_PATH/aiak_training_llm/train.py" \ + "${MODEL_ARGS[@]}" \ + "${DATA_ARGS[@]}" \ + "${TRAINING_ARGS[@]}" \ + "${MODEL_PARALLEL_ARGS[@]}" \ + "${LOGGING_ARGS[@]}" \ + 2>&1 | tee "$logfile" diff --git a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh index 91582e14..4d1cb02e 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1_alignment_mobilellm_140m.sh @@ -11,10 +11,10 @@ AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" # Model parallelism configuration TP="${1:-1}" # Tensor parallel PP="${2:-1}" # Pipeline parallel -SEQ_LEN="${3:-32768}" # Sequence length (reduced for testing) +SEQ_LEN="${3:-32768}" # Sequence length MBS="${4:-1}" # Micro batch size -GBS="${5:-4}" # Global batch size (for TP=1, PP=1, 8 GPUs -> DP=8) -NSTEP="${6:-100}" # Number of training iterations (1 step with 4 examples) +GBS="${5:-2}" # Global batch size (TP=1, PP=1, 2 GPUs -> DP=2, 1 sample/GPU) +NSTEP="${6:-2500}" # Total training iterations (resume from last checkpoint) # Data paths - UPDATE THESE FOR YOUR SETUP DATA_PATH="${DATA_PATH:-"$REPO_ROOT/data/LLaVA-558K-Webdataset"}" @@ -73,16 +73,16 @@ echo "Current Node Rank: ${NODE_RANK}" echo "Node Size: ${NNODES}" # Output directories -SAVE_CKPT_PATH=$(basename "$0" .sh) -TENSORBOARD_PATH="${SAVE_CKPT_PATH}/tensorboard" +SAVE_CKPT_PATH="${SAVE_CKPT_PATH:-$(basename "$0" .sh)}" +TENSORBOARD_PATH="${TENSORBOARD_PATH:-${SAVE_CKPT_PATH}/tensorboard}" mkdir -p "$SAVE_CKPT_PATH" mkdir -p "$TENSORBOARD_PATH" mkdir -p "$SAVE_CKPT_PATH/dataloader" -GPUS_PER_NODE=${GPUS_PER_NODE:-4} +GPUS_PER_NODE=${GPUS_PER_NODE:-2} MASTER_PORT=${MASTER_PORT:-26000} -SAVE_INTERVAL=${SAVE_INTERVAL:-1} +SAVE_INTERVAL=${SAVE_INTERVAL:-100} if [[ $SINGLE_NODE -eq 1 ]]; then DISTRIBUTED_ARGS=( @@ -138,7 +138,7 @@ TRAINING_ARGS=( --no-gradient-accumulation-fusion --seq-length "${SEQ_LEN}" --no-rope-fusion - --training-rice-vl-max-answer-length 32768 + --training-rice-vl-max-answer-length "${SEQ_LEN}" --transformer-impl local --max-position-embeddings 32768 # MobileLLM supports 32k context --init-method-std 0.02 diff --git a/tools/create_faithful_mobilellm_fastvit_checkpoint.py b/tools/create_faithful_mobilellm_fastvit_checkpoint.py new file mode 100644 index 00000000..9efec30e --- /dev/null +++ b/tools/create_faithful_mobilellm_fastvit_checkpoint.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Create a repo-faithful FastViT + MobileLLM checkpoint for LLaVA-OV. + +The original merged checkpoint stores MobileLLM weights under standalone +language-model names such as ``embedding.*`` and ``decoder.*``. The wrapped +LLaVA-OneVision model expects those weights under ``language_model.*``. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Dict, Tuple + +import torch + + +def _checkpoint_file(checkpoint_dir: Path) -> Path: + return checkpoint_dir / "release" / "mp_rank_00" / "model_optim_rng.pt" + + +def _pad_embedding( + weight: torch.Tensor, + target_vocab_size: int, + std: float, + seed: int, +) -> Tuple[torch.Tensor, int]: + current_vocab_size, hidden_size = weight.shape + if current_vocab_size == target_vocab_size: + return weight, 0 + if current_vocab_size > target_vocab_size: + raise ValueError( + f"Embedding has {current_vocab_size} rows, larger than target {target_vocab_size}." + ) + + generator = torch.Generator(device="cpu") + generator.manual_seed(seed) + extra = torch.empty( + target_vocab_size - current_vocab_size, + hidden_size, + dtype=weight.dtype, + device="cpu", + ) + extra.normal_(mean=0.0, std=std, generator=generator) + return torch.cat([weight.cpu(), extra], dim=0), extra.shape[0] + + +def _remap_language_key(key: str) -> str | None: + if key == "embedding.word_embeddings.weight": + return "language_model.embedding.word_embeddings.weight" + if key == "decoder.final_layernorm.weight": + return "language_model.decoder.final_layernorm.weight" + if "_extra_state" in key: + return None + if not key.startswith("decoder.layers."): + return None + + replacements = ( + (".self_attention.linear_qkv.weight", ".self_attention.linear_qkv.linear.weight"), + ( + ".self_attention.linear_qkv.layer_norm_weight", + ".self_attention.linear_qkv.ln.weight", + ), + (".self_attention.linear_proj.weight", ".self_attention.linear_proj.linear.weight"), + (".mlp.linear_fc1.weight", ".mlp.linear_fc1.linear.weight"), + (".mlp.linear_fc1.layer_norm_weight", ".mlp.linear_fc1.ln.weight"), + (".mlp.linear_fc2.weight", ".mlp.linear_fc2.linear.weight"), + ) + for old_suffix, new_suffix in replacements: + if key.endswith(old_suffix): + return "language_model." + key[: -len(old_suffix)] + new_suffix + return None + + +def create_checkpoint( + source_dir: Path, + output_dir: Path, + target_vocab_size: int, + embedding_init_std: float, + embedding_seed: int, +) -> None: + source_file = _checkpoint_file(source_dir) + output_file = _checkpoint_file(output_dir) + if not source_file.exists(): + raise FileNotFoundError(source_file) + if output_file.exists(): + raise FileExistsError( + f"{output_file} already exists. Move it aside if you want to recreate it." + ) + + checkpoint = torch.load(source_file, map_location="cpu", weights_only=False) + source_model: Dict[str, torch.Tensor] = checkpoint["model"] + output_model: Dict[str, torch.Tensor] = {} + + counts = { + "vision": 0, + "language": 0, + "language_ln_bias": 0, + "skipped_extra_state": 0, + "skipped_other": 0, + } + added_embedding_rows = 0 + + for key, value in source_model.items(): + if key.startswith("vision_model."): + output_model[key] = value.cpu() if torch.is_tensor(value) else value + counts["vision"] += 1 + continue + + new_key = _remap_language_key(key) + if new_key is None: + if "_extra_state" in key: + counts["skipped_extra_state"] += 1 + else: + counts["skipped_other"] += 1 + continue + + tensor = value.cpu() if torch.is_tensor(value) else value + if key == "embedding.word_embeddings.weight": + tensor, added_embedding_rows = _pad_embedding( + tensor, + target_vocab_size=target_vocab_size, + std=embedding_init_std, + seed=embedding_seed, + ) + output_model[new_key] = tensor + counts["language"] += 1 + + if new_key.endswith(".linear_qkv.ln.weight") or new_key.endswith(".linear_fc1.ln.weight"): + bias_key = new_key[:-len("weight")] + "bias" + output_model[bias_key] = torch.zeros_like(tensor) + counts["language_ln_bias"] += 1 + + if "language_model.embedding.word_embeddings.weight" not in output_model: + raise RuntimeError("Failed to remap MobileLLM embedding weight.") + if not any(key.startswith("vision_model.") for key in output_model): + raise RuntimeError("No FastViT vision weights were copied.") + if any(key.startswith(("embedding.", "decoder.")) for key in output_model): + raise RuntimeError("Unwrapped language keys leaked into output checkpoint.") + + output_file.parent.mkdir(parents=True, exist_ok=True) + torch.save( + { + "model": output_model, + "checkpoint_version": checkpoint.get("checkpoint_version", 3.0), + "args": checkpoint.get("args", None), + "iteration": 0, + }, + output_file, + ) + (output_dir / "latest_checkpointed_iteration.txt").write_text("release\n") + + print(f"source: {source_file}") + print(f"output: {output_file}") + print(f"keys written: {len(output_model)}") + print(f"vision keys copied: {counts['vision']}") + print(f"language tensors remapped: {counts['language']}") + print(f"layernorm biases added: {counts['language_ln_bias']}") + print(f"embedding rows added: {added_embedding_rows}") + print(f"extra_state keys skipped: {counts['skipped_extra_state']}") + print(f"other keys skipped: {counts['skipped_other']}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source", + type=Path, + default=Path("checkpoints/mobilellm-fastvit-merged-tp1-pp1"), + ) + parser.add_argument( + "--output", + type=Path, + default=Path("checkpoints/mobilellm-fastvit-merged-faithful-tp1-pp1"), + ) + parser.add_argument("--target-vocab-size", type=int, default=128384) + parser.add_argument("--embedding-init-std", type=float, default=0.02) + parser.add_argument("--embedding-seed", type=int, default=1234) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + create_checkpoint( + source_dir=args.source, + output_dir=args.output, + target_vocab_size=args.target_vocab_size, + embedding_init_std=args.embedding_init_std, + embedding_seed=args.embedding_seed, + ) + + +if __name__ == "__main__": + main() diff --git a/tools/prepare_hf_caption_parquet_to_energon_wds.py b/tools/prepare_hf_caption_parquet_to_energon_wds.py new file mode 100644 index 00000000..5aceb812 --- /dev/null +++ b/tools/prepare_hf_caption_parquet_to_energon_wds.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Stream HF image-caption parquet data into the Energon WebDataset format. + +This is intentionally small-slice friendly. Use it for one Mid-Training-85M +folder at a time, e.g. ImageNet-EN part00, instead of cloning/downloading the +whole Hugging Face dataset. +""" + +from __future__ import annotations + +import argparse +import io +import json +import shutil +import sys +from pathlib import Path +from typing import Any, Iterable + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "aiak_megatron")) + +from megatron.energon.epathlib import EPath # noqa: E402 +from megatron.energon.flavors import BaseWebdatasetFactory # noqa: E402 +from megatron.energon.flavors.webdataset import MAIN_FOLDER_NAME # noqa: E402 + + +def _image_to_jpeg_bytes(image: Any) -> bytes: + if image is None: + raise ValueError("sample has no image") + if isinstance(image, dict) and image.get("bytes") is not None: + return image["bytes"] + if isinstance(image, (bytes, bytearray)): + return bytes(image) + + # HF Image features usually decode to PIL.Image in non-streaming and + # streaming modes. Normalize to RGB JPEG so the Energon loader can use PIL. + if hasattr(image, "convert"): + image = image.convert("RGB") + buf = io.BytesIO() + image.save(buf, format="JPEG", quality=95) + return buf.getvalue() + + raise TypeError(f"Unsupported image value: {type(image)!r}") + + +def _safe_key(raw_id: Any, index: int) -> str: + text = str(raw_id or f"sample_{index:09d}") + text = text.replace("/", "_").replace(".", "_") + return "".join(ch if ch.isalnum() or ch in "_-" else "_" for ch in text) + + +def _sample_loader_template() -> str: + return "\n".join( + [ + "def sample_loader(sample: dict) -> dict:", + " data = sample['json']", + " images = [sample.get(f'img{i}.jpg') for i in range(len(data['images']))]", + " captions = data['captions']", + " prompts = data['prompts']", + " return dict(", + " __key__=sample['__key__'],", + " __restore_key__=sample['__restore_key__'],", + " captions=captions,", + " prompts=prompts,", + " images=images,", + " )", + "def part_filter(part: str) -> bool:", + " return True", + "", + ] + ) + + +def _write_metadata(output_dir: Path, tar_names: list[str], workers: int) -> None: + meta_dir = output_dir / MAIN_FOLDER_NAME + meta_dir.mkdir(parents=True, exist_ok=True) + + dataset_definition = { + "sample_type": { + "__module__": "aiak_training_llm.data.multimodal", + "__class__": "PackedCaptioningSample", + }, + "part_filter": "sample_loader.py:part_filter", + "sample_loader": "sample_loader.py:sample_loader", + } + (meta_dir / "dataset.yaml").write_text( + yaml.safe_dump(dataset_definition, sort_keys=False), + encoding="utf-8", + ) + (meta_dir / "sample_loader.py").write_text(_sample_loader_template(), encoding="utf-8") + + BaseWebdatasetFactory.prepare_dataset( + EPath(output_dir).absolute(), + tar_names, + split_parts_ratio=[("train", 1.0), ("val", 0.0), ("test", 0.0)], + tar_index_only=False, + workers=workers, + ) + + +def _iter_hf_samples(args: argparse.Namespace) -> Iterable[dict[str, Any]]: + from datasets import load_dataset + + data_files = {"train": args.data_files} + dataset = load_dataset( + args.repo_id, + data_files=data_files, + split="train", + streaming=True, + cache_dir=args.cache_dir, + ) + if args.shuffle_buffer_size > 0: + dataset = dataset.shuffle( + seed=args.seed, + buffer_size=args.shuffle_buffer_size, + ) + return dataset + + +def convert(args: argparse.Namespace) -> None: + import webdataset as wds + from tqdm import tqdm + + args.output_dir.mkdir(parents=True, exist_ok=True) + free_gb = shutil.disk_usage(args.output_dir).free / (1024**3) + if free_gb < args.min_free_gb: + raise RuntimeError( + f"{args.output_dir} has only {free_gb:.1f} GB free; " + f"need at least {args.min_free_gb:.1f} GB before writing shards" + ) + + pattern = str(args.output_dir / f"{args.shard_prefix}-%06d.tar") + written = 0 + + with wds.ShardWriter(pattern, maxcount=args.maxcount, maxsize=args.maxsize) as sink: + for index, row in enumerate(tqdm(_iter_hf_samples(args), desc="streaming HF rows")): + if args.max_samples and written >= args.max_samples: + break + + caption = row.get(args.caption_column) + image = row.get(args.image_column) + if not caption or image is None: + continue + + key = _safe_key(row.get(args.id_column), index) + payload = { + "images": [f"{key}.img0.jpg"], + "captions": [str(caption)], + "prompts": [args.prompt], + } + try: + image_bytes = _image_to_jpeg_bytes(image) + except Exception as exc: + print(f"[skip] {key}: {exc}", file=sys.stderr) + continue + + sink.write( + { + "__key__": key, + "img0.jpg": image_bytes, + "json": json.dumps(payload, ensure_ascii=False).encode("utf-8"), + } + ) + written += 1 + + tar_names = sorted(p.name for p in args.output_dir.glob(f"{args.shard_prefix}-*.tar")) + if not tar_names: + raise RuntimeError(f"No shards were written to {args.output_dir}") + _write_metadata(args.output_dir, tar_names, args.index_workers) + print(f"Wrote {written} samples to {args.output_dir}") + print("Use as DATA_PATH for Stage 1.5 midtraining.") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-id", default="mvp-lab/LLaVA-OneVision-1.5-Mid-Training-85M") + parser.add_argument( + "--data-files", + nargs="+", + default=["imagenet/EN/part00/*.parquet"], + help="Remote parquet files/globs inside the HF dataset repo.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("data/midtraining_imagenet_en_part00_webdataset"), + ) + parser.add_argument("--cache-dir", default="data/hf_cache_midtraining_stream") + parser.add_argument("--max-samples", type=int, default=10000) + parser.add_argument("--maxcount", type=int, default=1000) + parser.add_argument("--maxsize", type=int, default=2_000_000_000) + parser.add_argument("--min-free-gb", type=float, default=8.0) + parser.add_argument("--shard-prefix", default="imagenet-en") + parser.add_argument("--prompt", default="\nDescribe the image in detail.") + parser.add_argument("--id-column", default="id") + parser.add_argument("--image-column", default="image") + parser.add_argument("--caption-column", default="caption") + parser.add_argument("--shuffle-buffer-size", type=int, default=0) + parser.add_argument("--seed", type=int, default=1234) + parser.add_argument("--index-workers", type=int, default=8) + return parser.parse_args() + + +def main() -> None: + convert(parse_args()) + + +if __name__ == "__main__": + main() diff --git a/tools/preview_caption_wds_sample.py b/tools/preview_caption_wds_sample.py new file mode 100644 index 00000000..b060faf6 --- /dev/null +++ b/tools/preview_caption_wds_sample.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Print one caption WebDataset sample for run-log sanity checks.""" + +from __future__ import annotations + +import argparse +import io +import json +import tarfile +from pathlib import Path + + +def _truncate(text: str, limit: int) -> str: + if len(text) <= limit: + return text + return text[: limit - 3].rstrip() + "..." + + +def preview(data_path: Path, sample_index: int, max_chars: int) -> None: + tar_paths = sorted(data_path.glob("*.tar")) + if not tar_paths: + raise FileNotFoundError(f"No .tar shards found in {data_path}") + + seen = 0 + for tar_path in tar_paths: + with tarfile.open(tar_path, "r") as tar: + json_members = sorted( + (member for member in tar.getmembers() if member.isfile() and member.name.endswith(".json")), + key=lambda member: member.name, + ) + for member in json_members: + if seen != sample_index: + seen += 1 + continue + + raw = tar.extractfile(member) + if raw is None: + raise RuntimeError(f"Could not read {member.name} from {tar_path}") + payload = json.load(raw) + + key = member.name[: -len(".json")] + image_members = [ + name for name in tar.getnames() if name.startswith(f"{key}.img") + ] + + image_info = "none" + if image_members: + image_member = tar.getmember(image_members[0]) + image_info = f"{image_members[0]} ({image_member.size} bytes)" + try: + from PIL import Image + + image_file = tar.extractfile(image_member) + if image_file is not None: + with Image.open(io.BytesIO(image_file.read())) as image: + image_info += f", size={image.size[0]}x{image.size[1]}" + except Exception: + pass + + prompts = payload.get("prompts") or [] + captions = payload.get("captions") or [] + + print("") + print("========== DATA SAMPLE PREVIEW ==========") + print(f"DATA_PATH: {data_path}") + print(f"SHARD : {tar_path.name}") + print(f"KEY : {key}") + print(f"IMAGE : {image_info}") + print(f"PROMPT : {_truncate(str(prompts[0] if prompts else ''), max_chars)}") + print(f"CAPTION : {_truncate(str(captions[0] if captions else ''), max_chars)}") + print("=========================================") + print("") + return + + raise IndexError(f"sample_index={sample_index} out of range; found {seen} samples") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("data_path", type=Path) + parser.add_argument("--sample-index", type=int, default=0) + parser.add_argument("--max-chars", type=int, default=700) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + preview(args.data_path, args.sample_index, args.max_chars) + + +if __name__ == "__main__": + main() From 47690d6ac019d06da5f57c03bd5adf70df98e42a Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 8 May 2026 01:12:59 +0400 Subject: [PATCH 29/39] Remove tracked Stage 1 log artifacts --- .../launch_100_gpu4.out | 0 ..._tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log | 8137 -- ..._tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log | 8137 -- ...tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log | 16036 ---- ...tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log | 62359 ---------------- 5 files changed, 94669 deletions(-) delete mode 100644 stage_1_alignment_mobilellm_140m/launch_100_gpu4.out delete mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log delete mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log delete mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log delete mode 100644 stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log diff --git a/stage_1_alignment_mobilellm_140m/launch_100_gpu4.out b/stage_1_alignment_mobilellm_140m/launch_100_gpu4.out deleted file mode 100644 index e69de29b..00000000 diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log deleted file mode 100644 index 1a1667eb..00000000 --- a/stage_1_alignment_mobilellm_140m/run_2026-04-29_08:56:32_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log +++ /dev/null @@ -1,8137 +0,0 @@ -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] -W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] ***************************************** -W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0429 08:56:42.924000 3546059 site-packages/torch/distributed/run.py:803] ***************************************** -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -False -False -False -False -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -parse arguments function called -parse arguments function called -parse arguments function called -parse arguments function called -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft --------------- Configure model to llava-ov-mobilellm-140m -------------- -[DEBUG] Model config type: LlavaOnevision1_5Config -[DEBUG] Is MobileLLM: True - num_layers = 15 - hidden_size = 576 - ffn_hidden_size = 2048 - num_attention_heads = 9 - group_query_attention = True - num_query_groups = 3 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = False - vocab_size_in_config_file = 128256 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-05 - rotary_base = 8000000 - kv_channels = None - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 4, data-parallel size: 4, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... False - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.local - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... llama3 - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 4 - data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 15 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 32768 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 1 - fastvit_image_size .............................. 1024 - ffn_hidden_size ................................. 2048 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 4 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M - hidden_dropout .................................. 0 - hidden_size ..................................... 576 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_aspect_ratio .............................. pad - image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 64 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ None - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 20 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-mobilellm-140m - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 9 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3Using unified model type for training. - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 15 - - num_layers_at_end_in_bf16 ....................... 1 -Starting training... num_layers_at_start_in_bf16 ..................... 1 - - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 3 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 16 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 3 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 8000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_mobilellm_140m - save_ema ........................................ None - save_interval ................................... 1 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 32768 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 20 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 32768Using unified model type for training. - training_rice_vl_max_image_area ................. 1806336 - - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. False - -Starting training... use_checkpoint_args ............................. False - - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_fastvit ..................................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vision_tower_name ............................... mobileclip_l_1024 - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 128256 - vpp_scheduler ................................... None - wandb_exp_name .................................. mobilellm_integration - wandb_project ................................... llava-ov-1_5 - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 4 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -Using unified model type for training. - -Starting training... -Using unified model type for training. - -Starting training... -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 -> AIAK building HFTokenizer tokenizer ... -> setting tensorboard ... -INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 -WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. -WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. -WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). - > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -wandb: Currently logged in as: rana-zayed (rana-zayed-mbzuai) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin -wandb: creating run -wandb: Tracking run with wandb version 0.21.0 -wandb: Run data is saved locally in stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_085758-7dva7qn6 -wandb: Run `wandb offline` to turn off syncing. -wandb: Syncing run mobilellm_integration -wandb: ⭐️ View project at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 -wandb: 🚀 View run at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/7dva7qn6 -[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank [Gloo] Rank 32 is connected to 3 is connected to peer ranks. Expected number of connected peer ranks is : 33 peer ranks. -Expected number of connected peer ranks is : 3 -[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank [Gloo] Rank 32 is connected to 3 is connected to peer ranks. Expected number of connected peer ranks is : 33 peer ranks. -Expected number of connected peer ranks is : 3 -[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 2 is connected to [Gloo] Rank 3 peer ranks. Expected number of connected peer ranks is : 33 - is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so -make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.080 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W429 08:58:05.082128572 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.933 seconds -time to initialize megatron (seconds): 27.517 -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2026-04-29 08:58:22 -building llava-ov-mobilellm-140m model ... -[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m -[DEBUG PROVIDER] Args num_layers: 15 -[DEBUG PROVIDER] Args hidden_size: 576 -[DEBUG PROVIDER] Args vocab_size: 128260 -[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 -[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 -[DEBUG PROVIDER] Model family: llava_ov_1_5 -[DEBUG PROVIDER] use_mobilellm flag: True -[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone -[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 -[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Loading vision config... -[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 -[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json -[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd -[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ================================================ -[DEBUG PROVIDER] Loading adapter config... -[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 -[DEBUG PROVIDER] Building layer specs... -[DEBUG PROVIDER] ✓ Using MobileLLM layer specification -[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] MobileLLM layer spec created successfully -[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... -[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 -[INFO] Using MobileLLM as language backbone -[INFO] Using MobileLLM as language backbone -[INFO] Using MobileLLM as language backbone -[INFO] Using MobileLLM as language backbone -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (11216448 elements, 11216512 padded size): - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias - module.adapter.layernorm.bias - module.adapter.layernorm.weight -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[DEBUG] FastViT enabled: use_fastvit=True -[DEBUG] Before handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -FastViT enabled: Will load vision tower from pretrained checkpoint: /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -[DEBUG] After handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -Checkpoint file not found in load directory None attempting to finetune with checkpoint in /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - loading release checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -could not find arguments in the checkpoint ... -Loading checkpoint with device: cuda:0, strict=False -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel - checkpoint version 3.0 - successfully loaded checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 [ t 1/1, p 1/1 ] at iteration 0 - failed to find checkpoint /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1/release/mp_rank_00/model_optim_rng.pt in wandb -(min, max) time across ranks (ms): - load-checkpoint ................................: (3234.68, 3235.04) -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 08:58:28 -================================================================================ -FULL MODEL STRUCTURE: -================================================================================ - ---- Model 0 (pipeline rank 0) --- -DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -) -================================================================================ - -================================================================================ -PARAMETER TRAINABILITY STATUS: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -================================================================================ -Total trainable parameters: 11,216,448 (11.22M) -Total frozen parameters: 265,417,440 (265.42M) -Total parameters: 276,633,888 (276.63M) -Trainable percentage: 4.05% -================================================================================ -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 80 - validation: 400 - test: 400 -Using shared training tokenizer in FastVLM mode -Ensured multimodal special tokens for FastVLM tokenizer; added=0 -Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M -Initialized FastViT processor with image_size=1024 -Processor config: .TokenizerWrapper object at 0x7f762c463e20> -image_resolution: None -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 375), ] sum(count)=375 -rank=0, worker=1: shard_range=[pretrain-000003.tar[375, 750), ] sum(count)=375 -rank=0, worker=2: shard_range=[pretrain-000003.tar[750, 1125), ] sum(count)=375 -rank=0, worker=3: shard_range=[pretrain-000003.tar[1125, 1500), ] sum(count)=375 -rank=0, worker=4: shard_range=[pretrain-000003.tar[1500, 1875), ] sum(count)=375 -rank=0, worker=5: shard_range=[pretrain-000003.tar[1875, 2250), ] sum(count)=375 -rank=0, worker=6: shard_range=[pretrain-000003.tar[2250, 2625), ] sum(count)=375 -rank=0, worker=7: shard_range=[pretrain-000003.tar[2625, 2999), ] sum(count)=374 -rank=0, worker=8: shard_range=[pretrain-000003.tar[2999, 3374), ] sum(count)=375 -rank=0, worker=9: shard_range=[pretrain-000003.tar[3374, 3749), ] sum(count)=375 -rank=0, worker=10: shard_range=[pretrain-000003.tar[3749, 4124), ] sum(count)=375 -rank=0, worker=11: shard_range=[pretrain-000003.tar[4124, 4498), ] sum(count)=374 -rank=0, worker=12: shard_range=[pretrain-000003.tar[4498, 4873), ] sum(count)=375 -rank=0, worker=13: shard_range=[pretrain-000003.tar[4873, 5058), pretrain-000001.tar[0, 190), ] sum(count)=375 -rank=0, worker=14: shard_range=[pretrain-000001.tar[190, 565), ] sum(count)=375 -rank=0, worker=15: shard_range=[pretrain-000001.tar[565, 939), ] sum(count)=374 -rank=3, worker=0: shard_range=[pretrain-000004.tar[2858, 3233), ] sum(count)=375 -rank=3, worker=1: shard_range=[pretrain-000004.tar[3233, 3608), ] sum(count)=375 -rank=3, worker=2: shard_range=[pretrain-000004.tar[3608, 3821), pretrain-000000.tar[0, 162), ] sum(count)=375 -rank=3, worker=3: shard_range=[pretrain-000000.tar[162, 537), ] sum(count)=375 -rank=3, worker=4: shard_range=[pretrain-000000.tar[537, 912), ] sum(count)=375 -rank=3, worker=5: shard_range=[pretrain-000000.tar[912, 1287), ] sum(count)=375 -rank=3, worker=6: shard_range=[pretrain-000000.tar[1287, 1662), ] sum(count)=375 -rank=3, worker=7: shard_range=[pretrain-000000.tar[1662, 2036), ] sum(count)=374 -rank=3, worker=8: shard_range=[pretrain-000000.tar[2036, 2411), ] sum(count)=375 -rank=3, worker=9: shard_range=[pretrain-000000.tar[2411, 2786), ] sum(count)=375 -rank=3, worker=10: shard_range=[pretrain-000000.tar[2786, 3161), ] sum(count)=375 -rank=3, worker=11: shard_range=[pretrain-000000.tar[3161, 3535), ] sum(count)=374rank=2, worker=0: shard_range=[pretrain-000002.tar[1900, 2275), ] sum(count)=375 - -rank=2, worker=1: shard_range=[pretrain-000002.tar[2275, 2650), ] sum(count)=375 -rank=2, worker=2: shard_range=[pretrain-000002.tar[2650, 3025), ] sum(count)=375rank=3, worker=12: shard_range=[pretrain-000000.tar[3535, 3910), ] sum(count)=375 -rank=2, worker=3: shard_range=[pretrain-000002.tar[3025, 3400), ] sum(count)=375 - -rank=2, worker=4: shard_range=[pretrain-000002.tar[3400, 3775), ] sum(count)=375 -rank=2, worker=5: shard_range=[pretrain-000002.tar[3775, 4150), ] sum(count)=375 -rank=2, worker=6: shard_range=[pretrain-000002.tar[4150, 4525), ] sum(count)=375rank=3, worker=13: shard_range=[pretrain-000000.tar[3910, 4285), ] sum(count)=375 -rank=2, worker=7: shard_range=[pretrain-000002.tar[4525, 4899), ] sum(count)=374 - -rank=2, worker=8: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 235), ] sum(count)=375 -rank=2, worker=9: shard_range=[pretrain-000004.tar[235, 610), ] sum(count)=375 -rank=3, worker=14: shard_range=[pretrain-000000.tar[4285, 4660), ] sum(count)=375rank=2, worker=10: shard_range=[pretrain-000004.tar[610, 985), ] sum(count)=375 -rank=2, worker=11: shard_range=[pretrain-000004.tar[985, 1359), ] sum(count)=374 - -rank=2, worker=12: shard_range=[pretrain-000004.tar[1359, 1734), ] sum(count)=375 -rank=2, worker=13: shard_range=[pretrain-000004.tar[1734, 2109), ] sum(count)=375rank=3, worker=15: shard_range=[pretrain-000000.tar[4660, 5034), ] sum(count)=374 -rank=2, worker=14: shard_range=[pretrain-000004.tar[2109, 2484), ] sum(count)=375 - -rank=2, worker=15: shard_range=[pretrain-000004.tar[2484, 2858), ] sum(count)=374 -rank=1, worker=0: shard_range=[pretrain-000001.tar[939, 1314), ] sum(count)=375 -rank=1, worker=1: shard_range=[pretrain-000001.tar[1314, 1689), ] sum(count)=375 -rank=1, worker=2: shard_range=[pretrain-000001.tar[1689, 2064), ] sum(count)=375 -rank=1, worker=3: shard_range=[pretrain-000001.tar[2064, 2439), ] sum(count)=375 -rank=1, worker=4: shard_range=[pretrain-000001.tar[2439, 2814), ] sum(count)=375 -rank=1, worker=5: shard_range=[pretrain-000001.tar[2814, 3189), ] sum(count)=375 -rank=1, worker=6: shard_range=[pretrain-000001.tar[3189, 3564), ] sum(count)=375 -rank=1, worker=7: shard_range=[pretrain-000001.tar[3564, 3938), ] sum(count)=374 -rank=1, worker=8: shard_range=[pretrain-000001.tar[3938, 4313), ] sum(count)=375 -rank=1, worker=9: shard_range=[pretrain-000001.tar[4313, 4688), ] sum(count)=375 -rank=1, worker=10: shard_range=[pretrain-000001.tar[4688, 5036), pretrain-000002.tar[0, 27), ] sum(count)=375 -rank=1, worker=11: shard_range=[pretrain-000002.tar[27, 401), ] sum(count)=374 -rank=1, worker=12: shard_range=[pretrain-000002.tar[401, 776), ] sum(count)=375 -rank=1, worker=13: shard_range=[pretrain-000002.tar[776, 1151), ] sum(count)=375 -rank=1, worker=14: shard_range=[pretrain-000002.tar[1151, 1526), ] sum(count)=375 -rank=1, worker=15: shard_range=[pretrain-000002.tar[1526, 1900), ] sum(count)=374 -[after dataloaders are built] datetime: 2026-04-29 08:58:28 -done with setup ... -training ...(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (5522.85, 5523.23) - train/valid/test-data-iterators-setup ..........: (94.62, 94.80) - -================================================================================ -training with the following parameter status: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 - -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2026-04-29 08:58:28 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1430]) -Dataloader batch - labels shape: torch.Size([1, 1430]) -Dataloader batch - attn_mask shape: torch.Size([1, 1430]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1757]) -Dataloader batch - labels shape: torch.Size([1, 1757]) -Dataloader batch - attn_mask shape: torch.Size([1, 1757]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1909]) -Dataloader batch - labels shape: torch.Size([1, 1909]) -Dataloader batch - attn_mask shape: torch.Size([1, 1909]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1356]) -Dataloader batch - labels shape: torch.Size([1, 1356]) -Dataloader batch - attn_mask shape: torch.Size([1, 1356]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1909) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1909) torch.int64 - loss_mask : 532 / 1909 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 256, 3072) - out : (32, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1909, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1909, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1909, 1, 576) torch.bfloat16 - out : (1, 1909) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 532 tokens) : 11.7835 - top-1 accuracy : 1/532 = 0.2% - -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -Number of parameters in transformer layers in billions: 0.07 -Number of parameters in embedding layers in billions: 0.07 -Total number of parameters in billions: 0.14 -Number of parameters in most loaded shard in billions: 0.1403 -Theoretical memory footprints: weight and optimizer=1204.55 MB - [2026-04-29 08:58:52] iteration 1/ 20 | consumed samples: 4 | elapsed time per iteration (ms): 24530.1 | throughput (token/sec/GPU): 65.8 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.179645E+01 | loss scale: 1.0 | grad norm: 2.994 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 |[Rank 0] (after 1 iterations) memory (MB) | allocated: 709.1826171875 | max allocated: 5398.64892578125 | reserved: 6388.0 | max reserved: 6388.0 - -saving checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 1 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2495.67, 2499.58) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1781]) -Dataloader batch - labels shape: torch.Size([1, 1781]) -Dataloader batch - attn_mask shape: torch.Size([1, 1781]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - Dataloader batch - tokens shape: torch.Size([1, 1186]) -Dataloader batch - labels shape: torch.Size([1, 1186]) -Dataloader batch - attn_mask shape: torch.Size([1, 1186]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -torch.Size([1, 942]) -Dataloader batch - labels shape:Dataloader batch - tokens shape: torch.Size([1, 1114]) -Dataloader batch - labels shape: torch.Size([1, 1114]) -Dataloader batch - attn_mask shape: torch.Size([1, 1114]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -torch.Size([1, 942]) -Dataloader batch - attn_mask shape: torch.Size([1, 942]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1186) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1186) torch.int64 - loss_mask : 273 / 1186 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 256, 3072) - out : (22, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1186, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1186, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1186, 1, 576) torch.bfloat16 - out : (1, 1186) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 273 tokens) : 11.8206 - top-1 accuracy : 0/273 = 0.0% - - [2026-04-29 08:58:58] iteration 2/ 20 | consumed samples: 8 | elapsed time per iteration (ms): 3313.5 | throughput (token/sec/GPU): 379.0 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.182033E+01 | loss scale: 1.0 | grad norm: 3.165 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 2 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2544.79, 2545.35) -Dataloader batch - tokens shape: torch.Size([1, 1554]) -Dataloader batch - labels shape: torch.Size([1, 1554]) -Dataloader batch - attn_mask shape: torch.Size([1, 1554]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1849]) -Dataloader batch - labels shape: torch.Size([1, 1849]) -Dataloader batch - attn_mask shape: torch.Size([1, 1849]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1399]) -Dataloader batch - labels shape: torch.Size([1, 1399]) -Dataloader batch - attn_mask shape: torch.Size([1, 1399]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1741]) -Dataloader batch - labels shape: torch.Size([1, 1741]) -Dataloader batch - attn_mask shape: torch.Size([1, 1741]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1399) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1399) torch.int64 - loss_mask : 342 / 1399 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 256, 3072) - out : (26, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1399, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1399, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1399, 1, 576) torch.bfloat16 - out : (1, 1399) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.7172 - top-1 accuracy : 0/342 = 0.0% - - [2026-04-29 08:59:03] iteration 3/ 20 | consumed samples: 12 | elapsed time per iteration (ms): 2636.4 | throughput (token/sec/GPU): 620.4 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.168310E+01 | loss scale: 1.0 | grad norm: 1.864 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 3 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2523.37, 2523.95) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1585]) -torch.Size([1, 1544])Dataloader batch - labels shape: torch.Size([1, 1585]) - -Dataloader batch - attn_mask shape:Dataloader batch - labels shape: torch.Size([1, 1585])torch.Size([1, 1544]) - -Dataloader batch - image_grid_thw shape:Dataloader batch - attn_mask shape: torch.Size([1, 1544])torch.Size([29, 3]) - -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1042]) -Dataloader batch - labels shape: torch.Size([1, 1042]) -Dataloader batch - attn_mask shape: torch.Size([1, 1042]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1585) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1585) torch.int64 - loss_mask : 350 / 1585 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 256, 3072) - out : (29, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1585, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1585, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1585, 1, 576) torch.bfloat16 - out : (1, 1585) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.5905 - top-1 accuracy : 0/350 = 0.0% - - [2026-04-29 08:59:08] iteration 4/ 20 | consumed samples: 16 | elapsed time per iteration (ms): 2075.7 | throughput (token/sec/GPU): 676.3 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.163278E+01 | loss scale: 1.0 | grad norm: 1.456 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 4 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (3088.82, 3089.15) -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1412])Dataloader batch - tokens shape: torch.Size([1, 1214]) -Dataloader batch - labels shape: -torch.Size([1, 1214]) -Dataloader batch - attn_mask shape:Dataloader batch - labels shape: torch.Size([1, 1214]) - Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 367 / 1541 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 256, 3072) - out : (28, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.5435 - top-1 accuracy : 0/367 = 0.0% - - [2026-04-29 08:59:13] iteration 5/ 20 | consumed samples: 20 | elapsed time per iteration (ms): 2098.3 | throughput (token/sec/GPU): 638.4 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.153166E+01 | loss scale: 1.0 | grad norm: 1.349 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 5 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2403.65, 2403.98) -Dataloader batch - tokens shape: torch.Size([1, 1344]) -Dataloader batch - labels shape: torch.Size([1, 1344]) -Dataloader batch - attn_mask shape: torch.Size([1, 1344]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1498]) -Dataloader batch - labels shape: torch.Size([1, 1498]) -Dataloader batch - attn_mask shape: torch.Size([1, 1498]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1689]) -Dataloader batch - labels shape: torch.Size([1, 1689]) -Dataloader batch - attn_mask shape: torch.Size([1, 1689]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1344) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1344) torch.int64 - loss_mask : 399 / 1344 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 256, 3072) - out : (22, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1344, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1344, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1344, 1, 576) torch.bfloat16 - out : (1, 1344) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.5155 - top-1 accuracy : 1/399 = 0.3% - - [2026-04-29 08:59:18] iteration 6/ 20 | consumed samples: 24 | elapsed time per iteration (ms): 2397.1 | throughput (token/sec/GPU): 625.5 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.151996E+01 | loss scale: 1.0 | grad norm: 1.014 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 6 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2450.32, 2450.99) -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) -Dataloader batch - tokens shape: Dataloader batch - tokens shape:torch.Size([1, 1352]) -torch.Size([1, 1824])Dataloader batch - labels shape: - Dataloader batch - labels shape:torch.Size([1, 1352]) -torch.Size([1, 1824])Dataloader batch - attn_mask shape: - Dataloader batch - attn_mask shape: torch.Size([1, 1352])torch.Size([1, 1824]) - -Dataloader batch - image_grid_thw shape: torch.Size([32, 3])Dataloader batch - image_grid_thw shape: - torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1352) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1352) torch.int64 - loss_mask : 318 / 1352 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 256, 3072) - out : (25, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1352, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1352, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1352, 1, 576) torch.bfloat16 - out : (1, 1352) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.5107 - top-1 accuracy : 0/318 = 0.0% - - [2026-04-29 08:59:23] iteration 7/ 20 | consumed samples: 28 | elapsed time per iteration (ms): 2798.8 | throughput (token/sec/GPU): 557.4 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.149603E+01 | loss scale: 1.0 | grad norm: 0.963 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 7 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2402.95, 2403.50) -Dataloader batch - tokens shape: torch.Size([1, 1612]) -Dataloader batch - labels shape: torch.Size([1, 1612]) -Dataloader batch - attn_mask shape: torch.Size([1, 1612]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 995]) -Dataloader batch - labels shape: torch.Size([1, 995]) -Dataloader batch - attn_mask shape:torch.Size([1, 1527]) torch.Size([1, 995]) -Dataloader batch - image_grid_thw shape: - torch.Size([18, 3]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1558]) -Dataloader batch - labels shape: torch.Size([1, 1558]) -Dataloader batch - attn_mask shape: torch.Size([1, 1558]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 995) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 995) torch.int64 - loss_mask : 227 / 995 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 256, 3072) - out : (18, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (995, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (995, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (995, 1, 576) torch.bfloat16 - out : (1, 995) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 227 tokens) : 11.4816 - top-1 accuracy : 0/227 = 0.0% - - [2026-04-29 08:59:28] iteration 8/ 20 | consumed samples: 32 | elapsed time per iteration (ms): 2316.3 | throughput (token/sec/GPU): 614.3 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.145122E+01 | loss scale: 1.0 | grad norm: 0.860 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 8 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2311.63, 2311.82) -Dataloader batch - tokens shape: torch.Size([1, 1026]) -Dataloader batch - labels shape: torch.Size([1, 1026]) -Dataloader batch - attn_mask shape: torch.Size([1, 1026]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1529]) -Dataloader batch - labels shape: torch.Size([1, 1529]) -Dataloader batch - attn_mask shape: torch.Size([1, 1529]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1673]) -Dataloader batch - labels shape: torch.Size([1, 1673]) -Dataloader batch - attn_mask shape: torch.Size([1, 1673]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1175]) -Dataloader batch - labels shape: torch.Size([1, 1175]) -Dataloader batch - attn_mask shape: torch.Size([1, 1175]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1026) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1026) torch.int64 - loss_mask : 200 / 1026 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 256, 3072) - out : (20, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1026, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1026, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1026, 1, 576) torch.bfloat16 - out : (1, 1026) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 200 tokens) : 11.3715 - top-1 accuracy : 0/200 = 0.0% - - [2026-04-29 08:59:32] iteration 9/ 20 | consumed samples: 36 | elapsed time per iteration (ms): 1970.5 | throughput (token/sec/GPU): 685.5 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.143318E+01 | loss scale: 1.0 | grad norm: 0.920 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 9 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2160.61, 2160.88) -Dataloader batch - tokens shape: torch.Size([1, 1809]) -Dataloader batch - labels shape: torch.Size([1, 1809]) -Dataloader batch - attn_mask shape: torch.Size([1, 1809]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1396]) -Dataloader batch - labels shape: torch.Size([1, 1396]) -Dataloader batch - attn_mask shape: torch.Size([1, 1396]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1658]) -Dataloader batch - labels shape: torch.Size([1, 1658]) -Dataloader batch - attn_mask shape: torch.Size([1, 1658]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1804]) -Dataloader batch - labels shape: torch.Size([1, 1804]) -Dataloader batch - attn_mask shape: torch.Size([1, 1804]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1804) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1804) torch.int64 - loss_mask : 416 / 1804 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 256, 3072) - out : (32, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1804, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1804, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1804, 1, 576) torch.bfloat16 - out : (1, 1804) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.4491 - top-1 accuracy : 0/416 = 0.0% - - [2026-04-29 08:59:37] iteration 10/ 20 | consumed samples: 40 | elapsed time per iteration (ms): 2433.4 | throughput (token/sec/GPU): 684.9 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.144508E+01 | loss scale: 1.0 | grad norm: 0.838 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader -saving checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 10 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2352.68, 2353.32) -Dataloader batch - tokens shape: torch.Size([1, 1899]) -Dataloader batch - labels shape: torch.Size([1, 1899]) -Dataloader batch - attn_mask shape: torch.Size([1, 1899]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1134]) -Dataloader batch - labels shape: torch.Size([1, 1134]) -Dataloader batch - attn_mask shape: torch.Size([1, 1134]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1204]) -Dataloader batch - labels shape: torch.Size([1, 1204]) -Dataloader batch - attn_mask shape: torch.Size([1, 1204]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1134) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1134) torch.int64 - loss_mask : 319 / 1134 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 256, 3072) - out : (19, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1134, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1134, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1134, 1, 576) torch.bfloat16 - out : (1, 1134) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.5059 - top-1 accuracy : 3/319 = 0.9% - - [2026-04-29 08:59:41] iteration 11/ 20 | consumed samples: 44 | elapsed time per iteration (ms): 2028.4 | throughput (token/sec/GPU): 731.2 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.146556E+01 | loss scale: 1.0 | grad norm: 0.775 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 11 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1975.71, 1976.00) -Dataloader batch - tokens shape: torch.Size([1, 1115]) -Dataloader batch - labels shape: torch.Size([1, 1115]) -Dataloader batch - attn_mask shape: torch.Size([1, 1115]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1458]) -Dataloader batch - labels shape: torch.Size([1, 1458]) -Dataloader batch - attn_mask shape: torch.Size([1, 1458]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 295 / 1475 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 256, 3072) - out : (28, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.3431 - top-1 accuracy : 1/295 = 0.3% - - [2026-04-29 08:59:46] iteration 12/ 20 | consumed samples: 48 | elapsed time per iteration (ms): 2429.4 | throughput (token/sec/GPU): 562.7 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.136467E+01 | loss scale: 1.0 | grad norm: 0.852 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m in torch formatsaving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 12 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1950.65, 1951.35) -Dataloader batch - tokens shape: torch.Size([1, 1535]) -Dataloader batch - labels shape: torch.Size([1, 1535]) -Dataloader batch - attn_mask shape: torch.Size([1, 1535]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 963]) -Dataloader batch - labels shape: torch.Size([1, 963]) -Dataloader batch - attn_mask shape: torch.Size([1, 963]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1481]) -Dataloader batch - labels shape: torch.Size([1, 1481]) -Dataloader batch - attn_mask shape: torch.Size([1, 1481]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1481) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1481) torch.int64 - loss_mask : 350 / 1481 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 256, 3072) - out : (28, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1481, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1481, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1481, 1, 576) torch.bfloat16 - out : (1, 1481) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.3549 - top-1 accuracy : 6/350 = 1.7% - - [2026-04-29 08:59:50] iteration 13/ 20 | consumed samples: 52 | elapsed time per iteration (ms): 2254.6 | throughput (token/sec/GPU): 604.2 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.138722E+01 | loss scale: 1.0 | grad norm: 0.812 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 13 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1940.31, 1940.49) -Dataloader batch - tokens shape: torch.Size([1, 1848]) -Dataloader batch - labels shape: torch.Size([1, 1848]) -Dataloader batch - tokens shape:Dataloader batch - attn_mask shape: torch.Size([1, 1848]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -torch.Size([1, 1588]) -Dataloader batch - labels shape: torch.Size([1, 1588]) -Dataloader batch - attn_mask shape: torch.Size([1, 1588]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1534]) -Dataloader batch - labels shape: torch.Size([1, 1534]) -Dataloader batch - attn_mask shape: torch.Size([1, 1534]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1588) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1588) torch.int64 - loss_mask : 455 / 1588 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 256, 3072) - out : (27, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1588, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1588, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1588, 1, 576) torch.bfloat16 - out : (1, 1588) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 455 tokens) : 11.4433 - top-1 accuracy : 0/455 = 0.0% - - [2026-04-29 08:59:54] iteration 14/ 20 | consumed samples: 56 | elapsed time per iteration (ms): 2173.7 | throughput (token/sec/GPU): 735.3 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.144041E+01 | loss scale: 1.0 | grad norm: 0.641 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 14 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1797.54, 1797.87) -Dataloader batch - tokens shape: torch.Size([1, 1577]) -Dataloader batch - labels shape: torch.Size([1, 1577]) -Dataloader batch - attn_mask shape: torch.Size([1, 1577]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1007]) -Dataloader batch - labels shape: torch.Size([1, 1007]) -Dataloader batch - attn_mask shape: torch.Size([1, 1007]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1577) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1577) torch.int64 - loss_mask : 389 / 1577 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 256, 3072) - out : (28, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1577, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1577, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1577, 1, 576) torch.bfloat16 - out : (1, 1577) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.4508 - top-1 accuracy : 3/389 = 0.8% - - [2026-04-29 08:59:57] iteration 15/ 20 | consumed samples: 60 | elapsed time per iteration (ms): 1600.5 | throughput (token/sec/GPU): 841.4 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.138847E+01 | loss scale: 1.0 | grad norm: 0.718 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 15 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1845.50, 1845.65) -Dataloader batch - tokens shape: torch.Size([1, 1067]) -Dataloader batch - labels shape: torch.Size([1, 1067]) -Dataloader batch - attn_mask shape: torch.Size([1, 1067]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1575]) -Dataloader batch - labels shape: torch.Size([1, 1575]) -Dataloader batch - attn_mask shape: torch.Size([1, 1575]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1163]) -Dataloader batch - labels shape: torch.Size([1, 1163]) -Dataloader batch - attn_mask shape: torch.Size([1, 1163]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1441]) -Dataloader batch - labels shape: torch.Size([1, 1441]) -Dataloader batch - attn_mask shape: torch.Size([1, 1441]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1575) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1575) torch.int64 - loss_mask : 384 / 1575 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 256, 3072) - out : (28, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1575, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1575, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1575, 1, 576) torch.bfloat16 - out : (1, 1575) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.3409 - top-1 accuracy : 0/384 = 0.0% - - [2026-04-29 09:00:01] iteration 16/ 20 | consumed samples: 64 | elapsed time per iteration (ms): 2136.7 | throughput (token/sec/GPU): 613.8 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.142639E+01 | loss scale: 1.0 | grad norm: 0.756 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 16 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2297.64, 2297.76) -Dataloader batch - tokens shape: torch.Size([1, 1793]) -Dataloader batch - labels shape: torch.Size([1, 1793]) -Dataloader batch - attn_mask shape: torch.Size([1, 1793]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1395]) -Dataloader batch - labels shape: torch.Size([1, 1395]) -Dataloader batch - attn_mask shape: torch.Size([1, 1395]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1498]) -Dataloader batch - labels shape: torch.Size([1, 1498]) -Dataloader batch - attn_mask shape: torch.Size([1, 1498]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1581]) -Dataloader batch - labels shape: torch.Size([1, 1581]) -Dataloader batch - attn_mask shape: torch.Size([1, 1581]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1395) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1395) torch.int64 - loss_mask : 357 / 1395 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 256, 3072) - out : (25, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1395, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1395, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1395, 1, 576) torch.bfloat16 - out : (1, 1395) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3814 - top-1 accuracy : 3/357 = 0.8% - - [2026-04-29 09:00:06] iteration 17/ 20 | consumed samples: 68 | elapsed time per iteration (ms): 2179.7 | throughput (token/sec/GPU): 718.8 | learning rate: 6.416419E-06 | global batch size: 4 | lm loss: 1.139987E+01 | loss scale: 1.0 | grad norm: 0.644 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 17 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1828.11, 1828.17) -Dataloader batch - tokens shape: torch.Size([1, 1871]) -Dataloader batch - labels shape: torch.Size([1, 1871]) -Dataloader batch - attn_mask shape: torch.Size([1, 1871]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1825]) -Dataloader batch - labels shape: torch.Size([1, 1825]) -Dataloader batch - attn_mask shape: Dataloader batch - tokens shape:torch.Size([1, 1825])Dataloader batch - tokens shape: -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - torch.Size([1, 943]) -Dataloader batch - labels shape: torch.Size([1, 943]) -Dataloader batch - attn_mask shape: torch.Size([1, 943]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - torch.Size([1, 1451]) -Dataloader batch - labels shape: torch.Size([1, 1451]) -Dataloader batch - attn_mask shape: torch.Size([1, 1451]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1825) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1825) torch.int64 - loss_mask : 463 / 1825 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 256, 3072) - out : (32, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1825, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1825, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1825, 1, 576) torch.bfloat16 - out : (1, 1825) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 463 tokens) : 11.3953 - top-1 accuracy : 1/463 = 0.2% - - [2026-04-29 09:00:10] iteration 18/ 20 | consumed samples: 72 | elapsed time per iteration (ms): 2223.1 | throughput (token/sec/GPU): 684.9 | learning rate: 3.432342E-06 | global batch size: 4 | lm loss: 1.139225E+01 | loss scale: 1.0 | grad norm: 0.651 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader - - -saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 18 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1950.87, 1951.92) -Dataloader batch - tokens shape: torch.Size([1, 1478]) -Dataloader batch - labels shape: torch.Size([1, 1478]) -Dataloader batch - attn_mask shape: torch.Size([1, 1478]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1747]) -Dataloader batch - labels shape: torch.Size([1, 1747]) -Dataloader batch - attn_mask shape: torch.Size([1, 1747]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1394]) -Dataloader batch - labels shape: torch.Size([1, 1394]) -Dataloader batch - attn_mask shape: torch.Size([1, 1394]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - torch.Size([1, 1382]) -Dataloader batch - labels shape: torch.Size([1, 1382]) -Dataloader batch - attn_mask shape: torch.Size([1, 1382]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1747) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1747) torch.int64 - loss_mask : 390 / 1747 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 256, 3072) - out : (32, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1747, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1747, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1747, 1, 576) torch.bfloat16 - out : (1, 1747) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3102 - top-1 accuracy : 5/390 = 1.3% - - [2026-04-29 09:00:14] iteration 19/ 20 | consumed samples: 76 | elapsed time per iteration (ms): 2114.2 | throughput (token/sec/GPU): 709.6 | learning rate: 1.611867E-06 | global batch size: 4 | lm loss: 1.134015E+01 | loss scale: 1.0 | grad norm: 0.719 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader - - successfully saved checkpoint from iteration 19 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2256.53, 2256.75) -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1897]) -Dataloader batch - labels shape: torch.Size([1, 1897]) -Dataloader batch - attn_mask shape: torch.Size([1, 1897]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1457]) -Dataloader batch - labels shape: torch.Size([1, 1457]) -Dataloader batch - attn_mask shape: torch.Size([1, 1457]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1936]) -Dataloader batch - labels shape: torch.Size([1, 1936]) -Dataloader batch - attn_mask shape: torch.Size([1, 1936]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1936) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1936) torch.int64 - loss_mask : 569 / 1936 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 256, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 256, 3072) - out : (32, 256, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1936, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1936, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1936, 1, 576) torch.bfloat16 - out : (1, 1936) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 569 tokens) : 11.4828 - top-1 accuracy : 11/569 = 1.9% - - [2026-04-29 09:00:18] iteration 20/ 20 | consumed samples: 80 | elapsed time per iteration (ms): 1895.5 | throughput (token/sec/GPU): 898.0 | learning rate: 1.000000E-06 | global batch size: 4 | lm loss: 1.141360E+01 | loss scale: 1.0 | grad norm: 0.637 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (1857.53, 1857.59) - forward-compute ................................: (1405.63, 1406.53) - backward-compute ...............................: (322.51, 437.95) - batch-generator ................................: (275.09, 393.60) - layernorm-grads-all-reduce .....................: (0.02, 0.06) - embedding-grads-all-reduce .....................: (0.04, 0.09) - all-grads-sync .................................: (9.51, 10.53) - params-all-gather ..............................: (1.28, 1.36) - optimizer-copy-to-main-grad ....................: (0.10, 0.30) - optimizer-clip-main-grad .......................: (1.04, 1.19) - optimizer-count-zeros ..........................: (0.01, 0.04) - optimizer-inner-step ...........................: (0.64, 2.00) - optimizer-copy-main-to-model-params ............: (0.12, 0.45) - optimizer ......................................: (7.24, 7.32) -saving checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 20 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1846.10, 1846.25) -[after training is done] datetime: 2026-04-29 09:00:20 -wandb: uploading output.log; uploading config.yaml -wandb: uploading history steps 18-19, summary, console lines 1127-1145 -wandb: -wandb: -wandb: Run history: -wandb: Token throughput (per-sec-per-GPU) ▁▄▆▆▆▆▅▆▆▆▇▅▆▇█▆▆▆▆█ -wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: grad-norm ██▄▃▃▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁ -wandb: iteration-time █▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: learning-rate ███▇▇▇▆▆▅▅▄▃▃▂▂▂▁▁▁▁ -wandb: lm loss ██▆▅▄▄▃▃▂▃▃▁▂▂▂▂▂▂▁▂ -wandb: loss-scale ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: num-zeros ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: samples vs steps ▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██ -wandb: -wandb: Run summary: -wandb: Token throughput (per-sec-per-GPU) 898.04171 -wandb: batch-size 4 -wandb: grad-norm 0.6374 -wandb: iteration-time 1.89551 -wandb: learning-rate 0.0 -wandb: lm loss 11.4136 -wandb: loss-scale 1 -wandb: num-zeros 0 -wandb: samples vs steps 80 -wandb: -wandb: 🚀 View run mobilellm_integration at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/7dva7qn6 -wandb: ⭐️ View project at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 -wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) -wandb: Find logs at: stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_085758-7dva7qn6/logs -[rank0]:[W429 09:01:42.132834283 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W429 09:01:43.891072258 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W429 09:01:43.383694399 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W429 09:01:46.131858054 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log deleted file mode 100644 index f9ac6d7a..00000000 --- a/stage_1_alignment_mobilellm_140m/run_2026-04-29_09:11:51_tp1_pp1_seqlen32768_mbs1_gbs4_20steps.log +++ /dev/null @@ -1,8137 +0,0 @@ -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] -W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] ***************************************** -W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0429 09:11:53.099000 3555610 site-packages/torch/distributed/run.py:803] ***************************************** -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -FalseFalse - -False -False -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -parse arguments function called -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -parse arguments function called -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -Using unified model type for training. - -Starting training... -Using unified model type for training. - -Starting training... -> setting tensorboard ... -parse arguments function called --------------- Configure model to llava-ov-mobilellm-140m -------------- -[DEBUG] Model config type: LlavaOnevision1_5Config -[DEBUG] Is MobileLLM: True - num_layers = 15 - hidden_size = 576 - ffn_hidden_size = 2048 - num_attention_heads = 9 - group_query_attention = True - num_query_groups = 3 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = False - vocab_size_in_config_file = 128256 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-05 - rotary_base = 8000000 - kv_channels = None - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 4, data-parallel size: 4, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... False - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.local - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... llama3 - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 4 - data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 15 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 32768 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 1 - fastvit_image_size .............................. 1024 - ffn_hidden_size ................................. 2048 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 4 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M - hidden_dropout .................................. 0 - hidden_size ..................................... 576 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_aspect_ratio .............................. pad - image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 64 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ None - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 20 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-mobilellm-140m - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... None - no_load_rng ..................................... None - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 9 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 15 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 3 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 16 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 3 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 8000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_mobilellm_140m - save_ema ........................................ None - save_interval ................................... 1 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 32768 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 20 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 32768 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. False - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_fastvit ..................................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vision_tower_name ............................... mobileclip_l_1024 - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 128256 - vpp_scheduler ................................... None - wandb_exp_name .................................. mobilellm_integration - wandb_project ................................... llava-ov-1_5 - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 4 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -Using unified model type for training. - -Starting training... -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 -> AIAK building HFTokenizer tokenizer ... -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -parse arguments function called -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -Using unified model type for training. - -Starting training... -wandb: Currently logged in as: rana-zayed (rana-zayed-mbzuai) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin -INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 -WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. -WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. -WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). - > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -wandb: creating run -wandb: Tracking run with wandb version 0.21.0 -wandb: Run data is saved locally in stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_091210-1ll8kzkt -wandb: Run `wandb offline` to turn off syncing. -wandb: Syncing run mobilellm_integration -wandb: ⭐️ View project at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 -wandb: 🚀 View run at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/1ll8kzkt -[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 3 -3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 2 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 0 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 1 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -[Gloo] Rank 3 is connected to 3 peer ranks. Expected number of connected peer ranks is : 3 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so -make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.043 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W429 09:12:13.409480472 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.338 seconds -time to initialize megatron (seconds): 7.571 -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2026-04-29 09:12:16 -building llava-ov-mobilellm-140m model ... -[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m -[DEBUG PROVIDER] Args num_layers: 15 -[DEBUG PROVIDER] Args hidden_size: 576 -[DEBUG PROVIDER] Args vocab_size: 128260 -[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 -[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 -[DEBUG PROVIDER] Model family: llava_ov_1_5 -[DEBUG PROVIDER] use_mobilellm flag: True -[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone -[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 -[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Loading vision config... -[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 -[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json -[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd -[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ================================================ -[DEBUG PROVIDER] Loading adapter config... -[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 -[DEBUG PROVIDER] Building layer specs... -[DEBUG PROVIDER] ✓ Using MobileLLM layer specification -[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] MobileLLM layer spec created successfully -[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... -[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 -[INFO] Using MobileLLM as language backbone -[INFO] Using MobileLLM as language backbone -[INFO] Using MobileLLM as language backbone -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[INFO] Using MobileLLM as language backbone -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (11216448 elements, 11216512 padded size): - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.bias - module.adapter.layernorm.weight - module.adapter.linear_fc2.linear.weight - module.adapter.linear_fc1.linear.bias -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[DEBUG] FastViT enabled: use_fastvit=True -[DEBUG] Before handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -FastViT enabled: Will load vision tower from pretrained checkpoint: /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -[DEBUG] After handling: args.load=None, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -Checkpoint file not found in load directory None attempting to finetune with checkpoint in /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - loading release checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -could not find arguments in the checkpoint ... -Loading checkpoint with device: cuda:0, strict=False -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.layernorm.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc1.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.weight being removed from incompatible_keys.missing_keys in LlavaModel -WARNING:aiak_training_llm.models.llavaov_1_5.llavaov_1_5_model:adapter.linear_fc2.linear.bias being removed from incompatible_keys.missing_keys in LlavaModel - checkpoint version 3.0 - successfully loaded checkpoint from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 [ t 1/1, p 1/1 ] at iteration 0 - failed to find checkpoint /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1/release/mp_rank_00/model_optim_rng.pt in wandb -(min, max) time across ranks (ms): - load-checkpoint ................................: (3038.02, 3038.45) -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 09:12:21 -================================================================================ -FULL MODEL STRUCTURE: -================================================================================ - ---- Model 0 (pipeline rank 0) --- -DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -) -================================================================================ - -================================================================================ -PARAMETER TRAINABILITY STATUS: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -================================================================================ -Total trainable parameters: 11,216,448 (11.22M) -Total frozen parameters: 265,417,440 (265.42M) -Total parameters: 276,633,888 (276.63M) -Trainable percentage: 4.05% -================================================================================ -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 80 - validation: 400 - test: 400 -Using shared training tokenizer in FastVLM mode -Ensured multimodal special tokens for FastVLM tokenizer; added=0 -Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M -Initialized FastViT processor with image_size=1024 -Processor config: .TokenizerWrapper object at 0x7f1df8173eb0> -image_resolution: None -rank=3, worker=0: shard_range=[pretrain-000004.tar[2858, 3233), ] sum(count)=375rank=1, worker=0: shard_range=[pretrain-000001.tar[939, 1314), ] sum(count)=375 -rank=1, worker=1: shard_range=[pretrain-000001.tar[1314, 1689), ] sum(count)=375 -rank=1, worker=2: shard_range=[pretrain-000001.tar[1689, 2064), ] sum(count)=375 -rank=1, worker=3: shard_range=[pretrain-000001.tar[2064, 2439), ] sum(count)=375 -rank=1, worker=4: shard_range=[pretrain-000001.tar[2439, 2814), ] sum(count)=375 -rank=1, worker=5: shard_range=[pretrain-000001.tar[2814, 3189), ] sum(count)=375 -rank=1, worker=6: shard_range=[pretrain-000001.tar[3189, 3564), ] sum(count)=375 - -rank=1, worker=7: shard_range=[pretrain-000001.tar[3564, 3938), ] sum(count)=374 -rank=1, worker=8: shard_range=[pretrain-000001.tar[3938, 4313), ] sum(count)=375 -rank=3, worker=1: shard_range=[pretrain-000004.tar[3233, 3608), ] sum(count)=375rank=1, worker=9: shard_range=[pretrain-000001.tar[4313, 4688), ] sum(count)=375 -rank=1, worker=10: shard_range=[pretrain-000001.tar[4688, 5036), pretrain-000002.tar[0, 27), ] sum(count)=375 - -rank=1, worker=11: shard_range=[pretrain-000002.tar[27, 401), ] sum(count)=374 -rank=1, worker=12: shard_range=[pretrain-000002.tar[401, 776), ] sum(count)=375 -rank=3, worker=2: shard_range=[pretrain-000004.tar[3608, 3821), pretrain-000000.tar[0, 162), ] sum(count)=375rank=1, worker=13: shard_range=[pretrain-000002.tar[776, 1151), ] sum(count)=375 -rank=1, worker=14: shard_range=[pretrain-000002.tar[1151, 1526), ] sum(count)=375 - -rank=1, worker=15: shard_range=[pretrain-000002.tar[1526, 1900), ] sum(count)=374 -rank=3, worker=3: shard_range=[pretrain-000000.tar[162, 537), ] sum(count)=375 -rank=3, worker=4: shard_range=[pretrain-000000.tar[537, 912), ] sum(count)=375 -rank=3, worker=5: shard_range=[pretrain-000000.tar[912, 1287), ] sum(count)=375 -rank=3, worker=6: shard_range=[pretrain-000000.tar[1287, 1662), ] sum(count)=375 -rank=3, worker=7: shard_range=[pretrain-000000.tar[1662, 2036), ] sum(count)=374 -rank=3, worker=8: shard_range=[pretrain-000000.tar[2036, 2411), ] sum(count)=375 -rank=3, worker=9: shard_range=[pretrain-000000.tar[2411, 2786), ] sum(count)=375 -rank=3, worker=10: shard_range=[pretrain-000000.tar[2786, 3161), ] sum(count)=375 -rank=3, worker=11: shard_range=[pretrain-000000.tar[3161, 3535), ] sum(count)=374 -rank=3, worker=12: shard_range=[pretrain-000000.tar[3535, 3910), ] sum(count)=375 -rank=3, worker=13: shard_range=[pretrain-000000.tar[3910, 4285), ] sum(count)=375 -rank=3, worker=14: shard_range=[pretrain-000000.tar[4285, 4660), ] sum(count)=375 -rank=3, worker=15: shard_range=[pretrain-000000.tar[4660, 5034), ] sum(count)=374 -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 375), ] sum(count)=375 -rank=0, worker=1: shard_range=[pretrain-000003.tar[375, 750), ] sum(count)=375 -rank=0, worker=2: shard_range=[pretrain-000003.tar[750, 1125), ] sum(count)=375 -rank=0, worker=3: shard_range=[pretrain-000003.tar[1125, 1500), ] sum(count)=375 -rank=0, worker=4: shard_range=[pretrain-000003.tar[1500, 1875), ] sum(count)=375 -rank=0, worker=5: shard_range=[pretrain-000003.tar[1875, 2250), ] sum(count)=375 -rank=0, worker=6: shard_range=[pretrain-000003.tar[2250, 2625), ] sum(count)=375 -rank=0, worker=7: shard_range=[pretrain-000003.tar[2625, 2999), ] sum(count)=374 -rank=0, worker=8: shard_range=[pretrain-000003.tar[2999, 3374), ] sum(count)=375 -rank=0, worker=9: shard_range=[pretrain-000003.tar[3374, 3749), ] sum(count)=375 -rank=0, worker=10: shard_range=[pretrain-000003.tar[3749, 4124), ] sum(count)=375 -rank=0, worker=11: shard_range=[pretrain-000003.tar[4124, 4498), ] sum(count)=374 -rank=0, worker=12: shard_range=[pretrain-000003.tar[4498, 4873), ] sum(count)=375 -rank=0, worker=13: shard_range=[pretrain-000003.tar[4873, 5058), pretrain-000001.tar[0, 190), ] sum(count)=375 -rank=0, worker=14: shard_range=[pretrain-000001.tar[190, 565), ] sum(count)=375 -rank=0, worker=15: shard_range=[pretrain-000001.tar[565, 939), ] sum(count)=374 -rank=2, worker=0: shard_range=[pretrain-000002.tar[1900, 2275), ] sum(count)=375 -rank=2, worker=1: shard_range=[pretrain-000002.tar[2275, 2650), ] sum(count)=375 -rank=2, worker=2: shard_range=[pretrain-000002.tar[2650, 3025), ] sum(count)=375 -rank=2, worker=3: shard_range=[pretrain-000002.tar[3025, 3400), ] sum(count)=375 -rank=2, worker=4: shard_range=[pretrain-000002.tar[3400, 3775), ] sum(count)=375 -rank=2, worker=5: shard_range=[pretrain-000002.tar[3775, 4150), ] sum(count)=375 -rank=2, worker=6: shard_range=[pretrain-000002.tar[4150, 4525), ] sum(count)=375 -rank=2, worker=7: shard_range=[pretrain-000002.tar[4525, 4899), ] sum(count)=374 -rank=2, worker=8: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 235), ] sum(count)=375 -rank=2, worker=9: shard_range=[pretrain-000004.tar[235, 610), ] sum(count)=375 -rank=2, worker=10: shard_range=[pretrain-000004.tar[610, 985), ] sum(count)=375 -rank=2, worker=11: shard_range=[pretrain-000004.tar[985, 1359), ] sum(count)=374 -rank=2, worker=12: shard_range=[pretrain-000004.tar[1359, 1734), ] sum(count)=375 -rank=2, worker=13: shard_range=[pretrain-000004.tar[1734, 2109), ] sum(count)=375 -rank=2, worker=14: shard_range=[pretrain-000004.tar[2109, 2484), ] sum(count)=375 -rank=2, worker=15: shard_range=[pretrain-000004.tar[2484, 2858), ] sum(count)=374 -[after dataloaders are built] datetime: 2026-04-29 09:12:21 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (5402.02, 5415.21) - train/valid/test-data-iterators-setup ..........: (75.65, 81.61)training ... - -================================================================================ -training with the following parameter status: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 - -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -Setting rerun_state_machine.current_iteration to 0... -[before the start of training step] datetime: 2026-04-29 09:12:21 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 - -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1757]) -Dataloader batch - labels shape: torch.Size([1, 1757]) -Dataloader batch - attn_mask shape: torch.Size([1, 1757]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1356]) -Dataloader batch - labels shape: torch.Size([1, 1356]) -Dataloader batch - attn_mask shape: torch.Size([1, 1356]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1909]) -Dataloader batch - labels shape: torch.Size([1, 1909]) -Dataloader batch - attn_mask shape: torch.Size([1, 1909]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1909) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1909) torch.int64 - loss_mask : 532 / 1909 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1430]) -Dataloader batch - labels shape: torch.Size([1, 1430]) -Dataloader batch - attn_mask shape: torch.Size([1, 1430]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1909, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1909, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1909, 1, 576) torch.bfloat16 - out : (1, 1909) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 532 tokens) : 11.7829 - top-1 accuracy : 1/532 = 0.2% - -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -Number of parameters in transformer layers in billions: 0.07 -Number of parameters in embedding layers in billions: 0.07 -Total number of parameters in billions: 0.14 -Number of parameters in most loaded shard in billions: 0.1403 -Theoretical memory footprints: weight and optimizer=1204.55 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 709.1826171875 | max allocated: 5398.64892578125 | reserved: 6388.0 | max reserved: 6388.0 - [2026-04-29 09:12:42] iteration 1/ 20 | consumed samples: 4 | elapsed time per iteration (ms): 20330.5 | throughput (token/sec/GPU): 79.3 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.179576E+01 | loss scale: 1.0 | grad norm: 3.000 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 1 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 1 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (3113.10, 3115.03) -Dataloader batch - tokens shape: torch.Size([1, 1781]) -Dataloader batch - labels shape: torch.Size([1, 1781]) -Dataloader batch - attn_mask shape: torch.Size([1, 1781]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1186]) -Dataloader batch - labels shape: torch.Size([1, 1186]) -Dataloader batch - attn_mask shape: torch.Size([1, 1186]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1114]) -Dataloader batch - labels shape: torch.Size([1, 1114]) -Dataloader batch - attn_mask shape: torch.Size([1, 1114]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 942]) -Dataloader batch - labels shape: torch.Size([1, 942]) -Dataloader batch - attn_mask shape: torch.Size([1, 942]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1186) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1186) torch.int64 - loss_mask : 273 / 1186 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1186, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1186, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1186, 1, 576) torch.bfloat16 - out : (1, 1186) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 273 tokens) : 11.8201 - top-1 accuracy : 0/273 = 0.0% - - [2026-04-29 09:12:48] iteration 2/ 20 | consumed samples: 8 | elapsed time per iteration (ms): 2850.4 | throughput (token/sec/GPU): 440.6 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.181961E+01 | loss scale: 1.0 | grad norm: 3.164 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m in torch formatsaving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 2 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 2 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2986.92, 2988.67) -Dataloader batch - tokens shape: torch.Size([1, 1849]) -Dataloader batch - labels shape: torch.Size([1, 1849]) -Dataloader batch - attn_mask shape: torch.Size([1, 1849]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1741]) -Dataloader batch - labels shape: torch.Size([1, 1741]) -Dataloader batch - attn_mask shape: torch.Size([1, 1741]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1554]) -Dataloader batch - labels shape: torch.Size([1, 1554]) -Dataloader batch - attn_mask shape: torch.Size([1, 1554]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1399]) -Dataloader batch - labels shape: torch.Size([1, 1399]) -Dataloader batch - attn_mask shape: torch.Size([1, 1399]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1399) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1399) torch.int64 - loss_mask : 342 / 1399 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1399, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1399, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1399, 1, 576) torch.bfloat16 - out : (1, 1399) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.7174 - top-1 accuracy : 0/342 = 0.0% - - [2026-04-29 09:12:53] iteration 3/ 20 | consumed samples: 12 | elapsed time per iteration (ms): 2824.4 | throughput (token/sec/GPU): 579.1 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.168303E+01 | loss scale: 1.0 | grad norm: 1.876 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 3 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 3 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (3043.68, 3044.00) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1444]) -torch.Size([1, 1585])Dataloader batch - labels shape: -torch.Size([1, 1444])Dataloader batch - labels shape: - Dataloader batch - attn_mask shape:torch.Size([1, 1585]) -torch.Size([1, 1444])Dataloader batch - attn_mask shape: - Dataloader batch - image_grid_thw shape:torch.Size([1, 1585]) -torch.Size([24, 3])Dataloader batch - image_grid_thw shape: - torch.Size([29, 3]) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1544]) -Dataloader batch - labels shape: torch.Size([1, 1544]) -Dataloader batch - attn_mask shape: torch.Size([1, 1544]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3])torch.Size([1, 1042]) - -Dataloader batch - labels shape: torch.Size([1, 1042]) -Dataloader batch - attn_mask shape: torch.Size([1, 1042]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1585) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1585) torch.int64 - loss_mask : 350 / 1585 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1585, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1585, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1585, 1, 576) torch.bfloat16 - out : (1, 1585) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.5907 - top-1 accuracy : 0/350 = 0.0% - - [2026-04-29 09:12:58] iteration 4/ 20 | consumed samples: 16 | elapsed time per iteration (ms): 1773.9 | throughput (token/sec/GPU): 791.3 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.163304E+01 | loss scale: 1.0 | grad norm: 1.483 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 4 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 4 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2776.61, 2777.07) -Dataloader batch - tokens shape: torch.Size([1, 1214]) -Dataloader batch - labels shape: torch.Size([1, 1214]) -Dataloader batch - attn_mask shape: torch.Size([1, 1214]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 367 / 1541 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.5424 - top-1 accuracy : 0/367 = 0.0% - - [2026-04-29 09:13:03] iteration 5/ 20 | consumed samples: 20 | elapsed time per iteration (ms): 2378.8 | throughput (token/sec/GPU): 563.1 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.153108E+01 | loss scale: 1.0 | grad norm: 1.398 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 5 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 5 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2793.07, 2793.32) -Dataloader batch - tokens shape: torch.Size([1, 1498]) -Dataloader batch - labels shape: torch.Size([1, 1498]) -Dataloader batch - attn_mask shape: torch.Size([1, 1498]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1344]) -Dataloader batch - labels shape: torch.Size([1, 1344]) -Dataloader batch - attn_mask shape: torch.Size([1, 1344]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1689]) -Dataloader batch - labels shape: torch.Size([1, 1689]) -Dataloader batch - attn_mask shape: torch.Size([1, 1689]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1344) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1344) torch.int64 - loss_mask : 399 / 1344 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1344, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1344, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1344, 1, 576) torch.bfloat16 - out : (1, 1344) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.5134 - top-1 accuracy : 1/399 = 0.3% - - [2026-04-29 09:13:09] iteration 6/ 20 | consumed samples: 24 | elapsed time per iteration (ms): 2612.5 | throughput (token/sec/GPU): 574.0 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.151871E+01 | loss scale: 1.0 | grad norm: 1.023 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 6 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 6 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2767.65, 2767.84) -Dataloader batch - tokens shape: torch.Size([1, 1824]) -Dataloader batch - labels shape: torch.Size([1, 1824]) -Dataloader batch - attn_mask shape: torch.Size([1, 1824]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1352]) -Dataloader batch - labels shape: torch.Size([1, 1352]) -Dataloader batch - attn_mask shape: torch.Size([1, 1352]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1352) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1352) torch.int64 - loss_mask : 318 / 1352 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1352, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1352, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1352, 1, 576) torch.bfloat16 - out : (1, 1352) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.5097 - top-1 accuracy : 0/318 = 0.0% - - [2026-04-29 09:13:13] iteration 7/ 20 | consumed samples: 28 | elapsed time per iteration (ms): 2011.4 | throughput (token/sec/GPU): 775.6 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.149438E+01 | loss scale: 1.0 | grad norm: 0.972 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 7 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 7 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2697.84, 2698.53) -Dataloader batch - tokens shape: torch.Size([1, 1612]) -Dataloader batch - labels shape: torch.Size([1, 1612]) -Dataloader batch - attn_mask shape: torch.Size([1, 1612]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) -Dataloader batch - tokens shape: torch.Size([1, 995]) -Dataloader batch - labels shape: torch.Size([1, 995]) -Dataloader batch - attn_mask shape: torch.Size([1, 995]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1558]) -Dataloader batch - labels shape: torch.Size([1, 1558]) -Dataloader batch - attn_mask shape: torch.Size([1, 1558]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 1527]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 995) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 995) torch.int64 - loss_mask : 227 / 995 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (995, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (995, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (995, 1, 576) torch.bfloat16 - out : (1, 995) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 227 tokens) : 11.4815 - top-1 accuracy : 0/227 = 0.0% - - [2026-04-29 09:13:19] iteration 8/ 20 | consumed samples: 32 | elapsed time per iteration (ms): 2354.6 | throughput (token/sec/GPU): 604.4 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.145043E+01 | loss scale: 1.0 | grad norm: 0.868 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 8 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 8 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2993.46, 2993.78) -Dataloader batch - tokens shape: torch.Size([1, 1529]) -Dataloader batch - labels shape: torch.Size([1, 1529]) -Dataloader batch - attn_mask shape: torch.Size([1, 1529]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1175]) -Dataloader batch - tokens shape: Dataloader batch - labels shape:torch.Size([1, 1026]) -Dataloader batch - labels shape: torch.Size([1, 1026]) -Dataloader batch - attn_mask shape: torch.Size([1, 1026])torch.Size([1, 1175]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -Dataloader batch - attn_mask shape: torch.Size([1, 1175]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1673]) -Dataloader batch - labels shape: torch.Size([1, 1673]) -Dataloader batch - attn_mask shape: torch.Size([1, 1673]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1026) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1026) torch.int64 - loss_mask : 200 / 1026 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1026, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1026, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1026, 1, 576) torch.bfloat16 - out : (1, 1026) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 200 tokens) : 11.3712 - top-1 accuracy : 0/200 = 0.0% - - [2026-04-29 09:13:24] iteration 9/ 20 | consumed samples: 36 | elapsed time per iteration (ms): 2256.3 | throughput (token/sec/GPU): 598.7 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.143266E+01 | loss scale: 1.0 | grad norm: 0.944 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloadersaving checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m in torch format - - -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 9 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 9 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2398.93, 2399.30) -Dataloader batch - tokens shape: torch.Size([1, 1396]) -Dataloader batch - labels shape: torch.Size([1, 1396]) -Dataloader batch - attn_mask shape: torch.Size([1, 1396]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1804]) -Dataloader batch - labels shape: torch.Size([1, 1804]) -Dataloader batch - attn_mask shape: torch.Size([1, 1804]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1809]) -Dataloader batch - labels shape: torch.Size([1, 1809]) -Dataloader batch - attn_mask shape: torch.Size([1, 1809]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1658]) -Dataloader batch - labels shape: torch.Size([1, 1658]) -Dataloader batch - attn_mask shape: torch.Size([1, 1658]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1804) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1804) torch.int64 - loss_mask : 416 / 1804 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1804, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1804, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1804, 1, 576) torch.bfloat16 - out : (1, 1804) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.4488 - top-1 accuracy : 0/416 = 0.0% - - [2026-04-29 09:13:28] iteration 10/ 20 | consumed samples: 40 | elapsed time per iteration (ms): 2172.5 | throughput (token/sec/GPU): 767.2 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.144409E+01 | loss scale: 1.0 | grad norm: 0.851 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 10 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 10 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2500.80, 2501.73) -Dataloader batch - tokens shape: torch.Size([1, 1899]) -Dataloader batch - labels shape: torch.Size([1, 1899]) -Dataloader batch - attn_mask shape: torch.Size([1, 1899]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1134]) -Dataloader batch - labels shape: torch.Size([1, 1134]) -Dataloader batch - attn_mask shape: torch.Size([1, 1134]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1204]) -Dataloader batch - labels shape: torch.Size([1, 1204]) -Dataloader batch - attn_mask shape: torch.Size([1, 1204]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1134) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1134) torch.int64 - loss_mask : 319 / 1134 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1134, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1134, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1134, 1, 576) torch.bfloat16 - out : (1, 1134) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.5044 - top-1 accuracy : 3/319 = 0.9% - - [2026-04-29 09:13:33] iteration 11/ 20 | consumed samples: 44 | elapsed time per iteration (ms): 2299.5 | throughput (token/sec/GPU): 645.0 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.146383E+01 | loss scale: 1.0 | grad norm: 0.789 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 11 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 11 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2844.03, 2844.77) -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1115]) -Dataloader batch - labels shape: torch.Size([1, 1115]) -Dataloader batch - attn_mask shape: torch.Size([1, 1115]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1458]) -Dataloader batch - labels shape: torch.Size([1, 1458]) -Dataloader batch - attn_mask shape: torch.Size([1, 1458]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 295 / 1475 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.3404 - top-1 accuracy : 1/295 = 0.3% - - [2026-04-29 09:13:38] iteration 12/ 20 | consumed samples: 48 | elapsed time per iteration (ms): 2071.3 | throughput (token/sec/GPU): 660.0 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.136227E+01 | loss scale: 1.0 | grad norm: 0.882 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 12 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 12 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2613.00, 2613.48) -Dataloader batch - tokens shape: torch.Size([1, 1535]) -Dataloader batch - labels shape: torch.Size([1, 1535]) -Dataloader batch - attn_mask shape: torch.Size([1, 1535]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1481]) -Dataloader batch - labels shape: torch.Size([1, 1481]) -Dataloader batch - attn_mask shape: Dataloader batch - tokens shape:torch.Size([1, 1481]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 963]) -Dataloader batch - labels shape: torch.Size([1, 963]) -Dataloader batch - attn_mask shape: torch.Size([1, 963]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1481) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1481) torch.int64 - loss_mask : 350 / 1481 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1481, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1481, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1481, 1, 576) torch.bfloat16 - out : (1, 1481) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.3526 - top-1 accuracy : 6/350 = 1.7% - - [2026-04-29 09:13:43] iteration 13/ 20 | consumed samples: 52 | elapsed time per iteration (ms): 1814.6 | throughput (token/sec/GPU): 750.7 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.138488E+01 | loss scale: 1.0 | grad norm: 0.831 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloadersaving checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 13 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 13 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2329.09, 2329.61) -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1848]) -Dataloader batch - labels shape: torch.Size([1, 1848]) -Dataloader batch - attn_mask shape: torch.Size([1, 1848]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1588]) -Dataloader batch - labels shape: torch.Size([1, 1588]) -Dataloader batch - attn_mask shape: torch.Size([1, 1588]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1534]) -Dataloader batch - labels shape: torch.Size([1, 1534]) -Dataloader batch - attn_mask shape: torch.Size([1, 1534]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1588) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1588) torch.int64 - loss_mask : 455 / 1588 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1588, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1588, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1588, 1, 576) torch.bfloat16 - out : (1, 1588) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 455 tokens) : 11.4415 - top-1 accuracy : 0/455 = 0.0% - - [2026-04-29 09:13:47] iteration 14/ 20 | consumed samples: 56 | elapsed time per iteration (ms): 1856.3 | throughput (token/sec/GPU): 861.0 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.143828E+01 | loss scale: 1.0 | grad norm: 0.664 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m in torch formatsaving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 14 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 14 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2617.99, 2619.03) -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1577]) -Dataloader batch - labels shape: torch.Size([1, 1577]) -Dataloader batch - attn_mask shape: torch.Size([1, 1577]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1007]) -Dataloader batch - labels shape: torch.Size([1, 1007]) -Dataloader batch - attn_mask shape: torch.Size([1, 1007]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1577) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1577) torch.int64 - loss_mask : 389 / 1577 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1577, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1577, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1577, 1, 576) torch.bfloat16 - out : (1, 1577) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.4468 - top-1 accuracy : 6/389 = 1.5% - - [2026-04-29 09:13:51] iteration 15/ 20 | consumed samples: 60 | elapsed time per iteration (ms): 1877.1 | throughput (token/sec/GPU): 717.5 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.138525E+01 | loss scale: 1.0 | grad norm: 0.740 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 15 to stage_1_alignment_mobilellm_140m/dataloader - - successfully saved checkpoint from iteration 15 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2424.06, 2425.03) -Dataloader batch - tokens shape: torch.Size([1, 1067]) -Dataloader batch - labels shape: torch.Size([1, 1067]) -Dataloader batch - attn_mask shape: torch.Size([1, 1067]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1441]) -Dataloader batch - labels shape: torch.Size([1, 1441]) -Dataloader batch - attn_mask shape: torch.Size([1, 1441]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1575]) -Dataloader batch - labels shape: torch.Size([1, 1575]) -Dataloader batch - attn_mask shape: torch.Size([1, 1575]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1163]) -Dataloader batch - labels shape: torch.Size([1, 1163]) -Dataloader batch - attn_mask shape: torch.Size([1, 1163]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1575) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1575) torch.int64 - loss_mask : 384 / 1575 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1575, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1575, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1575, 1, 576) torch.bfloat16 - out : (1, 1575) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.3348 - top-1 accuracy : 0/384 = 0.0% - - [2026-04-29 09:13:56] iteration 16/ 20 | consumed samples: 64 | elapsed time per iteration (ms): 1966.1 | throughput (token/sec/GPU): 667.0 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.142319E+01 | loss scale: 1.0 | grad norm: 0.773 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 16 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 16 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2601.56, 2601.82) -Dataloader batch - tokens shape: torch.Size([1, 1498]) -Dataloader batch - labels shape: torch.Size([1, 1498]) -Dataloader batch - attn_mask shape: torch.Size([1, 1498]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: Dataloader batch - tokens shape: torch.Size([1, 1793]) -Dataloader batch - labels shape: torch.Size([1, 1793]) -Dataloader batch - attn_mask shape: torch.Size([1, 1793]) -Dataloader batch - image_grid_thw shape:torch.Size([1, 1581]) torch.Size([32, 3]) - -Dataloader batch - labels shape: torch.Size([1, 1581]) -Dataloader batch - attn_mask shape: torch.Size([1, 1581]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1395]) -Dataloader batch - labels shape: torch.Size([1, 1395]) -Dataloader batch - attn_mask shape: torch.Size([1, 1395]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1395) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1395) torch.int64 - loss_mask : 357 / 1395 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1395, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1395, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1395, 1, 576) torch.bfloat16 - out : (1, 1395) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3790 - top-1 accuracy : 3/357 = 0.8% - - [2026-04-29 09:14:00] iteration 17/ 20 | consumed samples: 68 | elapsed time per iteration (ms): 1828.7 | throughput (token/sec/GPU): 856.7 | learning rate: 6.416419E-06 | global batch size: 4 | lm loss: 1.139731E+01 | loss scale: 1.0 | grad norm: 0.660 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 17 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 17 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2378.09, 2378.67) -Dataloader batch - tokens shape: torch.Size([1, 943]) -Dataloader batch - labels shape: torch.Size([1, 943]) -Dataloader batch - attn_mask shape: torch.Size([1, 943]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1871]) -Dataloader batch - labels shape: torch.Size([1, 1871]) -Dataloader batch - attn_mask shape: torch.Size([1, 1871]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1825]) -Dataloader batch - labels shape: torch.Size([1, 1825]) -Dataloader batch - attn_mask shape: torch.Size([1, 1825]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1451]) -Dataloader batch - labels shape: torch.Size([1, 1451]) -Dataloader batch - attn_mask shape: torch.Size([1, 1451]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1825) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1825) torch.int64 - loss_mask : 463 / 1825 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1825, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1825, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1825, 1, 576) torch.bfloat16 - out : (1, 1825) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 463 tokens) : 11.3931 - top-1 accuracy : 6/463 = 1.3% - - [2026-04-29 09:14:05] iteration 18/ 20 | consumed samples: 72 | elapsed time per iteration (ms): 2162.8 | throughput (token/sec/GPU): 703.9 | learning rate: 3.432342E-06 | global batch size: 4 | lm loss: 1.139007E+01 | loss scale: 1.0 | grad norm: 0.671 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloadersaving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 18 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 18 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2485.46, 2485.70) -Dataloader batch - tokens shape: torch.Size([1, 1394]) -Dataloader batch - labels shape: torch.Size([1, 1394]) -Dataloader batch - attn_mask shape: torch.Size([1, 1394]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1747]) -Dataloader batch - labels shape: torch.Size([1, 1747]) -Dataloader batch - attn_mask shape: torch.Size([1, 1747]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1382]) -Dataloader batch - labels shape: torch.Size([1, 1382]) -Dataloader batch - attn_mask shape: Dataloader batch - tokens shape:torch.Size([1, 1382]) torch.Size([1, 1478]) - -Dataloader batch - labels shape: torch.Size([1, 1478]) -Dataloader batch - attn_mask shape: torch.Size([1, 1478]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1747) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1747) torch.int64 - loss_mask : 390 / 1747 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1747, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1747, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1747, 1, 576) torch.bfloat16 - out : (1, 1747) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3062 - top-1 accuracy : 5/390 = 1.3% - - [2026-04-29 09:14:09] iteration 19/ 20 | consumed samples: 76 | elapsed time per iteration (ms): 2270.7 | throughput (token/sec/GPU): 660.7 | learning rate: 1.611867E-06 | global batch size: 4 | lm loss: 1.133604E+01 | loss scale: 1.0 | grad norm: 0.744 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloadersaving checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader - -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 19 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 19 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2558.12, 2561.94) -Dataloader batch - tokens shape:Dataloader batch - tokens shape: torch.Size([1, 1897]) -Dataloader batch - labels shape: torch.Size([1, 1897]) -torch.Size([1, 1519])Dataloader batch - attn_mask shape: - Dataloader batch - labels shape:torch.Size([1, 1897]) - Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) -torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1457]) -Dataloader batch - labels shape: torch.Size([1, 1457]) -Dataloader batch - attn_mask shape: torch.Size([1, 1457]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) -Dataloader batch - tokens shape: torch.Size([1, 1936]) -Dataloader batch - labels shape: torch.Size([1, 1936]) -Dataloader batch - attn_mask shape: torch.Size([1, 1936]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1936) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1936) torch.int64 - loss_mask : 569 / 1936 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1936, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1936, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1936, 1, 576) torch.bfloat16 - out : (1, 1936) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 569 tokens) : 11.4807 - top-1 accuracy : 11/569 = 1.9% - - [2026-04-29 09:14:14] iteration 20/ 20 | consumed samples: 80 | elapsed time per iteration (ms): 2226.9 | throughput (token/sec/GPU): 764.4 | learning rate: 1.000000E-06 | global batch size: 4 | lm loss: 1.141078E+01 | loss scale: 1.0 | grad norm: 0.661 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (2193.58, 2194.32) - forward-compute ................................: (1623.25, 1625.61) - backward-compute ...............................: (388.65, 558.12) - batch-generator ................................: (299.87, 523.16) - layernorm-grads-all-reduce .....................: (0.02, 0.18) - embedding-grads-all-reduce .....................: (0.05, 0.15) - all-grads-sync .................................: (5.97, 6.40) - params-all-gather ..............................: (1.33, 1.39) - optimizer-copy-to-main-grad ....................: (0.10, 0.56) - optimizer-clip-main-grad .......................: (1.46, 2.83) - optimizer-count-zeros ..........................: (0.01, 0.29) - optimizer-inner-step ...........................: (0.70, 2.98) - optimizer-copy-main-to-model-params ............: (0.11, 0.67) - optimizer ......................................: (16.01, 16.23) -saving checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader -saving dataloader checkpoint at iteration 20 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 20 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2488.37, 2488.65) -[after training is done] datetime: 2026-04-29 09:14:17 -wandb: uploading artifact stage_1_alignment_mobilellm_140m; updating run metadata; uploading output.log -wandb: uploading artifact stage_1_alignment_mobilellm_140m -wandb: -wandb: -wandb: Run history: -wandb: Token throughput (per-sec-per-GPU) ▁▄▅▇▅▅▇▆▆▇▆▆▇█▇▆█▇▆▇ -wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: grad-norm ██▄▃▃▂▂▂▂▂▁▂▁▁▁▁▁▁▁▁ -wandb: iteration-time █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: learning-rate ███▇▇▇▆▆▅▅▄▃▃▂▂▂▁▁▁▁ -wandb: lm loss ██▆▅▄▄▃▃▂▃▃▁▂▂▂▂▂▂▁▂ -wandb: loss-scale ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: num-zeros ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: samples vs steps ▁▁▂▂▂▃▃▄▄▄▅▅▅▆▆▇▇▇██ -wandb: -wandb: Run summary: -wandb: Token throughput (per-sec-per-GPU) 764.39014 -wandb: batch-size 4 -wandb: grad-norm 0.66135 -wandb: iteration-time 2.22694 -wandb: learning-rate 0.0 -wandb: lm loss 11.41078 -wandb: loss-scale 1 -wandb: num-zeros 0 -wandb: samples vs steps 80 -wandb: -wandb: 🚀 View run mobilellm_integration at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/1ll8kzkt -wandb: ⭐️ View project at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 -wandb: Synced 5 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s) -wandb: Find logs at: stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_091210-1ll8kzkt/logs -[rank0]:[W429 09:15:39.766225268 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W429 09:15:40.686778180 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W429 09:15:40.170121945 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W429 09:15:43.622688209 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log deleted file mode 100644 index f848dedc..00000000 --- a/stage_1_alignment_mobilellm_140m/run_2026-04-29_12:22:06_tp1_pp1_seqlen32768_mbs1_gbs4_100steps.log +++ /dev/null @@ -1,16036 +0,0 @@ -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -False -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -parse arguments function called --------------- Configure model to llava-ov-mobilellm-140m -------------- -[DEBUG] Model config type: LlavaOnevision1_5Config -[DEBUG] Is MobileLLM: True - num_layers = 15 - hidden_size = 576 - ffn_hidden_size = 2048 - num_attention_heads = 9 - group_query_attention = True - num_query_groups = 3 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = False - vocab_size_in_config_file = 128256 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-05 - rotary_base = 8000000 - kv_channels = None - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... False - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.local - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... llama3 - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 15 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 32768 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 1 - fastvit_image_size .............................. 1024 - ffn_hidden_size ................................. 2048 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 4 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M - hidden_dropout .................................. 0 - hidden_size ..................................... 576 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_aspect_ratio .............................. pad - image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 64 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ stage_1_alignment_mobilellm_140m - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 100 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-mobilellm-140m - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... True - no_load_rng ..................................... True - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 9 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 15 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 3 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 16 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 3 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 8000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_mobilellm_140m - save_ema ........................................ None - save_interval ................................... 1 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 32768 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 100 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 32768 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. False - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_fastvit ..................................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vision_tower_name ............................... mobileclip_l_1024 - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 128256 - vpp_scheduler ................................... None - wandb_exp_name .................................. - wandb_project ................................... - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 1 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -Using unified model type for training. - -Starting training... -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 4 -> setting tensorboard ... -> AIAK building HFTokenizer tokenizer ... -INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 -WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. -WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. -WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). - > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so -make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.062 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W429 12:22:17.464874121 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.221 seconds -time to initialize megatron (seconds): 3.815 -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2026-04-29 12:22:19 -building llava-ov-mobilellm-140m model ... -[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m -[DEBUG PROVIDER] Args num_layers: 15 -[DEBUG PROVIDER] Args hidden_size: 576 -[DEBUG PROVIDER] Args vocab_size: 128260 -[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 -[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 -[DEBUG PROVIDER] Model family: llava_ov_1_5 -[DEBUG PROVIDER] use_mobilellm flag: True -[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone -[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 -[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Loading vision config... -[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 -[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json -[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd -[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ================================================ -[DEBUG PROVIDER] Loading adapter config... -[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 -[DEBUG PROVIDER] Building layer specs... -[DEBUG PROVIDER] ✓ Using MobileLLM layer specification -[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] MobileLLM layer spec created successfully -[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... -[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 -[INFO] Using MobileLLM as language backbone -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (11216448 elements, 11216512 padded size): - module.adapter.layernorm.bias - module.adapter.linear_fc1.linear.weight - module.adapter.layernorm.weight - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.bias - module.adapter.linear_fc2.linear.weight -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[DEBUG] FastViT enabled: use_fastvit=True -[DEBUG] Before handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -FastViT enabled: Will resume/load checkpoint from: stage_1_alignment_mobilellm_140m -[DEBUG] After handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - loading checkpoint from stage_1_alignment_mobilellm_140m at iteration 20 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 -WARNING:megatron.core.rerun_state_machine:RerunStateMachine disabled via CLI, ignoring machine state saved in checkpoint - successfully loaded checkpoint from stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] at iteration 20 -(min, max) time across ranks (ms): - load-checkpoint ................................: (514.72, 514.72) -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 12:22:22 -================================================================================ -FULL MODEL STRUCTURE: -================================================================================ - ---- Model 0 (pipeline rank 0) --- -DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -) -================================================================================ - -================================================================================ -PARAMETER TRAINABILITY STATUS: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -================================================================================ -Total trainable parameters: 11,216,448 (11.22M) -Total frozen parameters: 265,417,440 (265.42M) -Total parameters: 276,633,888 (276.63M) -Trainable percentage: 4.05% -================================================================================ -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 400 - validation: 400 - test: 400 -Using shared training tokenizer in FastVLM mode -Ensured multimodal special tokens for FastVLM tokenizer; added=0 -Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M -Initialized FastViT processor with image_size=1024 -Processor config: .TokenizerWrapper object at 0x7f57ec6afe20> -image_resolution: None -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 1500), ] sum(count)=1500 -rank=0, worker=1: shard_range=[pretrain-000003.tar[1500, 2999), ] sum(count)=1499 -rank=0, worker=2: shard_range=[pretrain-000003.tar[2999, 4498), ] sum(count)=1499 -rank=0, worker=3: shard_range=[pretrain-000003.tar[4498, 5058), pretrain-000001.tar[0, 939), ] sum(count)=1499 -rank=0, worker=4: shard_range=[pretrain-000001.tar[939, 2439), ] sum(count)=1500 -rank=0, worker=5: shard_range=[pretrain-000001.tar[2439, 3938), ] sum(count)=1499 -rank=0, worker=6: shard_range=[pretrain-000001.tar[3938, 5036), pretrain-000002.tar[0, 401), ] sum(count)=1499 -rank=0, worker=7: shard_range=[pretrain-000002.tar[401, 1900), ] sum(count)=1499 -rank=0, worker=8: shard_range=[pretrain-000002.tar[1900, 3400), ] sum(count)=1500 -rank=0, worker=9: shard_range=[pretrain-000002.tar[3400, 4899), ] sum(count)=1499 -rank=0, worker=10: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 1359), ] sum(count)=1499 -rank=0, worker=11: shard_range=[pretrain-000004.tar[1359, 2858), ] sum(count)=1499 -rank=0, worker=12: shard_range=[pretrain-000004.tar[2858, 3821), pretrain-000000.tar[0, 537), ] sum(count)=1500 -rank=0, worker=13: shard_range=[pretrain-000000.tar[537, 2036), ] sum(count)=1499 -rank=0, worker=14: shard_range=[pretrain-000000.tar[2036, 3535), ] sum(count)=1499 -rank=0, worker=15: shard_range=[pretrain-000000.tar[3535, 5034), ] sum(count)=1499 -loading dataset state failed. Skipping. Shard ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=0, count=375, byte_offset=0, byte_size=240076800) not found in [[[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=0, count=1500, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=1500, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=2999, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000003.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000003.tar'), offset=4498, count=560, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=0, count=939, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=939, count=1500, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=2439, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000001.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000001.tar'), offset=3938, count=1098, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=0, count=401, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=401, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=1900, count=1500, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=3400, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000002.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000002.tar'), offset=4899, count=140, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000004.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000004.tar'), offset=0, count=1359, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000004.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000004.tar'), offset=1359, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000004.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000004.tar'), offset=2858, count=963, byte_offset=None, byte_size=None)], [ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=0, count=537, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=537, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=2036, count=1499, byte_offset=None, byte_size=None)]], [[ShardInfo(name='pretrain-000000.tar', path=EPath('/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset/pretrain-000000.tar'), offset=3535, count=1499, byte_offset=None, byte_size=None)]]], states differ, not recoverable -[after dataloaders are built] datetime: 2026-04-29 12:22:22 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3066.93, 3066.93) - train/valid/test-data-iterators-setup ..........: (79.38, 79.38) -training ... - -================================================================================ -training with the following parameter status: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -Setting rerun_state_machine.current_iteration to 20... -[before the start of training step] datetime: 2026-04-29 12:22:23 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 1662]) -Dataloader batch - labels shape: torch.Size([1, 1662]) -Dataloader batch - attn_mask shape: torch.Size([1, 1662]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1662) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1662) torch.int64 - loss_mask : 412 / 1662 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1662, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1662, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1662, 1, 576) torch.bfloat16 - out : (1, 1662) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3501 - top-1 accuracy : 10/412 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1412) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1412) torch.int64 - loss_mask : 358 / 1412 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1412, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1412, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1412, 1, 576) torch.bfloat16 - out : (1, 1412) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.3680 - top-1 accuracy : 3/358 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1175]) -Dataloader batch - labels shape: torch.Size([1, 1175]) -Dataloader batch - attn_mask shape: torch.Size([1, 1175]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1175) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1175) torch.int64 - loss_mask : 278 / 1175 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1175, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1175, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1175, 1, 576) torch.bfloat16 - out : (1, 1175) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.3444 - top-1 accuracy : 11/278 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1470) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1470) torch.int64 - loss_mask : 421 / 1470 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1470, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1470, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1470, 1, 576) torch.bfloat16 - out : (1, 1470) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 421 tokens) : 11.4015 - top-1 accuracy : 0/421 = 0.0% - -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once - [2026-04-29 12:22:43] iteration 21/ 100 | consumed samples: 84 | elapsed time per iteration (ms): 20059.9 | throughput (token/sec/GPU): 285.1 | learning rate: 9.998430E-05 | global batch size: 4 | lm loss: 1.136812E+01 | loss scale: 1.0 | grad norm: 0.652 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer layers in billions: 0.07 -Number of parameters in embedding layers in billions: 0.07 -Total number of parameters in billions: 0.14 -Number of parameters in most loaded shard in billions: 0.1403 -Theoretical memory footprints: weight and optimizer=2409.10 MB -[Rank 0] (after 21 iterations) memory (MB) | allocated: 789.76123046875 | max allocated: 5130.18212890625 | reserved: 6046.0 | max reserved: 6046.0 -saving checkpoint at iteration 21 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 21 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 21 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1668.41, 1668.41) -Dataloader batch - tokens shape: torch.Size([1, 1909]) -Dataloader batch - labels shape: torch.Size([1, 1909]) -Dataloader batch - attn_mask shape: torch.Size([1, 1909]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1909) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1909) torch.int64 - loss_mask : 532 / 1909 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1909, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1909, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1909, 1, 576) torch.bfloat16 - out : (1, 1909) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 532 tokens) : 11.4621 - top-1 accuracy : 0/532 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 367 / 1541 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.3489 - top-1 accuracy : 15/367 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1026]) -Dataloader batch - labels shape: torch.Size([1, 1026]) -Dataloader batch - attn_mask shape: torch.Size([1, 1026]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1026) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1026) torch.int64 - loss_mask : 200 / 1026 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1026, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1026, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1026, 1, 576) torch.bfloat16 - out : (1, 1026) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 200 tokens) : 11.2604 - top-1 accuracy : 4/200 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1481]) -Dataloader batch - labels shape: torch.Size([1, 1481]) -Dataloader batch - attn_mask shape: torch.Size([1, 1481]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1481) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1481) torch.int64 - loss_mask : 350 / 1481 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1481, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1481, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1481, 1, 576) torch.bfloat16 - out : (1, 1481) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.3202 - top-1 accuracy : 11/350 = 3.1% - - [2026-04-29 12:22:49] iteration 22/ 100 | consumed samples: 88 | elapsed time per iteration (ms): 4845.6 | throughput (token/sec/GPU): 1229.4 | learning rate: 9.992056E-05 | global batch size: 4 | lm loss: 1.137132E+01 | loss scale: 1.0 | grad norm: 0.740 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 22 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 22 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 22 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1050.60, 1050.60) -Dataloader batch - tokens shape: torch.Size([1, 1757]) -Dataloader batch - labels shape: torch.Size([1, 1757]) -Dataloader batch - attn_mask shape: torch.Size([1, 1757]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1757) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1757) torch.int64 - loss_mask : 383 / 1757 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1757, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1757, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1757, 1, 576) torch.bfloat16 - out : (1, 1757) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.3092 - top-1 accuracy : 14/383 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1214]) -Dataloader batch - labels shape: torch.Size([1, 1214]) -Dataloader batch - attn_mask shape: torch.Size([1, 1214]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1214) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1214) torch.int64 - loss_mask : 289 / 1214 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1214, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1214, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1214, 1, 576) torch.bfloat16 - out : (1, 1214) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 289 tokens) : 11.3197 - top-1 accuracy : 2/289 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1673]) -Dataloader batch - labels shape: torch.Size([1, 1673]) -Dataloader batch - attn_mask shape: torch.Size([1, 1673]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1673) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1673) torch.int64 - loss_mask : 394 / 1673 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1673, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1673, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1673, 1, 576) torch.bfloat16 - out : (1, 1673) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3795 - top-1 accuracy : 10/394 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 963]) -Dataloader batch - labels shape: torch.Size([1, 963]) -Dataloader batch - attn_mask shape: torch.Size([1, 963]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 963) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 963) torch.int64 - loss_mask : 223 / 963 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (963, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (963, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (963, 1, 576) torch.bfloat16 - out : (1, 963) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 223 tokens) : 11.3192 - top-1 accuracy : 17/223 = 7.6% - - [2026-04-29 12:22:55] iteration 23/ 100 | consumed samples: 92 | elapsed time per iteration (ms): 4616.3 | throughput (token/sec/GPU): 1214.6 | learning rate: 9.980785E-05 | global batch size: 4 | lm loss: 1.133477E+01 | loss scale: 1.0 | grad norm: 0.832 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 23 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 23 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 23 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1180.81, 1180.81) -Dataloader batch - tokens shape: torch.Size([1, 1430]) -Dataloader batch - labels shape: torch.Size([1, 1430]) -Dataloader batch - attn_mask shape: torch.Size([1, 1430]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1430) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1430) torch.int64 - loss_mask : 382 / 1430 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1430, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1430, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1430, 1, 576) torch.bfloat16 - out : (1, 1430) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.3642 - top-1 accuracy : 16/382 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1191) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1191) torch.int64 - loss_mask : 280 / 1191 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1191, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1191, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1191, 1, 576) torch.bfloat16 - out : (1, 1191) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 280 tokens) : 11.2816 - top-1 accuracy : 19/280 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 392 / 1431 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3694 - top-1 accuracy : 21/392 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1535]) -Dataloader batch - labels shape: torch.Size([1, 1535]) -Dataloader batch - attn_mask shape: torch.Size([1, 1535]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1535) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1535) torch.int64 - loss_mask : 349 / 1535 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1535, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1535, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1535, 1, 576) torch.bfloat16 - out : (1, 1535) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.3177 - top-1 accuracy : 14/349 = 4.0% - - [2026-04-29 12:23:00] iteration 24/ 100 | consumed samples: 96 | elapsed time per iteration (ms): 4313.0 | throughput (token/sec/GPU): 1295.4 | learning rate: 9.964628E-05 | global batch size: 4 | lm loss: 1.133762E+01 | loss scale: 1.0 | grad norm: 1.035 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 24 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 24 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 24 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1271.39, 1271.39) -Dataloader batch - tokens shape: torch.Size([1, 1394]) -Dataloader batch - labels shape: torch.Size([1, 1394]) -Dataloader batch - attn_mask shape: torch.Size([1, 1394]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1394) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1394) torch.int64 - loss_mask : 317 / 1394 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1394, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1394, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1394, 1, 576) torch.bfloat16 - out : (1, 1394) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2622 - top-1 accuracy : 25/317 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1190]) -Dataloader batch - labels shape: torch.Size([1, 1190]) -Dataloader batch - attn_mask shape: torch.Size([1, 1190]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1190) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1190) torch.int64 - loss_mask : 278 / 1190 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1190, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1190, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1190, 1, 576) torch.bfloat16 - out : (1, 1190) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.2963 - top-1 accuracy : 22/278 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1554]) -Dataloader batch - labels shape: torch.Size([1, 1554]) -Dataloader batch - attn_mask shape: torch.Size([1, 1554]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1554) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1554) torch.int64 - loss_mask : 333 / 1554 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1554, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1554, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1554, 1, 576) torch.bfloat16 - out : (1, 1554) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.2884 - top-1 accuracy : 26/333 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 392 / 1443 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3861 - top-1 accuracy : 17/392 = 4.3% - - [2026-04-29 12:23:06] iteration 25/ 100 | consumed samples: 100 | elapsed time per iteration (ms): 4563.5 | throughput (token/sec/GPU): 1223.0 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.131279E+01 | loss scale: 1.0 | grad norm: 1.040 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 25 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 25 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 25 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1248.59, 1248.59) -Dataloader batch - tokens shape: torch.Size([1, 1395]) -Dataloader batch - labels shape: torch.Size([1, 1395]) -Dataloader batch - attn_mask shape: torch.Size([1, 1395]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 21 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1395) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1395) torch.int64 - loss_mask : 357 / 1395 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1395, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1395, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1395, 1, 576) torch.bfloat16 - out : (1, 1395) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3336 - top-1 accuracy : 19/357 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1553]) -Dataloader batch - labels shape: torch.Size([1, 1553]) -Dataloader batch - attn_mask shape: torch.Size([1, 1553]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 22 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1553) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1553) torch.int64 - loss_mask : 369 / 1553 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1553, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1553, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1553, 1, 576) torch.bfloat16 - out : (1, 1553) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2875 - top-1 accuracy : 27/369 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1177]) -Dataloader batch - labels shape: torch.Size([1, 1177]) -Dataloader batch - attn_mask shape: torch.Size([1, 1177]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 23 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1177) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1177) torch.int64 - loss_mask : 299 / 1177 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1177, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1177, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1177, 1, 576) torch.bfloat16 - out : (1, 1177) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 299 tokens) : 11.3431 - top-1 accuracy : 21/299 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1588]) -Dataloader batch - labels shape: torch.Size([1, 1588]) -Dataloader batch - attn_mask shape: torch.Size([1, 1588]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 24 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1588) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1588) torch.int64 - loss_mask : 455 / 1588 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1588, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1588, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1588, 1, 576) torch.bfloat16 - out : (1, 1588) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 455 tokens) : 11.3779 - top-1 accuracy : 16/455 = 3.5% - - [2026-04-29 12:23:12] iteration 26/ 100 | consumed samples: 104 | elapsed time per iteration (ms): 4397.4 | throughput (token/sec/GPU): 1299.2 | learning rate: 9.917726E-05 | global batch size: 4 | lm loss: 1.133764E+01 | loss scale: 1.0 | grad norm: 0.761 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 26 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 26 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 26 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1811.49, 1811.49) -Dataloader batch - tokens shape: torch.Size([1, 1498]) -Dataloader batch - labels shape: torch.Size([1, 1498]) -Dataloader batch - attn_mask shape: torch.Size([1, 1498]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 25 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1498) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1498) torch.int64 - loss_mask : 357 / 1498 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1498, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1498, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1498, 1, 576) torch.bfloat16 - out : (1, 1498) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3106 - top-1 accuracy : 16/357 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1553]) -Dataloader batch - labels shape: torch.Size([1, 1553]) -Dataloader batch - attn_mask shape: torch.Size([1, 1553]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 26 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1553) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1553) torch.int64 - loss_mask : 369 / 1553 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1553, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1553, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1553, 1, 576) torch.bfloat16 - out : (1, 1553) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.3102 - top-1 accuracy : 28/369 = 7.6% - -Dataloader batch - tokens shape: torch.Size([1, 1694]) -Dataloader batch - labels shape: torch.Size([1, 1694]) -Dataloader batch - attn_mask shape: torch.Size([1, 1694]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 27 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1694) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1694) torch.int64 - loss_mask : 348 / 1694 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1694, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1694, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1694, 1, 576) torch.bfloat16 - out : (1, 1694) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2493 - top-1 accuracy : 32/348 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1736]) -Dataloader batch - labels shape: torch.Size([1, 1736]) -Dataloader batch - attn_mask shape: torch.Size([1, 1736]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 28 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1736) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1736) torch.int64 - loss_mask : 365 / 1736 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1736, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1736, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1736, 1, 576) torch.bfloat16 - out : (1, 1736) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2502 - top-1 accuracy : 25/365 = 6.8% - - [2026-04-29 12:23:19] iteration 27/ 100 | consumed samples: 108 | elapsed time per iteration (ms): 5274.6 | throughput (token/sec/GPU): 1228.7 | learning rate: 9.887027E-05 | global batch size: 4 | lm loss: 1.128034E+01 | loss scale: 1.0 | grad norm: 0.887 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 27 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 27 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 27 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1079.35, 1079.35) -Dataloader batch - tokens shape: torch.Size([1, 1793]) -Dataloader batch - labels shape: torch.Size([1, 1793]) -Dataloader batch - attn_mask shape: torch.Size([1, 1793]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 29 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1793) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1793) torch.int64 - loss_mask : 435 / 1793 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1793, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1793, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1793, 1, 576) torch.bfloat16 - out : (1, 1793) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 435 tokens) : 11.3283 - top-1 accuracy : 28/435 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1457]) -Dataloader batch - labels shape: torch.Size([1, 1457]) -Dataloader batch - attn_mask shape: torch.Size([1, 1457]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 30 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1457) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1457) torch.int64 - loss_mask : 405 / 1457 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1457, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1457, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1457, 1, 576) torch.bfloat16 - out : (1, 1457) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.4073 - top-1 accuracy : 1/405 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1529]) -Dataloader batch - labels shape: torch.Size([1, 1529]) -Dataloader batch - attn_mask shape: torch.Size([1, 1529]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 31 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1529) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1529) torch.int64 - loss_mask : 390 / 1529 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1529, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1529, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1529, 1, 576) torch.bfloat16 - out : (1, 1529) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.2891 - top-1 accuracy : 2/390 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1381]) -Dataloader batch - labels shape: torch.Size([1, 1381]) -Dataloader batch - attn_mask shape: torch.Size([1, 1381]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 32 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1381) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1381) torch.int64 - loss_mask : 300 / 1381 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1381, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1381, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1381, 1, 576) torch.bfloat16 - out : (1, 1381) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.2697 - top-1 accuracy : 0/300 = 0.0% - - [2026-04-29 12:23:25] iteration 28/ 100 | consumed samples: 112 | elapsed time per iteration (ms): 4728.9 | throughput (token/sec/GPU): 1302.6 | learning rate: 9.851536E-05 | global batch size: 4 | lm loss: 1.132775E+01 | loss scale: 1.0 | grad norm: 0.749 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 28 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 28 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 28 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1587.02, 1587.02) -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 33 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 364 / 1542 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.3259 - top-1 accuracy : 21/364 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1399]) -Dataloader batch - labels shape: torch.Size([1, 1399]) -Dataloader batch - attn_mask shape: torch.Size([1, 1399]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 34 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1399) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1399) torch.int64 - loss_mask : 422 / 1399 tokens (30.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1399, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1399, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1399, 1, 576) torch.bfloat16 - out : (1, 1399) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.4425 - top-1 accuracy : 7/422 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1777]) -Dataloader batch - labels shape: torch.Size([1, 1777]) -Dataloader batch - attn_mask shape: torch.Size([1, 1777]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 35 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1777) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1777) torch.int64 - loss_mask : 409 / 1777 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1777, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1777, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1777, 1, 576) torch.bfloat16 - out : (1, 1777) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.2908 - top-1 accuracy : 22/409 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1731]) -Dataloader batch - labels shape: torch.Size([1, 1731]) -Dataloader batch - attn_mask shape: torch.Size([1, 1731]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 36 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1731) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1731) torch.int64 - loss_mask : 451 / 1731 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1731, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1731, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1731, 1, 576) torch.bfloat16 - out : (1, 1731) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 451 tokens) : 11.3920 - top-1 accuracy : 7/451 = 1.6% - - [2026-04-29 12:23:31] iteration 29/ 100 | consumed samples: 116 | elapsed time per iteration (ms): 4867.4 | throughput (token/sec/GPU): 1324.9 | learning rate: 9.811288E-05 | global batch size: 4 | lm loss: 1.136518E+01 | loss scale: 1.0 | grad norm: 0.752 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 29 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 29 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 29 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1150.08, 1150.08) -Dataloader batch - tokens shape: torch.Size([1, 1205]) -Dataloader batch - labels shape: torch.Size([1, 1205]) -Dataloader batch - attn_mask shape: torch.Size([1, 1205]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 37 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1205) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1205) torch.int64 - loss_mask : 300 / 1205 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1205, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1205, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1205, 1, 576) torch.bfloat16 - out : (1, 1205) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.3494 - top-1 accuracy : 10/300 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 38 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 377 / 1475 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.3295 - top-1 accuracy : 22/377 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1472]) -Dataloader batch - labels shape: torch.Size([1, 1472]) -Dataloader batch - attn_mask shape: torch.Size([1, 1472]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 39 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1472) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1472) torch.int64 - loss_mask : 312 / 1472 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1472, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1472, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1472, 1, 576) torch.bfloat16 - out : (1, 1472) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.2301 - top-1 accuracy : 23/312 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1040]) -Dataloader batch - labels shape: torch.Size([1, 1040]) -Dataloader batch - attn_mask shape: torch.Size([1, 1040]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 40 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1040) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1040) torch.int64 - loss_mask : 215 / 1040 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1040, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1040, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1040, 1, 576) torch.bfloat16 - out : (1, 1040) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 215 tokens) : 11.1874 - top-1 accuracy : 21/215 = 9.8% - - [2026-04-29 12:23:36] iteration 30/ 100 | consumed samples: 120 | elapsed time per iteration (ms): 4165.8 | throughput (token/sec/GPU): 1246.3 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.128331E+01 | loss scale: 1.0 | grad norm: 0.630 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 30 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 30 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 30 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1813.45, 1813.45) -Dataloader batch - tokens shape: torch.Size([1, 1432]) -Dataloader batch - labels shape: torch.Size([1, 1432]) -Dataloader batch - attn_mask shape: torch.Size([1, 1432]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 41 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1432) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1432) torch.int64 - loss_mask : 385 / 1432 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1432, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1432, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1432, 1, 576) torch.bfloat16 - out : (1, 1432) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.3653 - top-1 accuracy : 27/385 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1408]) -Dataloader batch - labels shape: torch.Size([1, 1408]) -Dataloader batch - attn_mask shape: torch.Size([1, 1408]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 42 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1408) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1408) torch.int64 - loss_mask : 348 / 1408 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1408, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1408, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1408, 1, 576) torch.bfloat16 - out : (1, 1408) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.3456 - top-1 accuracy : 15/348 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1087]) -Dataloader batch - labels shape: torch.Size([1, 1087]) -Dataloader batch - attn_mask shape: torch.Size([1, 1087]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 43 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1087) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1087) torch.int64 - loss_mask : 244 / 1087 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1087, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1087, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1087, 1, 576) torch.bfloat16 - out : (1, 1087) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 244 tokens) : 11.2286 - top-1 accuracy : 20/244 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1074]) -Dataloader batch - labels shape: torch.Size([1, 1074]) -Dataloader batch - attn_mask shape: torch.Size([1, 1074]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 44 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1074) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1074) torch.int64 - loss_mask : 310 / 1074 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1074, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1074, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1074, 1, 576) torch.bfloat16 - out : (1, 1074) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 310 tokens) : 11.3944 - top-1 accuracy : 14/310 = 4.5% - - [2026-04-29 12:23:42] iteration 31/ 100 | consumed samples: 124 | elapsed time per iteration (ms): 3981.7 | throughput (token/sec/GPU): 1256.0 | learning rate: 9.716682E-05 | global batch size: 4 | lm loss: 1.134109E+01 | loss scale: 1.0 | grad norm: 0.628 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 31 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 31 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 31 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1849.03, 1849.03) -Dataloader batch - tokens shape: torch.Size([1, 1383]) -Dataloader batch - labels shape: torch.Size([1, 1383]) -Dataloader batch - attn_mask shape: torch.Size([1, 1383]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 45 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1383) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1383) torch.int64 - loss_mask : 403 / 1383 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1383, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1383, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1383, 1, 576) torch.bfloat16 - out : (1, 1383) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.3868 - top-1 accuracy : 14/403 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1291]) -Dataloader batch - labels shape: torch.Size([1, 1291]) -Dataloader batch - attn_mask shape: torch.Size([1, 1291]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 46 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1291) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1291) torch.int64 - loss_mask : 396 / 1291 tokens (30.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1291, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1291, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1291, 1, 576) torch.bfloat16 - out : (1, 1291) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 396 tokens) : 11.3955 - top-1 accuracy : 0/396 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1228]) -Dataloader batch - labels shape: torch.Size([1, 1228]) -Dataloader batch - attn_mask shape: torch.Size([1, 1228]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 47 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1228) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1228) torch.int64 - loss_mask : 338 / 1228 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1228, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1228, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1228, 1, 576) torch.bfloat16 - out : (1, 1228) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.3653 - top-1 accuracy : 13/338 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1180]) -Dataloader batch - labels shape: torch.Size([1, 1180]) -Dataloader batch - attn_mask shape: torch.Size([1, 1180]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 48 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1180) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1180) torch.int64 - loss_mask : 260 / 1180 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1180, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1180, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1180, 1, 576) torch.bfloat16 - out : (1, 1180) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.3098 - top-1 accuracy : 5/260 = 1.9% - - [2026-04-29 12:23:48] iteration 32/ 100 | consumed samples: 128 | elapsed time per iteration (ms): 3917.0 | throughput (token/sec/GPU): 1297.4 | learning rate: 9.662419E-05 | global batch size: 4 | lm loss: 1.136973E+01 | loss scale: 1.0 | grad norm: 0.507 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 32 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 32 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 32 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2131.50, 2131.50) -Dataloader batch - tokens shape: torch.Size([1, 1219]) -Dataloader batch - labels shape: torch.Size([1, 1219]) -Dataloader batch - attn_mask shape: torch.Size([1, 1219]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 49 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1219) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1219) torch.int64 - loss_mask : 291 / 1219 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1219, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1219, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1219, 1, 576) torch.bfloat16 - out : (1, 1219) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 291 tokens) : 11.2999 - top-1 accuracy : 19/291 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1748]) -Dataloader batch - labels shape: torch.Size([1, 1748]) -Dataloader batch - attn_mask shape: torch.Size([1, 1748]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 50 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1748) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1748) torch.int64 - loss_mask : 380 / 1748 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1748, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1748, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1748, 1, 576) torch.bfloat16 - out : (1, 1748) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 380 tokens) : 11.2963 - top-1 accuracy : 24/380 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1873]) -Dataloader batch - labels shape: torch.Size([1, 1873]) -Dataloader batch - attn_mask shape: torch.Size([1, 1873]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 51 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1873) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1873) torch.int64 - loss_mask : 510 / 1873 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1873, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1873, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1873, 1, 576) torch.bfloat16 - out : (1, 1873) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 510 tokens) : 11.3447 - top-1 accuracy : 20/510 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1049]) -Dataloader batch - labels shape: torch.Size([1, 1049]) -Dataloader batch - attn_mask shape: torch.Size([1, 1049]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 52 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1049) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1049) torch.int64 - loss_mask : 265 / 1049 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1049, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1049, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1049, 1, 576) torch.bfloat16 - out : (1, 1049) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 265 tokens) : 11.3075 - top-1 accuracy : 1/265 = 0.4% - - [2026-04-29 12:23:55] iteration 33/ 100 | consumed samples: 132 | elapsed time per iteration (ms): 4549.3 | throughput (token/sec/GPU): 1294.5 | learning rate: 9.603585E-05 | global batch size: 4 | lm loss: 1.131618E+01 | loss scale: 1.0 | grad norm: 0.554 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 33 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 33 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 33 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2167.14, 2167.14) -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 53 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 392 / 1539 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3436 - top-1 accuracy : 18/392 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1454]) -Dataloader batch - labels shape: torch.Size([1, 1454]) -Dataloader batch - attn_mask shape: torch.Size([1, 1454]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 54 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1454) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1454) torch.int64 - loss_mask : 410 / 1454 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1454, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1454, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1454, 1, 576) torch.bfloat16 - out : (1, 1454) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.3232 - top-1 accuracy : 6/410 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1785]) -Dataloader batch - labels shape: torch.Size([1, 1785]) -Dataloader batch - attn_mask shape: torch.Size([1, 1785]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 55 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1785) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1785) torch.int64 - loss_mask : 426 / 1785 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1785, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1785, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1785, 1, 576) torch.bfloat16 - out : (1, 1785) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.3332 - top-1 accuracy : 21/426 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1565]) -Dataloader batch - labels shape: torch.Size([1, 1565]) -Dataloader batch - attn_mask shape: torch.Size([1, 1565]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 56 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1565) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1565) torch.int64 - loss_mask : 363 / 1565 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1565, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1565, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1565, 1, 576) torch.bfloat16 - out : (1, 1565) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2834 - top-1 accuracy : 10/363 = 2.8% - - [2026-04-29 12:24:02] iteration 34/ 100 | consumed samples: 136 | elapsed time per iteration (ms): 4818.4 | throughput (token/sec/GPU): 1316.4 | learning rate: 9.540239E-05 | global batch size: 4 | lm loss: 1.132181E+01 | loss scale: 1.0 | grad norm: 0.495 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 34 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 34 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 34 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1573.58, 1573.58) -Dataloader batch - tokens shape: torch.Size([1, 1025]) -Dataloader batch - labels shape: torch.Size([1, 1025]) -Dataloader batch - attn_mask shape: torch.Size([1, 1025]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 57 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1025) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1025) torch.int64 - loss_mask : 277 / 1025 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1025, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1025, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1025, 1, 576) torch.bfloat16 - out : (1, 1025) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 277 tokens) : 11.3252 - top-1 accuracy : 6/277 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1516]) -Dataloader batch - labels shape: torch.Size([1, 1516]) -Dataloader batch - attn_mask shape: torch.Size([1, 1516]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 58 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1516) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1516) torch.int64 - loss_mask : 325 / 1516 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1516, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1516, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1516, 1, 576) torch.bfloat16 - out : (1, 1516) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.2322 - top-1 accuracy : 20/325 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1899]) -Dataloader batch - labels shape: torch.Size([1, 1899]) -Dataloader batch - attn_mask shape: torch.Size([1, 1899]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 59 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1899) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1899) torch.int64 - loss_mask : 543 / 1899 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1899, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1899, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1899, 1, 576) torch.bfloat16 - out : (1, 1899) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 543 tokens) : 11.3966 - top-1 accuracy : 25/543 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1010]) -Dataloader batch - labels shape: torch.Size([1, 1010]) -Dataloader batch - attn_mask shape: torch.Size([1, 1010]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 60 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1010) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1010) torch.int64 - loss_mask : 225 / 1010 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1010, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1010, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1010, 1, 576) torch.bfloat16 - out : (1, 1010) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 225 tokens) : 11.2648 - top-1 accuracy : 14/225 = 6.2% - - [2026-04-29 12:24:08] iteration 35/ 100 | consumed samples: 140 | elapsed time per iteration (ms): 4297.1 | throughput (token/sec/GPU): 1268.3 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.132151E+01 | loss scale: 1.0 | grad norm: 0.625 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 35 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 35 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 35 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1235.81, 1235.81) -Dataloader batch - tokens shape: torch.Size([1, 1795]) -Dataloader batch - labels shape: torch.Size([1, 1795]) -Dataloader batch - attn_mask shape: torch.Size([1, 1795]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 61 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1795) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1795) torch.int64 - loss_mask : 428 / 1795 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1795, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1795, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1795, 1, 576) torch.bfloat16 - out : (1, 1795) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 428 tokens) : 11.2667 - top-1 accuracy : 21/428 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 62 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 389 / 1436 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.3462 - top-1 accuracy : 17/389 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1554]) -Dataloader batch - labels shape: torch.Size([1, 1554]) -Dataloader batch - attn_mask shape: torch.Size([1, 1554]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 63 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1554) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1554) torch.int64 - loss_mask : 387 / 1554 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1554, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1554, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1554, 1, 576) torch.bfloat16 - out : (1, 1554) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2994 - top-1 accuracy : 30/387 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 64 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 345 / 1434 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2860 - top-1 accuracy : 30/345 = 8.7% - - [2026-04-29 12:24:14] iteration 36/ 100 | consumed samples: 144 | elapsed time per iteration (ms): 4813.5 | throughput (token/sec/GPU): 1292.0 | learning rate: 9.400269E-05 | global batch size: 4 | lm loss: 1.129912E+01 | loss scale: 1.0 | grad norm: 0.441 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 36 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 36 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 36 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1601.60, 1601.60) -Dataloader batch - tokens shape: torch.Size([1, 1356]) -Dataloader batch - labels shape: torch.Size([1, 1356]) -Dataloader batch - attn_mask shape: torch.Size([1, 1356]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 65 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1356) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1356) torch.int64 - loss_mask : 280 / 1356 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1356, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1356, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1356, 1, 576) torch.bfloat16 - out : (1, 1356) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 280 tokens) : 11.1773 - top-1 accuracy : 25/280 = 8.9% - -Dataloader batch - tokens shape: torch.Size([1, 1244]) -Dataloader batch - labels shape: torch.Size([1, 1244]) -Dataloader batch - attn_mask shape: torch.Size([1, 1244]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 66 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1244) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1244) torch.int64 - loss_mask : 341 / 1244 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1244, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1244, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1244, 1, 576) torch.bfloat16 - out : (1, 1244) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.3379 - top-1 accuracy : 2/341 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 67 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 320 / 1400 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2470 - top-1 accuracy : 26/320 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1218]) -Dataloader batch - labels shape: torch.Size([1, 1218]) -Dataloader batch - attn_mask shape: torch.Size([1, 1218]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 68 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1218) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1218) torch.int64 - loss_mask : 319 / 1218 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1218, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1218, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1218, 1, 576) torch.bfloat16 - out : (1, 1218) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.3451 - top-1 accuracy : 0/319 = 0.0% - - [2026-04-29 12:24:19] iteration 37/ 100 | consumed samples: 148 | elapsed time per iteration (ms): 4129.5 | throughput (token/sec/GPU): 1263.6 | learning rate: 9.323782E-05 | global batch size: 4 | lm loss: 1.128093E+01 | loss scale: 1.0 | grad norm: 0.503 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 37 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 37 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 37 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1650.92, 1650.92) -Dataloader batch - tokens shape: torch.Size([1, 1614]) -Dataloader batch - labels shape: torch.Size([1, 1614]) -Dataloader batch - attn_mask shape: torch.Size([1, 1614]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 69 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1614) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1614) torch.int64 - loss_mask : 356 / 1614 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1614, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1614, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1614, 1, 576) torch.bfloat16 - out : (1, 1614) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2630 - top-1 accuracy : 25/356 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1556]) -Dataloader batch - labels shape: torch.Size([1, 1556]) -Dataloader batch - attn_mask shape: torch.Size([1, 1556]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 70 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1556) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1556) torch.int64 - loss_mask : 369 / 1556 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1556, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1556, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1556, 1, 576) torch.bfloat16 - out : (1, 1556) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2836 - top-1 accuracy : 31/369 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 1023]) -Dataloader batch - labels shape: torch.Size([1, 1023]) -Dataloader batch - attn_mask shape: torch.Size([1, 1023]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 71 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1023) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1023) torch.int64 - loss_mask : 258 / 1023 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1023, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1023, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1023, 1, 576) torch.bfloat16 - out : (1, 1023) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 258 tokens) : 11.2788 - top-1 accuracy : 17/258 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1560]) -Dataloader batch - labels shape: torch.Size([1, 1560]) -Dataloader batch - attn_mask shape: torch.Size([1, 1560]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 72 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1560) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1560) torch.int64 - loss_mask : 421 / 1560 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1560, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1560, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1560, 1, 576) torch.bfloat16 - out : (1, 1560) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 421 tokens) : 11.3482 - top-1 accuracy : 26/421 = 6.2% - - [2026-04-29 12:24:25] iteration 38/ 100 | consumed samples: 152 | elapsed time per iteration (ms): 4543.3 | throughput (token/sec/GPU): 1266.3 | learning rate: 9.243060E-05 | global batch size: 4 | lm loss: 1.129689E+01 | loss scale: 1.0 | grad norm: 0.513 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 38 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 38 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 38 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1617.58, 1617.58) -Dataloader batch - tokens shape: torch.Size([1, 1681]) -Dataloader batch - labels shape: torch.Size([1, 1681]) -Dataloader batch - attn_mask shape: torch.Size([1, 1681]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 73 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1681) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1681) torch.int64 - loss_mask : 408 / 1681 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1681, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1681, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1681, 1, 576) torch.bfloat16 - out : (1, 1681) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3177 - top-1 accuracy : 32/408 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1398]) -Dataloader batch - labels shape: torch.Size([1, 1398]) -Dataloader batch - attn_mask shape: torch.Size([1, 1398]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 74 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1398) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1398) torch.int64 - loss_mask : 328 / 1398 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1398, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1398, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1398, 1, 576) torch.bfloat16 - out : (1, 1398) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.2624 - top-1 accuracy : 15/328 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1853]) -Dataloader batch - labels shape: torch.Size([1, 1853]) -Dataloader batch - attn_mask shape: torch.Size([1, 1853]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 75 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1853) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1853) torch.int64 - loss_mask : 499 / 1853 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1853, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1853, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1853, 1, 576) torch.bfloat16 - out : (1, 1853) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 499 tokens) : 11.3689 - top-1 accuracy : 15/499 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 76 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 376 / 1539 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3072 - top-1 accuracy : 14/376 = 3.7% - - [2026-04-29 12:24:32] iteration 39/ 100 | consumed samples: 156 | elapsed time per iteration (ms): 4858.3 | throughput (token/sec/GPU): 1332.0 | learning rate: 9.158184E-05 | global batch size: 4 | lm loss: 1.131985E+01 | loss scale: 1.0 | grad norm: 0.429 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 39 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 39 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 39 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1698.07, 1698.07) -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 77 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 356 / 1434 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.3037 - top-1 accuracy : 28/356 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1562]) -Dataloader batch - labels shape: torch.Size([1, 1562]) -Dataloader batch - attn_mask shape: torch.Size([1, 1562]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 78 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1562) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1562) torch.int64 - loss_mask : 410 / 1562 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1562, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1562, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1562, 1, 576) torch.bfloat16 - out : (1, 1562) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.3960 - top-1 accuracy : 7/410 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1435]) -Dataloader batch - labels shape: torch.Size([1, 1435]) -Dataloader batch - attn_mask shape: torch.Size([1, 1435]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 79 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1435) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1435) torch.int64 - loss_mask : 376 / 1435 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1435, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1435, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1435, 1, 576) torch.bfloat16 - out : (1, 1435) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3384 - top-1 accuracy : 14/376 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1694]) -Dataloader batch - labels shape: torch.Size([1, 1694]) -Dataloader batch - attn_mask shape: torch.Size([1, 1694]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 80 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1694) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1694) torch.int64 - loss_mask : 344 / 1694 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1694, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1694, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1694, 1, 576) torch.bfloat16 - out : (1, 1694) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.1951 - top-1 accuracy : 34/344 = 9.9% - - [2026-04-29 12:24:38] iteration 40/ 100 | consumed samples: 160 | elapsed time per iteration (ms): 4832.3 | throughput (token/sec/GPU): 1267.5 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.131280E+01 | loss scale: 1.0 | grad norm: 0.450 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4821.77, 4821.77) - forward-compute ................................: (3607.96, 3607.96) - backward-compute ...............................: (1211.51, 1211.51) - batch-generator ................................: (463.04, 463.04) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.45, 0.45) - params-all-gather ..............................: (0.14, 0.14) - optimizer-copy-to-main-grad ....................: (0.13, 0.13) - optimizer-clip-main-grad .......................: (0.48, 0.48) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.28, 1.28) - optimizer-copy-main-to-model-params ............: (0.20, 0.20) - optimizer ......................................: (2.81, 2.81) -saving checkpoint at iteration 40 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 40 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 40 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1662.81, 1662.81) -Dataloader batch - tokens shape: torch.Size([1, 1581]) -Dataloader batch - labels shape: torch.Size([1, 1581]) -Dataloader batch - attn_mask shape: torch.Size([1, 1581]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 81 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1581) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1581) torch.int64 - loss_mask : 422 / 1581 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1581, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1581, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1581, 1, 576) torch.bfloat16 - out : (1, 1581) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.3586 - top-1 accuracy : 17/422 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1500]) -Dataloader batch - labels shape: torch.Size([1, 1500]) -Dataloader batch - attn_mask shape: torch.Size([1, 1500]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 82 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1500) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1500) torch.int64 - loss_mask : 391 / 1500 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1500, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1500, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1500, 1, 576) torch.bfloat16 - out : (1, 1500) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3426 - top-1 accuracy : 13/391 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1559]) -Dataloader batch - labels shape: torch.Size([1, 1559]) -Dataloader batch - attn_mask shape: torch.Size([1, 1559]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 83 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1559) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1559) torch.int64 - loss_mask : 340 / 1559 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1559, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1559, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1559, 1, 576) torch.bfloat16 - out : (1, 1559) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2193 - top-1 accuracy : 26/340 = 7.6% - -Dataloader batch - tokens shape: torch.Size([1, 1795]) -Dataloader batch - labels shape: torch.Size([1, 1795]) -Dataloader batch - attn_mask shape: torch.Size([1, 1795]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 84 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1795) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1795) torch.int64 - loss_mask : 432 / 1795 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1795, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1795, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1795, 1, 576) torch.bfloat16 - out : (1, 1795) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 432 tokens) : 11.3146 - top-1 accuracy : 7/432 = 1.6% - - [2026-04-29 12:24:45] iteration 41/ 100 | consumed samples: 164 | elapsed time per iteration (ms): 4887.2 | throughput (token/sec/GPU): 1316.7 | learning rate: 8.976308E-05 | global batch size: 4 | lm loss: 1.131278E+01 | loss scale: 1.0 | grad norm: 0.459 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 41 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 41 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 41 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1169.28, 1169.28) -Dataloader batch - tokens shape: torch.Size([1, 1732]) -Dataloader batch - labels shape: torch.Size([1, 1732]) -Dataloader batch - attn_mask shape: torch.Size([1, 1732]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 85 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1732) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1732) torch.int64 - loss_mask : 370 / 1732 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1732, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1732, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1732, 1, 576) torch.bfloat16 - out : (1, 1732) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.2719 - top-1 accuracy : 33/370 = 8.9% - -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 86 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1420) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1420) torch.int64 - loss_mask : 375 / 1420 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1420, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1420, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1420, 1, 576) torch.bfloat16 - out : (1, 1420) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.3466 - top-1 accuracy : 29/375 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1216]) -Dataloader batch - labels shape: torch.Size([1, 1216]) -Dataloader batch - attn_mask shape: torch.Size([1, 1216]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 87 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1216) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1216) torch.int64 - loss_mask : 298 / 1216 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1216, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1216, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1216, 1, 576) torch.bfloat16 - out : (1, 1216) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 298 tokens) : 11.3064 - top-1 accuracy : 9/298 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 88 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 356 / 1543 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2973 - top-1 accuracy : 27/356 = 7.6% - - [2026-04-29 12:24:51] iteration 42/ 100 | consumed samples: 168 | elapsed time per iteration (ms): 4640.8 | throughput (token/sec/GPU): 1273.7 | learning rate: 8.879489E-05 | global batch size: 4 | lm loss: 1.130576E+01 | loss scale: 1.0 | grad norm: 0.423 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 42 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 42 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 42 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1536.34, 1536.34) -Dataloader batch - tokens shape: torch.Size([1, 1072]) -Dataloader batch - labels shape: torch.Size([1, 1072]) -Dataloader batch - attn_mask shape: torch.Size([1, 1072]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 89 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1072) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1072) torch.int64 - loss_mask : 305 / 1072 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1072, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1072, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1072, 1, 576) torch.bfloat16 - out : (1, 1072) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.4321 - top-1 accuracy : 6/305 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 90 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 312 / 1400 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.2221 - top-1 accuracy : 30/312 = 9.6% - -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 91 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1696) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1696) torch.int64 - loss_mask : 385 / 1696 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1696, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1696, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1696, 1, 576) torch.bfloat16 - out : (1, 1696) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.2832 - top-1 accuracy : 29/385 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 848]) -Dataloader batch - labels shape: torch.Size([1, 848]) -Dataloader batch - attn_mask shape: torch.Size([1, 848]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 92 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 848) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 848) torch.int64 - loss_mask : 151 / 848 tokens (17.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (848, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (848, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (848, 1, 576) torch.bfloat16 - out : (1, 848) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 151 tokens) : 11.0210 - top-1 accuracy : 3/151 = 2.0% - - [2026-04-29 12:24:57] iteration 43/ 100 | consumed samples: 172 | elapsed time per iteration (ms): 4181.8 | throughput (token/sec/GPU): 1199.5 | learning rate: 8.778875E-05 | global batch size: 4 | lm loss: 1.127174E+01 | loss scale: 1.0 | grad norm: 0.462 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 43 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 43 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 43 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1901.25, 1901.25) -Dataloader batch - tokens shape: torch.Size([1, 995]) -Dataloader batch - labels shape: torch.Size([1, 995]) -Dataloader batch - attn_mask shape: torch.Size([1, 995]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 93 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 995) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 995) torch.int64 - loss_mask : 228 / 995 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (995, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (995, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (995, 1, 576) torch.bfloat16 - out : (1, 995) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 228 tokens) : 11.2677 - top-1 accuracy : 12/228 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1373]) -Dataloader batch - labels shape: torch.Size([1, 1373]) -Dataloader batch - attn_mask shape: torch.Size([1, 1373]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 94 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1373) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1373) torch.int64 - loss_mask : 327 / 1373 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1373, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1373, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1373, 1, 576) torch.bfloat16 - out : (1, 1373) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2624 - top-1 accuracy : 27/327 = 8.3% - -Dataloader batch - tokens shape: torch.Size([1, 1033]) -Dataloader batch - labels shape: torch.Size([1, 1033]) -Dataloader batch - attn_mask shape: torch.Size([1, 1033]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 95 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1033) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1033) torch.int64 - loss_mask : 201 / 1033 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1033, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1033, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1033, 1, 576) torch.bfloat16 - out : (1, 1033) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 201 tokens) : 11.1536 - top-1 accuracy : 23/201 = 11.4% - -Dataloader batch - tokens shape: torch.Size([1, 1707]) -Dataloader batch - labels shape: torch.Size([1, 1707]) -Dataloader batch - attn_mask shape: torch.Size([1, 1707]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 96 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1707) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1707) torch.int64 - loss_mask : 364 / 1707 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1707, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1707, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1707, 1, 576) torch.bfloat16 - out : (1, 1707) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.1961 - top-1 accuracy : 37/364 = 10.2% - - [2026-04-29 12:25:03] iteration 44/ 100 | consumed samples: 176 | elapsed time per iteration (ms): 4241.8 | throughput (token/sec/GPU): 1204.2 | learning rate: 8.674567E-05 | global batch size: 4 | lm loss: 1.122238E+01 | loss scale: 1.0 | grad norm: 0.528 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 44 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 44 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 44 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1371.31, 1371.31) -Dataloader batch - tokens shape: torch.Size([1, 1236]) -Dataloader batch - labels shape: torch.Size([1, 1236]) -Dataloader batch - attn_mask shape: torch.Size([1, 1236]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 97 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1236) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1236) torch.int64 - loss_mask : 340 / 1236 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1236, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1236, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1236, 1, 576) torch.bfloat16 - out : (1, 1236) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.3561 - top-1 accuracy : 14/340 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1481]) -Dataloader batch - labels shape: torch.Size([1, 1481]) -Dataloader batch - attn_mask shape: torch.Size([1, 1481]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 98 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1481) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1481) torch.int64 - loss_mask : 393 / 1481 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1481, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1481, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1481, 1, 576) torch.bfloat16 - out : (1, 1481) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 393 tokens) : 11.3337 - top-1 accuracy : 21/393 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1261]) -Dataloader batch - labels shape: torch.Size([1, 1261]) -Dataloader batch - attn_mask shape: torch.Size([1, 1261]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 99 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1261) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1261) torch.int64 - loss_mask : 366 / 1261 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1261, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1261, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1261, 1, 576) torch.bfloat16 - out : (1, 1261) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.4170 - top-1 accuracy : 1/366 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 100 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 360 / 1422 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.3636 - top-1 accuracy : 16/360 = 4.4% - - [2026-04-29 12:25:08] iteration 45/ 100 | consumed samples: 180 | elapsed time per iteration (ms): 4190.7 | throughput (token/sec/GPU): 1288.6 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.136718E+01 | loss scale: 1.0 | grad norm: 0.462 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 45 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 45 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 45 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1693.46, 1693.46) -Dataloader batch - tokens shape: torch.Size([1, 1706]) -Dataloader batch - labels shape: torch.Size([1, 1706]) -Dataloader batch - attn_mask shape: torch.Size([1, 1706]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 101 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1706) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1706) torch.int64 - loss_mask : 348 / 1706 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1706, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1706, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1706, 1, 576) torch.bfloat16 - out : (1, 1706) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2016 - top-1 accuracy : 31/348 = 8.9% - -Dataloader batch - tokens shape: torch.Size([1, 1323]) -Dataloader batch - labels shape: torch.Size([1, 1323]) -Dataloader batch - attn_mask shape: torch.Size([1, 1323]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 102 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1323) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1323) torch.int64 - loss_mask : 325 / 1323 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1323, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1323, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1323, 1, 576) torch.bfloat16 - out : (1, 1323) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.2947 - top-1 accuracy : 25/325 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1373]) -Dataloader batch - labels shape: torch.Size([1, 1373]) -Dataloader batch - attn_mask shape: torch.Size([1, 1373]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 103 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1373) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1373) torch.int64 - loss_mask : 293 / 1373 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1373, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1373, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1373, 1, 576) torch.bfloat16 - out : (1, 1373) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 293 tokens) : 11.2071 - top-1 accuracy : 28/293 = 9.6% - -Dataloader batch - tokens shape: torch.Size([1, 1869]) -Dataloader batch - labels shape: torch.Size([1, 1869]) -Dataloader batch - attn_mask shape: torch.Size([1, 1869]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 104 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1869) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1869) torch.int64 - loss_mask : 503 / 1869 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1869, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1869, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1869, 1, 576) torch.bfloat16 - out : (1, 1869) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 503 tokens) : 11.3337 - top-1 accuracy : 18/503 = 3.6% - - [2026-04-29 12:25:15] iteration 46/ 100 | consumed samples: 184 | elapsed time per iteration (ms): 5001.2 | throughput (token/sec/GPU): 1253.9 | learning rate: 8.455283E-05 | global batch size: 4 | lm loss: 1.126853E+01 | loss scale: 1.0 | grad norm: 0.415 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 46 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 46 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 46 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1153.95, 1153.95) -Dataloader batch - tokens shape: torch.Size([1, 1517]) -Dataloader batch - labels shape: torch.Size([1, 1517]) -Dataloader batch - attn_mask shape: torch.Size([1, 1517]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 105 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1517) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1517) torch.int64 - loss_mask : 349 / 1517 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1517, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1517, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1517, 1, 576) torch.bfloat16 - out : (1, 1517) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2658 - top-1 accuracy : 33/349 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1550]) -Dataloader batch - labels shape: torch.Size([1, 1550]) -Dataloader batch - attn_mask shape: torch.Size([1, 1550]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 106 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1550) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1550) torch.int64 - loss_mask : 424 / 1550 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1550, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1550, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1550, 1, 576) torch.bfloat16 - out : (1, 1550) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 424 tokens) : 11.3765 - top-1 accuracy : 19/424 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 107 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 365 / 1374 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.3379 - top-1 accuracy : 16/365 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 108 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 345 / 1722 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.1959 - top-1 accuracy : 23/345 = 6.7% - - [2026-04-29 12:25:21] iteration 47/ 100 | consumed samples: 188 | elapsed time per iteration (ms): 4868.2 | throughput (token/sec/GPU): 1266.0 | learning rate: 8.340524E-05 | global batch size: 4 | lm loss: 1.129895E+01 | loss scale: 1.0 | grad norm: 0.407 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 47 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 47 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 47 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1364.81, 1364.81) -Dataloader batch - tokens shape: torch.Size([1, 990]) -Dataloader batch - labels shape: torch.Size([1, 990]) -Dataloader batch - attn_mask shape: torch.Size([1, 990]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 109 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 990) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 990) torch.int64 - loss_mask : 218 / 990 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (990, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (990, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (990, 1, 576) torch.bfloat16 - out : (1, 990) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 218 tokens) : 11.2622 - top-1 accuracy : 12/218 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 110 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 344 / 1552 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.2738 - top-1 accuracy : 24/344 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1768]) -Dataloader batch - labels shape: torch.Size([1, 1768]) -Dataloader batch - attn_mask shape: torch.Size([1, 1768]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 111 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1768) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1768) torch.int64 - loss_mask : 427 / 1768 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1768, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1768, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1768, 1, 576) torch.bfloat16 - out : (1, 1768) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 427 tokens) : 11.2720 - top-1 accuracy : 33/427 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1553]) -Dataloader batch - labels shape: torch.Size([1, 1553]) -Dataloader batch - attn_mask shape: torch.Size([1, 1553]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 112 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1553) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1553) torch.int64 - loss_mask : 366 / 1553 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1553, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1553, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1553, 1, 576) torch.bfloat16 - out : (1, 1553) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.3104 - top-1 accuracy : 20/366 = 5.5% - - [2026-04-29 12:25:27] iteration 48/ 100 | consumed samples: 192 | elapsed time per iteration (ms): 4686.8 | throughput (token/sec/GPU): 1251.0 | learning rate: 8.222505E-05 | global batch size: 4 | lm loss: 1.128124E+01 | loss scale: 1.0 | grad norm: 0.385 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 48 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 48 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 48 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1597.34, 1597.34) -Dataloader batch - tokens shape: torch.Size([1, 1586]) -Dataloader batch - labels shape: torch.Size([1, 1586]) -Dataloader batch - attn_mask shape: torch.Size([1, 1586]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 113 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1586) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1586) torch.int64 - loss_mask : 450 / 1586 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1586, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1586, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1586, 1, 576) torch.bfloat16 - out : (1, 1586) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 450 tokens) : 11.3879 - top-1 accuracy : 16/450 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1051]) -Dataloader batch - labels shape: torch.Size([1, 1051]) -Dataloader batch - attn_mask shape: torch.Size([1, 1051]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 114 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1051) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1051) torch.int64 - loss_mask : 299 / 1051 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1051, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1051, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1051, 1, 576) torch.bfloat16 - out : (1, 1051) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 299 tokens) : 11.3156 - top-1 accuracy : 4/299 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1148]) -Dataloader batch - labels shape: torch.Size([1, 1148]) -Dataloader batch - attn_mask shape: torch.Size([1, 1148]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 115 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1148) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1148) torch.int64 - loss_mask : 315 / 1148 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1148, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1148, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1148, 1, 576) torch.bfloat16 - out : (1, 1148) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.3128 - top-1 accuracy : 11/315 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 116 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 397 / 1542 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.3414 - top-1 accuracy : 13/397 = 3.3% - - [2026-04-29 12:25:33] iteration 49/ 100 | consumed samples: 196 | elapsed time per iteration (ms): 3998.8 | throughput (token/sec/GPU): 1332.2 | learning rate: 8.101343E-05 | global batch size: 4 | lm loss: 1.134427E+01 | loss scale: 1.0 | grad norm: 0.400 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 49 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 49 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 49 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1734.44, 1734.44) -Dataloader batch - tokens shape: torch.Size([1, 1207]) -Dataloader batch - labels shape: torch.Size([1, 1207]) -Dataloader batch - attn_mask shape: torch.Size([1, 1207]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 117 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1207) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1207) torch.int64 - loss_mask : 302 / 1207 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1207, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1207, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1207, 1, 576) torch.bfloat16 - out : (1, 1207) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 302 tokens) : 11.3243 - top-1 accuracy : 1/302 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1769]) -Dataloader batch - labels shape: torch.Size([1, 1769]) -Dataloader batch - attn_mask shape: torch.Size([1, 1769]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 118 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1769) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1769) torch.int64 - loss_mask : 475 / 1769 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1769, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1769, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1769, 1, 576) torch.bfloat16 - out : (1, 1769) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 475 tokens) : 11.3283 - top-1 accuracy : 28/475 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1406]) -Dataloader batch - labels shape: torch.Size([1, 1406]) -Dataloader batch - attn_mask shape: torch.Size([1, 1406]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 119 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1406) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1406) torch.int64 - loss_mask : 374 / 1406 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1406, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1406, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1406, 1, 576) torch.bfloat16 - out : (1, 1406) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.3157 - top-1 accuracy : 16/374 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1735]) -Dataloader batch - labels shape: torch.Size([1, 1735]) -Dataloader batch - attn_mask shape: torch.Size([1, 1735]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 120 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1735) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1735) torch.int64 - loss_mask : 448 / 1735 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1735, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1735, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1735, 1, 576) torch.bfloat16 - out : (1, 1735) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 448 tokens) : 11.3200 - top-1 accuracy : 18/448 = 4.0% - - [2026-04-29 12:25:39] iteration 50/ 100 | consumed samples: 200 | elapsed time per iteration (ms): 4738.3 | throughput (token/sec/GPU): 1291.0 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.132229E+01 | loss scale: 1.0 | grad norm: 0.315 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 50 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 50 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 50 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1514.20, 1514.20) -Dataloader batch - tokens shape: torch.Size([1, 1904]) -Dataloader batch - labels shape: torch.Size([1, 1904]) -Dataloader batch - attn_mask shape: torch.Size([1, 1904]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 121 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1904) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1904) torch.int64 - loss_mask : 535 / 1904 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1904, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1904, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1904, 1, 576) torch.bfloat16 - out : (1, 1904) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 535 tokens) : 11.4000 - top-1 accuracy : 21/535 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1509]) -Dataloader batch - labels shape: torch.Size([1, 1509]) -Dataloader batch - attn_mask shape: torch.Size([1, 1509]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 122 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1509) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1509) torch.int64 - loss_mask : 335 / 1509 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1509, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1509, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1509, 1, 576) torch.bfloat16 - out : (1, 1509) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2216 - top-1 accuracy : 31/335 = 9.3% - -Dataloader batch - tokens shape: torch.Size([1, 1522]) -Dataloader batch - labels shape: torch.Size([1, 1522]) -Dataloader batch - attn_mask shape: torch.Size([1, 1522]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 123 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1522) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1522) torch.int64 - loss_mask : 371 / 1522 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1522, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1522, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1522, 1, 576) torch.bfloat16 - out : (1, 1522) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.3373 - top-1 accuracy : 19/371 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1786]) -Dataloader batch - labels shape: torch.Size([1, 1786]) -Dataloader batch - attn_mask shape: torch.Size([1, 1786]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 124 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1786) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1786) torch.int64 - loss_mask : 484 / 1786 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1786, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1786, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1786, 1, 576) torch.bfloat16 - out : (1, 1786) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 484 tokens) : 11.3710 - top-1 accuracy : 34/484 = 7.0% - - [2026-04-29 12:25:46] iteration 51/ 100 | consumed samples: 204 | elapsed time per iteration (ms): 5083.6 | throughput (token/sec/GPU): 1322.1 | learning rate: 7.850071E-05 | global batch size: 4 | lm loss: 1.134372E+01 | loss scale: 1.0 | grad norm: 0.341 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 51 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 51 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 51 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1047.42, 1047.42) -Dataloader batch - tokens shape: torch.Size([1, 1021]) -Dataloader batch - labels shape: torch.Size([1, 1021]) -Dataloader batch - attn_mask shape: torch.Size([1, 1021]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 125 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1021) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1021) torch.int64 - loss_mask : 262 / 1021 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1021, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1021, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1021, 1, 576) torch.bfloat16 - out : (1, 1021) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 262 tokens) : 11.2826 - top-1 accuracy : 24/262 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1516]) -Dataloader batch - labels shape: torch.Size([1, 1516]) -Dataloader batch - attn_mask shape: torch.Size([1, 1516]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 126 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1516) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1516) torch.int64 - loss_mask : 336 / 1516 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1516, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1516, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1516, 1, 576) torch.bfloat16 - out : (1, 1516) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.2628 - top-1 accuracy : 28/336 = 8.3% - -Dataloader batch - tokens shape: torch.Size([1, 1405]) -Dataloader batch - labels shape: torch.Size([1, 1405]) -Dataloader batch - attn_mask shape: torch.Size([1, 1405]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 127 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1405) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1405) torch.int64 - loss_mask : 348 / 1405 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1405, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1405, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1405, 1, 576) torch.bfloat16 - out : (1, 1405) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2902 - top-1 accuracy : 22/348 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 128 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 354 / 1539 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2948 - top-1 accuracy : 28/354 = 7.9% - - [2026-04-29 12:25:52] iteration 52/ 100 | consumed samples: 208 | elapsed time per iteration (ms): 4793.8 | throughput (token/sec/GPU): 1143.4 | learning rate: 7.720211E-05 | global batch size: 4 | lm loss: 1.128284E+01 | loss scale: 1.0 | grad norm: 0.434 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 52 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 52 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 52 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1556.26, 1556.26) -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 129 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 384 / 1386 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.3258 - top-1 accuracy : 5/384 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1621]) -Dataloader batch - labels shape: torch.Size([1, 1621]) -Dataloader batch - attn_mask shape: torch.Size([1, 1621]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 130 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1621) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1621) torch.int64 - loss_mask : 379 / 1621 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1621, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1621, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1621, 1, 576) torch.bfloat16 - out : (1, 1621) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.2974 - top-1 accuracy : 17/379 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1452]) -Dataloader batch - labels shape: torch.Size([1, 1452]) -Dataloader batch - attn_mask shape: torch.Size([1, 1452]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 131 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1452) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1452) torch.int64 - loss_mask : 397 / 1452 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1452, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1452, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1452, 1, 576) torch.bfloat16 - out : (1, 1452) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.3738 - top-1 accuracy : 11/397 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1575]) -Dataloader batch - labels shape: torch.Size([1, 1575]) -Dataloader batch - attn_mask shape: torch.Size([1, 1575]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 132 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1575) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1575) torch.int64 - loss_mask : 347 / 1575 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1575, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1575, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1575, 1, 576) torch.bfloat16 - out : (1, 1575) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2416 - top-1 accuracy : 7/347 = 2.0% - - [2026-04-29 12:25:58] iteration 53/ 100 | consumed samples: 212 | elapsed time per iteration (ms): 4636.4 | throughput (token/sec/GPU): 1301.4 | learning rate: 7.587705E-05 | global batch size: 4 | lm loss: 1.131192E+01 | loss scale: 1.0 | grad norm: 0.373 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 53 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 53 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 53 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1319.81, 1319.81) -Dataloader batch - tokens shape: torch.Size([1, 1721]) -Dataloader batch - labels shape: torch.Size([1, 1721]) -Dataloader batch - attn_mask shape: torch.Size([1, 1721]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 133 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1721) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1721) torch.int64 - loss_mask : 367 / 1721 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1721, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1721, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1721, 1, 576) torch.bfloat16 - out : (1, 1721) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2271 - top-1 accuracy : 21/367 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1895]) -Dataloader batch - labels shape: torch.Size([1, 1895]) -Dataloader batch - attn_mask shape: torch.Size([1, 1895]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 134 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1895) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1895) torch.int64 - loss_mask : 543 / 1895 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1895, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1895, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1895, 1, 576) torch.bfloat16 - out : (1, 1895) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 543 tokens) : 11.3798 - top-1 accuracy : 18/543 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1654]) -Dataloader batch - labels shape: torch.Size([1, 1654]) -Dataloader batch - attn_mask shape: torch.Size([1, 1654]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 135 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1654) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1654) torch.int64 - loss_mask : 374 / 1654 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1654, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1654, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1654, 1, 576) torch.bfloat16 - out : (1, 1654) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.2350 - top-1 accuracy : 32/374 = 8.6% - -Dataloader batch - tokens shape: torch.Size([1, 1206]) -Dataloader batch - labels shape: torch.Size([1, 1206]) -Dataloader batch - attn_mask shape: torch.Size([1, 1206]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 136 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1206) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1206) torch.int64 - loss_mask : 288 / 1206 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1206, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1206, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1206, 1, 576) torch.bfloat16 - out : (1, 1206) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 288 tokens) : 11.2934 - top-1 accuracy : 17/288 = 5.9% - - [2026-04-29 12:26:04] iteration 54/ 100 | consumed samples: 216 | elapsed time per iteration (ms): 4997.0 | throughput (token/sec/GPU): 1296.0 | learning rate: 7.452684E-05 | global batch size: 4 | lm loss: 1.129387E+01 | loss scale: 1.0 | grad norm: 0.407 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 54 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 54 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 54 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1801.55, 1801.55) -Dataloader batch - tokens shape: torch.Size([1, 1544]) -Dataloader batch - labels shape: torch.Size([1, 1544]) -Dataloader batch - attn_mask shape: torch.Size([1, 1544]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 137 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1544) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1544) torch.int64 - loss_mask : 399 / 1544 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1544, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1544, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1544, 1, 576) torch.bfloat16 - out : (1, 1544) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.3217 - top-1 accuracy : 10/399 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1551]) -Dataloader batch - labels shape: torch.Size([1, 1551]) -Dataloader batch - attn_mask shape: torch.Size([1, 1551]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 138 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1551) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1551) torch.int64 - loss_mask : 348 / 1551 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1551, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1551, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1551, 1, 576) torch.bfloat16 - out : (1, 1551) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2158 - top-1 accuracy : 4/348 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1693]) -Dataloader batch - labels shape: torch.Size([1, 1693]) -Dataloader batch - attn_mask shape: torch.Size([1, 1693]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 139 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1693) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1693) torch.int64 - loss_mask : 346 / 1693 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1693, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1693, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1693, 1, 576) torch.bfloat16 - out : (1, 1693) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.2066 - top-1 accuracy : 25/346 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 140 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 313 / 1496 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2078 - top-1 accuracy : 19/313 = 6.1% - - [2026-04-29 12:26:11] iteration 55/ 100 | consumed samples: 220 | elapsed time per iteration (ms): 5076.3 | throughput (token/sec/GPU): 1237.9 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.124179E+01 | loss scale: 1.0 | grad norm: 0.409 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 55 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 55 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 55 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1848.34, 1848.34) -Dataloader batch - tokens shape: torch.Size([1, 1553]) -Dataloader batch - labels shape: torch.Size([1, 1553]) -Dataloader batch - attn_mask shape: torch.Size([1, 1553]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 141 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1553) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1553) torch.int64 - loss_mask : 349 / 1553 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1553, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1553, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1553, 1, 576) torch.bfloat16 - out : (1, 1553) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.1922 - top-1 accuracy : 4/349 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 919]) -Dataloader batch - labels shape: torch.Size([1, 919]) -Dataloader batch - attn_mask shape: torch.Size([1, 919]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 142 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 919) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 919) torch.int64 - loss_mask : 176 / 919 tokens (19.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (919, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (919, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (919, 1, 576) torch.bfloat16 - out : (1, 919) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 176 tokens) : 11.2305 - top-1 accuracy : 0/176 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1714]) -Dataloader batch - labels shape: torch.Size([1, 1714]) -Dataloader batch - attn_mask shape: torch.Size([1, 1714]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 143 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1714) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1714) torch.int64 - loss_mask : 443 / 1714 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1714, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1714, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1714, 1, 576) torch.bfloat16 - out : (1, 1714) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 443 tokens) : 11.3247 - top-1 accuracy : 33/443 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1798]) -Dataloader batch - labels shape: torch.Size([1, 1798]) -Dataloader batch - attn_mask shape: torch.Size([1, 1798]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 144 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1798) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1798) torch.int64 - loss_mask : 450 / 1798 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1798, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1798, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1798, 1, 576) torch.bfloat16 - out : (1, 1798) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 450 tokens) : 11.2235 - top-1 accuracy : 10/450 = 2.2% - - [2026-04-29 12:26:18] iteration 56/ 100 | consumed samples: 224 | elapsed time per iteration (ms): 4880.6 | throughput (token/sec/GPU): 1226.1 | learning rate: 7.175637E-05 | global batch size: 4 | lm loss: 1.124832E+01 | loss scale: 1.0 | grad norm: 0.458 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 56 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 56 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 56 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1555.05, 1555.05) -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 145 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 313 / 1487 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.1997 - top-1 accuracy : 24/313 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 146 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1418) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1418) torch.int64 - loss_mask : 342 / 1418 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1418, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1418, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1418, 1, 576) torch.bfloat16 - out : (1, 1418) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2825 - top-1 accuracy : 28/342 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1585]) -Dataloader batch - labels shape: torch.Size([1, 1585]) -Dataloader batch - attn_mask shape: torch.Size([1, 1585]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 147 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1585) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1585) torch.int64 - loss_mask : 379 / 1585 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1585, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1585, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1585, 1, 576) torch.bfloat16 - out : (1, 1585) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.3026 - top-1 accuracy : 13/379 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1836]) -Dataloader batch - labels shape: torch.Size([1, 1836]) -Dataloader batch - attn_mask shape: torch.Size([1, 1836]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 148 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1836) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1836) torch.int64 - loss_mask : 458 / 1836 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1836, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1836, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1836, 1, 576) torch.bfloat16 - out : (1, 1836) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 458 tokens) : 11.3086 - top-1 accuracy : 3/458 = 0.7% - - [2026-04-29 12:26:24] iteration 57/ 100 | consumed samples: 228 | elapsed time per iteration (ms): 5054.9 | throughput (token/sec/GPU): 1251.5 | learning rate: 7.033885E-05 | global batch size: 4 | lm loss: 1.127826E+01 | loss scale: 1.0 | grad norm: 0.364 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 57 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 57 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 57 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1588.37, 1588.37) -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 149 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 369 / 1520 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2710 - top-1 accuracy : 15/369 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1480]) -Dataloader batch - labels shape: torch.Size([1, 1480]) -Dataloader batch - attn_mask shape: torch.Size([1, 1480]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 150 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1480) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1480) torch.int64 - loss_mask : 429 / 1480 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1480, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1480, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1480, 1, 576) torch.bfloat16 - out : (1, 1480) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 429 tokens) : 11.3819 - top-1 accuracy : 13/429 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 151 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1696) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1696) torch.int64 - loss_mask : 344 / 1696 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1696, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1696, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1696, 1, 576) torch.bfloat16 - out : (1, 1696) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.1824 - top-1 accuracy : 33/344 = 9.6% - -Dataloader batch - tokens shape: torch.Size([1, 1584]) -Dataloader batch - labels shape: torch.Size([1, 1584]) -Dataloader batch - attn_mask shape: torch.Size([1, 1584]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 152 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1584) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1584) torch.int64 - loss_mask : 386 / 1584 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1584, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1584, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1584, 1, 576) torch.bfloat16 - out : (1, 1584) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2020 - top-1 accuracy : 15/386 = 3.9% - - [2026-04-29 12:26:31] iteration 58/ 100 | consumed samples: 232 | elapsed time per iteration (ms): 4872.6 | throughput (token/sec/GPU): 1288.8 | learning rate: 6.890167E-05 | global batch size: 4 | lm loss: 1.126478E+01 | loss scale: 1.0 | grad norm: 0.358 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 58 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 58 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 58 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1330.42, 1330.42) -Dataloader batch - tokens shape: torch.Size([1, 1433]) -Dataloader batch - labels shape: torch.Size([1, 1433]) -Dataloader batch - attn_mask shape: torch.Size([1, 1433]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 153 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1433) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1433) torch.int64 - loss_mask : 375 / 1433 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1433, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1433, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1433, 1, 576) torch.bfloat16 - out : (1, 1433) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.3021 - top-1 accuracy : 28/375 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 154 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1470) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1470) torch.int64 - loss_mask : 372 / 1470 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1470, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1470, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1470, 1, 576) torch.bfloat16 - out : (1, 1470) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2992 - top-1 accuracy : 18/372 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1704]) -Dataloader batch - labels shape: torch.Size([1, 1704]) -Dataloader batch - attn_mask shape: torch.Size([1, 1704]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 155 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1704) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1704) torch.int64 - loss_mask : 442 / 1704 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1704, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1704, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1704, 1, 576) torch.bfloat16 - out : (1, 1704) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 442 tokens) : 11.2934 - top-1 accuracy : 17/442 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1005]) -Dataloader batch - labels shape: torch.Size([1, 1005]) -Dataloader batch - attn_mask shape: torch.Size([1, 1005]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 156 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1005) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1005) torch.int64 - loss_mask : 257 / 1005 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1005, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1005, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1005, 1, 576) torch.bfloat16 - out : (1, 1005) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 257 tokens) : 11.2970 - top-1 accuracy : 1/257 = 0.4% - - [2026-04-29 12:26:37] iteration 59/ 100 | consumed samples: 236 | elapsed time per iteration (ms): 4449.2 | throughput (token/sec/GPU): 1261.3 | learning rate: 6.744626E-05 | global batch size: 4 | lm loss: 1.129778E+01 | loss scale: 1.0 | grad norm: 0.387 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 59 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 59 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 59 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1377.92, 1377.92) -Dataloader batch - tokens shape: torch.Size([1, 1264]) -Dataloader batch - labels shape: torch.Size([1, 1264]) -Dataloader batch - attn_mask shape: torch.Size([1, 1264]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 157 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1264) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1264) torch.int64 - loss_mask : 327 / 1264 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1264, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1264, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1264, 1, 576) torch.bfloat16 - out : (1, 1264) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.3138 - top-1 accuracy : 3/327 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 158 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 363 / 1722 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2114 - top-1 accuracy : 24/363 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1623]) -Dataloader batch - labels shape: torch.Size([1, 1623]) -Dataloader batch - attn_mask shape: torch.Size([1, 1623]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 159 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1623) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1623) torch.int64 - loss_mask : 400 / 1623 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1623, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1623, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1623, 1, 576) torch.bfloat16 - out : (1, 1623) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.2533 - top-1 accuracy : 19/400 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1473]) -Dataloader batch - labels shape: torch.Size([1, 1473]) -Dataloader batch - attn_mask shape: torch.Size([1, 1473]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 160 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1473) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1473) torch.int64 - loss_mask : 332 / 1473 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1473, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1473, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1473, 1, 576) torch.bfloat16 - out : (1, 1473) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.2439 - top-1 accuracy : 24/332 = 7.2% - - [2026-04-29 12:26:43] iteration 60/ 100 | consumed samples: 240 | elapsed time per iteration (ms): 4941.9 | throughput (token/sec/GPU): 1230.7 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.125433E+01 | loss scale: 1.0 | grad norm: 0.345 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4929.49, 4929.49) - forward-compute ................................: (3649.73, 3649.73) - backward-compute ...............................: (1277.09, 1277.09) - batch-generator ................................: (398.04, 398.04) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.47, 0.47) - params-all-gather ..............................: (0.14, 0.14) - optimizer-copy-to-main-grad ....................: (0.17, 0.17) - optimizer-clip-main-grad .......................: (0.51, 0.51) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.29, 1.29) - optimizer-copy-main-to-model-params ............: (0.20, 0.20) - optimizer ......................................: (2.89, 2.89) -saving checkpoint at iteration 60 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 60 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 60 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1218.71, 1218.71) -Dataloader batch - tokens shape: torch.Size([1, 1589]) -Dataloader batch - labels shape: torch.Size([1, 1589]) -Dataloader batch - attn_mask shape: torch.Size([1, 1589]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 161 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1589) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1589) torch.int64 - loss_mask : 350 / 1589 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1589, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1589, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1589, 1, 576) torch.bfloat16 - out : (1, 1589) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.2381 - top-1 accuracy : 25/350 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 162 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 312 / 1444 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.1615 - top-1 accuracy : 13/312 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1029]) -Dataloader batch - labels shape: torch.Size([1, 1029]) -Dataloader batch - attn_mask shape: torch.Size([1, 1029]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 163 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1029) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1029) torch.int64 - loss_mask : 258 / 1029 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1029, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1029, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1029, 1, 576) torch.bfloat16 - out : (1, 1029) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 258 tokens) : 11.2989 - top-1 accuracy : 5/258 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1514]) -Dataloader batch - labels shape: torch.Size([1, 1514]) -Dataloader batch - attn_mask shape: torch.Size([1, 1514]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 164 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1514) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1514) torch.int64 - loss_mask : 404 / 1514 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1514, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1514, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1514, 1, 576) torch.bfloat16 - out : (1, 1514) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.2849 - top-1 accuracy : 29/404 = 7.2% - - [2026-04-29 12:26:49] iteration 61/ 100 | consumed samples: 244 | elapsed time per iteration (ms): 4487.5 | throughput (token/sec/GPU): 1242.6 | learning rate: 6.448653E-05 | global batch size: 4 | lm loss: 1.124620E+01 | loss scale: 1.0 | grad norm: 0.349 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 61 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 61 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 61 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1658.85, 1658.85) -Dataloader batch - tokens shape: torch.Size([1, 1613]) -Dataloader batch - labels shape: torch.Size([1, 1613]) -Dataloader batch - attn_mask shape: torch.Size([1, 1613]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 165 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1613) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1613) torch.int64 - loss_mask : 358 / 1613 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1613, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1613, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1613, 1, 576) torch.bfloat16 - out : (1, 1613) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.1937 - top-1 accuracy : 18/358 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1387]) -Dataloader batch - labels shape: torch.Size([1, 1387]) -Dataloader batch - attn_mask shape: torch.Size([1, 1387]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 166 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1387) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1387) torch.int64 - loss_mask : 313 / 1387 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1387, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1387, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1387, 1, 576) torch.bfloat16 - out : (1, 1387) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2465 - top-1 accuracy : 27/313 = 8.6% - -Dataloader batch - tokens shape: torch.Size([1, 1736]) -Dataloader batch - labels shape: torch.Size([1, 1736]) -Dataloader batch - attn_mask shape: torch.Size([1, 1736]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 167 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1736) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1736) torch.int64 - loss_mask : 375 / 1736 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1736, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1736, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1736, 1, 576) torch.bfloat16 - out : (1, 1736) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.2209 - top-1 accuracy : 37/375 = 9.9% - -Dataloader batch - tokens shape: torch.Size([1, 1384]) -Dataloader batch - labels shape: torch.Size([1, 1384]) -Dataloader batch - attn_mask shape: torch.Size([1, 1384]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 168 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1384) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1384) torch.int64 - loss_mask : 320 / 1384 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1384, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1384, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1384, 1, 576) torch.bfloat16 - out : (1, 1384) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.3126 - top-1 accuracy : 29/320 = 9.1% - - [2026-04-29 12:26:55] iteration 62/ 100 | consumed samples: 248 | elapsed time per iteration (ms): 4990.7 | throughput (token/sec/GPU): 1226.3 | learning rate: 6.298514E-05 | global batch size: 4 | lm loss: 1.124112E+01 | loss scale: 1.0 | grad norm: 0.384 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 62 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 62 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 62 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1475.24, 1475.24) -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 169 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 301 / 1364 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 301 tokens) : 11.2146 - top-1 accuracy : 17/301 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 170 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 392 / 1508 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3217 - top-1 accuracy : 10/392 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 171 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 312 / 1386 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.1921 - top-1 accuracy : 27/312 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1730]) -Dataloader batch - labels shape: torch.Size([1, 1730]) -Dataloader batch - attn_mask shape: torch.Size([1, 1730]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 172 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1730) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1730) torch.int64 - loss_mask : 372 / 1730 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1730, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1730, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1730, 1, 576) torch.bfloat16 - out : (1, 1730) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2114 - top-1 accuracy : 21/372 = 5.6% - - [2026-04-29 12:27:02] iteration 63/ 100 | consumed samples: 252 | elapsed time per iteration (ms): 4916.0 | throughput (token/sec/GPU): 1218.1 | learning rate: 6.147138E-05 | global batch size: 4 | lm loss: 1.123913E+01 | loss scale: 1.0 | grad norm: 0.344 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 63 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 63 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 63 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1527.72, 1527.72) -Dataloader batch - tokens shape: torch.Size([1, 1088]) -Dataloader batch - labels shape: torch.Size([1, 1088]) -Dataloader batch - attn_mask shape: torch.Size([1, 1088]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 173 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1088) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1088) torch.int64 - loss_mask : 248 / 1088 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1088, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1088, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1088, 1, 576) torch.bfloat16 - out : (1, 1088) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 248 tokens) : 11.2151 - top-1 accuracy : 10/248 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 174 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 355 / 1542 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2937 - top-1 accuracy : 20/355 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1851]) -Dataloader batch - labels shape: torch.Size([1, 1851]) -Dataloader batch - attn_mask shape: torch.Size([1, 1851]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 175 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1851) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1851) torch.int64 - loss_mask : 498 / 1851 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1851, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1851, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1851, 1, 576) torch.bfloat16 - out : (1, 1851) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 498 tokens) : 11.3649 - top-1 accuracy : 27/498 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1703]) -Dataloader batch - labels shape: torch.Size([1, 1703]) -Dataloader batch - attn_mask shape: torch.Size([1, 1703]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 176 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1703) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1703) torch.int64 - loss_mask : 337 / 1703 tokens (19.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1703, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1703, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1703, 1, 576) torch.bfloat16 - out : (1, 1703) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.1673 - top-1 accuracy : 25/337 = 7.4% - - [2026-04-29 12:27:08] iteration 64/ 100 | consumed samples: 256 | elapsed time per iteration (ms): 4962.2 | throughput (token/sec/GPU): 1246.2 | learning rate: 5.994675E-05 | global batch size: 4 | lm loss: 1.127519E+01 | loss scale: 1.0 | grad norm: 0.338 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 64 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 64 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 64 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1431.42, 1431.42) -Dataloader batch - tokens shape: torch.Size([1, 1367]) -Dataloader batch - labels shape: torch.Size([1, 1367]) -Dataloader batch - attn_mask shape: torch.Size([1, 1367]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 177 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1367) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1367) torch.int64 - loss_mask : 302 / 1367 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1367, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1367, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1367, 1, 576) torch.bfloat16 - out : (1, 1367) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 302 tokens) : 11.2701 - top-1 accuracy : 22/302 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1355]) -Dataloader batch - labels shape: torch.Size([1, 1355]) -Dataloader batch - attn_mask shape: torch.Size([1, 1355]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 178 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1355) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1355) torch.int64 - loss_mask : 291 / 1355 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1355, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1355, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1355, 1, 576) torch.bfloat16 - out : (1, 1355) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 291 tokens) : 11.1990 - top-1 accuracy : 20/291 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 179 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 289 / 1364 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 289 tokens) : 11.2091 - top-1 accuracy : 23/289 = 8.0% - -Dataloader batch - tokens shape: torch.Size([1, 1869]) -Dataloader batch - labels shape: torch.Size([1, 1869]) -Dataloader batch - attn_mask shape: torch.Size([1, 1869]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 180 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1869) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1869) torch.int64 - loss_mask : 503 / 1869 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1869, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1869, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1869, 1, 576) torch.bfloat16 - out : (1, 1869) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 503 tokens) : 11.3424 - top-1 accuracy : 11/503 = 2.2% - - [2026-04-29 12:27:14] iteration 65/ 100 | consumed samples: 260 | elapsed time per iteration (ms): 4773.4 | throughput (token/sec/GPU): 1247.5 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.126869E+01 | loss scale: 1.0 | grad norm: 0.296 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 65 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 65 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 65 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1088.13, 1088.13) -Dataloader batch - tokens shape: torch.Size([1, 1568]) -Dataloader batch - labels shape: torch.Size([1, 1568]) -Dataloader batch - attn_mask shape: torch.Size([1, 1568]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 181 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1568) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1568) torch.int64 - loss_mask : 342 / 1568 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1568, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1568, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1568, 1, 576) torch.bfloat16 - out : (1, 1568) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2536 - top-1 accuracy : 22/342 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1113]) -Dataloader batch - labels shape: torch.Size([1, 1113]) -Dataloader batch - attn_mask shape: torch.Size([1, 1113]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 182 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1113) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1113) torch.int64 - loss_mask : 280 / 1113 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1113, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1113, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1113, 1, 576) torch.bfloat16 - out : (1, 1113) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 280 tokens) : 11.3231 - top-1 accuracy : 13/280 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 183 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 339 / 1436 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2133 - top-1 accuracy : 21/339 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 184 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 335 / 1518 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2662 - top-1 accuracy : 23/335 = 6.9% - - [2026-04-29 12:27:20] iteration 66/ 100 | consumed samples: 264 | elapsed time per iteration (ms): 4496.5 | throughput (token/sec/GPU): 1253.2 | learning rate: 5.687092E-05 | global batch size: 4 | lm loss: 1.126131E+01 | loss scale: 1.0 | grad norm: 0.391 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 66 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 66 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 66 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1469.97, 1469.97) -Dataloader batch - tokens shape: torch.Size([1, 1642]) -Dataloader batch - labels shape: torch.Size([1, 1642]) -Dataloader batch - attn_mask shape: torch.Size([1, 1642]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 185 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1642) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1642) torch.int64 - loss_mask : 345 / 1642 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1642, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1642, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1642, 1, 576) torch.bfloat16 - out : (1, 1642) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2212 - top-1 accuracy : 35/345 = 10.1% - -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 186 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 311 / 1496 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.1933 - top-1 accuracy : 35/311 = 11.3% - -Dataloader batch - tokens shape: torch.Size([1, 1566]) -Dataloader batch - labels shape: torch.Size([1, 1566]) -Dataloader batch - attn_mask shape: torch.Size([1, 1566]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 187 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1566) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1566) torch.int64 - loss_mask : 431 / 1566 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1566, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1566, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1566, 1, 576) torch.bfloat16 - out : (1, 1566) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.3286 - top-1 accuracy : 0/431 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1086]) -Dataloader batch - labels shape: torch.Size([1, 1086]) -Dataloader batch - attn_mask shape: torch.Size([1, 1086]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 188 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1086) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1086) torch.int64 - loss_mask : 321 / 1086 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1086, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1086, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1086, 1, 576) torch.bfloat16 - out : (1, 1086) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.4415 - top-1 accuracy : 3/321 = 0.9% - - [2026-04-29 12:27:26] iteration 67/ 100 | consumed samples: 268 | elapsed time per iteration (ms): 4538.4 | throughput (token/sec/GPU): 1275.8 | learning rate: 5.532277E-05 | global batch size: 4 | lm loss: 1.129814E+01 | loss scale: 1.0 | grad norm: 0.351 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 67 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 67 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 67 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1547.58, 1547.58) -Dataloader batch - tokens shape: torch.Size([1, 1720]) -Dataloader batch - labels shape: torch.Size([1, 1720]) -Dataloader batch - attn_mask shape: torch.Size([1, 1720]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 189 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1720) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1720) torch.int64 - loss_mask : 349 / 1720 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1720, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1720, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1720, 1, 576) torch.bfloat16 - out : (1, 1720) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2152 - top-1 accuracy : 29/349 = 8.3% - -Dataloader batch - tokens shape: torch.Size([1, 1864]) -Dataloader batch - labels shape: torch.Size([1, 1864]) -Dataloader batch - attn_mask shape: torch.Size([1, 1864]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 190 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1864) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1864) torch.int64 - loss_mask : 491 / 1864 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1864, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1864, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1864, 1, 576) torch.bfloat16 - out : (1, 1864) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 491 tokens) : 11.3263 - top-1 accuracy : 25/491 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1404]) -Dataloader batch - labels shape: torch.Size([1, 1404]) -Dataloader batch - attn_mask shape: torch.Size([1, 1404]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 191 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1404) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1404) torch.int64 - loss_mask : 343 / 1404 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1404, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1404, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1404, 1, 576) torch.bfloat16 - out : (1, 1404) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.3536 - top-1 accuracy : 14/343 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1193]) -Dataloader batch - labels shape: torch.Size([1, 1193]) -Dataloader batch - attn_mask shape: torch.Size([1, 1193]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 192 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1193) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1193) torch.int64 - loss_mask : 260 / 1193 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1193, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1193, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1193, 1, 576) torch.bfloat16 - out : (1, 1193) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.1832 - top-1 accuracy : 30/260 = 11.5% - - [2026-04-29 12:27:32] iteration 68/ 100 | consumed samples: 272 | elapsed time per iteration (ms): 4821.2 | throughput (token/sec/GPU): 1282.0 | learning rate: 5.376985E-05 | global batch size: 4 | lm loss: 1.128015E+01 | loss scale: 1.0 | grad norm: 0.412 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 68 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 68 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 68 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1440.58, 1440.58) -Dataloader batch - tokens shape: torch.Size([1, 1894]) -Dataloader batch - labels shape: torch.Size([1, 1894]) -Dataloader batch - attn_mask shape: torch.Size([1, 1894]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 193 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1894) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1894) torch.int64 - loss_mask : 545 / 1894 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1894, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1894, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1894, 1, 576) torch.bfloat16 - out : (1, 1894) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 545 tokens) : 11.3696 - top-1 accuracy : 28/545 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1185]) -Dataloader batch - labels shape: torch.Size([1, 1185]) -Dataloader batch - attn_mask shape: torch.Size([1, 1185]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 194 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1185) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1185) torch.int64 - loss_mask : 284 / 1185 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1185, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1185, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1185, 1, 576) torch.bfloat16 - out : (1, 1185) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.2731 - top-1 accuracy : 8/284 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1829]) -Dataloader batch - labels shape: torch.Size([1, 1829]) -Dataloader batch - attn_mask shape: torch.Size([1, 1829]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 195 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1829) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1829) torch.int64 - loss_mask : 519 / 1829 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1829, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1829, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1829, 1, 576) torch.bfloat16 - out : (1, 1829) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 519 tokens) : 11.3598 - top-1 accuracy : 31/519 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1779]) -Dataloader batch - labels shape: torch.Size([1, 1779]) -Dataloader batch - attn_mask shape: torch.Size([1, 1779]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 196 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1779) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1779) torch.int64 - loss_mask : 406 / 1779 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1779, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1779, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1779, 1, 576) torch.bfloat16 - out : (1, 1779) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.2497 - top-1 accuracy : 15/406 = 3.7% - - [2026-04-29 12:27:39] iteration 69/ 100 | consumed samples: 276 | elapsed time per iteration (ms): 4982.9 | throughput (token/sec/GPU): 1342.0 | learning rate: 5.221368E-05 | global batch size: 4 | lm loss: 1.132334E+01 | loss scale: 1.0 | grad norm: 0.381 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 69 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 69 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 69 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1098.14, 1098.14) -Dataloader batch - tokens shape: torch.Size([1, 1485]) -Dataloader batch - labels shape: torch.Size([1, 1485]) -Dataloader batch - attn_mask shape: torch.Size([1, 1485]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 197 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1485) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1485) torch.int64 - loss_mask : 328 / 1485 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1485, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1485, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1485, 1, 576) torch.bfloat16 - out : (1, 1485) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.1742 - top-1 accuracy : 31/328 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1235]) -Dataloader batch - labels shape: torch.Size([1, 1235]) -Dataloader batch - attn_mask shape: torch.Size([1, 1235]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 198 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1235) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1235) torch.int64 - loss_mask : 295 / 1235 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1235, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1235, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1235, 1, 576) torch.bfloat16 - out : (1, 1235) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.2243 - top-1 accuracy : 17/295 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1448]) -Dataloader batch - labels shape: torch.Size([1, 1448]) -Dataloader batch - attn_mask shape: torch.Size([1, 1448]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 199 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1448) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1448) torch.int64 - loss_mask : 415 / 1448 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1448, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1448, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1448, 1, 576) torch.bfloat16 - out : (1, 1448) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.3514 - top-1 accuracy : 14/415 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1462]) -Dataloader batch - labels shape: torch.Size([1, 1462]) -Dataloader batch - attn_mask shape: torch.Size([1, 1462]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 200 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1462) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1462) torch.int64 - loss_mask : 314 / 1462 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1462, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1462, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1462, 1, 576) torch.bfloat16 - out : (1, 1462) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.2034 - top-1 accuracy : 22/314 = 7.0% - - [2026-04-29 12:27:44] iteration 70/ 100 | consumed samples: 280 | elapsed time per iteration (ms): 4601.4 | throughput (token/sec/GPU): 1223.5 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.124630E+01 | loss scale: 1.0 | grad norm: 0.378 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 70 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 70 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 70 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1313.10, 1313.10) -Dataloader batch - tokens shape: torch.Size([1, 1482]) -Dataloader batch - labels shape: torch.Size([1, 1482]) -Dataloader batch - attn_mask shape: torch.Size([1, 1482]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 201 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1482) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1482) torch.int64 - loss_mask : 322 / 1482 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1482, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1482, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1482, 1, 576) torch.bfloat16 - out : (1, 1482) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.2092 - top-1 accuracy : 14/322 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1589]) -Dataloader batch - labels shape: torch.Size([1, 1589]) -Dataloader batch - attn_mask shape: torch.Size([1, 1589]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 202 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1589) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1589) torch.int64 - loss_mask : 372 / 1589 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1589, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1589, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1589, 1, 576) torch.bfloat16 - out : (1, 1589) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2545 - top-1 accuracy : 30/372 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1706]) -Dataloader batch - labels shape: torch.Size([1, 1706]) -Dataloader batch - attn_mask shape: torch.Size([1, 1706]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 203 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1706) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1706) torch.int64 - loss_mask : 361 / 1706 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1706, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1706, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1706, 1, 576) torch.bfloat16 - out : (1, 1706) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.2415 - top-1 accuracy : 39/361 = 10.8% - -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 204 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 355 / 1520 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2564 - top-1 accuracy : 28/355 = 7.9% - - [2026-04-29 12:27:51] iteration 71/ 100 | consumed samples: 284 | elapsed time per iteration (ms): 4959.8 | throughput (token/sec/GPU): 1269.6 | learning rate: 4.909780E-05 | global batch size: 4 | lm loss: 1.124132E+01 | loss scale: 1.0 | grad norm: 0.359 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 71 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 71 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 71 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1108.27, 1108.27) -Dataloader batch - tokens shape: torch.Size([1, 972]) -Dataloader batch - labels shape: torch.Size([1, 972]) -Dataloader batch - attn_mask shape: torch.Size([1, 972]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 205 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 972) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 972) torch.int64 - loss_mask : 231 / 972 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (972, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (972, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (972, 1, 576) torch.bfloat16 - out : (1, 972) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 231 tokens) : 11.2373 - top-1 accuracy : 8/231 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1462]) -Dataloader batch - labels shape: torch.Size([1, 1462]) -Dataloader batch - attn_mask shape: torch.Size([1, 1462]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 206 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1462) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1462) torch.int64 - loss_mask : 392 / 1462 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1462, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1462, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1462, 1, 576) torch.bfloat16 - out : (1, 1462) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.2885 - top-1 accuracy : 42/392 = 10.7% - -Dataloader batch - tokens shape: torch.Size([1, 1658]) -Dataloader batch - labels shape: torch.Size([1, 1658]) -Dataloader batch - attn_mask shape: torch.Size([1, 1658]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 207 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1658) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1658) torch.int64 - loss_mask : 384 / 1658 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1658, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1658, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1658, 1, 576) torch.bfloat16 - out : (1, 1658) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.2922 - top-1 accuracy : 25/384 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 208 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1696) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1696) torch.int64 - loss_mask : 333 / 1696 tokens (19.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1696, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1696, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1696, 1, 576) torch.bfloat16 - out : (1, 1696) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.1868 - top-1 accuracy : 34/333 = 10.2% - - [2026-04-29 12:27:56] iteration 72/ 100 | consumed samples: 288 | elapsed time per iteration (ms): 4641.4 | throughput (token/sec/GPU): 1247.0 | learning rate: 4.754118E-05 | global batch size: 4 | lm loss: 1.125546E+01 | loss scale: 1.0 | grad norm: 0.422 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 72 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 72 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 72 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1136.22, 1136.22) -Dataloader batch - tokens shape: torch.Size([1, 1067]) -Dataloader batch - labels shape: torch.Size([1, 1067]) -Dataloader batch - attn_mask shape: torch.Size([1, 1067]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 209 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1067) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1067) torch.int64 - loss_mask : 282 / 1067 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1067, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1067, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1067, 1, 576) torch.bfloat16 - out : (1, 1067) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 282 tokens) : 11.3141 - top-1 accuracy : 7/282 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 210 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 398 / 1487 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.3548 - top-1 accuracy : 26/398 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1877]) -Dataloader batch - labels shape: torch.Size([1, 1877]) -Dataloader batch - attn_mask shape: torch.Size([1, 1877]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 211 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1877) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1877) torch.int64 - loss_mask : 519 / 1877 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1877, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1877, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1877, 1, 576) torch.bfloat16 - out : (1, 1877) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 519 tokens) : 11.3302 - top-1 accuracy : 17/519 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 798]) -Dataloader batch - labels shape: torch.Size([1, 798]) -Dataloader batch - attn_mask shape: torch.Size([1, 798]) -Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) - -==================================================================== - PIPELINE — STEP 212 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 798) torch.int64 - images : (16, 3, 1024, 1024) torch.bfloat16 - labels : (1, 798) torch.int64 - loss_mask : 139 / 798 tokens (17.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (16, 3, 1024, 1024) torch.bfloat16 - out : (16, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (16, 3072) - out : (16, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (798, 1, 576) torch.bfloat16 grad=False - image slots : 16 tokens replaced with vision embeddings - combined : (798, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (798, 1, 576) torch.bfloat16 - out : (1, 798) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 139 tokens) : 11.0157 - top-1 accuracy : 4/139 = 2.9% - - [2026-04-29 12:28:02] iteration 73/ 100 | consumed samples: 292 | elapsed time per iteration (ms): 4262.1 | throughput (token/sec/GPU): 1226.9 | learning rate: 4.598748E-05 | global batch size: 4 | lm loss: 1.130146E+01 | loss scale: 1.0 | grad norm: 0.397 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 73 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 73 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 73 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1702.11, 1702.11) -Dataloader batch - tokens shape: torch.Size([1, 1910]) -Dataloader batch - labels shape: torch.Size([1, 1910]) -Dataloader batch - attn_mask shape: torch.Size([1, 1910]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 213 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1910) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1910) torch.int64 - loss_mask : 552 / 1910 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1910, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1910, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1910, 1, 576) torch.bfloat16 - out : (1, 1910) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 552 tokens) : 11.3756 - top-1 accuracy : 20/552 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1461]) -Dataloader batch - labels shape: torch.Size([1, 1461]) -Dataloader batch - attn_mask shape: torch.Size([1, 1461]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 214 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1461) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1461) torch.int64 - loss_mask : 304 / 1461 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1461, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1461, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1461, 1, 576) torch.bfloat16 - out : (1, 1461) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 304 tokens) : 11.1184 - top-1 accuracy : 33/304 = 10.9% - -Dataloader batch - tokens shape: torch.Size([1, 1548]) -Dataloader batch - labels shape: torch.Size([1, 1548]) -Dataloader batch - attn_mask shape: torch.Size([1, 1548]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 215 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1548) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1548) torch.int64 - loss_mask : 406 / 1548 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1548, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1548, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1548, 1, 576) torch.bfloat16 - out : (1, 1548) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.3065 - top-1 accuracy : 19/406 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 216 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 426 / 1552 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.3406 - top-1 accuracy : 15/426 = 3.5% - - [2026-04-29 12:28:08] iteration 74/ 100 | consumed samples: 296 | elapsed time per iteration (ms): 4903.4 | throughput (token/sec/GPU): 1319.7 | learning rate: 4.443826E-05 | global batch size: 4 | lm loss: 1.130385E+01 | loss scale: 1.0 | grad norm: 0.273 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 74 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 74 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 74 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1279.16, 1279.16) -Dataloader batch - tokens shape: torch.Size([1, 1758]) -Dataloader batch - labels shape: torch.Size([1, 1758]) -Dataloader batch - attn_mask shape: torch.Size([1, 1758]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 217 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1758) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1758) torch.int64 - loss_mask : 409 / 1758 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1758, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1758, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1758, 1, 576) torch.bfloat16 - out : (1, 1758) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.2733 - top-1 accuracy : 33/409 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1129]) -Dataloader batch - labels shape: torch.Size([1, 1129]) -Dataloader batch - attn_mask shape: torch.Size([1, 1129]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 218 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1129) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1129) torch.int64 - loss_mask : 324 / 1129 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1129, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1129, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1129, 1, 576) torch.bfloat16 - out : (1, 1129) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.3479 - top-1 accuracy : 13/324 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 219 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 447 / 1475 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 447 tokens) : 11.4263 - top-1 accuracy : 5/447 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 220 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 420 / 1388 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 420 tokens) : 11.3947 - top-1 accuracy : 13/420 = 3.1% - - [2026-04-29 12:28:14] iteration 75/ 100 | consumed samples: 300 | elapsed time per iteration (ms): 4372.6 | throughput (token/sec/GPU): 1315.0 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.136302E+01 | loss scale: 1.0 | grad norm: 0.371 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 75 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 75 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 75 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1071.20, 1071.20) -Dataloader batch - tokens shape: torch.Size([1, 893]) -Dataloader batch - labels shape: torch.Size([1, 893]) -Dataloader batch - attn_mask shape: torch.Size([1, 893]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 221 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 893) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 893) torch.int64 - loss_mask : 200 / 893 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (893, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (893, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (893, 1, 576) torch.bfloat16 - out : (1, 893) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 200 tokens) : 11.2081 - top-1 accuracy : 8/200 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1165]) -Dataloader batch - labels shape: torch.Size([1, 1165]) -Dataloader batch - attn_mask shape: torch.Size([1, 1165]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 222 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1165) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1165) torch.int64 - loss_mask : 341 / 1165 tokens (29.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1165, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1165, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1165, 1, 576) torch.bfloat16 - out : (1, 1165) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.4365 - top-1 accuracy : 2/341 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 223 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1191) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1191) torch.int64 - loss_mask : 308 / 1191 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1191, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1191, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1191, 1, 576) torch.bfloat16 - out : (1, 1191) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.3200 - top-1 accuracy : 19/308 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1177]) -Dataloader batch - labels shape: torch.Size([1, 1177]) -Dataloader batch - attn_mask shape: torch.Size([1, 1177]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 224 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1177) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1177) torch.int64 - loss_mask : 254 / 1177 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1177, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1177, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1177, 1, 576) torch.bfloat16 - out : (1, 1177) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 254 tokens) : 11.1702 - top-1 accuracy : 18/254 = 7.1% - - [2026-04-29 12:28:19] iteration 76/ 100 | consumed samples: 304 | elapsed time per iteration (ms): 3588.8 | throughput (token/sec/GPU): 1233.3 | learning rate: 4.135936E-05 | global batch size: 4 | lm loss: 1.130125E+01 | loss scale: 1.0 | grad norm: 0.328 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 76 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 76 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 76 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1501.63, 1501.63) -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 225 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 366 / 1541 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2456 - top-1 accuracy : 32/366 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 226 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1420) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1420) torch.int64 - loss_mask : 334 / 1420 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1420, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1420, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1420, 1, 576) torch.bfloat16 - out : (1, 1420) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.2452 - top-1 accuracy : 27/334 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1054]) -Dataloader batch - labels shape: torch.Size([1, 1054]) -Dataloader batch - attn_mask shape: torch.Size([1, 1054]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 227 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1054) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1054) torch.int64 - loss_mask : 225 / 1054 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1054, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1054, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1054, 1, 576) torch.bfloat16 - out : (1, 1054) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 225 tokens) : 11.1657 - top-1 accuracy : 7/225 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1128]) -Dataloader batch - labels shape: torch.Size([1, 1128]) -Dataloader batch - attn_mask shape: torch.Size([1, 1128]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 228 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1128) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1128) torch.int64 - loss_mask : 327 / 1128 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1128, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1128, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1128, 1, 576) torch.bfloat16 - out : (1, 1128) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.3652 - top-1 accuracy : 11/327 = 3.4% - - [2026-04-29 12:28:24] iteration 77/ 100 | consumed samples: 308 | elapsed time per iteration (ms): 4186.2 | throughput (token/sec/GPU): 1228.6 | learning rate: 3.983273E-05 | global batch size: 4 | lm loss: 1.126238E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 77 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 77 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 77 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2038.45, 2038.45) -Dataloader batch - tokens shape: torch.Size([1, 957]) -Dataloader batch - labels shape: torch.Size([1, 957]) -Dataloader batch - attn_mask shape: torch.Size([1, 957]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 229 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 957) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 957) torch.int64 - loss_mask : 226 / 957 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (957, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (957, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (957, 1, 576) torch.bfloat16 - out : (1, 957) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 226 tokens) : 11.2447 - top-1 accuracy : 14/226 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1368]) -Dataloader batch - labels shape: torch.Size([1, 1368]) -Dataloader batch - attn_mask shape: torch.Size([1, 1368]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 230 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1368) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1368) torch.int64 - loss_mask : 319 / 1368 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1368, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1368, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1368, 1, 576) torch.bfloat16 - out : (1, 1368) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.2376 - top-1 accuracy : 7/319 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1735]) -Dataloader batch - labels shape: torch.Size([1, 1735]) -Dataloader batch - attn_mask shape: torch.Size([1, 1735]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 231 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1735) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1735) torch.int64 - loss_mask : 393 / 1735 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1735, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1735, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1735, 1, 576) torch.bfloat16 - out : (1, 1735) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 393 tokens) : 11.2205 - top-1 accuracy : 25/393 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1391]) -Dataloader batch - labels shape: torch.Size([1, 1391]) -Dataloader batch - attn_mask shape: torch.Size([1, 1391]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 232 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1391) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1391) torch.int64 - loss_mask : 334 / 1391 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1391, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1391, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1391, 1, 576) torch.bfloat16 - out : (1, 1391) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.2543 - top-1 accuracy : 9/334 = 2.7% - - [2026-04-29 12:28:31] iteration 78/ 100 | consumed samples: 312 | elapsed time per iteration (ms): 4391.0 | throughput (token/sec/GPU): 1241.4 | learning rate: 3.831667E-05 | global batch size: 4 | lm loss: 1.123792E+01 | loss scale: 1.0 | grad norm: 0.359 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 78 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 78 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 78 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1822.49, 1822.49) -Dataloader batch - tokens shape: torch.Size([1, 1650]) -Dataloader batch - labels shape: torch.Size([1, 1650]) -Dataloader batch - attn_mask shape: torch.Size([1, 1650]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 233 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1650) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1650) torch.int64 - loss_mask : 391 / 1650 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1650, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1650, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1650, 1, 576) torch.bfloat16 - out : (1, 1650) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2937 - top-1 accuracy : 26/391 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 234 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 348 / 1444 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2758 - top-1 accuracy : 18/348 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 235 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 290 / 1380 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.1802 - top-1 accuracy : 28/290 = 9.7% - -Dataloader batch - tokens shape: torch.Size([1, 1222]) -Dataloader batch - labels shape: torch.Size([1, 1222]) -Dataloader batch - attn_mask shape: torch.Size([1, 1222]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 236 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1222) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1222) torch.int64 - loss_mask : 328 / 1222 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1222, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1222, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1222, 1, 576) torch.bfloat16 - out : (1, 1222) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.3261 - top-1 accuracy : 11/328 = 3.4% - - [2026-04-29 12:28:37] iteration 79/ 100 | consumed samples: 316 | elapsed time per iteration (ms): 4710.6 | throughput (token/sec/GPU): 1209.2 | learning rate: 3.681269E-05 | global batch size: 4 | lm loss: 1.127269E+01 | loss scale: 1.0 | grad norm: 0.346 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 79 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 79 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 79 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1816.49, 1816.49) -Dataloader batch - tokens shape: torch.Size([1, 1161]) -Dataloader batch - labels shape: torch.Size([1, 1161]) -Dataloader batch - attn_mask shape: torch.Size([1, 1161]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 237 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1161) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1161) torch.int64 - loss_mask : 322 / 1161 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1161, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1161, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1161, 1, 576) torch.bfloat16 - out : (1, 1161) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.3296 - top-1 accuracy : 7/322 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 238 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 321 / 1374 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2287 - top-1 accuracy : 15/321 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 239 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 297 / 1422 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.1447 - top-1 accuracy : 8/297 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1133]) -Dataloader batch - labels shape: torch.Size([1, 1133]) -Dataloader batch - attn_mask shape: torch.Size([1, 1133]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 240 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1133) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1133) torch.int64 - loss_mask : 324 / 1133 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1133, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1133, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1133, 1, 576) torch.bfloat16 - out : (1, 1133) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.3818 - top-1 accuracy : 3/324 = 0.9% - - [2026-04-29 12:28:44] iteration 80/ 100 | consumed samples: 320 | elapsed time per iteration (ms): 4800.9 | throughput (token/sec/GPU): 1060.2 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.127392E+01 | loss scale: 1.0 | grad norm: 0.367 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4778.70, 4778.70) - forward-compute ................................: (3554.25, 3554.25) - backward-compute ...............................: (1220.09, 1220.09) - batch-generator ................................: (388.68, 388.68) - layernorm-grads-all-reduce .....................: (0.07, 0.07) - embedding-grads-all-reduce .....................: (0.11, 0.11) - all-grads-sync .................................: (1.30, 1.30) - params-all-gather ..............................: (0.42, 0.42) - optimizer-copy-to-main-grad ....................: (0.42, 0.42) - optimizer-clip-main-grad .......................: (1.46, 1.46) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.62, 2.62) - optimizer-copy-main-to-model-params ............: (0.63, 0.63) - optimizer ......................................: (7.71, 7.71) -saving checkpoint at iteration 80 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 80 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 80 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1855.26, 1855.26) -Dataloader batch - tokens shape: torch.Size([1, 957]) -Dataloader batch - labels shape: torch.Size([1, 957]) -Dataloader batch - attn_mask shape: torch.Size([1, 957]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 241 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 957) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 957) torch.int64 - loss_mask : 209 / 957 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (957, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (957, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (957, 1, 576) torch.bfloat16 - out : (1, 957) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 209 tokens) : 11.1600 - top-1 accuracy : 15/209 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1194]) -Dataloader batch - labels shape: torch.Size([1, 1194]) -Dataloader batch - attn_mask shape: torch.Size([1, 1194]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 242 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1194) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1194) torch.int64 - loss_mask : 288 / 1194 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1194, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1194, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1194, 1, 576) torch.bfloat16 - out : (1, 1194) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 288 tokens) : 11.2588 - top-1 accuracy : 17/288 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 985]) -Dataloader batch - labels shape: torch.Size([1, 985]) -Dataloader batch - attn_mask shape: torch.Size([1, 985]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 243 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 985) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 985) torch.int64 - loss_mask : 224 / 985 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (985, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (985, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (985, 1, 576) torch.bfloat16 - out : (1, 985) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 224 tokens) : 11.2230 - top-1 accuracy : 16/224 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1103]) -Dataloader batch - labels shape: torch.Size([1, 1103]) -Dataloader batch - attn_mask shape: torch.Size([1, 1103]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 244 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1103) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1103) torch.int64 - loss_mask : 296 / 1103 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1103, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1103, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1103, 1, 576) torch.bfloat16 - out : (1, 1103) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 296 tokens) : 11.3725 - top-1 accuracy : 14/296 = 4.7% - - [2026-04-29 12:28:50] iteration 81/ 100 | consumed samples: 324 | elapsed time per iteration (ms): 3694.8 | throughput (token/sec/GPU): 1147.3 | learning rate: 3.384688E-05 | global batch size: 4 | lm loss: 1.126372E+01 | loss scale: 1.0 | grad norm: 0.370 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 81 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 81 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 81 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1849.86, 1849.86) -Dataloader batch - tokens shape: torch.Size([1, 1645]) -Dataloader batch - labels shape: torch.Size([1, 1645]) -Dataloader batch - attn_mask shape: torch.Size([1, 1645]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 245 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1645) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1645) torch.int64 - loss_mask : 394 / 1645 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1645, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1645, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1645, 1, 576) torch.bfloat16 - out : (1, 1645) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.2224 - top-1 accuracy : 10/394 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1700]) -Dataloader batch - labels shape: torch.Size([1, 1700]) -Dataloader batch - attn_mask shape: torch.Size([1, 1700]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 246 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1700) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1700) torch.int64 - loss_mask : 351 / 1700 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1700, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1700, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1700, 1, 576) torch.bfloat16 - out : (1, 1700) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.1766 - top-1 accuracy : 30/351 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1366]) -Dataloader batch - labels shape: torch.Size([1, 1366]) -Dataloader batch - attn_mask shape: torch.Size([1, 1366]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 247 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1366) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1366) torch.int64 - loss_mask : 296 / 1366 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1366, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1366, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1366, 1, 576) torch.bfloat16 - out : (1, 1366) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 296 tokens) : 11.1789 - top-1 accuracy : 23/296 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 248 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 284 / 1364 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.1739 - top-1 accuracy : 27/284 = 9.5% - - [2026-04-29 12:28:57] iteration 82/ 100 | consumed samples: 328 | elapsed time per iteration (ms): 5080.6 | throughput (token/sec/GPU): 1195.7 | learning rate: 3.238799E-05 | global batch size: 4 | lm loss: 1.119016E+01 | loss scale: 1.0 | grad norm: 0.350 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 82 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 82 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 82 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1661.16, 1661.16) -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 249 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 343 / 1492 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.2033 - top-1 accuracy : 12/343 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1909]) -Dataloader batch - labels shape: torch.Size([1, 1909]) -Dataloader batch - attn_mask shape: torch.Size([1, 1909]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 250 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1909) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1909) torch.int64 - loss_mask : 551 / 1909 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1909, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1909, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1909, 1, 576) torch.bfloat16 - out : (1, 1909) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 551 tokens) : 11.3749 - top-1 accuracy : 19/551 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1838]) -Dataloader batch - labels shape: torch.Size([1, 1838]) -Dataloader batch - attn_mask shape: torch.Size([1, 1838]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 251 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1838) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1838) torch.int64 - loss_mask : 468 / 1838 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1838, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1838, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1838, 1, 576) torch.bfloat16 - out : (1, 1838) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 468 tokens) : 11.3236 - top-1 accuracy : 5/468 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1126]) -Dataloader batch - labels shape: torch.Size([1, 1126]) -Dataloader batch - attn_mask shape: torch.Size([1, 1126]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 252 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1126) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1126) torch.int64 - loss_mask : 320 / 1126 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1126, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1126, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1126, 1, 576) torch.bfloat16 - out : (1, 1126) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.3924 - top-1 accuracy : 11/320 = 3.4% - - [2026-04-29 12:29:04] iteration 83/ 100 | consumed samples: 332 | elapsed time per iteration (ms): 6265.7 | throughput (token/sec/GPU): 1015.8 | learning rate: 3.094706E-05 | global batch size: 4 | lm loss: 1.132897E+01 | loss scale: 1.0 | grad norm: 0.308 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 83 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 83 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 83 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1423.14, 1423.14) -Dataloader batch - tokens shape: torch.Size([1, 1416]) -Dataloader batch - labels shape: torch.Size([1, 1416]) -Dataloader batch - attn_mask shape: torch.Size([1, 1416]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 253 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1416) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1416) torch.int64 - loss_mask : 353 / 1416 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1416, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1416, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1416, 1, 576) torch.bfloat16 - out : (1, 1416) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.3436 - top-1 accuracy : 18/353 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 254 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 337 / 1507 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.2060 - top-1 accuracy : 30/337 = 8.9% - -Dataloader batch - tokens shape: torch.Size([1, 1932]) -Dataloader batch - labels shape: torch.Size([1, 1932]) -Dataloader batch - attn_mask shape: torch.Size([1, 1932]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 255 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1932) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1932) torch.int64 - loss_mask : 561 / 1932 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1932, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1932, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1932, 1, 576) torch.bfloat16 - out : (1, 1932) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 561 tokens) : 11.3814 - top-1 accuracy : 4/561 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1512]) -Dataloader batch - labels shape: torch.Size([1, 1512]) -Dataloader batch - attn_mask shape: torch.Size([1, 1512]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 256 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1512) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1512) torch.int64 - loss_mask : 373 / 1512 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1512, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1512, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1512, 1, 576) torch.bfloat16 - out : (1, 1512) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.2547 - top-1 accuracy : 23/373 = 6.2% - - [2026-04-29 12:29:12] iteration 84/ 100 | consumed samples: 336 | elapsed time per iteration (ms): 5891.0 | throughput (token/sec/GPU): 1080.8 | learning rate: 2.952549E-05 | global batch size: 4 | lm loss: 1.130770E+01 | loss scale: 1.0 | grad norm: 0.353 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 84 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 84 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 84 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1859.02, 1859.02) -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 257 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 347 / 1542 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2385 - top-1 accuracy : 19/347 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1528]) -Dataloader batch - labels shape: torch.Size([1, 1528]) -Dataloader batch - attn_mask shape: torch.Size([1, 1528]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 258 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1528) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1528) torch.int64 - loss_mask : 370 / 1528 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1528, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1528, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1528, 1, 576) torch.bfloat16 - out : (1, 1528) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.2657 - top-1 accuracy : 15/370 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1655]) -Dataloader batch - labels shape: torch.Size([1, 1655]) -Dataloader batch - attn_mask shape: torch.Size([1, 1655]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 259 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1655) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1655) torch.int64 - loss_mask : 380 / 1655 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1655, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1655, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1655, 1, 576) torch.bfloat16 - out : (1, 1655) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 380 tokens) : 11.2787 - top-1 accuracy : 25/380 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1867]) -Dataloader batch - labels shape: torch.Size([1, 1867]) -Dataloader batch - attn_mask shape: torch.Size([1, 1867]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 260 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1867) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1867) torch.int64 - loss_mask : 548 / 1867 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1867, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1867, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1867, 1, 576) torch.bfloat16 - out : (1, 1867) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 548 tokens) : 11.4075 - top-1 accuracy : 5/548 = 0.9% - - [2026-04-29 12:29:19] iteration 85/ 100 | consumed samples: 340 | elapsed time per iteration (ms): 5600.4 | throughput (token/sec/GPU): 1177.1 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.131021E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 85 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 85 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 85 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1274.96, 1274.96) -Dataloader batch - tokens shape: torch.Size([1, 1458]) -Dataloader batch - labels shape: torch.Size([1, 1458]) -Dataloader batch - attn_mask shape: torch.Size([1, 1458]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 261 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1458) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1458) torch.int64 - loss_mask : 310 / 1458 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1458, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1458, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1458, 1, 576) torch.bfloat16 - out : (1, 1458) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 310 tokens) : 11.1994 - top-1 accuracy : 13/310 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 945]) -Dataloader batch - labels shape: torch.Size([1, 945]) -Dataloader batch - attn_mask shape: torch.Size([1, 945]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 262 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 945) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 945) torch.int64 - loss_mask : 211 / 945 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (945, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (945, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (945, 1, 576) torch.bfloat16 - out : (1, 945) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 211 tokens) : 11.1300 - top-1 accuracy : 3/211 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1654]) -Dataloader batch - labels shape: torch.Size([1, 1654]) -Dataloader batch - attn_mask shape: torch.Size([1, 1654]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 263 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1654) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1654) torch.int64 - loss_mask : 409 / 1654 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1654, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1654, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1654, 1, 576) torch.bfloat16 - out : (1, 1654) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.2682 - top-1 accuracy : 32/409 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1177]) -Dataloader batch - labels shape: torch.Size([1, 1177]) -Dataloader batch - attn_mask shape: torch.Size([1, 1177]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 264 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1177) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1177) torch.int64 - loss_mask : 263 / 1177 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1177, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1177, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1177, 1, 576) torch.bfloat16 - out : (1, 1177) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.2061 - top-1 accuracy : 16/263 = 6.1% - - [2026-04-29 12:29:25] iteration 86/ 100 | consumed samples: 344 | elapsed time per iteration (ms): 4379.3 | throughput (token/sec/GPU): 1195.2 | learning rate: 2.674610E-05 | global batch size: 4 | lm loss: 1.121220E+01 | loss scale: 1.0 | grad norm: 0.376 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 86 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 86 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 86 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1554.00, 1554.00) -Dataloader batch - tokens shape: torch.Size([1, 1417]) -Dataloader batch - labels shape: torch.Size([1, 1417]) -Dataloader batch - attn_mask shape: torch.Size([1, 1417]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 265 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1417) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1417) torch.int64 - loss_mask : 401 / 1417 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1417, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1417, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1417, 1, 576) torch.bfloat16 - out : (1, 1417) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 401 tokens) : 11.3728 - top-1 accuracy : 4/401 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 974]) -Dataloader batch - labels shape: torch.Size([1, 974]) -Dataloader batch - attn_mask shape: torch.Size([1, 974]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 266 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 974) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 974) torch.int64 - loss_mask : 229 / 974 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (974, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (974, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (974, 1, 576) torch.bfloat16 - out : (1, 974) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 229 tokens) : 11.2548 - top-1 accuracy : 3/229 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1371]) -Dataloader batch - labels shape: torch.Size([1, 1371]) -Dataloader batch - attn_mask shape: torch.Size([1, 1371]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 267 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1371) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1371) torch.int64 - loss_mask : 324 / 1371 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1371, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1371, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1371, 1, 576) torch.bfloat16 - out : (1, 1371) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.2560 - top-1 accuracy : 24/324 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1781]) -Dataloader batch - labels shape: torch.Size([1, 1781]) -Dataloader batch - attn_mask shape: torch.Size([1, 1781]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 268 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1781) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1781) torch.int64 - loss_mask : 401 / 1781 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1781, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1781, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1781, 1, 576) torch.bfloat16 - out : (1, 1781) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 401 tokens) : 11.2737 - top-1 accuracy : 23/401 = 5.7% - - [2026-04-29 12:29:31] iteration 87/ 100 | consumed samples: 348 | elapsed time per iteration (ms): 4389.7 | throughput (token/sec/GPU): 1262.7 | learning rate: 2.539102E-05 | global batch size: 4 | lm loss: 1.129560E+01 | loss scale: 1.0 | grad norm: 0.322 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 87 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 87 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 87 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1476.73, 1476.73) -Dataloader batch - tokens shape: torch.Size([1, 1333]) -Dataloader batch - labels shape: torch.Size([1, 1333]) -Dataloader batch - attn_mask shape: torch.Size([1, 1333]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 269 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1333) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1333) torch.int64 - loss_mask : 371 / 1333 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1333, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1333, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1333, 1, 576) torch.bfloat16 - out : (1, 1333) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.3141 - top-1 accuracy : 15/371 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1441]) -Dataloader batch - labels shape: torch.Size([1, 1441]) -Dataloader batch - attn_mask shape: torch.Size([1, 1441]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 270 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1441) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1441) torch.int64 - loss_mask : 389 / 1441 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1441, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1441, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1441, 1, 576) torch.bfloat16 - out : (1, 1441) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.2916 - top-1 accuracy : 25/389 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1172]) -Dataloader batch - labels shape: torch.Size([1, 1172]) -Dataloader batch - attn_mask shape: torch.Size([1, 1172]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 271 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1172) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1172) torch.int64 - loss_mask : 312 / 1172 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1172, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1172, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1172, 1, 576) torch.bfloat16 - out : (1, 1172) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.3251 - top-1 accuracy : 8/312 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1839]) -Dataloader batch - labels shape: torch.Size([1, 1839]) -Dataloader batch - attn_mask shape: torch.Size([1, 1839]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 272 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1839) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1839) torch.int64 - loss_mask : 487 / 1839 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1839, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1839, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1839, 1, 576) torch.bfloat16 - out : (1, 1839) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 487 tokens) : 11.3547 - top-1 accuracy : 31/487 = 6.4% - - [2026-04-29 12:29:38] iteration 88/ 100 | consumed samples: 352 | elapsed time per iteration (ms): 5240.6 | throughput (token/sec/GPU): 1103.9 | learning rate: 2.406083E-05 | global batch size: 4 | lm loss: 1.132336E+01 | loss scale: 1.0 | grad norm: 0.314 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 88 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 88 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 88 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1164.40, 1164.40) -Dataloader batch - tokens shape: torch.Size([1, 1427]) -Dataloader batch - labels shape: torch.Size([1, 1427]) -Dataloader batch - attn_mask shape: torch.Size([1, 1427]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 273 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1427) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1427) torch.int64 - loss_mask : 391 / 1427 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1427, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1427, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1427, 1, 576) torch.bfloat16 - out : (1, 1427) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3749 - top-1 accuracy : 3/391 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1389]) -Dataloader batch - labels shape: torch.Size([1, 1389]) -Dataloader batch - attn_mask shape: torch.Size([1, 1389]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 274 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1389) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1389) torch.int64 - loss_mask : 308 / 1389 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1389, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1389, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1389, 1, 576) torch.bfloat16 - out : (1, 1389) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.2319 - top-1 accuracy : 13/308 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1909]) -Dataloader batch - labels shape: torch.Size([1, 1909]) -Dataloader batch - attn_mask shape: torch.Size([1, 1909]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 275 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1909) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1909) torch.int64 - loss_mask : 550 / 1909 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1909, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1909, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1909, 1, 576) torch.bfloat16 - out : (1, 1909) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 550 tokens) : 11.3745 - top-1 accuracy : 8/550 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 276 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 342 / 1518 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2667 - top-1 accuracy : 24/342 = 7.0% - - [2026-04-29 12:29:44] iteration 89/ 100 | consumed samples: 356 | elapsed time per iteration (ms): 5485.8 | throughput (token/sec/GPU): 1138.0 | learning rate: 2.275683E-05 | global batch size: 4 | lm loss: 1.132383E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 89 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 89 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 89 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1999.88, 1999.88) -Dataloader batch - tokens shape: torch.Size([1, 1589]) -Dataloader batch - labels shape: torch.Size([1, 1589]) -Dataloader batch - attn_mask shape: torch.Size([1, 1589]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 277 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1589) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1589) torch.int64 - loss_mask : 436 / 1589 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1589, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1589, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1589, 1, 576) torch.bfloat16 - out : (1, 1589) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.3796 - top-1 accuracy : 18/436 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1930]) -Dataloader batch - labels shape: torch.Size([1, 1930]) -Dataloader batch - attn_mask shape: torch.Size([1, 1930]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 278 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1930) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1930) torch.int64 - loss_mask : 565 / 1930 tokens (29.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1930, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1930, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1930, 1, 576) torch.bfloat16 - out : (1, 1930) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 565 tokens) : 11.4161 - top-1 accuracy : 4/565 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1721]) -Dataloader batch - labels shape: torch.Size([1, 1721]) -Dataloader batch - attn_mask shape: torch.Size([1, 1721]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 279 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1721) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1721) torch.int64 - loss_mask : 371 / 1721 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1721, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1721, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1721, 1, 576) torch.bfloat16 - out : (1, 1721) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.2466 - top-1 accuracy : 30/371 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1715]) -Dataloader batch - labels shape: torch.Size([1, 1715]) -Dataloader batch - attn_mask shape: torch.Size([1, 1715]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 280 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1715) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1715) torch.int64 - loss_mask : 395 / 1715 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1715, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1715, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1715, 1, 576) torch.bfloat16 - out : (1, 1715) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.2559 - top-1 accuracy : 25/395 = 6.3% - - [2026-04-29 12:29:52] iteration 90/ 100 | consumed samples: 360 | elapsed time per iteration (ms): 5365.9 | throughput (token/sec/GPU): 1296.1 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.133569E+01 | loss scale: 1.0 | grad norm: 0.313 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 90 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 90 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 90 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1063.59, 1063.59) -Dataloader batch - tokens shape: torch.Size([1, 1421]) -Dataloader batch - labels shape: torch.Size([1, 1421]) -Dataloader batch - attn_mask shape: torch.Size([1, 1421]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 281 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1421) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1421) torch.int64 - loss_mask : 320 / 1421 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1421, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1421, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1421, 1, 576) torch.bfloat16 - out : (1, 1421) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2452 - top-1 accuracy : 22/320 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 282 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 387 / 1504 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2935 - top-1 accuracy : 13/387 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1119]) -Dataloader batch - labels shape: torch.Size([1, 1119]) -Dataloader batch - attn_mask shape: torch.Size([1, 1119]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 283 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1119) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1119) torch.int64 - loss_mask : 269 / 1119 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1119, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1119, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1119, 1, 576) torch.bfloat16 - out : (1, 1119) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 269 tokens) : 11.2710 - top-1 accuracy : 12/269 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1180]) -Dataloader batch - labels shape: torch.Size([1, 1180]) -Dataloader batch - attn_mask shape: torch.Size([1, 1180]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 284 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1180) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1180) torch.int64 - loss_mask : 283 / 1180 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1180, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1180, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1180, 1, 576) torch.bfloat16 - out : (1, 1180) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 283 tokens) : 11.2364 - top-1 accuracy : 6/283 = 2.1% - - [2026-04-29 12:29:57] iteration 91/ 100 | consumed samples: 364 | elapsed time per iteration (ms): 4201.9 | throughput (token/sec/GPU): 1243.2 | learning rate: 2.023256E-05 | global batch size: 4 | lm loss: 1.126359E+01 | loss scale: 1.0 | grad norm: 0.317 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 91 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 91 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 91 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1592.82, 1592.82) -Dataloader batch - tokens shape: torch.Size([1, 1751]) -Dataloader batch - labels shape: torch.Size([1, 1751]) -Dataloader batch - attn_mask shape: torch.Size([1, 1751]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 285 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1751) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1751) torch.int64 - loss_mask : 381 / 1751 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1751, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1751, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1751, 1, 576) torch.bfloat16 - out : (1, 1751) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.1835 - top-1 accuracy : 26/381 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 286 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 386 / 1543 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2655 - top-1 accuracy : 9/386 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1497]) -Dataloader batch - labels shape: torch.Size([1, 1497]) -Dataloader batch - attn_mask shape: torch.Size([1, 1497]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 287 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1497) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1497) torch.int64 - loss_mask : 329 / 1497 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1497, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1497, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1497, 1, 576) torch.bfloat16 - out : (1, 1497) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2171 - top-1 accuracy : 28/329 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1903]) -Dataloader batch - labels shape: torch.Size([1, 1903]) -Dataloader batch - attn_mask shape: torch.Size([1, 1903]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 288 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1903) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1903) torch.int64 - loss_mask : 551 / 1903 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1903, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1903, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1903, 1, 576) torch.bfloat16 - out : (1, 1903) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 551 tokens) : 11.3362 - top-1 accuracy : 6/551 = 1.1% - - [2026-04-29 12:30:05] iteration 92/ 100 | consumed samples: 368 | elapsed time per iteration (ms): 6277.3 | throughput (token/sec/GPU): 1066.4 | learning rate: 1.901480E-05 | global batch size: 4 | lm loss: 1.126055E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 92 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 92 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 92 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1805.54, 1805.54) -Dataloader batch - tokens shape: torch.Size([1, 1703]) -Dataloader batch - labels shape: torch.Size([1, 1703]) -Dataloader batch - attn_mask shape: torch.Size([1, 1703]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 289 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1703) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1703) torch.int64 - loss_mask : 388 / 1703 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1703, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1703, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1703, 1, 576) torch.bfloat16 - out : (1, 1703) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2207 - top-1 accuracy : 27/388 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1491]) -Dataloader batch - labels shape: torch.Size([1, 1491]) -Dataloader batch - attn_mask shape: torch.Size([1, 1491]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 290 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1491) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1491) torch.int64 - loss_mask : 379 / 1491 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1491, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1491, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1491, 1, 576) torch.bfloat16 - out : (1, 1491) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.3128 - top-1 accuracy : 14/379 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 291 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 349 / 1431 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.3017 - top-1 accuracy : 22/349 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 292 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 314 / 1380 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.2128 - top-1 accuracy : 19/314 = 6.1% - - [2026-04-29 12:30:14] iteration 93/ 100 | consumed samples: 372 | elapsed time per iteration (ms): 7620.7 | throughput (token/sec/GPU): 788.0 | learning rate: 1.782823E-05 | global batch size: 4 | lm loss: 1.126315E+01 | loss scale: 1.0 | grad norm: 0.291 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 93 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 93 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 93 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1977.47, 1977.47) -Dataloader batch - tokens shape: torch.Size([1, 1549]) -Dataloader batch - labels shape: torch.Size([1, 1549]) -Dataloader batch - attn_mask shape: torch.Size([1, 1549]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 293 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1549) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1549) torch.int64 - loss_mask : 349 / 1549 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1549, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1549, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1549, 1, 576) torch.bfloat16 - out : (1, 1549) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2151 - top-1 accuracy : 17/349 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1531]) -Dataloader batch - labels shape: torch.Size([1, 1531]) -Dataloader batch - attn_mask shape: torch.Size([1, 1531]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 294 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1531) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1531) torch.int64 - loss_mask : 339 / 1531 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1531, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1531, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1531, 1, 576) torch.bfloat16 - out : (1, 1531) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2132 - top-1 accuracy : 20/339 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 295 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1519) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1519) torch.int64 - loss_mask : 378 / 1519 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1519, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1519, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1519, 1, 576) torch.bfloat16 - out : (1, 1519) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 378 tokens) : 11.2235 - top-1 accuracy : 21/378 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 296 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 360 / 1422 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.3049 - top-1 accuracy : 34/360 = 9.4% - - [2026-04-29 12:30:23] iteration 94/ 100 | consumed samples: 376 | elapsed time per iteration (ms): 7030.0 | throughput (token/sec/GPU): 856.5 | learning rate: 1.667403E-05 | global batch size: 4 | lm loss: 1.123955E+01 | loss scale: 1.0 | grad norm: 0.345 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 94 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 94 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 94 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1878.47, 1878.47) -Dataloader batch - tokens shape: torch.Size([1, 1417]) -Dataloader batch - labels shape: torch.Size([1, 1417]) -Dataloader batch - attn_mask shape: torch.Size([1, 1417]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 297 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1417) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1417) torch.int64 - loss_mask : 365 / 1417 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1417, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1417, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1417, 1, 576) torch.bfloat16 - out : (1, 1417) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2818 - top-1 accuracy : 19/365 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 298 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 409 / 1543 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.3387 - top-1 accuracy : 15/409 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1581]) -Dataloader batch - labels shape: torch.Size([1, 1581]) -Dataloader batch - attn_mask shape: torch.Size([1, 1581]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 299 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1581) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1581) torch.int64 - loss_mask : 433 / 1581 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1581, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1581, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1581, 1, 576) torch.bfloat16 - out : (1, 1581) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 433 tokens) : 11.3410 - top-1 accuracy : 29/433 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1554]) -Dataloader batch - labels shape: torch.Size([1, 1554]) -Dataloader batch - attn_mask shape: torch.Size([1, 1554]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 300 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1554) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1554) torch.int64 - loss_mask : 369 / 1554 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1554, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1554, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1554, 1, 576) torch.bfloat16 - out : (1, 1554) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2505 - top-1 accuracy : 24/369 = 6.5% - - [2026-04-29 12:30:31] iteration 95/ 100 | consumed samples: 380 | elapsed time per iteration (ms): 6064.4 | throughput (token/sec/GPU): 1005.0 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.130552E+01 | loss scale: 1.0 | grad norm: 0.291 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 95 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 95 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 95 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1836.51, 1836.51) -Dataloader batch - tokens shape: torch.Size([1, 1723]) -Dataloader batch - labels shape: torch.Size([1, 1723]) -Dataloader batch - attn_mask shape: torch.Size([1, 1723]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 301 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1723) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1723) torch.int64 - loss_mask : 367 / 1723 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1723, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1723, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1723, 1, 576) torch.bfloat16 - out : (1, 1723) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2070 - top-1 accuracy : 35/367 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1170]) -Dataloader batch - labels shape: torch.Size([1, 1170]) -Dataloader batch - attn_mask shape: torch.Size([1, 1170]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 302 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1170) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1170) torch.int64 - loss_mask : 313 / 1170 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1170, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1170, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1170, 1, 576) torch.bfloat16 - out : (1, 1170) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.3220 - top-1 accuracy : 20/313 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1556]) -Dataloader batch - labels shape: torch.Size([1, 1556]) -Dataloader batch - attn_mask shape: torch.Size([1, 1556]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 303 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1556) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1556) torch.int64 - loss_mask : 420 / 1556 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1556, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1556, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1556, 1, 576) torch.bfloat16 - out : (1, 1556) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 420 tokens) : 11.3500 - top-1 accuracy : 24/420 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1138]) -Dataloader batch - labels shape: torch.Size([1, 1138]) -Dataloader batch - attn_mask shape: torch.Size([1, 1138]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 304 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1138) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1138) torch.int64 - loss_mask : 290 / 1138 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1138, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1138, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1138, 1, 576) torch.bfloat16 - out : (1, 1138) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.3280 - top-1 accuracy : 11/290 = 3.8% - - [2026-04-29 12:30:39] iteration 96/ 100 | consumed samples: 384 | elapsed time per iteration (ms): 5940.6 | throughput (token/sec/GPU): 940.5 | learning rate: 1.446729E-05 | global batch size: 4 | lm loss: 1.130133E+01 | loss scale: 1.0 | grad norm: 0.298 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 96 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 96 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 96 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1852.10, 1852.10) -Dataloader batch - tokens shape: torch.Size([1, 1065]) -Dataloader batch - labels shape: torch.Size([1, 1065]) -Dataloader batch - attn_mask shape: torch.Size([1, 1065]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 305 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1065) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1065) torch.int64 - loss_mask : 291 / 1065 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1065, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1065, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1065, 1, 576) torch.bfloat16 - out : (1, 1065) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 291 tokens) : 11.3368 - top-1 accuracy : 12/291 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 306 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 385 / 1434 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.3011 - top-1 accuracy : 25/385 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 307 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 380 / 1507 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 380 tokens) : 11.3257 - top-1 accuracy : 11/380 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1085]) -Dataloader batch - labels shape: torch.Size([1, 1085]) -Dataloader batch - attn_mask shape: torch.Size([1, 1085]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 308 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1085) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1085) torch.int64 - loss_mask : 252 / 1085 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1085, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1085, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1085, 1, 576) torch.bfloat16 - out : (1, 1085) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 252 tokens) : 11.2164 - top-1 accuracy : 21/252 = 8.3% - - [2026-04-29 12:30:46] iteration 97/ 100 | consumed samples: 388 | elapsed time per iteration (ms): 5394.0 | throughput (token/sec/GPU): 943.8 | learning rate: 1.341694E-05 | global batch size: 4 | lm loss: 1.129986E+01 | loss scale: 1.0 | grad norm: 0.318 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 97 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 97 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 97 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2111.02, 2111.02) -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 309 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 327 / 1518 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2524 - top-1 accuracy : 30/327 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1833]) -Dataloader batch - labels shape: torch.Size([1, 1833]) -Dataloader batch - attn_mask shape: torch.Size([1, 1833]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 310 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1833) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1833) torch.int64 - loss_mask : 461 / 1833 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1833, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1833, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1833, 1, 576) torch.bfloat16 - out : (1, 1833) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 461 tokens) : 11.3209 - top-1 accuracy : 15/461 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1270]) -Dataloader batch - labels shape: torch.Size([1, 1270]) -Dataloader batch - attn_mask shape: torch.Size([1, 1270]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 311 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1270) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1270) torch.int64 - loss_mask : 373 / 1270 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1270, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1270, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1270, 1, 576) torch.bfloat16 - out : (1, 1270) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.3676 - top-1 accuracy : 0/373 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 987]) -Dataloader batch - labels shape: torch.Size([1, 987]) -Dataloader batch - attn_mask shape: torch.Size([1, 987]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 312 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 987) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 987) torch.int64 - loss_mask : 220 / 987 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (987, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (987, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (987, 1, 576) torch.bfloat16 - out : (1, 987) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 220 tokens) : 11.2477 - top-1 accuracy : 10/220 = 4.5% - - [2026-04-29 12:30:54] iteration 98/ 100 | consumed samples: 392 | elapsed time per iteration (ms): 6174.8 | throughput (token/sec/GPU): 908.2 | learning rate: 1.240333E-05 | global batch size: 4 | lm loss: 1.130563E+01 | loss scale: 1.0 | grad norm: 0.359 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 98 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 98 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 98 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1869.00, 1869.00) -Dataloader batch - tokens shape: torch.Size([1, 1144]) -Dataloader batch - labels shape: torch.Size([1, 1144]) -Dataloader batch - attn_mask shape: torch.Size([1, 1144]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 313 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1144) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1144) torch.int64 - loss_mask : 339 / 1144 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1144, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1144, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1144, 1, 576) torch.bfloat16 - out : (1, 1144) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.3535 - top-1 accuracy : 10/339 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1692]) -Dataloader batch - labels shape: torch.Size([1, 1692]) -Dataloader batch - attn_mask shape: torch.Size([1, 1692]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 314 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1692) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1692) torch.int64 - loss_mask : 349 / 1692 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1692, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1692, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1692, 1, 576) torch.bfloat16 - out : (1, 1692) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.1822 - top-1 accuracy : 22/349 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1070]) -Dataloader batch - labels shape: torch.Size([1, 1070]) -Dataloader batch - attn_mask shape: torch.Size([1, 1070]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 315 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1070) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1070) torch.int64 - loss_mask : 239 / 1070 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1070, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1070, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1070, 1, 576) torch.bfloat16 - out : (1, 1070) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 239 tokens) : 11.2533 - top-1 accuracy : 16/239 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1561]) -Dataloader batch - labels shape: torch.Size([1, 1561]) -Dataloader batch - attn_mask shape: torch.Size([1, 1561]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 316 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1561) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1561) torch.int64 - loss_mask : 337 / 1561 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1561, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1561, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1561, 1, 576) torch.bfloat16 - out : (1, 1561) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.1878 - top-1 accuracy : 21/337 = 6.2% - - [2026-04-29 12:31:03] iteration 99/ 100 | consumed samples: 396 | elapsed time per iteration (ms): 6639.8 | throughput (token/sec/GPU): 823.4 | learning rate: 1.142747E-05 | global batch size: 4 | lm loss: 1.124306E+01 | loss scale: 1.0 | grad norm: 0.338 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 99 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 99 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 99 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1897.21, 1897.21) -Dataloader batch - tokens shape: torch.Size([1, 1373]) -Dataloader batch - labels shape: torch.Size([1, 1373]) -Dataloader batch - attn_mask shape: torch.Size([1, 1373]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 317 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1373) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1373) torch.int64 - loss_mask : 415 / 1373 tokens (30.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1373, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1373, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1373, 1, 576) torch.bfloat16 - out : (1, 1373) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.4149 - top-1 accuracy : 17/415 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1755]) -Dataloader batch - labels shape: torch.Size([1, 1755]) -Dataloader batch - attn_mask shape: torch.Size([1, 1755]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 318 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1755) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1755) torch.int64 - loss_mask : 415 / 1755 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1755, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1755, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1755, 1, 576) torch.bfloat16 - out : (1, 1755) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.2686 - top-1 accuracy : 28/415 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1401]) -Dataloader batch - labels shape: torch.Size([1, 1401]) -Dataloader batch - attn_mask shape: torch.Size([1, 1401]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 319 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1401) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1401) torch.int64 - loss_mask : 321 / 1401 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1401, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1401, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1401, 1, 576) torch.bfloat16 - out : (1, 1401) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2353 - top-1 accuracy : 34/321 = 10.6% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 320 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 374 / 1431 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.3083 - top-1 accuracy : 3/374 = 0.8% - - [2026-04-29 12:31:10] iteration 100/ 100 | consumed samples: 400 | elapsed time per iteration (ms): 5435.8 | throughput (token/sec/GPU): 1096.4 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.131114E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (5415.53, 5415.53) - forward-compute ................................: (3924.16, 3924.16) - backward-compute ...............................: (1489.01, 1489.01) - batch-generator ................................: (468.12, 468.12) - layernorm-grads-all-reduce .....................: (0.03, 0.03) - embedding-grads-all-reduce .....................: (0.05, 0.05) - all-grads-sync .................................: (0.50, 0.50) - params-all-gather ..............................: (0.15, 0.15) - optimizer-copy-to-main-grad ....................: (0.15, 0.15) - optimizer-clip-main-grad .......................: (0.55, 0.55) - optimizer-count-zeros ..........................: (0.02, 0.02) - optimizer-inner-step ...........................: (1.32, 1.32) - optimizer-copy-main-to-model-params ............: (0.24, 0.24) - optimizer ......................................: (3.16, 3.16) -saving checkpoint at iteration 100 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 100 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 100 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1161.60, 1161.60) -[after training is done] datetime: 2026-04-29 12:31:11 -[rank0]:[W429 12:32:33.219356395 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) diff --git a/stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log b/stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log deleted file mode 100644 index b86cc391..00000000 --- a/stage_1_alignment_mobilellm_140m/run_2026-04-29_13:33:58_tp1_pp1_seqlen32768_mbs1_gbs4_500steps.log +++ /dev/null @@ -1,62359 +0,0 @@ -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. - import pynvml # type: ignore[import] -False -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'repr' attribute with value False was provided to the `Field()` function, which has no effect in the context it was used. 'repr' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py:2249: UnsupportedFieldAttributeWarning: The 'frozen' attribute with value True was provided to the `Field()` function, which has no effect in the context it was used. 'frozen' is field-specific metadata, and can only be attached to a model field using `Annotated` metadata or by assignment. This may have happened because an `Annotated` type alias using the `type` statement was used, or if the `Field()` function was attached to a single member of a union type. - warnings.warn( -parse arguments function called --------------- Configure model to llava-ov-mobilellm-140m -------------- -[DEBUG] Model config type: LlavaOnevision1_5Config -[DEBUG] Is MobileLLM: True - num_layers = 15 - hidden_size = 576 - ffn_hidden_size = 2048 - num_attention_heads = 9 - group_query_attention = True - num_query_groups = 3 - position_embedding_type = rope - add_position_embedding = False - rotary_interleaved = False - normalization = RMSNorm - swiglu = True - attention_dropout = 0 - hidden_dropout = 0 - add_bias_linear = False - add_qkv_bias = False - qk_layernorm = True - untie_embeddings_and_output_weights = False - vocab_size_in_config_file = 128256 - make_vocab_size_divisible_by = 128 - norm_epsilon = 1e-05 - rotary_base = 8000000 - kv_channels = None - num_experts = None - moe_ffn_hidden_size = None ----------------- End of configuration ---------------- -[DEBUG] Args now has: num_layers=15, hidden_size=576, num_attention_heads=9, vocab_size=128256 -INFO: Set dataloader type to external since --training-phase=SFT -WARNING: --sft-dataset-config is not specified, setup to default config (/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json) -using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: Nonetensor-model-parallel size: 1, encoder-tensor-model-parallel size: 0, pipeline-model-parallel size: 1, encoder-pipeline-model-parallel size: 0 -Number of virtual stages per pipeline stage: None -accumulate and all-reduce gradients in fp32 for bfloat16 data type. -using torch.bfloat16 for parameters ... -/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/training/arguments.py:743: UserWarning: Using async gradient all reduce requires setting the environment variable CUDA_DEVICE_MAX_CONNECTIONS to 1 - warnings.warn( ------------------------- arguments ------------------------ - account_for_embedding_in_pipeline_split ......... False - account_for_loss_in_pipeline_split .............. False - accumulate_allreduce_grads_in_fp32 .............. True - adam_beta1 ...................................... 0.9 - adam_beta2 ...................................... 0.99 - adam_eps ........................................ 1e-05 - add_bias_linear ................................. False - add_position_embedding .......................... False - add_qkv_bias .................................... False - add_question_in_pretrain ........................ False - additional_special_tokens ....................... None - adlr_autoresume ................................. False - adlr_autoresume_interval ........................ 1000 - align_grad_reduce ............................... True - align_param_gather .............................. False - app_tag_run_name ................................ None - app_tag_run_version ............................. 0.0.0 - apply_layernorm_1p .............................. False - apply_query_key_layer_scaling ................... False - apply_residual_connection_post_layernorm ........ False - apply_rope_fusion ............................... False - async_save ...................................... None - async_tensor_model_parallel_allreduce ........... True - attention_backend ............................... AttnBackend.local - attention_dropout ............................... 0 - attention_softmax_in_fp32 ....................... False - auto_detect_ckpt_format ......................... False - barrier_with_L1_time ............................ True - bert_binary_head ................................ True - bert_embedder_type .............................. megatron - bert_load ....................................... None - bf16 ............................................ True - bias_dropout_fusion ............................. True - bias_gelu_fusion ................................ False - bias_swiglu_fusion .............................. True - biencoder_projection_dim ........................ 0 - biencoder_shared_query_context_model ............ False - block_data_path ................................. None - calc_ft_timeouts ................................ False - calculate_per_token_loss ........................ False - caption_channels ................................ None - chat_template ................................... llama3 - check_for_large_grads ........................... False - check_for_nan_in_loss_and_grad .................. True - check_for_spiky_loss ............................ False - check_weight_hash_across_dp_replicas_interval ... None - ckpt_assume_constant_structure .................. False - ckpt_convert_format ............................. None - ckpt_convert_save ............................... None - ckpt_convert_update_legacy_dist_opt_format ...... False - ckpt_format ..................................... torch - ckpt_fully_parallel_load ........................ True - ckpt_fully_parallel_save ........................ True - ckpt_fully_parallel_save_deprecated ............. False - ckpt_step ....................................... None - classes_fraction ................................ 1.0 - clip_grad ....................................... 1.0 - clone_scatter_output_in_embedding ............... True - combined_1f1b ................................... False - combined_1f1b_recipe ............................ ep_a2a - config_logger_dir ............................... - consumed_train_samples .......................... 0 - consumed_valid_samples .......................... 0 - context_parallel_size ........................... 1 - context_parallel_ulysses_degree ................. 1 - cp_comm_type .................................... ['p2p'] - create_attention_mask_in_dataloader ............. True - cross_entropy_loss_fusion ....................... False - cuda_graph_warmup_steps ......................... 3 - custom_pipeline_layers .......................... None - custom_pipeline_recompute_layers ................ None - d2d_max_data_volume ............................. 0 - d2d_optimizer ................................... disabled - data_args_path .................................. None - data_cache_path ................................. None - data_parallel_random_init ....................... False - data_parallel_sharding_strategy ................. no_shard - data_parallel_size .............................. 1 - data_path ....................................... ['/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/data/LLaVA-558K-Webdataset'] - data_per_class_fraction ......................... 1.0 - data_sharding ................................... True - dataloader_save ................................. stage_1_alignment_mobilellm_140m/dataloader - dataloader_type ................................. external - ddp_average_in_collective ....................... False - ddp_bucket_size ................................. None - ddp_num_buckets ................................. None - ddp_pad_buckets_for_high_nccl_busbw ............. False - decoder_first_pipeline_num_layers ............... None - decoder_last_pipeline_num_layers ................ None - decoder_num_layers .............................. None - decoder_seq_length .............................. None - decoupled_lr .................................... None - decoupled_min_lr ................................ None - decrease_batch_size_if_needed ................... False - defer_embedding_wgrad_compute ................... False - dense_mlp_activation_func_recompute ............. False - deprecated_use_mcore_models ..................... False - detail_log_interval ............................. 20 - deterministic_mode .............................. False - dino_bottleneck_size ............................ 256 - dino_freeze_last_layer .......................... 1 - dino_head_hidden_size ........................... 2048 - dino_local_crops_number ......................... 10 - dino_local_img_size ............................. 96 - dino_norm_last_layer ............................ False - dino_teacher_temp ............................... 0.07 - dino_warmup_teacher_temp ........................ 0.04 - dino_warmup_teacher_temp_epochs ................. 30 - disable_straggler_on_startup .................... False - dist_ckpt_format_deprecated ..................... None - dist_ckpt_strictness ............................ assume_ok_unexpected - distribute_saved_activations .................... False - distributed_backend ............................. nccl - distributed_timeout_minutes ..................... 10 - ema_decay ....................................... 0.9999 - embedding_path .................................. None - empty_unused_memory_level ....................... 0 - enable_cuda_graph ............................... False - enable_discard_sample ........................... False - enable_ema ...................................... False - enable_fa_within_mla ............................ False - enable_fp8_comm ................................. False - enable_ft_package ............................... False - enable_gloo_process_groups ...................... True - enable_mem_monitor .............................. False - enable_one_logger ............................... False - enable_turn_off_bucketing ....................... True - encoder_num_layers .............................. 15 - encoder_pipeline_model_parallel_size ............ 0 - encoder_seq_length .............................. 32768 - encoder_tensor_model_parallel_size .............. 0 - end_weight_decay ................................ 0.0 - eod_mask_loss ................................... False - error_injection_rate ............................ 0 - error_injection_type ............................ transient_error - eval_interval ................................... 1000 - eval_iters ...................................... 100 - evidence_data_path .............................. None - exit_duration_in_mins ........................... None - exit_interval ................................... None - exit_on_missing_checkpoint ...................... False - exit_signal_handler ............................. False - exp_avg_dtype ................................... torch.float32 - exp_avg_sq_dtype ................................ torch.float32 - expert_model_parallel_size ...................... 1 - expert_tensor_parallel_size ..................... 1 - fastvit_image_size .............................. 1024 - ffn_hidden_size ................................. 2048 - finetune ........................................ False - first_last_layers_bf16 .......................... False - flash_decode .................................... False - fp16 ............................................ False - fp16_lm_cross_entropy ........................... False - fp32_residual_connection ........................ False - fp8 ............................................. None - fp8_amax_compute_algo ........................... most_recent - fp8_amax_history_len ............................ 1 - fp8_interval .................................... 1 - fp8_margin ...................................... 0 - fp8_param_gather ................................ False - fp8_recipe ...................................... delayed - fp8_wgrad ....................................... True - fps ............................................. 2.0 - fps_max_frames .................................. 768 - fps_min_frames .................................. 4 - frame_max_pixels ................................ 602112 - frame_min_pixels ................................ 100352 - global_batch_size ............................... 4 - grad_reduce_in_bf16 ............................. False - gradient_accumulation_fusion .................... False - gradient_reduce_div_fusion ...................... True - group_query_attention ........................... True - head_lr_mult .................................... 1.0 - hf_tokenizer_path ............................... facebook/MobileLLM-R1-140M - hidden_dropout .................................. 0 - hidden_size ..................................... 576 - hierarchical_context_parallel_sizes ............. None - hybrid_attention_ratio .......................... 0.0 - hybrid_mlp_ratio ................................ 0.0 - hybrid_override_pattern ......................... None - hysteresis ...................................... 2 - ict_head_size ................................... None - ict_load ........................................ None - image_aspect_ratio .............................. pad - image_grid_pinpoints ............................ [(384, 384), (768, 384), (384, 768), (768, 768)] - image_resolution ................................ None - img_h ........................................... 224 - img_w ........................................... 224 - indexer_batch_size .............................. 128 - indexer_log_interval ............................ 1000 - inference_batch_times_seqlen_threshold .......... -1 - inference_max_batch_size ........................ 8 - inference_max_seq_length ........................ 2560 - inference_rng_tracker ........................... False - init_method_std ................................. 0.02 - init_method_xavier_uniform ...................... False - init_model_with_meta_device ..................... False - initial_loss_scale .............................. 65536.0 - is_tokenized_data ............................... False - iter_per_epoch .................................. 1250 - iterations_to_skip .............................. [] - keep_fp8_transpose_cache_when_using_custom_fsdp . False - kv_channels ..................................... 64 - kv_lora_rank .................................... 32 - language_model_type ............................. None - latent_frame_interval ........................... 1 - latent_in_channels .............................. None - latent_out_channels ............................. None - latent_patch_size ............................... (1, 1, 1) - latent_space_scale .............................. 1.0 - latent_time_scale ............................... 1.0 - layernorm_recompute ............................. False - lazy_mpu_init ................................... None - length_sort_desc ................................ False - length_sort_pool_size ........................... 0 - load ............................................ stage_1_alignment_mobilellm_140m - load_ema ........................................ None - local_rank ...................................... 0 - log_detail ...................................... True - log_interval .................................... 1 - log_loss_scale_to_tensorboard ................... True - log_memory_to_tensorboard ....................... False - log_num_zeros_in_grad ........................... False - log_params_norm ................................. False - log_progress .................................... False - log_straggler ................................... False - log_throughput .................................. False - log_timers_to_tensorboard ....................... True - log_validation_ppl_to_tensorboard ............... False - log_world_size_to_tensorboard ................... False - logging_level ................................... None - loss_scale ...................................... None - loss_scale_window ............................... 1000 - lr .............................................. 0.0001 - lr_decay_iters .................................. 500 - lr_decay_samples ................................ None - lr_decay_style .................................. cosine - lr_warmup_fraction .............................. 0.002 - lr_warmup_init .................................. 0.0 - lr_warmup_iters ................................. 0 - lr_warmup_samples ............................... 0 - lr_wsd_decay_iters .............................. None - lr_wsd_decay_samples ............................ None - lr_wsd_decay_style .............................. exponential - main_grads_dtype ................................ torch.float32 - main_params_dtype ............................... torch.float32 - make_vocab_size_divisible_by .................... 128 - manual_gc ....................................... False - manual_gc_eval .................................. True - manual_gc_interval .............................. 0 - mask_factor ..................................... 1.0 - mask_prob ....................................... 0.15 - mask_type ....................................... random - masked_softmax_fusion ........................... True - max_latent_height ............................... None - max_latent_width ................................ None - max_pixels ...................................... 12845056 - max_position_embeddings ......................... 32768 - max_text_length ................................. None - max_tokens_to_oom ............................... 12000 - mem_monitor_force_print_token_threshold ......... 10000000 - mem_monitor_log ................................. None - memory_snapshot_path ............................ snapshot.pickle - merge_file ...................................... None - micro_batch_size ................................ 1 - microbatch_group_size_per_vp_stage .............. None - min_loss_scale .................................. 1.0 - min_lr .......................................... 1e-06 - min_pixels ...................................... 3136 - mla_recompute ................................... False - mmap_bin_files .................................. True - mock_data ....................................... False - model_family .................................... llava_ov_1_5 - model_name ...................................... llava-ov-mobilellm-140m - moe_aux_loss_coeff .............................. 0.0 - moe_enable_deepep ............................... False - moe_expert_capacity_factor ...................... None - moe_extended_tp ................................. False - moe_ffn_hidden_size ............................. None - moe_grouped_gemm ................................ False - moe_input_jitter_eps ............................ None - moe_layer_freq .................................. 1 - moe_layer_recompute ............................. False - moe_mlp_activation_func_recompute ............... False - moe_pad_expert_input_to_capacity ................ False - moe_per_layer_logging ........................... False - moe_permute_fusion .............................. False - moe_router_bias_update_rate ..................... 0.001 - moe_router_dtype ................................ None - moe_router_enable_expert_bias ................... False - moe_router_force_load_balancing ................. False - moe_router_group_topk ........................... None - moe_router_load_balancing_type .................. aux_loss - moe_router_num_groups ........................... None - moe_router_padding_for_fp8 ...................... False - moe_router_pre_softmax .......................... False - moe_router_score_function ....................... softmax - moe_router_topk ................................. 2 - moe_router_topk_scaling_factor .................. None - moe_shared_expert_intermediate_size ............. None - moe_shared_expert_overlap ....................... False - moe_token_dispatcher_type ....................... allgather - moe_token_drop_policy ........................... probs - moe_use_legacy_grouped_gemm ..................... False - moe_use_upcycling ............................... False - moe_z_loss_coeff ................................ None - mscale .......................................... 1.0 - mscale_all_dim .................................. 1.0 - mtp_loss_coef ................................... 0.1 - multi_latent_attention .......................... False - nccl_communicator_config_path ................... None - no_load_optim ................................... True - no_load_rng ..................................... True - no_persist_layer_norm ........................... False - no_save_optim ................................... None - no_save_rng ..................................... None - non_persistent_ckpt_type ........................ None - non_persistent_global_ckpt_dir .................. None - non_persistent_local_ckpt_algo .................. fully_parallel - non_persistent_local_ckpt_dir ................... None - non_persistent_save_interval .................... None - norm_epsilon .................................... 1e-05 - normalization ................................... RMSNorm - num_attention_heads ............................. 9 - num_bucket_build_workers ........................ 1 - num_channels .................................... 3 - num_classes ..................................... 1000 - num_dataset_builder_threads ..................... 1 - num_distributed_optimizer_instances ............. 1 - num_experts ..................................... None - num_latent_frames ............................... None - num_layers ...................................... 15 - num_layers_at_end_in_bf16 ....................... 1 - num_layers_at_start_in_bf16 ..................... 1 - num_layers_per_virtual_pipeline_stage ........... None - num_nextn_predict_layers ........................ 0 - num_query_groups ................................ 3 - num_virtual_stages_per_pipeline_rank ............ None - num_workers ..................................... 16 - one_logger_async ................................ False - one_logger_project .............................. megatron-lm - one_logger_run_name ............................. None - onnx_safe ....................................... None - openai_gelu ..................................... False - optimizer ....................................... adam - optimizer_cpu_offload ........................... False - optimizer_offload_fraction ...................... 1.0 - output_bert_embeddings .......................... False - overlap_cpu_optimizer_d2h_h2d ................... False - overlap_d2d_optimizer ........................... True - overlap_grad_reduce ............................. False - overlap_p2p_comm ................................ False - overlap_p2p_comm_warmup_flush ................... False - overlap_param_gather ............................ False - overlap_param_gather_with_optimizer_step ........ False - override_opt_param_scheduler .................... False - packing_batch_size .............................. None - packing_pretrain_data ........................... False - packing_sft_data ................................ False - padded_vocab_size ............................... None - padding_side .................................... right - params_dtype .................................... torch.bfloat16 - patch_dim ....................................... 16 - per_split_data_args_path ........................ None - perform_initialization .......................... True - pin_cpu_grads ................................... True - pin_cpu_params .................................. True - pipeline_model_parallel_comm_backend ............ None - pipeline_model_parallel_size .................... 1 - pipeline_model_parallel_split_rank .............. None - position_embedding_type ......................... rope - pretrained_checkpoint ........................... /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - print_mem_monitor_interval ...................... 100000 - profile ......................................... False - profile_ranks ................................... [0] - profile_step_end ................................ 12 - profile_step_start .............................. 10 - q_lora_rank ..................................... None - qk_head_dim ..................................... 128 - qk_layernorm .................................... True - qk_pos_emb_head_dim ............................. 64 - query_in_block_prob ............................. 0.1 - rampup_batch_size ............................... None - rank ............................................ 0 - recompute_granularity ........................... full - recompute_method ................................ uniform - recompute_num_layers ............................ 3 - record_memory_history ........................... False - reduced_data_volume_from_pre_stage .............. 0 - relative_attention_max_distance ................. 128 - relative_attention_num_buckets .................. 32 - replication ..................................... False - replication_factor .............................. 2 - replication_jump ................................ None - rerun_mode ...................................... disabled - reset_attention_mask ............................ False - reset_position_ids .............................. False - result_rejected_tracker_filename ................ None - retriever_report_topk_accuracies ................ [] - retriever_score_scaling ......................... False - retriever_seq_length ............................ 256 - retro_add_retriever ............................. False - retro_attention_gate ............................ 1 - retro_cyclic_train_iters ........................ None - retro_encoder_attention_dropout ................. 0.1 - retro_encoder_hidden_dropout .................... 0.1 - retro_encoder_layers ............................ 2 - retro_num_neighbors ............................. 2 - retro_num_retrieved_chunks ...................... 2 - retro_project_dir ............................... None - retro_verify_neighbor_count ..................... True - rope_in_fp32 .................................... True - rope_scaling_factor ............................. 8.0 - rotary_base ..................................... 8000000 - rotary_interleaved .............................. False - rotary_percent .................................. 1.0 - rotary_scaling_factor ........................... 1.0 - rotary_seq_len_interpolation_factor ............. None - sample_rate ..................................... 1.0 - save ............................................ stage_1_alignment_mobilellm_140m - save_ema ........................................ None - save_interval ................................... 50 - scatter_gather_tensors_in_pipeline .............. True - seed ............................................ 1234 - separate_layernorm_and_collinear ................ False - seq_length ...................................... 32768 - sequence_parallel ............................... False - sft_data_mix_strategy ........................... concat - sft_data_streaming .............................. False - sft_dataset ..................................... None - sft_dataset_config .............................. /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/configs/sft_dataset_config.json - sft_num_preprocess_workers ...................... None - sft_sort_batch .................................. False - sft_test_dataset ................................ None - sft_train_dataset ............................... None - sft_valid_dataset ............................... None - sgd_momentum .................................... 0.9 - short_seq_prob .................................. 0.1 - skip_train ...................................... False - skipped_train_samples ........................... 0 - spec ............................................ None - split ........................................... 100,0,0 - split_bw ........................................ False - split_special_tokens ............................ False - squared_relu .................................... False - start_weight_decay .............................. 0.0 - stdit_bucket_config ............................. None - straggler_ctrlr_port ............................ 65535 - straggler_minmax_count .......................... 1 - streaming_buffer_size ........................... 16384 - suggested_communication_unit_size ............... 400000000 - swiglu .......................................... True - swin_backbone_type .............................. tiny - te_rng_tracker .................................. False - tensor_model_parallel_size ...................... 1 - tensorboard_dir ................................. stage_1_alignment_mobilellm_140m/tensorboard - tensorboard_log_interval ........................ 1 - tensorboard_queue_size .......................... 1000 - test_data_path .................................. None - test_mode ....................................... False - tiktoken_num_special_tokens ..................... 1000 - tiktoken_pattern ................................ None - tiktoken_special_tokens ......................... None - timing_log_level ................................ 0 - timing_log_option ............................... minmax - titles_data_path ................................ None - tokenizer_model ................................. None - tokenizer_type .................................. HFTokenizer - tp_comm_bootstrap_backend ....................... nccl - tp_comm_bulk_dgrad .............................. True - tp_comm_bulk_wgrad .............................. True - tp_comm_overlap ................................. False - tp_comm_overlap_ag .............................. True - tp_comm_overlap_cfg ............................. None - tp_comm_overlap_rs .............................. True - tp_comm_overlap_rs_dgrad ........................ False - tp_comm_split_ag ................................ True - tp_comm_split_rs ................................ True - train_data_path ................................. None - train_iters ..................................... 500 - train_on_prompt ................................. False - train_samples ................................... None - train_sync_interval ............................. None - trainable_modules ............................... ['adapter'] - training_phase .................................. sft - training_rice_vl_max_answer_length .............. 32768 - training_rice_vl_max_image_area ................. 1806336 - transformer_impl ................................ local - transformer_pipeline_model_parallel_size ........ 1 - untie_embeddings_and_output_weights ............. False - use_checkpoint_args ............................. False - use_checkpoint_opt_param_scheduler .............. False - use_cpu_initialization .......................... None - use_custom_fsdp ................................. False - use_dist_ckpt ................................... False - use_dist_ckpt_deprecated ........................ False - use_distributed_optimizer ....................... True - use_fast_tokenizer .............................. False - use_fastvit ..................................... True - use_flash_attn .................................. False - use_legacy_models ............................... False - use_mp_args_from_checkpoint_args ................ False - use_normhead .................................... False - use_one_sent_docs ............................... False - use_persistent_ckpt_worker ...................... False - use_precision_aware_optimizer ................... False - use_pytorch_profiler ............................ False - use_ring_exchange_p2p ........................... False - use_rope_scaling ................................ False - use_rotary_position_embeddings .................. False - use_tokenizer_model_from_checkpoint_args ........ True - use_torch_fsdp2 ................................. False - use_torch_optimizer_for_cpu_offload ............. False - use_tp_pp_dp_mapping ............................ False - v_head_dim ...................................... 128 - valid_data_path ................................. None - variable_seq_lengths ............................ True - video_max_pixels ................................ 51380224 - virtual_pipeline_model_parallel_size ............ None - vision_backbone_type ............................ vit - vision_pretraining .............................. False - vision_pretraining_type ......................... classify - vision_tower_name ............................... mobileclip_l_1024 - vocab_extra_ids ................................. 0 - vocab_file ...................................... None - vocab_size ...................................... None - vocab_size_in_config_file ....................... 128256 - vpp_scheduler ................................... None - wandb_exp_name .................................. mobilellm_integration - wandb_project ................................... llava-ov-1_5 - wandb_save_dir .................................. - weight_decay .................................... 0.0 - weight_decay_incr_style ......................... constant - wgrad_deferral_limit ............................ 0 - world_size ...................................... 1 - yaml_cfg ........................................ None --------------------- end of arguments --------------------- -Building model trainer... -Model family: llava_ov_1_5 -Training phase: sft -Using unified model type for training. - -Starting training... -INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 4 -> setting tensorboard ... -wandb: Currently logged in as: rana-zayed (rana-zayed-mbzuai) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin -wandb: creating run -wandb: Tracking run with wandb version 0.21.0 -wandb: Run data is saved locally in stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_133502-3bn4iqdv -wandb: Run `wandb offline` to turn off syncing. -wandb: Syncing run mobilellm_integration -wandb: ⭐️ View project at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 -wandb: 🚀 View run at https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/3bn4iqdv -> AIAK building HFTokenizer tokenizer ... -INFO: ensured multimodal special tokens ['<|vision_start|>', '<|vision_end|>', '<|image_pad|>', '<|video_pad|>']; added=4 -WARNING: tokenizer vocab size increased from config size 128256 to 128260; updating args.vocab_size_in_config_file to avoid embedding index OOB. -WARNING: tokenizer already has an EOS token, replace <|eot_id|> with <|eot_id|>, and will add 0 new tokens to tokenizer. -WARNING: tokenizer does not have a pad token, setting to eos token(<|eot_id|>) (token id: 128009). - > padded vocab (size: 128260) with 124 dummy tokens (new size: 128384) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode disabled -> initializing torch distributed ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -> initialized tensor model parallel with size 1 -> initialized pipeline model parallel with size 1 -> setting random seeds to 1234 ... -> compiling dataset index builder ... -make: Entering directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' -Using existing compiled library: helpers_cpp.cpython-310-x86_64-linux-gnu.so -make: Leaving directory '/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_megatron/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.100 seconds -WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. -> compiling and loading fused kernels ... -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[rank0]:[W429 13:35:09.652713489 ProcessGroupNCCL.cpp:5072] Guessing device ID based on global rank. This can cause a hang if rank to GPU mapping is heterogeneous. You can specify device_id in init_process_group() ->>> done with compiling and loading fused kernels. Compilation time: 0.887 seconds -time to initialize megatron (seconds): 24.289 -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once -[after megatron is initialized] datetime: 2026-04-29 13:35:23 -building llava-ov-mobilellm-140m model ... -[DEBUG PROVIDER] Model name: llava-ov-mobilellm-140m -[DEBUG PROVIDER] Args num_layers: 15 -[DEBUG PROVIDER] Args hidden_size: 576 -[DEBUG PROVIDER] Args vocab_size: 128260 -[DEBUG PROVIDER] TransformerConfig built with num_layers: 15, hidden_size: 576 -[DEBUG PROVIDER] Initial language_config: layers=15, hidden=576 -[DEBUG PROVIDER] Model family: llava_ov_1_5 -[DEBUG PROVIDER] use_mobilellm flag: True -[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone -[DEBUG PROVIDER] Language config BEFORE override: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] Language config AFTER (should be same): layers=15, hidden=576, heads=9, query_groups=3 -[DEBUG PROVIDER] ========== LANGUAGE CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Loading vision config... -[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: mobileclip_l_1024 -[DEBUG PROVIDER] FastViT config loaded from /share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/aiak_training_llm/models/llavaov_1_5/../fastvit/mobileclip/configs/mobileclip_l.json -[DEBUG PROVIDER] image_cfg: embed_dim=3072, patch_size=64, model=fastvithd -[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=24, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=3072, num_attention_heads=48, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-05, layernorm_zero_centered_gamma=False, add_bias_linear=False, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='RMSNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ================================================ -[DEBUG PROVIDER] Loading adapter config... -[DEBUG PROVIDER] ========== ADAPTER CONFIG ========== -TransformerConfig(tensor_model_parallel_size=1, pipeline_model_parallel_comm_backend=None, pipeline_model_parallel_size=1, virtual_pipeline_model_parallel_size=None, sequence_parallel=False, context_parallel_size=1, hierarchical_context_parallel_sizes=None, expert_model_parallel_size=1, expert_tensor_parallel_size=1, moe_extended_tp=False, perform_initialization=True, use_cpu_initialization=None, fp16=False, bf16=True, params_dtype=torch.bfloat16, timers=None, finalize_model_grads_func=None, grad_scale_func=None, no_sync_func=None, grad_sync_func=None, param_sync_func=None, deterministic_mode=False, enable_autocast=False, autocast_dtype=torch.bfloat16, num_microbatches_with_partial_activation_checkpoints=None, gradient_accumulation_fusion=False, async_tensor_model_parallel_allreduce=True, use_te_rng_tracker=False, tp_comm_overlap=False, tp_comm_bulk_wgrad=True, tp_comm_bulk_dgrad=True, tp_comm_overlap_ag=True, tp_comm_overlap_rs=True, tp_comm_overlap_rs_dgrad=False, tp_comm_split_ag=True, tp_comm_atomic_ag=False, tp_comm_split_rs=True, tp_comm_atomic_rs=False, cross_entropy_loss_fusion=False, tp_comm_overlap_disable_qkv=False, tp_comm_overlap_disable_fc1=False, tp_comm_bootstrap_backend='nccl', pipeline_dtype=torch.bfloat16, variable_seq_lengths=True, overlap_p2p_comm=False, batch_p2p_comm=True, batch_p2p_sync=True, use_ring_exchange_p2p=False, deallocate_pipeline_outputs=True, defer_embedding_wgrad_compute=False, wgrad_deferral_limit=0, pipeline_model_parallel_split_rank=None, overlap_p2p_comm_warmup_flush=False, microbatch_group_size_per_vp_stage=1, vpp_scheduler=None, combined_1f1b=False, combined_1f1b_recipe='ep_a2a', split_bw=False, cpu_offloading=False, cpu_offloading_num_layers=0, _cpu_offloading_context=None, cpu_offloading_activations=True, cpu_offloading_weights=True, barrier_with_L1_time=True, custom_pipeline_layers=None, custom_pipeline_recompute_layers=None, flash_attn_checkpoint=False, single_all_to_all=False, dense_mlp_activation_func_recompute=False, layernorm_recompute=False, moe_mlp_activation_func_recompute=False, mla_recompute=False, num_layers=15, num_layers_in_first_pipeline_stage=None, num_layers_in_last_pipeline_stage=None, account_for_embedding_in_pipeline_split=False, account_for_loss_in_pipeline_split=False, hidden_size=576, num_attention_heads=9, attention_backend=, softmax_scale=None, num_query_groups=3, ffn_hidden_size=2048, kv_channels=64, hidden_dropout=0, attention_dropout=0, fp32_residual_connection=False, apply_residual_connection_post_layernorm=False, layernorm_epsilon=1e-06, layernorm_zero_centered_gamma=False, add_bias_linear=True, add_qkv_bias=False, gated_linear_unit=True, activation_func=, activation_func_fp8_input_store=False, num_moe_experts=None, rotary_interleaved=False, window_size=None, normalization='LayerNorm', position_embedding_type='rope', qk_layernorm=True, test_mode=False, calculate_per_token_loss=False, multi_latent_attention=False, init_method=functools.partial(, mean=0.0, std=0.02), output_layer_init_method=functools.partial(, mean=0.0, std=0.0036514837167011074), init_method_std=0.02, init_model_with_meta_device=False, apply_query_key_layer_scaling=False, attention_softmax_in_fp32=False, bias_activation_fusion=True, masked_softmax_fusion=True, persist_layer_norm=True, memory_efficient_layer_norm=False, bias_dropout_fusion=True, apply_rope_fusion=False, recompute_granularity='full', recompute_method='uniform', recompute_num_layers=3, distribute_saved_activations=False, fp8=None, fp8_recipe='delayed', enable_fp8_comm=False, fp8_param=False, fp8_margin=0, fp8_interval=1, fp8_amax_history_len=1, fp8_amax_compute_algo='most_recent', fp8_wgrad=True, fp8_dot_product_attention=False, fp8_multi_head_attention=False, tp_only_amax_red=False, first_last_layers_bf16=False, num_layers_at_start_in_bf16=1, num_layers_at_end_in_bf16=1, moe_shared_expert_intermediate_size=None, moe_shared_expert_overlap=False, moe_layer_freq=1, moe_ffn_hidden_size=None, moe_router_load_balancing_type='aux_loss', moe_router_topk=2, moe_router_topk_limited_devices=None, moe_router_padding_for_fp8=False, moe_router_num_groups=None, moe_router_group_topk=None, moe_router_pre_softmax=False, moe_router_topk_scaling_factor=None, moe_router_score_function='softmax', moe_router_enable_expert_bias=False, moe_router_bias_update_rate=0.001, moe_router_force_load_balancing=False, moe_grouped_gemm=False, moe_use_legacy_grouped_gemm=False, moe_aux_loss_coeff=0.0, moe_z_loss_coeff=None, moe_input_jitter_eps=None, moe_token_dropping=False, moe_token_dispatcher_type='allgather', moe_enable_deepep=False, moe_per_layer_logging=False, moe_expert_capacity_factor=None, moe_pad_expert_input_to_capacity=False, moe_token_drop_policy='probs', moe_layer_recompute=False, moe_permute_fusion=False, cp_comm_type='p2p', enable_cuda_graph=False, cuda_graph_use_single_mempool=False, cuda_graph_retain_backward_graph=False, cuda_graph_warmup_steps=3, external_cuda_graph=False, clone_scatter_output_in_embedding=True, disable_parameter_transpose_cache=False, config_logger_dir='', flash_decode=False, inference_rng_tracker=False, use_custom_fsdp=False, enable_mem_monitor=False, print_mem_monitor_interval=100000, mem_monitor_log=None, mem_monitor_force_print_token_threshold=10000000, moe_router_dtype=None) -[DEBUG PROVIDER] ====================================== -[DEBUG PROVIDER] Resolved vision token ids from tokenizer: image_token_id=128258, video_token_id=128259 -[DEBUG PROVIDER] Building layer specs... -[DEBUG PROVIDER] ✓ Using MobileLLM layer specification -[DEBUG PROVIDER] MobileLLM layer config: layers=15, hidden=576, heads=9 -[DEBUG PROVIDER] MobileLLM layer spec created successfully -[DEBUG PROVIDER] Creating LlavaOnevision1_5 model... -[DEBUG PROVIDER] Final language_config: layers=15, hidden=576, vocab=128384, rotary_base=8000000 -[INFO] Using MobileLLM as language backbone -[Attention] apply_rotary_fn bound to: megatron.core.models.common.embeddings.rope_utils.apply_rotary_pos_emb -[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully! - > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 276633888 -INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=400000000, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False) -INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 1 -Params for bucket 1 (11216448 elements, 11216512 padded size): - module.adapter.layernorm.weight - module.adapter.linear_fc2.linear.bias - module.adapter.linear_fc1.linear.weight - module.adapter.linear_fc2.linear.weight - module.adapter.layernorm.bias - module.adapter.linear_fc1.linear.bias -[DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -)] -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-06, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.0, fp8_recipe='delayed', fp16=False, bf16=True, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=65536.0, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.99, adam_eps=1e-05, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='', d2d_optimizer='disabled', overlap_d2d_optimizer=True, d2d_max_data_volume=0, reduced_data_volume_from_pre_stage=0) -AdamW ( -Parameter Group 0 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 0.0 - weight_decay: 0.0 - -Parameter Group 1 - amsgrad: False - betas: (0.9, 0.99) - capturable: False - decoupled_weight_decay: True - differentiable: False - eps: 1e-05 - foreach: None - fused: None - is_decoupled_lr: False - is_expert_parallel: False - lr: 0.0001 - lr_mult: 1.0 - max_lr: 0.0001 - maximize: False - min_lr: 1e-06 - wd_mult: 1.0 - weight_decay: 0.0 -) -INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[DEBUG] FastViT enabled: use_fastvit=True -[DEBUG] Before handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 -FastViT enabled: Will resume/load checkpoint from: stage_1_alignment_mobilellm_140m -[DEBUG] After handling: args.load=stage_1_alignment_mobilellm_140m, args.pretrained_checkpoint=/share/data/drive_3/mobile_vlm/LLaVA-OneVision-1.5/checkpoints/mobilellm-fastvit-merged-tp1-pp1 - loading checkpoint from stage_1_alignment_mobilellm_140m at iteration 100 -Loading checkpoint with device: cuda:0, strict=False - checkpoint version 3.0 -WARNING:megatron.core.rerun_state_machine:RerunStateMachine disabled via CLI, ignoring machine state saved in checkpoint - successfully loaded checkpoint from stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] at iteration 100 -(min, max) time across ranks (ms): - load-checkpoint ................................: (5203.56, 5203.56) -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-04-29 13:35:31 -================================================================================ -FULL MODEL STRUCTURE: -================================================================================ - ---- Model 0 (pipeline rank 0) --- -DistributedDataParallel( - (module): Float16Module( - (module): LlavaOnevision1_5( - (vision_model): FastViTModel( - (vision_tower): MobileCLIPVisionTower( - (vision_tower): MCi( - (model): FastViT( - (patch_embed): Sequential( - (0): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(3, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=96) - ) - (2): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(96, 96, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - (network): ModuleList( - (0): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(96, 96, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=96) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(96, 96, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=96, bias=False) - (bn): BatchNorm2d(96, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(96, 384, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(384, 96, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (1): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(96, 192, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=96) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(192, 192, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (2): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(192, 192, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=192) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(192, 192, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=192, bias=False) - (bn): BatchNorm2d(192, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(192, 768, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(768, 192, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (3): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(192, 384, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=192) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(384, 384, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (4): Sequential( - (0): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (4): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (5): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (6): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (7): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (8): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (9): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (10): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (11): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (12): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (13): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (14): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (15): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (16): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (17): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (18): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (19): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (20): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (21): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (22): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (23): RepMixerBlock( - (token_mixer): RepMixer( - (reparam_conv): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=384) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(384, 384, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=384, bias=False) - (bn): BatchNorm2d(384, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(384, 1536, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(1536, 384, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (5): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(384, 768, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=384) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(768, 768, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (6): RepCPE( - (reparam_conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768) - ) - (7): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (2): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (3): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=768, out_features=2304, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=768, out_features=768, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(768, 768, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=768, bias=False) - (bn): BatchNorm2d(768, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(768, 3072, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(3072, 768, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - (8): PatchEmbed( - (proj): Sequential( - (0): ReparamLargeKernelConv( - (activation): GELU(approximate='none') - (se): Identity() - (lkb_reparam): Conv2d(768, 1536, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), groups=768) - ) - (1): MobileOneBlock( - (se): Identity() - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 1536, kernel_size=(1, 1), stride=(1, 1)) - ) - ) - ) - (9): RepCPE( - (reparam_conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536) - ) - (10): Sequential( - (0): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - (1): AttentionBlock( - (norm): LayerNormChannel() - (token_mixer): MHSA( - (qkv): Linear(in_features=1536, out_features=4608, bias=False) - (attn_drop): Dropout(p=0.0, inplace=False) - (proj): Linear(in_features=1536, out_features=1536, bias=True) - (proj_drop): Dropout(p=0.0, inplace=False) - ) - (convffn): ConvFFN( - (conv): Sequential( - (conv): Conv2d(1536, 1536, kernel_size=(7, 7), stride=(1, 1), padding=(3, 3), groups=1536, bias=False) - (bn): BatchNorm2d(1536, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) - ) - (fc1): Conv2d(1536, 6144, kernel_size=(1, 1), stride=(1, 1)) - (act): GELU(approximate='none') - (fc2): Conv2d(6144, 1536, kernel_size=(1, 1), stride=(1, 1)) - (drop): Dropout(p=0.0, inplace=False) - ) - (drop_path): Identity() - ) - ) - ) - (conv_exp): MobileOneBlock( - (se): SEBlock( - (reduce): Conv2d(3072, 192, kernel_size=(1, 1), stride=(1, 1)) - (expand): Conv2d(192, 3072, kernel_size=(1, 1), stride=(1, 1)) - ) - (activation): GELU(approximate='none') - (reparam_conv): Conv2d(1536, 3072, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=1536) - ) - (head): GlobalPool2D() - ) - ) - ) - ) - (adapter): Adapter( - (layernorm): FusedLayerNorm() - (linear_fc1): TELinear( - (linear): Linear(in_features=3072, out_features=3072, bias=True) - ) - (linear_fc2): TELinear( - (linear): Linear(in_features=3072, out_features=576, bias=True) - ) - ) - (language_model): MobileLLMModel( - (embedding): LanguageModelEmbedding( - (word_embeddings): VocabParallelEmbedding() - (embedding_dropout): Dropout(p=0, inplace=False) - ) - (rotary_pos_emb): RotaryEmbedding() - (decoder): TransformerBlock( - (layers): ModuleList( - (0-14): 15 x TransformerLayer( - (input_layernorm): IdentityOp() - (self_attention): SelfAttention( - (core_attention): DotProductAttention( - (attention_dropout): Dropout(p=0, inplace=False) - ) - (linear_proj): TERowParallelLinear( - (linear): Linear(in_features=576, out_features=576, bias=False) - ) - (linear_qkv): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=960, bias=False) - ) - (q_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - (k_layernorm): TENorm( - (ln): LayerNorm((64,), eps=1e-05, elementwise_affine=True) - ) - ) - (pre_cross_attn_layernorm): IdentityOp() - (cross_attention): IdentityOp() - (cross_attn_bda): IdentityFuncOp() - (pre_mlp_layernorm): IdentityOp() - (mlp): MLP( - (linear_fc1): TELayerNormColumnParallelLinear( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - (linear): Linear(in_features=576, out_features=4096, bias=False) - ) - (activation_func): ActivationFuncModule() - (linear_fc2): TERowParallelLinear( - (linear): Linear(in_features=2048, out_features=576, bias=False) - ) - ) - ) - ) - (final_layernorm): TENorm( - (ln): LayerNorm((576,), eps=1e-05, elementwise_affine=True) - ) - ) - (output_layer): ColumnParallelLinear(in_features=576, out_features=128384, bias=False, TP=1) - ) - ) - ) -) -================================================================================ - -================================================================================ -PARAMETER TRAINABILITY STATUS: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -================================================================================ -Total trainable parameters: 11,216,448 (11.22M) -Total frozen parameters: 265,417,440 (265.42M) -Total parameters: 276,633,888 (276.63M) -Trainable percentage: 4.05% -================================================================================ -> building train, validation, and test datasets ... - > datasets target sizes (minimum size): - train: 2000 - validation: 400 - test: 400 -Using shared training tokenizer in FastVLM mode -Ensured multimodal special tokens for FastVLM tokenizer; added=0 -Loaded tokenizer (FastVLM mode) from facebook/MobileLLM-R1-140M -Initialized FastViT processor with image_size=1024 -Processor config: .TokenizerWrapper object at 0x7f2e946e7e20> -image_resolution: None -rank=0, worker=0: shard_range=[pretrain-000003.tar[0, 1500), ] sum(count)=1500 -rank=0, worker=1: shard_range=[pretrain-000003.tar[1500, 2999), ] sum(count)=1499 -rank=0, worker=2: shard_range=[pretrain-000003.tar[2999, 4498), ] sum(count)=1499 -rank=0, worker=3: shard_range=[pretrain-000003.tar[4498, 5058), pretrain-000001.tar[0, 939), ] sum(count)=1499 -rank=0, worker=4: shard_range=[pretrain-000001.tar[939, 2439), ] sum(count)=1500 -rank=0, worker=5: shard_range=[pretrain-000001.tar[2439, 3938), ] sum(count)=1499 -rank=0, worker=6: shard_range=[pretrain-000001.tar[3938, 5036), pretrain-000002.tar[0, 401), ] sum(count)=1499 -rank=0, worker=7: shard_range=[pretrain-000002.tar[401, 1900), ] sum(count)=1499 -rank=0, worker=8: shard_range=[pretrain-000002.tar[1900, 3400), ] sum(count)=1500 -rank=0, worker=9: shard_range=[pretrain-000002.tar[3400, 4899), ] sum(count)=1499 -rank=0, worker=10: shard_range=[pretrain-000002.tar[4899, 5039), pretrain-000004.tar[0, 1359), ] sum(count)=1499 -rank=0, worker=11: shard_range=[pretrain-000004.tar[1359, 2858), ] sum(count)=1499 -rank=0, worker=12: shard_range=[pretrain-000004.tar[2858, 3821), pretrain-000000.tar[0, 537), ] sum(count)=1500 -rank=0, worker=13: shard_range=[pretrain-000000.tar[537, 2036), ] sum(count)=1499 -rank=0, worker=14: shard_range=[pretrain-000000.tar[2036, 3535), ] sum(count)=1499 -rank=0, worker=15: shard_range=[pretrain-000000.tar[3535, 5034), ] sum(count)=1499 -restored dataset state from stage_1_alignment_mobilellm_140m/dataloader/iter_0000100/mp_rank_00/train_dataloader_dprank000.pt -[after dataloaders are built] datetime: 2026-04-29 13:35:31 -done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (7978.23, 7978.23) - train/valid/test-data-iterators-setup ..........: (86.80, 86.80) -training ... - -================================================================================ -training with the following parameter status: -================================================================================ -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.weight | shape: (96, 3, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.0.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.1.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.weight | shape: (96, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.patch_embed.2.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.0.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.layer_scale | shape: (96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.weight | shape: (96, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.token_mixer.reparam_conv.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.conv.weight | shape: (96, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.weight | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.conv.bn.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.weight | shape: (384, 96, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc1.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.weight | shape: (96, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.0.1.convffn.fc2.bias | shape: (96,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.0.lkb_reparam.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.weight | shape: (192, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.1.proj.1.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.0.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.1.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.2.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.3.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.4.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.5.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.6.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.7.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.8.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.9.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.10.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.layer_scale | shape: (192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.weight | shape: (192, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.token_mixer.reparam_conv.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.conv.weight | shape: (192, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.weight | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.conv.bn.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.weight | shape: (768, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc1.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.weight | shape: (192, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.2.11.convffn.fc2.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.0.lkb_reparam.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.weight | shape: (384, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.3.proj.1.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.0.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.1.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.2.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.3.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.4.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.5.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.6.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.7.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.8.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.9.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.10.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.11.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.12.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.13.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.14.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.15.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.16.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.17.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.18.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.19.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.20.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.21.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.22.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.layer_scale | shape: (384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.weight | shape: (384, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.token_mixer.reparam_conv.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.conv.weight | shape: (384, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.weight | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.conv.bn.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.weight | shape: (1536, 384, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc1.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.weight | shape: (384, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.4.23.convffn.fc2.bias | shape: (384,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.0.lkb_reparam.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.weight | shape: (768, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.5.proj.1.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.6.reparam_conv.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.0.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.1.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.2.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_1 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.layer_scale_2 | shape: (768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.norm.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.qkv.weight | shape: (2304, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.weight | shape: (768, 768) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.token_mixer.proj.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.conv.weight | shape: (768, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.weight | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.conv.bn.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.weight | shape: (3072, 768, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc1.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.weight | shape: (768, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.7.3.convffn.fc2.bias | shape: (768,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.0.lkb_reparam.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.weight | shape: (1536, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.8.proj.1.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.9.reparam_conv.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.0.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_1 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.layer_scale_2 | shape: (1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.norm.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.qkv.weight | shape: (4608, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.weight | shape: (1536, 1536) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.token_mixer.proj.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.conv.weight | shape: (1536, 1, 7, 7) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.weight | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.conv.bn.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.weight | shape: (6144, 1536, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc1.bias | shape: (6144,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.weight | shape: (1536, 6144, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.network.10.1.convffn.fc2.bias | shape: (1536,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.weight | shape: (192, 3072, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.reduce.bias | shape: (192,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.weight | shape: (3072, 192, 1, 1) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.se.expand.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.weight | shape: (3072, 1, 3, 3) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.conv_exp.reparam_conv.bias | shape: (3072,) | dtype: torch.bfloat16 -FROZEN | module.module.vision_model.vision_tower.vision_tower.model.head.proj | shape: (3072, 768) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.weight | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.layernorm.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.weight | shape: (3072, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc1.linear.bias | shape: (3072,) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.weight | shape: (576, 3072) | dtype: torch.bfloat16 -TRAINABLE | module.module.adapter.linear_fc2.linear.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.embedding.word_embeddings.weight | shape: (128384, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.0.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.1.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.2.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.3.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.4.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.5.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.6.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.7.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.8.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.9.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.10.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.11.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.12.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.13.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_proj.linear.weight | shape: (576, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.linear_qkv.linear.weight | shape: (960, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.q_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.weight | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.self_attention.k_layernorm.ln.bias | shape: (64,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.ln.bias | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc1.linear.weight | shape: (4096, 576) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.layers.14.mlp.linear_fc2.linear.weight | shape: (576, 2048) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.weight | shape: (576,) | dtype: torch.bfloat16 -FROZEN | module.module.language_model.decoder.final_layernorm.ln.bias | shape: (576,) | dtype: torch.bfloat16 -Setting rerun_state_machine.current_iteration to 100... -[before the start of training step] datetime: 2026-04-29 13:35:31 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 - -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -[DEBUG PREPROCESS TOKENS] image_token='<|image_pad|>' id=128258, vision_start='<|vision_start|>' id=128256, vision_end='<|vision_end|>' id=128257, counts_in_input_ids: image=1, vision_start=1 -You're using a PreTrainedTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding. -Dataloader batch - tokens shape: torch.Size([1, 985]) -Dataloader batch - labels shape: torch.Size([1, 985]) -Dataloader batch - attn_mask shape: torch.Size([1, 985]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 985) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 985) torch.int64 - loss_mask : 213 / 985 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (985, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (985, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (985, 1, 576) torch.bfloat16 - out : (1, 985) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 213 tokens) : 11.2059 - top-1 accuracy : 16/213 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 2 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 404 / 1459 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.3175 - top-1 accuracy : 12/404 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 3 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 337 / 1400 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.2449 - top-1 accuracy : 5/337 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1872]) -Dataloader batch - labels shape: torch.Size([1, 1872]) -Dataloader batch - attn_mask shape: torch.Size([1, 1872]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 4 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1872) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1872) torch.int64 - loss_mask : 514 / 1872 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1872, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1872, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1872, 1, 576) torch.bfloat16 - out : (1, 1872) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 514 tokens) : 11.3168 - top-1 accuracy : 37/514 = 7.2% - -/home/ashaker/miniconda3/envs/llava-ov-4b-clean/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning. - warnings.warn( # warn only once - [2026-04-29 13:36:34] iteration 101/ 500 | consumed samples: 404 | elapsed time per iteration (ms): 62508.2 | throughput (token/sec/GPU): 91.4 | learning rate: 1.000000E-04 | global batch size: 4 | lm loss: 1.128442E+01 | loss scale: 1.0 | grad norm: 0.352 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer layers in billions: 0.07 -Number of parameters in embedding layers in billions: 0.07 -Total number of parameters in billions: 0.14 -Number of parameters in most loaded shard in billions: 0.1403 -Theoretical memory footprints: weight and optimizer=2409.10 MB -[Rank 0] (after 101 iterations) memory (MB) | allocated: 793.81787109375 | max allocated: 5484.03466796875 | reserved: 9622.0 | max reserved: 9622.0 -Dataloader batch - tokens shape: torch.Size([1, 1449]) -Dataloader batch - labels shape: torch.Size([1, 1449]) -Dataloader batch - attn_mask shape: torch.Size([1, 1449]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 5 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1449) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1449) torch.int64 - loss_mask : 353 / 1449 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1449, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1449, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1449, 1, 576) torch.bfloat16 - out : (1, 1449) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.2924 - top-1 accuracy : 7/353 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 6 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1412) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1412) torch.int64 - loss_mask : 335 / 1412 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1412, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1412, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1412, 1, 576) torch.bfloat16 - out : (1, 1412) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.3050 - top-1 accuracy : 18/335 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1365]) -Dataloader batch - labels shape: torch.Size([1, 1365]) -Dataloader batch - attn_mask shape: torch.Size([1, 1365]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 7 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1365) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1365) torch.int64 - loss_mask : 315 / 1365 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1365, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1365, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1365, 1, 576) torch.bfloat16 - out : (1, 1365) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.2518 - top-1 accuracy : 19/315 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1582]) -Dataloader batch - labels shape: torch.Size([1, 1582]) -Dataloader batch - attn_mask shape: torch.Size([1, 1582]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 8 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1582) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1582) torch.int64 - loss_mask : 439 / 1582 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1582, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1582, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1582, 1, 576) torch.bfloat16 - out : (1, 1582) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 439 tokens) : 11.3241 - top-1 accuracy : 18/439 = 4.1% - - [2026-04-29 13:36:40] iteration 102/ 500 | consumed samples: 408 | elapsed time per iteration (ms): 6273.4 | throughput (token/sec/GPU): 925.8 | learning rate: 9.999902E-05 | global batch size: 4 | lm loss: 1.129612E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1895]) -Dataloader batch - labels shape: torch.Size([1, 1895]) -Dataloader batch - attn_mask shape: torch.Size([1, 1895]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 9 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1895) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1895) torch.int64 - loss_mask : 531 / 1895 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1895, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1895, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1895, 1, 576) torch.bfloat16 - out : (1, 1895) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 531 tokens) : 11.3597 - top-1 accuracy : 23/531 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1489]) -Dataloader batch - labels shape: torch.Size([1, 1489]) -Dataloader batch - attn_mask shape: torch.Size([1, 1489]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 10 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1489) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1489) torch.int64 - loss_mask : 313 / 1489 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1489, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1489, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1489, 1, 576) torch.bfloat16 - out : (1, 1489) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2138 - top-1 accuracy : 30/313 = 9.6% - -Dataloader batch - tokens shape: torch.Size([1, 1752]) -Dataloader batch - labels shape: torch.Size([1, 1752]) -Dataloader batch - attn_mask shape: torch.Size([1, 1752]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 11 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1752) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1752) torch.int64 - loss_mask : 394 / 1752 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1752, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1752, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1752, 1, 576) torch.bfloat16 - out : (1, 1752) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.2502 - top-1 accuracy : 26/394 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1495]) -Dataloader batch - labels shape: torch.Size([1, 1495]) -Dataloader batch - attn_mask shape: torch.Size([1, 1495]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 12 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1495) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1495) torch.int64 - loss_mask : 354 / 1495 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1495, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1495, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1495, 1, 576) torch.bfloat16 - out : (1, 1495) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2848 - top-1 accuracy : 20/354 = 5.6% - - [2026-04-29 13:36:47] iteration 103/ 500 | consumed samples: 412 | elapsed time per iteration (ms): 7095.3 | throughput (token/sec/GPU): 934.6 | learning rate: 9.999608E-05 | global batch size: 4 | lm loss: 1.128726E+01 | loss scale: 1.0 | grad norm: 0.549 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 999]) -Dataloader batch - labels shape: torch.Size([1, 999]) -Dataloader batch - attn_mask shape: torch.Size([1, 999]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 13 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 999) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 999) torch.int64 - loss_mask : 238 / 999 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (999, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (999, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (999, 1, 576) torch.bfloat16 - out : (1, 999) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 238 tokens) : 11.2851 - top-1 accuracy : 16/238 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 14 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 359 / 1444 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.2496 - top-1 accuracy : 22/359 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1387]) -Dataloader batch - labels shape: torch.Size([1, 1387]) -Dataloader batch - attn_mask shape: torch.Size([1, 1387]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 15 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1387) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1387) torch.int64 - loss_mask : 330 / 1387 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1387, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1387, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1387, 1, 576) torch.bfloat16 - out : (1, 1387) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.2850 - top-1 accuracy : 26/330 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1686]) -Dataloader batch - labels shape: torch.Size([1, 1686]) -Dataloader batch - attn_mask shape: torch.Size([1, 1686]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 16 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1686) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1686) torch.int64 - loss_mask : 410 / 1686 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1686, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1686, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1686, 1, 576) torch.bfloat16 - out : (1, 1686) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.2497 - top-1 accuracy : 13/410 = 3.2% - - [2026-04-29 13:36:53] iteration 104/ 500 | consumed samples: 416 | elapsed time per iteration (ms): 6029.8 | throughput (token/sec/GPU): 914.8 | learning rate: 9.999117E-05 | global batch size: 4 | lm loss: 1.126469E+01 | loss scale: 1.0 | grad norm: 0.473 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1534]) -Dataloader batch - labels shape: torch.Size([1, 1534]) -Dataloader batch - attn_mask shape: torch.Size([1, 1534]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 17 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1534) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1534) torch.int64 - loss_mask : 399 / 1534 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1534, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1534, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1534, 1, 576) torch.bfloat16 - out : (1, 1534) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.3398 - top-1 accuracy : 23/399 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1421]) -Dataloader batch - labels shape: torch.Size([1, 1421]) -Dataloader batch - attn_mask shape: torch.Size([1, 1421]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 18 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1421) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1421) torch.int64 - loss_mask : 358 / 1421 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1421, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1421, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1421, 1, 576) torch.bfloat16 - out : (1, 1421) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.3165 - top-1 accuracy : 20/358 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 19 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 398 / 1459 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.3363 - top-1 accuracy : 2/398 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1227]) -Dataloader batch - labels shape: torch.Size([1, 1227]) -Dataloader batch - attn_mask shape: torch.Size([1, 1227]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 20 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1227) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1227) torch.int64 - loss_mask : 325 / 1227 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1227, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1227, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1227, 1, 576) torch.bfloat16 - out : (1, 1227) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.3268 - top-1 accuracy : 10/325 = 3.1% - - [2026-04-29 13:36:59] iteration 105/ 500 | consumed samples: 420 | elapsed time per iteration (ms): 5807.2 | throughput (token/sec/GPU): 971.4 | learning rate: 9.998430E-05 | global batch size: 4 | lm loss: 1.133039E+01 | loss scale: 1.0 | grad norm: 0.470 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 1527]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 21 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1527) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1527) torch.int64 - loss_mask : 327 / 1527 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1527, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1527, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1527, 1, 576) torch.bfloat16 - out : (1, 1527) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.1968 - top-1 accuracy : 15/327 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1174]) -Dataloader batch - labels shape: torch.Size([1, 1174]) -Dataloader batch - attn_mask shape: torch.Size([1, 1174]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 22 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1174) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1174) torch.int64 - loss_mask : 326 / 1174 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1174, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1174, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1174, 1, 576) torch.bfloat16 - out : (1, 1174) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.3892 - top-1 accuracy : 6/326 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1781]) -Dataloader batch - labels shape: torch.Size([1, 1781]) -Dataloader batch - attn_mask shape: torch.Size([1, 1781]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 23 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1781) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1781) torch.int64 - loss_mask : 406 / 1781 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1781, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1781, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1781, 1, 576) torch.bfloat16 - out : (1, 1781) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.2330 - top-1 accuracy : 21/406 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1107]) -Dataloader batch - labels shape: torch.Size([1, 1107]) -Dataloader batch - attn_mask shape: torch.Size([1, 1107]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 24 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1107) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1107) torch.int64 - loss_mask : 299 / 1107 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1107, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1107, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1107, 1, 576) torch.bfloat16 - out : (1, 1107) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 299 tokens) : 11.3542 - top-1 accuracy : 9/299 = 3.0% - - [2026-04-29 13:37:05] iteration 106/ 500 | consumed samples: 424 | elapsed time per iteration (ms): 5897.8 | throughput (token/sec/GPU): 947.6 | learning rate: 9.997548E-05 | global batch size: 4 | lm loss: 1.128846E+01 | loss scale: 1.0 | grad norm: 0.379 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 25 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 364 / 1518 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2845 - top-1 accuracy : 5/364 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1453]) -Dataloader batch - labels shape: torch.Size([1, 1453]) -Dataloader batch - attn_mask shape: torch.Size([1, 1453]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 26 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1453) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1453) torch.int64 - loss_mask : 404 / 1453 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1453, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1453, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1453, 1, 576) torch.bfloat16 - out : (1, 1453) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.3466 - top-1 accuracy : 2/404 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1029]) -Dataloader batch - labels shape: torch.Size([1, 1029]) -Dataloader batch - attn_mask shape: torch.Size([1, 1029]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 27 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1029) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1029) torch.int64 - loss_mask : 263 / 1029 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1029, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1029, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1029, 1, 576) torch.bfloat16 - out : (1, 1029) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.3044 - top-1 accuracy : 6/263 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1081]) -Dataloader batch - labels shape: torch.Size([1, 1081]) -Dataloader batch - attn_mask shape: torch.Size([1, 1081]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 28 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1081) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1081) torch.int64 - loss_mask : 275 / 1081 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1081, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1081, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1081, 1, 576) torch.bfloat16 - out : (1, 1081) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 275 tokens) : 11.3373 - top-1 accuracy : 17/275 = 6.2% - - [2026-04-29 13:37:10] iteration 107/ 500 | consumed samples: 428 | elapsed time per iteration (ms): 5778.0 | throughput (token/sec/GPU): 879.4 | learning rate: 9.996469E-05 | global batch size: 4 | lm loss: 1.131884E+01 | loss scale: 1.0 | grad norm: 0.361 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1270]) -Dataloader batch - labels shape: torch.Size([1, 1270]) -Dataloader batch - attn_mask shape: torch.Size([1, 1270]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 29 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1270) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1270) torch.int64 - loss_mask : 343 / 1270 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1270, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1270, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1270, 1, 576) torch.bfloat16 - out : (1, 1270) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.3434 - top-1 accuracy : 11/343 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1047]) -Dataloader batch - labels shape: torch.Size([1, 1047]) -Dataloader batch - attn_mask shape: torch.Size([1, 1047]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 30 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1047) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1047) torch.int64 - loss_mask : 219 / 1047 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1047, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1047, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1047, 1, 576) torch.bfloat16 - out : (1, 1047) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 219 tokens) : 11.1684 - top-1 accuracy : 6/219 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1656]) -Dataloader batch - labels shape: torch.Size([1, 1656]) -Dataloader batch - attn_mask shape: torch.Size([1, 1656]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 31 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1656) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1656) torch.int64 - loss_mask : 413 / 1656 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1656, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1656, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1656, 1, 576) torch.bfloat16 - out : (1, 1656) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2663 - top-1 accuracy : 2/413 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 992]) -Dataloader batch - labels shape: torch.Size([1, 992]) -Dataloader batch - attn_mask shape: torch.Size([1, 992]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 32 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 992) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 992) torch.int64 - loss_mask : 222 / 992 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (992, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (992, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (992, 1, 576) torch.bfloat16 - out : (1, 992) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 222 tokens) : 11.2836 - top-1 accuracy : 11/222 = 5.0% - - [2026-04-29 13:37:16] iteration 108/ 500 | consumed samples: 432 | elapsed time per iteration (ms): 5692.2 | throughput (token/sec/GPU): 872.2 | learning rate: 9.995194E-05 | global batch size: 4 | lm loss: 1.127368E+01 | loss scale: 1.0 | grad norm: 0.422 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 33 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 413 / 1468 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.4118 - top-1 accuracy : 6/413 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1361]) -Dataloader batch - labels shape: torch.Size([1, 1361]) -Dataloader batch - attn_mask shape: torch.Size([1, 1361]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 34 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1361) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1361) torch.int64 - loss_mask : 286 / 1361 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1361, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1361, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1361, 1, 576) torch.bfloat16 - out : (1, 1361) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 286 tokens) : 11.1924 - top-1 accuracy : 15/286 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1084]) -Dataloader batch - labels shape: torch.Size([1, 1084]) -Dataloader batch - attn_mask shape: torch.Size([1, 1084]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 35 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1084) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1084) torch.int64 - loss_mask : 256 / 1084 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1084, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1084, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1084, 1, 576) torch.bfloat16 - out : (1, 1084) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 256 tokens) : 11.3106 - top-1 accuracy : 5/256 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1000]) -Dataloader batch - labels shape: torch.Size([1, 1000]) -Dataloader batch - attn_mask shape: torch.Size([1, 1000]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 36 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1000) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1000) torch.int64 - loss_mask : 248 / 1000 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1000, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1000, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1000, 1, 576) torch.bfloat16 - out : (1, 1000) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 248 tokens) : 11.2977 - top-1 accuracy : 7/248 = 2.8% - - [2026-04-29 13:37:22] iteration 109/ 500 | consumed samples: 436 | elapsed time per iteration (ms): 5420.4 | throughput (token/sec/GPU): 906.4 | learning rate: 9.993723E-05 | global batch size: 4 | lm loss: 1.131460E+01 | loss scale: 1.0 | grad norm: 0.482 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 37 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1696) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1696) torch.int64 - loss_mask : 337 / 1696 tokens (19.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1696, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1696, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1696, 1, 576) torch.bfloat16 - out : (1, 1696) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.2151 - top-1 accuracy : 37/337 = 11.0% - -Dataloader batch - tokens shape: torch.Size([1, 1522]) -Dataloader batch - labels shape: torch.Size([1, 1522]) -Dataloader batch - attn_mask shape: torch.Size([1, 1522]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 38 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1522) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1522) torch.int64 - loss_mask : 357 / 1522 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1522, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1522, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1522, 1, 576) torch.bfloat16 - out : (1, 1522) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2466 - top-1 accuracy : 27/357 = 7.6% - -Dataloader batch - tokens shape: torch.Size([1, 1692]) -Dataloader batch - labels shape: torch.Size([1, 1692]) -Dataloader batch - attn_mask shape: torch.Size([1, 1692]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 39 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1692) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1692) torch.int64 - loss_mask : 347 / 1692 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1692, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1692, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1692, 1, 576) torch.bfloat16 - out : (1, 1692) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.1795 - top-1 accuracy : 29/347 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 1564]) -Dataloader batch - labels shape: torch.Size([1, 1564]) -Dataloader batch - attn_mask shape: torch.Size([1, 1564]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 40 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1564) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1564) torch.int64 - loss_mask : 362 / 1564 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1564, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1564, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1564, 1, 576) torch.bfloat16 - out : (1, 1564) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2334 - top-1 accuracy : 21/362 = 5.8% - - [2026-04-29 13:37:29] iteration 110/ 500 | consumed samples: 440 | elapsed time per iteration (ms): 7161.7 | throughput (token/sec/GPU): 904.0 | learning rate: 9.992056E-05 | global batch size: 4 | lm loss: 1.121903E+01 | loss scale: 1.0 | grad norm: 0.422 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1564]) -Dataloader batch - labels shape: torch.Size([1, 1564]) -Dataloader batch - attn_mask shape: torch.Size([1, 1564]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 41 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1564) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1564) torch.int64 - loss_mask : 387 / 1564 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1564, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1564, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1564, 1, 576) torch.bfloat16 - out : (1, 1564) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2561 - top-1 accuracy : 10/387 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1457]) -Dataloader batch - labels shape: torch.Size([1, 1457]) -Dataloader batch - attn_mask shape: torch.Size([1, 1457]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 42 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1457) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1457) torch.int64 - loss_mask : 312 / 1457 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1457, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1457, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1457, 1, 576) torch.bfloat16 - out : (1, 1457) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.1624 - top-1 accuracy : 8/312 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1531]) -Dataloader batch - labels shape: torch.Size([1, 1531]) -Dataloader batch - attn_mask shape: torch.Size([1, 1531]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 43 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1531) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1531) torch.int64 - loss_mask : 399 / 1531 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1531, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1531, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1531, 1, 576) torch.bfloat16 - out : (1, 1531) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.2714 - top-1 accuracy : 21/399 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 44 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 317 / 1459 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.1703 - top-1 accuracy : 10/317 = 3.2% - - [2026-04-29 13:37:35] iteration 111/ 500 | consumed samples: 444 | elapsed time per iteration (ms): 6593.5 | throughput (token/sec/GPU): 911.7 | learning rate: 9.990193E-05 | global batch size: 4 | lm loss: 1.122050E+01 | loss scale: 1.0 | grad norm: 0.382 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1881]) -Dataloader batch - labels shape: torch.Size([1, 1881]) -Dataloader batch - attn_mask shape: torch.Size([1, 1881]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 45 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1881) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1881) torch.int64 - loss_mask : 506 / 1881 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1881, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1881, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1881, 1, 576) torch.bfloat16 - out : (1, 1881) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 506 tokens) : 11.3654 - top-1 accuracy : 13/506 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1453]) -Dataloader batch - labels shape: torch.Size([1, 1453]) -Dataloader batch - attn_mask shape: torch.Size([1, 1453]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 46 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1453) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1453) torch.int64 - loss_mask : 398 / 1453 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1453, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1453, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1453, 1, 576) torch.bfloat16 - out : (1, 1453) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.3884 - top-1 accuracy : 16/398 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1159]) -Dataloader batch - labels shape: torch.Size([1, 1159]) -Dataloader batch - attn_mask shape: torch.Size([1, 1159]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 47 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1159) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1159) torch.int64 - loss_mask : 247 / 1159 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1159, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1159, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1159, 1, 576) torch.bfloat16 - out : (1, 1159) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 247 tokens) : 11.1677 - top-1 accuracy : 19/247 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1837]) -Dataloader batch - labels shape: torch.Size([1, 1837]) -Dataloader batch - attn_mask shape: torch.Size([1, 1837]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 48 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1837) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1837) torch.int64 - loss_mask : 455 / 1837 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1837, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1837, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1837, 1, 576) torch.bfloat16 - out : (1, 1837) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 455 tokens) : 11.3281 - top-1 accuracy : 19/455 = 4.2% - - [2026-04-29 13:37:42] iteration 112/ 500 | consumed samples: 448 | elapsed time per iteration (ms): 6614.8 | throughput (token/sec/GPU): 956.9 | learning rate: 9.988135E-05 | global batch size: 4 | lm loss: 1.133011E+01 | loss scale: 1.0 | grad norm: 0.365 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1570]) -Dataloader batch - labels shape: torch.Size([1, 1570]) -Dataloader batch - attn_mask shape: torch.Size([1, 1570]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 49 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1570) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1570) torch.int64 - loss_mask : 335 / 1570 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1570, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1570, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1570, 1, 576) torch.bfloat16 - out : (1, 1570) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2413 - top-1 accuracy : 15/335 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1269]) -Dataloader batch - labels shape: torch.Size([1, 1269]) -Dataloader batch - attn_mask shape: torch.Size([1, 1269]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 50 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1269) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1269) torch.int64 - loss_mask : 345 / 1269 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1269, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1269, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1269, 1, 576) torch.bfloat16 - out : (1, 1269) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.3770 - top-1 accuracy : 9/345 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1545]) -Dataloader batch - labels shape: torch.Size([1, 1545]) -Dataloader batch - attn_mask shape: torch.Size([1, 1545]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 51 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1545) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1545) torch.int64 - loss_mask : 335 / 1545 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1545, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1545, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1545, 1, 576) torch.bfloat16 - out : (1, 1545) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.1921 - top-1 accuracy : 30/335 = 9.0% - -Dataloader batch - tokens shape: torch.Size([1, 1766]) -Dataloader batch - labels shape: torch.Size([1, 1766]) -Dataloader batch - attn_mask shape: torch.Size([1, 1766]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 52 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1766) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1766) torch.int64 - loss_mask : 387 / 1766 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1766, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1766, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1766, 1, 576) torch.bfloat16 - out : (1, 1766) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2228 - top-1 accuracy : 18/387 = 4.7% - - [2026-04-29 13:37:49] iteration 113/ 500 | consumed samples: 452 | elapsed time per iteration (ms): 6828.3 | throughput (token/sec/GPU): 900.7 | learning rate: 9.985880E-05 | global batch size: 4 | lm loss: 1.125784E+01 | loss scale: 1.0 | grad norm: 0.337 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1489]) -Dataloader batch - labels shape: torch.Size([1, 1489]) -Dataloader batch - attn_mask shape: torch.Size([1, 1489]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 53 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1489) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1489) torch.int64 - loss_mask : 441 / 1489 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1489, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1489, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1489, 1, 576) torch.bfloat16 - out : (1, 1489) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 441 tokens) : 11.3808 - top-1 accuracy : 18/441 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1214]) -Dataloader batch - labels shape: torch.Size([1, 1214]) -Dataloader batch - attn_mask shape: torch.Size([1, 1214]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 54 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1214) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1214) torch.int64 - loss_mask : 283 / 1214 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1214, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1214, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1214, 1, 576) torch.bfloat16 - out : (1, 1214) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 283 tokens) : 11.2809 - top-1 accuracy : 20/283 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1059]) -Dataloader batch - labels shape: torch.Size([1, 1059]) -Dataloader batch - attn_mask shape: torch.Size([1, 1059]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 55 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1059) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1059) torch.int64 - loss_mask : 220 / 1059 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1059, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1059, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1059, 1, 576) torch.bfloat16 - out : (1, 1059) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 220 tokens) : 11.2013 - top-1 accuracy : 21/220 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1402]) -Dataloader batch - labels shape: torch.Size([1, 1402]) -Dataloader batch - attn_mask shape: torch.Size([1, 1402]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 56 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1402) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1402) torch.int64 - loss_mask : 355 / 1402 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1402, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1402, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1402, 1, 576) torch.bfloat16 - out : (1, 1402) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2965 - top-1 accuracy : 2/355 = 0.6% - - [2026-04-29 13:37:54] iteration 114/ 500 | consumed samples: 456 | elapsed time per iteration (ms): 5735.0 | throughput (token/sec/GPU): 900.4 | learning rate: 9.983430E-05 | global batch size: 4 | lm loss: 1.130560E+01 | loss scale: 1.0 | grad norm: 0.436 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1080]) -Dataloader batch - labels shape: torch.Size([1, 1080]) -Dataloader batch - attn_mask shape: torch.Size([1, 1080]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 57 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1080) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1080) torch.int64 - loss_mask : 333 / 1080 tokens (30.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1080, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1080, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1080, 1, 576) torch.bfloat16 - out : (1, 1080) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.3696 - top-1 accuracy : 4/333 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 58 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1418) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1418) torch.int64 - loss_mask : 350 / 1418 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1418, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1418, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1418, 1, 576) torch.bfloat16 - out : (1, 1418) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.2830 - top-1 accuracy : 3/350 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1122]) -Dataloader batch - labels shape: torch.Size([1, 1122]) -Dataloader batch - attn_mask shape: torch.Size([1, 1122]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 59 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1122) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1122) torch.int64 - loss_mask : 271 / 1122 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1122, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1122, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1122, 1, 576) torch.bfloat16 - out : (1, 1122) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 271 tokens) : 11.3065 - top-1 accuracy : 18/271 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1413]) -Dataloader batch - labels shape: torch.Size([1, 1413]) -Dataloader batch - attn_mask shape: torch.Size([1, 1413]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 60 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1413) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1413) torch.int64 - loss_mask : 261 / 1413 tokens (18.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1413, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1413, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1413, 1, 576) torch.bfloat16 - out : (1, 1413) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 261 tokens) : 11.0933 - top-1 accuracy : 24/261 = 9.2% - - [2026-04-29 13:38:00] iteration 115/ 500 | consumed samples: 460 | elapsed time per iteration (ms): 5585.5 | throughput (token/sec/GPU): 901.1 | learning rate: 9.980785E-05 | global batch size: 4 | lm loss: 1.127125E+01 | loss scale: 1.0 | grad norm: 0.342 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 61 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 381 / 1543 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.3198 - top-1 accuracy : 7/381 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1859]) -Dataloader batch - labels shape: torch.Size([1, 1859]) -Dataloader batch - attn_mask shape: torch.Size([1, 1859]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 62 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1859) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1859) torch.int64 - loss_mask : 490 / 1859 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1859, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1859, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1859, 1, 576) torch.bfloat16 - out : (1, 1859) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 490 tokens) : 11.3163 - top-1 accuracy : 20/490 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1174]) -Dataloader batch - labels shape: torch.Size([1, 1174]) -Dataloader batch - attn_mask shape: torch.Size([1, 1174]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 63 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1174) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1174) torch.int64 - loss_mask : 315 / 1174 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1174, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1174, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1174, 1, 576) torch.bfloat16 - out : (1, 1174) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.3298 - top-1 accuracy : 4/315 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1147]) -Dataloader batch - labels shape: torch.Size([1, 1147]) -Dataloader batch - attn_mask shape: torch.Size([1, 1147]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 64 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1147) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1147) torch.int64 - loss_mask : 305 / 1147 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1147, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1147, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1147, 1, 576) torch.bfloat16 - out : (1, 1147) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.2815 - top-1 accuracy : 21/305 = 6.9% - - [2026-04-29 13:38:06] iteration 116/ 500 | consumed samples: 464 | elapsed time per iteration (ms): 6082.3 | throughput (token/sec/GPU): 940.9 | learning rate: 9.977943E-05 | global batch size: 4 | lm loss: 1.131290E+01 | loss scale: 1.0 | grad norm: 0.389 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1556]) -Dataloader batch - labels shape: torch.Size([1, 1556]) -Dataloader batch - attn_mask shape: torch.Size([1, 1556]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 65 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1556) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1556) torch.int64 - loss_mask : 424 / 1556 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1556, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1556, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1556, 1, 576) torch.bfloat16 - out : (1, 1556) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 424 tokens) : 11.3758 - top-1 accuracy : 17/424 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1472]) -Dataloader batch - labels shape: torch.Size([1, 1472]) -Dataloader batch - attn_mask shape: torch.Size([1, 1472]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 66 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1472) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1472) torch.int64 - loss_mask : 317 / 1472 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1472, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1472, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1472, 1, 576) torch.bfloat16 - out : (1, 1472) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.1791 - top-1 accuracy : 33/317 = 10.4% - -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 67 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1423) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1423) torch.int64 - loss_mask : 311 / 1423 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1423, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1423, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1423, 1, 576) torch.bfloat16 - out : (1, 1423) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.1934 - top-1 accuracy : 24/311 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1082]) -Dataloader batch - labels shape: torch.Size([1, 1082]) -Dataloader batch - attn_mask shape: torch.Size([1, 1082]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 68 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1082) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1082) torch.int64 - loss_mask : 326 / 1082 tokens (30.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1082, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1082, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1082, 1, 576) torch.bfloat16 - out : (1, 1082) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.3697 - top-1 accuracy : 4/326 = 1.2% - - [2026-04-29 13:38:12] iteration 117/ 500 | consumed samples: 468 | elapsed time per iteration (ms): 5870.1 | throughput (token/sec/GPU): 942.6 | learning rate: 9.974907E-05 | global batch size: 4 | lm loss: 1.128796E+01 | loss scale: 1.0 | grad norm: 0.356 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1327]) -Dataloader batch - labels shape: torch.Size([1, 1327]) -Dataloader batch - attn_mask shape: torch.Size([1, 1327]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 69 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1327) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1327) torch.int64 - loss_mask : 379 / 1327 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1327, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1327, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1327, 1, 576) torch.bfloat16 - out : (1, 1327) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.4011 - top-1 accuracy : 5/379 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 70 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 402 / 1518 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.3245 - top-1 accuracy : 13/402 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 795]) -Dataloader batch - labels shape: torch.Size([1, 795]) -Dataloader batch - attn_mask shape: torch.Size([1, 795]) -Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) - -==================================================================== - PIPELINE — STEP 71 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 795) torch.int64 - images : (16, 3, 1024, 1024) torch.bfloat16 - labels : (1, 795) torch.int64 - loss_mask : 150 / 795 tokens (18.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (16, 3, 1024, 1024) torch.bfloat16 - out : (16, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (16, 3072) - out : (16, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (795, 1, 576) torch.bfloat16 grad=False - image slots : 16 tokens replaced with vision embeddings - combined : (795, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (795, 1, 576) torch.bfloat16 - out : (1, 795) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 150 tokens) : 11.1070 - top-1 accuracy : 3/150 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1325]) -Dataloader batch - labels shape: torch.Size([1, 1325]) -Dataloader batch - attn_mask shape: torch.Size([1, 1325]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 72 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1325) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1325) torch.int64 - loss_mask : 385 / 1325 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1325, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1325, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1325, 1, 576) torch.bfloat16 - out : (1, 1325) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.3468 - top-1 accuracy : 0/385 = 0.0% - - [2026-04-29 13:38:17] iteration 118/ 500 | consumed samples: 472 | elapsed time per iteration (ms): 5425.7 | throughput (token/sec/GPU): 915.1 | learning rate: 9.971676E-05 | global batch size: 4 | lm loss: 1.132831E+01 | loss scale: 1.0 | grad norm: 0.406 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1405]) -Dataloader batch - labels shape: torch.Size([1, 1405]) -Dataloader batch - attn_mask shape: torch.Size([1, 1405]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 73 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1405) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1405) torch.int64 - loss_mask : 304 / 1405 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1405, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1405, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1405, 1, 576) torch.bfloat16 - out : (1, 1405) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 304 tokens) : 11.2126 - top-1 accuracy : 29/304 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 908]) -Dataloader batch - labels shape: torch.Size([1, 908]) -Dataloader batch - attn_mask shape: torch.Size([1, 908]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 74 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 908) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 908) torch.int64 - loss_mask : 173 / 908 tokens (19.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (908, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (908, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (908, 1, 576) torch.bfloat16 - out : (1, 908) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 173 tokens) : 11.1921 - top-1 accuracy : 12/173 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1419]) -Dataloader batch - labels shape: torch.Size([1, 1419]) -Dataloader batch - attn_mask shape: torch.Size([1, 1419]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 75 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1419) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1419) torch.int64 - loss_mask : 442 / 1419 tokens (31.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1419, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1419, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1419, 1, 576) torch.bfloat16 - out : (1, 1419) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 442 tokens) : 11.4428 - top-1 accuracy : 2/442 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1258]) -Dataloader batch - labels shape: torch.Size([1, 1258]) -Dataloader batch - attn_mask shape: torch.Size([1, 1258]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 76 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1258) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1258) torch.int64 - loss_mask : 364 / 1258 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1258, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1258, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1258, 1, 576) torch.bfloat16 - out : (1, 1258) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.4051 - top-1 accuracy : 1/364 = 0.3% - - [2026-04-29 13:38:23] iteration 119/ 500 | consumed samples: 476 | elapsed time per iteration (ms): 5695.3 | throughput (token/sec/GPU): 876.2 | learning rate: 9.968249E-05 | global batch size: 4 | lm loss: 1.134378E+01 | loss scale: 1.0 | grad norm: 0.374 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1863]) -Dataloader batch - labels shape: torch.Size([1, 1863]) -Dataloader batch - attn_mask shape: torch.Size([1, 1863]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 77 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1863) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1863) torch.int64 - loss_mask : 474 / 1863 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1863, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1863, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1863, 1, 576) torch.bfloat16 - out : (1, 1863) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 474 tokens) : 11.3282 - top-1 accuracy : 14/474 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1513]) -Dataloader batch - labels shape: torch.Size([1, 1513]) -Dataloader batch - attn_mask shape: torch.Size([1, 1513]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 78 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1513) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1513) torch.int64 - loss_mask : 374 / 1513 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1513, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1513, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1513, 1, 576) torch.bfloat16 - out : (1, 1513) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.2422 - top-1 accuracy : 11/374 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1854]) -Dataloader batch - labels shape: torch.Size([1, 1854]) -Dataloader batch - attn_mask shape: torch.Size([1, 1854]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 79 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1854) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1854) torch.int64 - loss_mask : 476 / 1854 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1854, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1854, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1854, 1, 576) torch.bfloat16 - out : (1, 1854) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 476 tokens) : 11.3314 - top-1 accuracy : 8/476 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 972]) -Dataloader batch - labels shape: torch.Size([1, 972]) -Dataloader batch - attn_mask shape: torch.Size([1, 972]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 80 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 972) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 972) torch.int64 - loss_mask : 190 / 972 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (972, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (972, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (972, 1, 576) torch.bfloat16 - out : (1, 972) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 190 tokens) : 11.2078 - top-1 accuracy : 11/190 = 5.8% - - [2026-04-29 13:38:30] iteration 120/ 500 | consumed samples: 480 | elapsed time per iteration (ms): 6451.8 | throughput (token/sec/GPU): 961.3 | learning rate: 9.964628E-05 | global batch size: 4 | lm loss: 1.129286E+01 | loss scale: 1.0 | grad norm: 0.396 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (6382.55, 6382.55) - forward-compute ................................: (5025.89, 5025.89) - backward-compute ...............................: (1335.75, 1335.75) - batch-generator ................................: (463.31, 463.31) - layernorm-grads-all-reduce .....................: (0.03, 0.03) - embedding-grads-all-reduce .....................: (0.04, 0.04) - all-grads-sync .................................: (5.11, 5.11) - params-all-gather ..............................: (0.20, 0.20) - optimizer-copy-to-main-grad ....................: (0.17, 0.17) - optimizer-clip-main-grad .......................: (0.94, 0.94) - optimizer-count-zeros ..........................: (0.02, 0.02) - optimizer-inner-step ...........................: (1.62, 1.62) - optimizer-copy-main-to-model-params ............: (0.34, 0.34) - optimizer ......................................: (5.49, 5.49) -Dataloader batch - tokens shape: torch.Size([1, 1005]) -Dataloader batch - labels shape: torch.Size([1, 1005]) -Dataloader batch - attn_mask shape: torch.Size([1, 1005]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 81 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1005) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1005) torch.int64 - loss_mask : 233 / 1005 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1005, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1005, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1005, 1, 576) torch.bfloat16 - out : (1, 1005) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 233 tokens) : 11.2586 - top-1 accuracy : 14/233 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 82 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 324 / 1424 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.2872 - top-1 accuracy : 21/324 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 955]) -Dataloader batch - labels shape: torch.Size([1, 955]) -Dataloader batch - attn_mask shape: torch.Size([1, 955]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 83 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 955) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 955) torch.int64 - loss_mask : 210 / 955 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (955, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (955, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (955, 1, 576) torch.bfloat16 - out : (1, 955) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 210 tokens) : 11.2135 - top-1 accuracy : 10/210 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1355]) -Dataloader batch - labels shape: torch.Size([1, 1355]) -Dataloader batch - attn_mask shape: torch.Size([1, 1355]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 84 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1355) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1355) torch.int64 - loss_mask : 307 / 1355 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1355, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1355, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1355, 1, 576) torch.bfloat16 - out : (1, 1355) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.2149 - top-1 accuracy : 26/307 = 8.5% - - [2026-04-29 13:38:35] iteration 121/ 500 | consumed samples: 484 | elapsed time per iteration (ms): 5278.0 | throughput (token/sec/GPU): 897.9 | learning rate: 9.960811E-05 | global batch size: 4 | lm loss: 1.124593E+01 | loss scale: 1.0 | grad norm: 0.416 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 966]) -Dataloader batch - labels shape: torch.Size([1, 966]) -Dataloader batch - attn_mask shape: torch.Size([1, 966]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 85 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 966) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 966) torch.int64 - loss_mask : 190 / 966 tokens (19.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (966, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (966, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (966, 1, 576) torch.bfloat16 - out : (1, 966) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 190 tokens) : 11.1053 - top-1 accuracy : 7/190 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1702]) -Dataloader batch - labels shape: torch.Size([1, 1702]) -Dataloader batch - attn_mask shape: torch.Size([1, 1702]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 86 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1702) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1702) torch.int64 - loss_mask : 375 / 1702 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1702, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1702, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1702, 1, 576) torch.bfloat16 - out : (1, 1702) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.2441 - top-1 accuracy : 17/375 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1523]) -Dataloader batch - labels shape: torch.Size([1, 1523]) -Dataloader batch - attn_mask shape: torch.Size([1, 1523]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 87 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1523) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1523) torch.int64 - loss_mask : 339 / 1523 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1523, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1523, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1523, 1, 576) torch.bfloat16 - out : (1, 1523) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2565 - top-1 accuracy : 7/339 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1871]) -Dataloader batch - labels shape: torch.Size([1, 1871]) -Dataloader batch - attn_mask shape: torch.Size([1, 1871]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 88 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1871) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1871) torch.int64 - loss_mask : 521 / 1871 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1871, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1871, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1871, 1, 576) torch.bfloat16 - out : (1, 1871) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 521 tokens) : 11.3472 - top-1 accuracy : 26/521 = 5.0% - - [2026-04-29 13:38:42] iteration 122/ 500 | consumed samples: 488 | elapsed time per iteration (ms): 6901.9 | throughput (token/sec/GPU): 878.3 | learning rate: 9.956800E-05 | global batch size: 4 | lm loss: 1.126623E+01 | loss scale: 1.0 | grad norm: 0.454 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1550]) -Dataloader batch - labels shape: torch.Size([1, 1550]) -Dataloader batch - attn_mask shape: torch.Size([1, 1550]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 89 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1550) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1550) torch.int64 - loss_mask : 391 / 1550 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1550, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1550, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1550, 1, 576) torch.bfloat16 - out : (1, 1550) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2893 - top-1 accuracy : 9/391 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1147]) -Dataloader batch - labels shape: torch.Size([1, 1147]) -Dataloader batch - attn_mask shape: torch.Size([1, 1147]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 90 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1147) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1147) torch.int64 - loss_mask : 295 / 1147 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1147, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1147, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1147, 1, 576) torch.bfloat16 - out : (1, 1147) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.3031 - top-1 accuracy : 1/295 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1472]) -Dataloader batch - labels shape: torch.Size([1, 1472]) -Dataloader batch - attn_mask shape: torch.Size([1, 1472]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 91 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1472) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1472) torch.int64 - loss_mask : 311 / 1472 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1472, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1472, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1472, 1, 576) torch.bfloat16 - out : (1, 1472) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.1260 - top-1 accuracy : 32/311 = 10.3% - -Dataloader batch - tokens shape: torch.Size([1, 1887]) -Dataloader batch - labels shape: torch.Size([1, 1887]) -Dataloader batch - attn_mask shape: torch.Size([1, 1887]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 92 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1887) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1887) torch.int64 - loss_mask : 529 / 1887 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1887, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1887, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1887, 1, 576) torch.bfloat16 - out : (1, 1887) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 529 tokens) : 11.4178 - top-1 accuracy : 27/529 = 5.1% - - [2026-04-29 13:38:48] iteration 123/ 500 | consumed samples: 492 | elapsed time per iteration (ms): 6489.9 | throughput (token/sec/GPU): 933.1 | learning rate: 9.952595E-05 | global batch size: 4 | lm loss: 1.130322E+01 | loss scale: 1.0 | grad norm: 0.356 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1479]) -Dataloader batch - labels shape: torch.Size([1, 1479]) -Dataloader batch - attn_mask shape: torch.Size([1, 1479]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 93 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1479) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1479) torch.int64 - loss_mask : 326 / 1479 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1479, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1479, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1479, 1, 576) torch.bfloat16 - out : (1, 1479) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.1881 - top-1 accuracy : 32/326 = 9.8% - -Dataloader batch - tokens shape: torch.Size([1, 1729]) -Dataloader batch - labels shape: torch.Size([1, 1729]) -Dataloader batch - attn_mask shape: torch.Size([1, 1729]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 94 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1729) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1729) torch.int64 - loss_mask : 354 / 1729 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1729, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1729, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1729, 1, 576) torch.bfloat16 - out : (1, 1729) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2429 - top-1 accuracy : 26/354 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1684]) -Dataloader batch - labels shape: torch.Size([1, 1684]) -Dataloader batch - attn_mask shape: torch.Size([1, 1684]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 95 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1684) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1684) torch.int64 - loss_mask : 336 / 1684 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1684, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1684, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1684, 1, 576) torch.bfloat16 - out : (1, 1684) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.1762 - top-1 accuracy : 35/336 = 10.4% - -Dataloader batch - tokens shape: torch.Size([1, 1390]) -Dataloader batch - labels shape: torch.Size([1, 1390]) -Dataloader batch - attn_mask shape: torch.Size([1, 1390]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 96 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1390) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1390) torch.int64 - loss_mask : 334 / 1390 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1390, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1390, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1390, 1, 576) torch.bfloat16 - out : (1, 1390) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.2615 - top-1 accuracy : 6/334 = 1.8% - - [2026-04-29 13:38:55] iteration 124/ 500 | consumed samples: 496 | elapsed time per iteration (ms): 6799.2 | throughput (token/sec/GPU): 923.9 | learning rate: 9.948195E-05 | global batch size: 4 | lm loss: 1.121768E+01 | loss scale: 1.0 | grad norm: 0.365 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1783]) -Dataloader batch - labels shape: torch.Size([1, 1783]) -Dataloader batch - attn_mask shape: torch.Size([1, 1783]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 97 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1783) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1783) torch.int64 - loss_mask : 413 / 1783 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1783, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1783, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1783, 1, 576) torch.bfloat16 - out : (1, 1783) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2926 - top-1 accuracy : 32/413 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1129]) -Dataloader batch - labels shape: torch.Size([1, 1129]) -Dataloader batch - attn_mask shape: torch.Size([1, 1129]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 98 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1129) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1129) torch.int64 - loss_mask : 261 / 1129 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1129, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1129, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1129, 1, 576) torch.bfloat16 - out : (1, 1129) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 261 tokens) : 11.2471 - top-1 accuracy : 6/261 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 993]) -Dataloader batch - labels shape: torch.Size([1, 993]) -Dataloader batch - attn_mask shape: torch.Size([1, 993]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 99 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 993) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 993) torch.int64 - loss_mask : 243 / 993 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (993, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (993, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (993, 1, 576) torch.bfloat16 - out : (1, 993) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 243 tokens) : 11.3329 - top-1 accuracy : 19/243 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1943]) -Dataloader batch - labels shape: torch.Size([1, 1943]) -Dataloader batch - attn_mask shape: torch.Size([1, 1943]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 100 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1943) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1943) torch.int64 - loss_mask : 585 / 1943 tokens (30.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1943, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1943, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1943, 1, 576) torch.bfloat16 - out : (1, 1943) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 585 tokens) : 11.4314 - top-1 accuracy : 25/585 = 4.3% - - [2026-04-29 13:39:01] iteration 125/ 500 | consumed samples: 500 | elapsed time per iteration (ms): 6161.9 | throughput (token/sec/GPU): 949.1 | learning rate: 9.943601E-05 | global batch size: 4 | lm loss: 1.134525E+01 | loss scale: 1.0 | grad norm: 0.337 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1724]) -Dataloader batch - labels shape: torch.Size([1, 1724]) -Dataloader batch - attn_mask shape: torch.Size([1, 1724]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 101 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1724) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1724) torch.int64 - loss_mask : 376 / 1724 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1724, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1724, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1724, 1, 576) torch.bfloat16 - out : (1, 1724) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.2196 - top-1 accuracy : 4/376 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1793]) -Dataloader batch - labels shape: torch.Size([1, 1793]) -Dataloader batch - attn_mask shape: torch.Size([1, 1793]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 102 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1793) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1793) torch.int64 - loss_mask : 422 / 1793 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1793, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1793, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1793, 1, 576) torch.bfloat16 - out : (1, 1793) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.2956 - top-1 accuracy : 12/422 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1501]) -Dataloader batch - labels shape: torch.Size([1, 1501]) -Dataloader batch - attn_mask shape: torch.Size([1, 1501]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 103 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1501) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1501) torch.int64 - loss_mask : 376 / 1501 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1501, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1501, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1501, 1, 576) torch.bfloat16 - out : (1, 1501) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3195 - top-1 accuracy : 29/376 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1366]) -Dataloader batch - labels shape: torch.Size([1, 1366]) -Dataloader batch - attn_mask shape: torch.Size([1, 1366]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 104 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1366) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1366) torch.int64 - loss_mask : 307 / 1366 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1366, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1366, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1366, 1, 576) torch.bfloat16 - out : (1, 1366) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.1803 - top-1 accuracy : 21/307 = 6.8% - - [2026-04-29 13:39:08] iteration 126/ 500 | consumed samples: 504 | elapsed time per iteration (ms): 6732.4 | throughput (token/sec/GPU): 948.3 | learning rate: 9.938813E-05 | global batch size: 4 | lm loss: 1.125848E+01 | loss scale: 1.0 | grad norm: 0.296 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 105 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 359 / 1722 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.2120 - top-1 accuracy : 21/359 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1024]) -Dataloader batch - labels shape: torch.Size([1, 1024]) -Dataloader batch - attn_mask shape: torch.Size([1, 1024]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 106 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1024) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1024) torch.int64 - loss_mask : 253 / 1024 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1024, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1024, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1024, 1, 576) torch.bfloat16 - out : (1, 1024) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 253 tokens) : 11.2729 - top-1 accuracy : 4/253 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1071]) -Dataloader batch - labels shape: torch.Size([1, 1071]) -Dataloader batch - attn_mask shape: torch.Size([1, 1071]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 107 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1071) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1071) torch.int64 - loss_mask : 236 / 1071 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1071, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1071, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1071, 1, 576) torch.bfloat16 - out : (1, 1071) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 236 tokens) : 11.1930 - top-1 accuracy : 14/236 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1430]) -Dataloader batch - labels shape: torch.Size([1, 1430]) -Dataloader batch - attn_mask shape: torch.Size([1, 1430]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 108 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1430) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1430) torch.int64 - loss_mask : 378 / 1430 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1430, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1430, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1430, 1, 576) torch.bfloat16 - out : (1, 1430) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 378 tokens) : 11.3211 - top-1 accuracy : 16/378 = 4.2% - - [2026-04-29 13:39:14] iteration 127/ 500 | consumed samples: 508 | elapsed time per iteration (ms): 5641.1 | throughput (token/sec/GPU): 930.1 | learning rate: 9.933831E-05 | global batch size: 4 | lm loss: 1.125455E+01 | loss scale: 1.0 | grad norm: 0.306 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1192]) -Dataloader batch - labels shape: torch.Size([1, 1192]) -Dataloader batch - attn_mask shape: torch.Size([1, 1192]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 109 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1192) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1192) torch.int64 - loss_mask : 297 / 1192 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1192, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1192, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1192, 1, 576) torch.bfloat16 - out : (1, 1192) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.3082 - top-1 accuracy : 6/297 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 110 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 297 / 1374 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.1230 - top-1 accuracy : 16/297 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1417]) -Dataloader batch - labels shape: torch.Size([1, 1417]) -Dataloader batch - attn_mask shape: torch.Size([1, 1417]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 111 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1417) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1417) torch.int64 - loss_mask : 333 / 1417 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1417, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1417, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1417, 1, 576) torch.bfloat16 - out : (1, 1417) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.2373 - top-1 accuracy : 10/333 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1845]) -Dataloader batch - labels shape: torch.Size([1, 1845]) -Dataloader batch - attn_mask shape: torch.Size([1, 1845]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 112 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1845) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1845) torch.int64 - loss_mask : 477 / 1845 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1845, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1845, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1845, 1, 576) torch.bfloat16 - out : (1, 1845) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 477 tokens) : 11.2511 - top-1 accuracy : 15/477 = 3.1% - - [2026-04-29 13:39:20] iteration 128/ 500 | consumed samples: 512 | elapsed time per iteration (ms): 6345.3 | throughput (token/sec/GPU): 918.5 | learning rate: 9.928656E-05 | global batch size: 4 | lm loss: 1.123283E+01 | loss scale: 1.0 | grad norm: 0.373 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1405]) -Dataloader batch - labels shape: torch.Size([1, 1405]) -Dataloader batch - attn_mask shape: torch.Size([1, 1405]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 113 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1405) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1405) torch.int64 - loss_mask : 386 / 1405 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1405, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1405, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1405, 1, 576) torch.bfloat16 - out : (1, 1405) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3666 - top-1 accuracy : 1/386 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1031]) -Dataloader batch - labels shape: torch.Size([1, 1031]) -Dataloader batch - attn_mask shape: torch.Size([1, 1031]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 114 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1031) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1031) torch.int64 - loss_mask : 271 / 1031 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1031, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1031, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1031, 1, 576) torch.bfloat16 - out : (1, 1031) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 271 tokens) : 11.3204 - top-1 accuracy : 19/271 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1150]) -Dataloader batch - labels shape: torch.Size([1, 1150]) -Dataloader batch - attn_mask shape: torch.Size([1, 1150]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 115 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1150) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1150) torch.int64 - loss_mask : 343 / 1150 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1150, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1150, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1150, 1, 576) torch.bfloat16 - out : (1, 1150) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.4182 - top-1 accuracy : 4/343 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 116 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 306 / 1444 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.2358 - top-1 accuracy : 17/306 = 5.6% - - [2026-04-29 13:39:26] iteration 129/ 500 | consumed samples: 516 | elapsed time per iteration (ms): 5636.3 | throughput (token/sec/GPU): 892.4 | learning rate: 9.923288E-05 | global batch size: 4 | lm loss: 1.133993E+01 | loss scale: 1.0 | grad norm: 0.356 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1842]) -Dataloader batch - labels shape: torch.Size([1, 1842]) -Dataloader batch - attn_mask shape: torch.Size([1, 1842]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 117 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1842) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1842) torch.int64 - loss_mask : 493 / 1842 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1842, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1842, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1842, 1, 576) torch.bfloat16 - out : (1, 1842) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 493 tokens) : 11.3380 - top-1 accuracy : 11/493 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1432]) -Dataloader batch - labels shape: torch.Size([1, 1432]) -Dataloader batch - attn_mask shape: torch.Size([1, 1432]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 118 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1432) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1432) torch.int64 - loss_mask : 389 / 1432 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1432, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1432, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1432, 1, 576) torch.bfloat16 - out : (1, 1432) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.3105 - top-1 accuracy : 24/389 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1555]) -Dataloader batch - labels shape: torch.Size([1, 1555]) -Dataloader batch - attn_mask shape: torch.Size([1, 1555]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 119 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1555) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1555) torch.int64 - loss_mask : 395 / 1555 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1555, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1555, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1555, 1, 576) torch.bfloat16 - out : (1, 1555) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.3287 - top-1 accuracy : 3/395 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1463]) -Dataloader batch - labels shape: torch.Size([1, 1463]) -Dataloader batch - attn_mask shape: torch.Size([1, 1463]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 120 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1463) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1463) torch.int64 - loss_mask : 492 / 1463 tokens (33.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1463, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1463, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1463, 1, 576) torch.bfloat16 - out : (1, 1463) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 492 tokens) : 11.4790 - top-1 accuracy : 3/492 = 0.6% - - [2026-04-29 13:39:32] iteration 130/ 500 | consumed samples: 520 | elapsed time per iteration (ms): 6369.8 | throughput (token/sec/GPU): 987.8 | learning rate: 9.917726E-05 | global batch size: 4 | lm loss: 1.136906E+01 | loss scale: 1.0 | grad norm: 0.305 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1463]) -Dataloader batch - labels shape: torch.Size([1, 1463]) -Dataloader batch - attn_mask shape: torch.Size([1, 1463]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 121 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1463) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1463) torch.int64 - loss_mask : 354 / 1463 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1463, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1463, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1463, 1, 576) torch.bfloat16 - out : (1, 1463) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.3426 - top-1 accuracy : 20/354 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1955]) -Dataloader batch - labels shape: torch.Size([1, 1955]) -Dataloader batch - attn_mask shape: torch.Size([1, 1955]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 122 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1955) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1955) torch.int64 - loss_mask : 577 / 1955 tokens (29.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1955, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1955, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1955, 1, 576) torch.bfloat16 - out : (1, 1955) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 577 tokens) : 11.3841 - top-1 accuracy : 4/577 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 123 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 405 / 1543 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.3773 - top-1 accuracy : 2/405 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1656]) -Dataloader batch - labels shape: torch.Size([1, 1656]) -Dataloader batch - attn_mask shape: torch.Size([1, 1656]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 124 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1656) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1656) torch.int64 - loss_mask : 407 / 1656 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1656, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1656, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1656, 1, 576) torch.bfloat16 - out : (1, 1656) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.2852 - top-1 accuracy : 6/407 = 1.5% - - [2026-04-29 13:39:39] iteration 131/ 500 | consumed samples: 524 | elapsed time per iteration (ms): 6635.3 | throughput (token/sec/GPU): 997.2 | learning rate: 9.911971E-05 | global batch size: 4 | lm loss: 1.135101E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1014]) -Dataloader batch - labels shape: torch.Size([1, 1014]) -Dataloader batch - attn_mask shape: torch.Size([1, 1014]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 125 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1014) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1014) torch.int64 - loss_mask : 241 / 1014 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1014, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1014, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1014, 1, 576) torch.bfloat16 - out : (1, 1014) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 241 tokens) : 11.2600 - top-1 accuracy : 9/241 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1761]) -Dataloader batch - labels shape: torch.Size([1, 1761]) -Dataloader batch - attn_mask shape: torch.Size([1, 1761]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 126 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1761) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1761) torch.int64 - loss_mask : 391 / 1761 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1761, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1761, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1761, 1, 576) torch.bfloat16 - out : (1, 1761) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2597 - top-1 accuracy : 24/391 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1011]) -Dataloader batch - labels shape: torch.Size([1, 1011]) -Dataloader batch - attn_mask shape: torch.Size([1, 1011]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 127 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1011) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1011) torch.int64 - loss_mask : 237 / 1011 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1011, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1011, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1011, 1, 576) torch.bfloat16 - out : (1, 1011) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 237 tokens) : 11.2926 - top-1 accuracy : 4/237 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1867]) -Dataloader batch - labels shape: torch.Size([1, 1867]) -Dataloader batch - attn_mask shape: torch.Size([1, 1867]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 128 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1867) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1867) torch.int64 - loss_mask : 493 / 1867 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1867, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1867, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1867, 1, 576) torch.bfloat16 - out : (1, 1867) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 493 tokens) : 11.3229 - top-1 accuracy : 2/493 = 0.4% - - [2026-04-29 13:39:45] iteration 132/ 500 | consumed samples: 528 | elapsed time per iteration (ms): 5899.2 | throughput (token/sec/GPU): 958.3 | learning rate: 9.906023E-05 | global batch size: 4 | lm loss: 1.128837E+01 | loss scale: 1.0 | grad norm: 0.364 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1560]) -Dataloader batch - labels shape: torch.Size([1, 1560]) -Dataloader batch - attn_mask shape: torch.Size([1, 1560]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 129 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1560) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1560) torch.int64 - loss_mask : 377 / 1560 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1560, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1560, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1560, 1, 576) torch.bfloat16 - out : (1, 1560) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.2536 - top-1 accuracy : 22/377 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 130 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 311 / 1508 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.1884 - top-1 accuracy : 31/311 = 10.0% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 131 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 369 / 1507 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.1881 - top-1 accuracy : 12/369 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 132 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1412) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1412) torch.int64 - loss_mask : 315 / 1412 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1412, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1412, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1412, 1, 576) torch.bfloat16 - out : (1, 1412) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.2320 - top-1 accuracy : 16/315 = 5.1% - - [2026-04-29 13:39:51] iteration 133/ 500 | consumed samples: 532 | elapsed time per iteration (ms): 6505.5 | throughput (token/sec/GPU): 920.3 | learning rate: 9.899884E-05 | global batch size: 4 | lm loss: 1.121625E+01 | loss scale: 1.0 | grad norm: 0.369 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1089]) -Dataloader batch - labels shape: torch.Size([1, 1089]) -Dataloader batch - attn_mask shape: torch.Size([1, 1089]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 133 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1089) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1089) torch.int64 - loss_mask : 319 / 1089 tokens (29.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1089, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1089, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1089, 1, 576) torch.bfloat16 - out : (1, 1089) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.3691 - top-1 accuracy : 1/319 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1159]) -Dataloader batch - labels shape: torch.Size([1, 1159]) -Dataloader batch - attn_mask shape: torch.Size([1, 1159]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 134 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1159) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1159) torch.int64 - loss_mask : 282 / 1159 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1159, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1159, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1159, 1, 576) torch.bfloat16 - out : (1, 1159) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 282 tokens) : 11.3114 - top-1 accuracy : 15/282 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1196]) -Dataloader batch - labels shape: torch.Size([1, 1196]) -Dataloader batch - attn_mask shape: torch.Size([1, 1196]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 135 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1196) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1196) torch.int64 - loss_mask : 303 / 1196 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1196, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1196, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1196, 1, 576) torch.bfloat16 - out : (1, 1196) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.2786 - top-1 accuracy : 17/303 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1733]) -Dataloader batch - labels shape: torch.Size([1, 1733]) -Dataloader batch - attn_mask shape: torch.Size([1, 1733]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 136 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1733) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1733) torch.int64 - loss_mask : 379 / 1733 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1733, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1733, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1733, 1, 576) torch.bfloat16 - out : (1, 1733) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.1875 - top-1 accuracy : 33/379 = 8.7% - - [2026-04-29 13:39:57] iteration 134/ 500 | consumed samples: 536 | elapsed time per iteration (ms): 5562.2 | throughput (token/sec/GPU): 930.7 | learning rate: 9.893552E-05 | global batch size: 4 | lm loss: 1.128139E+01 | loss scale: 1.0 | grad norm: 0.311 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1718]) -Dataloader batch - labels shape: torch.Size([1, 1718]) -Dataloader batch - attn_mask shape: torch.Size([1, 1718]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 137 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1718) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1718) torch.int64 - loss_mask : 440 / 1718 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1718, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1718, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1718, 1, 576) torch.bfloat16 - out : (1, 1718) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 440 tokens) : 11.3084 - top-1 accuracy : 8/440 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1411]) -Dataloader batch - labels shape: torch.Size([1, 1411]) -Dataloader batch - attn_mask shape: torch.Size([1, 1411]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 138 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1411) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1411) torch.int64 - loss_mask : 363 / 1411 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1411, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1411, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1411, 1, 576) torch.bfloat16 - out : (1, 1411) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2617 - top-1 accuracy : 11/363 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1521]) -Dataloader batch - labels shape: torch.Size([1, 1521]) -Dataloader batch - attn_mask shape: torch.Size([1, 1521]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 139 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1521) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1521) torch.int64 - loss_mask : 346 / 1521 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1521, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1521, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1521, 1, 576) torch.bfloat16 - out : (1, 1521) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.2014 - top-1 accuracy : 25/346 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1800]) -Dataloader batch - labels shape: torch.Size([1, 1800]) -Dataloader batch - attn_mask shape: torch.Size([1, 1800]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 140 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1800) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1800) torch.int64 - loss_mask : 436 / 1800 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1800, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1800, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1800, 1, 576) torch.bfloat16 - out : (1, 1800) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.2668 - top-1 accuracy : 20/436 = 4.6% - - [2026-04-29 13:40:03] iteration 135/ 500 | consumed samples: 540 | elapsed time per iteration (ms): 6892.3 | throughput (token/sec/GPU): 935.8 | learning rate: 9.887027E-05 | global batch size: 4 | lm loss: 1.126288E+01 | loss scale: 1.0 | grad norm: 0.302 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 141 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 348 / 1496 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2142 - top-1 accuracy : 5/348 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1397]) -Dataloader batch - labels shape: torch.Size([1, 1397]) -Dataloader batch - attn_mask shape: torch.Size([1, 1397]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 142 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1397) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1397) torch.int64 - loss_mask : 333 / 1397 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1397, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1397, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1397, 1, 576) torch.bfloat16 - out : (1, 1397) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.2797 - top-1 accuracy : 13/333 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1748]) -Dataloader batch - labels shape: torch.Size([1, 1748]) -Dataloader batch - attn_mask shape: torch.Size([1, 1748]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 143 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1748) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1748) torch.int64 - loss_mask : 384 / 1748 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1748, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1748, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1748, 1, 576) torch.bfloat16 - out : (1, 1748) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.2823 - top-1 accuracy : 24/384 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 988]) -Dataloader batch - labels shape: torch.Size([1, 988]) -Dataloader batch - attn_mask shape: torch.Size([1, 988]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 144 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 988) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 988) torch.int64 - loss_mask : 223 / 988 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (988, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (988, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (988, 1, 576) torch.bfloat16 - out : (1, 988) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 223 tokens) : 11.2595 - top-1 accuracy : 20/223 = 9.0% - - [2026-04-29 13:40:10] iteration 136/ 500 | consumed samples: 544 | elapsed time per iteration (ms): 6420.5 | throughput (token/sec/GPU): 876.7 | learning rate: 9.880312E-05 | global batch size: 4 | lm loss: 1.125928E+01 | loss scale: 1.0 | grad norm: 0.323 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 145 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 287 / 1459 tokens (19.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 287 tokens) : 11.1366 - top-1 accuracy : 34/287 = 11.8% - -Dataloader batch - tokens shape: torch.Size([1, 1036]) -Dataloader batch - labels shape: torch.Size([1, 1036]) -Dataloader batch - attn_mask shape: torch.Size([1, 1036]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 146 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1036) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1036) torch.int64 - loss_mask : 220 / 1036 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1036, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1036, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1036, 1, 576) torch.bfloat16 - out : (1, 1036) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 220 tokens) : 11.1850 - top-1 accuracy : 4/220 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1356]) -Dataloader batch - labels shape: torch.Size([1, 1356]) -Dataloader batch - attn_mask shape: torch.Size([1, 1356]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 147 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1356) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1356) torch.int64 - loss_mask : 293 / 1356 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1356, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1356, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1356, 1, 576) torch.bfloat16 - out : (1, 1356) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 293 tokens) : 11.1797 - top-1 accuracy : 5/293 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1773]) -Dataloader batch - labels shape: torch.Size([1, 1773]) -Dataloader batch - attn_mask shape: torch.Size([1, 1773]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 148 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1773) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1773) torch.int64 - loss_mask : 413 / 1773 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1773, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1773, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1773, 1, 576) torch.bfloat16 - out : (1, 1773) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2286 - top-1 accuracy : 20/413 = 4.8% - - [2026-04-29 13:40:16] iteration 137/ 500 | consumed samples: 548 | elapsed time per iteration (ms): 6276.8 | throughput (token/sec/GPU): 896.0 | learning rate: 9.873405E-05 | global batch size: 4 | lm loss: 1.118712E+01 | loss scale: 1.0 | grad norm: 0.431 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1489]) -Dataloader batch - labels shape: torch.Size([1, 1489]) -Dataloader batch - attn_mask shape: torch.Size([1, 1489]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 149 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1489) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1489) torch.int64 - loss_mask : 386 / 1489 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1489, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1489, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1489, 1, 576) torch.bfloat16 - out : (1, 1489) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3305 - top-1 accuracy : 4/386 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1179]) -Dataloader batch - labels shape: torch.Size([1, 1179]) -Dataloader batch - attn_mask shape: torch.Size([1, 1179]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 150 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1179) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1179) torch.int64 - loss_mask : 315 / 1179 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1179, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1179, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1179, 1, 576) torch.bfloat16 - out : (1, 1179) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.2878 - top-1 accuracy : 4/315 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1906]) -Dataloader batch - labels shape: torch.Size([1, 1906]) -Dataloader batch - attn_mask shape: torch.Size([1, 1906]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 151 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1906) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1906) torch.int64 - loss_mask : 522 / 1906 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1906, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1906, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1906, 1, 576) torch.bfloat16 - out : (1, 1906) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 522 tokens) : 11.3656 - top-1 accuracy : 10/522 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1865]) -Dataloader batch - labels shape: torch.Size([1, 1865]) -Dataloader batch - attn_mask shape: torch.Size([1, 1865]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 152 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1865) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1865) torch.int64 - loss_mask : 500 / 1865 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1865, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1865, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1865, 1, 576) torch.bfloat16 - out : (1, 1865) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 500 tokens) : 11.3122 - top-1 accuracy : 16/500 = 3.2% - - [2026-04-29 13:40:23] iteration 138/ 500 | consumed samples: 552 | elapsed time per iteration (ms): 6658.9 | throughput (token/sec/GPU): 967.0 | learning rate: 9.866305E-05 | global batch size: 4 | lm loss: 1.132802E+01 | loss scale: 1.0 | grad norm: 0.325 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1197]) -Dataloader batch - labels shape: torch.Size([1, 1197]) -Dataloader batch - attn_mask shape: torch.Size([1, 1197]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 153 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1197) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1197) torch.int64 - loss_mask : 273 / 1197 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1197, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1197, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1197, 1, 576) torch.bfloat16 - out : (1, 1197) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 273 tokens) : 11.2877 - top-1 accuracy : 17/273 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1671]) -Dataloader batch - labels shape: torch.Size([1, 1671]) -Dataloader batch - attn_mask shape: torch.Size([1, 1671]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 154 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1671) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1671) torch.int64 - loss_mask : 385 / 1671 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1671, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1671, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1671, 1, 576) torch.bfloat16 - out : (1, 1671) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.2324 - top-1 accuracy : 27/385 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1860]) -Dataloader batch - labels shape: torch.Size([1, 1860]) -Dataloader batch - attn_mask shape: torch.Size([1, 1860]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 155 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1860) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1860) torch.int64 - loss_mask : 497 / 1860 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1860, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1860, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1860, 1, 576) torch.bfloat16 - out : (1, 1860) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 497 tokens) : 11.3333 - top-1 accuracy : 7/497 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 156 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 340 / 1518 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2361 - top-1 accuracy : 24/340 = 7.1% - - [2026-04-29 13:40:30] iteration 139/ 500 | consumed samples: 556 | elapsed time per iteration (ms): 7093.1 | throughput (token/sec/GPU): 880.6 | learning rate: 9.859016E-05 | global batch size: 4 | lm loss: 1.127688E+01 | loss scale: 1.0 | grad norm: 0.326 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 157 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 395 / 1552 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.3067 - top-1 accuracy : 25/395 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 158 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 419 / 1507 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.3629 - top-1 accuracy : 18/419 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1044]) -Dataloader batch - labels shape: torch.Size([1, 1044]) -Dataloader batch - attn_mask shape: torch.Size([1, 1044]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 159 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1044) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1044) torch.int64 - loss_mask : 284 / 1044 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1044, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1044, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1044, 1, 576) torch.bfloat16 - out : (1, 1044) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.3443 - top-1 accuracy : 3/284 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1879]) -Dataloader batch - labels shape: torch.Size([1, 1879]) -Dataloader batch - attn_mask shape: torch.Size([1, 1879]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 160 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1879) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1879) torch.int64 - loss_mask : 535 / 1879 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1879, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1879, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1879, 1, 576) torch.bfloat16 - out : (1, 1879) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 535 tokens) : 11.3024 - top-1 accuracy : 21/535 = 3.9% - - [2026-04-29 13:40:36] iteration 140/ 500 | consumed samples: 560 | elapsed time per iteration (ms): 6228.8 | throughput (token/sec/GPU): 960.4 | learning rate: 9.851536E-05 | global batch size: 4 | lm loss: 1.132625E+01 | loss scale: 1.0 | grad norm: 0.315 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (6168.38, 6168.38) - forward-compute ................................: (4876.52, 4876.52) - backward-compute ...............................: (1281.10, 1281.10) - batch-generator ................................: (448.33, 448.33) - layernorm-grads-all-reduce .....................: (0.03, 0.03) - embedding-grads-all-reduce .....................: (0.06, 0.06) - all-grads-sync .................................: (1.01, 1.01) - params-all-gather ..............................: (0.22, 0.22) - optimizer-copy-to-main-grad ....................: (0.19, 0.19) - optimizer-clip-main-grad .......................: (0.95, 0.95) - optimizer-count-zeros ..........................: (0.03, 0.03) - optimizer-inner-step ...........................: (1.55, 1.55) - optimizer-copy-main-to-model-params ............: (0.34, 0.34) - optimizer ......................................: (5.40, 5.40) -Dataloader batch - tokens shape: torch.Size([1, 1731]) -Dataloader batch - labels shape: torch.Size([1, 1731]) -Dataloader batch - attn_mask shape: torch.Size([1, 1731]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 161 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1731) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1731) torch.int64 - loss_mask : 366 / 1731 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1731, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1731, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1731, 1, 576) torch.bfloat16 - out : (1, 1731) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2368 - top-1 accuracy : 23/366 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1743]) -Dataloader batch - labels shape: torch.Size([1, 1743]) -Dataloader batch - attn_mask shape: torch.Size([1, 1743]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 162 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1743) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1743) torch.int64 - loss_mask : 388 / 1743 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1743, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1743, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1743, 1, 576) torch.bfloat16 - out : (1, 1743) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2276 - top-1 accuracy : 31/388 = 8.0% - -Dataloader batch - tokens shape: torch.Size([1, 1902]) -Dataloader batch - labels shape: torch.Size([1, 1902]) -Dataloader batch - attn_mask shape: torch.Size([1, 1902]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 163 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1902) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1902) torch.int64 - loss_mask : 550 / 1902 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1902, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1902, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1902, 1, 576) torch.bfloat16 - out : (1, 1902) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 550 tokens) : 11.3860 - top-1 accuracy : 12/550 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1708]) -Dataloader batch - labels shape: torch.Size([1, 1708]) -Dataloader batch - attn_mask shape: torch.Size([1, 1708]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 164 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1708) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1708) torch.int64 - loss_mask : 345 / 1708 tokens (20.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1708, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1708, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1708, 1, 576) torch.bfloat16 - out : (1, 1708) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.1546 - top-1 accuracy : 20/345 = 5.8% - - [2026-04-29 13:40:44] iteration 141/ 500 | consumed samples: 564 | elapsed time per iteration (ms): 7734.9 | throughput (token/sec/GPU): 915.8 | learning rate: 9.843866E-05 | global batch size: 4 | lm loss: 1.126720E+01 | loss scale: 1.0 | grad norm: 0.413 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1124]) -Dataloader batch - labels shape: torch.Size([1, 1124]) -Dataloader batch - attn_mask shape: torch.Size([1, 1124]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 165 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1124) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1124) torch.int64 - loss_mask : 278 / 1124 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1124, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1124, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1124, 1, 576) torch.bfloat16 - out : (1, 1124) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.3252 - top-1 accuracy : 0/278 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 166 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 392 / 1542 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3219 - top-1 accuracy : 27/392 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 167 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 278 / 1444 tokens (19.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.2068 - top-1 accuracy : 19/278 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1456]) -Dataloader batch - labels shape: torch.Size([1, 1456]) -Dataloader batch - attn_mask shape: torch.Size([1, 1456]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 168 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1456) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1456) torch.int64 - loss_mask : 306 / 1456 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1456, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1456, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1456, 1, 576) torch.bfloat16 - out : (1, 1456) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.2417 - top-1 accuracy : 16/306 = 5.2% - - [2026-04-29 13:40:50] iteration 142/ 500 | consumed samples: 568 | elapsed time per iteration (ms): 6053.5 | throughput (token/sec/GPU): 919.5 | learning rate: 9.836005E-05 | global batch size: 4 | lm loss: 1.127754E+01 | loss scale: 1.0 | grad norm: 0.369 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1147]) -Dataloader batch - labels shape: torch.Size([1, 1147]) -Dataloader batch - attn_mask shape: torch.Size([1, 1147]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 169 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1147) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1147) torch.int64 - loss_mask : 335 / 1147 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1147, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1147, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1147, 1, 576) torch.bfloat16 - out : (1, 1147) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.3285 - top-1 accuracy : 6/335 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1588]) -Dataloader batch - labels shape: torch.Size([1, 1588]) -Dataloader batch - attn_mask shape: torch.Size([1, 1588]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 170 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1588) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1588) torch.int64 - loss_mask : 332 / 1588 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1588, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1588, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1588, 1, 576) torch.bfloat16 - out : (1, 1588) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.1775 - top-1 accuracy : 34/332 = 10.2% - -Dataloader batch - tokens shape: torch.Size([1, 1866]) -Dataloader batch - labels shape: torch.Size([1, 1866]) -Dataloader batch - attn_mask shape: torch.Size([1, 1866]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 171 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1866) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1866) torch.int64 - loss_mask : 508 / 1866 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1866, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1866, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1866, 1, 576) torch.bfloat16 - out : (1, 1866) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 508 tokens) : 11.3412 - top-1 accuracy : 23/508 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1108]) -Dataloader batch - labels shape: torch.Size([1, 1108]) -Dataloader batch - attn_mask shape: torch.Size([1, 1108]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 172 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1108) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1108) torch.int64 - loss_mask : 299 / 1108 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1108, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1108, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1108, 1, 576) torch.bfloat16 - out : (1, 1108) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 299 tokens) : 11.3469 - top-1 accuracy : 2/299 = 0.7% - - [2026-04-29 13:40:56] iteration 143/ 500 | consumed samples: 572 | elapsed time per iteration (ms): 5952.9 | throughput (token/sec/GPU): 959.0 | learning rate: 9.827955E-05 | global batch size: 4 | lm loss: 1.130261E+01 | loss scale: 1.0 | grad norm: 0.289 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1404]) -Dataloader batch - labels shape: torch.Size([1, 1404]) -Dataloader batch - attn_mask shape: torch.Size([1, 1404]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 173 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1404) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1404) torch.int64 - loss_mask : 399 / 1404 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1404, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1404, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1404, 1, 576) torch.bfloat16 - out : (1, 1404) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.3410 - top-1 accuracy : 9/399 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 174 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 354 / 1438 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.3888 - top-1 accuracy : 10/354 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1669]) -Dataloader batch - labels shape: torch.Size([1, 1669]) -Dataloader batch - attn_mask shape: torch.Size([1, 1669]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 175 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1669) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1669) torch.int64 - loss_mask : 419 / 1669 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1669, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1669, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1669, 1, 576) torch.bfloat16 - out : (1, 1669) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.2859 - top-1 accuracy : 15/419 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1390]) -Dataloader batch - labels shape: torch.Size([1, 1390]) -Dataloader batch - attn_mask shape: torch.Size([1, 1390]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 176 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1390) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1390) torch.int64 - loss_mask : 307 / 1390 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1390, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1390, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1390, 1, 576) torch.bfloat16 - out : (1, 1390) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.2623 - top-1 accuracy : 18/307 = 5.9% - - [2026-04-29 13:41:02] iteration 144/ 500 | consumed samples: 576 | elapsed time per iteration (ms): 6147.8 | throughput (token/sec/GPU): 959.9 | learning rate: 9.819716E-05 | global batch size: 4 | lm loss: 1.132049E+01 | loss scale: 1.0 | grad norm: 0.281 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1557]) -Dataloader batch - labels shape: torch.Size([1, 1557]) -Dataloader batch - attn_mask shape: torch.Size([1, 1557]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 177 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1557) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1557) torch.int64 - loss_mask : 408 / 1557 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1557, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1557, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1557, 1, 576) torch.bfloat16 - out : (1, 1557) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3568 - top-1 accuracy : 21/408 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1464]) -Dataloader batch - labels shape: torch.Size([1, 1464]) -Dataloader batch - attn_mask shape: torch.Size([1, 1464]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 178 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1464) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1464) torch.int64 - loss_mask : 406 / 1464 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1464, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1464, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1464, 1, 576) torch.bfloat16 - out : (1, 1464) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.3400 - top-1 accuracy : 0/406 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1538]) -Dataloader batch - labels shape: torch.Size([1, 1538]) -Dataloader batch - attn_mask shape: torch.Size([1, 1538]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 179 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1538) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1538) torch.int64 - loss_mask : 384 / 1538 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1538, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1538, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1538, 1, 576) torch.bfloat16 - out : (1, 1538) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.2916 - top-1 accuracy : 16/384 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1165]) -Dataloader batch - labels shape: torch.Size([1, 1165]) -Dataloader batch - attn_mask shape: torch.Size([1, 1165]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 180 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1165) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1165) torch.int64 - loss_mask : 233 / 1165 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1165, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1165, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1165, 1, 576) torch.bfloat16 - out : (1, 1165) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 233 tokens) : 11.2029 - top-1 accuracy : 21/233 = 9.0% - - [2026-04-29 13:41:08] iteration 145/ 500 | consumed samples: 580 | elapsed time per iteration (ms): 6158.6 | throughput (token/sec/GPU): 929.4 | learning rate: 9.811288E-05 | global batch size: 4 | lm loss: 1.130945E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1007]) -Dataloader batch - labels shape: torch.Size([1, 1007]) -Dataloader batch - attn_mask shape: torch.Size([1, 1007]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 181 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1007) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1007) torch.int64 - loss_mask : 244 / 1007 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1007, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1007, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1007, 1, 576) torch.bfloat16 - out : (1, 1007) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 244 tokens) : 11.2514 - top-1 accuracy : 18/244 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1703]) -Dataloader batch - labels shape: torch.Size([1, 1703]) -Dataloader batch - attn_mask shape: torch.Size([1, 1703]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 182 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1703) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1703) torch.int64 - loss_mask : 375 / 1703 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1703, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1703, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1703, 1, 576) torch.bfloat16 - out : (1, 1703) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.1896 - top-1 accuracy : 42/375 = 11.2% - -Dataloader batch - tokens shape: torch.Size([1, 1032]) -Dataloader batch - labels shape: torch.Size([1, 1032]) -Dataloader batch - attn_mask shape: torch.Size([1, 1032]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 183 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1032) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1032) torch.int64 - loss_mask : 275 / 1032 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1032, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1032, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1032, 1, 576) torch.bfloat16 - out : (1, 1032) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 275 tokens) : 11.2489 - top-1 accuracy : 11/275 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1033]) -Dataloader batch - labels shape: torch.Size([1, 1033]) -Dataloader batch - attn_mask shape: torch.Size([1, 1033]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 184 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1033) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1033) torch.int64 - loss_mask : 228 / 1033 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1033, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1033, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1033, 1, 576) torch.bfloat16 - out : (1, 1033) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 228 tokens) : 11.1860 - top-1 accuracy : 7/228 = 3.1% - - [2026-04-29 13:41:14] iteration 146/ 500 | consumed samples: 584 | elapsed time per iteration (ms): 5455.1 | throughput (token/sec/GPU): 875.3 | learning rate: 9.802671E-05 | global batch size: 4 | lm loss: 1.121686E+01 | loss scale: 1.0 | grad norm: 0.343 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1579]) -Dataloader batch - labels shape: torch.Size([1, 1579]) -Dataloader batch - attn_mask shape: torch.Size([1, 1579]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 185 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1579) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1579) torch.int64 - loss_mask : 363 / 1579 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1579, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1579, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1579, 1, 576) torch.bfloat16 - out : (1, 1579) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2095 - top-1 accuracy : 24/363 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 186 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 295 / 1400 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.2497 - top-1 accuracy : 28/295 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 187 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 320 / 1380 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2186 - top-1 accuracy : 13/320 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1506]) -Dataloader batch - labels shape: torch.Size([1, 1506]) -Dataloader batch - attn_mask shape: torch.Size([1, 1506]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 188 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1506) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1506) torch.int64 - loss_mask : 386 / 1506 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1506, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1506, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1506, 1, 576) torch.bfloat16 - out : (1, 1506) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2727 - top-1 accuracy : 27/386 = 7.0% - - [2026-04-29 13:41:20] iteration 147/ 500 | consumed samples: 588 | elapsed time per iteration (ms): 6399.2 | throughput (token/sec/GPU): 916.5 | learning rate: 9.793865E-05 | global batch size: 4 | lm loss: 1.123823E+01 | loss scale: 1.0 | grad norm: 0.283 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1152]) -Dataloader batch - labels shape: torch.Size([1, 1152]) -Dataloader batch - attn_mask shape: torch.Size([1, 1152]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 189 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1152) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1152) torch.int64 - loss_mask : 278 / 1152 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1152, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1152, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1152, 1, 576) torch.bfloat16 - out : (1, 1152) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.3118 - top-1 accuracy : 4/278 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1476]) -Dataloader batch - labels shape: torch.Size([1, 1476]) -Dataloader batch - attn_mask shape: torch.Size([1, 1476]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 190 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1476) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1476) torch.int64 - loss_mask : 411 / 1476 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1476, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1476, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1476, 1, 576) torch.bfloat16 - out : (1, 1476) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.3982 - top-1 accuracy : 3/411 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1599]) -Dataloader batch - labels shape: torch.Size([1, 1599]) -Dataloader batch - attn_mask shape: torch.Size([1, 1599]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 191 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1599) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1599) torch.int64 - loss_mask : 467 / 1599 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1599, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1599, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1599, 1, 576) torch.bfloat16 - out : (1, 1599) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 467 tokens) : 11.3466 - top-1 accuracy : 18/467 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1485]) -Dataloader batch - labels shape: torch.Size([1, 1485]) -Dataloader batch - attn_mask shape: torch.Size([1, 1485]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 192 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1485) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1485) torch.int64 - loss_mask : 316 / 1485 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1485, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1485, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1485, 1, 576) torch.bfloat16 - out : (1, 1485) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.2275 - top-1 accuracy : 33/316 = 10.4% - - [2026-04-29 13:41:26] iteration 148/ 500 | consumed samples: 592 | elapsed time per iteration (ms): 5977.4 | throughput (token/sec/GPU): 955.6 | learning rate: 9.784872E-05 | global batch size: 4 | lm loss: 1.132890E+01 | loss scale: 1.0 | grad norm: 0.326 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1372]) -Dataloader batch - labels shape: torch.Size([1, 1372]) -Dataloader batch - attn_mask shape: torch.Size([1, 1372]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 193 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1372) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1372) torch.int64 - loss_mask : 312 / 1372 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1372, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1372, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1372, 1, 576) torch.bfloat16 - out : (1, 1372) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.3006 - top-1 accuracy : 8/312 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1891]) -Dataloader batch - labels shape: torch.Size([1, 1891]) -Dataloader batch - attn_mask shape: torch.Size([1, 1891]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 194 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1891) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1891) torch.int64 - loss_mask : 519 / 1891 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1891, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1891, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1891, 1, 576) torch.bfloat16 - out : (1, 1891) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 519 tokens) : 11.3463 - top-1 accuracy : 12/519 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1749]) -Dataloader batch - labels shape: torch.Size([1, 1749]) -Dataloader batch - attn_mask shape: torch.Size([1, 1749]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 195 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1749) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1749) torch.int64 - loss_mask : 388 / 1749 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1749, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1749, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1749, 1, 576) torch.bfloat16 - out : (1, 1749) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2287 - top-1 accuracy : 25/388 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1655]) -Dataloader batch - labels shape: torch.Size([1, 1655]) -Dataloader batch - attn_mask shape: torch.Size([1, 1655]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 196 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1655) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1655) torch.int64 - loss_mask : 378 / 1655 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1655, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1655, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1655, 1, 576) torch.bfloat16 - out : (1, 1655) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 378 tokens) : 11.3206 - top-1 accuracy : 24/378 = 6.3% - - [2026-04-29 13:41:33] iteration 149/ 500 | consumed samples: 596 | elapsed time per iteration (ms): 6897.5 | throughput (token/sec/GPU): 966.6 | learning rate: 9.775690E-05 | global batch size: 4 | lm loss: 1.130273E+01 | loss scale: 1.0 | grad norm: 0.296 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1531]) -Dataloader batch - labels shape: torch.Size([1, 1531]) -Dataloader batch - attn_mask shape: torch.Size([1, 1531]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 197 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1531) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1531) torch.int64 - loss_mask : 382 / 1531 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1531, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1531, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1531, 1, 576) torch.bfloat16 - out : (1, 1531) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.3077 - top-1 accuracy : 18/382 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1870]) -Dataloader batch - labels shape: torch.Size([1, 1870]) -Dataloader batch - attn_mask shape: torch.Size([1, 1870]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 198 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1870) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1870) torch.int64 - loss_mask : 520 / 1870 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1870, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1870, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1870, 1, 576) torch.bfloat16 - out : (1, 1870) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 520 tokens) : 11.3502 - top-1 accuracy : 3/520 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1138]) -Dataloader batch - labels shape: torch.Size([1, 1138]) -Dataloader batch - attn_mask shape: torch.Size([1, 1138]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 199 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1138) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1138) torch.int64 - loss_mask : 328 / 1138 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1138, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1138, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1138, 1, 576) torch.bfloat16 - out : (1, 1138) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.3422 - top-1 accuracy : 2/328 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1396]) -Dataloader batch - labels shape: torch.Size([1, 1396]) -Dataloader batch - attn_mask shape: torch.Size([1, 1396]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 200 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1396) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1396) torch.int64 - loss_mask : 316 / 1396 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1396, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1396, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1396, 1, 576) torch.bfloat16 - out : (1, 1396) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.2467 - top-1 accuracy : 27/316 = 8.5% - - [2026-04-29 13:41:39] iteration 150/ 500 | consumed samples: 600 | elapsed time per iteration (ms): 6135.0 | throughput (token/sec/GPU): 967.4 | learning rate: 9.766321E-05 | global batch size: 4 | lm loss: 1.131686E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 150 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 150 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 150 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1894.41, 1894.41) -Dataloader batch - tokens shape: torch.Size([1, 1705]) -Dataloader batch - labels shape: torch.Size([1, 1705]) -Dataloader batch - attn_mask shape: torch.Size([1, 1705]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 201 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1705) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1705) torch.int64 - loss_mask : 357 / 1705 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1705, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1705, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1705, 1, 576) torch.bfloat16 - out : (1, 1705) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.1504 - top-1 accuracy : 34/357 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1012]) -Dataloader batch - labels shape: torch.Size([1, 1012]) -Dataloader batch - attn_mask shape: torch.Size([1, 1012]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 202 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1012) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1012) torch.int64 - loss_mask : 261 / 1012 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1012, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1012, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1012, 1, 576) torch.bfloat16 - out : (1, 1012) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 261 tokens) : 11.2761 - top-1 accuracy : 3/261 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1557]) -Dataloader batch - labels shape: torch.Size([1, 1557]) -Dataloader batch - attn_mask shape: torch.Size([1, 1557]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 203 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1557) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1557) torch.int64 - loss_mask : 436 / 1557 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1557, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1557, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1557, 1, 576) torch.bfloat16 - out : (1, 1557) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.3434 - top-1 accuracy : 29/436 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1373]) -Dataloader batch - labels shape: torch.Size([1, 1373]) -Dataloader batch - attn_mask shape: torch.Size([1, 1373]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 204 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1373) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1373) torch.int64 - loss_mask : 312 / 1373 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1373, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1373, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1373, 1, 576) torch.bfloat16 - out : (1, 1373) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.2506 - top-1 accuracy : 17/312 = 5.4% - - [2026-04-29 13:41:47] iteration 151/ 500 | consumed samples: 604 | elapsed time per iteration (ms): 5973.0 | throughput (token/sec/GPU): 945.4 | learning rate: 9.756766E-05 | global batch size: 4 | lm loss: 1.125890E+01 | loss scale: 1.0 | grad norm: 0.309 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1426]) -Dataloader batch - labels shape: torch.Size([1, 1426]) -Dataloader batch - attn_mask shape: torch.Size([1, 1426]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 205 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1426) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1426) torch.int64 - loss_mask : 369 / 1426 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1426, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1426, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1426, 1, 576) torch.bfloat16 - out : (1, 1426) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2960 - top-1 accuracy : 21/369 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1203]) -Dataloader batch - labels shape: torch.Size([1, 1203]) -Dataloader batch - attn_mask shape: torch.Size([1, 1203]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 206 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1203) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1203) torch.int64 - loss_mask : 307 / 1203 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1203, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1203, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1203, 1, 576) torch.bfloat16 - out : (1, 1203) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.2837 - top-1 accuracy : 7/307 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1707]) -Dataloader batch - labels shape: torch.Size([1, 1707]) -Dataloader batch - attn_mask shape: torch.Size([1, 1707]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 207 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1707) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1707) torch.int64 - loss_mask : 351 / 1707 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1707, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1707, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1707, 1, 576) torch.bfloat16 - out : (1, 1707) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.1582 - top-1 accuracy : 38/351 = 10.8% - -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 208 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1423) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1423) torch.int64 - loss_mask : 317 / 1423 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1423, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1423, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1423, 1, 576) torch.bfloat16 - out : (1, 1423) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2470 - top-1 accuracy : 10/317 = 3.2% - - [2026-04-29 13:41:52] iteration 152/ 500 | consumed samples: 608 | elapsed time per iteration (ms): 4608.9 | throughput (token/sec/GPU): 1249.5 | learning rate: 9.747023E-05 | global batch size: 4 | lm loss: 1.124564E+01 | loss scale: 1.0 | grad norm: 0.297 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 209 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 351 / 1532 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.2095 - top-1 accuracy : 19/351 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 210 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1385) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1385) torch.int64 - loss_mask : 311 / 1385 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1385, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1385, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1385, 1, 576) torch.bfloat16 - out : (1, 1385) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2142 - top-1 accuracy : 13/311 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1465]) -Dataloader batch - labels shape: torch.Size([1, 1465]) -Dataloader batch - attn_mask shape: torch.Size([1, 1465]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 211 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1465) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1465) torch.int64 - loss_mask : 412 / 1465 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1465, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1465, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1465, 1, 576) torch.bfloat16 - out : (1, 1465) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3653 - top-1 accuracy : 2/412 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1057]) -Dataloader batch - labels shape: torch.Size([1, 1057]) -Dataloader batch - attn_mask shape: torch.Size([1, 1057]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 212 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1057) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1057) torch.int64 - loss_mask : 288 / 1057 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1057, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1057, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1057, 1, 576) torch.bfloat16 - out : (1, 1057) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 288 tokens) : 11.3375 - top-1 accuracy : 9/288 = 3.1% - - [2026-04-29 13:41:56] iteration 153/ 500 | consumed samples: 612 | elapsed time per iteration (ms): 4375.9 | throughput (token/sec/GPU): 1242.9 | learning rate: 9.737095E-05 | global batch size: 4 | lm loss: 1.128477E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1464]) -Dataloader batch - labels shape: torch.Size([1, 1464]) -Dataloader batch - attn_mask shape: torch.Size([1, 1464]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 213 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1464) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1464) torch.int64 - loss_mask : 412 / 1464 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1464, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1464, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1464, 1, 576) torch.bfloat16 - out : (1, 1464) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3665 - top-1 accuracy : 12/412 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 214 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 370 / 1504 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.2783 - top-1 accuracy : 0/370 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1403]) -Dataloader batch - labels shape: torch.Size([1, 1403]) -Dataloader batch - attn_mask shape: torch.Size([1, 1403]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 215 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1403) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1403) torch.int64 - loss_mask : 354 / 1403 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1403, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1403, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1403, 1, 576) torch.bfloat16 - out : (1, 1403) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.3243 - top-1 accuracy : 17/354 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1571]) -Dataloader batch - labels shape: torch.Size([1, 1571]) -Dataloader batch - attn_mask shape: torch.Size([1, 1571]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 216 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1571) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1571) torch.int64 - loss_mask : 354 / 1571 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1571, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1571, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1571, 1, 576) torch.bfloat16 - out : (1, 1571) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.1740 - top-1 accuracy : 36/354 = 10.2% - - [2026-04-29 13:42:01] iteration 154/ 500 | consumed samples: 616 | elapsed time per iteration (ms): 4675.6 | throughput (token/sec/GPU): 1270.8 | learning rate: 9.726981E-05 | global batch size: 4 | lm loss: 1.128883E+01 | loss scale: 1.0 | grad norm: 0.315 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1465]) -Dataloader batch - labels shape: torch.Size([1, 1465]) -Dataloader batch - attn_mask shape: torch.Size([1, 1465]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 217 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1465) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1465) torch.int64 - loss_mask : 292 / 1465 tokens (19.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1465, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1465, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1465, 1, 576) torch.bfloat16 - out : (1, 1465) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 292 tokens) : 11.1569 - top-1 accuracy : 30/292 = 10.3% - -Dataloader batch - tokens shape: torch.Size([1, 1562]) -Dataloader batch - labels shape: torch.Size([1, 1562]) -Dataloader batch - attn_mask shape: torch.Size([1, 1562]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 218 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1562) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1562) torch.int64 - loss_mask : 413 / 1562 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1562, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1562, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1562, 1, 576) torch.bfloat16 - out : (1, 1562) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.3073 - top-1 accuracy : 25/413 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1821]) -Dataloader batch - labels shape: torch.Size([1, 1821]) -Dataloader batch - attn_mask shape: torch.Size([1, 1821]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 219 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1821) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1821) torch.int64 - loss_mask : 472 / 1821 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1821, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1821, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1821, 1, 576) torch.bfloat16 - out : (1, 1821) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 472 tokens) : 11.2717 - top-1 accuracy : 3/472 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1831]) -Dataloader batch - labels shape: torch.Size([1, 1831]) -Dataloader batch - attn_mask shape: torch.Size([1, 1831]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 220 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1831) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1831) torch.int64 - loss_mask : 460 / 1831 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1831, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1831, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1831, 1, 576) torch.bfloat16 - out : (1, 1831) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 460 tokens) : 11.3079 - top-1 accuracy : 10/460 = 2.2% - - [2026-04-29 13:42:06] iteration 155/ 500 | consumed samples: 620 | elapsed time per iteration (ms): 5185.7 | throughput (token/sec/GPU): 1288.0 | learning rate: 9.716682E-05 | global batch size: 4 | lm loss: 1.127035E+01 | loss scale: 1.0 | grad norm: 0.294 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 221 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 393 / 1750 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 393 tokens) : 11.2208 - top-1 accuracy : 27/393 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1737]) -Dataloader batch - labels shape: torch.Size([1, 1737]) -Dataloader batch - attn_mask shape: torch.Size([1, 1737]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 222 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1737) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1737) torch.int64 - loss_mask : 367 / 1737 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1737, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1737, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1737, 1, 576) torch.bfloat16 - out : (1, 1737) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2263 - top-1 accuracy : 35/367 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1398]) -Dataloader batch - labels shape: torch.Size([1, 1398]) -Dataloader batch - attn_mask shape: torch.Size([1, 1398]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 223 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1398) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1398) torch.int64 - loss_mask : 326 / 1398 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1398, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1398, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1398, 1, 576) torch.bfloat16 - out : (1, 1398) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2029 - top-1 accuracy : 7/326 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 224 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 359 / 1422 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.3056 - top-1 accuracy : 17/359 = 4.7% - - [2026-04-29 13:42:11] iteration 156/ 500 | consumed samples: 624 | elapsed time per iteration (ms): 4904.8 | throughput (token/sec/GPU): 1285.9 | learning rate: 9.706197E-05 | global batch size: 4 | lm loss: 1.123921E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1244]) -Dataloader batch - labels shape: torch.Size([1, 1244]) -Dataloader batch - attn_mask shape: torch.Size([1, 1244]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 225 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1244) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1244) torch.int64 - loss_mask : 361 / 1244 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1244, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1244, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1244, 1, 576) torch.bfloat16 - out : (1, 1244) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.3676 - top-1 accuracy : 1/361 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1408]) -Dataloader batch - labels shape: torch.Size([1, 1408]) -Dataloader batch - attn_mask shape: torch.Size([1, 1408]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 226 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1408) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1408) torch.int64 - loss_mask : 339 / 1408 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1408, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1408, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1408, 1, 576) torch.bfloat16 - out : (1, 1408) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2908 - top-1 accuracy : 25/339 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 227 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 357 / 1520 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2839 - top-1 accuracy : 4/357 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1583]) -Dataloader batch - labels shape: torch.Size([1, 1583]) -Dataloader batch - attn_mask shape: torch.Size([1, 1583]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 228 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1583) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1583) torch.int64 - loss_mask : 429 / 1583 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1583, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1583, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1583, 1, 576) torch.bfloat16 - out : (1, 1583) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 429 tokens) : 11.3458 - top-1 accuracy : 6/429 = 1.4% - - [2026-04-29 13:42:15] iteration 157/ 500 | consumed samples: 628 | elapsed time per iteration (ms): 4460.2 | throughput (token/sec/GPU): 1290.3 | learning rate: 9.695529E-05 | global batch size: 4 | lm loss: 1.132369E+01 | loss scale: 1.0 | grad norm: 0.334 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1164]) -Dataloader batch - labels shape: torch.Size([1, 1164]) -Dataloader batch - attn_mask shape: torch.Size([1, 1164]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 229 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1164) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1164) torch.int64 - loss_mask : 311 / 1164 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1164, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1164, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1164, 1, 576) torch.bfloat16 - out : (1, 1164) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.3596 - top-1 accuracy : 14/311 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 230 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1385) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1385) torch.int64 - loss_mask : 311 / 1385 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1385, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1385, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1385, 1, 576) torch.bfloat16 - out : (1, 1385) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2497 - top-1 accuracy : 10/311 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 231 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 440 / 1431 tokens (30.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 440 tokens) : 11.4447 - top-1 accuracy : 2/440 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1115]) -Dataloader batch - labels shape: torch.Size([1, 1115]) -Dataloader batch - attn_mask shape: torch.Size([1, 1115]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 232 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1115) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1115) torch.int64 - loss_mask : 275 / 1115 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1115, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1115, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1115, 1, 576) torch.bfloat16 - out : (1, 1115) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 275 tokens) : 11.2796 - top-1 accuracy : 21/275 = 7.6% - - [2026-04-29 13:42:19] iteration 158/ 500 | consumed samples: 632 | elapsed time per iteration (ms): 4117.9 | throughput (token/sec/GPU): 1237.3 | learning rate: 9.684676E-05 | global batch size: 4 | lm loss: 1.134556E+01 | loss scale: 1.0 | grad norm: 0.317 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1625]) -Dataloader batch - labels shape: torch.Size([1, 1625]) -Dataloader batch - attn_mask shape: torch.Size([1, 1625]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 233 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1625) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1625) torch.int64 - loss_mask : 353 / 1625 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1625, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1625, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1625, 1, 576) torch.bfloat16 - out : (1, 1625) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.2092 - top-1 accuracy : 30/353 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1221]) -Dataloader batch - labels shape: torch.Size([1, 1221]) -Dataloader batch - attn_mask shape: torch.Size([1, 1221]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 234 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1221) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1221) torch.int64 - loss_mask : 290 / 1221 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1221, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1221, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1221, 1, 576) torch.bfloat16 - out : (1, 1221) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.2792 - top-1 accuracy : 19/290 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1775]) -Dataloader batch - labels shape: torch.Size([1, 1775]) -Dataloader batch - attn_mask shape: torch.Size([1, 1775]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 235 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1775) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1775) torch.int64 - loss_mask : 412 / 1775 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1775, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1775, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1775, 1, 576) torch.bfloat16 - out : (1, 1775) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.2117 - top-1 accuracy : 19/412 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1259]) -Dataloader batch - labels shape: torch.Size([1, 1259]) -Dataloader batch - attn_mask shape: torch.Size([1, 1259]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 236 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1259) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1259) torch.int64 - loss_mask : 367 / 1259 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1259, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1259, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1259, 1, 576) torch.bfloat16 - out : (1, 1259) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.3836 - top-1 accuracy : 6/367 = 1.6% - - [2026-04-29 13:42:24] iteration 159/ 500 | consumed samples: 636 | elapsed time per iteration (ms): 4585.2 | throughput (token/sec/GPU): 1282.4 | learning rate: 9.673639E-05 | global batch size: 4 | lm loss: 1.126923E+01 | loss scale: 1.0 | grad norm: 0.266 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1447]) -Dataloader batch - labels shape: torch.Size([1, 1447]) -Dataloader batch - attn_mask shape: torch.Size([1, 1447]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 237 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1447) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1447) torch.int64 - loss_mask : 391 / 1447 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1447, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1447, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1447, 1, 576) torch.bfloat16 - out : (1, 1447) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3101 - top-1 accuracy : 7/391 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1530]) -Dataloader batch - labels shape: torch.Size([1, 1530]) -Dataloader batch - attn_mask shape: torch.Size([1, 1530]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 238 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1530) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1530) torch.int64 - loss_mask : 342 / 1530 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1530, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1530, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1530, 1, 576) torch.bfloat16 - out : (1, 1530) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2195 - top-1 accuracy : 32/342 = 9.4% - -Dataloader batch - tokens shape: torch.Size([1, 1408]) -Dataloader batch - labels shape: torch.Size([1, 1408]) -Dataloader batch - attn_mask shape: torch.Size([1, 1408]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 239 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1408) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1408) torch.int64 - loss_mask : 300 / 1408 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1408, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1408, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1408, 1, 576) torch.bfloat16 - out : (1, 1408) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.1858 - top-1 accuracy : 10/300 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1497]) -Dataloader batch - labels shape: torch.Size([1, 1497]) -Dataloader batch - attn_mask shape: torch.Size([1, 1497]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 240 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1497) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1497) torch.int64 - loss_mask : 448 / 1497 tokens (29.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1497, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1497, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1497, 1, 576) torch.bfloat16 - out : (1, 1497) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 448 tokens) : 11.3919 - top-1 accuracy : 7/448 = 1.6% - - [2026-04-29 13:42:28] iteration 160/ 500 | consumed samples: 640 | elapsed time per iteration (ms): 4524.2 | throughput (token/sec/GPU): 1300.1 | learning rate: 9.662419E-05 | global batch size: 4 | lm loss: 1.128874E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4508.89, 4508.89) - forward-compute ................................: (3366.43, 3366.43) - backward-compute ...............................: (1139.48, 1139.48) - batch-generator ................................: (415.21, 415.21) - layernorm-grads-all-reduce .....................: (0.04, 0.04) - embedding-grads-all-reduce .....................: (0.07, 0.07) - all-grads-sync .................................: (0.74, 0.74) - params-all-gather ..............................: (0.19, 0.19) - optimizer-copy-to-main-grad ....................: (0.26, 0.26) - optimizer-clip-main-grad .......................: (0.93, 0.93) - optimizer-count-zeros ..........................: (0.03, 0.03) - optimizer-inner-step ...........................: (1.51, 1.51) - optimizer-copy-main-to-model-params ............: (0.29, 0.29) - optimizer ......................................: (4.29, 4.29) -Dataloader batch - tokens shape: torch.Size([1, 914]) -Dataloader batch - labels shape: torch.Size([1, 914]) -Dataloader batch - attn_mask shape: torch.Size([1, 914]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 241 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 914) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 914) torch.int64 - loss_mask : 184 / 914 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (914, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (914, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (914, 1, 576) torch.bfloat16 - out : (1, 914) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 184 tokens) : 11.1975 - top-1 accuracy : 6/184 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1613]) -Dataloader batch - labels shape: torch.Size([1, 1613]) -Dataloader batch - attn_mask shape: torch.Size([1, 1613]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 242 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1613) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1613) torch.int64 - loss_mask : 387 / 1613 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1613, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1613, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1613, 1, 576) torch.bfloat16 - out : (1, 1613) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2886 - top-1 accuracy : 21/387 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1915]) -Dataloader batch - labels shape: torch.Size([1, 1915]) -Dataloader batch - attn_mask shape: torch.Size([1, 1915]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 243 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1915) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1915) torch.int64 - loss_mask : 565 / 1915 tokens (29.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1915, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1915, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1915, 1, 576) torch.bfloat16 - out : (1, 1915) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 565 tokens) : 11.3847 - top-1 accuracy : 10/565 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1564]) -Dataloader batch - labels shape: torch.Size([1, 1564]) -Dataloader batch - attn_mask shape: torch.Size([1, 1564]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 244 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1564) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1564) torch.int64 - loss_mask : 411 / 1564 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1564, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1564, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1564, 1, 576) torch.bfloat16 - out : (1, 1564) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.3499 - top-1 accuracy : 24/411 = 5.8% - - [2026-04-29 13:42:33] iteration 161/ 500 | consumed samples: 644 | elapsed time per iteration (ms): 4611.3 | throughput (token/sec/GPU): 1302.5 | learning rate: 9.651016E-05 | global batch size: 4 | lm loss: 1.132917E+01 | loss scale: 1.0 | grad norm: 0.413 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1517]) -Dataloader batch - labels shape: torch.Size([1, 1517]) -Dataloader batch - attn_mask shape: torch.Size([1, 1517]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 245 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1517) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1517) torch.int64 - loss_mask : 335 / 1517 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1517, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1517, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1517, 1, 576) torch.bfloat16 - out : (1, 1517) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2188 - top-1 accuracy : 30/335 = 9.0% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 246 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 307 / 1400 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.1862 - top-1 accuracy : 28/307 = 9.1% - -Dataloader batch - tokens shape: torch.Size([1, 1360]) -Dataloader batch - labels shape: torch.Size([1, 1360]) -Dataloader batch - attn_mask shape: torch.Size([1, 1360]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 247 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1360) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1360) torch.int64 - loss_mask : 277 / 1360 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1360, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1360, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1360, 1, 576) torch.bfloat16 - out : (1, 1360) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 277 tokens) : 11.2347 - top-1 accuracy : 32/277 = 11.6% - -Dataloader batch - tokens shape: torch.Size([1, 963]) -Dataloader batch - labels shape: torch.Size([1, 963]) -Dataloader batch - attn_mask shape: torch.Size([1, 963]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 248 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 963) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 963) torch.int64 - loss_mask : 210 / 963 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (963, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (963, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (963, 1, 576) torch.bfloat16 - out : (1, 963) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 210 tokens) : 11.2188 - top-1 accuracy : 25/210 = 11.9% - - [2026-04-29 13:42:37] iteration 162/ 500 | consumed samples: 648 | elapsed time per iteration (ms): 4426.6 | throughput (token/sec/GPU): 1183.7 | learning rate: 9.639430E-05 | global batch size: 4 | lm loss: 1.121383E+01 | loss scale: 1.0 | grad norm: 0.381 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1537]) -Dataloader batch - labels shape: torch.Size([1, 1537]) -Dataloader batch - attn_mask shape: torch.Size([1, 1537]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 249 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1537) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1537) torch.int64 - loss_mask : 317 / 1537 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1537, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1537, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1537, 1, 576) torch.bfloat16 - out : (1, 1537) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.1850 - top-1 accuracy : 24/317 = 7.6% - -Dataloader batch - tokens shape: torch.Size([1, 1368]) -Dataloader batch - labels shape: torch.Size([1, 1368]) -Dataloader batch - attn_mask shape: torch.Size([1, 1368]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 250 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1368) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1368) torch.int64 - loss_mask : 317 / 1368 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1368, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1368, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1368, 1, 576) torch.bfloat16 - out : (1, 1368) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2662 - top-1 accuracy : 21/317 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1232]) -Dataloader batch - labels shape: torch.Size([1, 1232]) -Dataloader batch - attn_mask shape: torch.Size([1, 1232]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 251 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1232) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1232) torch.int64 - loss_mask : 317 / 1232 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1232, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1232, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1232, 1, 576) torch.bfloat16 - out : (1, 1232) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.3233 - top-1 accuracy : 23/317 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1598]) -Dataloader batch - labels shape: torch.Size([1, 1598]) -Dataloader batch - attn_mask shape: torch.Size([1, 1598]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 252 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1598) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1598) torch.int64 - loss_mask : 353 / 1598 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1598, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1598, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1598, 1, 576) torch.bfloat16 - out : (1, 1598) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.1798 - top-1 accuracy : 30/353 = 8.5% - - [2026-04-29 13:42:42] iteration 163/ 500 | consumed samples: 652 | elapsed time per iteration (ms): 4674.9 | throughput (token/sec/GPU): 1226.8 | learning rate: 9.627664E-05 | global batch size: 4 | lm loss: 1.123696E+01 | loss scale: 1.0 | grad norm: 0.285 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 253 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1385) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1385) torch.int64 - loss_mask : 321 / 1385 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1385, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1385, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1385, 1, 576) torch.bfloat16 - out : (1, 1385) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2612 - top-1 accuracy : 7/321 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 254 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 364 / 1542 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2835 - top-1 accuracy : 18/364 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1578]) -Dataloader batch - labels shape: torch.Size([1, 1578]) -Dataloader batch - attn_mask shape: torch.Size([1, 1578]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 255 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1578) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1578) torch.int64 - loss_mask : 368 / 1578 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1578, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1578, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1578, 1, 576) torch.bfloat16 - out : (1, 1578) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2166 - top-1 accuracy : 32/368 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1107]) -Dataloader batch - labels shape: torch.Size([1, 1107]) -Dataloader batch - attn_mask shape: torch.Size([1, 1107]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 256 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1107) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1107) torch.int64 - loss_mask : 300 / 1107 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1107, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1107, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1107, 1, 576) torch.bfloat16 - out : (1, 1107) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.4149 - top-1 accuracy : 19/300 = 6.3% - - [2026-04-29 13:42:46] iteration 164/ 500 | consumed samples: 656 | elapsed time per iteration (ms): 4335.2 | throughput (token/sec/GPU): 1294.5 | learning rate: 9.615714E-05 | global batch size: 4 | lm loss: 1.128914E+01 | loss scale: 1.0 | grad norm: 0.300 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1483]) -Dataloader batch - labels shape: torch.Size([1, 1483]) -Dataloader batch - attn_mask shape: torch.Size([1, 1483]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 257 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1483) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1483) torch.int64 - loss_mask : 355 / 1483 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1483, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1483, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1483, 1, 576) torch.bfloat16 - out : (1, 1483) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2968 - top-1 accuracy : 17/355 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1578]) -Dataloader batch - labels shape: torch.Size([1, 1578]) -Dataloader batch - attn_mask shape: torch.Size([1, 1578]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 258 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1578) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1578) torch.int64 - loss_mask : 352 / 1578 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1578, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1578, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1578, 1, 576) torch.bfloat16 - out : (1, 1578) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.2530 - top-1 accuracy : 36/352 = 10.2% - -Dataloader batch - tokens shape: torch.Size([1, 1395]) -Dataloader batch - labels shape: torch.Size([1, 1395]) -Dataloader batch - attn_mask shape: torch.Size([1, 1395]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 259 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1395) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1395) torch.int64 - loss_mask : 318 / 1395 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1395, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1395, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1395, 1, 576) torch.bfloat16 - out : (1, 1395) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.1702 - top-1 accuracy : 29/318 = 9.1% - -Dataloader batch - tokens shape: torch.Size([1, 1131]) -Dataloader batch - labels shape: torch.Size([1, 1131]) -Dataloader batch - attn_mask shape: torch.Size([1, 1131]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 260 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1131) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1131) torch.int64 - loss_mask : 289 / 1131 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1131, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1131, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1131, 1, 576) torch.bfloat16 - out : (1, 1131) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 289 tokens) : 11.3129 - top-1 accuracy : 3/289 = 1.0% - - [2026-04-29 13:42:51] iteration 165/ 500 | consumed samples: 660 | elapsed time per iteration (ms): 4643.0 | throughput (token/sec/GPU): 1203.3 | learning rate: 9.603585E-05 | global batch size: 4 | lm loss: 1.125794E+01 | loss scale: 1.0 | grad norm: 0.319 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1799]) -Dataloader batch - labels shape: torch.Size([1, 1799]) -Dataloader batch - attn_mask shape: torch.Size([1, 1799]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 261 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1799) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1799) torch.int64 - loss_mask : 445 / 1799 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1799, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1799, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1799, 1, 576) torch.bfloat16 - out : (1, 1799) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 445 tokens) : 11.2735 - top-1 accuracy : 19/445 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1399]) -Dataloader batch - labels shape: torch.Size([1, 1399]) -Dataloader batch - attn_mask shape: torch.Size([1, 1399]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 262 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1399) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1399) torch.int64 - loss_mask : 338 / 1399 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1399, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1399, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1399, 1, 576) torch.bfloat16 - out : (1, 1399) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.2952 - top-1 accuracy : 9/338 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1540]) -Dataloader batch - labels shape: torch.Size([1, 1540]) -Dataloader batch - attn_mask shape: torch.Size([1, 1540]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 263 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1540) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1540) torch.int64 - loss_mask : 357 / 1540 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1540, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1540, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1540, 1, 576) torch.bfloat16 - out : (1, 1540) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2358 - top-1 accuracy : 15/357 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1146]) -Dataloader batch - labels shape: torch.Size([1, 1146]) -Dataloader batch - attn_mask shape: torch.Size([1, 1146]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 264 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1146) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1146) torch.int64 - loss_mask : 367 / 1146 tokens (32.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1146, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1146, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1146, 1, 576) torch.bfloat16 - out : (1, 1146) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.4037 - top-1 accuracy : 16/367 = 4.4% - - [2026-04-29 13:42:56] iteration 166/ 500 | consumed samples: 664 | elapsed time per iteration (ms): 4594.9 | throughput (token/sec/GPU): 1280.6 | learning rate: 9.591275E-05 | global batch size: 4 | lm loss: 1.130115E+01 | loss scale: 1.0 | grad norm: 0.308 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 949]) -Dataloader batch - labels shape: torch.Size([1, 949]) -Dataloader batch - attn_mask shape: torch.Size([1, 949]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 265 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 949) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 949) torch.int64 - loss_mask : 215 / 949 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (949, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (949, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (949, 1, 576) torch.bfloat16 - out : (1, 949) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 215 tokens) : 11.2074 - top-1 accuracy : 22/215 = 10.2% - -Dataloader batch - tokens shape: torch.Size([1, 1691]) -Dataloader batch - labels shape: torch.Size([1, 1691]) -Dataloader batch - attn_mask shape: torch.Size([1, 1691]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 266 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1691) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1691) torch.int64 - loss_mask : 350 / 1691 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1691, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1691, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1691, 1, 576) torch.bfloat16 - out : (1, 1691) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.1673 - top-1 accuracy : 38/350 = 10.9% - -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 267 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 371 / 1434 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.3048 - top-1 accuracy : 19/371 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1914]) -Dataloader batch - labels shape: torch.Size([1, 1914]) -Dataloader batch - attn_mask shape: torch.Size([1, 1914]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 268 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1914) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1914) torch.int64 - loss_mask : 535 / 1914 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1914, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1914, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1914, 1, 576) torch.bfloat16 - out : (1, 1914) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 535 tokens) : 11.3702 - top-1 accuracy : 4/535 = 0.7% - - [2026-04-29 13:43:00] iteration 167/ 500 | consumed samples: 668 | elapsed time per iteration (ms): 4645.7 | throughput (token/sec/GPU): 1288.9 | learning rate: 9.578785E-05 | global batch size: 4 | lm loss: 1.128162E+01 | loss scale: 1.0 | grad norm: 0.332 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1731]) -Dataloader batch - labels shape: torch.Size([1, 1731]) -Dataloader batch - attn_mask shape: torch.Size([1, 1731]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 269 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1731) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1731) torch.int64 - loss_mask : 386 / 1731 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1731, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1731, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1731, 1, 576) torch.bfloat16 - out : (1, 1731) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2664 - top-1 accuracy : 41/386 = 10.6% - -Dataloader batch - tokens shape: torch.Size([1, 1530]) -Dataloader batch - labels shape: torch.Size([1, 1530]) -Dataloader batch - attn_mask shape: torch.Size([1, 1530]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 270 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1530) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1530) torch.int64 - loss_mask : 325 / 1530 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1530, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1530, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1530, 1, 576) torch.bfloat16 - out : (1, 1530) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.1774 - top-1 accuracy : 17/325 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1077]) -Dataloader batch - labels shape: torch.Size([1, 1077]) -Dataloader batch - attn_mask shape: torch.Size([1, 1077]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 271 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1077) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1077) torch.int64 - loss_mask : 250 / 1077 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1077, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1077, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1077, 1, 576) torch.bfloat16 - out : (1, 1077) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 250 tokens) : 11.2262 - top-1 accuracy : 6/250 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1367]) -Dataloader batch - labels shape: torch.Size([1, 1367]) -Dataloader batch - attn_mask shape: torch.Size([1, 1367]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 272 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1367) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1367) torch.int64 - loss_mask : 306 / 1367 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1367, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1367, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1367, 1, 576) torch.bfloat16 - out : (1, 1367) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.2776 - top-1 accuracy : 12/306 = 3.9% - - [2026-04-29 13:43:05] iteration 168/ 500 | consumed samples: 672 | elapsed time per iteration (ms): 4702.6 | throughput (token/sec/GPU): 1213.2 | learning rate: 9.566115E-05 | global batch size: 4 | lm loss: 1.123833E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1878]) -Dataloader batch - labels shape: torch.Size([1, 1878]) -Dataloader batch - attn_mask shape: torch.Size([1, 1878]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 273 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1878) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1878) torch.int64 - loss_mask : 526 / 1878 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1878, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1878, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1878, 1, 576) torch.bfloat16 - out : (1, 1878) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 526 tokens) : 11.3863 - top-1 accuracy : 5/526 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1195]) -Dataloader batch - labels shape: torch.Size([1, 1195]) -Dataloader batch - attn_mask shape: torch.Size([1, 1195]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 274 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1195) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1195) torch.int64 - loss_mask : 298 / 1195 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1195, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1195, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1195, 1, 576) torch.bfloat16 - out : (1, 1195) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 298 tokens) : 11.3107 - top-1 accuracy : 12/298 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1212]) -Dataloader batch - labels shape: torch.Size([1, 1212]) -Dataloader batch - attn_mask shape: torch.Size([1, 1212]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 275 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1212) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1212) torch.int64 - loss_mask : 322 / 1212 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1212, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1212, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1212, 1, 576) torch.bfloat16 - out : (1, 1212) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.3386 - top-1 accuracy : 9/322 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 276 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 369 / 1518 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.3247 - top-1 accuracy : 23/369 = 6.2% - - [2026-04-29 13:43:10] iteration 169/ 500 | consumed samples: 676 | elapsed time per iteration (ms): 4537.1 | throughput (token/sec/GPU): 1279.0 | learning rate: 9.553267E-05 | global batch size: 4 | lm loss: 1.134629E+01 | loss scale: 1.0 | grad norm: 0.325 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1726]) -Dataloader batch - labels shape: torch.Size([1, 1726]) -Dataloader batch - attn_mask shape: torch.Size([1, 1726]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 277 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1726) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1726) torch.int64 - loss_mask : 373 / 1726 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1726, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1726, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1726, 1, 576) torch.bfloat16 - out : (1, 1726) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.2296 - top-1 accuracy : 40/373 = 10.7% - -Dataloader batch - tokens shape: torch.Size([1, 1440]) -Dataloader batch - labels shape: torch.Size([1, 1440]) -Dataloader batch - attn_mask shape: torch.Size([1, 1440]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 278 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1440) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1440) torch.int64 - loss_mask : 378 / 1440 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1440, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1440, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1440, 1, 576) torch.bfloat16 - out : (1, 1440) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 378 tokens) : 11.3160 - top-1 accuracy : 6/378 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1180]) -Dataloader batch - labels shape: torch.Size([1, 1180]) -Dataloader batch - attn_mask shape: torch.Size([1, 1180]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 279 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1180) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1180) torch.int64 - loss_mask : 262 / 1180 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1180, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1180, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1180, 1, 576) torch.bfloat16 - out : (1, 1180) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 262 tokens) : 11.1533 - top-1 accuracy : 8/262 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1550]) -Dataloader batch - labels shape: torch.Size([1, 1550]) -Dataloader batch - attn_mask shape: torch.Size([1, 1550]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 280 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1550) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1550) torch.int64 - loss_mask : 418 / 1550 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1550, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1550, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1550, 1, 576) torch.bfloat16 - out : (1, 1550) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 418 tokens) : 11.3402 - top-1 accuracy : 23/418 = 5.5% - - [2026-04-29 13:43:14] iteration 170/ 500 | consumed samples: 680 | elapsed time per iteration (ms): 4758.4 | throughput (token/sec/GPU): 1239.1 | learning rate: 9.540239E-05 | global batch size: 4 | lm loss: 1.127076E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1482]) -Dataloader batch - labels shape: torch.Size([1, 1482]) -Dataloader batch - attn_mask shape: torch.Size([1, 1482]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 281 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1482) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1482) torch.int64 - loss_mask : 345 / 1482 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1482, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1482, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1482, 1, 576) torch.bfloat16 - out : (1, 1482) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2699 - top-1 accuracy : 9/345 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 282 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1412) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1412) torch.int64 - loss_mask : 370 / 1412 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1412, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1412, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1412, 1, 576) torch.bfloat16 - out : (1, 1412) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.3071 - top-1 accuracy : 12/370 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1889]) -Dataloader batch - labels shape: torch.Size([1, 1889]) -Dataloader batch - attn_mask shape: torch.Size([1, 1889]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 283 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1889) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1889) torch.int64 - loss_mask : 533 / 1889 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1889, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1889, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1889, 1, 576) torch.bfloat16 - out : (1, 1889) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 533 tokens) : 11.4134 - top-1 accuracy : 8/533 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 284 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 353 / 1539 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.2026 - top-1 accuracy : 2/353 = 0.6% - - [2026-04-29 13:43:19] iteration 171/ 500 | consumed samples: 684 | elapsed time per iteration (ms): 4926.0 | throughput (token/sec/GPU): 1283.4 | learning rate: 9.527035E-05 | global batch size: 4 | lm loss: 1.131145E+01 | loss scale: 1.0 | grad norm: 0.252 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1266]) -Dataloader batch - labels shape: torch.Size([1, 1266]) -Dataloader batch - attn_mask shape: torch.Size([1, 1266]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 285 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1266) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1266) torch.int64 - loss_mask : 376 / 1266 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1266, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1266, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1266, 1, 576) torch.bfloat16 - out : (1, 1266) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3827 - top-1 accuracy : 1/376 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1945]) -Dataloader batch - labels shape: torch.Size([1, 1945]) -Dataloader batch - attn_mask shape: torch.Size([1, 1945]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 286 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1945) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1945) torch.int64 - loss_mask : 587 / 1945 tokens (30.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1945, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1945, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1945, 1, 576) torch.bfloat16 - out : (1, 1945) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 587 tokens) : 11.4183 - top-1 accuracy : 25/587 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1530]) -Dataloader batch - labels shape: torch.Size([1, 1530]) -Dataloader batch - attn_mask shape: torch.Size([1, 1530]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 287 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1530) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1530) torch.int64 - loss_mask : 346 / 1530 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1530, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1530, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1530, 1, 576) torch.bfloat16 - out : (1, 1530) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.2267 - top-1 accuracy : 26/346 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 288 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 372 / 1722 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2020 - top-1 accuracy : 15/372 = 4.0% - - [2026-04-29 13:43:24] iteration 172/ 500 | consumed samples: 688 | elapsed time per iteration (ms): 4828.1 | throughput (token/sec/GPU): 1338.6 | learning rate: 9.513652E-05 | global batch size: 4 | lm loss: 1.132302E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 289 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 316 / 1520 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.2010 - top-1 accuracy : 18/316 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1392]) -Dataloader batch - labels shape: torch.Size([1, 1392]) -Dataloader batch - attn_mask shape: torch.Size([1, 1392]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 290 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1392) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1392) torch.int64 - loss_mask : 349 / 1392 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1392, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1392, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1392, 1, 576) torch.bfloat16 - out : (1, 1392) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2803 - top-1 accuracy : 25/349 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1025]) -Dataloader batch - labels shape: torch.Size([1, 1025]) -Dataloader batch - attn_mask shape: torch.Size([1, 1025]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 291 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1025) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1025) torch.int64 - loss_mask : 250 / 1025 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1025, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1025, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1025, 1, 576) torch.bfloat16 - out : (1, 1025) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 250 tokens) : 11.2853 - top-1 accuracy : 2/250 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1759]) -Dataloader batch - labels shape: torch.Size([1, 1759]) -Dataloader batch - attn_mask shape: torch.Size([1, 1759]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 292 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1759) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1759) torch.int64 - loss_mask : 408 / 1759 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1759, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1759, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1759, 1, 576) torch.bfloat16 - out : (1, 1759) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.2754 - top-1 accuracy : 35/408 = 8.6% - - [2026-04-29 13:43:29] iteration 173/ 500 | consumed samples: 692 | elapsed time per iteration (ms): 4608.4 | throughput (token/sec/GPU): 1236.0 | learning rate: 9.500093E-05 | global batch size: 4 | lm loss: 1.126078E+01 | loss scale: 1.0 | grad norm: 0.305 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1291]) -Dataloader batch - labels shape: torch.Size([1, 1291]) -Dataloader batch - attn_mask shape: torch.Size([1, 1291]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 293 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1291) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1291) torch.int64 - loss_mask : 331 / 1291 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1291, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1291, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1291, 1, 576) torch.bfloat16 - out : (1, 1291) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.3046 - top-1 accuracy : 16/331 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 294 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 365 / 1422 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.3062 - top-1 accuracy : 4/365 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1226]) -Dataloader batch - labels shape: torch.Size([1, 1226]) -Dataloader batch - attn_mask shape: torch.Size([1, 1226]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 295 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1226) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1226) torch.int64 - loss_mask : 332 / 1226 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1226, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1226, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1226, 1, 576) torch.bfloat16 - out : (1, 1226) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.3724 - top-1 accuracy : 8/332 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1447]) -Dataloader batch - labels shape: torch.Size([1, 1447]) -Dataloader batch - attn_mask shape: torch.Size([1, 1447]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 296 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1447) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1447) torch.int64 - loss_mask : 413 / 1447 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1447, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1447, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1447, 1, 576) torch.bfloat16 - out : (1, 1447) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.3644 - top-1 accuracy : 4/413 = 1.0% - - [2026-04-29 13:43:33] iteration 174/ 500 | consumed samples: 696 | elapsed time per iteration (ms): 4200.0 | throughput (token/sec/GPU): 1282.4 | learning rate: 9.486356E-05 | global batch size: 4 | lm loss: 1.133775E+01 | loss scale: 1.0 | grad norm: 0.309 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 297 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1191) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1191) torch.int64 - loss_mask : 270 / 1191 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1191, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1191, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1191, 1, 576) torch.bfloat16 - out : (1, 1191) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 270 tokens) : 11.2172 - top-1 accuracy : 15/270 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1379]) -Dataloader batch - labels shape: torch.Size([1, 1379]) -Dataloader batch - attn_mask shape: torch.Size([1, 1379]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 298 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1379) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1379) torch.int64 - loss_mask : 366 / 1379 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1379, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1379, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1379, 1, 576) torch.bfloat16 - out : (1, 1379) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.3090 - top-1 accuracy : 5/366 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1251]) -Dataloader batch - labels shape: torch.Size([1, 1251]) -Dataloader batch - attn_mask shape: torch.Size([1, 1251]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 299 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1251) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1251) torch.int64 - loss_mask : 329 / 1251 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1251, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1251, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1251, 1, 576) torch.bfloat16 - out : (1, 1251) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.3762 - top-1 accuracy : 8/329 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1528]) -Dataloader batch - labels shape: torch.Size([1, 1528]) -Dataloader batch - attn_mask shape: torch.Size([1, 1528]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 300 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1528) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1528) torch.int64 - loss_mask : 345 / 1528 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1528, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1528, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1528, 1, 576) torch.bfloat16 - out : (1, 1528) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2602 - top-1 accuracy : 16/345 = 4.6% - - [2026-04-29 13:43:37] iteration 175/ 500 | consumed samples: 700 | elapsed time per iteration (ms): 4286.6 | throughput (token/sec/GPU): 1247.8 | learning rate: 9.472445E-05 | global batch size: 4 | lm loss: 1.129409E+01 | loss scale: 1.0 | grad norm: 0.254 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1188]) -Dataloader batch - labels shape: torch.Size([1, 1188]) -Dataloader batch - attn_mask shape: torch.Size([1, 1188]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 301 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1188) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1188) torch.int64 - loss_mask : 268 / 1188 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1188, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1188, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1188, 1, 576) torch.bfloat16 - out : (1, 1188) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 268 tokens) : 11.2893 - top-1 accuracy : 11/268 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 302 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 357 / 1542 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2346 - top-1 accuracy : 14/357 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1809]) -Dataloader batch - labels shape: torch.Size([1, 1809]) -Dataloader batch - attn_mask shape: torch.Size([1, 1809]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 303 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1809) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1809) torch.int64 - loss_mask : 449 / 1809 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1809, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1809, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1809, 1, 576) torch.bfloat16 - out : (1, 1809) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 449 tokens) : 11.3085 - top-1 accuracy : 3/449 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1578]) -Dataloader batch - labels shape: torch.Size([1, 1578]) -Dataloader batch - attn_mask shape: torch.Size([1, 1578]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 304 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1578) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1578) torch.int64 - loss_mask : 388 / 1578 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1578, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1578, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1578, 1, 576) torch.bfloat16 - out : (1, 1578) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2770 - top-1 accuracy : 4/388 = 1.0% - - [2026-04-29 13:43:42] iteration 176/ 500 | consumed samples: 704 | elapsed time per iteration (ms): 4851.2 | throughput (token/sec/GPU): 1260.9 | learning rate: 9.458358E-05 | global batch size: 4 | lm loss: 1.127858E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1406]) -Dataloader batch - labels shape: torch.Size([1, 1406]) -Dataloader batch - attn_mask shape: torch.Size([1, 1406]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 305 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1406) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1406) torch.int64 - loss_mask : 314 / 1406 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1406, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1406, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1406, 1, 576) torch.bfloat16 - out : (1, 1406) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.2625 - top-1 accuracy : 21/314 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 306 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 390 / 1525 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3259 - top-1 accuracy : 5/390 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 961]) -Dataloader batch - labels shape: torch.Size([1, 961]) -Dataloader batch - attn_mask shape: torch.Size([1, 961]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 307 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 961) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 961) torch.int64 - loss_mask : 195 / 961 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (961, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (961, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (961, 1, 576) torch.bfloat16 - out : (1, 961) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 195 tokens) : 11.1604 - top-1 accuracy : 14/195 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1778]) -Dataloader batch - labels shape: torch.Size([1, 1778]) -Dataloader batch - attn_mask shape: torch.Size([1, 1778]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 308 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1778) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1778) torch.int64 - loss_mask : 404 / 1778 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1778, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1778, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1778, 1, 576) torch.bfloat16 - out : (1, 1778) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.2243 - top-1 accuracy : 7/404 = 1.7% - - [2026-04-29 13:43:47] iteration 177/ 500 | consumed samples: 708 | elapsed time per iteration (ms): 4614.2 | throughput (token/sec/GPU): 1228.8 | learning rate: 9.444096E-05 | global batch size: 4 | lm loss: 1.125436E+01 | loss scale: 1.0 | grad norm: 0.324 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1457]) -Dataloader batch - labels shape: torch.Size([1, 1457]) -Dataloader batch - attn_mask shape: torch.Size([1, 1457]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 309 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1457) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1457) torch.int64 - loss_mask : 355 / 1457 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1457, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1457, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1457, 1, 576) torch.bfloat16 - out : (1, 1457) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2392 - top-1 accuracy : 6/355 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1461]) -Dataloader batch - labels shape: torch.Size([1, 1461]) -Dataloader batch - attn_mask shape: torch.Size([1, 1461]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 310 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1461) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1461) torch.int64 - loss_mask : 399 / 1461 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1461, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1461, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1461, 1, 576) torch.bfloat16 - out : (1, 1461) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.3639 - top-1 accuracy : 9/399 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1574]) -Dataloader batch - labels shape: torch.Size([1, 1574]) -Dataloader batch - attn_mask shape: torch.Size([1, 1574]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 311 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1574) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1574) torch.int64 - loss_mask : 358 / 1574 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1574, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1574, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1574, 1, 576) torch.bfloat16 - out : (1, 1574) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2107 - top-1 accuracy : 23/358 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1498]) -Dataloader batch - labels shape: torch.Size([1, 1498]) -Dataloader batch - attn_mask shape: torch.Size([1, 1498]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 312 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1498) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1498) torch.int64 - loss_mask : 457 / 1498 tokens (30.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1498, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1498, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1498, 1, 576) torch.bfloat16 - out : (1, 1498) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 457 tokens) : 11.3999 - top-1 accuracy : 28/457 = 6.1% - - [2026-04-29 13:43:51] iteration 178/ 500 | consumed samples: 712 | elapsed time per iteration (ms): 4645.4 | throughput (token/sec/GPU): 1289.4 | learning rate: 9.429661E-05 | global batch size: 4 | lm loss: 1.131122E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 974]) -Dataloader batch - labels shape: torch.Size([1, 974]) -Dataloader batch - attn_mask shape: torch.Size([1, 974]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 313 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 974) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 974) torch.int64 - loss_mask : 227 / 974 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (974, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (974, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (974, 1, 576) torch.bfloat16 - out : (1, 974) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 227 tokens) : 11.2183 - top-1 accuracy : 6/227 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1862]) -Dataloader batch - labels shape: torch.Size([1, 1862]) -Dataloader batch - attn_mask shape: torch.Size([1, 1862]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 314 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1862) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1862) torch.int64 - loss_mask : 493 / 1862 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1862, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1862, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1862, 1, 576) torch.bfloat16 - out : (1, 1862) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 493 tokens) : 11.3464 - top-1 accuracy : 17/493 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1403]) -Dataloader batch - labels shape: torch.Size([1, 1403]) -Dataloader batch - attn_mask shape: torch.Size([1, 1403]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 315 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1403) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1403) torch.int64 - loss_mask : 346 / 1403 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1403, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1403, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1403, 1, 576) torch.bfloat16 - out : (1, 1403) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.2659 - top-1 accuracy : 14/346 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 316 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 395 / 1438 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.2918 - top-1 accuracy : 15/395 = 3.8% - - [2026-04-29 13:43:56] iteration 179/ 500 | consumed samples: 716 | elapsed time per iteration (ms): 4488.4 | throughput (token/sec/GPU): 1264.8 | learning rate: 9.415051E-05 | global batch size: 4 | lm loss: 1.129267E+01 | loss scale: 1.0 | grad norm: 0.215 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1429]) -Dataloader batch - labels shape: torch.Size([1, 1429]) -Dataloader batch - attn_mask shape: torch.Size([1, 1429]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 317 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1429) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1429) torch.int64 - loss_mask : 392 / 1429 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1429, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1429, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1429, 1, 576) torch.bfloat16 - out : (1, 1429) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.2772 - top-1 accuracy : 10/392 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1405]) -Dataloader batch - labels shape: torch.Size([1, 1405]) -Dataloader batch - attn_mask shape: torch.Size([1, 1405]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 318 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1405) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1405) torch.int64 - loss_mask : 376 / 1405 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1405, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1405, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1405, 1, 576) torch.bfloat16 - out : (1, 1405) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3091 - top-1 accuracy : 3/376 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1497]) -Dataloader batch - labels shape: torch.Size([1, 1497]) -Dataloader batch - attn_mask shape: torch.Size([1, 1497]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 319 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1497) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1497) torch.int64 - loss_mask : 331 / 1497 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1497, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1497, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1497, 1, 576) torch.bfloat16 - out : (1, 1497) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.2059 - top-1 accuracy : 23/331 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1382]) -Dataloader batch - labels shape: torch.Size([1, 1382]) -Dataloader batch - attn_mask shape: torch.Size([1, 1382]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 320 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1382) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1382) torch.int64 - loss_mask : 325 / 1382 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1382, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1382, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1382, 1, 576) torch.bfloat16 - out : (1, 1382) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.2728 - top-1 accuracy : 17/325 = 5.2% - - [2026-04-29 13:44:00] iteration 180/ 500 | consumed samples: 720 | elapsed time per iteration (ms): 4528.9 | throughput (token/sec/GPU): 1261.5 | learning rate: 9.400269E-05 | global batch size: 4 | lm loss: 1.126805E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4503.08, 4503.08) - forward-compute ................................: (3360.17, 3360.17) - backward-compute ...............................: (1138.82, 1138.82) - batch-generator ................................: (446.58, 446.58) - layernorm-grads-all-reduce .....................: (0.07, 0.07) - embedding-grads-all-reduce .....................: (0.10, 0.10) - all-grads-sync .................................: (1.03, 1.03) - params-all-gather ..............................: (0.36, 0.36) - optimizer-copy-to-main-grad ....................: (0.40, 0.40) - optimizer-clip-main-grad .......................: (1.30, 1.30) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.02, 2.02) - optimizer-copy-main-to-model-params ............: (0.59, 0.59) - optimizer ......................................: (6.70, 6.70) -Dataloader batch - tokens shape: torch.Size([1, 1430]) -Dataloader batch - labels shape: torch.Size([1, 1430]) -Dataloader batch - attn_mask shape: torch.Size([1, 1430]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 321 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1430) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1430) torch.int64 - loss_mask : 377 / 1430 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1430, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1430, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1430, 1, 576) torch.bfloat16 - out : (1, 1430) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.3293 - top-1 accuracy : 16/377 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1123]) -Dataloader batch - labels shape: torch.Size([1, 1123]) -Dataloader batch - attn_mask shape: torch.Size([1, 1123]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 322 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1123) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1123) torch.int64 - loss_mask : 264 / 1123 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1123, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1123, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1123, 1, 576) torch.bfloat16 - out : (1, 1123) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 264 tokens) : 11.2490 - top-1 accuracy : 15/264 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 994]) -Dataloader batch - labels shape: torch.Size([1, 994]) -Dataloader batch - attn_mask shape: torch.Size([1, 994]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 323 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 994) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 994) torch.int64 - loss_mask : 253 / 994 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (994, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (994, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (994, 1, 576) torch.bfloat16 - out : (1, 994) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 253 tokens) : 11.3054 - top-1 accuracy : 1/253 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 324 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 303 / 1386 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.2160 - top-1 accuracy : 24/303 = 7.9% - - [2026-04-29 13:44:04] iteration 181/ 500 | consumed samples: 724 | elapsed time per iteration (ms): 4109.7 | throughput (token/sec/GPU): 1200.3 | learning rate: 9.385314E-05 | global batch size: 4 | lm loss: 1.127788E+01 | loss scale: 1.0 | grad norm: 0.300 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 325 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 382 / 1422 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.2646 - top-1 accuracy : 11/382 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 991]) -Dataloader batch - labels shape: torch.Size([1, 991]) -Dataloader batch - attn_mask shape: torch.Size([1, 991]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 326 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 991) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 991) torch.int64 - loss_mask : 250 / 991 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (991, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (991, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (991, 1, 576) torch.bfloat16 - out : (1, 991) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 250 tokens) : 11.2223 - top-1 accuracy : 3/250 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1770]) -Dataloader batch - labels shape: torch.Size([1, 1770]) -Dataloader batch - attn_mask shape: torch.Size([1, 1770]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 327 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1770) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1770) torch.int64 - loss_mask : 408 / 1770 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1770, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1770, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1770, 1, 576) torch.bfloat16 - out : (1, 1770) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.2677 - top-1 accuracy : 24/408 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1381]) -Dataloader batch - labels shape: torch.Size([1, 1381]) -Dataloader batch - attn_mask shape: torch.Size([1, 1381]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 328 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1381) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1381) torch.int64 - loss_mask : 315 / 1381 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1381, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1381, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1381, 1, 576) torch.bfloat16 - out : (1, 1381) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.2868 - top-1 accuracy : 19/315 = 6.0% - - [2026-04-29 13:44:09] iteration 182/ 500 | consumed samples: 728 | elapsed time per iteration (ms): 4416.1 | throughput (token/sec/GPU): 1259.9 | learning rate: 9.370187E-05 | global batch size: 4 | lm loss: 1.126289E+01 | loss scale: 1.0 | grad norm: 0.254 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1806]) -Dataloader batch - labels shape: torch.Size([1, 1806]) -Dataloader batch - attn_mask shape: torch.Size([1, 1806]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 329 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1806) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1806) torch.int64 - loss_mask : 441 / 1806 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1806, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1806, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1806, 1, 576) torch.bfloat16 - out : (1, 1806) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 441 tokens) : 11.2953 - top-1 accuracy : 26/441 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 998]) -Dataloader batch - labels shape: torch.Size([1, 998]) -Dataloader batch - attn_mask shape: torch.Size([1, 998]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 330 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 998) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 998) torch.int64 - loss_mask : 230 / 998 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (998, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (998, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (998, 1, 576) torch.bfloat16 - out : (1, 998) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 230 tokens) : 11.2059 - top-1 accuracy : 15/230 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1486]) -Dataloader batch - labels shape: torch.Size([1, 1486]) -Dataloader batch - attn_mask shape: torch.Size([1, 1486]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 331 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1486) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1486) torch.int64 - loss_mask : 306 / 1486 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1486, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1486, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1486, 1, 576) torch.bfloat16 - out : (1, 1486) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.1441 - top-1 accuracy : 25/306 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1196]) -Dataloader batch - labels shape: torch.Size([1, 1196]) -Dataloader batch - attn_mask shape: torch.Size([1, 1196]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 332 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1196) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1196) torch.int64 - loss_mask : 274 / 1196 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1196, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1196, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1196, 1, 576) torch.bfloat16 - out : (1, 1196) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 274 tokens) : 11.2014 - top-1 accuracy : 15/274 = 5.5% - - [2026-04-29 13:44:13] iteration 183/ 500 | consumed samples: 732 | elapsed time per iteration (ms): 4574.3 | throughput (token/sec/GPU): 1199.3 | learning rate: 9.354889E-05 | global batch size: 4 | lm loss: 1.122132E+01 | loss scale: 1.0 | grad norm: 0.305 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1020]) -Dataloader batch - labels shape: torch.Size([1, 1020]) -Dataloader batch - attn_mask shape: torch.Size([1, 1020]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 333 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1020) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1020) torch.int64 - loss_mask : 246 / 1020 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1020, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1020, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1020, 1, 576) torch.bfloat16 - out : (1, 1020) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 246 tokens) : 11.2754 - top-1 accuracy : 5/246 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1911]) -Dataloader batch - labels shape: torch.Size([1, 1911]) -Dataloader batch - attn_mask shape: torch.Size([1, 1911]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 334 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1911) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1911) torch.int64 - loss_mask : 549 / 1911 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1911, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1911, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1911, 1, 576) torch.bfloat16 - out : (1, 1911) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 549 tokens) : 11.3770 - top-1 accuracy : 23/549 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1664]) -Dataloader batch - labels shape: torch.Size([1, 1664]) -Dataloader batch - attn_mask shape: torch.Size([1, 1664]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 335 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1664) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1664) torch.int64 - loss_mask : 388 / 1664 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1664, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1664, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1664, 1, 576) torch.bfloat16 - out : (1, 1664) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2644 - top-1 accuracy : 24/388 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1570]) -Dataloader batch - labels shape: torch.Size([1, 1570]) -Dataloader batch - attn_mask shape: torch.Size([1, 1570]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 336 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1570) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1570) torch.int64 - loss_mask : 379 / 1570 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1570, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1570, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1570, 1, 576) torch.bfloat16 - out : (1, 1570) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.2297 - top-1 accuracy : 13/379 = 3.4% - - [2026-04-29 13:44:19] iteration 184/ 500 | consumed samples: 736 | elapsed time per iteration (ms): 5134.2 | throughput (token/sec/GPU): 1200.8 | learning rate: 9.339421E-05 | global batch size: 4 | lm loss: 1.129728E+01 | loss scale: 1.0 | grad norm: 0.249 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1425]) -Dataloader batch - labels shape: torch.Size([1, 1425]) -Dataloader batch - attn_mask shape: torch.Size([1, 1425]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 337 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1425) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1425) torch.int64 - loss_mask : 352 / 1425 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1425, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1425, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1425, 1, 576) torch.bfloat16 - out : (1, 1425) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.2481 - top-1 accuracy : 17/352 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 338 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 284 / 1475 tokens (19.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.1190 - top-1 accuracy : 19/284 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1350]) -Dataloader batch - labels shape: torch.Size([1, 1350]) -Dataloader batch - attn_mask shape: torch.Size([1, 1350]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 339 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1350) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1350) torch.int64 - loss_mask : 280 / 1350 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1350, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1350, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1350, 1, 576) torch.bfloat16 - out : (1, 1350) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 280 tokens) : 11.1728 - top-1 accuracy : 23/280 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1026]) -Dataloader batch - labels shape: torch.Size([1, 1026]) -Dataloader batch - attn_mask shape: torch.Size([1, 1026]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 340 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1026) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1026) torch.int64 - loss_mask : 258 / 1026 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1026, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1026, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1026, 1, 576) torch.bfloat16 - out : (1, 1026) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 258 tokens) : 11.3299 - top-1 accuracy : 8/258 = 3.1% - - [2026-04-29 13:44:23] iteration 185/ 500 | consumed samples: 740 | elapsed time per iteration (ms): 4465.4 | throughput (token/sec/GPU): 1181.5 | learning rate: 9.323782E-05 | global batch size: 4 | lm loss: 1.121688E+01 | loss scale: 1.0 | grad norm: 0.303 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1427]) -Dataloader batch - labels shape: torch.Size([1, 1427]) -Dataloader batch - attn_mask shape: torch.Size([1, 1427]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 341 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1427) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1427) torch.int64 - loss_mask : 338 / 1427 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1427, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1427, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1427, 1, 576) torch.bfloat16 - out : (1, 1427) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.3074 - top-1 accuracy : 9/338 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1785]) -Dataloader batch - labels shape: torch.Size([1, 1785]) -Dataloader batch - attn_mask shape: torch.Size([1, 1785]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 342 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1785) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1785) torch.int64 - loss_mask : 404 / 1785 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1785, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1785, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1785, 1, 576) torch.bfloat16 - out : (1, 1785) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.2300 - top-1 accuracy : 15/404 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 343 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 326 / 1460 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2304 - top-1 accuracy : 7/326 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1216]) -Dataloader batch - labels shape: torch.Size([1, 1216]) -Dataloader batch - attn_mask shape: torch.Size([1, 1216]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 344 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1216) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1216) torch.int64 - loss_mask : 315 / 1216 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1216, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1216, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1216, 1, 576) torch.bfloat16 - out : (1, 1216) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.3364 - top-1 accuracy : 2/315 = 0.6% - - [2026-04-29 13:44:28] iteration 186/ 500 | consumed samples: 744 | elapsed time per iteration (ms): 4794.8 | throughput (token/sec/GPU): 1228.0 | learning rate: 9.307974E-05 | global batch size: 4 | lm loss: 1.127323E+01 | loss scale: 1.0 | grad norm: 0.367 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 345 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 349 / 1400 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2422 - top-1 accuracy : 6/349 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 346 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 386 / 1750 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2030 - top-1 accuracy : 9/386 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 347 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 349 / 1434 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2609 - top-1 accuracy : 13/349 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1367]) -Dataloader batch - labels shape: torch.Size([1, 1367]) -Dataloader batch - attn_mask shape: torch.Size([1, 1367]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 348 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1367) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1367) torch.int64 - loss_mask : 316 / 1367 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1367, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1367, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1367, 1, 576) torch.bfloat16 - out : (1, 1367) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.2279 - top-1 accuracy : 17/316 = 5.4% - - [2026-04-29 13:44:33] iteration 187/ 500 | consumed samples: 748 | elapsed time per iteration (ms): 4795.2 | throughput (token/sec/GPU): 1241.0 | learning rate: 9.291997E-05 | global batch size: 4 | lm loss: 1.123283E+01 | loss scale: 1.0 | grad norm: 0.273 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1315]) -Dataloader batch - labels shape: torch.Size([1, 1315]) -Dataloader batch - attn_mask shape: torch.Size([1, 1315]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 349 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1315) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1315) torch.int64 - loss_mask : 343 / 1315 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1315, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1315, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1315, 1, 576) torch.bfloat16 - out : (1, 1315) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.3417 - top-1 accuracy : 2/343 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1692]) -Dataloader batch - labels shape: torch.Size([1, 1692]) -Dataloader batch - attn_mask shape: torch.Size([1, 1692]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 350 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1692) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1692) torch.int64 - loss_mask : 366 / 1692 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1692, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1692, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1692, 1, 576) torch.bfloat16 - out : (1, 1692) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.1516 - top-1 accuracy : 37/366 = 10.1% - -Dataloader batch - tokens shape: torch.Size([1, 1372]) -Dataloader batch - labels shape: torch.Size([1, 1372]) -Dataloader batch - attn_mask shape: torch.Size([1, 1372]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 351 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1372) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1372) torch.int64 - loss_mask : 309 / 1372 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1372, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1372, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1372, 1, 576) torch.bfloat16 - out : (1, 1372) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.1674 - top-1 accuracy : 27/309 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1198]) -Dataloader batch - labels shape: torch.Size([1, 1198]) -Dataloader batch - attn_mask shape: torch.Size([1, 1198]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 352 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1198) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1198) torch.int64 - loss_mask : 305 / 1198 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1198, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1198, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1198, 1, 576) torch.bfloat16 - out : (1, 1198) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.3158 - top-1 accuracy : 21/305 = 6.9% - - [2026-04-29 13:44:37] iteration 188/ 500 | consumed samples: 752 | elapsed time per iteration (ms): 4530.1 | throughput (token/sec/GPU): 1231.1 | learning rate: 9.275852E-05 | global batch size: 4 | lm loss: 1.124243E+01 | loss scale: 1.0 | grad norm: 0.266 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1795]) -Dataloader batch - labels shape: torch.Size([1, 1795]) -Dataloader batch - attn_mask shape: torch.Size([1, 1795]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 353 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1795) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1795) torch.int64 - loss_mask : 427 / 1795 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1795, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1795, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1795, 1, 576) torch.bfloat16 - out : (1, 1795) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 427 tokens) : 11.2869 - top-1 accuracy : 24/427 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1019]) -Dataloader batch - labels shape: torch.Size([1, 1019]) -Dataloader batch - attn_mask shape: torch.Size([1, 1019]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 354 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1019) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1019) torch.int64 - loss_mask : 270 / 1019 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1019, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1019, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1019, 1, 576) torch.bfloat16 - out : (1, 1019) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 270 tokens) : 11.3109 - top-1 accuracy : 6/270 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1524]) -Dataloader batch - labels shape: torch.Size([1, 1524]) -Dataloader batch - attn_mask shape: torch.Size([1, 1524]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 355 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1524) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1524) torch.int64 - loss_mask : 358 / 1524 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1524, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1524, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1524, 1, 576) torch.bfloat16 - out : (1, 1524) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2273 - top-1 accuracy : 19/358 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1846]) -Dataloader batch - labels shape: torch.Size([1, 1846]) -Dataloader batch - attn_mask shape: torch.Size([1, 1846]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 356 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1846) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1846) torch.int64 - loss_mask : 469 / 1846 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1846, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1846, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1846, 1, 576) torch.bfloat16 - out : (1, 1846) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 469 tokens) : 11.3113 - top-1 accuracy : 10/469 = 2.1% - - [2026-04-29 13:44:42] iteration 189/ 500 | consumed samples: 756 | elapsed time per iteration (ms): 5006.1 | throughput (token/sec/GPU): 1235.3 | learning rate: 9.259539E-05 | global batch size: 4 | lm loss: 1.128466E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1495]) -Dataloader batch - labels shape: torch.Size([1, 1495]) -Dataloader batch - attn_mask shape: torch.Size([1, 1495]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 357 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1495) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1495) torch.int64 - loss_mask : 339 / 1495 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1495, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1495, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1495, 1, 576) torch.bfloat16 - out : (1, 1495) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2649 - top-1 accuracy : 6/339 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1103]) -Dataloader batch - labels shape: torch.Size([1, 1103]) -Dataloader batch - attn_mask shape: torch.Size([1, 1103]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 358 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1103) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1103) torch.int64 - loss_mask : 259 / 1103 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1103, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1103, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1103, 1, 576) torch.bfloat16 - out : (1, 1103) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 259 tokens) : 11.2382 - top-1 accuracy : 22/259 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1463]) -Dataloader batch - labels shape: torch.Size([1, 1463]) -Dataloader batch - attn_mask shape: torch.Size([1, 1463]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 359 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1463) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1463) torch.int64 - loss_mask : 307 / 1463 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1463, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1463, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1463, 1, 576) torch.bfloat16 - out : (1, 1463) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.1645 - top-1 accuracy : 17/307 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1584]) -Dataloader batch - labels shape: torch.Size([1, 1584]) -Dataloader batch - attn_mask shape: torch.Size([1, 1584]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 360 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1584) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1584) torch.int64 - loss_mask : 361 / 1584 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1584, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1584, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1584, 1, 576) torch.bfloat16 - out : (1, 1584) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.2761 - top-1 accuracy : 20/361 = 5.5% - - [2026-04-29 13:44:47] iteration 190/ 500 | consumed samples: 760 | elapsed time per iteration (ms): 4685.3 | throughput (token/sec/GPU): 1204.8 | learning rate: 9.243060E-05 | global batch size: 4 | lm loss: 1.123829E+01 | loss scale: 1.0 | grad norm: 0.324 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 361 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 409 / 1374 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.3890 - top-1 accuracy : 9/409 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1366]) -Dataloader batch - labels shape: torch.Size([1, 1366]) -Dataloader batch - attn_mask shape: torch.Size([1, 1366]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 362 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1366) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1366) torch.int64 - loss_mask : 325 / 1366 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1366, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1366, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1366, 1, 576) torch.bfloat16 - out : (1, 1366) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.2536 - top-1 accuracy : 18/325 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 363 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 350 / 1504 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.2290 - top-1 accuracy : 9/350 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1361]) -Dataloader batch - labels shape: torch.Size([1, 1361]) -Dataloader batch - attn_mask shape: torch.Size([1, 1361]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 364 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1361) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1361) torch.int64 - loss_mask : 311 / 1361 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1361, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1361, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1361, 1, 576) torch.bfloat16 - out : (1, 1361) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.1839 - top-1 accuracy : 10/311 = 3.2% - - [2026-04-29 13:44:51] iteration 191/ 500 | consumed samples: 764 | elapsed time per iteration (ms): 4610.3 | throughput (token/sec/GPU): 1215.7 | learning rate: 9.226415E-05 | global batch size: 4 | lm loss: 1.127158E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1538]) -Dataloader batch - labels shape: torch.Size([1, 1538]) -Dataloader batch - attn_mask shape: torch.Size([1, 1538]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 365 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1538) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1538) torch.int64 - loss_mask : 364 / 1538 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1538, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1538, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1538, 1, 576) torch.bfloat16 - out : (1, 1538) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2152 - top-1 accuracy : 18/364 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1036]) -Dataloader batch - labels shape: torch.Size([1, 1036]) -Dataloader batch - attn_mask shape: torch.Size([1, 1036]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 366 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1036) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1036) torch.int64 - loss_mask : 270 / 1036 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1036, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1036, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1036, 1, 576) torch.bfloat16 - out : (1, 1036) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 270 tokens) : 11.3402 - top-1 accuracy : 0/270 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1576]) -Dataloader batch - labels shape: torch.Size([1, 1576]) -Dataloader batch - attn_mask shape: torch.Size([1, 1576]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 367 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1576) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1576) torch.int64 - loss_mask : 364 / 1576 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1576, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1576, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1576, 1, 576) torch.bfloat16 - out : (1, 1576) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2500 - top-1 accuracy : 6/364 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 965]) -Dataloader batch - labels shape: torch.Size([1, 965]) -Dataloader batch - attn_mask shape: torch.Size([1, 965]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 368 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 965) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 965) torch.int64 - loss_mask : 213 / 965 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (965, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (965, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (965, 1, 576) torch.bfloat16 - out : (1, 965) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 213 tokens) : 11.2473 - top-1 accuracy : 14/213 = 6.6% - - [2026-04-29 13:44:56] iteration 192/ 500 | consumed samples: 768 | elapsed time per iteration (ms): 4306.1 | throughput (token/sec/GPU): 1187.9 | learning rate: 9.209604E-05 | global batch size: 4 | lm loss: 1.125918E+01 | loss scale: 1.0 | grad norm: 0.308 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1529]) -Dataloader batch - labels shape: torch.Size([1, 1529]) -Dataloader batch - attn_mask shape: torch.Size([1, 1529]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 369 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1529) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1529) torch.int64 - loss_mask : 341 / 1529 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1529, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1529, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1529, 1, 576) torch.bfloat16 - out : (1, 1529) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.2515 - top-1 accuracy : 17/341 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 370 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 335 / 1424 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2377 - top-1 accuracy : 9/335 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1566]) -Dataloader batch - labels shape: torch.Size([1, 1566]) -Dataloader batch - attn_mask shape: torch.Size([1, 1566]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 371 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1566) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1566) torch.int64 - loss_mask : 367 / 1566 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1566, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1566, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1566, 1, 576) torch.bfloat16 - out : (1, 1566) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2608 - top-1 accuracy : 20/367 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1604]) -Dataloader batch - labels shape: torch.Size([1, 1604]) -Dataloader batch - attn_mask shape: torch.Size([1, 1604]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 372 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1604) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1604) torch.int64 - loss_mask : 363 / 1604 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1604, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1604, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1604, 1, 576) torch.bfloat16 - out : (1, 1604) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2557 - top-1 accuracy : 25/363 = 6.9% - - [2026-04-29 13:45:01] iteration 193/ 500 | consumed samples: 772 | elapsed time per iteration (ms): 4877.1 | throughput (token/sec/GPU): 1255.5 | learning rate: 9.192628E-05 | global batch size: 4 | lm loss: 1.125173E+01 | loss scale: 1.0 | grad norm: 0.284 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1505]) -Dataloader batch - labels shape: torch.Size([1, 1505]) -Dataloader batch - attn_mask shape: torch.Size([1, 1505]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 373 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1505) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1505) torch.int64 - loss_mask : 336 / 1505 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1505, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1505, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1505, 1, 576) torch.bfloat16 - out : (1, 1505) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.2005 - top-1 accuracy : 29/336 = 8.6% - -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 374 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 308 / 1459 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.1818 - top-1 accuracy : 32/308 = 10.4% - -Dataloader batch - tokens shape: torch.Size([1, 1249]) -Dataloader batch - labels shape: torch.Size([1, 1249]) -Dataloader batch - attn_mask shape: torch.Size([1, 1249]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 375 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1249) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1249) torch.int64 - loss_mask : 330 / 1249 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1249, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1249, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1249, 1, 576) torch.bfloat16 - out : (1, 1249) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.3623 - top-1 accuracy : 11/330 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 376 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 391 / 1750 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2128 - top-1 accuracy : 1/391 = 0.3% - - [2026-04-29 13:45:05] iteration 194/ 500 | consumed samples: 776 | elapsed time per iteration (ms): 4838.6 | throughput (token/sec/GPU): 1232.4 | learning rate: 9.175488E-05 | global batch size: 4 | lm loss: 1.123893E+01 | loss scale: 1.0 | grad norm: 0.286 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1579]) -Dataloader batch - labels shape: torch.Size([1, 1579]) -Dataloader batch - attn_mask shape: torch.Size([1, 1579]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 377 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1579) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1579) torch.int64 - loss_mask : 379 / 1579 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1579, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1579, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1579, 1, 576) torch.bfloat16 - out : (1, 1579) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.2553 - top-1 accuracy : 9/379 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1449]) -Dataloader batch - labels shape: torch.Size([1, 1449]) -Dataloader batch - attn_mask shape: torch.Size([1, 1449]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 378 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1449) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1449) torch.int64 - loss_mask : 358 / 1449 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1449, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1449, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1449, 1, 576) torch.bfloat16 - out : (1, 1449) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2758 - top-1 accuracy : 11/358 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 824]) -Dataloader batch - labels shape: torch.Size([1, 824]) -Dataloader batch - attn_mask shape: torch.Size([1, 824]) -Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) - -==================================================================== - PIPELINE — STEP 379 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 824) torch.int64 - images : (16, 3, 1024, 1024) torch.bfloat16 - labels : (1, 824) torch.int64 - loss_mask : 157 / 824 tokens (19.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (16, 3, 1024, 1024) torch.bfloat16 - out : (16, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (16, 3072) - out : (16, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (824, 1, 576) torch.bfloat16 grad=False - image slots : 16 tokens replaced with vision embeddings - combined : (824, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (824, 1, 576) torch.bfloat16 - out : (1, 824) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 157 tokens) : 11.1482 - top-1 accuracy : 5/157 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1547]) -Dataloader batch - labels shape: torch.Size([1, 1547]) -Dataloader batch - attn_mask shape: torch.Size([1, 1547]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 380 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1547) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1547) torch.int64 - loss_mask : 393 / 1547 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1547, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1547, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1547, 1, 576) torch.bfloat16 - out : (1, 1547) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 393 tokens) : 11.2929 - top-1 accuracy : 5/393 = 1.3% - - [2026-04-29 13:45:10] iteration 195/ 500 | consumed samples: 780 | elapsed time per iteration (ms): 4363.0 | throughput (token/sec/GPU): 1237.5 | learning rate: 9.158184E-05 | global batch size: 4 | lm loss: 1.125941E+01 | loss scale: 1.0 | grad norm: 0.351 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1719]) -Dataloader batch - labels shape: torch.Size([1, 1719]) -Dataloader batch - attn_mask shape: torch.Size([1, 1719]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 381 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1719) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1719) torch.int64 - loss_mask : 435 / 1719 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1719, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1719, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1719, 1, 576) torch.bfloat16 - out : (1, 1719) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 435 tokens) : 11.3306 - top-1 accuracy : 12/435 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1023]) -Dataloader batch - labels shape: torch.Size([1, 1023]) -Dataloader batch - attn_mask shape: torch.Size([1, 1023]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 382 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1023) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1023) torch.int64 - loss_mask : 250 / 1023 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1023, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1023, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1023, 1, 576) torch.bfloat16 - out : (1, 1023) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 250 tokens) : 11.2997 - top-1 accuracy : 11/250 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1538]) -Dataloader batch - labels shape: torch.Size([1, 1538]) -Dataloader batch - attn_mask shape: torch.Size([1, 1538]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 383 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1538) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1538) torch.int64 - loss_mask : 400 / 1538 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1538, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1538, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1538, 1, 576) torch.bfloat16 - out : (1, 1538) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.3108 - top-1 accuracy : 14/400 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1557]) -Dataloader batch - labels shape: torch.Size([1, 1557]) -Dataloader batch - attn_mask shape: torch.Size([1, 1557]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 384 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1557) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1557) torch.int64 - loss_mask : 398 / 1557 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1557, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1557, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1557, 1, 576) torch.bfloat16 - out : (1, 1557) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.2873 - top-1 accuracy : 15/398 = 3.8% - - [2026-04-29 13:45:14] iteration 196/ 500 | consumed samples: 784 | elapsed time per iteration (ms): 4406.6 | throughput (token/sec/GPU): 1324.6 | learning rate: 9.140718E-05 | global batch size: 4 | lm loss: 1.130843E+01 | loss scale: 1.0 | grad norm: 0.311 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1635]) -Dataloader batch - labels shape: torch.Size([1, 1635]) -Dataloader batch - attn_mask shape: torch.Size([1, 1635]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 385 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1635) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1635) torch.int64 - loss_mask : 377 / 1635 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1635, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1635, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1635, 1, 576) torch.bfloat16 - out : (1, 1635) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.2690 - top-1 accuracy : 29/377 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1394]) -Dataloader batch - labels shape: torch.Size([1, 1394]) -Dataloader batch - attn_mask shape: torch.Size([1, 1394]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 386 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1394) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1394) torch.int64 - loss_mask : 381 / 1394 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1394, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1394, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1394, 1, 576) torch.bfloat16 - out : (1, 1394) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.2850 - top-1 accuracy : 16/381 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1537]) -Dataloader batch - labels shape: torch.Size([1, 1537]) -Dataloader batch - attn_mask shape: torch.Size([1, 1537]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 387 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1537) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1537) torch.int64 - loss_mask : 399 / 1537 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1537, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1537, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1537, 1, 576) torch.bfloat16 - out : (1, 1537) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.2806 - top-1 accuracy : 12/399 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1573]) -Dataloader batch - labels shape: torch.Size([1, 1573]) -Dataloader batch - attn_mask shape: torch.Size([1, 1573]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 388 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1573) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1573) torch.int64 - loss_mask : 386 / 1573 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1573, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1573, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1573, 1, 576) torch.bfloat16 - out : (1, 1573) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2661 - top-1 accuracy : 14/386 = 3.6% - - [2026-04-29 13:45:19] iteration 197/ 500 | consumed samples: 788 | elapsed time per iteration (ms): 4856.8 | throughput (token/sec/GPU): 1264.0 | learning rate: 9.123089E-05 | global batch size: 4 | lm loss: 1.127522E+01 | loss scale: 1.0 | grad norm: 0.269 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1321]) -Dataloader batch - labels shape: torch.Size([1, 1321]) -Dataloader batch - attn_mask shape: torch.Size([1, 1321]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 389 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1321) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1321) torch.int64 - loss_mask : 356 / 1321 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1321, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1321, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1321, 1, 576) torch.bfloat16 - out : (1, 1321) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.3477 - top-1 accuracy : 10/356 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1369]) -Dataloader batch - labels shape: torch.Size([1, 1369]) -Dataloader batch - attn_mask shape: torch.Size([1, 1369]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 390 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1369) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1369) torch.int64 - loss_mask : 390 / 1369 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1369, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1369, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1369, 1, 576) torch.bfloat16 - out : (1, 1369) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3565 - top-1 accuracy : 3/390 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1076]) -Dataloader batch - labels shape: torch.Size([1, 1076]) -Dataloader batch - attn_mask shape: torch.Size([1, 1076]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 391 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1076) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1076) torch.int64 - loss_mask : 313 / 1076 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1076, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1076, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1076, 1, 576) torch.bfloat16 - out : (1, 1076) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.3643 - top-1 accuracy : 5/313 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1760]) -Dataloader batch - labels shape: torch.Size([1, 1760]) -Dataloader batch - attn_mask shape: torch.Size([1, 1760]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 392 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1760) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1760) torch.int64 - loss_mask : 383 / 1760 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1760, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1760, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1760, 1, 576) torch.bfloat16 - out : (1, 1760) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.1964 - top-1 accuracy : 23/383 = 6.0% - - [2026-04-29 13:45:23] iteration 198/ 500 | consumed samples: 792 | elapsed time per iteration (ms): 4329.7 | throughput (token/sec/GPU): 1276.3 | learning rate: 9.105299E-05 | global batch size: 4 | lm loss: 1.131351E+01 | loss scale: 1.0 | grad norm: 0.248 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1451]) -Dataloader batch - labels shape: torch.Size([1, 1451]) -Dataloader batch - attn_mask shape: torch.Size([1, 1451]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 393 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1451) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1451) torch.int64 - loss_mask : 387 / 1451 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1451, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1451, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1451, 1, 576) torch.bfloat16 - out : (1, 1451) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.3138 - top-1 accuracy : 14/387 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1227]) -Dataloader batch - labels shape: torch.Size([1, 1227]) -Dataloader batch - attn_mask shape: torch.Size([1, 1227]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 394 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1227) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1227) torch.int64 - loss_mask : 355 / 1227 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1227, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1227, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1227, 1, 576) torch.bfloat16 - out : (1, 1227) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.3596 - top-1 accuracy : 2/355 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1219]) -Dataloader batch - labels shape: torch.Size([1, 1219]) -Dataloader batch - attn_mask shape: torch.Size([1, 1219]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 395 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1219) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1219) torch.int64 - loss_mask : 284 / 1219 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1219, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1219, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1219, 1, 576) torch.bfloat16 - out : (1, 1219) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.2002 - top-1 accuracy : 11/284 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1411]) -Dataloader batch - labels shape: torch.Size([1, 1411]) -Dataloader batch - attn_mask shape: torch.Size([1, 1411]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 396 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1411) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1411) torch.int64 - loss_mask : 323 / 1411 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1411, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1411, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1411, 1, 576) torch.bfloat16 - out : (1, 1411) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.2023 - top-1 accuracy : 13/323 = 4.0% - - [2026-04-29 13:45:28] iteration 199/ 500 | consumed samples: 796 | elapsed time per iteration (ms): 4240.9 | throughput (token/sec/GPU): 1251.6 | learning rate: 9.087348E-05 | global batch size: 4 | lm loss: 1.127524E+01 | loss scale: 1.0 | grad norm: 0.277 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1723]) -Dataloader batch - labels shape: torch.Size([1, 1723]) -Dataloader batch - attn_mask shape: torch.Size([1, 1723]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 397 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1723) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1723) torch.int64 - loss_mask : 367 / 1723 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1723, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1723, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1723, 1, 576) torch.bfloat16 - out : (1, 1723) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2190 - top-1 accuracy : 25/367 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1691]) -Dataloader batch - labels shape: torch.Size([1, 1691]) -Dataloader batch - attn_mask shape: torch.Size([1, 1691]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 398 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1691) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1691) torch.int64 - loss_mask : 343 / 1691 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1691, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1691, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1691, 1, 576) torch.bfloat16 - out : (1, 1691) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.1668 - top-1 accuracy : 32/343 = 9.3% - -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 399 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 315 / 1468 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.1500 - top-1 accuracy : 12/315 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1904]) -Dataloader batch - labels shape: torch.Size([1, 1904]) -Dataloader batch - attn_mask shape: torch.Size([1, 1904]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 400 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1904) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1904) torch.int64 - loss_mask : 538 / 1904 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1904, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1904, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1904, 1, 576) torch.bfloat16 - out : (1, 1904) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 538 tokens) : 11.3801 - top-1 accuracy : 2/538 = 0.4% - - [2026-04-29 13:45:33] iteration 200/ 500 | consumed samples: 800 | elapsed time per iteration (ms): 5267.3 | throughput (token/sec/GPU): 1288.3 | learning rate: 9.069237E-05 | global batch size: 4 | lm loss: 1.124909E+01 | loss scale: 1.0 | grad norm: 0.350 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (5254.65, 5254.65) - forward-compute ................................: (3949.06, 3949.06) - backward-compute ...............................: (1303.69, 1303.69) - batch-generator ................................: (511.03, 511.03) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.35, 0.35) - params-all-gather ..............................: (0.11, 0.11) - optimizer-copy-to-main-grad ....................: (0.10, 0.10) - optimizer-clip-main-grad .......................: (0.40, 0.40) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.09, 1.09) - optimizer-copy-main-to-model-params ............: (0.18, 0.18) - optimizer ......................................: (2.43, 2.43) -saving checkpoint at iteration 200 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 200 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 200 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1978.33, 1978.33) -Dataloader batch - tokens shape: torch.Size([1, 1556]) -Dataloader batch - labels shape: torch.Size([1, 1556]) -Dataloader batch - attn_mask shape: torch.Size([1, 1556]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 401 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1556) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1556) torch.int64 - loss_mask : 359 / 1556 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1556, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1556, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1556, 1, 576) torch.bfloat16 - out : (1, 1556) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.3096 - top-1 accuracy : 19/359 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1410]) -Dataloader batch - labels shape: torch.Size([1, 1410]) -Dataloader batch - attn_mask shape: torch.Size([1, 1410]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 402 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1410) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1410) torch.int64 - loss_mask : 362 / 1410 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1410, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1410, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1410, 1, 576) torch.bfloat16 - out : (1, 1410) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2956 - top-1 accuracy : 21/362 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1049]) -Dataloader batch - labels shape: torch.Size([1, 1049]) -Dataloader batch - attn_mask shape: torch.Size([1, 1049]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 403 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1049) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1049) torch.int64 - loss_mask : 285 / 1049 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1049, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1049, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1049, 1, 576) torch.bfloat16 - out : (1, 1049) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 285 tokens) : 11.3920 - top-1 accuracy : 5/285 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1791]) -Dataloader batch - labels shape: torch.Size([1, 1791]) -Dataloader batch - attn_mask shape: torch.Size([1, 1791]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 404 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1791) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1791) torch.int64 - loss_mask : 406 / 1791 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1791, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1791, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1791, 1, 576) torch.bfloat16 - out : (1, 1791) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.2465 - top-1 accuracy : 4/406 = 1.0% - - [2026-04-29 13:45:40] iteration 201/ 500 | consumed samples: 804 | elapsed time per iteration (ms): 4697.2 | throughput (token/sec/GPU): 1236.1 | learning rate: 9.050967E-05 | global batch size: 4 | lm loss: 1.130451E+01 | loss scale: 1.0 | grad norm: 0.244 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1407]) -Dataloader batch - labels shape: torch.Size([1, 1407]) -Dataloader batch - attn_mask shape: torch.Size([1, 1407]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 405 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1407) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1407) torch.int64 - loss_mask : 349 / 1407 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1407, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1407, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1407, 1, 576) torch.bfloat16 - out : (1, 1407) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.3175 - top-1 accuracy : 8/349 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1836]) -Dataloader batch - labels shape: torch.Size([1, 1836]) -Dataloader batch - attn_mask shape: torch.Size([1, 1836]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 406 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1836) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1836) torch.int64 - loss_mask : 487 / 1836 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1836, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1836, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1836, 1, 576) torch.bfloat16 - out : (1, 1836) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 487 tokens) : 11.3269 - top-1 accuracy : 13/487 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 995]) -Dataloader batch - labels shape: torch.Size([1, 995]) -Dataloader batch - attn_mask shape: torch.Size([1, 995]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 407 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 995) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 995) torch.int64 - loss_mask : 241 / 995 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (995, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (995, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (995, 1, 576) torch.bfloat16 - out : (1, 995) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 241 tokens) : 11.2427 - top-1 accuracy : 0/241 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1928]) -Dataloader batch - labels shape: torch.Size([1, 1928]) -Dataloader batch - attn_mask shape: torch.Size([1, 1928]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 408 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1928) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1928) torch.int64 - loss_mask : 576 / 1928 tokens (29.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1928, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1928, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1928, 1, 576) torch.bfloat16 - out : (1, 1928) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 576 tokens) : 11.4123 - top-1 accuracy : 16/576 = 2.8% - - [2026-04-29 13:45:44] iteration 202/ 500 | consumed samples: 808 | elapsed time per iteration (ms): 4689.9 | throughput (token/sec/GPU): 1314.7 | learning rate: 9.032539E-05 | global batch size: 4 | lm loss: 1.134238E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 409 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1519) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1519) torch.int64 - loss_mask : 352 / 1519 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1519, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1519, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1519, 1, 576) torch.bfloat16 - out : (1, 1519) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.2670 - top-1 accuracy : 13/352 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1634]) -Dataloader batch - labels shape: torch.Size([1, 1634]) -Dataloader batch - attn_mask shape: torch.Size([1, 1634]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 410 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1634) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1634) torch.int64 - loss_mask : 368 / 1634 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1634, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1634, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1634, 1, 576) torch.bfloat16 - out : (1, 1634) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2171 - top-1 accuracy : 13/368 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1781]) -Dataloader batch - labels shape: torch.Size([1, 1781]) -Dataloader batch - attn_mask shape: torch.Size([1, 1781]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 411 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1781) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1781) torch.int64 - loss_mask : 416 / 1781 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1781, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1781, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1781, 1, 576) torch.bfloat16 - out : (1, 1781) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.2827 - top-1 accuracy : 20/416 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1399]) -Dataloader batch - labels shape: torch.Size([1, 1399]) -Dataloader batch - attn_mask shape: torch.Size([1, 1399]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 412 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1399) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1399) torch.int64 - loss_mask : 338 / 1399 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1399, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1399, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1399, 1, 576) torch.bfloat16 - out : (1, 1399) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.2969 - top-1 accuracy : 15/338 = 4.4% - - [2026-04-29 13:45:49] iteration 203/ 500 | consumed samples: 812 | elapsed time per iteration (ms): 4931.6 | throughput (token/sec/GPU): 1284.2 | learning rate: 9.013952E-05 | global batch size: 4 | lm loss: 1.126583E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1591]) -Dataloader batch - labels shape: torch.Size([1, 1591]) -Dataloader batch - attn_mask shape: torch.Size([1, 1591]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 413 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1591) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1591) torch.int64 - loss_mask : 367 / 1591 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1591, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1591, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1591, 1, 576) torch.bfloat16 - out : (1, 1591) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2681 - top-1 accuracy : 17/367 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1483]) -Dataloader batch - labels shape: torch.Size([1, 1483]) -Dataloader batch - attn_mask shape: torch.Size([1, 1483]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 414 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1483) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1483) torch.int64 - loss_mask : 396 / 1483 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1483, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1483, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1483, 1, 576) torch.bfloat16 - out : (1, 1483) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 396 tokens) : 11.3274 - top-1 accuracy : 7/396 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 415 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 286 / 1374 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 286 tokens) : 11.2492 - top-1 accuracy : 21/286 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1086]) -Dataloader batch - labels shape: torch.Size([1, 1086]) -Dataloader batch - attn_mask shape: torch.Size([1, 1086]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 416 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1086) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1086) torch.int64 - loss_mask : 329 / 1086 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1086, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1086, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1086, 1, 576) torch.bfloat16 - out : (1, 1086) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.4023 - top-1 accuracy : 1/329 = 0.3% - - [2026-04-29 13:45:54] iteration 204/ 500 | consumed samples: 816 | elapsed time per iteration (ms): 4457.6 | throughput (token/sec/GPU): 1241.5 | learning rate: 8.995208E-05 | global batch size: 4 | lm loss: 1.131327E+01 | loss scale: 1.0 | grad norm: 0.283 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1015]) -Dataloader batch - labels shape: torch.Size([1, 1015]) -Dataloader batch - attn_mask shape: torch.Size([1, 1015]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 417 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1015) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1015) torch.int64 - loss_mask : 265 / 1015 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1015, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1015, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1015, 1, 576) torch.bfloat16 - out : (1, 1015) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 265 tokens) : 11.3250 - top-1 accuracy : 13/265 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1497]) -Dataloader batch - labels shape: torch.Size([1, 1497]) -Dataloader batch - attn_mask shape: torch.Size([1, 1497]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 418 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1497) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1497) torch.int64 - loss_mask : 316 / 1497 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1497, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1497, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1497, 1, 576) torch.bfloat16 - out : (1, 1497) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.1814 - top-1 accuracy : 20/316 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1435]) -Dataloader batch - labels shape: torch.Size([1, 1435]) -Dataloader batch - attn_mask shape: torch.Size([1, 1435]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 419 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1435) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1435) torch.int64 - loss_mask : 358 / 1435 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1435, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1435, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1435, 1, 576) torch.bfloat16 - out : (1, 1435) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2913 - top-1 accuracy : 11/358 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1528]) -Dataloader batch - labels shape: torch.Size([1, 1528]) -Dataloader batch - attn_mask shape: torch.Size([1, 1528]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 420 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1528) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1528) torch.int64 - loss_mask : 336 / 1528 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1528, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1528, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1528, 1, 576) torch.bfloat16 - out : (1, 1528) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.2521 - top-1 accuracy : 9/336 = 2.7% - - [2026-04-29 13:45:58] iteration 205/ 500 | consumed samples: 820 | elapsed time per iteration (ms): 4566.8 | throughput (token/sec/GPU): 1198.9 | learning rate: 8.976308E-05 | global batch size: 4 | lm loss: 1.126074E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1110]) -Dataloader batch - labels shape: torch.Size([1, 1110]) -Dataloader batch - attn_mask shape: torch.Size([1, 1110]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 421 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1110) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1110) torch.int64 - loss_mask : 249 / 1110 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1110, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1110, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1110, 1, 576) torch.bfloat16 - out : (1, 1110) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 249 tokens) : 11.1566 - top-1 accuracy : 7/249 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1004]) -Dataloader batch - labels shape: torch.Size([1, 1004]) -Dataloader batch - attn_mask shape: torch.Size([1, 1004]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 422 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1004) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1004) torch.int64 - loss_mask : 251 / 1004 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1004, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1004, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1004, 1, 576) torch.bfloat16 - out : (1, 1004) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2837 - top-1 accuracy : 4/251 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1529]) -Dataloader batch - labels shape: torch.Size([1, 1529]) -Dataloader batch - attn_mask shape: torch.Size([1, 1529]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 423 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1529) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1529) torch.int64 - loss_mask : 330 / 1529 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1529, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1529, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1529, 1, 576) torch.bfloat16 - out : (1, 1529) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.1655 - top-1 accuracy : 22/330 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1419]) -Dataloader batch - labels shape: torch.Size([1, 1419]) -Dataloader batch - attn_mask shape: torch.Size([1, 1419]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 424 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1419) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1419) torch.int64 - loss_mask : 364 / 1419 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1419, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1419, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1419, 1, 576) torch.bfloat16 - out : (1, 1419) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2944 - top-1 accuracy : 15/364 = 4.1% - - [2026-04-29 13:46:02] iteration 206/ 500 | consumed samples: 824 | elapsed time per iteration (ms): 4146.9 | throughput (token/sec/GPU): 1220.7 | learning rate: 8.957252E-05 | global batch size: 4 | lm loss: 1.122781E+01 | loss scale: 1.0 | grad norm: 0.278 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1890]) -Dataloader batch - labels shape: torch.Size([1, 1890]) -Dataloader batch - attn_mask shape: torch.Size([1, 1890]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 425 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1890) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1890) torch.int64 - loss_mask : 524 / 1890 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1890, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1890, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1890, 1, 576) torch.bfloat16 - out : (1, 1890) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 524 tokens) : 11.3317 - top-1 accuracy : 23/524 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1578]) -Dataloader batch - labels shape: torch.Size([1, 1578]) -Dataloader batch - attn_mask shape: torch.Size([1, 1578]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 426 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1578) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1578) torch.int64 - loss_mask : 354 / 1578 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1578, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1578, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1578, 1, 576) torch.bfloat16 - out : (1, 1578) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2565 - top-1 accuracy : 6/354 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1152]) -Dataloader batch - labels shape: torch.Size([1, 1152]) -Dataloader batch - attn_mask shape: torch.Size([1, 1152]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 427 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1152) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1152) torch.int64 - loss_mask : 294 / 1152 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1152, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1152, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1152, 1, 576) torch.bfloat16 - out : (1, 1152) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.3737 - top-1 accuracy : 2/294 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1382]) -Dataloader batch - labels shape: torch.Size([1, 1382]) -Dataloader batch - attn_mask shape: torch.Size([1, 1382]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 428 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1382) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1382) torch.int64 - loss_mask : 328 / 1382 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1382, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1382, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1382, 1, 576) torch.bfloat16 - out : (1, 1382) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.2141 - top-1 accuracy : 9/328 = 2.7% - - [2026-04-29 13:46:07] iteration 207/ 500 | consumed samples: 828 | elapsed time per iteration (ms): 4705.6 | throughput (token/sec/GPU): 1275.5 | learning rate: 8.938042E-05 | global batch size: 4 | lm loss: 1.129644E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1377]) -Dataloader batch - labels shape: torch.Size([1, 1377]) -Dataloader batch - attn_mask shape: torch.Size([1, 1377]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 429 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1377) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1377) torch.int64 - loss_mask : 312 / 1377 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1377, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1377, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1377, 1, 576) torch.bfloat16 - out : (1, 1377) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.2704 - top-1 accuracy : 8/312 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1537]) -Dataloader batch - labels shape: torch.Size([1, 1537]) -Dataloader batch - attn_mask shape: torch.Size([1, 1537]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 430 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1537) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1537) torch.int64 - loss_mask : 358 / 1537 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1537, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1537, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1537, 1, 576) torch.bfloat16 - out : (1, 1537) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2511 - top-1 accuracy : 28/358 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1868]) -Dataloader batch - labels shape: torch.Size([1, 1868]) -Dataloader batch - attn_mask shape: torch.Size([1, 1868]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 431 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1868) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1868) torch.int64 - loss_mask : 508 / 1868 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1868, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1868, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1868, 1, 576) torch.bfloat16 - out : (1, 1868) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 508 tokens) : 11.3621 - top-1 accuracy : 7/508 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1000]) -Dataloader batch - labels shape: torch.Size([1, 1000]) -Dataloader batch - attn_mask shape: torch.Size([1, 1000]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 432 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1000) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1000) torch.int64 - loss_mask : 237 / 1000 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1000, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1000, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1000, 1, 576) torch.bfloat16 - out : (1, 1000) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 237 tokens) : 11.2796 - top-1 accuracy : 4/237 = 1.7% - - [2026-04-29 13:46:12] iteration 208/ 500 | consumed samples: 832 | elapsed time per iteration (ms): 4547.2 | throughput (token/sec/GPU): 1271.6 | learning rate: 8.918677E-05 | global batch size: 4 | lm loss: 1.129996E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1133]) -Dataloader batch - labels shape: torch.Size([1, 1133]) -Dataloader batch - attn_mask shape: torch.Size([1, 1133]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 433 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1133) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1133) torch.int64 - loss_mask : 336 / 1133 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1133, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1133, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1133, 1, 576) torch.bfloat16 - out : (1, 1133) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.4126 - top-1 accuracy : 4/336 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1001]) -Dataloader batch - labels shape: torch.Size([1, 1001]) -Dataloader batch - attn_mask shape: torch.Size([1, 1001]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 434 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1001) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1001) torch.int64 - loss_mask : 225 / 1001 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1001, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1001, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1001, 1, 576) torch.bfloat16 - out : (1, 1001) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 225 tokens) : 11.2407 - top-1 accuracy : 12/225 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1786]) -Dataloader batch - labels shape: torch.Size([1, 1786]) -Dataloader batch - attn_mask shape: torch.Size([1, 1786]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 435 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1786) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1786) torch.int64 - loss_mask : 415 / 1786 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1786, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1786, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1786, 1, 576) torch.bfloat16 - out : (1, 1786) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.2586 - top-1 accuracy : 27/415 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 436 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 407 / 1539 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.2975 - top-1 accuracy : 4/407 = 1.0% - - [2026-04-29 13:46:16] iteration 209/ 500 | consumed samples: 836 | elapsed time per iteration (ms): 4319.5 | throughput (token/sec/GPU): 1263.8 | learning rate: 8.899160E-05 | global batch size: 4 | lm loss: 1.130454E+01 | loss scale: 1.0 | grad norm: 0.264 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1485]) -Dataloader batch - labels shape: torch.Size([1, 1485]) -Dataloader batch - attn_mask shape: torch.Size([1, 1485]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 437 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1485) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1485) torch.int64 - loss_mask : 423 / 1485 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1485, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1485, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1485, 1, 576) torch.bfloat16 - out : (1, 1485) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 423 tokens) : 11.3531 - top-1 accuracy : 5/423 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1210]) -Dataloader batch - labels shape: torch.Size([1, 1210]) -Dataloader batch - attn_mask shape: torch.Size([1, 1210]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 438 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1210) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1210) torch.int64 - loss_mask : 323 / 1210 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1210, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1210, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1210, 1, 576) torch.bfloat16 - out : (1, 1210) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.3007 - top-1 accuracy : 3/323 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 439 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 341 / 1695 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.1731 - top-1 accuracy : 31/341 = 9.1% - -Dataloader batch - tokens shape: torch.Size([1, 1916]) -Dataloader batch - labels shape: torch.Size([1, 1916]) -Dataloader batch - attn_mask shape: torch.Size([1, 1916]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 440 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1916) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1916) torch.int64 - loss_mask : 562 / 1916 tokens (29.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1916, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1916, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1916, 1, 576) torch.bfloat16 - out : (1, 1916) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 562 tokens) : 11.3773 - top-1 accuracy : 20/562 = 3.6% - - [2026-04-29 13:46:21] iteration 210/ 500 | consumed samples: 840 | elapsed time per iteration (ms): 4846.4 | throughput (token/sec/GPU): 1301.2 | learning rate: 8.879489E-05 | global batch size: 4 | lm loss: 1.131386E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1410]) -Dataloader batch - labels shape: torch.Size([1, 1410]) -Dataloader batch - attn_mask shape: torch.Size([1, 1410]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 441 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1410) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1410) torch.int64 - loss_mask : 321 / 1410 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1410, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1410, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1410, 1, 576) torch.bfloat16 - out : (1, 1410) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2486 - top-1 accuracy : 14/321 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1164]) -Dataloader batch - labels shape: torch.Size([1, 1164]) -Dataloader batch - attn_mask shape: torch.Size([1, 1164]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 442 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1164) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1164) torch.int64 - loss_mask : 313 / 1164 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1164, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1164, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1164, 1, 576) torch.bfloat16 - out : (1, 1164) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.3809 - top-1 accuracy : 2/313 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1777]) -Dataloader batch - labels shape: torch.Size([1, 1777]) -Dataloader batch - attn_mask shape: torch.Size([1, 1777]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 443 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1777) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1777) torch.int64 - loss_mask : 406 / 1777 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1777, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1777, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1777, 1, 576) torch.bfloat16 - out : (1, 1777) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.2695 - top-1 accuracy : 24/406 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 1527]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 444 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1527) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1527) torch.int64 - loss_mask : 390 / 1527 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1527, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1527, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1527, 1, 576) torch.bfloat16 - out : (1, 1527) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.2838 - top-1 accuracy : 7/390 = 1.8% - - [2026-04-29 13:46:25] iteration 211/ 500 | consumed samples: 844 | elapsed time per iteration (ms): 4637.2 | throughput (token/sec/GPU): 1267.6 | learning rate: 8.859667E-05 | global batch size: 4 | lm loss: 1.129310E+01 | loss scale: 1.0 | grad norm: 0.244 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 445 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 348 / 1695 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.1641 - top-1 accuracy : 25/348 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1471]) -Dataloader batch - labels shape: torch.Size([1, 1471]) -Dataloader batch - attn_mask shape: torch.Size([1, 1471]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 446 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1471) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1471) torch.int64 - loss_mask : 303 / 1471 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1471, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1471, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1471, 1, 576) torch.bfloat16 - out : (1, 1471) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.1466 - top-1 accuracy : 28/303 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1773]) -Dataloader batch - labels shape: torch.Size([1, 1773]) -Dataloader batch - attn_mask shape: torch.Size([1, 1773]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 447 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1773) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1773) torch.int64 - loss_mask : 420 / 1773 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1773, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1773, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1773, 1, 576) torch.bfloat16 - out : (1, 1773) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 420 tokens) : 11.2642 - top-1 accuracy : 22/420 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1649]) -Dataloader batch - labels shape: torch.Size([1, 1649]) -Dataloader batch - attn_mask shape: torch.Size([1, 1649]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 448 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1649) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1649) torch.int64 - loss_mask : 431 / 1649 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1649, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1649, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1649, 1, 576) torch.bfloat16 - out : (1, 1649) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.3131 - top-1 accuracy : 4/431 = 0.9% - - [2026-04-29 13:46:31] iteration 212/ 500 | consumed samples: 848 | elapsed time per iteration (ms): 5195.4 | throughput (token/sec/GPU): 1268.0 | learning rate: 8.839693E-05 | global batch size: 4 | lm loss: 1.123132E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1514]) -Dataloader batch - labels shape: torch.Size([1, 1514]) -Dataloader batch - attn_mask shape: torch.Size([1, 1514]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 449 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1514) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1514) torch.int64 - loss_mask : 380 / 1514 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1514, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1514, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1514, 1, 576) torch.bfloat16 - out : (1, 1514) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 380 tokens) : 11.2525 - top-1 accuracy : 6/380 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 992]) -Dataloader batch - labels shape: torch.Size([1, 992]) -Dataloader batch - attn_mask shape: torch.Size([1, 992]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 450 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 992) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 992) torch.int64 - loss_mask : 242 / 992 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (992, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (992, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (992, 1, 576) torch.bfloat16 - out : (1, 992) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 242 tokens) : 11.3191 - top-1 accuracy : 21/242 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1522]) -Dataloader batch - labels shape: torch.Size([1, 1522]) -Dataloader batch - attn_mask shape: torch.Size([1, 1522]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 451 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1522) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1522) torch.int64 - loss_mask : 384 / 1522 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1522, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1522, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1522, 1, 576) torch.bfloat16 - out : (1, 1522) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.2507 - top-1 accuracy : 12/384 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1747]) -Dataloader batch - labels shape: torch.Size([1, 1747]) -Dataloader batch - attn_mask shape: torch.Size([1, 1747]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 452 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1747) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1747) torch.int64 - loss_mask : 382 / 1747 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1747, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1747, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1747, 1, 576) torch.bfloat16 - out : (1, 1747) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.1784 - top-1 accuracy : 25/382 = 6.5% - - [2026-04-29 13:46:35] iteration 213/ 500 | consumed samples: 852 | elapsed time per iteration (ms): 4559.4 | throughput (token/sec/GPU): 1266.6 | learning rate: 8.819569E-05 | global batch size: 4 | lm loss: 1.124325E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1752]) -Dataloader batch - labels shape: torch.Size([1, 1752]) -Dataloader batch - attn_mask shape: torch.Size([1, 1752]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 453 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1752) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1752) torch.int64 - loss_mask : 391 / 1752 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1752, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1752, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1752, 1, 576) torch.bfloat16 - out : (1, 1752) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2034 - top-1 accuracy : 23/391 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 454 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 295 / 1436 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.1666 - top-1 accuracy : 4/295 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 455 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 298 / 1380 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 298 tokens) : 11.2077 - top-1 accuracy : 23/298 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 456 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 357 / 1424 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3626 - top-1 accuracy : 17/357 = 4.8% - - [2026-04-29 13:46:40] iteration 214/ 500 | consumed samples: 856 | elapsed time per iteration (ms): 4825.5 | throughput (token/sec/GPU): 1241.7 | learning rate: 8.799296E-05 | global batch size: 4 | lm loss: 1.123865E+01 | loss scale: 1.0 | grad norm: 0.300 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 457 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 351 / 1695 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.1756 - top-1 accuracy : 21/351 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1220]) -Dataloader batch - labels shape: torch.Size([1, 1220]) -Dataloader batch - attn_mask shape: torch.Size([1, 1220]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 458 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1220) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1220) torch.int64 - loss_mask : 300 / 1220 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1220, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1220, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1220, 1, 576) torch.bfloat16 - out : (1, 1220) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.2240 - top-1 accuracy : 22/300 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 459 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 394 / 1539 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3416 - top-1 accuracy : 21/394 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1445]) -Dataloader batch - labels shape: torch.Size([1, 1445]) -Dataloader batch - attn_mask shape: torch.Size([1, 1445]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 460 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1445) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1445) torch.int64 - loss_mask : 368 / 1445 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1445, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1445, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1445, 1, 576) torch.bfloat16 - out : (1, 1445) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2998 - top-1 accuracy : 14/368 = 3.8% - - [2026-04-29 13:46:45] iteration 215/ 500 | consumed samples: 860 | elapsed time per iteration (ms): 4758.7 | throughput (token/sec/GPU): 1239.6 | learning rate: 8.778875E-05 | global batch size: 4 | lm loss: 1.126451E+01 | loss scale: 1.0 | grad norm: 0.255 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1355]) -Dataloader batch - labels shape: torch.Size([1, 1355]) -Dataloader batch - attn_mask shape: torch.Size([1, 1355]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 461 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1355) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1355) torch.int64 - loss_mask : 278 / 1355 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1355, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1355, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1355, 1, 576) torch.bfloat16 - out : (1, 1355) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.1681 - top-1 accuracy : 22/278 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1759]) -Dataloader batch - labels shape: torch.Size([1, 1759]) -Dataloader batch - attn_mask shape: torch.Size([1, 1759]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 462 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1759) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1759) torch.int64 - loss_mask : 434 / 1759 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1759, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1759, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1759, 1, 576) torch.bfloat16 - out : (1, 1759) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 434 tokens) : 11.3506 - top-1 accuracy : 23/434 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1494]) -Dataloader batch - labels shape: torch.Size([1, 1494]) -Dataloader batch - attn_mask shape: torch.Size([1, 1494]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 463 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1494) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1494) torch.int64 - loss_mask : 331 / 1494 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1494, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1494, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1494, 1, 576) torch.bfloat16 - out : (1, 1494) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.1546 - top-1 accuracy : 18/331 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 774]) -Dataloader batch - labels shape: torch.Size([1, 774]) -Dataloader batch - attn_mask shape: torch.Size([1, 774]) -Dataloader batch - image_grid_thw shape: torch.Size([15, 3]) - -==================================================================== - PIPELINE — STEP 464 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 774) torch.int64 - images : (15, 3, 1024, 1024) torch.bfloat16 - labels : (1, 774) torch.int64 - loss_mask : 160 / 774 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (15, 3, 1024, 1024) torch.bfloat16 - out : (15, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (15, 3072) - out : (15, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (774, 1, 576) torch.bfloat16 grad=False - image slots : 15 tokens replaced with vision embeddings - combined : (774, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (774, 1, 576) torch.bfloat16 - out : (1, 774) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 160 tokens) : 11.1209 - top-1 accuracy : 1/160 = 0.6% - - [2026-04-29 13:46:49] iteration 216/ 500 | consumed samples: 864 | elapsed time per iteration (ms): 4374.2 | throughput (token/sec/GPU): 1230.4 | learning rate: 8.758306E-05 | global batch size: 4 | lm loss: 1.122394E+01 | loss scale: 1.0 | grad norm: 0.310 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1204]) -Dataloader batch - labels shape: torch.Size([1, 1204]) -Dataloader batch - attn_mask shape: torch.Size([1, 1204]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 465 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1204) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1204) torch.int64 - loss_mask : 363 / 1204 tokens (30.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1204, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1204, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1204, 1, 576) torch.bfloat16 - out : (1, 1204) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.4012 - top-1 accuracy : 13/363 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1162]) -Dataloader batch - labels shape: torch.Size([1, 1162]) -Dataloader batch - attn_mask shape: torch.Size([1, 1162]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 466 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1162) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1162) torch.int64 - loss_mask : 260 / 1162 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1162, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1162, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1162, 1, 576) torch.bfloat16 - out : (1, 1162) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.1802 - top-1 accuracy : 10/260 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1506]) -Dataloader batch - labels shape: torch.Size([1, 1506]) -Dataloader batch - attn_mask shape: torch.Size([1, 1506]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 467 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1506) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1506) torch.int64 - loss_mask : 343 / 1506 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1506, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1506, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1506, 1, 576) torch.bfloat16 - out : (1, 1506) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.1910 - top-1 accuracy : 21/343 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1559]) -Dataloader batch - labels shape: torch.Size([1, 1559]) -Dataloader batch - attn_mask shape: torch.Size([1, 1559]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 468 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1559) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1559) torch.int64 - loss_mask : 359 / 1559 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1559, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1559, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1559, 1, 576) torch.bfloat16 - out : (1, 1559) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.2222 - top-1 accuracy : 20/359 = 5.6% - - [2026-04-29 13:46:54] iteration 217/ 500 | consumed samples: 868 | elapsed time per iteration (ms): 4547.7 | throughput (token/sec/GPU): 1194.2 | learning rate: 8.737589E-05 | global batch size: 4 | lm loss: 1.125494E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1432]) -Dataloader batch - labels shape: torch.Size([1, 1432]) -Dataloader batch - attn_mask shape: torch.Size([1, 1432]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 469 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1432) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1432) torch.int64 - loss_mask : 345 / 1432 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1432, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1432, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1432, 1, 576) torch.bfloat16 - out : (1, 1432) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2362 - top-1 accuracy : 3/345 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1531]) -Dataloader batch - labels shape: torch.Size([1, 1531]) -Dataloader batch - attn_mask shape: torch.Size([1, 1531]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 470 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1531) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1531) torch.int64 - loss_mask : 419 / 1531 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1531, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1531, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1531, 1, 576) torch.bfloat16 - out : (1, 1531) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.4014 - top-1 accuracy : 2/419 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 471 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 329 / 1508 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2274 - top-1 accuracy : 28/329 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1549]) -Dataloader batch - labels shape: torch.Size([1, 1549]) -Dataloader batch - attn_mask shape: torch.Size([1, 1549]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 472 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1549) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1549) torch.int64 - loss_mask : 403 / 1549 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1549, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1549, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1549, 1, 576) torch.bfloat16 - out : (1, 1549) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.3472 - top-1 accuracy : 3/403 = 0.7% - - [2026-04-29 13:46:58] iteration 218/ 500 | consumed samples: 872 | elapsed time per iteration (ms): 4676.2 | throughput (token/sec/GPU): 1287.4 | learning rate: 8.716727E-05 | global batch size: 4 | lm loss: 1.131041E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1872]) -Dataloader batch - labels shape: torch.Size([1, 1872]) -Dataloader batch - attn_mask shape: torch.Size([1, 1872]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 473 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1872) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1872) torch.int64 - loss_mask : 501 / 1872 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1872, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1872, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1872, 1, 576) torch.bfloat16 - out : (1, 1872) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 501 tokens) : 11.3389 - top-1 accuracy : 18/501 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1116]) -Dataloader batch - labels shape: torch.Size([1, 1116]) -Dataloader batch - attn_mask shape: torch.Size([1, 1116]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 474 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1116) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1116) torch.int64 - loss_mask : 263 / 1116 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1116, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1116, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1116, 1, 576) torch.bfloat16 - out : (1, 1116) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.2071 - top-1 accuracy : 1/263 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 475 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 344 / 1695 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.1295 - top-1 accuracy : 34/344 = 9.9% - -Dataloader batch - tokens shape: torch.Size([1, 1710]) -Dataloader batch - labels shape: torch.Size([1, 1710]) -Dataloader batch - attn_mask shape: torch.Size([1, 1710]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 476 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1710) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1710) torch.int64 - loss_mask : 369 / 1710 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1710, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1710, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1710, 1, 576) torch.bfloat16 - out : (1, 1710) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.1695 - top-1 accuracy : 38/369 = 10.3% - - [2026-04-29 13:47:03] iteration 219/ 500 | consumed samples: 876 | elapsed time per iteration (ms): 5107.9 | throughput (token/sec/GPU): 1251.6 | learning rate: 8.695719E-05 | global batch size: 4 | lm loss: 1.122432E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 477 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1420) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1420) torch.int64 - loss_mask : 352 / 1420 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1420, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1420, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1420, 1, 576) torch.bfloat16 - out : (1, 1420) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.3035 - top-1 accuracy : 15/352 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 478 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 370 / 1552 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.2539 - top-1 accuracy : 12/370 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1108]) -Dataloader batch - labels shape: torch.Size([1, 1108]) -Dataloader batch - attn_mask shape: torch.Size([1, 1108]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 479 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1108) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1108) torch.int64 - loss_mask : 298 / 1108 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1108, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1108, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1108, 1, 576) torch.bfloat16 - out : (1, 1108) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 298 tokens) : 11.3259 - top-1 accuracy : 3/298 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1540]) -Dataloader batch - labels shape: torch.Size([1, 1540]) -Dataloader batch - attn_mask shape: torch.Size([1, 1540]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 480 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1540) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1540) torch.int64 - loss_mask : 327 / 1540 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1540, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1540, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1540, 1, 576) torch.bfloat16 - out : (1, 1540) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2054 - top-1 accuracy : 33/327 = 10.1% - - [2026-04-29 13:47:08] iteration 220/ 500 | consumed samples: 880 | elapsed time per iteration (ms): 4417.1 | throughput (token/sec/GPU): 1272.3 | learning rate: 8.674567E-05 | global batch size: 4 | lm loss: 1.127101E+01 | loss scale: 1.0 | grad norm: 0.276 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4404.10, 4404.10) - forward-compute ................................: (3293.57, 3293.57) - backward-compute ...............................: (1108.59, 1108.59) - batch-generator ................................: (440.77, 440.77) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.41, 0.41) - params-all-gather ..............................: (0.12, 0.12) - optimizer-copy-to-main-grad ....................: (0.11, 0.11) - optimizer-clip-main-grad .......................: (0.43, 0.43) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.13, 1.13) - optimizer-copy-main-to-model-params ............: (0.19, 0.19) - optimizer ......................................: (2.55, 2.55) -Dataloader batch - tokens shape: torch.Size([1, 1862]) -Dataloader batch - labels shape: torch.Size([1, 1862]) -Dataloader batch - attn_mask shape: torch.Size([1, 1862]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 481 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1862) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1862) torch.int64 - loss_mask : 527 / 1862 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1862, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1862, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1862, 1, 576) torch.bfloat16 - out : (1, 1862) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 527 tokens) : 11.3271 - top-1 accuracy : 4/527 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 482 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 462 / 1436 tokens (32.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 462 tokens) : 11.4364 - top-1 accuracy : 2/462 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1766]) -Dataloader batch - labels shape: torch.Size([1, 1766]) -Dataloader batch - attn_mask shape: torch.Size([1, 1766]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 483 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1766) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1766) torch.int64 - loss_mask : 405 / 1766 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1766, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1766, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1766, 1, 576) torch.bfloat16 - out : (1, 1766) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.2233 - top-1 accuracy : 16/405 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 484 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 314 / 1388 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.2069 - top-1 accuracy : 15/314 = 4.8% - - [2026-04-29 13:47:13] iteration 221/ 500 | consumed samples: 884 | elapsed time per iteration (ms): 4984.5 | throughput (token/sec/GPU): 1294.4 | learning rate: 8.653271E-05 | global batch size: 4 | lm loss: 1.130995E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1085]) -Dataloader batch - labels shape: torch.Size([1, 1085]) -Dataloader batch - attn_mask shape: torch.Size([1, 1085]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 485 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1085) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1085) torch.int64 - loss_mask : 238 / 1085 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1085, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1085, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1085, 1, 576) torch.bfloat16 - out : (1, 1085) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 238 tokens) : 11.2145 - top-1 accuracy : 2/238 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1706]) -Dataloader batch - labels shape: torch.Size([1, 1706]) -Dataloader batch - attn_mask shape: torch.Size([1, 1706]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 486 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1706) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1706) torch.int64 - loss_mask : 347 / 1706 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1706, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1706, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1706, 1, 576) torch.bfloat16 - out : (1, 1706) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2175 - top-1 accuracy : 32/347 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 487 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 304 / 1388 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 304 tokens) : 11.2348 - top-1 accuracy : 18/304 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 918]) -Dataloader batch - labels shape: torch.Size([1, 918]) -Dataloader batch - attn_mask shape: torch.Size([1, 918]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 488 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 918) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 918) torch.int64 - loss_mask : 194 / 918 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (918, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (918, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (918, 1, 576) torch.bfloat16 - out : (1, 918) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 194 tokens) : 11.1938 - top-1 accuracy : 7/194 = 3.6% - - [2026-04-29 13:47:17] iteration 222/ 500 | consumed samples: 888 | elapsed time per iteration (ms): 4337.4 | throughput (token/sec/GPU): 1175.1 | learning rate: 8.631832E-05 | global batch size: 4 | lm loss: 1.121742E+01 | loss scale: 1.0 | grad norm: 0.314 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1274]) -Dataloader batch - labels shape: torch.Size([1, 1274]) -Dataloader batch - attn_mask shape: torch.Size([1, 1274]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 489 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1274) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1274) torch.int64 - loss_mask : 377 / 1274 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1274, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1274, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1274, 1, 576) torch.bfloat16 - out : (1, 1274) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.4184 - top-1 accuracy : 7/377 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1521]) -Dataloader batch - labels shape: torch.Size([1, 1521]) -Dataloader batch - attn_mask shape: torch.Size([1, 1521]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 490 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1521) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1521) torch.int64 - loss_mask : 365 / 1521 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1521, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1521, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1521, 1, 576) torch.bfloat16 - out : (1, 1521) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2738 - top-1 accuracy : 6/365 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1221]) -Dataloader batch - labels shape: torch.Size([1, 1221]) -Dataloader batch - attn_mask shape: torch.Size([1, 1221]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 491 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1221) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1221) torch.int64 - loss_mask : 293 / 1221 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1221, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1221, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1221, 1, 576) torch.bfloat16 - out : (1, 1221) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 293 tokens) : 11.2267 - top-1 accuracy : 6/293 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1233]) -Dataloader batch - labels shape: torch.Size([1, 1233]) -Dataloader batch - attn_mask shape: torch.Size([1, 1233]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 492 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1233) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1233) torch.int64 - loss_mask : 346 / 1233 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1233, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1233, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1233, 1, 576) torch.bfloat16 - out : (1, 1233) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.3293 - top-1 accuracy : 1/346 = 0.3% - - [2026-04-29 13:47:21] iteration 223/ 500 | consumed samples: 892 | elapsed time per iteration (ms): 4093.2 | throughput (token/sec/GPU): 1282.4 | learning rate: 8.610251E-05 | global batch size: 4 | lm loss: 1.131719E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1283]) -Dataloader batch - labels shape: torch.Size([1, 1283]) -Dataloader batch - attn_mask shape: torch.Size([1, 1283]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 493 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1283) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1283) torch.int64 - loss_mask : 392 / 1283 tokens (30.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1283, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1283, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1283, 1, 576) torch.bfloat16 - out : (1, 1283) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3674 - top-1 accuracy : 4/392 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 494 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 311 / 1492 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2056 - top-1 accuracy : 23/311 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1931]) -Dataloader batch - labels shape: torch.Size([1, 1931]) -Dataloader batch - attn_mask shape: torch.Size([1, 1931]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 495 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1931) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1931) torch.int64 - loss_mask : 561 / 1931 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1931, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1931, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1931, 1, 576) torch.bfloat16 - out : (1, 1931) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 561 tokens) : 11.4010 - top-1 accuracy : 14/561 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1285]) -Dataloader batch - labels shape: torch.Size([1, 1285]) -Dataloader batch - attn_mask shape: torch.Size([1, 1285]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 496 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1285) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1285) torch.int64 - loss_mask : 387 / 1285 tokens (30.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1285, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1285, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1285, 1, 576) torch.bfloat16 - out : (1, 1285) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.4330 - top-1 accuracy : 10/387 = 2.6% - - [2026-04-29 13:47:26] iteration 224/ 500 | consumed samples: 896 | elapsed time per iteration (ms): 4491.8 | throughput (token/sec/GPU): 1333.8 | learning rate: 8.588529E-05 | global batch size: 4 | lm loss: 1.136372E+01 | loss scale: 1.0 | grad norm: 0.311 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 497 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 363 / 1504 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.3138 - top-1 accuracy : 19/363 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1547]) -Dataloader batch - labels shape: torch.Size([1, 1547]) -Dataloader batch - attn_mask shape: torch.Size([1, 1547]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 498 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1547) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1547) torch.int64 - loss_mask : 426 / 1547 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1547, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1547, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1547, 1, 576) torch.bfloat16 - out : (1, 1547) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.3493 - top-1 accuracy : 19/426 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1491]) -Dataloader batch - labels shape: torch.Size([1, 1491]) -Dataloader batch - attn_mask shape: torch.Size([1, 1491]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 499 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1491) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1491) torch.int64 - loss_mask : 367 / 1491 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1491, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1491, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1491, 1, 576) torch.bfloat16 - out : (1, 1491) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2209 - top-1 accuracy : 9/367 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 500 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1423) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1423) torch.int64 - loss_mask : 273 / 1423 tokens (19.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1423, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1423, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1423, 1, 576) torch.bfloat16 - out : (1, 1423) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 273 tokens) : 11.0597 - top-1 accuracy : 10/273 = 3.7% - - [2026-04-29 13:47:31] iteration 225/ 500 | consumed samples: 900 | elapsed time per iteration (ms): 4760.3 | throughput (token/sec/GPU): 1253.1 | learning rate: 8.566667E-05 | global batch size: 4 | lm loss: 1.125199E+01 | loss scale: 1.0 | grad norm: 0.291 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1794]) -Dataloader batch - labels shape: torch.Size([1, 1794]) -Dataloader batch - attn_mask shape: torch.Size([1, 1794]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 501 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1794) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1794) torch.int64 - loss_mask : 411 / 1794 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1794, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1794, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1794, 1, 576) torch.bfloat16 - out : (1, 1794) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.2474 - top-1 accuracy : 4/411 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1362]) -Dataloader batch - labels shape: torch.Size([1, 1362]) -Dataloader batch - attn_mask shape: torch.Size([1, 1362]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 502 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1362) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1362) torch.int64 - loss_mask : 404 / 1362 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1362, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1362, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1362, 1, 576) torch.bfloat16 - out : (1, 1362) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.3835 - top-1 accuracy : 10/404 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1151]) -Dataloader batch - labels shape: torch.Size([1, 1151]) -Dataloader batch - attn_mask shape: torch.Size([1, 1151]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 503 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1151) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1151) torch.int64 - loss_mask : 306 / 1151 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1151, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1151, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1151, 1, 576) torch.bfloat16 - out : (1, 1151) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.3540 - top-1 accuracy : 3/306 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1780]) -Dataloader batch - labels shape: torch.Size([1, 1780]) -Dataloader batch - attn_mask shape: torch.Size([1, 1780]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 504 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1780) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1780) torch.int64 - loss_mask : 397 / 1780 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1780, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1780, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1780, 1, 576) torch.bfloat16 - out : (1, 1780) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.2061 - top-1 accuracy : 19/397 = 4.8% - - [2026-04-29 13:47:35] iteration 226/ 500 | consumed samples: 904 | elapsed time per iteration (ms): 4715.1 | throughput (token/sec/GPU): 1291.0 | learning rate: 8.544665E-05 | global batch size: 4 | lm loss: 1.129431E+01 | loss scale: 1.0 | grad norm: 0.260 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 945]) -Dataloader batch - labels shape: torch.Size([1, 945]) -Dataloader batch - attn_mask shape: torch.Size([1, 945]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 505 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 945) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 945) torch.int64 - loss_mask : 190 / 945 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (945, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (945, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (945, 1, 576) torch.bfloat16 - out : (1, 945) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 190 tokens) : 11.1338 - top-1 accuracy : 8/190 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1524]) -Dataloader batch - labels shape: torch.Size([1, 1524]) -Dataloader batch - attn_mask shape: torch.Size([1, 1524]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 506 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1524) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1524) torch.int64 - loss_mask : 371 / 1524 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1524, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1524, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1524, 1, 576) torch.bfloat16 - out : (1, 1524) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.2563 - top-1 accuracy : 19/371 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1512]) -Dataloader batch - labels shape: torch.Size([1, 1512]) -Dataloader batch - attn_mask shape: torch.Size([1, 1512]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 507 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1512) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1512) torch.int64 - loss_mask : 383 / 1512 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1512, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1512, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1512, 1, 576) torch.bfloat16 - out : (1, 1512) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.3320 - top-1 accuracy : 32/383 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 1782]) -Dataloader batch - labels shape: torch.Size([1, 1782]) -Dataloader batch - attn_mask shape: torch.Size([1, 1782]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 508 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1782) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1782) torch.int64 - loss_mask : 420 / 1782 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1782, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1782, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1782, 1, 576) torch.bfloat16 - out : (1, 1782) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 420 tokens) : 11.2179 - top-1 accuracy : 0/420 = 0.0% - - [2026-04-29 13:47:40] iteration 227/ 500 | consumed samples: 908 | elapsed time per iteration (ms): 4628.4 | throughput (token/sec/GPU): 1245.1 | learning rate: 8.522525E-05 | global batch size: 4 | lm loss: 1.124868E+01 | loss scale: 1.0 | grad norm: 0.344 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1031]) -Dataloader batch - labels shape: torch.Size([1, 1031]) -Dataloader batch - attn_mask shape: torch.Size([1, 1031]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 509 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1031) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1031) torch.int64 - loss_mask : 264 / 1031 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1031, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1031, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1031, 1, 576) torch.bfloat16 - out : (1, 1031) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 264 tokens) : 11.2990 - top-1 accuracy : 1/264 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1766]) -Dataloader batch - labels shape: torch.Size([1, 1766]) -Dataloader batch - attn_mask shape: torch.Size([1, 1766]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 510 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1766) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1766) torch.int64 - loss_mask : 397 / 1766 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1766, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1766, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1766, 1, 576) torch.bfloat16 - out : (1, 1766) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.2319 - top-1 accuracy : 20/397 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1509]) -Dataloader batch - labels shape: torch.Size([1, 1509]) -Dataloader batch - attn_mask shape: torch.Size([1, 1509]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 511 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1509) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1509) torch.int64 - loss_mask : 363 / 1509 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1509, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1509, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1509, 1, 576) torch.bfloat16 - out : (1, 1509) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2742 - top-1 accuracy : 11/363 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1573]) -Dataloader batch - labels shape: torch.Size([1, 1573]) -Dataloader batch - attn_mask shape: torch.Size([1, 1573]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 512 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1573) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1573) torch.int64 - loss_mask : 392 / 1573 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1573, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1573, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1573, 1, 576) torch.bfloat16 - out : (1, 1573) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.2872 - top-1 accuracy : 17/392 = 4.3% - - [2026-04-29 13:47:45] iteration 228/ 500 | consumed samples: 912 | elapsed time per iteration (ms): 4600.3 | throughput (token/sec/GPU): 1277.9 | learning rate: 8.500247E-05 | global batch size: 4 | lm loss: 1.127060E+01 | loss scale: 1.0 | grad norm: 0.280 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1479]) -Dataloader batch - labels shape: torch.Size([1, 1479]) -Dataloader batch - attn_mask shape: torch.Size([1, 1479]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 513 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1479) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1479) torch.int64 - loss_mask : 441 / 1479 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1479, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1479, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1479, 1, 576) torch.bfloat16 - out : (1, 1479) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 441 tokens) : 11.3359 - top-1 accuracy : 5/441 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1680]) -Dataloader batch - labels shape: torch.Size([1, 1680]) -Dataloader batch - attn_mask shape: torch.Size([1, 1680]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 514 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1680) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1680) torch.int64 - loss_mask : 405 / 1680 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1680, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1680, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1680, 1, 576) torch.bfloat16 - out : (1, 1680) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.2861 - top-1 accuracy : 6/405 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1453]) -Dataloader batch - labels shape: torch.Size([1, 1453]) -Dataloader batch - attn_mask shape: torch.Size([1, 1453]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 515 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1453) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1453) torch.int64 - loss_mask : 393 / 1453 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1453, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1453, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1453, 1, 576) torch.bfloat16 - out : (1, 1453) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 393 tokens) : 11.3628 - top-1 accuracy : 6/393 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 946]) -Dataloader batch - labels shape: torch.Size([1, 946]) -Dataloader batch - attn_mask shape: torch.Size([1, 946]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 516 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 946) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 946) torch.int64 - loss_mask : 201 / 946 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (946, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (946, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (946, 1, 576) torch.bfloat16 - out : (1, 946) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 201 tokens) : 11.1687 - top-1 accuracy : 10/201 = 5.0% - - [2026-04-29 13:47:49] iteration 229/ 500 | consumed samples: 916 | elapsed time per iteration (ms): 4512.5 | throughput (token/sec/GPU): 1231.7 | learning rate: 8.477833E-05 | global batch size: 4 | lm loss: 1.130591E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1373]) -Dataloader batch - labels shape: torch.Size([1, 1373]) -Dataloader batch - attn_mask shape: torch.Size([1, 1373]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 517 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1373) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1373) torch.int64 - loss_mask : 331 / 1373 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1373, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1373, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1373, 1, 576) torch.bfloat16 - out : (1, 1373) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.2309 - top-1 accuracy : 9/331 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 518 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 341 / 1518 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.2506 - top-1 accuracy : 8/341 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1435]) -Dataloader batch - labels shape: torch.Size([1, 1435]) -Dataloader batch - attn_mask shape: torch.Size([1, 1435]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 519 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1435) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1435) torch.int64 - loss_mask : 344 / 1435 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1435, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1435, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1435, 1, 576) torch.bfloat16 - out : (1, 1435) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.2845 - top-1 accuracy : 17/344 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1855]) -Dataloader batch - labels shape: torch.Size([1, 1855]) -Dataloader batch - attn_mask shape: torch.Size([1, 1855]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 520 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1855) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1855) torch.int64 - loss_mask : 503 / 1855 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1855, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1855, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1855, 1, 576) torch.bfloat16 - out : (1, 1855) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 503 tokens) : 11.3650 - top-1 accuracy : 13/503 = 2.6% - - [2026-04-29 13:47:54] iteration 230/ 500 | consumed samples: 920 | elapsed time per iteration (ms): 4859.4 | throughput (token/sec/GPU): 1272.0 | learning rate: 8.455283E-05 | global batch size: 4 | lm loss: 1.129185E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1563]) -Dataloader batch - labels shape: torch.Size([1, 1563]) -Dataloader batch - attn_mask shape: torch.Size([1, 1563]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 521 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1563) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1563) torch.int64 - loss_mask : 368 / 1563 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1563, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1563, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1563, 1, 576) torch.bfloat16 - out : (1, 1563) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2350 - top-1 accuracy : 21/368 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 962]) -Dataloader batch - labels shape: torch.Size([1, 962]) -Dataloader batch - attn_mask shape: torch.Size([1, 962]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 522 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 962) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 962) torch.int64 - loss_mask : 205 / 962 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (962, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (962, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (962, 1, 576) torch.bfloat16 - out : (1, 962) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 205 tokens) : 11.1561 - top-1 accuracy : 5/205 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1138]) -Dataloader batch - labels shape: torch.Size([1, 1138]) -Dataloader batch - attn_mask shape: torch.Size([1, 1138]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 523 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1138) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1138) torch.int64 - loss_mask : 322 / 1138 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1138, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1138, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1138, 1, 576) torch.bfloat16 - out : (1, 1138) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.3748 - top-1 accuracy : 3/322 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1398]) -Dataloader batch - labels shape: torch.Size([1, 1398]) -Dataloader batch - attn_mask shape: torch.Size([1, 1398]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 524 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1398) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1398) torch.int64 - loss_mask : 335 / 1398 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1398, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1398, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1398, 1, 576) torch.bfloat16 - out : (1, 1398) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2448 - top-1 accuracy : 5/335 = 1.5% - - [2026-04-29 13:47:58] iteration 231/ 500 | consumed samples: 924 | elapsed time per iteration (ms): 4077.6 | throughput (token/sec/GPU): 1241.2 | learning rate: 8.432597E-05 | global batch size: 4 | lm loss: 1.126110E+01 | loss scale: 1.0 | grad norm: 0.261 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1931]) -Dataloader batch - labels shape: torch.Size([1, 1931]) -Dataloader batch - attn_mask shape: torch.Size([1, 1931]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 525 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1931) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1931) torch.int64 - loss_mask : 550 / 1931 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1931, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1931, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1931, 1, 576) torch.bfloat16 - out : (1, 1931) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 550 tokens) : 11.3169 - top-1 accuracy : 3/550 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 1527]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 526 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1527) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1527) torch.int64 - loss_mask : 421 / 1527 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1527, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1527, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1527, 1, 576) torch.bfloat16 - out : (1, 1527) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 421 tokens) : 11.4007 - top-1 accuracy : 1/421 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1092]) -Dataloader batch - labels shape: torch.Size([1, 1092]) -Dataloader batch - attn_mask shape: torch.Size([1, 1092]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 527 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1092) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1092) torch.int64 - loss_mask : 263 / 1092 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1092, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1092, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1092, 1, 576) torch.bfloat16 - out : (1, 1092) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.1970 - top-1 accuracy : 7/263 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1817]) -Dataloader batch - labels shape: torch.Size([1, 1817]) -Dataloader batch - attn_mask shape: torch.Size([1, 1817]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 528 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1817) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1817) torch.int64 - loss_mask : 451 / 1817 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1817, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1817, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1817, 1, 576) torch.bfloat16 - out : (1, 1817) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 451 tokens) : 11.2696 - top-1 accuracy : 6/451 = 1.3% - - [2026-04-29 13:48:03] iteration 232/ 500 | consumed samples: 928 | elapsed time per iteration (ms): 4713.3 | throughput (token/sec/GPU): 1350.8 | learning rate: 8.409777E-05 | global batch size: 4 | lm loss: 1.130646E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 529 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 391 / 1541 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3296 - top-1 accuracy : 26/391 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1488]) -Dataloader batch - labels shape: torch.Size([1, 1488]) -Dataloader batch - attn_mask shape: torch.Size([1, 1488]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 530 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1488) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1488) torch.int64 - loss_mask : 431 / 1488 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1488, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1488, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1488, 1, 576) torch.bfloat16 - out : (1, 1488) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.4230 - top-1 accuracy : 15/431 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1580]) -Dataloader batch - labels shape: torch.Size([1, 1580]) -Dataloader batch - attn_mask shape: torch.Size([1, 1580]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 531 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1580) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1580) torch.int64 - loss_mask : 365 / 1580 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1580, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1580, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1580, 1, 576) torch.bfloat16 - out : (1, 1580) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.1810 - top-1 accuracy : 5/365 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1742]) -Dataloader batch - labels shape: torch.Size([1, 1742]) -Dataloader batch - attn_mask shape: torch.Size([1, 1742]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 532 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1742) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1742) torch.int64 - loss_mask : 403 / 1742 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1742, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1742, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1742, 1, 576) torch.bfloat16 - out : (1, 1742) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.2238 - top-1 accuracy : 33/403 = 8.2% - - [2026-04-29 13:48:08] iteration 233/ 500 | consumed samples: 932 | elapsed time per iteration (ms): 5031.5 | throughput (token/sec/GPU): 1262.2 | learning rate: 8.386825E-05 | global batch size: 4 | lm loss: 1.129397E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1074]) -Dataloader batch - labels shape: torch.Size([1, 1074]) -Dataloader batch - attn_mask shape: torch.Size([1, 1074]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 533 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1074) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1074) torch.int64 - loss_mask : 253 / 1074 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1074, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1074, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1074, 1, 576) torch.bfloat16 - out : (1, 1074) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 253 tokens) : 11.2783 - top-1 accuracy : 3/253 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1808]) -Dataloader batch - labels shape: torch.Size([1, 1808]) -Dataloader batch - attn_mask shape: torch.Size([1, 1808]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 534 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1808) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1808) torch.int64 - loss_mask : 432 / 1808 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1808, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1808, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1808, 1, 576) torch.bfloat16 - out : (1, 1808) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 432 tokens) : 11.2805 - top-1 accuracy : 6/432 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1393]) -Dataloader batch - labels shape: torch.Size([1, 1393]) -Dataloader batch - attn_mask shape: torch.Size([1, 1393]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 535 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1393) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1393) torch.int64 - loss_mask : 329 / 1393 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1393, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1393, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1393, 1, 576) torch.bfloat16 - out : (1, 1393) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2249 - top-1 accuracy : 17/329 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1881]) -Dataloader batch - labels shape: torch.Size([1, 1881]) -Dataloader batch - attn_mask shape: torch.Size([1, 1881]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 536 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1881) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1881) torch.int64 - loss_mask : 542 / 1881 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1881, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1881, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1881, 1, 576) torch.bfloat16 - out : (1, 1881) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 542 tokens) : 11.3620 - top-1 accuracy : 18/542 = 3.3% - - [2026-04-29 13:48:12] iteration 234/ 500 | consumed samples: 936 | elapsed time per iteration (ms): 4799.4 | throughput (token/sec/GPU): 1282.7 | learning rate: 8.363740E-05 | global batch size: 4 | lm loss: 1.129676E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1408]) -Dataloader batch - labels shape: torch.Size([1, 1408]) -Dataloader batch - attn_mask shape: torch.Size([1, 1408]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 537 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1408) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1408) torch.int64 - loss_mask : 365 / 1408 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1408, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1408, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1408, 1, 576) torch.bfloat16 - out : (1, 1408) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2807 - top-1 accuracy : 9/365 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 538 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 356 / 1460 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2424 - top-1 accuracy : 15/356 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1669]) -Dataloader batch - labels shape: torch.Size([1, 1669]) -Dataloader batch - attn_mask shape: torch.Size([1, 1669]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 539 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1669) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1669) torch.int64 - loss_mask : 403 / 1669 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1669, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1669, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1669, 1, 576) torch.bfloat16 - out : (1, 1669) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.2617 - top-1 accuracy : 18/403 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1886]) -Dataloader batch - labels shape: torch.Size([1, 1886]) -Dataloader batch - attn_mask shape: torch.Size([1, 1886]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 540 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1886) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1886) torch.int64 - loss_mask : 532 / 1886 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1886, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1886, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1886, 1, 576) torch.bfloat16 - out : (1, 1886) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 532 tokens) : 11.3356 - top-1 accuracy : 24/532 = 4.5% - - [2026-04-29 13:48:17] iteration 235/ 500 | consumed samples: 940 | elapsed time per iteration (ms): 4841.4 | throughput (token/sec/GPU): 1326.7 | learning rate: 8.340524E-05 | global batch size: 4 | lm loss: 1.128547E+01 | loss scale: 1.0 | grad norm: 0.275 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1401]) -Dataloader batch - labels shape: torch.Size([1, 1401]) -Dataloader batch - attn_mask shape: torch.Size([1, 1401]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 541 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1401) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1401) torch.int64 - loss_mask : 349 / 1401 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1401, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1401, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1401, 1, 576) torch.bfloat16 - out : (1, 1401) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2817 - top-1 accuracy : 22/349 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1503]) -Dataloader batch - labels shape: torch.Size([1, 1503]) -Dataloader batch - attn_mask shape: torch.Size([1, 1503]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 542 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1503) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1503) torch.int64 - loss_mask : 358 / 1503 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1503, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1503, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1503, 1, 576) torch.bfloat16 - out : (1, 1503) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2394 - top-1 accuracy : 15/358 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1415]) -Dataloader batch - labels shape: torch.Size([1, 1415]) -Dataloader batch - attn_mask shape: torch.Size([1, 1415]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 543 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1415) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1415) torch.int64 - loss_mask : 354 / 1415 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1415, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1415, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1415, 1, 576) torch.bfloat16 - out : (1, 1415) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2997 - top-1 accuracy : 10/354 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1175]) -Dataloader batch - labels shape: torch.Size([1, 1175]) -Dataloader batch - attn_mask shape: torch.Size([1, 1175]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 544 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1175) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1175) torch.int64 - loss_mask : 271 / 1175 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1175, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1175, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1175, 1, 576) torch.bfloat16 - out : (1, 1175) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 271 tokens) : 11.2387 - top-1 accuracy : 2/271 = 0.7% - - [2026-04-29 13:48:22] iteration 236/ 500 | consumed samples: 944 | elapsed time per iteration (ms): 4469.5 | throughput (token/sec/GPU): 1229.2 | learning rate: 8.317177E-05 | global batch size: 4 | lm loss: 1.126637E+01 | loss scale: 1.0 | grad norm: 0.273 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1653]) -Dataloader batch - labels shape: torch.Size([1, 1653]) -Dataloader batch - attn_mask shape: torch.Size([1, 1653]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 545 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1653) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1653) torch.int64 - loss_mask : 381 / 1653 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1653, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1653, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1653, 1, 576) torch.bfloat16 - out : (1, 1653) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.2768 - top-1 accuracy : 12/381 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1067]) -Dataloader batch - labels shape: torch.Size([1, 1067]) -Dataloader batch - attn_mask shape: torch.Size([1, 1067]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 546 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1067) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1067) torch.int64 - loss_mask : 241 / 1067 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1067, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1067, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1067, 1, 576) torch.bfloat16 - out : (1, 1067) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 241 tokens) : 11.2334 - top-1 accuracy : 4/241 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 547 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 328 / 1487 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.1875 - top-1 accuracy : 36/328 = 11.0% - -Dataloader batch - tokens shape: torch.Size([1, 1413]) -Dataloader batch - labels shape: torch.Size([1, 1413]) -Dataloader batch - attn_mask shape: torch.Size([1, 1413]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 548 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1413) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1413) torch.int64 - loss_mask : 342 / 1413 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1413, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1413, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1413, 1, 576) torch.bfloat16 - out : (1, 1413) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2416 - top-1 accuracy : 2/342 = 0.6% - - [2026-04-29 13:48:26] iteration 237/ 500 | consumed samples: 948 | elapsed time per iteration (ms): 4479.3 | throughput (token/sec/GPU): 1254.7 | learning rate: 8.293701E-05 | global batch size: 4 | lm loss: 1.123670E+01 | loss scale: 1.0 | grad norm: 0.304 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 549 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 331 / 1400 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.2436 - top-1 accuracy : 5/331 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1077]) -Dataloader batch - labels shape: torch.Size([1, 1077]) -Dataloader batch - attn_mask shape: torch.Size([1, 1077]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 550 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1077) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1077) torch.int64 - loss_mask : 233 / 1077 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1077, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1077, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1077, 1, 576) torch.bfloat16 - out : (1, 1077) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 233 tokens) : 11.2093 - top-1 accuracy : 24/233 = 10.3% - -Dataloader batch - tokens shape: torch.Size([1, 1512]) -Dataloader batch - labels shape: torch.Size([1, 1512]) -Dataloader batch - attn_mask shape: torch.Size([1, 1512]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 551 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1512) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1512) torch.int64 - loss_mask : 408 / 1512 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1512, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1512, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1512, 1, 576) torch.bfloat16 - out : (1, 1512) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3340 - top-1 accuracy : 4/408 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 969]) -Dataloader batch - labels shape: torch.Size([1, 969]) -Dataloader batch - attn_mask shape: torch.Size([1, 969]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 552 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 969) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 969) torch.int64 - loss_mask : 228 / 969 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (969, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (969, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (969, 1, 576) torch.bfloat16 - out : (1, 969) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 228 tokens) : 11.2370 - top-1 accuracy : 19/228 = 8.3% - - [2026-04-29 13:48:30] iteration 238/ 500 | consumed samples: 952 | elapsed time per iteration (ms): 4122.9 | throughput (token/sec/GPU): 1202.6 | learning rate: 8.270096E-05 | global batch size: 4 | lm loss: 1.126643E+01 | loss scale: 1.0 | grad norm: 0.276 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 553 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 328 / 1388 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.2827 - top-1 accuracy : 13/328 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1594]) -Dataloader batch - labels shape: torch.Size([1, 1594]) -Dataloader batch - attn_mask shape: torch.Size([1, 1594]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 554 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1594) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1594) torch.int64 - loss_mask : 360 / 1594 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1594, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1594, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1594, 1, 576) torch.bfloat16 - out : (1, 1594) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.1865 - top-1 accuracy : 18/360 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1707]) -Dataloader batch - labels shape: torch.Size([1, 1707]) -Dataloader batch - attn_mask shape: torch.Size([1, 1707]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 555 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1707) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1707) torch.int64 - loss_mask : 365 / 1707 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1707, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1707, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1707, 1, 576) torch.bfloat16 - out : (1, 1707) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.1523 - top-1 accuracy : 24/365 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1493]) -Dataloader batch - labels shape: torch.Size([1, 1493]) -Dataloader batch - attn_mask shape: torch.Size([1, 1493]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 556 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1493) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1493) torch.int64 - loss_mask : 341 / 1493 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1493, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1493, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1493, 1, 576) torch.bfloat16 - out : (1, 1493) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.2345 - top-1 accuracy : 21/341 = 6.2% - - [2026-04-29 13:48:35] iteration 239/ 500 | consumed samples: 956 | elapsed time per iteration (ms): 4922.0 | throughput (token/sec/GPU): 1256.0 | learning rate: 8.246364E-05 | global batch size: 4 | lm loss: 1.121191E+01 | loss scale: 1.0 | grad norm: 0.250 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1481]) -Dataloader batch - labels shape: torch.Size([1, 1481]) -Dataloader batch - attn_mask shape: torch.Size([1, 1481]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 557 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1481) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1481) torch.int64 - loss_mask : 296 / 1481 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1481, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1481, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1481, 1, 576) torch.bfloat16 - out : (1, 1481) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 296 tokens) : 11.1696 - top-1 accuracy : 30/296 = 10.1% - -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 558 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1385) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1385) torch.int64 - loss_mask : 327 / 1385 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1385, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1385, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1385, 1, 576) torch.bfloat16 - out : (1, 1385) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2907 - top-1 accuracy : 10/327 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1394]) -Dataloader batch - labels shape: torch.Size([1, 1394]) -Dataloader batch - attn_mask shape: torch.Size([1, 1394]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 559 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1394) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1394) torch.int64 - loss_mask : 332 / 1394 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1394, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1394, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1394, 1, 576) torch.bfloat16 - out : (1, 1394) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.2747 - top-1 accuracy : 14/332 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 980]) -Dataloader batch - labels shape: torch.Size([1, 980]) -Dataloader batch - attn_mask shape: torch.Size([1, 980]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 560 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 980) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 980) torch.int64 - loss_mask : 236 / 980 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (980, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (980, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (980, 1, 576) torch.bfloat16 - out : (1, 980) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 236 tokens) : 11.2374 - top-1 accuracy : 18/236 = 7.6% - - [2026-04-29 13:48:40] iteration 240/ 500 | consumed samples: 960 | elapsed time per iteration (ms): 4397.8 | throughput (token/sec/GPU): 1191.5 | learning rate: 8.222505E-05 | global batch size: 4 | lm loss: 1.124556E+01 | loss scale: 1.0 | grad norm: 0.286 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4370.82, 4370.82) - forward-compute ................................: (3238.27, 3238.27) - backward-compute ...............................: (1128.51, 1128.51) - batch-generator ................................: (419.38, 419.38) - layernorm-grads-all-reduce .....................: (0.07, 0.07) - embedding-grads-all-reduce .....................: (0.11, 0.11) - all-grads-sync .................................: (1.09, 1.09) - params-all-gather ..............................: (0.41, 0.41) - optimizer-copy-to-main-grad ....................: (0.34, 0.34) - optimizer-clip-main-grad .......................: (1.35, 1.35) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.21, 2.21) - optimizer-copy-main-to-model-params ............: (0.58, 0.58) - optimizer ......................................: (6.95, 6.95) -Dataloader batch - tokens shape: torch.Size([1, 1958]) -Dataloader batch - labels shape: torch.Size([1, 1958]) -Dataloader batch - attn_mask shape: torch.Size([1, 1958]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 561 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1958) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1958) torch.int64 - loss_mask : 576 / 1958 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1958, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1958, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1958, 1, 576) torch.bfloat16 - out : (1, 1958) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 576 tokens) : 11.3796 - top-1 accuracy : 7/576 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1545]) -Dataloader batch - labels shape: torch.Size([1, 1545]) -Dataloader batch - attn_mask shape: torch.Size([1, 1545]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 562 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1545) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1545) torch.int64 - loss_mask : 400 / 1545 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1545, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1545, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1545, 1, 576) torch.bfloat16 - out : (1, 1545) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.2796 - top-1 accuracy : 11/400 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1536]) -Dataloader batch - labels shape: torch.Size([1, 1536]) -Dataloader batch - attn_mask shape: torch.Size([1, 1536]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 563 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1536) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1536) torch.int64 - loss_mask : 386 / 1536 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1536, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1536, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1536, 1, 576) torch.bfloat16 - out : (1, 1536) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3054 - top-1 accuracy : 7/386 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1511]) -Dataloader batch - labels shape: torch.Size([1, 1511]) -Dataloader batch - attn_mask shape: torch.Size([1, 1511]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 564 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1511) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1511) torch.int64 - loss_mask : 347 / 1511 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1511, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1511, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1511, 1, 576) torch.bfloat16 - out : (1, 1511) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2341 - top-1 accuracy : 11/347 = 3.2% - - [2026-04-29 13:48:45] iteration 241/ 500 | consumed samples: 964 | elapsed time per iteration (ms): 5063.1 | throughput (token/sec/GPU): 1293.7 | learning rate: 8.198521E-05 | global batch size: 4 | lm loss: 1.130989E+01 | loss scale: 1.0 | grad norm: 0.259 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1047]) -Dataloader batch - labels shape: torch.Size([1, 1047]) -Dataloader batch - attn_mask shape: torch.Size([1, 1047]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 565 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1047) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1047) torch.int64 - loss_mask : 295 / 1047 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1047, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1047, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1047, 1, 576) torch.bfloat16 - out : (1, 1047) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.3607 - top-1 accuracy : 5/295 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1425]) -Dataloader batch - labels shape: torch.Size([1, 1425]) -Dataloader batch - attn_mask shape: torch.Size([1, 1425]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 566 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1425) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1425) torch.int64 - loss_mask : 365 / 1425 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1425, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1425, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1425, 1, 576) torch.bfloat16 - out : (1, 1425) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2794 - top-1 accuracy : 16/365 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1015]) -Dataloader batch - labels shape: torch.Size([1, 1015]) -Dataloader batch - attn_mask shape: torch.Size([1, 1015]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 567 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1015) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1015) torch.int64 - loss_mask : 234 / 1015 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1015, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1015, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1015, 1, 576) torch.bfloat16 - out : (1, 1015) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 234 tokens) : 11.2312 - top-1 accuracy : 6/234 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1495]) -Dataloader batch - labels shape: torch.Size([1, 1495]) -Dataloader batch - attn_mask shape: torch.Size([1, 1495]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 568 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1495) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1495) torch.int64 - loss_mask : 316 / 1495 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1495, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1495, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1495, 1, 576) torch.bfloat16 - out : (1, 1495) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.1997 - top-1 accuracy : 32/316 = 10.1% - - [2026-04-29 13:48:49] iteration 242/ 500 | consumed samples: 968 | elapsed time per iteration (ms): 4115.2 | throughput (token/sec/GPU): 1210.6 | learning rate: 8.174411E-05 | global batch size: 4 | lm loss: 1.126909E+01 | loss scale: 1.0 | grad norm: 0.258 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 970]) -Dataloader batch - labels shape: torch.Size([1, 970]) -Dataloader batch - attn_mask shape: torch.Size([1, 970]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 569 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 970) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 970) torch.int64 - loss_mask : 237 / 970 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (970, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (970, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (970, 1, 576) torch.bfloat16 - out : (1, 970) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 237 tokens) : 11.2576 - top-1 accuracy : 7/237 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1512]) -Dataloader batch - labels shape: torch.Size([1, 1512]) -Dataloader batch - attn_mask shape: torch.Size([1, 1512]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 570 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1512) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1512) torch.int64 - loss_mask : 373 / 1512 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1512, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1512, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1512, 1, 576) torch.bfloat16 - out : (1, 1512) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.3169 - top-1 accuracy : 6/373 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1478]) -Dataloader batch - labels shape: torch.Size([1, 1478]) -Dataloader batch - attn_mask shape: torch.Size([1, 1478]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 571 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1478) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1478) torch.int64 - loss_mask : 327 / 1478 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1478, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1478, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1478, 1, 576) torch.bfloat16 - out : (1, 1478) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2127 - top-1 accuracy : 11/327 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1829]) -Dataloader batch - labels shape: torch.Size([1, 1829]) -Dataloader batch - attn_mask shape: torch.Size([1, 1829]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 572 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1829) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1829) torch.int64 - loss_mask : 451 / 1829 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1829, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1829, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1829, 1, 576) torch.bfloat16 - out : (1, 1829) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 451 tokens) : 11.2620 - top-1 accuracy : 4/451 = 0.9% - - [2026-04-29 13:48:54] iteration 243/ 500 | consumed samples: 972 | elapsed time per iteration (ms): 4617.9 | throughput (token/sec/GPU): 1253.6 | learning rate: 8.150178E-05 | global batch size: 4 | lm loss: 1.126440E+01 | loss scale: 1.0 | grad norm: 0.261 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1851]) -Dataloader batch - labels shape: torch.Size([1, 1851]) -Dataloader batch - attn_mask shape: torch.Size([1, 1851]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 573 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1851) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1851) torch.int64 - loss_mask : 501 / 1851 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1851, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1851, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1851, 1, 576) torch.bfloat16 - out : (1, 1851) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 501 tokens) : 11.3152 - top-1 accuracy : 8/501 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1051]) -Dataloader batch - labels shape: torch.Size([1, 1051]) -Dataloader batch - attn_mask shape: torch.Size([1, 1051]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 574 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1051) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1051) torch.int64 - loss_mask : 307 / 1051 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1051, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1051, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1051, 1, 576) torch.bfloat16 - out : (1, 1051) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.3630 - top-1 accuracy : 4/307 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1821]) -Dataloader batch - labels shape: torch.Size([1, 1821]) -Dataloader batch - attn_mask shape: torch.Size([1, 1821]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 575 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1821) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1821) torch.int64 - loss_mask : 456 / 1821 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1821, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1821, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1821, 1, 576) torch.bfloat16 - out : (1, 1821) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 456 tokens) : 11.3164 - top-1 accuracy : 21/456 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1698]) -Dataloader batch - labels shape: torch.Size([1, 1698]) -Dataloader batch - attn_mask shape: torch.Size([1, 1698]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 576 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1698) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1698) torch.int64 - loss_mask : 348 / 1698 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1698, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1698, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1698, 1, 576) torch.bfloat16 - out : (1, 1698) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2076 - top-1 accuracy : 27/348 = 7.8% - - [2026-04-29 13:48:58] iteration 244/ 500 | consumed samples: 976 | elapsed time per iteration (ms): 4939.9 | throughput (token/sec/GPU): 1299.8 | learning rate: 8.125821E-05 | global batch size: 4 | lm loss: 1.130139E+01 | loss scale: 1.0 | grad norm: 0.274 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 577 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 381 / 1475 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.3554 - top-1 accuracy : 10/381 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 578 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 374 / 1496 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.2579 - top-1 accuracy : 28/374 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1429]) -Dataloader batch - labels shape: torch.Size([1, 1429]) -Dataloader batch - attn_mask shape: torch.Size([1, 1429]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 579 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1429) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1429) torch.int64 - loss_mask : 339 / 1429 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1429, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1429, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1429, 1, 576) torch.bfloat16 - out : (1, 1429) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2925 - top-1 accuracy : 5/339 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1472]) -Dataloader batch - labels shape: torch.Size([1, 1472]) -Dataloader batch - attn_mask shape: torch.Size([1, 1472]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 580 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1472) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1472) torch.int64 - loss_mask : 310 / 1472 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1472, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1472, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1472, 1, 576) torch.bfloat16 - out : (1, 1472) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 310 tokens) : 11.1635 - top-1 accuracy : 31/310 = 10.0% - - [2026-04-29 13:49:03] iteration 245/ 500 | consumed samples: 980 | elapsed time per iteration (ms): 4919.4 | throughput (token/sec/GPU): 1193.6 | learning rate: 8.101343E-05 | global batch size: 4 | lm loss: 1.127187E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1751]) -Dataloader batch - labels shape: torch.Size([1, 1751]) -Dataloader batch - attn_mask shape: torch.Size([1, 1751]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 581 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1751) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1751) torch.int64 - loss_mask : 399 / 1751 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1751, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1751, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1751, 1, 576) torch.bfloat16 - out : (1, 1751) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.2240 - top-1 accuracy : 9/399 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1515]) -Dataloader batch - labels shape: torch.Size([1, 1515]) -Dataloader batch - attn_mask shape: torch.Size([1, 1515]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 582 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1515) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1515) torch.int64 - loss_mask : 366 / 1515 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1515, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1515, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1515, 1, 576) torch.bfloat16 - out : (1, 1515) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2448 - top-1 accuracy : 29/366 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 583 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 318 / 1422 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.2878 - top-1 accuracy : 24/318 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 584 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1467) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1467) torch.int64 - loss_mask : 314 / 1467 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1467, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1467, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1467, 1, 576) torch.bfloat16 - out : (1, 1467) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.2162 - top-1 accuracy : 12/314 = 3.8% - - [2026-04-29 13:49:08] iteration 246/ 500 | consumed samples: 984 | elapsed time per iteration (ms): 4951.4 | throughput (token/sec/GPU): 1243.1 | learning rate: 8.076744E-05 | global batch size: 4 | lm loss: 1.124220E+01 | loss scale: 1.0 | grad norm: 0.301 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1410]) -Dataloader batch - labels shape: torch.Size([1, 1410]) -Dataloader batch - attn_mask shape: torch.Size([1, 1410]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 585 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1410) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1410) torch.int64 - loss_mask : 340 / 1410 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1410, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1410, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1410, 1, 576) torch.bfloat16 - out : (1, 1410) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2743 - top-1 accuracy : 19/340 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1369]) -Dataloader batch - labels shape: torch.Size([1, 1369]) -Dataloader batch - attn_mask shape: torch.Size([1, 1369]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 586 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1369) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1369) torch.int64 - loss_mask : 293 / 1369 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1369, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1369, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1369, 1, 576) torch.bfloat16 - out : (1, 1369) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 293 tokens) : 11.1861 - top-1 accuracy : 9/293 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 587 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 297 / 1475 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.1892 - top-1 accuracy : 32/297 = 10.8% - -Dataloader batch - tokens shape: torch.Size([1, 1633]) -Dataloader batch - labels shape: torch.Size([1, 1633]) -Dataloader batch - attn_mask shape: torch.Size([1, 1633]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 588 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1633) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1633) torch.int64 - loss_mask : 389 / 1633 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1633, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1633, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1633, 1, 576) torch.bfloat16 - out : (1, 1633) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.2494 - top-1 accuracy : 10/389 = 2.6% - - [2026-04-29 13:49:13] iteration 247/ 500 | consumed samples: 988 | elapsed time per iteration (ms): 4856.3 | throughput (token/sec/GPU): 1212.2 | learning rate: 8.052024E-05 | global batch size: 4 | lm loss: 1.122821E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 589 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 343 / 1492 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.2952 - top-1 accuracy : 25/343 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1135]) -Dataloader batch - labels shape: torch.Size([1, 1135]) -Dataloader batch - attn_mask shape: torch.Size([1, 1135]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 590 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1135) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1135) torch.int64 - loss_mask : 339 / 1135 tokens (29.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1135, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1135, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1135, 1, 576) torch.bfloat16 - out : (1, 1135) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.4150 - top-1 accuracy : 15/339 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1884]) -Dataloader batch - labels shape: torch.Size([1, 1884]) -Dataloader batch - attn_mask shape: torch.Size([1, 1884]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 591 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1884) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1884) torch.int64 - loss_mask : 540 / 1884 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1884, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1884, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1884, 1, 576) torch.bfloat16 - out : (1, 1884) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 540 tokens) : 11.3763 - top-1 accuracy : 25/540 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 999]) -Dataloader batch - labels shape: torch.Size([1, 999]) -Dataloader batch - attn_mask shape: torch.Size([1, 999]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 592 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 999) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 999) torch.int64 - loss_mask : 234 / 999 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (999, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (999, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (999, 1, 576) torch.bfloat16 - out : (1, 999) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 234 tokens) : 11.2550 - top-1 accuracy : 18/234 = 7.7% - - [2026-04-29 13:49:18] iteration 248/ 500 | consumed samples: 992 | elapsed time per iteration (ms): 4306.4 | throughput (token/sec/GPU): 1279.5 | learning rate: 8.027186E-05 | global batch size: 4 | lm loss: 1.134673E+01 | loss scale: 1.0 | grad norm: 0.266 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 974]) -Dataloader batch - labels shape: torch.Size([1, 974]) -Dataloader batch - attn_mask shape: torch.Size([1, 974]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 593 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 974) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 974) torch.int64 - loss_mask : 224 / 974 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (974, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (974, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (974, 1, 576) torch.bfloat16 - out : (1, 974) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 224 tokens) : 11.2626 - top-1 accuracy : 5/224 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1562]) -Dataloader batch - labels shape: torch.Size([1, 1562]) -Dataloader batch - attn_mask shape: torch.Size([1, 1562]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 594 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1562) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1562) torch.int64 - loss_mask : 423 / 1562 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1562, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1562, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1562, 1, 576) torch.bfloat16 - out : (1, 1562) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 423 tokens) : 11.3397 - top-1 accuracy : 9/423 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 595 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 386 / 1438 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3639 - top-1 accuracy : 24/386 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1442]) -Dataloader batch - labels shape: torch.Size([1, 1442]) -Dataloader batch - attn_mask shape: torch.Size([1, 1442]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 596 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1442) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1442) torch.int64 - loss_mask : 345 / 1442 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1442, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1442, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1442, 1, 576) torch.bfloat16 - out : (1, 1442) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.3160 - top-1 accuracy : 6/345 = 1.7% - - [2026-04-29 13:49:22] iteration 249/ 500 | consumed samples: 996 | elapsed time per iteration (ms): 4457.5 | throughput (token/sec/GPU): 1215.0 | learning rate: 8.002231E-05 | global batch size: 4 | lm loss: 1.132802E+01 | loss scale: 1.0 | grad norm: 0.275 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1882]) -Dataloader batch - labels shape: torch.Size([1, 1882]) -Dataloader batch - attn_mask shape: torch.Size([1, 1882]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 597 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1882) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1882) torch.int64 - loss_mask : 510 / 1882 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1882, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1882, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1882, 1, 576) torch.bfloat16 - out : (1, 1882) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 510 tokens) : 11.4072 - top-1 accuracy : 12/510 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 598 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 300 / 1492 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.1580 - top-1 accuracy : 27/300 = 9.0% - -Dataloader batch - tokens shape: torch.Size([1, 1464]) -Dataloader batch - labels shape: torch.Size([1, 1464]) -Dataloader batch - attn_mask shape: torch.Size([1, 1464]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 599 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1464) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1464) torch.int64 - loss_mask : 413 / 1464 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1464, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1464, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1464, 1, 576) torch.bfloat16 - out : (1, 1464) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.3740 - top-1 accuracy : 6/413 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1471]) -Dataloader batch - labels shape: torch.Size([1, 1471]) -Dataloader batch - attn_mask shape: torch.Size([1, 1471]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 600 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1471) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1471) torch.int64 - loss_mask : 295 / 1471 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1471, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1471, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1471, 1, 576) torch.bfloat16 - out : (1, 1471) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.1726 - top-1 accuracy : 35/295 = 11.9% - - [2026-04-29 13:49:27] iteration 250/ 500 | consumed samples: 1000 | elapsed time per iteration (ms): 4899.7 | throughput (token/sec/GPU): 1287.6 | learning rate: 7.977157E-05 | global batch size: 4 | lm loss: 1.130331E+01 | loss scale: 1.0 | grad norm: 0.231 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 250 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 250 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 250 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2086.61, 2086.61) -Dataloader batch - tokens shape: torch.Size([1, 1061]) -Dataloader batch - labels shape: torch.Size([1, 1061]) -Dataloader batch - attn_mask shape: torch.Size([1, 1061]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 601 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1061) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1061) torch.int64 - loss_mask : 207 / 1061 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1061, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1061, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1061, 1, 576) torch.bfloat16 - out : (1, 1061) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 207 tokens) : 11.1948 - top-1 accuracy : 16/207 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1870]) -Dataloader batch - labels shape: torch.Size([1, 1870]) -Dataloader batch - attn_mask shape: torch.Size([1, 1870]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 602 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1870) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1870) torch.int64 - loss_mask : 509 / 1870 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1870, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1870, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1870, 1, 576) torch.bfloat16 - out : (1, 1870) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 509 tokens) : 11.3227 - top-1 accuracy : 9/509 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1494]) -Dataloader batch - labels shape: torch.Size([1, 1494]) -Dataloader batch - attn_mask shape: torch.Size([1, 1494]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 603 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1494) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1494) torch.int64 - loss_mask : 317 / 1494 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1494, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1494, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1494, 1, 576) torch.bfloat16 - out : (1, 1494) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2014 - top-1 accuracy : 26/317 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1699]) -Dataloader batch - labels shape: torch.Size([1, 1699]) -Dataloader batch - attn_mask shape: torch.Size([1, 1699]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 604 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1699) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1699) torch.int64 - loss_mask : 353 / 1699 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1699, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1699, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1699, 1, 576) torch.bfloat16 - out : (1, 1699) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.1603 - top-1 accuracy : 16/353 = 4.5% - - [2026-04-29 13:49:34] iteration 251/ 500 | consumed samples: 1004 | elapsed time per iteration (ms): 4825.2 | throughput (token/sec/GPU): 1269.2 | learning rate: 7.951968E-05 | global batch size: 4 | lm loss: 1.123449E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 759]) -Dataloader batch - labels shape: torch.Size([1, 759]) -Dataloader batch - attn_mask shape: torch.Size([1, 759]) -Dataloader batch - image_grid_thw shape: torch.Size([15, 3]) - -==================================================================== - PIPELINE — STEP 605 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 759) torch.int64 - images : (15, 3, 1024, 1024) torch.bfloat16 - labels : (1, 759) torch.int64 - loss_mask : 156 / 759 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (15, 3, 1024, 1024) torch.bfloat16 - out : (15, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (15, 3072) - out : (15, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (759, 1, 576) torch.bfloat16 grad=False - image slots : 15 tokens replaced with vision embeddings - combined : (759, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (759, 1, 576) torch.bfloat16 - out : (1, 759) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 156 tokens) : 11.1549 - top-1 accuracy : 3/156 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1576]) -Dataloader batch - labels shape: torch.Size([1, 1576]) -Dataloader batch - attn_mask shape: torch.Size([1, 1576]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 606 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1576) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1576) torch.int64 - loss_mask : 353 / 1576 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1576, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1576, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1576, 1, 576) torch.bfloat16 - out : (1, 1576) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.2427 - top-1 accuracy : 12/353 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1530]) -Dataloader batch - labels shape: torch.Size([1, 1530]) -Dataloader batch - attn_mask shape: torch.Size([1, 1530]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 607 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1530) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1530) torch.int64 - loss_mask : 410 / 1530 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1530, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1530, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1530, 1, 576) torch.bfloat16 - out : (1, 1530) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.2824 - top-1 accuracy : 5/410 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 608 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1420) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1420) torch.int64 - loss_mask : 362 / 1420 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1420, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1420, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1420, 1, 576) torch.bfloat16 - out : (1, 1420) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.3190 - top-1 accuracy : 5/362 = 1.4% - - [2026-04-29 13:49:38] iteration 252/ 500 | consumed samples: 1008 | elapsed time per iteration (ms): 4322.0 | throughput (token/sec/GPU): 1222.8 | learning rate: 7.926664E-05 | global batch size: 4 | lm loss: 1.126627E+01 | loss scale: 1.0 | grad norm: 0.247 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1033]) -Dataloader batch - labels shape: torch.Size([1, 1033]) -Dataloader batch - attn_mask shape: torch.Size([1, 1033]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 609 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1033) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1033) torch.int64 - loss_mask : 250 / 1033 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1033, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1033, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1033, 1, 576) torch.bfloat16 - out : (1, 1033) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 250 tokens) : 11.3139 - top-1 accuracy : 8/250 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 610 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1191) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1191) torch.int64 - loss_mask : 259 / 1191 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1191, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1191, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1191, 1, 576) torch.bfloat16 - out : (1, 1191) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 259 tokens) : 11.2276 - top-1 accuracy : 17/259 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 611 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 386 / 1532 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3463 - top-1 accuracy : 14/386 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 612 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 457 / 1504 tokens (30.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 457 tokens) : 11.3903 - top-1 accuracy : 3/457 = 0.7% - - [2026-04-29 13:49:42] iteration 253/ 500 | consumed samples: 1012 | elapsed time per iteration (ms): 4285.0 | throughput (token/sec/GPU): 1227.5 | learning rate: 7.901245E-05 | global batch size: 4 | lm loss: 1.133243E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1510]) -Dataloader batch - labels shape: torch.Size([1, 1510]) -Dataloader batch - attn_mask shape: torch.Size([1, 1510]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 613 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1510) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1510) torch.int64 - loss_mask : 366 / 1510 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1510, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1510, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1510, 1, 576) torch.bfloat16 - out : (1, 1510) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2483 - top-1 accuracy : 22/366 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1080]) -Dataloader batch - labels shape: torch.Size([1, 1080]) -Dataloader batch - attn_mask shape: torch.Size([1, 1080]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 614 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1080) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1080) torch.int64 - loss_mask : 312 / 1080 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1080, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1080, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1080, 1, 576) torch.bfloat16 - out : (1, 1080) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.3477 - top-1 accuracy : 3/312 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1753]) -Dataloader batch - labels shape: torch.Size([1, 1753]) -Dataloader batch - attn_mask shape: torch.Size([1, 1753]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 615 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1753) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1753) torch.int64 - loss_mask : 397 / 1753 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1753, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1753, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1753, 1, 576) torch.bfloat16 - out : (1, 1753) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.2399 - top-1 accuracy : 31/397 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1659]) -Dataloader batch - labels shape: torch.Size([1, 1659]) -Dataloader batch - attn_mask shape: torch.Size([1, 1659]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 616 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1659) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1659) torch.int64 - loss_mask : 363 / 1659 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1659, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1659, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1659, 1, 576) torch.bfloat16 - out : (1, 1659) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.1810 - top-1 accuracy : 33/363 = 9.1% - - [2026-04-29 13:49:47] iteration 254/ 500 | consumed samples: 1016 | elapsed time per iteration (ms): 4821.8 | throughput (token/sec/GPU): 1244.8 | learning rate: 7.875715E-05 | global batch size: 4 | lm loss: 1.125052E+01 | loss scale: 1.0 | grad norm: 0.220 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1471]) -Dataloader batch - labels shape: torch.Size([1, 1471]) -Dataloader batch - attn_mask shape: torch.Size([1, 1471]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 617 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1471) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1471) torch.int64 - loss_mask : 323 / 1471 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1471, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1471, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1471, 1, 576) torch.bfloat16 - out : (1, 1471) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.1689 - top-1 accuracy : 9/323 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 618 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 326 / 1424 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2091 - top-1 accuracy : 21/326 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1365]) -Dataloader batch - labels shape: torch.Size([1, 1365]) -Dataloader batch - attn_mask shape: torch.Size([1, 1365]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 619 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1365) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1365) torch.int64 - loss_mask : 297 / 1365 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1365, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1365, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1365, 1, 576) torch.bfloat16 - out : (1, 1365) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.1847 - top-1 accuracy : 21/297 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 620 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 311 / 1386 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2080 - top-1 accuracy : 29/311 = 9.3% - - [2026-04-29 13:49:52] iteration 255/ 500 | consumed samples: 1020 | elapsed time per iteration (ms): 4630.2 | throughput (token/sec/GPU): 1219.4 | learning rate: 7.850071E-05 | global batch size: 4 | lm loss: 1.119271E+01 | loss scale: 1.0 | grad norm: 0.283 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1454]) -Dataloader batch - labels shape: torch.Size([1, 1454]) -Dataloader batch - attn_mask shape: torch.Size([1, 1454]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 621 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1454) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1454) torch.int64 - loss_mask : 309 / 1454 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1454, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1454, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1454, 1, 576) torch.bfloat16 - out : (1, 1454) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.2007 - top-1 accuracy : 34/309 = 11.0% - -Dataloader batch - tokens shape: torch.Size([1, 1361]) -Dataloader batch - labels shape: torch.Size([1, 1361]) -Dataloader batch - attn_mask shape: torch.Size([1, 1361]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 622 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1361) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1361) torch.int64 - loss_mask : 398 / 1361 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1361, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1361, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1361, 1, 576) torch.bfloat16 - out : (1, 1361) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.3750 - top-1 accuracy : 6/398 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 897]) -Dataloader batch - labels shape: torch.Size([1, 897]) -Dataloader batch - attn_mask shape: torch.Size([1, 897]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 623 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 897) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 897) torch.int64 - loss_mask : 195 / 897 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (897, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (897, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (897, 1, 576) torch.bfloat16 - out : (1, 897) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 195 tokens) : 11.1898 - top-1 accuracy : 8/195 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 624 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 376 / 1552 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.2855 - top-1 accuracy : 21/376 = 5.6% - - [2026-04-29 13:49:56] iteration 256/ 500 | consumed samples: 1024 | elapsed time per iteration (ms): 4441.7 | throughput (token/sec/GPU): 1185.1 | learning rate: 7.824317E-05 | global batch size: 4 | lm loss: 1.127826E+01 | loss scale: 1.0 | grad norm: 0.290 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1749]) -Dataloader batch - labels shape: torch.Size([1, 1749]) -Dataloader batch - attn_mask shape: torch.Size([1, 1749]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 625 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1749) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1749) torch.int64 - loss_mask : 448 / 1749 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1749, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1749, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1749, 1, 576) torch.bfloat16 - out : (1, 1749) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 448 tokens) : 11.2940 - top-1 accuracy : 41/448 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1815]) -Dataloader batch - labels shape: torch.Size([1, 1815]) -Dataloader batch - attn_mask shape: torch.Size([1, 1815]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 626 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1815) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1815) torch.int64 - loss_mask : 462 / 1815 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1815, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1815, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1815, 1, 576) torch.bfloat16 - out : (1, 1815) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 462 tokens) : 11.2445 - top-1 accuracy : 23/462 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1362]) -Dataloader batch - labels shape: torch.Size([1, 1362]) -Dataloader batch - attn_mask shape: torch.Size([1, 1362]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 627 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1362) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1362) torch.int64 - loss_mask : 303 / 1362 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1362, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1362, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1362, 1, 576) torch.bfloat16 - out : (1, 1362) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.2056 - top-1 accuracy : 14/303 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1178]) -Dataloader batch - labels shape: torch.Size([1, 1178]) -Dataloader batch - attn_mask shape: torch.Size([1, 1178]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 628 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1178) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1178) torch.int64 - loss_mask : 252 / 1178 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1178, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1178, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1178, 1, 576) torch.bfloat16 - out : (1, 1178) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 252 tokens) : 11.1717 - top-1 accuracy : 15/252 = 6.0% - - [2026-04-29 13:50:01] iteration 257/ 500 | consumed samples: 1028 | elapsed time per iteration (ms): 4953.3 | throughput (token/sec/GPU): 1232.3 | learning rate: 7.798453E-05 | global batch size: 4 | lm loss: 1.123906E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1503]) -Dataloader batch - labels shape: torch.Size([1, 1503]) -Dataloader batch - attn_mask shape: torch.Size([1, 1503]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 629 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1503) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1503) torch.int64 - loss_mask : 314 / 1503 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1503, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1503, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1503, 1, 576) torch.bfloat16 - out : (1, 1503) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.1723 - top-1 accuracy : 22/314 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1842]) -Dataloader batch - labels shape: torch.Size([1, 1842]) -Dataloader batch - attn_mask shape: torch.Size([1, 1842]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 630 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1842) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1842) torch.int64 - loss_mask : 464 / 1842 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1842, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1842, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1842, 1, 576) torch.bfloat16 - out : (1, 1842) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 464 tokens) : 11.2825 - top-1 accuracy : 20/464 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 631 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1467) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1467) torch.int64 - loss_mask : 412 / 1467 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1467, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1467, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1467, 1, 576) torch.bfloat16 - out : (1, 1467) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3147 - top-1 accuracy : 1/412 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1448]) -Dataloader batch - labels shape: torch.Size([1, 1448]) -Dataloader batch - attn_mask shape: torch.Size([1, 1448]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 632 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1448) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1448) torch.int64 - loss_mask : 401 / 1448 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1448, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1448, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1448, 1, 576) torch.bfloat16 - out : (1, 1448) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 401 tokens) : 11.3615 - top-1 accuracy : 21/401 = 5.2% - - [2026-04-29 13:50:06] iteration 258/ 500 | consumed samples: 1032 | elapsed time per iteration (ms): 4895.6 | throughput (token/sec/GPU): 1278.7 | learning rate: 7.772480E-05 | global batch size: 4 | lm loss: 1.128899E+01 | loss scale: 1.0 | grad norm: 0.261 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1246]) -Dataloader batch - labels shape: torch.Size([1, 1246]) -Dataloader batch - attn_mask shape: torch.Size([1, 1246]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 633 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1246) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1246) torch.int64 - loss_mask : 355 / 1246 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1246, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1246, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1246, 1, 576) torch.bfloat16 - out : (1, 1246) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.3667 - top-1 accuracy : 5/355 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1379]) -Dataloader batch - labels shape: torch.Size([1, 1379]) -Dataloader batch - attn_mask shape: torch.Size([1, 1379]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 634 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1379) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1379) torch.int64 - loss_mask : 327 / 1379 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1379, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1379, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1379, 1, 576) torch.bfloat16 - out : (1, 1379) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2559 - top-1 accuracy : 31/327 = 9.5% - -Dataloader batch - tokens shape: torch.Size([1, 1366]) -Dataloader batch - labels shape: torch.Size([1, 1366]) -Dataloader batch - attn_mask shape: torch.Size([1, 1366]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 635 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1366) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1366) torch.int64 - loss_mask : 305 / 1366 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1366, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1366, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1366, 1, 576) torch.bfloat16 - out : (1, 1366) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.1560 - top-1 accuracy : 21/305 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1086]) -Dataloader batch - labels shape: torch.Size([1, 1086]) -Dataloader batch - attn_mask shape: torch.Size([1, 1086]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 636 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1086) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1086) torch.int64 - loss_mask : 249 / 1086 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1086, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1086, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1086, 1, 576) torch.bfloat16 - out : (1, 1086) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 249 tokens) : 11.2462 - top-1 accuracy : 3/249 = 1.2% - - [2026-04-29 13:50:10] iteration 259/ 500 | consumed samples: 1036 | elapsed time per iteration (ms): 4272.9 | throughput (token/sec/GPU): 1188.2 | learning rate: 7.746398E-05 | global batch size: 4 | lm loss: 1.126113E+01 | loss scale: 1.0 | grad norm: 0.244 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1390]) -Dataloader batch - labels shape: torch.Size([1, 1390]) -Dataloader batch - attn_mask shape: torch.Size([1, 1390]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 637 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1390) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1390) torch.int64 - loss_mask : 328 / 1390 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1390, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1390, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1390, 1, 576) torch.bfloat16 - out : (1, 1390) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.2255 - top-1 accuracy : 24/328 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1265]) -Dataloader batch - labels shape: torch.Size([1, 1265]) -Dataloader batch - attn_mask shape: torch.Size([1, 1265]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 638 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1265) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1265) torch.int64 - loss_mask : 327 / 1265 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1265, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1265, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1265, 1, 576) torch.bfloat16 - out : (1, 1265) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.3300 - top-1 accuracy : 5/327 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1245]) -Dataloader batch - labels shape: torch.Size([1, 1245]) -Dataloader batch - attn_mask shape: torch.Size([1, 1245]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 639 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1245) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1245) torch.int64 - loss_mask : 322 / 1245 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1245, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1245, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1245, 1, 576) torch.bfloat16 - out : (1, 1245) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.3017 - top-1 accuracy : 13/322 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1366]) -Dataloader batch - labels shape: torch.Size([1, 1366]) -Dataloader batch - attn_mask shape: torch.Size([1, 1366]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 640 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1366) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1366) torch.int64 - loss_mask : 310 / 1366 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1366, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1366, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1366, 1, 576) torch.bfloat16 - out : (1, 1366) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 310 tokens) : 11.2402 - top-1 accuracy : 20/310 = 6.5% - - [2026-04-29 13:50:15] iteration 260/ 500 | consumed samples: 1040 | elapsed time per iteration (ms): 4232.0 | throughput (token/sec/GPU): 1244.3 | learning rate: 7.720211E-05 | global batch size: 4 | lm loss: 1.127465E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4210.14, 4210.14) - forward-compute ................................: (3110.32, 3110.32) - backward-compute ...............................: (1096.22, 1096.22) - batch-generator ................................: (405.77, 405.77) - layernorm-grads-all-reduce .....................: (0.07, 0.07) - embedding-grads-all-reduce .....................: (0.10, 0.10) - all-grads-sync .................................: (1.00, 1.00) - params-all-gather ..............................: (0.38, 0.38) - optimizer-copy-to-main-grad ....................: (0.32, 0.32) - optimizer-clip-main-grad .......................: (1.20, 1.20) - optimizer-count-zeros ..........................: (0.06, 0.06) - optimizer-inner-step ...........................: (2.07, 2.07) - optimizer-copy-main-to-model-params ............: (0.53, 0.53) - optimizer ......................................: (6.51, 6.51) -Dataloader batch - tokens shape: torch.Size([1, 1506]) -Dataloader batch - labels shape: torch.Size([1, 1506]) -Dataloader batch - attn_mask shape: torch.Size([1, 1506]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 641 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1506) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1506) torch.int64 - loss_mask : 372 / 1506 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1506, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1506, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1506, 1, 576) torch.bfloat16 - out : (1, 1506) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.3374 - top-1 accuracy : 15/372 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1391]) -Dataloader batch - labels shape: torch.Size([1, 1391]) -Dataloader batch - attn_mask shape: torch.Size([1, 1391]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 642 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1391) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1391) torch.int64 - loss_mask : 314 / 1391 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1391, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1391, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1391, 1, 576) torch.bfloat16 - out : (1, 1391) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.1935 - top-1 accuracy : 17/314 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1794]) -Dataloader batch - labels shape: torch.Size([1, 1794]) -Dataloader batch - attn_mask shape: torch.Size([1, 1794]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 643 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1794) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1794) torch.int64 - loss_mask : 415 / 1794 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1794, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1794, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1794, 1, 576) torch.bfloat16 - out : (1, 1794) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.2880 - top-1 accuracy : 34/415 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1163]) -Dataloader batch - labels shape: torch.Size([1, 1163]) -Dataloader batch - attn_mask shape: torch.Size([1, 1163]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 644 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1163) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1163) torch.int64 - loss_mask : 304 / 1163 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1163, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1163, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1163, 1, 576) torch.bfloat16 - out : (1, 1163) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 304 tokens) : 11.3571 - top-1 accuracy : 5/304 = 1.6% - - [2026-04-29 13:50:19] iteration 261/ 500 | consumed samples: 1044 | elapsed time per iteration (ms): 4687.3 | throughput (token/sec/GPU): 1248.9 | learning rate: 7.693917E-05 | global batch size: 4 | lm loss: 1.129491E+01 | loss scale: 1.0 | grad norm: 0.214 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1378]) -Dataloader batch - labels shape: torch.Size([1, 1378]) -Dataloader batch - attn_mask shape: torch.Size([1, 1378]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 645 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1378) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1378) torch.int64 - loss_mask : 311 / 1378 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1378, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1378, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1378, 1, 576) torch.bfloat16 - out : (1, 1378) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2390 - top-1 accuracy : 19/311 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1832]) -Dataloader batch - labels shape: torch.Size([1, 1832]) -Dataloader batch - attn_mask shape: torch.Size([1, 1832]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 646 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1832) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1832) torch.int64 - loss_mask : 461 / 1832 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1832, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1832, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1832, 1, 576) torch.bfloat16 - out : (1, 1832) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 461 tokens) : 11.3077 - top-1 accuracy : 16/461 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1196]) -Dataloader batch - labels shape: torch.Size([1, 1196]) -Dataloader batch - attn_mask shape: torch.Size([1, 1196]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 647 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1196) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1196) torch.int64 - loss_mask : 301 / 1196 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1196, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1196, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1196, 1, 576) torch.bfloat16 - out : (1, 1196) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 301 tokens) : 11.3275 - top-1 accuracy : 3/301 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1122]) -Dataloader batch - labels shape: torch.Size([1, 1122]) -Dataloader batch - attn_mask shape: torch.Size([1, 1122]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 648 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1122) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1122) torch.int64 - loss_mask : 294 / 1122 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1122, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1122, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1122, 1, 576) torch.bfloat16 - out : (1, 1122) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.3454 - top-1 accuracy : 17/294 = 5.8% - - [2026-04-29 13:50:24] iteration 262/ 500 | consumed samples: 1048 | elapsed time per iteration (ms): 4414.0 | throughput (token/sec/GPU): 1252.4 | learning rate: 7.667518E-05 | global batch size: 4 | lm loss: 1.130456E+01 | loss scale: 1.0 | grad norm: 0.255 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1054]) -Dataloader batch - labels shape: torch.Size([1, 1054]) -Dataloader batch - attn_mask shape: torch.Size([1, 1054]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 649 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1054) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1054) torch.int64 - loss_mask : 218 / 1054 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1054, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1054, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1054, 1, 576) torch.bfloat16 - out : (1, 1054) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 218 tokens) : 11.1912 - top-1 accuracy : 22/218 = 10.1% - -Dataloader batch - tokens shape: torch.Size([1, 1428]) -Dataloader batch - labels shape: torch.Size([1, 1428]) -Dataloader batch - attn_mask shape: torch.Size([1, 1428]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 650 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1428) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1428) torch.int64 - loss_mask : 389 / 1428 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1428, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1428, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1428, 1, 576) torch.bfloat16 - out : (1, 1428) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.3336 - top-1 accuracy : 8/389 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1086]) -Dataloader batch - labels shape: torch.Size([1, 1086]) -Dataloader batch - attn_mask shape: torch.Size([1, 1086]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 651 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1086) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1086) torch.int64 - loss_mask : 264 / 1086 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1086, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1086, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1086, 1, 576) torch.bfloat16 - out : (1, 1086) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 264 tokens) : 11.2869 - top-1 accuracy : 12/264 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1809]) -Dataloader batch - labels shape: torch.Size([1, 1809]) -Dataloader batch - attn_mask shape: torch.Size([1, 1809]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 652 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1809) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1809) torch.int64 - loss_mask : 451 / 1809 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1809, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1809, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1809, 1, 576) torch.bfloat16 - out : (1, 1809) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 451 tokens) : 11.2545 - top-1 accuracy : 12/451 = 2.7% - - [2026-04-29 13:50:29] iteration 263/ 500 | consumed samples: 1052 | elapsed time per iteration (ms): 4840.0 | throughput (token/sec/GPU): 1111.0 | learning rate: 7.641016E-05 | global batch size: 4 | lm loss: 1.127381E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1422]) -Dataloader batch - labels shape: torch.Size([1, 1422]) -Dataloader batch - attn_mask shape: torch.Size([1, 1422]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 653 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1422) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1422) torch.int64 - loss_mask : 421 / 1422 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1422, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1422, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1422, 1, 576) torch.bfloat16 - out : (1, 1422) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 421 tokens) : 11.4200 - top-1 accuracy : 5/421 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1748]) -Dataloader batch - labels shape: torch.Size([1, 1748]) -Dataloader batch - attn_mask shape: torch.Size([1, 1748]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 654 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1748) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1748) torch.int64 - loss_mask : 383 / 1748 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1748, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1748, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1748, 1, 576) torch.bfloat16 - out : (1, 1748) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.2389 - top-1 accuracy : 36/383 = 9.4% - -Dataloader batch - tokens shape: torch.Size([1, 1751]) -Dataloader batch - labels shape: torch.Size([1, 1751]) -Dataloader batch - attn_mask shape: torch.Size([1, 1751]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 655 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1751) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1751) torch.int64 - loss_mask : 392 / 1751 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1751, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1751, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1751, 1, 576) torch.bfloat16 - out : (1, 1751) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.2100 - top-1 accuracy : 33/392 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 963]) -Dataloader batch - labels shape: torch.Size([1, 963]) -Dataloader batch - attn_mask shape: torch.Size([1, 963]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 656 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 963) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 963) torch.int64 - loss_mask : 199 / 963 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (963, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (963, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (963, 1, 576) torch.bfloat16 - out : (1, 963) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 199 tokens) : 11.1615 - top-1 accuracy : 1/199 = 0.5% - - [2026-04-29 13:50:33] iteration 264/ 500 | consumed samples: 1056 | elapsed time per iteration (ms): 4773.7 | throughput (token/sec/GPU): 1232.6 | learning rate: 7.614411E-05 | global batch size: 4 | lm loss: 1.127439E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1158]) -Dataloader batch - labels shape: torch.Size([1, 1158]) -Dataloader batch - attn_mask shape: torch.Size([1, 1158]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 657 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1158) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1158) torch.int64 - loss_mask : 254 / 1158 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1158, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1158, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1158, 1, 576) torch.bfloat16 - out : (1, 1158) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 254 tokens) : 11.1873 - top-1 accuracy : 10/254 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1415]) -Dataloader batch - labels shape: torch.Size([1, 1415]) -Dataloader batch - attn_mask shape: torch.Size([1, 1415]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 658 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1415) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1415) torch.int64 - loss_mask : 372 / 1415 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1415, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1415, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1415, 1, 576) torch.bfloat16 - out : (1, 1415) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.3125 - top-1 accuracy : 19/372 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1214]) -Dataloader batch - labels shape: torch.Size([1, 1214]) -Dataloader batch - attn_mask shape: torch.Size([1, 1214]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 659 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1214) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1214) torch.int64 - loss_mask : 294 / 1214 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1214, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1214, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1214, 1, 576) torch.bfloat16 - out : (1, 1214) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.3079 - top-1 accuracy : 8/294 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1563]) -Dataloader batch - labels shape: torch.Size([1, 1563]) -Dataloader batch - attn_mask shape: torch.Size([1, 1563]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 660 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1563) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1563) torch.int64 - loss_mask : 355 / 1563 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1563, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1563, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1563, 1, 576) torch.bfloat16 - out : (1, 1563) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2101 - top-1 accuracy : 8/355 = 2.3% - - [2026-04-29 13:50:38] iteration 265/ 500 | consumed samples: 1060 | elapsed time per iteration (ms): 4510.7 | throughput (token/sec/GPU): 1186.1 | learning rate: 7.587705E-05 | global batch size: 4 | lm loss: 1.125798E+01 | loss scale: 1.0 | grad norm: 0.299 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1858]) -Dataloader batch - labels shape: torch.Size([1, 1858]) -Dataloader batch - attn_mask shape: torch.Size([1, 1858]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 661 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1858) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1858) torch.int64 - loss_mask : 492 / 1858 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1858, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1858, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1858, 1, 576) torch.bfloat16 - out : (1, 1858) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 492 tokens) : 11.3262 - top-1 accuracy : 3/492 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1362]) -Dataloader batch - labels shape: torch.Size([1, 1362]) -Dataloader batch - attn_mask shape: torch.Size([1, 1362]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 662 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1362) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1362) torch.int64 - loss_mask : 310 / 1362 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1362, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1362, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1362, 1, 576) torch.bfloat16 - out : (1, 1362) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 310 tokens) : 11.2466 - top-1 accuracy : 5/310 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1210]) -Dataloader batch - labels shape: torch.Size([1, 1210]) -Dataloader batch - attn_mask shape: torch.Size([1, 1210]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 663 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1210) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1210) torch.int64 - loss_mask : 292 / 1210 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1210, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1210, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1210, 1, 576) torch.bfloat16 - out : (1, 1210) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 292 tokens) : 11.2307 - top-1 accuracy : 11/292 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1879]) -Dataloader batch - labels shape: torch.Size([1, 1879]) -Dataloader batch - attn_mask shape: torch.Size([1, 1879]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 664 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1879) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1879) torch.int64 - loss_mask : 527 / 1879 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1879, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1879, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1879, 1, 576) torch.bfloat16 - out : (1, 1879) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 527 tokens) : 11.3276 - top-1 accuracy : 14/527 = 2.7% - - [2026-04-29 13:50:43] iteration 266/ 500 | consumed samples: 1064 | elapsed time per iteration (ms): 4909.4 | throughput (token/sec/GPU): 1285.1 | learning rate: 7.560897E-05 | global batch size: 4 | lm loss: 1.129424E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1426]) -Dataloader batch - labels shape: torch.Size([1, 1426]) -Dataloader batch - attn_mask shape: torch.Size([1, 1426]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 665 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1426) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1426) torch.int64 - loss_mask : 368 / 1426 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1426, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1426, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1426, 1, 576) torch.bfloat16 - out : (1, 1426) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.3675 - top-1 accuracy : 22/368 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1453]) -Dataloader batch - labels shape: torch.Size([1, 1453]) -Dataloader batch - attn_mask shape: torch.Size([1, 1453]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 666 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1453) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1453) torch.int64 - loss_mask : 356 / 1453 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1453, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1453, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1453, 1, 576) torch.bfloat16 - out : (1, 1453) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2471 - top-1 accuracy : 14/356 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1749]) -Dataloader batch - labels shape: torch.Size([1, 1749]) -Dataloader batch - attn_mask shape: torch.Size([1, 1749]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 667 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1749) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1749) torch.int64 - loss_mask : 391 / 1749 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1749, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1749, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1749, 1, 576) torch.bfloat16 - out : (1, 1749) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2469 - top-1 accuracy : 24/391 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1749]) -Dataloader batch - labels shape: torch.Size([1, 1749]) -Dataloader batch - attn_mask shape: torch.Size([1, 1749]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 668 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1749) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1749) torch.int64 - loss_mask : 404 / 1749 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1749, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1749, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1749, 1, 576) torch.bfloat16 - out : (1, 1749) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.2460 - top-1 accuracy : 17/404 = 4.2% - - [2026-04-29 13:50:48] iteration 267/ 500 | consumed samples: 1068 | elapsed time per iteration (ms): 5042.6 | throughput (token/sec/GPU): 1264.6 | learning rate: 7.533991E-05 | global batch size: 4 | lm loss: 1.127594E+01 | loss scale: 1.0 | grad norm: 0.250 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 669 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 322 / 1380 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.2771 - top-1 accuracy : 21/322 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1892]) -Dataloader batch - labels shape: torch.Size([1, 1892]) -Dataloader batch - attn_mask shape: torch.Size([1, 1892]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 670 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1892) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1892) torch.int64 - loss_mask : 534 / 1892 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1892, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1892, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1892, 1, 576) torch.bfloat16 - out : (1, 1892) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 534 tokens) : 11.3710 - top-1 accuracy : 10/534 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1846]) -Dataloader batch - labels shape: torch.Size([1, 1846]) -Dataloader batch - attn_mask shape: torch.Size([1, 1846]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 671 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1846) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1846) torch.int64 - loss_mask : 532 / 1846 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1846, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1846, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1846, 1, 576) torch.bfloat16 - out : (1, 1846) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 532 tokens) : 11.4057 - top-1 accuracy : 4/532 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1806]) -Dataloader batch - labels shape: torch.Size([1, 1806]) -Dataloader batch - attn_mask shape: torch.Size([1, 1806]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 672 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1806) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1806) torch.int64 - loss_mask : 426 / 1806 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1806, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1806, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1806, 1, 576) torch.bfloat16 - out : (1, 1806) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.2804 - top-1 accuracy : 10/426 = 2.3% - - [2026-04-29 13:50:53] iteration 268/ 500 | consumed samples: 1072 | elapsed time per iteration (ms): 5125.8 | throughput (token/sec/GPU): 1350.8 | learning rate: 7.506986E-05 | global batch size: 4 | lm loss: 1.134323E+01 | loss scale: 1.0 | grad norm: 0.254 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1639]) -Dataloader batch - labels shape: torch.Size([1, 1639]) -Dataloader batch - attn_mask shape: torch.Size([1, 1639]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 673 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1639) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1639) torch.int64 - loss_mask : 366 / 1639 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1639, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1639, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1639, 1, 576) torch.bfloat16 - out : (1, 1639) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2519 - top-1 accuracy : 19/366 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1531]) -Dataloader batch - labels shape: torch.Size([1, 1531]) -Dataloader batch - attn_mask shape: torch.Size([1, 1531]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 674 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1531) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1531) torch.int64 - loss_mask : 386 / 1531 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1531, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1531, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1531, 1, 576) torch.bfloat16 - out : (1, 1531) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2765 - top-1 accuracy : 23/386 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1046]) -Dataloader batch - labels shape: torch.Size([1, 1046]) -Dataloader batch - attn_mask shape: torch.Size([1, 1046]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 675 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1046) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1046) torch.int64 - loss_mask : 277 / 1046 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1046, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1046, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1046, 1, 576) torch.bfloat16 - out : (1, 1046) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 277 tokens) : 11.3179 - top-1 accuracy : 0/277 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 676 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 403 / 1695 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.2510 - top-1 accuracy : 18/403 = 4.5% - - [2026-04-29 13:50:58] iteration 269/ 500 | consumed samples: 1076 | elapsed time per iteration (ms): 4702.0 | throughput (token/sec/GPU): 1257.1 | learning rate: 7.479883E-05 | global batch size: 4 | lm loss: 1.127104E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1228]) -Dataloader batch - labels shape: torch.Size([1, 1228]) -Dataloader batch - attn_mask shape: torch.Size([1, 1228]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 677 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1228) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1228) torch.int64 - loss_mask : 306 / 1228 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1228, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1228, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1228, 1, 576) torch.bfloat16 - out : (1, 1228) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.2757 - top-1 accuracy : 1/306 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 678 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 286 / 1460 tokens (19.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 286 tokens) : 11.1679 - top-1 accuracy : 30/286 = 10.5% - -Dataloader batch - tokens shape: torch.Size([1, 1606]) -Dataloader batch - labels shape: torch.Size([1, 1606]) -Dataloader batch - attn_mask shape: torch.Size([1, 1606]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 679 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1606) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1606) torch.int64 - loss_mask : 386 / 1606 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1606, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1606, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1606, 1, 576) torch.bfloat16 - out : (1, 1606) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3314 - top-1 accuracy : 16/386 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1549]) -Dataloader batch - labels shape: torch.Size([1, 1549]) -Dataloader batch - attn_mask shape: torch.Size([1, 1549]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 680 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1549) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1549) torch.int64 - loss_mask : 334 / 1549 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1549, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1549, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1549, 1, 576) torch.bfloat16 - out : (1, 1549) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.2006 - top-1 accuracy : 26/334 = 7.8% - - [2026-04-29 13:51:02] iteration 270/ 500 | consumed samples: 1080 | elapsed time per iteration (ms): 4723.7 | throughput (token/sec/GPU): 1236.9 | learning rate: 7.452684E-05 | global batch size: 4 | lm loss: 1.124949E+01 | loss scale: 1.0 | grad norm: 0.274 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1736]) -Dataloader batch - labels shape: torch.Size([1, 1736]) -Dataloader batch - attn_mask shape: torch.Size([1, 1736]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 681 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1736) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1736) torch.int64 - loss_mask : 417 / 1736 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1736, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1736, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1736, 1, 576) torch.bfloat16 - out : (1, 1736) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 417 tokens) : 11.2740 - top-1 accuracy : 24/417 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 682 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 461 / 1520 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 461 tokens) : 11.4030 - top-1 accuracy : 1/461 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1703]) -Dataloader batch - labels shape: torch.Size([1, 1703]) -Dataloader batch - attn_mask shape: torch.Size([1, 1703]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 683 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1703) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1703) torch.int64 - loss_mask : 351 / 1703 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1703, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1703, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1703, 1, 576) torch.bfloat16 - out : (1, 1703) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.1612 - top-1 accuracy : 30/351 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 684 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 336 / 1507 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.2205 - top-1 accuracy : 34/336 = 10.1% - - [2026-04-29 13:51:07] iteration 271/ 500 | consumed samples: 1084 | elapsed time per iteration (ms): 5048.3 | throughput (token/sec/GPU): 1280.8 | learning rate: 7.425390E-05 | global batch size: 4 | lm loss: 1.127521E+01 | loss scale: 1.0 | grad norm: 0.280 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1559]) -Dataloader batch - labels shape: torch.Size([1, 1559]) -Dataloader batch - attn_mask shape: torch.Size([1, 1559]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 685 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1559) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1559) torch.int64 - loss_mask : 381 / 1559 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1559, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1559, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1559, 1, 576) torch.bfloat16 - out : (1, 1559) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.2352 - top-1 accuracy : 24/381 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1338]) -Dataloader batch - labels shape: torch.Size([1, 1338]) -Dataloader batch - attn_mask shape: torch.Size([1, 1338]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 686 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1338) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1338) torch.int64 - loss_mask : 389 / 1338 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1338, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1338, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1338, 1, 576) torch.bfloat16 - out : (1, 1338) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.3675 - top-1 accuracy : 4/389 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 687 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1467) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1467) torch.int64 - loss_mask : 408 / 1467 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1467, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1467, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1467, 1, 576) torch.bfloat16 - out : (1, 1467) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3507 - top-1 accuracy : 1/408 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 688 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 395 / 1443 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.3366 - top-1 accuracy : 5/395 = 1.3% - - [2026-04-29 13:51:12] iteration 272/ 500 | consumed samples: 1088 | elapsed time per iteration (ms): 4461.7 | throughput (token/sec/GPU): 1301.5 | learning rate: 7.398002E-05 | global batch size: 4 | lm loss: 1.132333E+01 | loss scale: 1.0 | grad norm: 0.288 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1478]) -Dataloader batch - labels shape: torch.Size([1, 1478]) -Dataloader batch - attn_mask shape: torch.Size([1, 1478]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 689 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1478) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1478) torch.int64 - loss_mask : 309 / 1478 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1478, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1478, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1478, 1, 576) torch.bfloat16 - out : (1, 1478) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.1034 - top-1 accuracy : 30/309 = 9.7% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 690 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 369 / 1431 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.3652 - top-1 accuracy : 10/369 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 857]) -Dataloader batch - labels shape: torch.Size([1, 857]) -Dataloader batch - attn_mask shape: torch.Size([1, 857]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 691 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 857) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 857) torch.int64 - loss_mask : 169 / 857 tokens (19.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (857, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (857, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (857, 1, 576) torch.bfloat16 - out : (1, 857) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 169 tokens) : 11.0726 - top-1 accuracy : 4/169 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1580]) -Dataloader batch - labels shape: torch.Size([1, 1580]) -Dataloader batch - attn_mask shape: torch.Size([1, 1580]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 692 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1580) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1580) torch.int64 - loss_mask : 435 / 1580 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1580, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1580, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1580, 1, 576) torch.bfloat16 - out : (1, 1580) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 435 tokens) : 11.3118 - top-1 accuracy : 13/435 = 3.0% - - [2026-04-29 13:51:16] iteration 273/ 500 | consumed samples: 1092 | elapsed time per iteration (ms): 4289.2 | throughput (token/sec/GPU): 1246.4 | learning rate: 7.370520E-05 | global batch size: 4 | lm loss: 1.124543E+01 | loss scale: 1.0 | grad norm: 0.294 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1555]) -Dataloader batch - labels shape: torch.Size([1, 1555]) -Dataloader batch - attn_mask shape: torch.Size([1, 1555]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 693 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1555) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1555) torch.int64 - loss_mask : 368 / 1555 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1555, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1555, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1555, 1, 576) torch.bfloat16 - out : (1, 1555) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2541 - top-1 accuracy : 31/368 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 1548]) -Dataloader batch - labels shape: torch.Size([1, 1548]) -Dataloader batch - attn_mask shape: torch.Size([1, 1548]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 694 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1548) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1548) torch.int64 - loss_mask : 398 / 1548 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1548, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1548, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1548, 1, 576) torch.bfloat16 - out : (1, 1548) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.3251 - top-1 accuracy : 25/398 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1376]) -Dataloader batch - labels shape: torch.Size([1, 1376]) -Dataloader batch - attn_mask shape: torch.Size([1, 1376]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 695 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1376) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1376) torch.int64 - loss_mask : 295 / 1376 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1376, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1376, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1376, 1, 576) torch.bfloat16 - out : (1, 1376) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.1526 - top-1 accuracy : 18/295 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1150]) -Dataloader batch - labels shape: torch.Size([1, 1150]) -Dataloader batch - attn_mask shape: torch.Size([1, 1150]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 696 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1150) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1150) torch.int64 - loss_mask : 307 / 1150 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1150, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1150, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1150, 1, 576) torch.bfloat16 - out : (1, 1150) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.3657 - top-1 accuracy : 7/307 = 2.3% - - [2026-04-29 13:51:21] iteration 274/ 500 | consumed samples: 1096 | elapsed time per iteration (ms): 4478.0 | throughput (token/sec/GPU): 1257.0 | learning rate: 7.342947E-05 | global batch size: 4 | lm loss: 1.127790E+01 | loss scale: 1.0 | grad norm: 0.197 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1536]) -Dataloader batch - labels shape: torch.Size([1, 1536]) -Dataloader batch - attn_mask shape: torch.Size([1, 1536]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 697 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1536) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1536) torch.int64 - loss_mask : 394 / 1536 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1536, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1536, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1536, 1, 576) torch.bfloat16 - out : (1, 1536) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3483 - top-1 accuracy : 4/394 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1557]) -Dataloader batch - labels shape: torch.Size([1, 1557]) -Dataloader batch - attn_mask shape: torch.Size([1, 1557]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 698 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1557) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1557) torch.int64 - loss_mask : 410 / 1557 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1557, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1557, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1557, 1, 576) torch.bfloat16 - out : (1, 1557) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.3608 - top-1 accuracy : 6/410 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1035]) -Dataloader batch - labels shape: torch.Size([1, 1035]) -Dataloader batch - attn_mask shape: torch.Size([1, 1035]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 699 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1035) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1035) torch.int64 - loss_mask : 262 / 1035 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1035, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1035, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1035, 1, 576) torch.bfloat16 - out : (1, 1035) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 262 tokens) : 11.3067 - top-1 accuracy : 0/262 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1242]) -Dataloader batch - labels shape: torch.Size([1, 1242]) -Dataloader batch - attn_mask shape: torch.Size([1, 1242]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 700 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1242) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1242) torch.int64 - loss_mask : 348 / 1242 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1242, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1242, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1242, 1, 576) torch.bfloat16 - out : (1, 1242) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.3236 - top-1 accuracy : 2/348 = 0.6% - - [2026-04-29 13:51:25] iteration 275/ 500 | consumed samples: 1100 | elapsed time per iteration (ms): 4169.6 | throughput (token/sec/GPU): 1287.9 | learning rate: 7.315283E-05 | global batch size: 4 | lm loss: 1.133816E+01 | loss scale: 1.0 | grad norm: 0.271 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1517]) -Dataloader batch - labels shape: torch.Size([1, 1517]) -Dataloader batch - attn_mask shape: torch.Size([1, 1517]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 701 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1517) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1517) torch.int64 - loss_mask : 375 / 1517 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1517, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1517, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1517, 1, 576) torch.bfloat16 - out : (1, 1517) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.2995 - top-1 accuracy : 32/375 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1458]) -Dataloader batch - labels shape: torch.Size([1, 1458]) -Dataloader batch - attn_mask shape: torch.Size([1, 1458]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 702 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1458) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1458) torch.int64 - loss_mask : 368 / 1458 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1458, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1458, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1458, 1, 576) torch.bfloat16 - out : (1, 1458) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.3178 - top-1 accuracy : 18/368 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1441]) -Dataloader batch - labels shape: torch.Size([1, 1441]) -Dataloader batch - attn_mask shape: torch.Size([1, 1441]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 703 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1441) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1441) torch.int64 - loss_mask : 386 / 1441 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1441, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1441, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1441, 1, 576) torch.bfloat16 - out : (1, 1441) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2888 - top-1 accuracy : 3/386 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 704 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 277 / 1460 tokens (19.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 277 tokens) : 11.0981 - top-1 accuracy : 28/277 = 10.1% - - [2026-04-29 13:51:30] iteration 276/ 500 | consumed samples: 1104 | elapsed time per iteration (ms): 4741.8 | throughput (token/sec/GPU): 1239.2 | learning rate: 7.287529E-05 | global batch size: 4 | lm loss: 1.126169E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 705 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1423) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1423) torch.int64 - loss_mask : 349 / 1423 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1423, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1423, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1423, 1, 576) torch.bfloat16 - out : (1, 1423) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2960 - top-1 accuracy : 5/349 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1446]) -Dataloader batch - labels shape: torch.Size([1, 1446]) -Dataloader batch - attn_mask shape: torch.Size([1, 1446]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 706 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1446) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1446) torch.int64 - loss_mask : 397 / 1446 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1446, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1446, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1446, 1, 576) torch.bfloat16 - out : (1, 1446) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.3163 - top-1 accuracy : 4/397 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1894]) -Dataloader batch - labels shape: torch.Size([1, 1894]) -Dataloader batch - attn_mask shape: torch.Size([1, 1894]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 707 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1894) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1894) torch.int64 - loss_mask : 526 / 1894 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1894, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1894, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1894, 1, 576) torch.bfloat16 - out : (1, 1894) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 526 tokens) : 11.3506 - top-1 accuracy : 23/526 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1064]) -Dataloader batch - labels shape: torch.Size([1, 1064]) -Dataloader batch - attn_mask shape: torch.Size([1, 1064]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 708 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1064) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1064) torch.int64 - loss_mask : 242 / 1064 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1064, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1064, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1064, 1, 576) torch.bfloat16 - out : (1, 1064) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 242 tokens) : 11.1667 - top-1 accuracy : 17/242 = 7.0% - - [2026-04-29 13:51:34] iteration 277/ 500 | consumed samples: 1108 | elapsed time per iteration (ms): 4664.7 | throughput (token/sec/GPU): 1249.2 | learning rate: 7.259686E-05 | global batch size: 4 | lm loss: 1.129963E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1501]) -Dataloader batch - labels shape: torch.Size([1, 1501]) -Dataloader batch - attn_mask shape: torch.Size([1, 1501]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 709 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1501) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1501) torch.int64 - loss_mask : 352 / 1501 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1501, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1501, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1501, 1, 576) torch.bfloat16 - out : (1, 1501) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.2599 - top-1 accuracy : 4/352 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 710 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 281 / 1443 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 281 tokens) : 11.1571 - top-1 accuracy : 32/281 = 11.4% - -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 711 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 405 / 1543 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.3351 - top-1 accuracy : 9/405 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1346]) -Dataloader batch - labels shape: torch.Size([1, 1346]) -Dataloader batch - attn_mask shape: torch.Size([1, 1346]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 712 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1346) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1346) torch.int64 - loss_mask : 367 / 1346 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1346, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1346, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1346, 1, 576) torch.bfloat16 - out : (1, 1346) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.3757 - top-1 accuracy : 18/367 = 4.9% - - [2026-04-29 13:51:39] iteration 278/ 500 | consumed samples: 1112 | elapsed time per iteration (ms): 4617.6 | throughput (token/sec/GPU): 1263.2 | learning rate: 7.231756E-05 | global batch size: 4 | lm loss: 1.129127E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1943]) -Dataloader batch - labels shape: torch.Size([1, 1943]) -Dataloader batch - attn_mask shape: torch.Size([1, 1943]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 713 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1943) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1943) torch.int64 - loss_mask : 568 / 1943 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1943, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1943, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1943, 1, 576) torch.bfloat16 - out : (1, 1943) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 568 tokens) : 11.3889 - top-1 accuracy : 3/568 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 714 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 358 / 1434 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2795 - top-1 accuracy : 19/358 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1484]) -Dataloader batch - labels shape: torch.Size([1, 1484]) -Dataloader batch - attn_mask shape: torch.Size([1, 1484]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 715 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1484) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1484) torch.int64 - loss_mask : 308 / 1484 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1484, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1484, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1484, 1, 576) torch.bfloat16 - out : (1, 1484) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.1932 - top-1 accuracy : 32/308 = 10.4% - -Dataloader batch - tokens shape: torch.Size([1, 1409]) -Dataloader batch - labels shape: torch.Size([1, 1409]) -Dataloader batch - attn_mask shape: torch.Size([1, 1409]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 716 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1409) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1409) torch.int64 - loss_mask : 311 / 1409 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1409, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1409, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1409, 1, 576) torch.bfloat16 - out : (1, 1409) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2310 - top-1 accuracy : 12/311 = 3.9% - - [2026-04-29 13:51:44] iteration 279/ 500 | consumed samples: 1116 | elapsed time per iteration (ms): 4968.4 | throughput (token/sec/GPU): 1262.0 | learning rate: 7.203739E-05 | global batch size: 4 | lm loss: 1.129274E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 717 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 297 / 1459 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.1207 - top-1 accuracy : 10/297 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 718 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 346 / 1532 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.2485 - top-1 accuracy : 19/346 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1528]) -Dataloader batch - labels shape: torch.Size([1, 1528]) -Dataloader batch - attn_mask shape: torch.Size([1, 1528]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 719 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1528) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1528) torch.int64 - loss_mask : 350 / 1528 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1528, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1528, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1528, 1, 576) torch.bfloat16 - out : (1, 1528) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.2209 - top-1 accuracy : 16/350 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1071]) -Dataloader batch - labels shape: torch.Size([1, 1071]) -Dataloader batch - attn_mask shape: torch.Size([1, 1071]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 720 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1071) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1071) torch.int64 - loss_mask : 308 / 1071 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1071, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1071, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1071, 1, 576) torch.bfloat16 - out : (1, 1071) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.3766 - top-1 accuracy : 13/308 = 4.2% - - [2026-04-29 13:51:48] iteration 280/ 500 | consumed samples: 1120 | elapsed time per iteration (ms): 4601.4 | throughput (token/sec/GPU): 1214.9 | learning rate: 7.175637E-05 | global batch size: 4 | lm loss: 1.124224E+01 | loss scale: 1.0 | grad norm: 0.217 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4587.02, 4587.02) - forward-compute ................................: (3409.11, 3409.11) - backward-compute ...............................: (1175.09, 1175.09) - batch-generator ................................: (432.75, 432.75) - layernorm-grads-all-reduce .....................: (0.03, 0.03) - embedding-grads-all-reduce .....................: (0.04, 0.04) - all-grads-sync .................................: (0.71, 0.71) - params-all-gather ..............................: (0.20, 0.20) - optimizer-copy-to-main-grad ....................: (0.19, 0.19) - optimizer-clip-main-grad .......................: (0.72, 0.72) - optimizer-count-zeros ..........................: (0.02, 0.02) - optimizer-inner-step ...........................: (1.44, 1.44) - optimizer-copy-main-to-model-params ............: (0.27, 0.27) - optimizer ......................................: (3.62, 3.62) -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 721 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 329 / 1424 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2586 - top-1 accuracy : 13/329 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1039]) -Dataloader batch - labels shape: torch.Size([1, 1039]) -Dataloader batch - attn_mask shape: torch.Size([1, 1039]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 722 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1039) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1039) torch.int64 - loss_mask : 254 / 1039 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1039, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1039, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1039, 1, 576) torch.bfloat16 - out : (1, 1039) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 254 tokens) : 11.3268 - top-1 accuracy : 4/254 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1265]) -Dataloader batch - labels shape: torch.Size([1, 1265]) -Dataloader batch - attn_mask shape: torch.Size([1, 1265]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 723 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1265) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1265) torch.int64 - loss_mask : 388 / 1265 tokens (30.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1265, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1265, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1265, 1, 576) torch.bfloat16 - out : (1, 1265) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3968 - top-1 accuracy : 3/388 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1440]) -Dataloader batch - labels shape: torch.Size([1, 1440]) -Dataloader batch - attn_mask shape: torch.Size([1, 1440]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 724 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1440) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1440) torch.int64 - loss_mask : 455 / 1440 tokens (31.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1440, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1440, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1440, 1, 576) torch.bfloat16 - out : (1, 1440) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 455 tokens) : 11.4545 - top-1 accuracy : 2/455 = 0.4% - - [2026-04-29 13:51:52] iteration 281/ 500 | consumed samples: 1124 | elapsed time per iteration (ms): 4041.1 | throughput (token/sec/GPU): 1278.9 | learning rate: 7.147451E-05 | global batch size: 4 | lm loss: 1.137086E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1725]) -Dataloader batch - labels shape: torch.Size([1, 1725]) -Dataloader batch - attn_mask shape: torch.Size([1, 1725]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 725 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1725) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1725) torch.int64 - loss_mask : 416 / 1725 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1725, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1725, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1725, 1, 576) torch.bfloat16 - out : (1, 1725) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.2919 - top-1 accuracy : 27/416 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1717]) -Dataloader batch - labels shape: torch.Size([1, 1717]) -Dataloader batch - attn_mask shape: torch.Size([1, 1717]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 726 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1717) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1717) torch.int64 - loss_mask : 368 / 1717 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1717, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1717, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1717, 1, 576) torch.bfloat16 - out : (1, 1717) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2139 - top-1 accuracy : 27/368 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 727 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 324 / 1424 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.2316 - top-1 accuracy : 19/324 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1880]) -Dataloader batch - labels shape: torch.Size([1, 1880]) -Dataloader batch - attn_mask shape: torch.Size([1, 1880]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 728 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1880) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1880) torch.int64 - loss_mask : 519 / 1880 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1880, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1880, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1880, 1, 576) torch.bfloat16 - out : (1, 1880) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 519 tokens) : 11.3738 - top-1 accuracy : 6/519 = 1.2% - - [2026-04-29 13:51:58] iteration 282/ 500 | consumed samples: 1128 | elapsed time per iteration (ms): 5373.3 | throughput (token/sec/GPU): 1255.5 | learning rate: 7.119181E-05 | global batch size: 4 | lm loss: 1.128837E+01 | loss scale: 1.0 | grad norm: 0.226 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1389]) -Dataloader batch - labels shape: torch.Size([1, 1389]) -Dataloader batch - attn_mask shape: torch.Size([1, 1389]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 729 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1389) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1389) torch.int64 - loss_mask : 422 / 1389 tokens (30.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1389, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1389, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1389, 1, 576) torch.bfloat16 - out : (1, 1389) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.4389 - top-1 accuracy : 1/422 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1510]) -Dataloader batch - labels shape: torch.Size([1, 1510]) -Dataloader batch - attn_mask shape: torch.Size([1, 1510]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 730 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1510) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1510) torch.int64 - loss_mask : 377 / 1510 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1510, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1510, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1510, 1, 576) torch.bfloat16 - out : (1, 1510) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.2736 - top-1 accuracy : 21/377 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1482]) -Dataloader batch - labels shape: torch.Size([1, 1482]) -Dataloader batch - attn_mask shape: torch.Size([1, 1482]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 731 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1482) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1482) torch.int64 - loss_mask : 375 / 1482 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1482, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1482, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1482, 1, 576) torch.bfloat16 - out : (1, 1482) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.2996 - top-1 accuracy : 0/375 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1348]) -Dataloader batch - labels shape: torch.Size([1, 1348]) -Dataloader batch - attn_mask shape: torch.Size([1, 1348]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 732 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1348) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1348) torch.int64 - loss_mask : 373 / 1348 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1348, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1348, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1348, 1, 576) torch.bfloat16 - out : (1, 1348) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.3464 - top-1 accuracy : 14/373 = 3.8% - - [2026-04-29 13:52:02] iteration 283/ 500 | consumed samples: 1132 | elapsed time per iteration (ms): 4661.7 | throughput (token/sec/GPU): 1228.9 | learning rate: 7.090830E-05 | global batch size: 4 | lm loss: 1.134256E+01 | loss scale: 1.0 | grad norm: 0.226 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1543]) -Dataloader batch - labels shape: torch.Size([1, 1543]) -Dataloader batch - attn_mask shape: torch.Size([1, 1543]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 733 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1543) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1543) torch.int64 - loss_mask : 368 / 1543 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1543, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1543, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1543, 1, 576) torch.bfloat16 - out : (1, 1543) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2656 - top-1 accuracy : 3/368 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 734 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 413 / 1507 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.3163 - top-1 accuracy : 3/413 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1972]) -Dataloader batch - labels shape: torch.Size([1, 1972]) -Dataloader batch - attn_mask shape: torch.Size([1, 1972]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 735 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1972) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1972) torch.int64 - loss_mask : 618 / 1972 tokens (31.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1972, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1972, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1972, 1, 576) torch.bfloat16 - out : (1, 1972) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 618 tokens) : 11.4126 - top-1 accuracy : 12/618 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 736 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 392 / 1532 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.2278 - top-1 accuracy : 17/392 = 4.3% - - [2026-04-29 13:52:07] iteration 284/ 500 | consumed samples: 1136 | elapsed time per iteration (ms): 4960.7 | throughput (token/sec/GPU): 1321.2 | learning rate: 7.062397E-05 | global batch size: 4 | lm loss: 1.131973E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1202]) -Dataloader batch - labels shape: torch.Size([1, 1202]) -Dataloader batch - attn_mask shape: torch.Size([1, 1202]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 737 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1202) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1202) torch.int64 - loss_mask : 292 / 1202 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1202, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1202, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1202, 1, 576) torch.bfloat16 - out : (1, 1202) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 292 tokens) : 11.2533 - top-1 accuracy : 6/292 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 738 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 304 / 1364 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 304 tokens) : 11.2509 - top-1 accuracy : 24/304 = 7.9% - -Dataloader batch - tokens shape: torch.Size([1, 1413]) -Dataloader batch - labels shape: torch.Size([1, 1413]) -Dataloader batch - attn_mask shape: torch.Size([1, 1413]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 739 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1413) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1413) torch.int64 - loss_mask : 366 / 1413 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1413, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1413, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1413, 1, 576) torch.bfloat16 - out : (1, 1413) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2968 - top-1 accuracy : 9/366 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1015]) -Dataloader batch - labels shape: torch.Size([1, 1015]) -Dataloader batch - attn_mask shape: torch.Size([1, 1015]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 740 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1015) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1015) torch.int64 - loss_mask : 197 / 1015 tokens (19.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1015, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1015, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1015, 1, 576) torch.bfloat16 - out : (1, 1015) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 197 tokens) : 11.1506 - top-1 accuracy : 8/197 = 4.1% - - [2026-04-29 13:52:12] iteration 285/ 500 | consumed samples: 1140 | elapsed time per iteration (ms): 4391.8 | throughput (token/sec/GPU): 1137.1 | learning rate: 7.033885E-05 | global batch size: 4 | lm loss: 1.124896E+01 | loss scale: 1.0 | grad norm: 0.246 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1213]) -Dataloader batch - labels shape: torch.Size([1, 1213]) -Dataloader batch - attn_mask shape: torch.Size([1, 1213]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 741 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1213) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1213) torch.int64 - loss_mask : 326 / 1213 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1213, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1213, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1213, 1, 576) torch.bfloat16 - out : (1, 1213) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.4121 - top-1 accuracy : 19/326 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 742 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 372 / 1434 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.3330 - top-1 accuracy : 10/372 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1073]) -Dataloader batch - labels shape: torch.Size([1, 1073]) -Dataloader batch - attn_mask shape: torch.Size([1, 1073]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 743 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1073) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1073) torch.int64 - loss_mask : 237 / 1073 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1073, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1073, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1073, 1, 576) torch.bfloat16 - out : (1, 1073) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 237 tokens) : 11.1760 - top-1 accuracy : 17/237 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1544]) -Dataloader batch - labels shape: torch.Size([1, 1544]) -Dataloader batch - attn_mask shape: torch.Size([1, 1544]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 744 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1544) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1544) torch.int64 - loss_mask : 391 / 1544 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1544, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1544, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1544, 1, 576) torch.bfloat16 - out : (1, 1544) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3222 - top-1 accuracy : 16/391 = 4.1% - - [2026-04-29 13:52:16] iteration 286/ 500 | consumed samples: 1144 | elapsed time per iteration (ms): 4372.9 | throughput (token/sec/GPU): 1203.8 | learning rate: 7.005294E-05 | global batch size: 4 | lm loss: 1.132121E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1850]) -Dataloader batch - labels shape: torch.Size([1, 1850]) -Dataloader batch - attn_mask shape: torch.Size([1, 1850]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 745 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1850) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1850) torch.int64 - loss_mask : 508 / 1850 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1850, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1850, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1850, 1, 576) torch.bfloat16 - out : (1, 1850) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 508 tokens) : 11.3665 - top-1 accuracy : 25/508 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1561]) -Dataloader batch - labels shape: torch.Size([1, 1561]) -Dataloader batch - attn_mask shape: torch.Size([1, 1561]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 746 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1561) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1561) torch.int64 - loss_mask : 368 / 1561 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1561, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1561, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1561, 1, 576) torch.bfloat16 - out : (1, 1561) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2653 - top-1 accuracy : 26/368 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1847]) -Dataloader batch - labels shape: torch.Size([1, 1847]) -Dataloader batch - attn_mask shape: torch.Size([1, 1847]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 747 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1847) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1847) torch.int64 - loss_mask : 484 / 1847 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1847, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1847, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1847, 1, 576) torch.bfloat16 - out : (1, 1847) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 484 tokens) : 11.3058 - top-1 accuracy : 26/484 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1666]) -Dataloader batch - labels shape: torch.Size([1, 1666]) -Dataloader batch - attn_mask shape: torch.Size([1, 1666]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 748 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1666) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1666) torch.int64 - loss_mask : 431 / 1666 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1666, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1666, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1666, 1, 576) torch.bfloat16 - out : (1, 1666) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.1686 - top-1 accuracy : 23/431 = 5.3% - - [2026-04-29 13:52:22] iteration 287/ 500 | consumed samples: 1148 | elapsed time per iteration (ms): 5412.0 | throughput (token/sec/GPU): 1279.4 | learning rate: 6.976626E-05 | global batch size: 4 | lm loss: 1.128168E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1561]) -Dataloader batch - labels shape: torch.Size([1, 1561]) -Dataloader batch - attn_mask shape: torch.Size([1, 1561]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 749 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1561) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1561) torch.int64 - loss_mask : 407 / 1561 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1561, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1561, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1561, 1, 576) torch.bfloat16 - out : (1, 1561) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.2501 - top-1 accuracy : 11/407 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 750 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 300 / 1364 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.1984 - top-1 accuracy : 16/300 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1446]) -Dataloader batch - labels shape: torch.Size([1, 1446]) -Dataloader batch - attn_mask shape: torch.Size([1, 1446]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 751 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1446) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1446) torch.int64 - loss_mask : 390 / 1446 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1446, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1446, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1446, 1, 576) torch.bfloat16 - out : (1, 1446) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3699 - top-1 accuracy : 6/390 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1538]) -Dataloader batch - labels shape: torch.Size([1, 1538]) -Dataloader batch - attn_mask shape: torch.Size([1, 1538]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 752 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1538) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1538) torch.int64 - loss_mask : 371 / 1538 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1538, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1538, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1538, 1, 576) torch.bfloat16 - out : (1, 1538) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.2653 - top-1 accuracy : 10/371 = 2.7% - - [2026-04-29 13:52:26] iteration 288/ 500 | consumed samples: 1152 | elapsed time per iteration (ms): 4662.4 | throughput (token/sec/GPU): 1267.4 | learning rate: 6.947881E-05 | global batch size: 4 | lm loss: 1.127518E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 753 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 283 / 1443 tokens (19.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 283 tokens) : 11.1549 - top-1 accuracy : 7/283 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1169]) -Dataloader batch - labels shape: torch.Size([1, 1169]) -Dataloader batch - attn_mask shape: torch.Size([1, 1169]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 754 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1169) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1169) torch.int64 - loss_mask : 374 / 1169 tokens (32.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1169, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1169, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1169, 1, 576) torch.bfloat16 - out : (1, 1169) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.4394 - top-1 accuracy : 17/374 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1099]) -Dataloader batch - labels shape: torch.Size([1, 1099]) -Dataloader batch - attn_mask shape: torch.Size([1, 1099]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 755 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1099) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1099) torch.int64 - loss_mask : 297 / 1099 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1099, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1099, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1099, 1, 576) torch.bfloat16 - out : (1, 1099) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.3653 - top-1 accuracy : 2/297 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1700]) -Dataloader batch - labels shape: torch.Size([1, 1700]) -Dataloader batch - attn_mask shape: torch.Size([1, 1700]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 756 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1700) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1700) torch.int64 - loss_mask : 338 / 1700 tokens (19.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1700, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1700, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1700, 1, 576) torch.bfloat16 - out : (1, 1700) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.1767 - top-1 accuracy : 30/338 = 8.9% - - [2026-04-29 13:52:31] iteration 289/ 500 | consumed samples: 1156 | elapsed time per iteration (ms): 4672.6 | throughput (token/sec/GPU): 1158.0 | learning rate: 6.919062E-05 | global batch size: 4 | lm loss: 1.129132E+01 | loss scale: 1.0 | grad norm: 0.277 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1763]) -Dataloader batch - labels shape: torch.Size([1, 1763]) -Dataloader batch - attn_mask shape: torch.Size([1, 1763]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 757 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1763) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1763) torch.int64 - loss_mask : 409 / 1763 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1763, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1763, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1763, 1, 576) torch.bfloat16 - out : (1, 1763) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.2354 - top-1 accuracy : 38/409 = 9.3% - -Dataloader batch - tokens shape: torch.Size([1, 1846]) -Dataloader batch - labels shape: torch.Size([1, 1846]) -Dataloader batch - attn_mask shape: torch.Size([1, 1846]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 758 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1846) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1846) torch.int64 - loss_mask : 493 / 1846 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1846, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1846, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1846, 1, 576) torch.bfloat16 - out : (1, 1846) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 493 tokens) : 11.3386 - top-1 accuracy : 17/493 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1370]) -Dataloader batch - labels shape: torch.Size([1, 1370]) -Dataloader batch - attn_mask shape: torch.Size([1, 1370]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 759 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1370) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1370) torch.int64 - loss_mask : 273 / 1370 tokens (19.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1370, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1370, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1370, 1, 576) torch.bfloat16 - out : (1, 1370) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 273 tokens) : 11.1756 - top-1 accuracy : 22/273 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 760 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 331 / 1496 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.2429 - top-1 accuracy : 5/331 = 1.5% - - [2026-04-29 13:52:36] iteration 290/ 500 | consumed samples: 1160 | elapsed time per iteration (ms): 5188.0 | throughput (token/sec/GPU): 1248.1 | learning rate: 6.890167E-05 | global batch size: 4 | lm loss: 1.126001E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 986]) -Dataloader batch - labels shape: torch.Size([1, 986]) -Dataloader batch - attn_mask shape: torch.Size([1, 986]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 761 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 986) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 986) torch.int64 - loss_mask : 226 / 986 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (986, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (986, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (986, 1, 576) torch.bfloat16 - out : (1, 986) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 226 tokens) : 11.2126 - top-1 accuracy : 4/226 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1723]) -Dataloader batch - labels shape: torch.Size([1, 1723]) -Dataloader batch - attn_mask shape: torch.Size([1, 1723]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 762 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1723) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1723) torch.int64 - loss_mask : 376 / 1723 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1723, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1723, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1723, 1, 576) torch.bfloat16 - out : (1, 1723) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.2139 - top-1 accuracy : 19/376 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 763 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1519) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1519) torch.int64 - loss_mask : 323 / 1519 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1519, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1519, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1519, 1, 576) torch.bfloat16 - out : (1, 1519) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.2029 - top-1 accuracy : 28/323 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1215]) -Dataloader batch - labels shape: torch.Size([1, 1215]) -Dataloader batch - attn_mask shape: torch.Size([1, 1215]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 764 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1215) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1215) torch.int64 - loss_mask : 270 / 1215 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1215, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1215, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1215, 1, 576) torch.bfloat16 - out : (1, 1215) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 270 tokens) : 11.2507 - top-1 accuracy : 14/270 = 5.2% - - [2026-04-29 13:52:41] iteration 291/ 500 | consumed samples: 1164 | elapsed time per iteration (ms): 4584.8 | throughput (token/sec/GPU): 1187.2 | learning rate: 6.861200E-05 | global batch size: 4 | lm loss: 1.121901E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1135]) -Dataloader batch - labels shape: torch.Size([1, 1135]) -Dataloader batch - attn_mask shape: torch.Size([1, 1135]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 765 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1135) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1135) torch.int64 - loss_mask : 225 / 1135 tokens (19.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1135, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1135, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1135, 1, 576) torch.bfloat16 - out : (1, 1135) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 225 tokens) : 11.0769 - top-1 accuracy : 8/225 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1077]) -Dataloader batch - labels shape: torch.Size([1, 1077]) -Dataloader batch - attn_mask shape: torch.Size([1, 1077]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 766 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1077) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1077) torch.int64 - loss_mask : 266 / 1077 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1077, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1077, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1077, 1, 576) torch.bfloat16 - out : (1, 1077) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 266 tokens) : 11.2910 - top-1 accuracy : 10/266 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1707]) -Dataloader batch - labels shape: torch.Size([1, 1707]) -Dataloader batch - attn_mask shape: torch.Size([1, 1707]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 767 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1707) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1707) torch.int64 - loss_mask : 364 / 1707 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1707, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1707, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1707, 1, 576) torch.bfloat16 - out : (1, 1707) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2174 - top-1 accuracy : 38/364 = 10.4% - -Dataloader batch - tokens shape: torch.Size([1, 1751]) -Dataloader batch - labels shape: torch.Size([1, 1751]) -Dataloader batch - attn_mask shape: torch.Size([1, 1751]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 768 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1751) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1751) torch.int64 - loss_mask : 401 / 1751 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1751, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1751, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1751, 1, 576) torch.bfloat16 - out : (1, 1751) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 401 tokens) : 11.2243 - top-1 accuracy : 27/401 = 6.7% - - [2026-04-29 13:52:45] iteration 292/ 500 | consumed samples: 1168 | elapsed time per iteration (ms): 4654.5 | throughput (token/sec/GPU): 1218.2 | learning rate: 6.832162E-05 | global batch size: 4 | lm loss: 1.121000E+01 | loss scale: 1.0 | grad norm: 0.264 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1757]) -Dataloader batch - labels shape: torch.Size([1, 1757]) -Dataloader batch - attn_mask shape: torch.Size([1, 1757]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 769 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1757) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1757) torch.int64 - loss_mask : 409 / 1757 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1757, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1757, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1757, 1, 576) torch.bfloat16 - out : (1, 1757) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.2299 - top-1 accuracy : 28/409 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1693]) -Dataloader batch - labels shape: torch.Size([1, 1693]) -Dataloader batch - attn_mask shape: torch.Size([1, 1693]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 770 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1693) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1693) torch.int64 - loss_mask : 334 / 1693 tokens (19.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1693, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1693, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1693, 1, 576) torch.bfloat16 - out : (1, 1693) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.1727 - top-1 accuracy : 25/334 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 771 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 378 / 1525 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 378 tokens) : 11.2548 - top-1 accuracy : 9/378 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1859]) -Dataloader batch - labels shape: torch.Size([1, 1859]) -Dataloader batch - attn_mask shape: torch.Size([1, 1859]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 772 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1859) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1859) torch.int64 - loss_mask : 500 / 1859 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1859, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1859, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1859, 1, 576) torch.bfloat16 - out : (1, 1859) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 500 tokens) : 11.3416 - top-1 accuracy : 16/500 = 3.2% - - [2026-04-29 13:52:51] iteration 293/ 500 | consumed samples: 1172 | elapsed time per iteration (ms): 5563.9 | throughput (token/sec/GPU): 1228.3 | learning rate: 6.803052E-05 | global batch size: 4 | lm loss: 1.125836E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1602]) -Dataloader batch - labels shape: torch.Size([1, 1602]) -Dataloader batch - attn_mask shape: torch.Size([1, 1602]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 773 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1602) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1602) torch.int64 - loss_mask : 452 / 1602 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1602, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1602, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1602, 1, 576) torch.bfloat16 - out : (1, 1602) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 452 tokens) : 11.3449 - top-1 accuracy : 2/452 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1407]) -Dataloader batch - labels shape: torch.Size([1, 1407]) -Dataloader batch - attn_mask shape: torch.Size([1, 1407]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 774 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1407) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1407) torch.int64 - loss_mask : 327 / 1407 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1407, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1407, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1407, 1, 576) torch.bfloat16 - out : (1, 1407) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2324 - top-1 accuracy : 13/327 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 775 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 324 / 1386 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.2145 - top-1 accuracy : 21/324 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1702]) -Dataloader batch - labels shape: torch.Size([1, 1702]) -Dataloader batch - attn_mask shape: torch.Size([1, 1702]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 776 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1702) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1702) torch.int64 - loss_mask : 364 / 1702 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1702, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1702, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1702, 1, 576) torch.bfloat16 - out : (1, 1702) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2410 - top-1 accuracy : 28/364 = 7.7% - - [2026-04-29 13:52:56] iteration 294/ 500 | consumed samples: 1176 | elapsed time per iteration (ms): 5023.5 | throughput (token/sec/GPU): 1213.7 | learning rate: 6.773874E-05 | global batch size: 4 | lm loss: 1.126527E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1232]) -Dataloader batch - labels shape: torch.Size([1, 1232]) -Dataloader batch - attn_mask shape: torch.Size([1, 1232]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 777 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1232) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1232) torch.int64 - loss_mask : 322 / 1232 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1232, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1232, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1232, 1, 576) torch.bfloat16 - out : (1, 1232) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.2782 - top-1 accuracy : 18/322 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1484]) -Dataloader batch - labels shape: torch.Size([1, 1484]) -Dataloader batch - attn_mask shape: torch.Size([1, 1484]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 778 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1484) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1484) torch.int64 - loss_mask : 332 / 1484 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1484, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1484, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1484, 1, 576) torch.bfloat16 - out : (1, 1484) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.1786 - top-1 accuracy : 13/332 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1026]) -Dataloader batch - labels shape: torch.Size([1, 1026]) -Dataloader batch - attn_mask shape: torch.Size([1, 1026]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 779 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1026) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1026) torch.int64 - loss_mask : 252 / 1026 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1026, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1026, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1026, 1, 576) torch.bfloat16 - out : (1, 1026) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 252 tokens) : 11.2754 - top-1 accuracy : 8/252 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1879]) -Dataloader batch - labels shape: torch.Size([1, 1879]) -Dataloader batch - attn_mask shape: torch.Size([1, 1879]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 780 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1879) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1879) torch.int64 - loss_mask : 507 / 1879 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1879, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1879, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1879, 1, 576) torch.bfloat16 - out : (1, 1879) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 507 tokens) : 11.3323 - top-1 accuracy : 7/507 = 1.4% - - [2026-04-29 13:53:01] iteration 295/ 500 | consumed samples: 1180 | elapsed time per iteration (ms): 4545.6 | throughput (token/sec/GPU): 1236.6 | learning rate: 6.744626E-05 | global batch size: 4 | lm loss: 1.127371E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1483]) -Dataloader batch - labels shape: torch.Size([1, 1483]) -Dataloader batch - attn_mask shape: torch.Size([1, 1483]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 781 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1483) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1483) torch.int64 - loss_mask : 399 / 1483 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1483, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1483, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1483, 1, 576) torch.bfloat16 - out : (1, 1483) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.3535 - top-1 accuracy : 13/399 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1567]) -Dataloader batch - labels shape: torch.Size([1, 1567]) -Dataloader batch - attn_mask shape: torch.Size([1, 1567]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 782 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1567) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1567) torch.int64 - loss_mask : 350 / 1567 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1567, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1567, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1567, 1, 576) torch.bfloat16 - out : (1, 1567) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.2637 - top-1 accuracy : 28/350 = 8.0% - -Dataloader batch - tokens shape: torch.Size([1, 1425]) -Dataloader batch - labels shape: torch.Size([1, 1425]) -Dataloader batch - attn_mask shape: torch.Size([1, 1425]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 783 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1425) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1425) torch.int64 - loss_mask : 395 / 1425 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1425, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1425, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1425, 1, 576) torch.bfloat16 - out : (1, 1425) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.4067 - top-1 accuracy : 8/395 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1536]) -Dataloader batch - labels shape: torch.Size([1, 1536]) -Dataloader batch - attn_mask shape: torch.Size([1, 1536]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 784 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1536) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1536) torch.int64 - loss_mask : 391 / 1536 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1536, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1536, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1536, 1, 576) torch.bfloat16 - out : (1, 1536) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3068 - top-1 accuracy : 14/391 = 3.6% - - [2026-04-29 13:53:05] iteration 296/ 500 | consumed samples: 1184 | elapsed time per iteration (ms): 4749.4 | throughput (token/sec/GPU): 1265.6 | learning rate: 6.715312E-05 | global batch size: 4 | lm loss: 1.133482E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 785 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 316 / 1443 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.2380 - top-1 accuracy : 8/316 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1863]) -Dataloader batch - labels shape: torch.Size([1, 1863]) -Dataloader batch - attn_mask shape: torch.Size([1, 1863]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 786 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1863) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1863) torch.int64 - loss_mask : 498 / 1863 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1863, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1863, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1863, 1, 576) torch.bfloat16 - out : (1, 1863) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 498 tokens) : 11.3808 - top-1 accuracy : 24/498 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1083]) -Dataloader batch - labels shape: torch.Size([1, 1083]) -Dataloader batch - attn_mask shape: torch.Size([1, 1083]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 787 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1083) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1083) torch.int64 - loss_mask : 260 / 1083 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1083, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1083, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1083, 1, 576) torch.bfloat16 - out : (1, 1083) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.2421 - top-1 accuracy : 8/260 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 788 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 361 / 1542 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.2525 - top-1 accuracy : 40/361 = 11.1% - - [2026-04-29 13:53:10] iteration 297/ 500 | consumed samples: 1188 | elapsed time per iteration (ms): 4827.6 | throughput (token/sec/GPU): 1228.6 | learning rate: 6.685932E-05 | global batch size: 4 | lm loss: 1.129194E+01 | loss scale: 1.0 | grad norm: 0.230 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 789 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 397 / 1525 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.3282 - top-1 accuracy : 32/397 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1056]) -Dataloader batch - labels shape: torch.Size([1, 1056]) -Dataloader batch - attn_mask shape: torch.Size([1, 1056]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 790 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1056) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1056) torch.int64 - loss_mask : 214 / 1056 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1056, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1056, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1056, 1, 576) torch.bfloat16 - out : (1, 1056) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 214 tokens) : 11.1449 - top-1 accuracy : 16/214 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1036]) -Dataloader batch - labels shape: torch.Size([1, 1036]) -Dataloader batch - attn_mask shape: torch.Size([1, 1036]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 791 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1036) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1036) torch.int64 - loss_mask : 268 / 1036 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1036, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1036, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1036, 1, 576) torch.bfloat16 - out : (1, 1036) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 268 tokens) : 11.3434 - top-1 accuracy : 1/268 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1788]) -Dataloader batch - labels shape: torch.Size([1, 1788]) -Dataloader batch - attn_mask shape: torch.Size([1, 1788]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 792 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1788) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1788) torch.int64 - loss_mask : 499 / 1788 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1788, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1788, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1788, 1, 576) torch.bfloat16 - out : (1, 1788) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 499 tokens) : 11.3617 - top-1 accuracy : 8/499 = 1.6% - - [2026-04-29 13:53:14] iteration 298/ 500 | consumed samples: 1192 | elapsed time per iteration (ms): 4298.4 | throughput (token/sec/GPU): 1257.4 | learning rate: 6.656487E-05 | global batch size: 4 | lm loss: 1.131483E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1426]) -Dataloader batch - labels shape: torch.Size([1, 1426]) -Dataloader batch - attn_mask shape: torch.Size([1, 1426]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 793 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1426) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1426) torch.int64 - loss_mask : 394 / 1426 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1426, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1426, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1426, 1, 576) torch.bfloat16 - out : (1, 1426) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3528 - top-1 accuracy : 11/394 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1379]) -Dataloader batch - labels shape: torch.Size([1, 1379]) -Dataloader batch - attn_mask shape: torch.Size([1, 1379]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 794 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1379) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1379) torch.int64 - loss_mask : 330 / 1379 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1379, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1379, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1379, 1, 576) torch.bfloat16 - out : (1, 1379) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.1995 - top-1 accuracy : 17/330 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 795 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 326 / 1388 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2750 - top-1 accuracy : 18/326 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1791]) -Dataloader batch - labels shape: torch.Size([1, 1791]) -Dataloader batch - attn_mask shape: torch.Size([1, 1791]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 796 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1791) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1791) torch.int64 - loss_mask : 436 / 1791 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1791, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1791, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1791, 1, 576) torch.bfloat16 - out : (1, 1791) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.2931 - top-1 accuracy : 24/436 = 5.5% - - [2026-04-29 13:53:19] iteration 299/ 500 | consumed samples: 1196 | elapsed time per iteration (ms): 4828.4 | throughput (token/sec/GPU): 1239.3 | learning rate: 6.626978E-05 | global batch size: 4 | lm loss: 1.128416E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 797 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 339 / 1532 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.1874 - top-1 accuracy : 20/339 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1512]) -Dataloader batch - labels shape: torch.Size([1, 1512]) -Dataloader batch - attn_mask shape: torch.Size([1, 1512]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 798 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1512) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1512) torch.int64 - loss_mask : 330 / 1512 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1512, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1512, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1512, 1, 576) torch.bfloat16 - out : (1, 1512) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.2136 - top-1 accuracy : 19/330 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 799 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 330 / 1518 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.2983 - top-1 accuracy : 20/330 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 800 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 335 / 1492 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2358 - top-1 accuracy : 32/335 = 9.6% - - [2026-04-29 13:53:24] iteration 300/ 500 | consumed samples: 1200 | elapsed time per iteration (ms): 5048.5 | throughput (token/sec/GPU): 1199.2 | learning rate: 6.597406E-05 | global batch size: 4 | lm loss: 1.123348E+01 | loss scale: 1.0 | grad norm: 0.318 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (5025.47, 5025.47) - forward-compute ................................: (3736.69, 3736.69) - backward-compute ...............................: (1284.79, 1284.79) - batch-generator ................................: (524.45, 524.45) - layernorm-grads-all-reduce .....................: (0.08, 0.08) - embedding-grads-all-reduce .....................: (0.10, 0.10) - all-grads-sync .................................: (0.99, 0.99) - params-all-gather ..............................: (0.38, 0.38) - optimizer-copy-to-main-grad ....................: (0.34, 0.34) - optimizer-clip-main-grad .......................: (1.30, 1.30) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.16, 2.16) - optimizer-copy-main-to-model-params ............: (0.55, 0.55) - optimizer ......................................: (6.87, 6.87) -saving checkpoint at iteration 300 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 300 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 300 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2518.93, 2518.93) -Dataloader batch - tokens shape: torch.Size([1, 1725]) -Dataloader batch - labels shape: torch.Size([1, 1725]) -Dataloader batch - attn_mask shape: torch.Size([1, 1725]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 801 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1725) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1725) torch.int64 - loss_mask : 347 / 1725 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1725, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1725, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1725, 1, 576) torch.bfloat16 - out : (1, 1725) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2150 - top-1 accuracy : 25/347 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 802 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 515 / 1492 tokens (34.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 515 tokens) : 11.4955 - top-1 accuracy : 2/515 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1378]) -Dataloader batch - labels shape: torch.Size([1, 1378]) -Dataloader batch - attn_mask shape: torch.Size([1, 1378]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 803 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1378) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1378) torch.int64 - loss_mask : 283 / 1378 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1378, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1378, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1378, 1, 576) torch.bfloat16 - out : (1, 1378) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 283 tokens) : 11.1375 - top-1 accuracy : 20/283 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1062]) -Dataloader batch - labels shape: torch.Size([1, 1062]) -Dataloader batch - attn_mask shape: torch.Size([1, 1062]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 804 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1062) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1062) torch.int64 - loss_mask : 290 / 1062 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1062, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1062, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1062, 1, 576) torch.bfloat16 - out : (1, 1062) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.3077 - top-1 accuracy : 5/290 = 1.7% - - [2026-04-29 13:53:31] iteration 301/ 500 | consumed samples: 1204 | elapsed time per iteration (ms): 4487.9 | throughput (token/sec/GPU): 1260.5 | learning rate: 6.567774E-05 | global batch size: 4 | lm loss: 1.131911E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1446]) -Dataloader batch - labels shape: torch.Size([1, 1446]) -Dataloader batch - attn_mask shape: torch.Size([1, 1446]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 805 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1446) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1446) torch.int64 - loss_mask : 408 / 1446 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1446, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1446, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1446, 1, 576) torch.bfloat16 - out : (1, 1446) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3660 - top-1 accuracy : 8/408 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1565]) -Dataloader batch - labels shape: torch.Size([1, 1565]) -Dataloader batch - attn_mask shape: torch.Size([1, 1565]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 806 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1565) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1565) torch.int64 - loss_mask : 385 / 1565 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1565, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1565, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1565, 1, 576) torch.bfloat16 - out : (1, 1565) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.3328 - top-1 accuracy : 21/385 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1477]) -Dataloader batch - labels shape: torch.Size([1, 1477]) -Dataloader batch - attn_mask shape: torch.Size([1, 1477]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 807 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1477) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1477) torch.int64 - loss_mask : 426 / 1477 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1477, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1477, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1477, 1, 576) torch.bfloat16 - out : (1, 1477) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.3927 - top-1 accuracy : 7/426 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1095]) -Dataloader batch - labels shape: torch.Size([1, 1095]) -Dataloader batch - attn_mask shape: torch.Size([1, 1095]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 808 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1095) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1095) torch.int64 - loss_mask : 274 / 1095 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1095, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1095, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1095, 1, 576) torch.bfloat16 - out : (1, 1095) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 274 tokens) : 11.3498 - top-1 accuracy : 0/274 = 0.0% - - [2026-04-29 13:53:36] iteration 302/ 500 | consumed samples: 1208 | elapsed time per iteration (ms): 4337.4 | throughput (token/sec/GPU): 1287.2 | learning rate: 6.538081E-05 | global batch size: 4 | lm loss: 1.136208E+01 | loss scale: 1.0 | grad norm: 0.290 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1232]) -Dataloader batch - labels shape: torch.Size([1, 1232]) -Dataloader batch - attn_mask shape: torch.Size([1, 1232]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 809 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1232) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1232) torch.int64 - loss_mask : 294 / 1232 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1232, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1232, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1232, 1, 576) torch.bfloat16 - out : (1, 1232) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.2672 - top-1 accuracy : 5/294 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1059]) -Dataloader batch - labels shape: torch.Size([1, 1059]) -Dataloader batch - attn_mask shape: torch.Size([1, 1059]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 810 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1059) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1059) torch.int64 - loss_mask : 287 / 1059 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1059, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1059, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1059, 1, 576) torch.bfloat16 - out : (1, 1059) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 287 tokens) : 11.4238 - top-1 accuracy : 1/287 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1500]) -Dataloader batch - labels shape: torch.Size([1, 1500]) -Dataloader batch - attn_mask shape: torch.Size([1, 1500]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 811 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1500) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1500) torch.int64 - loss_mask : 354 / 1500 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1500, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1500, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1500, 1, 576) torch.bfloat16 - out : (1, 1500) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2623 - top-1 accuracy : 31/354 = 8.8% - -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 812 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1420) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1420) torch.int64 - loss_mask : 361 / 1420 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1420, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1420, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1420, 1, 576) torch.bfloat16 - out : (1, 1420) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.2838 - top-1 accuracy : 3/361 = 0.8% - - [2026-04-29 13:53:40] iteration 303/ 500 | consumed samples: 1212 | elapsed time per iteration (ms): 4261.1 | throughput (token/sec/GPU): 1222.9 | learning rate: 6.508329E-05 | global batch size: 4 | lm loss: 1.130517E+01 | loss scale: 1.0 | grad norm: 0.295 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1534]) -Dataloader batch - labels shape: torch.Size([1, 1534]) -Dataloader batch - attn_mask shape: torch.Size([1, 1534]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 813 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1534) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1534) torch.int64 - loss_mask : 384 / 1534 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1534, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1534, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1534, 1, 576) torch.bfloat16 - out : (1, 1534) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.2821 - top-1 accuracy : 4/384 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1087]) -Dataloader batch - labels shape: torch.Size([1, 1087]) -Dataloader batch - attn_mask shape: torch.Size([1, 1087]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 814 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1087) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1087) torch.int64 - loss_mask : 245 / 1087 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1087, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1087, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1087, 1, 576) torch.bfloat16 - out : (1, 1087) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 245 tokens) : 11.2385 - top-1 accuracy : 12/245 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1653]) -Dataloader batch - labels shape: torch.Size([1, 1653]) -Dataloader batch - attn_mask shape: torch.Size([1, 1653]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 815 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1653) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1653) torch.int64 - loss_mask : 426 / 1653 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1653, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1653, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1653, 1, 576) torch.bfloat16 - out : (1, 1653) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.3039 - top-1 accuracy : 7/426 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1414]) -Dataloader batch - labels shape: torch.Size([1, 1414]) -Dataloader batch - attn_mask shape: torch.Size([1, 1414]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 816 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1414) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1414) torch.int64 - loss_mask : 357 / 1414 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1414, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1414, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1414, 1, 576) torch.bfloat16 - out : (1, 1414) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3077 - top-1 accuracy : 6/357 = 1.7% - - [2026-04-29 13:53:44] iteration 304/ 500 | consumed samples: 1216 | elapsed time per iteration (ms): 4600.8 | throughput (token/sec/GPU): 1236.3 | learning rate: 6.478519E-05 | global batch size: 4 | lm loss: 1.128759E+01 | loss scale: 1.0 | grad norm: 0.259 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1555]) -Dataloader batch - labels shape: torch.Size([1, 1555]) -Dataloader batch - attn_mask shape: torch.Size([1, 1555]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 817 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1555) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1555) torch.int64 - loss_mask : 404 / 1555 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1555, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1555, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1555, 1, 576) torch.bfloat16 - out : (1, 1555) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.3053 - top-1 accuracy : 25/404 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1396]) -Dataloader batch - labels shape: torch.Size([1, 1396]) -Dataloader batch - attn_mask shape: torch.Size([1, 1396]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 818 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1396) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1396) torch.int64 - loss_mask : 349 / 1396 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1396, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1396, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1396, 1, 576) torch.bfloat16 - out : (1, 1396) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2456 - top-1 accuracy : 3/349 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1075]) -Dataloader batch - labels shape: torch.Size([1, 1075]) -Dataloader batch - attn_mask shape: torch.Size([1, 1075]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 819 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1075) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1075) torch.int64 - loss_mask : 238 / 1075 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1075, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1075, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1075, 1, 576) torch.bfloat16 - out : (1, 1075) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 238 tokens) : 11.1995 - top-1 accuracy : 16/238 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 820 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 338 / 1518 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.2233 - top-1 accuracy : 23/338 = 6.8% - - [2026-04-29 13:53:49] iteration 305/ 500 | consumed samples: 1220 | elapsed time per iteration (ms): 4549.8 | throughput (token/sec/GPU): 1218.5 | learning rate: 6.448653E-05 | global batch size: 4 | lm loss: 1.124986E+01 | loss scale: 1.0 | grad norm: 0.252 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 821 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 373 / 1424 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.3341 - top-1 accuracy : 7/373 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1590]) -Dataloader batch - labels shape: torch.Size([1, 1590]) -Dataloader batch - attn_mask shape: torch.Size([1, 1590]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 822 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1590) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1590) torch.int64 - loss_mask : 357 / 1590 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1590, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1590, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1590, 1, 576) torch.bfloat16 - out : (1, 1590) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2384 - top-1 accuracy : 6/357 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1249]) -Dataloader batch - labels shape: torch.Size([1, 1249]) -Dataloader batch - attn_mask shape: torch.Size([1, 1249]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 823 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1249) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1249) torch.int64 - loss_mask : 324 / 1249 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1249, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1249, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1249, 1, 576) torch.bfloat16 - out : (1, 1249) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.3390 - top-1 accuracy : 4/324 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1614]) -Dataloader batch - labels shape: torch.Size([1, 1614]) -Dataloader batch - attn_mask shape: torch.Size([1, 1614]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 824 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1614) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1614) torch.int64 - loss_mask : 467 / 1614 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1614, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1614, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1614, 1, 576) torch.bfloat16 - out : (1, 1614) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 467 tokens) : 11.3937 - top-1 accuracy : 10/467 = 2.1% - - [2026-04-29 13:53:54] iteration 306/ 500 | consumed samples: 1224 | elapsed time per iteration (ms): 4564.1 | throughput (token/sec/GPU): 1287.7 | learning rate: 6.418731E-05 | global batch size: 4 | lm loss: 1.133099E+01 | loss scale: 1.0 | grad norm: 0.255 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1561]) -Dataloader batch - labels shape: torch.Size([1, 1561]) -Dataloader batch - attn_mask shape: torch.Size([1, 1561]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 825 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1561) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1561) torch.int64 - loss_mask : 355 / 1561 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1561, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1561, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1561, 1, 576) torch.bfloat16 - out : (1, 1561) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2311 - top-1 accuracy : 15/355 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 826 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1470) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1470) torch.int64 - loss_mask : 376 / 1470 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1470, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1470, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1470, 1, 576) torch.bfloat16 - out : (1, 1470) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3464 - top-1 accuracy : 21/376 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1644]) -Dataloader batch - labels shape: torch.Size([1, 1644]) -Dataloader batch - attn_mask shape: torch.Size([1, 1644]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 827 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1644) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1644) torch.int64 - loss_mask : 425 / 1644 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1644, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1644, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1644, 1, 576) torch.bfloat16 - out : (1, 1644) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 425 tokens) : 11.3117 - top-1 accuracy : 9/425 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1762]) -Dataloader batch - labels shape: torch.Size([1, 1762]) -Dataloader batch - attn_mask shape: torch.Size([1, 1762]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 828 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1762) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1762) torch.int64 - loss_mask : 406 / 1762 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1762, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1762, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1762, 1, 576) torch.bfloat16 - out : (1, 1762) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.2442 - top-1 accuracy : 5/406 = 1.2% - - [2026-04-29 13:53:59] iteration 307/ 500 | consumed samples: 1228 | elapsed time per iteration (ms): 5056.2 | throughput (token/sec/GPU): 1273.1 | learning rate: 6.388756E-05 | global batch size: 4 | lm loss: 1.128419E+01 | loss scale: 1.0 | grad norm: 0.272 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1792]) -Dataloader batch - labels shape: torch.Size([1, 1792]) -Dataloader batch - attn_mask shape: torch.Size([1, 1792]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 829 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1792) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1792) torch.int64 - loss_mask : 479 / 1792 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1792, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1792, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1792, 1, 576) torch.bfloat16 - out : (1, 1792) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 479 tokens) : 11.3722 - top-1 accuracy : 12/479 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1562]) -Dataloader batch - labels shape: torch.Size([1, 1562]) -Dataloader batch - attn_mask shape: torch.Size([1, 1562]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 830 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1562) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1562) torch.int64 - loss_mask : 414 / 1562 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1562, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1562, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1562, 1, 576) torch.bfloat16 - out : (1, 1562) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 414 tokens) : 11.3163 - top-1 accuracy : 3/414 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1114]) -Dataloader batch - labels shape: torch.Size([1, 1114]) -Dataloader batch - attn_mask shape: torch.Size([1, 1114]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 831 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1114) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1114) torch.int64 - loss_mask : 324 / 1114 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1114, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1114, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1114, 1, 576) torch.bfloat16 - out : (1, 1114) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.3721 - top-1 accuracy : 3/324 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1872]) -Dataloader batch - labels shape: torch.Size([1, 1872]) -Dataloader batch - attn_mask shape: torch.Size([1, 1872]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 832 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1872) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1872) torch.int64 - loss_mask : 506 / 1872 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1872, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1872, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1872, 1, 576) torch.bfloat16 - out : (1, 1872) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 506 tokens) : 11.3467 - top-1 accuracy : 7/506 = 1.4% - - [2026-04-29 13:54:03] iteration 308/ 500 | consumed samples: 1232 | elapsed time per iteration (ms): 4825.3 | throughput (token/sec/GPU): 1313.9 | learning rate: 6.358726E-05 | global batch size: 4 | lm loss: 1.135125E+01 | loss scale: 1.0 | grad norm: 0.269 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 934]) -Dataloader batch - labels shape: torch.Size([1, 934]) -Dataloader batch - attn_mask shape: torch.Size([1, 934]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 833 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 934) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 934) torch.int64 - loss_mask : 192 / 934 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (934, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (934, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (934, 1, 576) torch.bfloat16 - out : (1, 934) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 192 tokens) : 11.2180 - top-1 accuracy : 7/192 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1828]) -Dataloader batch - labels shape: torch.Size([1, 1828]) -Dataloader batch - attn_mask shape: torch.Size([1, 1828]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 834 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1828) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1828) torch.int64 - loss_mask : 470 / 1828 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1828, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1828, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1828, 1, 576) torch.bfloat16 - out : (1, 1828) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 470 tokens) : 11.2937 - top-1 accuracy : 16/470 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1506]) -Dataloader batch - labels shape: torch.Size([1, 1506]) -Dataloader batch - attn_mask shape: torch.Size([1, 1506]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 835 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1506) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1506) torch.int64 - loss_mask : 320 / 1506 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1506, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1506, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1506, 1, 576) torch.bfloat16 - out : (1, 1506) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2612 - top-1 accuracy : 27/320 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 1401]) -Dataloader batch - labels shape: torch.Size([1, 1401]) -Dataloader batch - attn_mask shape: torch.Size([1, 1401]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 836 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1401) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1401) torch.int64 - loss_mask : 319 / 1401 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1401, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1401, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1401, 1, 576) torch.bfloat16 - out : (1, 1401) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.2236 - top-1 accuracy : 5/319 = 1.6% - - [2026-04-29 13:54:08] iteration 309/ 500 | consumed samples: 1236 | elapsed time per iteration (ms): 4681.2 | throughput (token/sec/GPU): 1211.0 | learning rate: 6.328645E-05 | global batch size: 4 | lm loss: 1.125733E+01 | loss scale: 1.0 | grad norm: 0.319 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1573]) -Dataloader batch - labels shape: torch.Size([1, 1573]) -Dataloader batch - attn_mask shape: torch.Size([1, 1573]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 837 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1573) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1573) torch.int64 - loss_mask : 419 / 1573 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1573, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1573, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1573, 1, 576) torch.bfloat16 - out : (1, 1573) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.3685 - top-1 accuracy : 30/419 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1055]) -Dataloader batch - labels shape: torch.Size([1, 1055]) -Dataloader batch - attn_mask shape: torch.Size([1, 1055]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 838 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1055) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1055) torch.int64 - loss_mask : 284 / 1055 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1055, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1055, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1055, 1, 576) torch.bfloat16 - out : (1, 1055) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.2930 - top-1 accuracy : 6/284 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1031]) -Dataloader batch - labels shape: torch.Size([1, 1031]) -Dataloader batch - attn_mask shape: torch.Size([1, 1031]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 839 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1031) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1031) torch.int64 - loss_mask : 284 / 1031 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1031, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1031, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1031, 1, 576) torch.bfloat16 - out : (1, 1031) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 284 tokens) : 11.3438 - top-1 accuracy : 3/284 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1405]) -Dataloader batch - labels shape: torch.Size([1, 1405]) -Dataloader batch - attn_mask shape: torch.Size([1, 1405]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 840 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1405) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1405) torch.int64 - loss_mask : 340 / 1405 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1405, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1405, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1405, 1, 576) torch.bfloat16 - out : (1, 1405) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2146 - top-1 accuracy : 3/340 = 0.9% - - [2026-04-29 13:54:12] iteration 310/ 500 | consumed samples: 1240 | elapsed time per iteration (ms): 4028.3 | throughput (token/sec/GPU): 1257.1 | learning rate: 6.298514E-05 | global batch size: 4 | lm loss: 1.130762E+01 | loss scale: 1.0 | grad norm: 0.262 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1584]) -Dataloader batch - labels shape: torch.Size([1, 1584]) -Dataloader batch - attn_mask shape: torch.Size([1, 1584]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 841 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1584) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1584) torch.int64 - loss_mask : 372 / 1584 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1584, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1584, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1584, 1, 576) torch.bfloat16 - out : (1, 1584) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2595 - top-1 accuracy : 20/372 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 842 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 371 / 1750 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.2345 - top-1 accuracy : 10/371 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 843 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 345 / 1438 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2397 - top-1 accuracy : 6/345 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1817]) -Dataloader batch - labels shape: torch.Size([1, 1817]) -Dataloader batch - attn_mask shape: torch.Size([1, 1817]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 844 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1817) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1817) torch.int64 - loss_mask : 476 / 1817 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1817, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1817, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1817, 1, 576) torch.bfloat16 - out : (1, 1817) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 476 tokens) : 11.3707 - top-1 accuracy : 22/476 = 4.6% - - [2026-04-29 13:54:17] iteration 311/ 500 | consumed samples: 1244 | elapsed time per iteration (ms): 5201.6 | throughput (token/sec/GPU): 1266.7 | learning rate: 6.268333E-05 | global batch size: 4 | lm loss: 1.128303E+01 | loss scale: 1.0 | grad norm: 0.278 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1706]) -Dataloader batch - labels shape: torch.Size([1, 1706]) -Dataloader batch - attn_mask shape: torch.Size([1, 1706]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 845 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1706) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1706) torch.int64 - loss_mask : 343 / 1706 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1706, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1706, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1706, 1, 576) torch.bfloat16 - out : (1, 1706) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.1543 - top-1 accuracy : 29/343 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1403]) -Dataloader batch - labels shape: torch.Size([1, 1403]) -Dataloader batch - attn_mask shape: torch.Size([1, 1403]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 846 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1403) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1403) torch.int64 - loss_mask : 370 / 1403 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1403, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1403, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1403, 1, 576) torch.bfloat16 - out : (1, 1403) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.3146 - top-1 accuracy : 22/370 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1399]) -Dataloader batch - labels shape: torch.Size([1, 1399]) -Dataloader batch - attn_mask shape: torch.Size([1, 1399]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 847 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1399) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1399) torch.int64 - loss_mask : 333 / 1399 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1399, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1399, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1399, 1, 576) torch.bfloat16 - out : (1, 1399) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.2866 - top-1 accuracy : 3/333 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 848 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 325 / 1380 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.2293 - top-1 accuracy : 20/325 = 6.2% - - [2026-04-29 13:54:22] iteration 312/ 500 | consumed samples: 1248 | elapsed time per iteration (ms): 4652.5 | throughput (token/sec/GPU): 1265.6 | learning rate: 6.238103E-05 | global batch size: 4 | lm loss: 1.124750E+01 | loss scale: 1.0 | grad norm: 0.309 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1891]) -Dataloader batch - labels shape: torch.Size([1, 1891]) -Dataloader batch - attn_mask shape: torch.Size([1, 1891]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 849 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1891) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1891) torch.int64 - loss_mask : 524 / 1891 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1891, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1891, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1891, 1, 576) torch.bfloat16 - out : (1, 1891) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 524 tokens) : 11.3559 - top-1 accuracy : 19/524 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1428]) -Dataloader batch - labels shape: torch.Size([1, 1428]) -Dataloader batch - attn_mask shape: torch.Size([1, 1428]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 850 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1428) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1428) torch.int64 - loss_mask : 357 / 1428 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1428, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1428, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1428, 1, 576) torch.bfloat16 - out : (1, 1428) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2707 - top-1 accuracy : 8/357 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1049]) -Dataloader batch - labels shape: torch.Size([1, 1049]) -Dataloader batch - attn_mask shape: torch.Size([1, 1049]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 851 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1049) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1049) torch.int64 - loss_mask : 279 / 1049 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1049, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1049, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1049, 1, 576) torch.bfloat16 - out : (1, 1049) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 279 tokens) : 11.3610 - top-1 accuracy : 1/279 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 852 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 335 / 1388 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2425 - top-1 accuracy : 13/335 = 3.9% - - [2026-04-29 13:54:27] iteration 313/ 500 | consumed samples: 1252 | elapsed time per iteration (ms): 4497.5 | throughput (token/sec/GPU): 1279.8 | learning rate: 6.207827E-05 | global batch size: 4 | lm loss: 1.131110E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1932]) -Dataloader batch - labels shape: torch.Size([1, 1932]) -Dataloader batch - attn_mask shape: torch.Size([1, 1932]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 853 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1932) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1932) torch.int64 - loss_mask : 558 / 1932 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1932, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1932, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1932, 1, 576) torch.bfloat16 - out : (1, 1932) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 558 tokens) : 11.4057 - top-1 accuracy : 0/558 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1046]) -Dataloader batch - labels shape: torch.Size([1, 1046]) -Dataloader batch - attn_mask shape: torch.Size([1, 1046]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 854 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1046) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1046) torch.int64 - loss_mask : 213 / 1046 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1046, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1046, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1046, 1, 576) torch.bfloat16 - out : (1, 1046) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 213 tokens) : 11.1960 - top-1 accuracy : 9/213 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 855 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 350 / 1532 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.1967 - top-1 accuracy : 20/350 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1408]) -Dataloader batch - labels shape: torch.Size([1, 1408]) -Dataloader batch - attn_mask shape: torch.Size([1, 1408]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 856 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1408) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1408) torch.int64 - loss_mask : 319 / 1408 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1408, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1408, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1408, 1, 576) torch.bfloat16 - out : (1, 1408) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.1899 - top-1 accuracy : 16/319 = 5.0% - - [2026-04-29 13:54:31] iteration 314/ 500 | consumed samples: 1256 | elapsed time per iteration (ms): 4670.2 | throughput (token/sec/GPU): 1267.2 | learning rate: 6.177505E-05 | global batch size: 4 | lm loss: 1.127609E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 996]) -Dataloader batch - labels shape: torch.Size([1, 996]) -Dataloader batch - attn_mask shape: torch.Size([1, 996]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 857 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 996) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 996) torch.int64 - loss_mask : 248 / 996 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (996, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (996, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (996, 1, 576) torch.bfloat16 - out : (1, 996) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 248 tokens) : 11.2687 - top-1 accuracy : 11/248 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1573]) -Dataloader batch - labels shape: torch.Size([1, 1573]) -Dataloader batch - attn_mask shape: torch.Size([1, 1573]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 858 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1573) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1573) torch.int64 - loss_mask : 373 / 1573 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1573, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1573, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1573, 1, 576) torch.bfloat16 - out : (1, 1573) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.2641 - top-1 accuracy : 11/373 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1720]) -Dataloader batch - labels shape: torch.Size([1, 1720]) -Dataloader batch - attn_mask shape: torch.Size([1, 1720]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 859 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1720) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1720) torch.int64 - loss_mask : 362 / 1720 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1720, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1720, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1720, 1, 576) torch.bfloat16 - out : (1, 1720) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2497 - top-1 accuracy : 8/362 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1114]) -Dataloader batch - labels shape: torch.Size([1, 1114]) -Dataloader batch - attn_mask shape: torch.Size([1, 1114]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 860 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1114) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1114) torch.int64 - loss_mask : 285 / 1114 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1114, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1114, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1114, 1, 576) torch.bfloat16 - out : (1, 1114) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 285 tokens) : 11.3027 - top-1 accuracy : 13/285 = 4.6% - - [2026-04-29 13:54:36] iteration 315/ 500 | consumed samples: 1260 | elapsed time per iteration (ms): 4494.3 | throughput (token/sec/GPU): 1202.2 | learning rate: 6.147138E-05 | global batch size: 4 | lm loss: 1.126956E+01 | loss scale: 1.0 | grad norm: 0.242 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1830]) -Dataloader batch - labels shape: torch.Size([1, 1830]) -Dataloader batch - attn_mask shape: torch.Size([1, 1830]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 861 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1830) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1830) torch.int64 - loss_mask : 465 / 1830 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1830, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1830, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1830, 1, 576) torch.bfloat16 - out : (1, 1830) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 465 tokens) : 11.3182 - top-1 accuracy : 15/465 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1882]) -Dataloader batch - labels shape: torch.Size([1, 1882]) -Dataloader batch - attn_mask shape: torch.Size([1, 1882]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 862 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1882) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1882) torch.int64 - loss_mask : 531 / 1882 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1882, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1882, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1882, 1, 576) torch.bfloat16 - out : (1, 1882) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 531 tokens) : 11.3638 - top-1 accuracy : 9/531 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1815]) -Dataloader batch - labels shape: torch.Size([1, 1815]) -Dataloader batch - attn_mask shape: torch.Size([1, 1815]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 863 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1815) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1815) torch.int64 - loss_mask : 444 / 1815 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1815, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1815, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1815, 1, 576) torch.bfloat16 - out : (1, 1815) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 444 tokens) : 11.3118 - top-1 accuracy : 4/444 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 864 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1467) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1467) torch.int64 - loss_mask : 365 / 1467 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1467, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1467, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1467, 1, 576) torch.bfloat16 - out : (1, 1467) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2638 - top-1 accuracy : 13/365 = 3.6% - - [2026-04-29 13:54:41] iteration 316/ 500 | consumed samples: 1264 | elapsed time per iteration (ms): 5312.7 | throughput (token/sec/GPU): 1316.5 | learning rate: 6.116727E-05 | global batch size: 4 | lm loss: 1.131905E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1179]) -Dataloader batch - labels shape: torch.Size([1, 1179]) -Dataloader batch - attn_mask shape: torch.Size([1, 1179]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 865 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1179) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1179) torch.int64 - loss_mask : 326 / 1179 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1179, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1179, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1179, 1, 576) torch.bfloat16 - out : (1, 1179) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.4158 - top-1 accuracy : 4/326 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1050]) -Dataloader batch - labels shape: torch.Size([1, 1050]) -Dataloader batch - attn_mask shape: torch.Size([1, 1050]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 866 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1050) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1050) torch.int64 - loss_mask : 223 / 1050 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1050, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1050, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1050, 1, 576) torch.bfloat16 - out : (1, 1050) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 223 tokens) : 11.1546 - top-1 accuracy : 4/223 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1876]) -Dataloader batch - labels shape: torch.Size([1, 1876]) -Dataloader batch - attn_mask shape: torch.Size([1, 1876]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 867 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1876) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1876) torch.int64 - loss_mask : 525 / 1876 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1876, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1876, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1876, 1, 576) torch.bfloat16 - out : (1, 1876) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 525 tokens) : 11.3396 - top-1 accuracy : 15/525 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1131]) -Dataloader batch - labels shape: torch.Size([1, 1131]) -Dataloader batch - attn_mask shape: torch.Size([1, 1131]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 868 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1131) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1131) torch.int64 - loss_mask : 308 / 1131 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1131, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1131, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1131, 1, 576) torch.bfloat16 - out : (1, 1131) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.3749 - top-1 accuracy : 0/308 = 0.0% - - [2026-04-29 13:54:45] iteration 317/ 500 | consumed samples: 1268 | elapsed time per iteration (ms): 4202.0 | throughput (token/sec/GPU): 1246.1 | learning rate: 6.086275E-05 | global batch size: 4 | lm loss: 1.133559E+01 | loss scale: 1.0 | grad norm: 0.231 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1458]) -Dataloader batch - labels shape: torch.Size([1, 1458]) -Dataloader batch - attn_mask shape: torch.Size([1, 1458]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 869 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1458) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1458) torch.int64 - loss_mask : 408 / 1458 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1458, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1458, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1458, 1, 576) torch.bfloat16 - out : (1, 1458) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3372 - top-1 accuracy : 3/408 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1376]) -Dataloader batch - labels shape: torch.Size([1, 1376]) -Dataloader batch - attn_mask shape: torch.Size([1, 1376]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 870 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1376) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1376) torch.int64 - loss_mask : 305 / 1376 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1376, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1376, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1376, 1, 576) torch.bfloat16 - out : (1, 1376) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.2448 - top-1 accuracy : 17/305 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1481]) -Dataloader batch - labels shape: torch.Size([1, 1481]) -Dataloader batch - attn_mask shape: torch.Size([1, 1481]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 871 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1481) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1481) torch.int64 - loss_mask : 340 / 1481 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1481, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1481, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1481, 1, 576) torch.bfloat16 - out : (1, 1481) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2106 - top-1 accuracy : 6/340 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1060]) -Dataloader batch - labels shape: torch.Size([1, 1060]) -Dataloader batch - attn_mask shape: torch.Size([1, 1060]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 872 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1060) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1060) torch.int64 - loss_mask : 287 / 1060 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1060, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1060, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1060, 1, 576) torch.bfloat16 - out : (1, 1060) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 287 tokens) : 11.3580 - top-1 accuracy : 0/287 = 0.0% - - [2026-04-29 13:54:50] iteration 318/ 500 | consumed samples: 1272 | elapsed time per iteration (ms): 4390.6 | throughput (token/sec/GPU): 1224.2 | learning rate: 6.055781E-05 | global batch size: 4 | lm loss: 1.128850E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1449]) -Dataloader batch - labels shape: torch.Size([1, 1449]) -Dataloader batch - attn_mask shape: torch.Size([1, 1449]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 873 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1449) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1449) torch.int64 - loss_mask : 431 / 1449 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1449, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1449, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1449, 1, 576) torch.bfloat16 - out : (1, 1449) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.3417 - top-1 accuracy : 5/431 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 874 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1412) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1412) torch.int64 - loss_mask : 354 / 1412 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1412, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1412, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1412, 1, 576) torch.bfloat16 - out : (1, 1412) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2390 - top-1 accuracy : 17/354 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1204]) -Dataloader batch - labels shape: torch.Size([1, 1204]) -Dataloader batch - attn_mask shape: torch.Size([1, 1204]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 875 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1204) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1204) torch.int64 - loss_mask : 290 / 1204 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1204, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1204, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1204, 1, 576) torch.bfloat16 - out : (1, 1204) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.2183 - top-1 accuracy : 20/290 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1495]) -Dataloader batch - labels shape: torch.Size([1, 1495]) -Dataloader batch - attn_mask shape: torch.Size([1, 1495]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 876 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1495) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1495) torch.int64 - loss_mask : 306 / 1495 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1495, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1495, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1495, 1, 576) torch.bfloat16 - out : (1, 1495) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.2252 - top-1 accuracy : 26/306 = 8.5% - - [2026-04-29 13:54:54] iteration 319/ 500 | consumed samples: 1276 | elapsed time per iteration (ms): 4403.3 | throughput (token/sec/GPU): 1262.7 | learning rate: 6.025247E-05 | global batch size: 4 | lm loss: 1.126364E+01 | loss scale: 1.0 | grad norm: 0.251 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1442]) -Dataloader batch - labels shape: torch.Size([1, 1442]) -Dataloader batch - attn_mask shape: torch.Size([1, 1442]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 877 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1442) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1442) torch.int64 - loss_mask : 395 / 1442 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1442, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1442, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1442, 1, 576) torch.bfloat16 - out : (1, 1442) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.3309 - top-1 accuracy : 1/395 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1557]) -Dataloader batch - labels shape: torch.Size([1, 1557]) -Dataloader batch - attn_mask shape: torch.Size([1, 1557]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 878 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1557) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1557) torch.int64 - loss_mask : 414 / 1557 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1557, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1557, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1557, 1, 576) torch.bfloat16 - out : (1, 1557) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 414 tokens) : 11.2652 - top-1 accuracy : 14/414 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1872]) -Dataloader batch - labels shape: torch.Size([1, 1872]) -Dataloader batch - attn_mask shape: torch.Size([1, 1872]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 879 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1872) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1872) torch.int64 - loss_mask : 519 / 1872 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1872, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1872, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1872, 1, 576) torch.bfloat16 - out : (1, 1872) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 519 tokens) : 11.4007 - top-1 accuracy : 22/519 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1533]) -Dataloader batch - labels shape: torch.Size([1, 1533]) -Dataloader batch - attn_mask shape: torch.Size([1, 1533]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 880 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1533) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1533) torch.int64 - loss_mask : 347 / 1533 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1533, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1533, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1533, 1, 576) torch.bfloat16 - out : (1, 1533) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2151 - top-1 accuracy : 14/347 = 4.0% - - [2026-04-29 13:54:59] iteration 320/ 500 | consumed samples: 1280 | elapsed time per iteration (ms): 4991.9 | throughput (token/sec/GPU): 1282.9 | learning rate: 5.994675E-05 | global batch size: 4 | lm loss: 1.131231E+01 | loss scale: 1.0 | grad norm: 0.193 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4979.16, 4979.16) - forward-compute ................................: (3647.24, 3647.24) - backward-compute ...............................: (1329.65, 1329.65) - batch-generator ................................: (412.78, 412.78) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.04, 0.04) - all-grads-sync .................................: (0.56, 0.56) - params-all-gather ..............................: (0.15, 0.15) - optimizer-copy-to-main-grad ....................: (0.15, 0.15) - optimizer-clip-main-grad .......................: (0.57, 0.57) - optimizer-count-zeros ..........................: (0.02, 0.02) - optimizer-inner-step ...........................: (1.31, 1.31) - optimizer-copy-main-to-model-params ............: (0.24, 0.24) - optimizer ......................................: (3.14, 3.14) -Dataloader batch - tokens shape: torch.Size([1, 1819]) -Dataloader batch - labels shape: torch.Size([1, 1819]) -Dataloader batch - attn_mask shape: torch.Size([1, 1819]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 881 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1819) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1819) torch.int64 - loss_mask : 443 / 1819 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1819, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1819, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1819, 1, 576) torch.bfloat16 - out : (1, 1819) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 443 tokens) : 11.2853 - top-1 accuracy : 2/443 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1066]) -Dataloader batch - labels shape: torch.Size([1, 1066]) -Dataloader batch - attn_mask shape: torch.Size([1, 1066]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 882 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1066) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1066) torch.int64 - loss_mask : 246 / 1066 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1066, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1066, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1066, 1, 576) torch.bfloat16 - out : (1, 1066) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 246 tokens) : 11.2503 - top-1 accuracy : 10/246 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1497]) -Dataloader batch - labels shape: torch.Size([1, 1497]) -Dataloader batch - attn_mask shape: torch.Size([1, 1497]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 883 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1497) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1497) torch.int64 - loss_mask : 313 / 1497 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1497, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1497, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1497, 1, 576) torch.bfloat16 - out : (1, 1497) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.1832 - top-1 accuracy : 28/313 = 8.9% - -Dataloader batch - tokens shape: torch.Size([1, 1104]) -Dataloader batch - labels shape: torch.Size([1, 1104]) -Dataloader batch - attn_mask shape: torch.Size([1, 1104]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 884 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1104) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1104) torch.int64 - loss_mask : 255 / 1104 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1104, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1104, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1104, 1, 576) torch.bfloat16 - out : (1, 1104) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 255 tokens) : 11.2770 - top-1 accuracy : 17/255 = 6.7% - - [2026-04-29 13:55:04] iteration 321/ 500 | consumed samples: 1284 | elapsed time per iteration (ms): 4516.1 | throughput (token/sec/GPU): 1214.8 | learning rate: 5.964065E-05 | global batch size: 4 | lm loss: 1.125132E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1450]) -Dataloader batch - labels shape: torch.Size([1, 1450]) -Dataloader batch - attn_mask shape: torch.Size([1, 1450]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 885 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1450) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1450) torch.int64 - loss_mask : 342 / 1450 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1450, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1450, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1450, 1, 576) torch.bfloat16 - out : (1, 1450) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2942 - top-1 accuracy : 12/342 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1330]) -Dataloader batch - labels shape: torch.Size([1, 1330]) -Dataloader batch - attn_mask shape: torch.Size([1, 1330]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 886 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1330) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1330) torch.int64 - loss_mask : 428 / 1330 tokens (32.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1330, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1330, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1330, 1, 576) torch.bfloat16 - out : (1, 1330) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 428 tokens) : 11.4512 - top-1 accuracy : 0/428 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 887 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 295 / 1443 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.1824 - top-1 accuracy : 4/295 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 888 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 431 / 1487 tokens (29.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.3769 - top-1 accuracy : 3/431 = 0.7% - - [2026-04-29 13:55:08] iteration 322/ 500 | consumed samples: 1288 | elapsed time per iteration (ms): 4470.0 | throughput (token/sec/GPU): 1277.4 | learning rate: 5.933419E-05 | global batch size: 4 | lm loss: 1.134088E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1530]) -Dataloader batch - labels shape: torch.Size([1, 1530]) -Dataloader batch - attn_mask shape: torch.Size([1, 1530]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 889 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1530) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1530) torch.int64 - loss_mask : 329 / 1530 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1530, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1530, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1530, 1, 576) torch.bfloat16 - out : (1, 1530) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.1901 - top-1 accuracy : 19/329 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1901]) -Dataloader batch - labels shape: torch.Size([1, 1901]) -Dataloader batch - attn_mask shape: torch.Size([1, 1901]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 890 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1901) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1901) torch.int64 - loss_mask : 539 / 1901 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1901, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1901, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1901, 1, 576) torch.bfloat16 - out : (1, 1901) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 539 tokens) : 11.3533 - top-1 accuracy : 4/539 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1601]) -Dataloader batch - labels shape: torch.Size([1, 1601]) -Dataloader batch - attn_mask shape: torch.Size([1, 1601]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 891 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1601) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1601) torch.int64 - loss_mask : 390 / 1601 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1601, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1601, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1601, 1, 576) torch.bfloat16 - out : (1, 1601) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.2660 - top-1 accuracy : 19/390 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1398]) -Dataloader batch - labels shape: torch.Size([1, 1398]) -Dataloader batch - attn_mask shape: torch.Size([1, 1398]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 892 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1398) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1398) torch.int64 - loss_mask : 317 / 1398 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1398, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1398, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1398, 1, 576) torch.bfloat16 - out : (1, 1398) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2300 - top-1 accuracy : 16/317 = 5.0% - - [2026-04-29 13:55:13] iteration 323/ 500 | consumed samples: 1292 | elapsed time per iteration (ms): 5142.6 | throughput (token/sec/GPU): 1250.3 | learning rate: 5.902737E-05 | global batch size: 4 | lm loss: 1.127278E+01 | loss scale: 1.0 | grad norm: 0.268 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 952]) -Dataloader batch - labels shape: torch.Size([1, 952]) -Dataloader batch - attn_mask shape: torch.Size([1, 952]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 893 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 952) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 952) torch.int64 - loss_mask : 229 / 952 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (952, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (952, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (952, 1, 576) torch.bfloat16 - out : (1, 952) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 229 tokens) : 11.2717 - top-1 accuracy : 5/229 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1462]) -Dataloader batch - labels shape: torch.Size([1, 1462]) -Dataloader batch - attn_mask shape: torch.Size([1, 1462]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 894 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1462) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1462) torch.int64 - loss_mask : 407 / 1462 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1462, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1462, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1462, 1, 576) torch.bfloat16 - out : (1, 1462) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.3624 - top-1 accuracy : 8/407 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1826]) -Dataloader batch - labels shape: torch.Size([1, 1826]) -Dataloader batch - attn_mask shape: torch.Size([1, 1826]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 895 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1826) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1826) torch.int64 - loss_mask : 475 / 1826 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1826, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1826, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1826, 1, 576) torch.bfloat16 - out : (1, 1826) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 475 tokens) : 11.2921 - top-1 accuracy : 24/475 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 896 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 276 / 1436 tokens (19.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 276 tokens) : 11.1015 - top-1 accuracy : 22/276 = 8.0% - - [2026-04-29 13:55:18] iteration 324/ 500 | consumed samples: 1296 | elapsed time per iteration (ms): 4628.1 | throughput (token/sec/GPU): 1226.4 | learning rate: 5.872023E-05 | global batch size: 4 | lm loss: 1.127145E+01 | loss scale: 1.0 | grad norm: 0.245 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1180]) -Dataloader batch - labels shape: torch.Size([1, 1180]) -Dataloader batch - attn_mask shape: torch.Size([1, 1180]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 897 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1180) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1180) torch.int64 - loss_mask : 278 / 1180 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1180, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1180, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1180, 1, 576) torch.bfloat16 - out : (1, 1180) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.2969 - top-1 accuracy : 5/278 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1817]) -Dataloader batch - labels shape: torch.Size([1, 1817]) -Dataloader batch - attn_mask shape: torch.Size([1, 1817]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 898 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1817) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1817) torch.int64 - loss_mask : 461 / 1817 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1817, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1817, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1817, 1, 576) torch.bfloat16 - out : (1, 1817) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 461 tokens) : 11.2819 - top-1 accuracy : 13/461 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1130]) -Dataloader batch - labels shape: torch.Size([1, 1130]) -Dataloader batch - attn_mask shape: torch.Size([1, 1130]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 899 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1130) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1130) torch.int64 - loss_mask : 323 / 1130 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1130, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1130, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1130, 1, 576) torch.bfloat16 - out : (1, 1130) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.3937 - top-1 accuracy : 1/323 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1137]) -Dataloader batch - labels shape: torch.Size([1, 1137]) -Dataloader batch - attn_mask shape: torch.Size([1, 1137]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 900 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1137) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1137) torch.int64 - loss_mask : 334 / 1137 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1137, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1137, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1137, 1, 576) torch.bfloat16 - out : (1, 1137) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.4002 - top-1 accuracy : 12/334 = 3.6% - - [2026-04-29 13:55:22] iteration 325/ 500 | consumed samples: 1300 | elapsed time per iteration (ms): 4279.8 | throughput (token/sec/GPU): 1230.0 | learning rate: 5.841275E-05 | global batch size: 4 | lm loss: 1.133906E+01 | loss scale: 1.0 | grad norm: 0.206 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 969]) -Dataloader batch - labels shape: torch.Size([1, 969]) -Dataloader batch - attn_mask shape: torch.Size([1, 969]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 901 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 969) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 969) torch.int64 - loss_mask : 219 / 969 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (969, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (969, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (969, 1, 576) torch.bfloat16 - out : (1, 969) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 219 tokens) : 11.2580 - top-1 accuracy : 12/219 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1087]) -Dataloader batch - labels shape: torch.Size([1, 1087]) -Dataloader batch - attn_mask shape: torch.Size([1, 1087]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 902 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1087) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1087) torch.int64 - loss_mask : 313 / 1087 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1087, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1087, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1087, 1, 576) torch.bfloat16 - out : (1, 1087) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.3460 - top-1 accuracy : 6/313 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1522]) -Dataloader batch - labels shape: torch.Size([1, 1522]) -Dataloader batch - attn_mask shape: torch.Size([1, 1522]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 903 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1522) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1522) torch.int64 - loss_mask : 376 / 1522 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1522, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1522, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1522, 1, 576) torch.bfloat16 - out : (1, 1522) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.2661 - top-1 accuracy : 8/376 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1157]) -Dataloader batch - labels shape: torch.Size([1, 1157]) -Dataloader batch - attn_mask shape: torch.Size([1, 1157]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 904 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1157) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1157) torch.int64 - loss_mask : 346 / 1157 tokens (29.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1157, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1157, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1157, 1, 576) torch.bfloat16 - out : (1, 1157) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.3766 - top-1 accuracy : 0/346 = 0.0% - - [2026-04-29 13:55:26] iteration 326/ 500 | consumed samples: 1304 | elapsed time per iteration (ms): 3907.1 | throughput (token/sec/GPU): 1211.9 | learning rate: 5.810496E-05 | global batch size: 4 | lm loss: 1.131513E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1476]) -Dataloader batch - labels shape: torch.Size([1, 1476]) -Dataloader batch - attn_mask shape: torch.Size([1, 1476]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 905 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1476) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1476) torch.int64 - loss_mask : 313 / 1476 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1476, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1476, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1476, 1, 576) torch.bfloat16 - out : (1, 1476) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.1845 - top-1 accuracy : 19/313 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1837]) -Dataloader batch - labels shape: torch.Size([1, 1837]) -Dataloader batch - attn_mask shape: torch.Size([1, 1837]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 906 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1837) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1837) torch.int64 - loss_mask : 516 / 1837 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1837, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1837, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1837, 1, 576) torch.bfloat16 - out : (1, 1837) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 516 tokens) : 11.3626 - top-1 accuracy : 7/516 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1826]) -Dataloader batch - labels shape: torch.Size([1, 1826]) -Dataloader batch - attn_mask shape: torch.Size([1, 1826]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 907 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1826) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1826) torch.int64 - loss_mask : 465 / 1826 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1826, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1826, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1826, 1, 576) torch.bfloat16 - out : (1, 1826) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 465 tokens) : 11.3408 - top-1 accuracy : 9/465 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1153]) -Dataloader batch - labels shape: torch.Size([1, 1153]) -Dataloader batch - attn_mask shape: torch.Size([1, 1153]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 908 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1153) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1153) torch.int64 - loss_mask : 298 / 1153 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1153, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1153, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1153, 1, 576) torch.bfloat16 - out : (1, 1153) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 298 tokens) : 11.3195 - top-1 accuracy : 17/298 = 5.7% - - [2026-04-29 13:55:31] iteration 327/ 500 | consumed samples: 1308 | elapsed time per iteration (ms): 4971.4 | throughput (token/sec/GPU): 1265.6 | learning rate: 5.779687E-05 | global batch size: 4 | lm loss: 1.131317E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1779]) -Dataloader batch - labels shape: torch.Size([1, 1779]) -Dataloader batch - attn_mask shape: torch.Size([1, 1779]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 909 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1779) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1779) torch.int64 - loss_mask : 418 / 1779 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1779, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1779, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1779, 1, 576) torch.bfloat16 - out : (1, 1779) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 418 tokens) : 11.2545 - top-1 accuracy : 7/418 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1804]) -Dataloader batch - labels shape: torch.Size([1, 1804]) -Dataloader batch - attn_mask shape: torch.Size([1, 1804]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 910 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1804) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1804) torch.int64 - loss_mask : 438 / 1804 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1804, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1804, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1804, 1, 576) torch.bfloat16 - out : (1, 1804) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 438 tokens) : 11.2279 - top-1 accuracy : 4/438 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1338]) -Dataloader batch - labels shape: torch.Size([1, 1338]) -Dataloader batch - attn_mask shape: torch.Size([1, 1338]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 911 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1338) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1338) torch.int64 - loss_mask : 353 / 1338 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1338, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1338, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1338, 1, 576) torch.bfloat16 - out : (1, 1338) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.3521 - top-1 accuracy : 2/353 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1647]) -Dataloader batch - labels shape: torch.Size([1, 1647]) -Dataloader batch - attn_mask shape: torch.Size([1, 1647]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 912 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1647) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1647) torch.int64 - loss_mask : 375 / 1647 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1647, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1647, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1647, 1, 576) torch.bfloat16 - out : (1, 1647) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.2388 - top-1 accuracy : 17/375 = 4.5% - - [2026-04-29 13:55:36] iteration 328/ 500 | consumed samples: 1312 | elapsed time per iteration (ms): 5301.0 | throughput (token/sec/GPU): 1239.0 | learning rate: 5.748849E-05 | global batch size: 4 | lm loss: 1.126519E+01 | loss scale: 1.0 | grad norm: 0.201 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1880]) -Dataloader batch - labels shape: torch.Size([1, 1880]) -Dataloader batch - attn_mask shape: torch.Size([1, 1880]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 913 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1880) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1880) torch.int64 - loss_mask : 512 / 1880 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1880, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1880, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1880, 1, 576) torch.bfloat16 - out : (1, 1880) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 512 tokens) : 11.3286 - top-1 accuracy : 1/512 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 914 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 290 / 1364 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.2135 - top-1 accuracy : 31/290 = 10.7% - -Dataloader batch - tokens shape: torch.Size([1, 1731]) -Dataloader batch - labels shape: torch.Size([1, 1731]) -Dataloader batch - attn_mask shape: torch.Size([1, 1731]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 915 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1731) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1731) torch.int64 - loss_mask : 375 / 1731 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1731, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1731, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1731, 1, 576) torch.bfloat16 - out : (1, 1731) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 375 tokens) : 11.2214 - top-1 accuracy : 24/375 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 916 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 381 / 1438 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.3446 - top-1 accuracy : 23/381 = 6.0% - - [2026-04-29 13:55:41] iteration 329/ 500 | consumed samples: 1316 | elapsed time per iteration (ms): 4948.9 | throughput (token/sec/GPU): 1295.9 | learning rate: 5.717984E-05 | global batch size: 4 | lm loss: 1.128527E+01 | loss scale: 1.0 | grad norm: 0.231 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1523]) -Dataloader batch - labels shape: torch.Size([1, 1523]) -Dataloader batch - attn_mask shape: torch.Size([1, 1523]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 917 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1523) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1523) torch.int64 - loss_mask : 416 / 1523 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1523, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1523, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1523, 1, 576) torch.bfloat16 - out : (1, 1523) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.3269 - top-1 accuracy : 5/416 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 918 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 452 / 1520 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 452 tokens) : 11.4017 - top-1 accuracy : 3/452 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1387]) -Dataloader batch - labels shape: torch.Size([1, 1387]) -Dataloader batch - attn_mask shape: torch.Size([1, 1387]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 919 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1387) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1387) torch.int64 - loss_mask : 322 / 1387 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1387, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1387, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1387, 1, 576) torch.bfloat16 - out : (1, 1387) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.2627 - top-1 accuracy : 15/322 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 951]) -Dataloader batch - labels shape: torch.Size([1, 951]) -Dataloader batch - attn_mask shape: torch.Size([1, 951]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 920 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 951) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 951) torch.int64 - loss_mask : 213 / 951 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (951, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (951, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (951, 1, 576) torch.bfloat16 - out : (1, 951) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 213 tokens) : 11.2445 - top-1 accuracy : 4/213 = 1.9% - - [2026-04-29 13:55:46] iteration 330/ 500 | consumed samples: 1320 | elapsed time per iteration (ms): 4346.5 | throughput (token/sec/GPU): 1238.0 | learning rate: 5.687092E-05 | global batch size: 4 | lm loss: 1.132376E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1537]) -Dataloader batch - labels shape: torch.Size([1, 1537]) -Dataloader batch - attn_mask shape: torch.Size([1, 1537]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 921 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1537) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1537) torch.int64 - loss_mask : 385 / 1537 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1537, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1537, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1537, 1, 576) torch.bfloat16 - out : (1, 1537) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.3648 - top-1 accuracy : 7/385 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1442]) -Dataloader batch - labels shape: torch.Size([1, 1442]) -Dataloader batch - attn_mask shape: torch.Size([1, 1442]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 922 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1442) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1442) torch.int64 - loss_mask : 420 / 1442 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1442, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1442, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1442, 1, 576) torch.bfloat16 - out : (1, 1442) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 420 tokens) : 11.3774 - top-1 accuracy : 2/420 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1806]) -Dataloader batch - labels shape: torch.Size([1, 1806]) -Dataloader batch - attn_mask shape: torch.Size([1, 1806]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 923 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1806) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1806) torch.int64 - loss_mask : 444 / 1806 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1806, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1806, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1806, 1, 576) torch.bfloat16 - out : (1, 1806) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 444 tokens) : 11.2784 - top-1 accuracy : 7/444 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1958]) -Dataloader batch - labels shape: torch.Size([1, 1958]) -Dataloader batch - attn_mask shape: torch.Size([1, 1958]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 924 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1958) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1958) torch.int64 - loss_mask : 595 / 1958 tokens (30.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1958, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1958, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1958, 1, 576) torch.bfloat16 - out : (1, 1958) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 595 tokens) : 11.3636 - top-1 accuracy : 2/595 = 0.3% - - [2026-04-29 13:55:51] iteration 331/ 500 | consumed samples: 1324 | elapsed time per iteration (ms): 5033.0 | throughput (token/sec/GPU): 1339.8 | learning rate: 5.656174E-05 | global batch size: 4 | lm loss: 1.134648E+01 | loss scale: 1.0 | grad norm: 0.228 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1411]) -Dataloader batch - labels shape: torch.Size([1, 1411]) -Dataloader batch - attn_mask shape: torch.Size([1, 1411]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 925 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1411) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1411) torch.int64 - loss_mask : 405 / 1411 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1411, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1411, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1411, 1, 576) torch.bfloat16 - out : (1, 1411) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.3565 - top-1 accuracy : 3/405 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1454]) -Dataloader batch - labels shape: torch.Size([1, 1454]) -Dataloader batch - attn_mask shape: torch.Size([1, 1454]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 926 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1454) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1454) torch.int64 - loss_mask : 403 / 1454 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1454, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1454, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1454, 1, 576) torch.bfloat16 - out : (1, 1454) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.3530 - top-1 accuracy : 18/403 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1156]) -Dataloader batch - labels shape: torch.Size([1, 1156]) -Dataloader batch - attn_mask shape: torch.Size([1, 1156]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 927 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1156) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1156) torch.int64 - loss_mask : 217 / 1156 tokens (18.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1156, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1156, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1156, 1, 576) torch.bfloat16 - out : (1, 1156) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 217 tokens) : 11.1400 - top-1 accuracy : 16/217 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1437]) -Dataloader batch - labels shape: torch.Size([1, 1437]) -Dataloader batch - attn_mask shape: torch.Size([1, 1437]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 928 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1437) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1437) torch.int64 - loss_mask : 272 / 1437 tokens (18.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1437, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1437, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1437, 1, 576) torch.bfloat16 - out : (1, 1437) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 272 tokens) : 11.0737 - top-1 accuracy : 9/272 = 3.3% - - [2026-04-29 13:55:55] iteration 332/ 500 | consumed samples: 1328 | elapsed time per iteration (ms): 4513.3 | throughput (token/sec/GPU): 1209.3 | learning rate: 5.625233E-05 | global batch size: 4 | lm loss: 1.125987E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1567]) -Dataloader batch - labels shape: torch.Size([1, 1567]) -Dataloader batch - attn_mask shape: torch.Size([1, 1567]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 929 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1567) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1567) torch.int64 - loss_mask : 335 / 1567 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1567, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1567, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1567, 1, 576) torch.bfloat16 - out : (1, 1567) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.1742 - top-1 accuracy : 29/335 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1181]) -Dataloader batch - labels shape: torch.Size([1, 1181]) -Dataloader batch - attn_mask shape: torch.Size([1, 1181]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 930 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1181) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1181) torch.int64 - loss_mask : 263 / 1181 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1181, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1181, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1181, 1, 576) torch.bfloat16 - out : (1, 1181) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.2300 - top-1 accuracy : 18/263 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1401]) -Dataloader batch - labels shape: torch.Size([1, 1401]) -Dataloader batch - attn_mask shape: torch.Size([1, 1401]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 931 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1401) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1401) torch.int64 - loss_mask : 339 / 1401 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1401, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1401, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1401, 1, 576) torch.bfloat16 - out : (1, 1401) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.3157 - top-1 accuracy : 9/339 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 932 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 350 / 1546 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.1960 - top-1 accuracy : 23/350 = 6.6% - - [2026-04-29 13:56:00] iteration 333/ 500 | consumed samples: 1332 | elapsed time per iteration (ms): 4758.9 | throughput (token/sec/GPU): 1196.7 | learning rate: 5.594269E-05 | global batch size: 4 | lm loss: 1.122881E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1077]) -Dataloader batch - labels shape: torch.Size([1, 1077]) -Dataloader batch - attn_mask shape: torch.Size([1, 1077]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 933 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1077) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1077) torch.int64 - loss_mask : 306 / 1077 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1077, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1077, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1077, 1, 576) torch.bfloat16 - out : (1, 1077) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.3616 - top-1 accuracy : 0/306 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1462]) -Dataloader batch - labels shape: torch.Size([1, 1462]) -Dataloader batch - attn_mask shape: torch.Size([1, 1462]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 934 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1462) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1462) torch.int64 - loss_mask : 302 / 1462 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1462, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1462, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1462, 1, 576) torch.bfloat16 - out : (1, 1462) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 302 tokens) : 11.0565 - top-1 accuracy : 31/302 = 10.3% - -Dataloader batch - tokens shape: torch.Size([1, 1536]) -Dataloader batch - labels shape: torch.Size([1, 1536]) -Dataloader batch - attn_mask shape: torch.Size([1, 1536]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 935 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1536) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1536) torch.int64 - loss_mask : 380 / 1536 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1536, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1536, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1536, 1, 576) torch.bfloat16 - out : (1, 1536) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 380 tokens) : 11.2979 - top-1 accuracy : 1/380 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 936 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 313 / 1386 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2505 - top-1 accuracy : 24/313 = 7.7% - - [2026-04-29 13:56:04] iteration 334/ 500 | consumed samples: 1336 | elapsed time per iteration (ms): 4546.5 | throughput (token/sec/GPU): 1201.1 | learning rate: 5.563283E-05 | global batch size: 4 | lm loss: 1.124543E+01 | loss scale: 1.0 | grad norm: 0.214 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1640]) -Dataloader batch - labels shape: torch.Size([1, 1640]) -Dataloader batch - attn_mask shape: torch.Size([1, 1640]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 937 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1640) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1640) torch.int64 - loss_mask : 374 / 1640 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1640, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1640, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1640, 1, 576) torch.bfloat16 - out : (1, 1640) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.2883 - top-1 accuracy : 25/374 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1882]) -Dataloader batch - labels shape: torch.Size([1, 1882]) -Dataloader batch - attn_mask shape: torch.Size([1, 1882]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 938 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1882) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1882) torch.int64 - loss_mask : 531 / 1882 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1882, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1882, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1882, 1, 576) torch.bfloat16 - out : (1, 1882) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 531 tokens) : 11.3494 - top-1 accuracy : 20/531 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1595]) -Dataloader batch - labels shape: torch.Size([1, 1595]) -Dataloader batch - attn_mask shape: torch.Size([1, 1595]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 939 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1595) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1595) torch.int64 - loss_mask : 401 / 1595 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1595, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1595, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1595, 1, 576) torch.bfloat16 - out : (1, 1595) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 401 tokens) : 11.2730 - top-1 accuracy : 11/401 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 940 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 345 / 1436 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2492 - top-1 accuracy : 17/345 = 4.9% - - [2026-04-29 13:56:10] iteration 335/ 500 | consumed samples: 1340 | elapsed time per iteration (ms): 5187.8 | throughput (token/sec/GPU): 1263.2 | learning rate: 5.532277E-05 | global batch size: 4 | lm loss: 1.129604E+01 | loss scale: 1.0 | grad norm: 0.211 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1154]) -Dataloader batch - labels shape: torch.Size([1, 1154]) -Dataloader batch - attn_mask shape: torch.Size([1, 1154]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 941 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1154) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1154) torch.int64 - loss_mask : 321 / 1154 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1154, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1154, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1154, 1, 576) torch.bfloat16 - out : (1, 1154) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.3122 - top-1 accuracy : 4/321 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1735]) -Dataloader batch - labels shape: torch.Size([1, 1735]) -Dataloader batch - attn_mask shape: torch.Size([1, 1735]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 942 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1735) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1735) torch.int64 - loss_mask : 363 / 1735 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1735, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1735, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1735, 1, 576) torch.bfloat16 - out : (1, 1735) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2142 - top-1 accuracy : 10/363 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1521]) -Dataloader batch - labels shape: torch.Size([1, 1521]) -Dataloader batch - attn_mask shape: torch.Size([1, 1521]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 943 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1521) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1521) torch.int64 - loss_mask : 362 / 1521 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1521, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1521, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1521, 1, 576) torch.bfloat16 - out : (1, 1521) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2723 - top-1 accuracy : 12/362 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1033]) -Dataloader batch - labels shape: torch.Size([1, 1033]) -Dataloader batch - attn_mask shape: torch.Size([1, 1033]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 944 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1033) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1033) torch.int64 - loss_mask : 259 / 1033 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1033, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1033, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1033, 1, 576) torch.bfloat16 - out : (1, 1033) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 259 tokens) : 11.3653 - top-1 accuracy : 15/259 = 5.8% - - [2026-04-29 13:56:14] iteration 336/ 500 | consumed samples: 1344 | elapsed time per iteration (ms): 4379.4 | throughput (token/sec/GPU): 1242.9 | learning rate: 5.501252E-05 | global batch size: 4 | lm loss: 1.128441E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1821]) -Dataloader batch - labels shape: torch.Size([1, 1821]) -Dataloader batch - attn_mask shape: torch.Size([1, 1821]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 945 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1821) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1821) torch.int64 - loss_mask : 436 / 1821 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1821, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1821, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1821, 1, 576) torch.bfloat16 - out : (1, 1821) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.2873 - top-1 accuracy : 5/436 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1502]) -Dataloader batch - labels shape: torch.Size([1, 1502]) -Dataloader batch - attn_mask shape: torch.Size([1, 1502]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 946 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1502) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1502) torch.int64 - loss_mask : 386 / 1502 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1502, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1502, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1502, 1, 576) torch.bfloat16 - out : (1, 1502) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2078 - top-1 accuracy : 22/386 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 947 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 307 / 1487 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 307 tokens) : 11.1836 - top-1 accuracy : 25/307 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 979]) -Dataloader batch - labels shape: torch.Size([1, 979]) -Dataloader batch - attn_mask shape: torch.Size([1, 979]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 948 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 979) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 979) torch.int64 - loss_mask : 218 / 979 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (979, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (979, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (979, 1, 576) torch.bfloat16 - out : (1, 979) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 218 tokens) : 11.2000 - top-1 accuracy : 1/218 = 0.5% - - [2026-04-29 13:56:19] iteration 337/ 500 | consumed samples: 1348 | elapsed time per iteration (ms): 4568.8 | throughput (token/sec/GPU): 1267.1 | learning rate: 5.470209E-05 | global batch size: 4 | lm loss: 1.122676E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1477]) -Dataloader batch - labels shape: torch.Size([1, 1477]) -Dataloader batch - attn_mask shape: torch.Size([1, 1477]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 949 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1477) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1477) torch.int64 - loss_mask : 432 / 1477 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1477, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1477, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1477, 1, 576) torch.bfloat16 - out : (1, 1477) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 432 tokens) : 11.4038 - top-1 accuracy : 20/432 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 950 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 400 / 1532 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.3064 - top-1 accuracy : 1/400 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 951 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1519) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1519) torch.int64 - loss_mask : 394 / 1519 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1519, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1519, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1519, 1, 576) torch.bfloat16 - out : (1, 1519) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3194 - top-1 accuracy : 12/394 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 952 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 410 / 1468 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.2888 - top-1 accuracy : 3/410 = 0.7% - - [2026-04-29 13:56:23] iteration 338/ 500 | consumed samples: 1352 | elapsed time per iteration (ms): 4824.4 | throughput (token/sec/GPU): 1242.9 | learning rate: 5.439149E-05 | global batch size: 4 | lm loss: 1.133085E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 953 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 321 / 1468 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2213 - top-1 accuracy : 14/321 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1495]) -Dataloader batch - labels shape: torch.Size([1, 1495]) -Dataloader batch - attn_mask shape: torch.Size([1, 1495]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 954 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1495) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1495) torch.int64 - loss_mask : 362 / 1495 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1495, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1495, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1495, 1, 576) torch.bfloat16 - out : (1, 1495) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2927 - top-1 accuracy : 23/362 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 974]) -Dataloader batch - labels shape: torch.Size([1, 974]) -Dataloader batch - attn_mask shape: torch.Size([1, 974]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 955 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 974) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 974) torch.int64 - loss_mask : 203 / 974 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (974, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (974, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (974, 1, 576) torch.bfloat16 - out : (1, 974) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 203 tokens) : 11.1417 - top-1 accuracy : 9/203 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1720]) -Dataloader batch - labels shape: torch.Size([1, 1720]) -Dataloader batch - attn_mask shape: torch.Size([1, 1720]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 956 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1720) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1720) torch.int64 - loss_mask : 394 / 1720 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1720, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1720, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1720, 1, 576) torch.bfloat16 - out : (1, 1720) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.2494 - top-1 accuracy : 14/394 = 3.6% - - [2026-04-29 13:56:28] iteration 339/ 500 | consumed samples: 1356 | elapsed time per iteration (ms): 4645.0 | throughput (token/sec/GPU): 1217.9 | learning rate: 5.408074E-05 | global batch size: 4 | lm loss: 1.123755E+01 | loss scale: 1.0 | grad norm: 0.240 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 957 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 392 / 1468 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3491 - top-1 accuracy : 16/392 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1762]) -Dataloader batch - labels shape: torch.Size([1, 1762]) -Dataloader batch - attn_mask shape: torch.Size([1, 1762]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 958 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1762) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1762) torch.int64 - loss_mask : 404 / 1762 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1762, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1762, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1762, 1, 576) torch.bfloat16 - out : (1, 1762) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.2998 - top-1 accuracy : 26/404 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1191]) -Dataloader batch - labels shape: torch.Size([1, 1191]) -Dataloader batch - attn_mask shape: torch.Size([1, 1191]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 959 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1191) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1191) torch.int64 - loss_mask : 297 / 1191 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1191, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1191, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1191, 1, 576) torch.bfloat16 - out : (1, 1191) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.3176 - top-1 accuracy : 3/297 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1705]) -Dataloader batch - labels shape: torch.Size([1, 1705]) -Dataloader batch - attn_mask shape: torch.Size([1, 1705]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 960 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1705) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1705) torch.int64 - loss_mask : 355 / 1705 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1705, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1705, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1705, 1, 576) torch.bfloat16 - out : (1, 1705) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.1939 - top-1 accuracy : 22/355 = 6.2% - - [2026-04-29 13:56:33] iteration 340/ 500 | consumed samples: 1360 | elapsed time per iteration (ms): 4995.5 | throughput (token/sec/GPU): 1226.3 | learning rate: 5.376985E-05 | global batch size: 4 | lm loss: 1.129081E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4979.61, 4979.61) - forward-compute ................................: (3702.41, 3702.41) - backward-compute ...............................: (1274.45, 1274.45) - batch-generator ................................: (439.64, 439.64) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.05, 0.05) - all-grads-sync .................................: (0.74, 0.74) - params-all-gather ..............................: (0.23, 0.23) - optimizer-copy-to-main-grad ....................: (0.15, 0.15) - optimizer-clip-main-grad .......................: (0.68, 0.68) - optimizer-count-zeros ..........................: (0.03, 0.03) - optimizer-inner-step ...........................: (1.45, 1.45) - optimizer-copy-main-to-model-params ............: (0.26, 0.26) - optimizer ......................................: (3.44, 3.44) -Dataloader batch - tokens shape: torch.Size([1, 1687]) -Dataloader batch - labels shape: torch.Size([1, 1687]) -Dataloader batch - attn_mask shape: torch.Size([1, 1687]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 961 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1687) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1687) torch.int64 - loss_mask : 387 / 1687 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1687, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1687, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1687, 1, 576) torch.bfloat16 - out : (1, 1687) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2173 - top-1 accuracy : 36/387 = 9.3% - -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 962 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 355 / 1722 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2223 - top-1 accuracy : 8/355 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 963 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 434 / 1436 tokens (30.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 434 tokens) : 11.3800 - top-1 accuracy : 13/434 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1465]) -Dataloader batch - labels shape: torch.Size([1, 1465]) -Dataloader batch - attn_mask shape: torch.Size([1, 1465]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 964 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1465) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1465) torch.int64 - loss_mask : 414 / 1465 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1465, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1465, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1465, 1, 576) torch.bfloat16 - out : (1, 1465) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 414 tokens) : 11.3845 - top-1 accuracy : 3/414 = 0.7% - - [2026-04-29 13:56:38] iteration 341/ 500 | consumed samples: 1364 | elapsed time per iteration (ms): 4972.6 | throughput (token/sec/GPU): 1269.0 | learning rate: 5.345883E-05 | global batch size: 4 | lm loss: 1.130637E+01 | loss scale: 1.0 | grad norm: 0.173 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1826]) -Dataloader batch - labels shape: torch.Size([1, 1826]) -Dataloader batch - attn_mask shape: torch.Size([1, 1826]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 965 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1826) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1826) torch.int64 - loss_mask : 465 / 1826 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1826, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1826, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1826, 1, 576) torch.bfloat16 - out : (1, 1826) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 465 tokens) : 11.3305 - top-1 accuracy : 23/465 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1413]) -Dataloader batch - labels shape: torch.Size([1, 1413]) -Dataloader batch - attn_mask shape: torch.Size([1, 1413]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 966 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1413) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1413) torch.int64 - loss_mask : 332 / 1413 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1413, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1413, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1413, 1, 576) torch.bfloat16 - out : (1, 1413) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.2784 - top-1 accuracy : 19/332 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1482]) -Dataloader batch - labels shape: torch.Size([1, 1482]) -Dataloader batch - attn_mask shape: torch.Size([1, 1482]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 967 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1482) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1482) torch.int64 - loss_mask : 456 / 1482 tokens (30.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1482, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1482, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1482, 1, 576) torch.bfloat16 - out : (1, 1482) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 456 tokens) : 11.4199 - top-1 accuracy : 8/456 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1871]) -Dataloader batch - labels shape: torch.Size([1, 1871]) -Dataloader batch - attn_mask shape: torch.Size([1, 1871]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 968 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1871) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1871) torch.int64 - loss_mask : 520 / 1871 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1871, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1871, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1871, 1, 576) torch.bfloat16 - out : (1, 1871) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 520 tokens) : 11.4320 - top-1 accuracy : 20/520 = 3.8% - - [2026-04-29 13:56:43] iteration 342/ 500 | consumed samples: 1368 | elapsed time per iteration (ms): 5140.9 | throughput (token/sec/GPU): 1282.3 | learning rate: 5.314768E-05 | global batch size: 4 | lm loss: 1.137350E+01 | loss scale: 1.0 | grad norm: 0.175 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1264]) -Dataloader batch - labels shape: torch.Size([1, 1264]) -Dataloader batch - attn_mask shape: torch.Size([1, 1264]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 969 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1264) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1264) torch.int64 - loss_mask : 342 / 1264 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1264, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1264, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1264, 1, 576) torch.bfloat16 - out : (1, 1264) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.3185 - top-1 accuracy : 0/342 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 970 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 436 / 1496 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.3827 - top-1 accuracy : 2/436 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1084]) -Dataloader batch - labels shape: torch.Size([1, 1084]) -Dataloader batch - attn_mask shape: torch.Size([1, 1084]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 971 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1084) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1084) torch.int64 - loss_mask : 275 / 1084 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1084, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1084, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1084, 1, 576) torch.bfloat16 - out : (1, 1084) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 275 tokens) : 11.2849 - top-1 accuracy : 16/275 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1859]) -Dataloader batch - labels shape: torch.Size([1, 1859]) -Dataloader batch - attn_mask shape: torch.Size([1, 1859]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 972 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1859) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1859) torch.int64 - loss_mask : 517 / 1859 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1859, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1859, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1859, 1, 576) torch.bfloat16 - out : (1, 1859) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 517 tokens) : 11.3247 - top-1 accuracy : 14/517 = 2.7% - - [2026-04-29 13:56:48] iteration 343/ 500 | consumed samples: 1372 | elapsed time per iteration (ms): 4422.8 | throughput (token/sec/GPU): 1289.4 | learning rate: 5.283644E-05 | global batch size: 4 | lm loss: 1.133250E+01 | loss scale: 1.0 | grad norm: 0.209 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1049]) -Dataloader batch - labels shape: torch.Size([1, 1049]) -Dataloader batch - attn_mask shape: torch.Size([1, 1049]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 973 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1049) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1049) torch.int64 - loss_mask : 270 / 1049 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1049, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1049, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1049, 1, 576) torch.bfloat16 - out : (1, 1049) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 270 tokens) : 11.3136 - top-1 accuracy : 2/270 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1022]) -Dataloader batch - labels shape: torch.Size([1, 1022]) -Dataloader batch - attn_mask shape: torch.Size([1, 1022]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 974 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1022) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1022) torch.int64 - loss_mask : 254 / 1022 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1022, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1022, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1022, 1, 576) torch.bfloat16 - out : (1, 1022) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 254 tokens) : 11.3475 - top-1 accuracy : 15/254 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1507]) -Dataloader batch - labels shape: torch.Size([1, 1507]) -Dataloader batch - attn_mask shape: torch.Size([1, 1507]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 975 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1507) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1507) torch.int64 - loss_mask : 320 / 1507 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1507, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1507, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1507, 1, 576) torch.bfloat16 - out : (1, 1507) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.1346 - top-1 accuracy : 26/320 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1545]) -Dataloader batch - labels shape: torch.Size([1, 1545]) -Dataloader batch - attn_mask shape: torch.Size([1, 1545]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 976 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1545) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1545) torch.int64 - loss_mask : 389 / 1545 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1545, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1545, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1545, 1, 576) torch.bfloat16 - out : (1, 1545) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.2404 - top-1 accuracy : 3/389 = 0.8% - - [2026-04-29 13:56:52] iteration 344/ 500 | consumed samples: 1376 | elapsed time per iteration (ms): 4451.2 | throughput (token/sec/GPU): 1150.9 | learning rate: 5.252510E-05 | global batch size: 4 | lm loss: 1.125102E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 999]) -Dataloader batch - labels shape: torch.Size([1, 999]) -Dataloader batch - attn_mask shape: torch.Size([1, 999]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 977 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 999) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 999) torch.int64 - loss_mask : 227 / 999 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (999, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (999, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (999, 1, 576) torch.bfloat16 - out : (1, 999) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 227 tokens) : 11.1883 - top-1 accuracy : 1/227 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1623]) -Dataloader batch - labels shape: torch.Size([1, 1623]) -Dataloader batch - attn_mask shape: torch.Size([1, 1623]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 978 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1623) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1623) torch.int64 - loss_mask : 383 / 1623 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1623, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1623, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1623, 1, 576) torch.bfloat16 - out : (1, 1623) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.2411 - top-1 accuracy : 16/383 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1495]) -Dataloader batch - labels shape: torch.Size([1, 1495]) -Dataloader batch - attn_mask shape: torch.Size([1, 1495]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 979 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1495) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1495) torch.int64 - loss_mask : 437 / 1495 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1495, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1495, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1495, 1, 576) torch.bfloat16 - out : (1, 1495) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 437 tokens) : 11.4063 - top-1 accuracy : 4/437 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1789]) -Dataloader batch - labels shape: torch.Size([1, 1789]) -Dataloader batch - attn_mask shape: torch.Size([1, 1789]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 980 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1789) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1789) torch.int64 - loss_mask : 426 / 1789 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1789, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1789, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1789, 1, 576) torch.bfloat16 - out : (1, 1789) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.2884 - top-1 accuracy : 11/426 = 2.6% - - [2026-04-29 13:56:57] iteration 345/ 500 | consumed samples: 1380 | elapsed time per iteration (ms): 5079.6 | throughput (token/sec/GPU): 1162.7 | learning rate: 5.221368E-05 | global batch size: 4 | lm loss: 1.129565E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1080]) -Dataloader batch - labels shape: torch.Size([1, 1080]) -Dataloader batch - attn_mask shape: torch.Size([1, 1080]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 981 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1080) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1080) torch.int64 - loss_mask : 251 / 1080 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1080, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1080, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1080, 1, 576) torch.bfloat16 - out : (1, 1080) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2192 - top-1 accuracy : 23/251 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1512]) -Dataloader batch - labels shape: torch.Size([1, 1512]) -Dataloader batch - attn_mask shape: torch.Size([1, 1512]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 982 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1512) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1512) torch.int64 - loss_mask : 332 / 1512 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1512, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1512, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1512, 1, 576) torch.bfloat16 - out : (1, 1512) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.1931 - top-1 accuracy : 20/332 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1900]) -Dataloader batch - labels shape: torch.Size([1, 1900]) -Dataloader batch - attn_mask shape: torch.Size([1, 1900]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 983 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1900) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1900) torch.int64 - loss_mask : 534 / 1900 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1900, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1900, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1900, 1, 576) torch.bfloat16 - out : (1, 1900) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 534 tokens) : 11.3531 - top-1 accuracy : 6/534 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1204]) -Dataloader batch - labels shape: torch.Size([1, 1204]) -Dataloader batch - attn_mask shape: torch.Size([1, 1204]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 984 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1204) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1204) torch.int64 - loss_mask : 267 / 1204 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1204, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1204, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1204, 1, 576) torch.bfloat16 - out : (1, 1204) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 267 tokens) : 11.2612 - top-1 accuracy : 9/267 = 3.4% - - [2026-04-29 13:57:02] iteration 346/ 500 | consumed samples: 1384 | elapsed time per iteration (ms): 4824.0 | throughput (token/sec/GPU): 1180.8 | learning rate: 5.190220E-05 | global batch size: 4 | lm loss: 1.127270E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1582]) -Dataloader batch - labels shape: torch.Size([1, 1582]) -Dataloader batch - attn_mask shape: torch.Size([1, 1582]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 985 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1582) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1582) torch.int64 - loss_mask : 366 / 1582 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1582, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1582, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1582, 1, 576) torch.bfloat16 - out : (1, 1582) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2671 - top-1 accuracy : 10/366 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1372]) -Dataloader batch - labels shape: torch.Size([1, 1372]) -Dataloader batch - attn_mask shape: torch.Size([1, 1372]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 986 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1372) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1372) torch.int64 - loss_mask : 391 / 1372 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1372, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1372, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1372, 1, 576) torch.bfloat16 - out : (1, 1372) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3973 - top-1 accuracy : 5/391 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 987 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 311 / 1400 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2031 - top-1 accuracy : 15/311 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 944]) -Dataloader batch - labels shape: torch.Size([1, 944]) -Dataloader batch - attn_mask shape: torch.Size([1, 944]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 988 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 944) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 944) torch.int64 - loss_mask : 215 / 944 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (944, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (944, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (944, 1, 576) torch.bfloat16 - out : (1, 944) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 215 tokens) : 11.1965 - top-1 accuracy : 15/215 = 7.0% - - [2026-04-29 13:57:06] iteration 347/ 500 | consumed samples: 1388 | elapsed time per iteration (ms): 4380.7 | throughput (token/sec/GPU): 1209.4 | learning rate: 5.159065E-05 | global batch size: 4 | lm loss: 1.127941E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1893]) -Dataloader batch - labels shape: torch.Size([1, 1893]) -Dataloader batch - attn_mask shape: torch.Size([1, 1893]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 989 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1893) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1893) torch.int64 - loss_mask : 543 / 1893 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1893, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1893, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1893, 1, 576) torch.bfloat16 - out : (1, 1893) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 543 tokens) : 11.3796 - top-1 accuracy : 2/543 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1180]) -Dataloader batch - labels shape: torch.Size([1, 1180]) -Dataloader batch - attn_mask shape: torch.Size([1, 1180]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 990 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1180) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1180) torch.int64 - loss_mask : 331 / 1180 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1180, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1180, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1180, 1, 576) torch.bfloat16 - out : (1, 1180) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.3708 - top-1 accuracy : 6/331 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1662]) -Dataloader batch - labels shape: torch.Size([1, 1662]) -Dataloader batch - attn_mask shape: torch.Size([1, 1662]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 991 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1662) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1662) torch.int64 - loss_mask : 391 / 1662 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1662, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1662, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1662, 1, 576) torch.bfloat16 - out : (1, 1662) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2758 - top-1 accuracy : 10/391 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1960]) -Dataloader batch - labels shape: torch.Size([1, 1960]) -Dataloader batch - attn_mask shape: torch.Size([1, 1960]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 992 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1960) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1960) torch.int64 - loss_mask : 583 / 1960 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1960, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1960, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1960, 1, 576) torch.bfloat16 - out : (1, 1960) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 583 tokens) : 11.3441 - top-1 accuracy : 2/583 = 0.3% - - [2026-04-29 13:57:11] iteration 348/ 500 | consumed samples: 1392 | elapsed time per iteration (ms): 5141.9 | throughput (token/sec/GPU): 1302.1 | learning rate: 5.127907E-05 | global batch size: 4 | lm loss: 1.134486E+01 | loss scale: 1.0 | grad norm: 0.198 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 993 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 352 / 1722 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.1740 - top-1 accuracy : 34/352 = 9.7% - -Dataloader batch - tokens shape: torch.Size([1, 1054]) -Dataloader batch - labels shape: torch.Size([1, 1054]) -Dataloader batch - attn_mask shape: torch.Size([1, 1054]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 994 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1054) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1054) torch.int64 - loss_mask : 268 / 1054 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1054, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1054, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1054, 1, 576) torch.bfloat16 - out : (1, 1054) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 268 tokens) : 11.2669 - top-1 accuracy : 7/268 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1358]) -Dataloader batch - labels shape: torch.Size([1, 1358]) -Dataloader batch - attn_mask shape: torch.Size([1, 1358]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 995 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1358) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1358) torch.int64 - loss_mask : 387 / 1358 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1358, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1358, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1358, 1, 576) torch.bfloat16 - out : (1, 1358) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.3668 - top-1 accuracy : 1/387 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1636]) -Dataloader batch - labels shape: torch.Size([1, 1636]) -Dataloader batch - attn_mask shape: torch.Size([1, 1636]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 996 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1636) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1636) torch.int64 - loss_mask : 400 / 1636 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1636, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1636, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1636, 1, 576) torch.bfloat16 - out : (1, 1636) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.2860 - top-1 accuracy : 10/400 = 2.5% - - [2026-04-29 13:57:17] iteration 349/ 500 | consumed samples: 1396 | elapsed time per iteration (ms): 5237.8 | throughput (token/sec/GPU): 1101.6 | learning rate: 5.096745E-05 | global batch size: 4 | lm loss: 1.127656E+01 | loss scale: 1.0 | grad norm: 0.217 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1163]) -Dataloader batch - labels shape: torch.Size([1, 1163]) -Dataloader batch - attn_mask shape: torch.Size([1, 1163]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 997 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1163) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1163) torch.int64 - loss_mask : 320 / 1163 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1163, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1163, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1163, 1, 576) torch.bfloat16 - out : (1, 1163) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.3239 - top-1 accuracy : 3/320 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 998 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 465 / 1431 tokens (32.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 465 tokens) : 11.4584 - top-1 accuracy : 5/465 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1767]) -Dataloader batch - labels shape: torch.Size([1, 1767]) -Dataloader batch - attn_mask shape: torch.Size([1, 1767]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 999 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1767) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1767) torch.int64 - loss_mask : 409 / 1767 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1767, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1767, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1767, 1, 576) torch.bfloat16 - out : (1, 1767) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 409 tokens) : 11.2964 - top-1 accuracy : 9/409 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1046]) -Dataloader batch - labels shape: torch.Size([1, 1046]) -Dataloader batch - attn_mask shape: torch.Size([1, 1046]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1000 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1046) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1046) torch.int64 - loss_mask : 251 / 1046 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1046, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1046, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1046, 1, 576) torch.bfloat16 - out : (1, 1046) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2705 - top-1 accuracy : 13/251 = 5.2% - - [2026-04-29 13:57:21] iteration 350/ 500 | consumed samples: 1400 | elapsed time per iteration (ms): 4217.0 | throughput (token/sec/GPU): 1282.2 | learning rate: 5.065582E-05 | global batch size: 4 | lm loss: 1.135012E+01 | loss scale: 1.0 | grad norm: 0.173 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 350 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 350 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 350 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2414.00, 2414.00) -Dataloader batch - tokens shape: torch.Size([1, 1432]) -Dataloader batch - labels shape: torch.Size([1, 1432]) -Dataloader batch - attn_mask shape: torch.Size([1, 1432]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1001 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1432) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1432) torch.int64 - loss_mask : 361 / 1432 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1432, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1432, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1432, 1, 576) torch.bfloat16 - out : (1, 1432) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.2519 - top-1 accuracy : 19/361 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1002 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 321 / 1374 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2630 - top-1 accuracy : 14/321 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1070]) -Dataloader batch - labels shape: torch.Size([1, 1070]) -Dataloader batch - attn_mask shape: torch.Size([1, 1070]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1003 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1070) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1070) torch.int64 - loss_mask : 305 / 1070 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1070, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1070, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1070, 1, 576) torch.bfloat16 - out : (1, 1070) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.3676 - top-1 accuracy : 0/305 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1836]) -Dataloader batch - labels shape: torch.Size([1, 1836]) -Dataloader batch - attn_mask shape: torch.Size([1, 1836]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1004 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1836) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1836) torch.int64 - loss_mask : 471 / 1836 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1836, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1836, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1836, 1, 576) torch.bfloat16 - out : (1, 1836) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 471 tokens) : 11.2949 - top-1 accuracy : 8/471 = 1.7% - - [2026-04-29 13:57:28] iteration 351/ 500 | consumed samples: 1404 | elapsed time per iteration (ms): 4616.5 | throughput (token/sec/GPU): 1237.3 | learning rate: 5.034418E-05 | global batch size: 4 | lm loss: 1.129241E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1397]) -Dataloader batch - labels shape: torch.Size([1, 1397]) -Dataloader batch - attn_mask shape: torch.Size([1, 1397]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1005 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1397) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1397) torch.int64 - loss_mask : 423 / 1397 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1397, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1397, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1397, 1, 576) torch.bfloat16 - out : (1, 1397) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 423 tokens) : 11.4415 - top-1 accuracy : 1/423 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1756]) -Dataloader batch - labels shape: torch.Size([1, 1756]) -Dataloader batch - attn_mask shape: torch.Size([1, 1756]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1006 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1756) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1756) torch.int64 - loss_mask : 411 / 1756 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1756, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1756, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1756, 1, 576) torch.bfloat16 - out : (1, 1756) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.2645 - top-1 accuracy : 32/411 = 7.8% - -Dataloader batch - tokens shape: torch.Size([1, 1806]) -Dataloader batch - labels shape: torch.Size([1, 1806]) -Dataloader batch - attn_mask shape: torch.Size([1, 1806]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1007 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1806) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1806) torch.int64 - loss_mask : 442 / 1806 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1806, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1806, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1806, 1, 576) torch.bfloat16 - out : (1, 1806) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 442 tokens) : 11.2491 - top-1 accuracy : 20/442 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1608]) -Dataloader batch - labels shape: torch.Size([1, 1608]) -Dataloader batch - attn_mask shape: torch.Size([1, 1608]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1008 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1608) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1608) torch.int64 - loss_mask : 400 / 1608 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1608, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1608, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1608, 1, 576) torch.bfloat16 - out : (1, 1608) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.3232 - top-1 accuracy : 11/400 = 2.8% - - [2026-04-29 13:57:33] iteration 352/ 500 | consumed samples: 1408 | elapsed time per iteration (ms): 5114.8 | throughput (token/sec/GPU): 1283.9 | learning rate: 5.003254E-05 | global batch size: 4 | lm loss: 1.131911E+01 | loss scale: 1.0 | grad norm: 0.193 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1553]) -Dataloader batch - labels shape: torch.Size([1, 1553]) -Dataloader batch - attn_mask shape: torch.Size([1, 1553]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1009 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1553) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1553) torch.int64 - loss_mask : 418 / 1553 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1553, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1553, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1553, 1, 576) torch.bfloat16 - out : (1, 1553) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 418 tokens) : 11.2991 - top-1 accuracy : 7/418 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1788]) -Dataloader batch - labels shape: torch.Size([1, 1788]) -Dataloader batch - attn_mask shape: torch.Size([1, 1788]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1010 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1788) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1788) torch.int64 - loss_mask : 427 / 1788 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1788, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1788, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1788, 1, 576) torch.bfloat16 - out : (1, 1788) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 427 tokens) : 11.2867 - top-1 accuracy : 24/427 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1111]) -Dataloader batch - labels shape: torch.Size([1, 1111]) -Dataloader batch - attn_mask shape: torch.Size([1, 1111]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1011 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1111) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1111) torch.int64 - loss_mask : 288 / 1111 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1111, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1111, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1111, 1, 576) torch.bfloat16 - out : (1, 1111) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 288 tokens) : 11.3084 - top-1 accuracy : 19/288 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1012 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 484 / 1468 tokens (33.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 484 tokens) : 11.4655 - top-1 accuracy : 2/484 = 0.4% - - [2026-04-29 13:57:38] iteration 353/ 500 | consumed samples: 1412 | elapsed time per iteration (ms): 4557.1 | throughput (token/sec/GPU): 1299.1 | learning rate: 4.972093E-05 | global batch size: 4 | lm loss: 1.134729E+01 | loss scale: 1.0 | grad norm: 0.214 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1724]) -Dataloader batch - labels shape: torch.Size([1, 1724]) -Dataloader batch - attn_mask shape: torch.Size([1, 1724]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1013 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1724) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1724) torch.int64 - loss_mask : 351 / 1724 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1724, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1724, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1724, 1, 576) torch.bfloat16 - out : (1, 1724) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.1848 - top-1 accuracy : 21/351 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1401]) -Dataloader batch - labels shape: torch.Size([1, 1401]) -Dataloader batch - attn_mask shape: torch.Size([1, 1401]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1014 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1401) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1401) torch.int64 - loss_mask : 334 / 1401 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1401, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1401, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1401, 1, 576) torch.bfloat16 - out : (1, 1401) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.2648 - top-1 accuracy : 16/334 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1882]) -Dataloader batch - labels shape: torch.Size([1, 1882]) -Dataloader batch - attn_mask shape: torch.Size([1, 1882]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1015 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1882) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1882) torch.int64 - loss_mask : 523 / 1882 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1882, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1882, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1882, 1, 576) torch.bfloat16 - out : (1, 1882) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 523 tokens) : 11.3413 - top-1 accuracy : 11/523 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1560]) -Dataloader batch - labels shape: torch.Size([1, 1560]) -Dataloader batch - attn_mask shape: torch.Size([1, 1560]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1016 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1560) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1560) torch.int64 - loss_mask : 350 / 1560 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1560, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1560, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1560, 1, 576) torch.bfloat16 - out : (1, 1560) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.2323 - top-1 accuracy : 15/350 = 4.3% - - [2026-04-29 13:57:43] iteration 354/ 500 | consumed samples: 1416 | elapsed time per iteration (ms): 5134.4 | throughput (token/sec/GPU): 1279.0 | learning rate: 4.940934E-05 | global batch size: 4 | lm loss: 1.126514E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1017 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 397 / 1542 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.3078 - top-1 accuracy : 12/397 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1156]) -Dataloader batch - labels shape: torch.Size([1, 1156]) -Dataloader batch - attn_mask shape: torch.Size([1, 1156]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1018 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1156) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1156) torch.int64 - loss_mask : 311 / 1156 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1156, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1156, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1156, 1, 576) torch.bfloat16 - out : (1, 1156) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.3059 - top-1 accuracy : 9/311 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1837]) -Dataloader batch - labels shape: torch.Size([1, 1837]) -Dataloader batch - attn_mask shape: torch.Size([1, 1837]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1019 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1837) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1837) torch.int64 - loss_mask : 481 / 1837 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1837, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1837, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1837, 1, 576) torch.bfloat16 - out : (1, 1837) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 481 tokens) : 11.3186 - top-1 accuracy : 23/481 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1544]) -Dataloader batch - labels shape: torch.Size([1, 1544]) -Dataloader batch - attn_mask shape: torch.Size([1, 1544]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1020 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1544) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1544) torch.int64 - loss_mask : 386 / 1544 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1544, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1544, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1544, 1, 576) torch.bfloat16 - out : (1, 1544) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2710 - top-1 accuracy : 20/386 = 5.2% - - [2026-04-29 13:57:47] iteration 355/ 500 | consumed samples: 1420 | elapsed time per iteration (ms): 4775.2 | throughput (token/sec/GPU): 1273.0 | learning rate: 4.909780E-05 | global batch size: 4 | lm loss: 1.130169E+01 | loss scale: 1.0 | grad norm: 0.248 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 1021 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 381 / 1388 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.3582 - top-1 accuracy : 1/381 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1806]) -Dataloader batch - labels shape: torch.Size([1, 1806]) -Dataloader batch - attn_mask shape: torch.Size([1, 1806]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1022 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1806) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1806) torch.int64 - loss_mask : 454 / 1806 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1806, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1806, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1806, 1, 576) torch.bfloat16 - out : (1, 1806) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 454 tokens) : 11.2647 - top-1 accuracy : 11/454 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1081]) -Dataloader batch - labels shape: torch.Size([1, 1081]) -Dataloader batch - attn_mask shape: torch.Size([1, 1081]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1023 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1081) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1081) torch.int64 - loss_mask : 274 / 1081 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1081, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1081, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1081, 1, 576) torch.bfloat16 - out : (1, 1081) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 274 tokens) : 11.3217 - top-1 accuracy : 4/274 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1368]) -Dataloader batch - labels shape: torch.Size([1, 1368]) -Dataloader batch - attn_mask shape: torch.Size([1, 1368]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1024 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1368) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1368) torch.int64 - loss_mask : 283 / 1368 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1368, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1368, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1368, 1, 576) torch.bfloat16 - out : (1, 1368) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 283 tokens) : 11.1985 - top-1 accuracy : 29/283 = 10.2% - - [2026-04-29 13:57:52] iteration 356/ 500 | consumed samples: 1424 | elapsed time per iteration (ms): 4375.1 | throughput (token/sec/GPU): 1289.8 | learning rate: 4.878632E-05 | global batch size: 4 | lm loss: 1.128806E+01 | loss scale: 1.0 | grad norm: 0.185 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1782]) -Dataloader batch - labels shape: torch.Size([1, 1782]) -Dataloader batch - attn_mask shape: torch.Size([1, 1782]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1025 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1782) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1782) torch.int64 - loss_mask : 424 / 1782 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1782, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1782, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1782, 1, 576) torch.bfloat16 - out : (1, 1782) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 424 tokens) : 11.2310 - top-1 accuracy : 29/424 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1788]) -Dataloader batch - labels shape: torch.Size([1, 1788]) -Dataloader batch - attn_mask shape: torch.Size([1, 1788]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1026 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1788) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1788) torch.int64 - loss_mask : 411 / 1788 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1788, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1788, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1788, 1, 576) torch.bfloat16 - out : (1, 1788) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.2802 - top-1 accuracy : 20/411 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1416]) -Dataloader batch - labels shape: torch.Size([1, 1416]) -Dataloader batch - attn_mask shape: torch.Size([1, 1416]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1027 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1416) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1416) torch.int64 - loss_mask : 353 / 1416 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1416, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1416, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1416, 1, 576) torch.bfloat16 - out : (1, 1416) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.2966 - top-1 accuracy : 20/353 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1360]) -Dataloader batch - labels shape: torch.Size([1, 1360]) -Dataloader batch - attn_mask shape: torch.Size([1, 1360]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1028 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1360) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1360) torch.int64 - loss_mask : 302 / 1360 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1360, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1360, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1360, 1, 576) torch.bfloat16 - out : (1, 1360) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 302 tokens) : 11.2117 - top-1 accuracy : 2/302 = 0.7% - - [2026-04-29 13:57:57] iteration 357/ 500 | consumed samples: 1428 | elapsed time per iteration (ms): 5012.9 | throughput (token/sec/GPU): 1265.9 | learning rate: 4.847490E-05 | global batch size: 4 | lm loss: 1.125619E+01 | loss scale: 1.0 | grad norm: 0.211 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1838]) -Dataloader batch - labels shape: torch.Size([1, 1838]) -Dataloader batch - attn_mask shape: torch.Size([1, 1838]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1029 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1838) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1838) torch.int64 - loss_mask : 476 / 1838 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1838, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1838, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1838, 1, 576) torch.bfloat16 - out : (1, 1838) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 476 tokens) : 11.3329 - top-1 accuracy : 9/476 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1030 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 364 / 1504 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2674 - top-1 accuracy : 19/364 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 906]) -Dataloader batch - labels shape: torch.Size([1, 906]) -Dataloader batch - attn_mask shape: torch.Size([1, 906]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1031 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 906) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 906) torch.int64 - loss_mask : 194 / 906 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (906, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (906, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (906, 1, 576) torch.bfloat16 - out : (1, 906) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 194 tokens) : 11.1805 - top-1 accuracy : 4/194 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1425]) -Dataloader batch - labels shape: torch.Size([1, 1425]) -Dataloader batch - attn_mask shape: torch.Size([1, 1425]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1032 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1425) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1425) torch.int64 - loss_mask : 366 / 1425 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1425, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1425, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1425, 1, 576) torch.bfloat16 - out : (1, 1425) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.3322 - top-1 accuracy : 8/366 = 2.2% - - [2026-04-29 13:58:01] iteration 358/ 500 | consumed samples: 1432 | elapsed time per iteration (ms): 4647.5 | throughput (token/sec/GPU): 1220.6 | learning rate: 4.816356E-05 | global batch size: 4 | lm loss: 1.129460E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1072]) -Dataloader batch - labels shape: torch.Size([1, 1072]) -Dataloader batch - attn_mask shape: torch.Size([1, 1072]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1033 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1072) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1072) torch.int64 - loss_mask : 223 / 1072 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1072, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1072, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1072, 1, 576) torch.bfloat16 - out : (1, 1072) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 223 tokens) : 11.2327 - top-1 accuracy : 18/223 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 817]) -Dataloader batch - labels shape: torch.Size([1, 817]) -Dataloader batch - attn_mask shape: torch.Size([1, 817]) -Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) - -==================================================================== - PIPELINE — STEP 1034 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 817) torch.int64 - images : (16, 3, 1024, 1024) torch.bfloat16 - labels : (1, 817) torch.int64 - loss_mask : 169 / 817 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (16, 3, 1024, 1024) torch.bfloat16 - out : (16, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (16, 3072) - out : (16, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (817, 1, 576) torch.bfloat16 grad=False - image slots : 16 tokens replaced with vision embeddings - combined : (817, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (817, 1, 576) torch.bfloat16 - out : (1, 817) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 169 tokens) : 11.1595 - top-1 accuracy : 5/169 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1035 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 361 / 1508 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.2628 - top-1 accuracy : 20/361 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1036 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 402 / 1444 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.3792 - top-1 accuracy : 6/402 = 1.5% - - [2026-04-29 13:58:06] iteration 359/ 500 | consumed samples: 1436 | elapsed time per iteration (ms): 4037.3 | throughput (token/sec/GPU): 1199.1 | learning rate: 4.785231E-05 | global batch size: 4 | lm loss: 1.128237E+01 | loss scale: 1.0 | grad norm: 0.293 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1556]) -Dataloader batch - labels shape: torch.Size([1, 1556]) -Dataloader batch - attn_mask shape: torch.Size([1, 1556]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1037 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1556) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1556) torch.int64 - loss_mask : 422 / 1556 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1556, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1556, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1556, 1, 576) torch.bfloat16 - out : (1, 1556) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.3195 - top-1 accuracy : 10/422 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1038 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 347 / 1695 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.1955 - top-1 accuracy : 28/347 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1857]) -Dataloader batch - labels shape: torch.Size([1, 1857]) -Dataloader batch - attn_mask shape: torch.Size([1, 1857]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1039 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1857) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1857) torch.int64 - loss_mask : 494 / 1857 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1857, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1857, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1857, 1, 576) torch.bfloat16 - out : (1, 1857) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 494 tokens) : 11.3447 - top-1 accuracy : 6/494 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 881]) -Dataloader batch - labels shape: torch.Size([1, 881]) -Dataloader batch - attn_mask shape: torch.Size([1, 881]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 1040 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 881) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 881) torch.int64 - loss_mask : 181 / 881 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (881, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (881, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (881, 1, 576) torch.bfloat16 - out : (1, 881) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 181 tokens) : 11.1350 - top-1 accuracy : 8/181 = 4.4% - - [2026-04-29 13:58:10] iteration 360/ 500 | consumed samples: 1440 | elapsed time per iteration (ms): 4741.0 | throughput (token/sec/GPU): 1263.2 | learning rate: 4.754118E-05 | global batch size: 4 | lm loss: 1.127518E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4727.33, 4727.33) - forward-compute ................................: (3551.50, 3551.50) - backward-compute ...............................: (1173.85, 1173.85) - batch-generator ................................: (482.54, 482.54) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.40, 0.40) - params-all-gather ..............................: (0.14, 0.14) - optimizer-copy-to-main-grad ....................: (0.11, 0.11) - optimizer-clip-main-grad .......................: (0.45, 0.45) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.18, 1.18) - optimizer-copy-main-to-model-params ............: (0.20, 0.20) - optimizer ......................................: (2.66, 2.66) -Dataloader batch - tokens shape: torch.Size([1, 1474]) -Dataloader batch - labels shape: torch.Size([1, 1474]) -Dataloader batch - attn_mask shape: torch.Size([1, 1474]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1041 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1474) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1474) torch.int64 - loss_mask : 294 / 1474 tokens (19.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1474, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1474, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1474, 1, 576) torch.bfloat16 - out : (1, 1474) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.1084 - top-1 accuracy : 32/294 = 10.9% - -Dataloader batch - tokens shape: torch.Size([1, 871]) -Dataloader batch - labels shape: torch.Size([1, 871]) -Dataloader batch - attn_mask shape: torch.Size([1, 871]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 1042 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 871) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 871) torch.int64 - loss_mask : 182 / 871 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (871, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (871, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (871, 1, 576) torch.bfloat16 - out : (1, 871) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 182 tokens) : 11.1721 - top-1 accuracy : 3/182 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 1527]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1043 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1527) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1527) torch.int64 - loss_mask : 351 / 1527 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1527, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1527, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1527, 1, 576) torch.bfloat16 - out : (1, 1527) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.2551 - top-1 accuracy : 17/351 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1286]) -Dataloader batch - labels shape: torch.Size([1, 1286]) -Dataloader batch - attn_mask shape: torch.Size([1, 1286]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1044 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1286) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1286) torch.int64 - loss_mask : 344 / 1286 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1286, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1286, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1286, 1, 576) torch.bfloat16 - out : (1, 1286) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.3522 - top-1 accuracy : 4/344 = 1.2% - - [2026-04-29 13:58:15] iteration 361/ 500 | consumed samples: 1444 | elapsed time per iteration (ms): 4255.5 | throughput (token/sec/GPU): 1212.1 | learning rate: 4.723015E-05 | global batch size: 4 | lm loss: 1.123388E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1588]) -Dataloader batch - labels shape: torch.Size([1, 1588]) -Dataloader batch - attn_mask shape: torch.Size([1, 1588]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1045 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1588) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1588) torch.int64 - loss_mask : 339 / 1588 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1588, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1588, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1588, 1, 576) torch.bfloat16 - out : (1, 1588) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2255 - top-1 accuracy : 15/339 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1427]) -Dataloader batch - labels shape: torch.Size([1, 1427]) -Dataloader batch - attn_mask shape: torch.Size([1, 1427]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1046 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1427) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1427) torch.int64 - loss_mask : 373 / 1427 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1427, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1427, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1427, 1, 576) torch.bfloat16 - out : (1, 1427) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.3047 - top-1 accuracy : 17/373 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1367]) -Dataloader batch - labels shape: torch.Size([1, 1367]) -Dataloader batch - attn_mask shape: torch.Size([1, 1367]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1047 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1367) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1367) torch.int64 - loss_mask : 321 / 1367 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1367, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1367, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1367, 1, 576) torch.bfloat16 - out : (1, 1367) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2430 - top-1 accuracy : 7/321 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1458]) -Dataloader batch - labels shape: torch.Size([1, 1458]) -Dataloader batch - attn_mask shape: torch.Size([1, 1458]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1048 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1458) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1458) torch.int64 - loss_mask : 293 / 1458 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1458, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1458, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1458, 1, 576) torch.bfloat16 - out : (1, 1458) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 293 tokens) : 11.1044 - top-1 accuracy : 22/293 = 7.5% - - [2026-04-29 13:58:19] iteration 362/ 500 | consumed samples: 1448 | elapsed time per iteration (ms): 4663.9 | throughput (token/sec/GPU): 1252.2 | learning rate: 4.691926E-05 | global batch size: 4 | lm loss: 1.122524E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1049 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 336 / 1431 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 336 tokens) : 11.1879 - top-1 accuracy : 17/336 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1669]) -Dataloader batch - labels shape: torch.Size([1, 1669]) -Dataloader batch - attn_mask shape: torch.Size([1, 1669]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1050 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1669) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1669) torch.int64 - loss_mask : 368 / 1669 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1669, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1669, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1669, 1, 576) torch.bfloat16 - out : (1, 1669) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2006 - top-1 accuracy : 16/368 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1011]) -Dataloader batch - labels shape: torch.Size([1, 1011]) -Dataloader batch - attn_mask shape: torch.Size([1, 1011]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1051 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1011) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1011) torch.int64 - loss_mask : 246 / 1011 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1011, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1011, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1011, 1, 576) torch.bfloat16 - out : (1, 1011) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 246 tokens) : 11.3239 - top-1 accuracy : 1/246 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1762]) -Dataloader batch - labels shape: torch.Size([1, 1762]) -Dataloader batch - attn_mask shape: torch.Size([1, 1762]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1052 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1762) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1762) torch.int64 - loss_mask : 402 / 1762 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1762, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1762, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1762, 1, 576) torch.bfloat16 - out : (1, 1762) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.2913 - top-1 accuracy : 26/402 = 6.5% - - [2026-04-29 13:58:24] iteration 363/ 500 | consumed samples: 1452 | elapsed time per iteration (ms): 4941.5 | throughput (token/sec/GPU): 1188.5 | learning rate: 4.660851E-05 | global batch size: 4 | lm loss: 1.124685E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1049]) -Dataloader batch - labels shape: torch.Size([1, 1049]) -Dataloader batch - attn_mask shape: torch.Size([1, 1049]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1053 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1049) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1049) torch.int64 - loss_mask : 289 / 1049 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1049, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1049, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1049, 1, 576) torch.bfloat16 - out : (1, 1049) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 289 tokens) : 11.3170 - top-1 accuracy : 2/289 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1055]) -Dataloader batch - labels shape: torch.Size([1, 1055]) -Dataloader batch - attn_mask shape: torch.Size([1, 1055]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1054 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1055) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1055) torch.int64 - loss_mask : 292 / 1055 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1055, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1055, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1055, 1, 576) torch.bfloat16 - out : (1, 1055) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 292 tokens) : 11.3656 - top-1 accuracy : 4/292 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1383]) -Dataloader batch - labels shape: torch.Size([1, 1383]) -Dataloader batch - attn_mask shape: torch.Size([1, 1383]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1055 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1383) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1383) torch.int64 - loss_mask : 303 / 1383 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1383, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1383, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1383, 1, 576) torch.bfloat16 - out : (1, 1383) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.1863 - top-1 accuracy : 15/303 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1069]) -Dataloader batch - labels shape: torch.Size([1, 1069]) -Dataloader batch - attn_mask shape: torch.Size([1, 1069]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1056 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1069) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1069) torch.int64 - loss_mask : 234 / 1069 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1069, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1069, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1069, 1, 576) torch.bfloat16 - out : (1, 1069) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 234 tokens) : 11.1886 - top-1 accuracy : 6/234 = 2.6% - - [2026-04-29 13:58:28] iteration 364/ 500 | consumed samples: 1456 | elapsed time per iteration (ms): 3750.4 | throughput (token/sec/GPU): 1214.8 | learning rate: 4.629791E-05 | global batch size: 4 | lm loss: 1.126741E+01 | loss scale: 1.0 | grad norm: 0.229 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1551]) -Dataloader batch - labels shape: torch.Size([1, 1551]) -Dataloader batch - attn_mask shape: torch.Size([1, 1551]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1057 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1551) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1551) torch.int64 - loss_mask : 354 / 1551 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1551, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1551, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1551, 1, 576) torch.bfloat16 - out : (1, 1551) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2948 - top-1 accuracy : 19/354 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1074]) -Dataloader batch - labels shape: torch.Size([1, 1074]) -Dataloader batch - attn_mask shape: torch.Size([1, 1074]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1058 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1074) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1074) torch.int64 - loss_mask : 248 / 1074 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1074, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1074, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1074, 1, 576) torch.bfloat16 - out : (1, 1074) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 248 tokens) : 11.2453 - top-1 accuracy : 16/248 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1513]) -Dataloader batch - labels shape: torch.Size([1, 1513]) -Dataloader batch - attn_mask shape: torch.Size([1, 1513]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1059 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1513) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1513) torch.int64 - loss_mask : 364 / 1513 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1513, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1513, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1513, 1, 576) torch.bfloat16 - out : (1, 1513) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2880 - top-1 accuracy : 5/364 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1060 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 381 / 1518 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.2840 - top-1 accuracy : 24/381 = 6.3% - - [2026-04-29 13:58:32] iteration 365/ 500 | consumed samples: 1460 | elapsed time per iteration (ms): 4607.3 | throughput (token/sec/GPU): 1227.6 | learning rate: 4.598748E-05 | global batch size: 4 | lm loss: 1.128077E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1123]) -Dataloader batch - labels shape: torch.Size([1, 1123]) -Dataloader batch - attn_mask shape: torch.Size([1, 1123]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1061 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1123) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1123) torch.int64 - loss_mask : 260 / 1123 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1123, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1123, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1123, 1, 576) torch.bfloat16 - out : (1, 1123) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.2309 - top-1 accuracy : 4/260 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1453]) -Dataloader batch - labels shape: torch.Size([1, 1453]) -Dataloader batch - attn_mask shape: torch.Size([1, 1453]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1062 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1453) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1453) torch.int64 - loss_mask : 301 / 1453 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1453, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1453, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1453, 1, 576) torch.bfloat16 - out : (1, 1453) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 301 tokens) : 11.1620 - top-1 accuracy : 15/301 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1472]) -Dataloader batch - labels shape: torch.Size([1, 1472]) -Dataloader batch - attn_mask shape: torch.Size([1, 1472]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1063 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1472) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1472) torch.int64 - loss_mask : 325 / 1472 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1472, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1472, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1472, 1, 576) torch.bfloat16 - out : (1, 1472) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.1569 - top-1 accuracy : 8/325 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1972]) -Dataloader batch - labels shape: torch.Size([1, 1972]) -Dataloader batch - attn_mask shape: torch.Size([1, 1972]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1064 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1972) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1972) torch.int64 - loss_mask : 585 / 1972 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1972, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1972, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1972, 1, 576) torch.bfloat16 - out : (1, 1972) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 585 tokens) : 11.3934 - top-1 accuracy : 4/585 = 0.7% - - [2026-04-29 13:58:37] iteration 366/ 500 | consumed samples: 1464 | elapsed time per iteration (ms): 4884.9 | throughput (token/sec/GPU): 1232.4 | learning rate: 4.567723E-05 | global batch size: 4 | lm loss: 1.126509E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1065 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 358 / 1525 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2351 - top-1 accuracy : 2/358 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1799]) -Dataloader batch - labels shape: torch.Size([1, 1799]) -Dataloader batch - attn_mask shape: torch.Size([1, 1799]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1066 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1799) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1799) torch.int64 - loss_mask : 421 / 1799 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1799, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1799, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1799, 1, 576) torch.bfloat16 - out : (1, 1799) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 421 tokens) : 11.2524 - top-1 accuracy : 5/421 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1613]) -Dataloader batch - labels shape: torch.Size([1, 1613]) -Dataloader batch - attn_mask shape: torch.Size([1, 1613]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1067 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1613) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1613) torch.int64 - loss_mask : 377 / 1613 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1613, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1613, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1613, 1, 576) torch.bfloat16 - out : (1, 1613) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 377 tokens) : 11.2111 - top-1 accuracy : 22/377 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1095]) -Dataloader batch - labels shape: torch.Size([1, 1095]) -Dataloader batch - attn_mask shape: torch.Size([1, 1095]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1068 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1095) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1095) torch.int64 - loss_mask : 290 / 1095 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1095, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1095, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1095, 1, 576) torch.bfloat16 - out : (1, 1095) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.2850 - top-1 accuracy : 14/290 = 4.8% - - [2026-04-29 13:58:42] iteration 367/ 500 | consumed samples: 1468 | elapsed time per iteration (ms): 4782.1 | throughput (token/sec/GPU): 1261.4 | learning rate: 4.536717E-05 | global batch size: 4 | lm loss: 1.124389E+01 | loss scale: 1.0 | grad norm: 0.257 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1095]) -Dataloader batch - labels shape: torch.Size([1, 1095]) -Dataloader batch - attn_mask shape: torch.Size([1, 1095]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1069 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1095) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1095) torch.int64 - loss_mask : 246 / 1095 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1095, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1095, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1095, 1, 576) torch.bfloat16 - out : (1, 1095) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 246 tokens) : 11.1865 - top-1 accuracy : 20/246 = 8.1% - -Dataloader batch - tokens shape: torch.Size([1, 1488]) -Dataloader batch - labels shape: torch.Size([1, 1488]) -Dataloader batch - attn_mask shape: torch.Size([1, 1488]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1070 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1488) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1488) torch.int64 - loss_mask : 444 / 1488 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1488, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1488, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1488, 1, 576) torch.bfloat16 - out : (1, 1488) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 444 tokens) : 11.3704 - top-1 accuracy : 9/444 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1071 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 300 / 1388 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.2449 - top-1 accuracy : 11/300 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1793]) -Dataloader batch - labels shape: torch.Size([1, 1793]) -Dataloader batch - attn_mask shape: torch.Size([1, 1793]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1072 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1793) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1793) torch.int64 - loss_mask : 442 / 1793 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1793, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1793, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1793, 1, 576) torch.bfloat16 - out : (1, 1793) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 442 tokens) : 11.2679 - top-1 accuracy : 13/442 = 2.9% - - [2026-04-29 13:58:47] iteration 368/ 500 | consumed samples: 1472 | elapsed time per iteration (ms): 4547.7 | throughput (token/sec/GPU): 1267.5 | learning rate: 4.505731E-05 | global batch size: 4 | lm loss: 1.128090E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1581]) -Dataloader batch - labels shape: torch.Size([1, 1581]) -Dataloader batch - attn_mask shape: torch.Size([1, 1581]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1073 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1581) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1581) torch.int64 - loss_mask : 354 / 1581 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1581, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1581, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1581, 1, 576) torch.bfloat16 - out : (1, 1581) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 354 tokens) : 11.2056 - top-1 accuracy : 13/354 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1274]) -Dataloader batch - labels shape: torch.Size([1, 1274]) -Dataloader batch - attn_mask shape: torch.Size([1, 1274]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1074 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1274) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1274) torch.int64 - loss_mask : 385 / 1274 tokens (30.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1274, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1274, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1274, 1, 576) torch.bfloat16 - out : (1, 1274) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.3444 - top-1 accuracy : 2/385 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1075 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 290 / 1364 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.1767 - top-1 accuracy : 17/290 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1842]) -Dataloader batch - labels shape: torch.Size([1, 1842]) -Dataloader batch - attn_mask shape: torch.Size([1, 1842]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1076 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1842) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1842) torch.int64 - loss_mask : 483 / 1842 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1842, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1842, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1842, 1, 576) torch.bfloat16 - out : (1, 1842) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 483 tokens) : 11.3102 - top-1 accuracy : 9/483 = 1.9% - - [2026-04-29 13:58:52] iteration 369/ 500 | consumed samples: 1476 | elapsed time per iteration (ms): 4821.7 | throughput (token/sec/GPU): 1257.0 | learning rate: 4.474767E-05 | global batch size: 4 | lm loss: 1.126881E+01 | loss scale: 1.0 | grad norm: 0.219 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1434]) -Dataloader batch - labels shape: torch.Size([1, 1434]) -Dataloader batch - attn_mask shape: torch.Size([1, 1434]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1077 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1434) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1434) torch.int64 - loss_mask : 368 / 1434 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1434, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1434, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1434, 1, 576) torch.bfloat16 - out : (1, 1434) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2993 - top-1 accuracy : 15/368 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 962]) -Dataloader batch - labels shape: torch.Size([1, 962]) -Dataloader batch - attn_mask shape: torch.Size([1, 962]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1078 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 962) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 962) torch.int64 - loss_mask : 207 / 962 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (962, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (962, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (962, 1, 576) torch.bfloat16 - out : (1, 962) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 207 tokens) : 11.2307 - top-1 accuracy : 5/207 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 991]) -Dataloader batch - labels shape: torch.Size([1, 991]) -Dataloader batch - attn_mask shape: torch.Size([1, 991]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1079 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 991) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 991) torch.int64 - loss_mask : 214 / 991 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (991, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (991, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (991, 1, 576) torch.bfloat16 - out : (1, 991) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 214 tokens) : 11.2075 - top-1 accuracy : 1/214 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1413]) -Dataloader batch - labels shape: torch.Size([1, 1413]) -Dataloader batch - attn_mask shape: torch.Size([1, 1413]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1080 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1413) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1413) torch.int64 - loss_mask : 352 / 1413 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1413, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1413, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1413, 1, 576) torch.bfloat16 - out : (1, 1413) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 352 tokens) : 11.2869 - top-1 accuracy : 7/352 = 2.0% - - [2026-04-29 13:58:55] iteration 370/ 500 | consumed samples: 1480 | elapsed time per iteration (ms): 3954.5 | throughput (token/sec/GPU): 1213.8 | learning rate: 4.443826E-05 | global batch size: 4 | lm loss: 1.126581E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1081 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 325 / 1518 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.1727 - top-1 accuracy : 4/325 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1393]) -Dataloader batch - labels shape: torch.Size([1, 1393]) -Dataloader batch - attn_mask shape: torch.Size([1, 1393]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1082 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1393) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1393) torch.int64 - loss_mask : 342 / 1393 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1393, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1393, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1393, 1, 576) torch.bfloat16 - out : (1, 1393) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.3199 - top-1 accuracy : 8/342 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1398]) -Dataloader batch - labels shape: torch.Size([1, 1398]) -Dataloader batch - attn_mask shape: torch.Size([1, 1398]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1083 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1398) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1398) torch.int64 - loss_mask : 330 / 1398 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1398, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1398, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1398, 1, 576) torch.bfloat16 - out : (1, 1398) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.2642 - top-1 accuracy : 14/330 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1215]) -Dataloader batch - labels shape: torch.Size([1, 1215]) -Dataloader batch - attn_mask shape: torch.Size([1, 1215]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1084 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1215) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1215) torch.int64 - loss_mask : 294 / 1215 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1215, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1215, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1215, 1, 576) torch.bfloat16 - out : (1, 1215) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.2217 - top-1 accuracy : 16/294 = 5.4% - - [2026-04-29 13:59:00] iteration 371/ 500 | consumed samples: 1484 | elapsed time per iteration (ms): 4506.2 | throughput (token/sec/GPU): 1225.9 | learning rate: 4.412908E-05 | global batch size: 4 | lm loss: 1.124625E+01 | loss scale: 1.0 | grad norm: 0.200 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1215]) -Dataloader batch - labels shape: torch.Size([1, 1215]) -Dataloader batch - attn_mask shape: torch.Size([1, 1215]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1085 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1215) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1215) torch.int64 - loss_mask : 332 / 1215 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1215, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1215, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1215, 1, 576) torch.bfloat16 - out : (1, 1215) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.3406 - top-1 accuracy : 4/332 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1753]) -Dataloader batch - labels shape: torch.Size([1, 1753]) -Dataloader batch - attn_mask shape: torch.Size([1, 1753]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1086 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1753) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1753) torch.int64 - loss_mask : 369 / 1753 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1753, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1753, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1753, 1, 576) torch.bfloat16 - out : (1, 1753) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2784 - top-1 accuracy : 22/369 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1138]) -Dataloader batch - labels shape: torch.Size([1, 1138]) -Dataloader batch - attn_mask shape: torch.Size([1, 1138]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1087 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1138) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1138) torch.int64 - loss_mask : 345 / 1138 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1138, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1138, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1138, 1, 576) torch.bfloat16 - out : (1, 1138) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.4305 - top-1 accuracy : 4/345 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1581]) -Dataloader batch - labels shape: torch.Size([1, 1581]) -Dataloader batch - attn_mask shape: torch.Size([1, 1581]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1088 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1581) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1581) torch.int64 - loss_mask : 357 / 1581 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1581, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1581, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1581, 1, 576) torch.bfloat16 - out : (1, 1581) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2334 - top-1 accuracy : 27/357 = 7.6% - - [2026-04-29 13:59:05] iteration 372/ 500 | consumed samples: 1488 | elapsed time per iteration (ms): 4651.7 | throughput (token/sec/GPU): 1222.6 | learning rate: 4.382016E-05 | global batch size: 4 | lm loss: 1.131907E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1972]) -Dataloader batch - labels shape: torch.Size([1, 1972]) -Dataloader batch - attn_mask shape: torch.Size([1, 1972]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1089 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1972) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1972) torch.int64 - loss_mask : 622 / 1972 tokens (31.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1972, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1972, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1972, 1, 576) torch.bfloat16 - out : (1, 1972) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 622 tokens) : 11.4017 - top-1 accuracy : 7/622 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1721]) -Dataloader batch - labels shape: torch.Size([1, 1721]) -Dataloader batch - attn_mask shape: torch.Size([1, 1721]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1090 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1721) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1721) torch.int64 - loss_mask : 367 / 1721 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1721, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1721, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1721, 1, 576) torch.bfloat16 - out : (1, 1721) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.1900 - top-1 accuracy : 30/367 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1091 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1385) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1385) torch.int64 - loss_mask : 286 / 1385 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1385, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1385, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1385, 1, 576) torch.bfloat16 - out : (1, 1385) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 286 tokens) : 11.1631 - top-1 accuracy : 18/286 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1092 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1423) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1423) torch.int64 - loss_mask : 344 / 1423 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1423, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1423, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1423, 1, 576) torch.bfloat16 - out : (1, 1423) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.2725 - top-1 accuracy : 6/344 = 1.7% - - [2026-04-29 13:59:10] iteration 373/ 500 | consumed samples: 1492 | elapsed time per iteration (ms): 5142.8 | throughput (token/sec/GPU): 1264.1 | learning rate: 4.351151E-05 | global batch size: 4 | lm loss: 1.128413E+01 | loss scale: 1.0 | grad norm: 0.213 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1568]) -Dataloader batch - labels shape: torch.Size([1, 1568]) -Dataloader batch - attn_mask shape: torch.Size([1, 1568]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1093 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1568) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1568) torch.int64 - loss_mask : 372 / 1568 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1568, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1568, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1568, 1, 576) torch.bfloat16 - out : (1, 1568) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2502 - top-1 accuracy : 13/372 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1098]) -Dataloader batch - labels shape: torch.Size([1, 1098]) -Dataloader batch - attn_mask shape: torch.Size([1, 1098]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1094 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1098) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1098) torch.int64 - loss_mask : 238 / 1098 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1098, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1098, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1098, 1, 576) torch.bfloat16 - out : (1, 1098) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 238 tokens) : 11.1905 - top-1 accuracy : 3/238 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1926]) -Dataloader batch - labels shape: torch.Size([1, 1926]) -Dataloader batch - attn_mask shape: torch.Size([1, 1926]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1095 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1926) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1926) torch.int64 - loss_mask : 572 / 1926 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1926, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1926, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1926, 1, 576) torch.bfloat16 - out : (1, 1926) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 572 tokens) : 11.3856 - top-1 accuracy : 5/572 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1401]) -Dataloader batch - labels shape: torch.Size([1, 1401]) -Dataloader batch - attn_mask shape: torch.Size([1, 1401]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1096 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1401) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1401) torch.int64 - loss_mask : 333 / 1401 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1401, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1401, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1401, 1, 576) torch.bfloat16 - out : (1, 1401) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 333 tokens) : 11.2494 - top-1 accuracy : 11/333 = 3.3% - - [2026-04-29 13:59:14] iteration 374/ 500 | consumed samples: 1496 | elapsed time per iteration (ms): 4648.0 | throughput (token/sec/GPU): 1289.4 | learning rate: 4.320313E-05 | global batch size: 4 | lm loss: 1.129174E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1540]) -Dataloader batch - labels shape: torch.Size([1, 1540]) -Dataloader batch - attn_mask shape: torch.Size([1, 1540]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1097 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1540) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1540) torch.int64 - loss_mask : 334 / 1540 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1540, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1540, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1540, 1, 576) torch.bfloat16 - out : (1, 1540) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.2512 - top-1 accuracy : 20/334 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1098 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 386 / 1431 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3472 - top-1 accuracy : 9/386 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1490]) -Dataloader batch - labels shape: torch.Size([1, 1490]) -Dataloader batch - attn_mask shape: torch.Size([1, 1490]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1099 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1490) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1490) torch.int64 - loss_mask : 313 / 1490 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1490, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1490, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1490, 1, 576) torch.bfloat16 - out : (1, 1490) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2076 - top-1 accuracy : 17/313 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1100 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 387 / 1750 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2667 - top-1 accuracy : 6/387 = 1.6% - - [2026-04-29 13:59:19] iteration 375/ 500 | consumed samples: 1500 | elapsed time per iteration (ms): 4961.0 | throughput (token/sec/GPU): 1252.0 | learning rate: 4.289504E-05 | global batch size: 4 | lm loss: 1.127192E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1556]) -Dataloader batch - labels shape: torch.Size([1, 1556]) -Dataloader batch - attn_mask shape: torch.Size([1, 1556]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1101 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1556) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1556) torch.int64 - loss_mask : 343 / 1556 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1556, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1556, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1556, 1, 576) torch.bfloat16 - out : (1, 1556) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.2294 - top-1 accuracy : 13/343 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 983]) -Dataloader batch - labels shape: torch.Size([1, 983]) -Dataloader batch - attn_mask shape: torch.Size([1, 983]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1102 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 983) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 983) torch.int64 - loss_mask : 198 / 983 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (983, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (983, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (983, 1, 576) torch.bfloat16 - out : (1, 983) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 198 tokens) : 11.0921 - top-1 accuracy : 10/198 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1103 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1470) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1470) torch.int64 - loss_mask : 317 / 1470 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1470, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1470, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1470, 1, 576) torch.bfloat16 - out : (1, 1470) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.1450 - top-1 accuracy : 4/317 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1017]) -Dataloader batch - labels shape: torch.Size([1, 1017]) -Dataloader batch - attn_mask shape: torch.Size([1, 1017]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1104 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1017) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1017) torch.int64 - loss_mask : 250 / 1017 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1017, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1017, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1017, 1, 576) torch.bfloat16 - out : (1, 1017) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 250 tokens) : 11.2293 - top-1 accuracy : 7/250 = 2.8% - - [2026-04-29 13:59:24] iteration 376/ 500 | consumed samples: 1504 | elapsed time per iteration (ms): 4305.0 | throughput (token/sec/GPU): 1167.5 | learning rate: 4.258725E-05 | global batch size: 4 | lm loss: 1.118069E+01 | loss scale: 1.0 | grad norm: 0.279 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1036]) -Dataloader batch - labels shape: torch.Size([1, 1036]) -Dataloader batch - attn_mask shape: torch.Size([1, 1036]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1105 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1036) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1036) torch.int64 - loss_mask : 219 / 1036 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1036, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1036, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1036, 1, 576) torch.bfloat16 - out : (1, 1036) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 219 tokens) : 11.2017 - top-1 accuracy : 7/219 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1523]) -Dataloader batch - labels shape: torch.Size([1, 1523]) -Dataloader batch - attn_mask shape: torch.Size([1, 1523]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1106 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1523) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1523) torch.int64 - loss_mask : 335 / 1523 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1523, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1523, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1523, 1, 576) torch.bfloat16 - out : (1, 1523) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2483 - top-1 accuracy : 28/335 = 8.4% - -Dataloader batch - tokens shape: torch.Size([1, 1514]) -Dataloader batch - labels shape: torch.Size([1, 1514]) -Dataloader batch - attn_mask shape: torch.Size([1, 1514]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1107 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1514) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1514) torch.int64 - loss_mask : 320 / 1514 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1514, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1514, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1514, 1, 576) torch.bfloat16 - out : (1, 1514) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.1660 - top-1 accuracy : 24/320 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1393]) -Dataloader batch - labels shape: torch.Size([1, 1393]) -Dataloader batch - attn_mask shape: torch.Size([1, 1393]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1108 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1393) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1393) torch.int64 - loss_mask : 326 / 1393 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1393, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1393, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1393, 1, 576) torch.bfloat16 - out : (1, 1393) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2349 - top-1 accuracy : 14/326 = 4.3% - - [2026-04-29 13:59:28] iteration 377/ 500 | consumed samples: 1508 | elapsed time per iteration (ms): 4560.3 | throughput (token/sec/GPU): 1198.6 | learning rate: 4.227977E-05 | global batch size: 4 | lm loss: 1.121422E+01 | loss scale: 1.0 | grad norm: 0.228 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1074]) -Dataloader batch - labels shape: torch.Size([1, 1074]) -Dataloader batch - attn_mask shape: torch.Size([1, 1074]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1109 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1074) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1074) torch.int64 - loss_mask : 316 / 1074 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1074, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1074, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1074, 1, 576) torch.bfloat16 - out : (1, 1074) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.4098 - top-1 accuracy : 10/316 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1875]) -Dataloader batch - labels shape: torch.Size([1, 1875]) -Dataloader batch - attn_mask shape: torch.Size([1, 1875]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1110 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1875) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1875) torch.int64 - loss_mask : 517 / 1875 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1875, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1875, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1875, 1, 576) torch.bfloat16 - out : (1, 1875) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 517 tokens) : 11.3414 - top-1 accuracy : 6/517 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1296]) -Dataloader batch - labels shape: torch.Size([1, 1296]) -Dataloader batch - attn_mask shape: torch.Size([1, 1296]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1111 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1296) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1296) torch.int64 - loss_mask : 356 / 1296 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1296, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1296, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1296, 1, 576) torch.bfloat16 - out : (1, 1296) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.3303 - top-1 accuracy : 5/356 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1514]) -Dataloader batch - labels shape: torch.Size([1, 1514]) -Dataloader batch - attn_mask shape: torch.Size([1, 1514]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1112 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1514) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1514) torch.int64 - loss_mask : 392 / 1514 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1514, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1514, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1514, 1, 576) torch.bfloat16 - out : (1, 1514) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3133 - top-1 accuracy : 17/392 = 4.3% - - [2026-04-29 13:59:33] iteration 378/ 500 | consumed samples: 1512 | elapsed time per iteration (ms): 4420.0 | throughput (token/sec/GPU): 1302.9 | learning rate: 4.197262E-05 | global batch size: 4 | lm loss: 1.134558E+01 | loss scale: 1.0 | grad norm: 0.246 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1523]) -Dataloader batch - labels shape: torch.Size([1, 1523]) -Dataloader batch - attn_mask shape: torch.Size([1, 1523]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1113 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1523) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1523) torch.int64 - loss_mask : 382 / 1523 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1523, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1523, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1523, 1, 576) torch.bfloat16 - out : (1, 1523) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.2803 - top-1 accuracy : 5/382 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1761]) -Dataloader batch - labels shape: torch.Size([1, 1761]) -Dataloader batch - attn_mask shape: torch.Size([1, 1761]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1114 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1761) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1761) torch.int64 - loss_mask : 406 / 1761 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1761, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1761, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1761, 1, 576) torch.bfloat16 - out : (1, 1761) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.2842 - top-1 accuracy : 14/406 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 937]) -Dataloader batch - labels shape: torch.Size([1, 937]) -Dataloader batch - attn_mask shape: torch.Size([1, 937]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1115 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 937) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 937) torch.int64 - loss_mask : 204 / 937 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (937, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (937, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (937, 1, 576) torch.bfloat16 - out : (1, 937) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 204 tokens) : 11.1731 - top-1 accuracy : 2/204 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1562]) -Dataloader batch - labels shape: torch.Size([1, 1562]) -Dataloader batch - attn_mask shape: torch.Size([1, 1562]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1116 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1562) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1562) torch.int64 - loss_mask : 415 / 1562 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1562, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1562, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1562, 1, 576) torch.bfloat16 - out : (1, 1562) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.3281 - top-1 accuracy : 19/415 = 4.6% - - [2026-04-29 13:59:37] iteration 379/ 500 | consumed samples: 1516 | elapsed time per iteration (ms): 4658.6 | throughput (token/sec/GPU): 1241.4 | learning rate: 4.166581E-05 | global batch size: 4 | lm loss: 1.127998E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1522]) -Dataloader batch - labels shape: torch.Size([1, 1522]) -Dataloader batch - attn_mask shape: torch.Size([1, 1522]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1117 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1522) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1522) torch.int64 - loss_mask : 366 / 1522 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1522, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1522, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1522, 1, 576) torch.bfloat16 - out : (1, 1522) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2778 - top-1 accuracy : 18/366 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1599]) -Dataloader batch - labels shape: torch.Size([1, 1599]) -Dataloader batch - attn_mask shape: torch.Size([1, 1599]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1118 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1599) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1599) torch.int64 - loss_mask : 458 / 1599 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1599, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1599, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1599, 1, 576) torch.bfloat16 - out : (1, 1599) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 458 tokens) : 11.3888 - top-1 accuracy : 3/458 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1874]) -Dataloader batch - labels shape: torch.Size([1, 1874]) -Dataloader batch - attn_mask shape: torch.Size([1, 1874]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1119 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1874) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1874) torch.int64 - loss_mask : 505 / 1874 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1874, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1874, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1874, 1, 576) torch.bfloat16 - out : (1, 1874) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 505 tokens) : 11.3527 - top-1 accuracy : 6/505 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1696]) -Dataloader batch - labels shape: torch.Size([1, 1696]) -Dataloader batch - attn_mask shape: torch.Size([1, 1696]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1120 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1696) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1696) torch.int64 - loss_mask : 363 / 1696 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1696, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1696, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1696, 1, 576) torch.bfloat16 - out : (1, 1696) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2030 - top-1 accuracy : 16/363 = 4.4% - - [2026-04-29 13:59:42] iteration 380/ 500 | consumed samples: 1520 | elapsed time per iteration (ms): 5087.9 | throughput (token/sec/GPU): 1315.1 | learning rate: 4.135936E-05 | global batch size: 4 | lm loss: 1.131415E+01 | loss scale: 1.0 | grad norm: 0.201 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (5077.75, 5077.75) - forward-compute ................................: (3824.77, 3824.77) - backward-compute ...............................: (1251.10, 1251.10) - batch-generator ................................: (502.07, 502.07) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.38, 0.38) - params-all-gather ..............................: (0.11, 0.11) - optimizer-copy-to-main-grad ....................: (0.10, 0.10) - optimizer-clip-main-grad .......................: (0.41, 0.41) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.13, 1.13) - optimizer-copy-main-to-model-params ............: (0.18, 0.18) - optimizer ......................................: (2.48, 2.48) -Dataloader batch - tokens shape: torch.Size([1, 1397]) -Dataloader batch - labels shape: torch.Size([1, 1397]) -Dataloader batch - attn_mask shape: torch.Size([1, 1397]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1121 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1397) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1397) torch.int64 - loss_mask : 378 / 1397 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1397, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1397, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1397, 1, 576) torch.bfloat16 - out : (1, 1397) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 378 tokens) : 11.3612 - top-1 accuracy : 9/378 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1165]) -Dataloader batch - labels shape: torch.Size([1, 1165]) -Dataloader batch - attn_mask shape: torch.Size([1, 1165]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1122 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1165) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1165) torch.int64 - loss_mask : 311 / 1165 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1165, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1165, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1165, 1, 576) torch.bfloat16 - out : (1, 1165) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.3554 - top-1 accuracy : 5/311 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1871]) -Dataloader batch - labels shape: torch.Size([1, 1871]) -Dataloader batch - attn_mask shape: torch.Size([1, 1871]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1123 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1871) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1871) torch.int64 - loss_mask : 504 / 1871 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1871, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1871, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1871, 1, 576) torch.bfloat16 - out : (1, 1871) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 504 tokens) : 11.3120 - top-1 accuracy : 6/504 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1377]) -Dataloader batch - labels shape: torch.Size([1, 1377]) -Dataloader batch - attn_mask shape: torch.Size([1, 1377]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1124 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1377) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1377) torch.int64 - loss_mask : 316 / 1377 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1377, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1377, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1377, 1, 576) torch.bfloat16 - out : (1, 1377) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 316 tokens) : 11.2109 - top-1 accuracy : 16/316 = 5.1% - - [2026-04-29 13:59:47] iteration 381/ 500 | consumed samples: 1524 | elapsed time per iteration (ms): 4593.1 | throughput (token/sec/GPU): 1264.9 | learning rate: 4.105326E-05 | global batch size: 4 | lm loss: 1.131208E+01 | loss scale: 1.0 | grad norm: 0.181 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1125 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 314 / 1460 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.1359 - top-1 accuracy : 15/314 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1739]) -Dataloader batch - labels shape: torch.Size([1, 1739]) -Dataloader batch - attn_mask shape: torch.Size([1, 1739]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1126 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1739) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1739) torch.int64 - loss_mask : 385 / 1739 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1739, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1739, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1739, 1, 576) torch.bfloat16 - out : (1, 1739) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 385 tokens) : 11.2371 - top-1 accuracy : 29/385 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1017]) -Dataloader batch - labels shape: torch.Size([1, 1017]) -Dataloader batch - attn_mask shape: torch.Size([1, 1017]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1127 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1017) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1017) torch.int64 - loss_mask : 261 / 1017 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1017, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1017, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1017, 1, 576) torch.bfloat16 - out : (1, 1017) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 261 tokens) : 11.2826 - top-1 accuracy : 6/261 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1497]) -Dataloader batch - labels shape: torch.Size([1, 1497]) -Dataloader batch - attn_mask shape: torch.Size([1, 1497]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1128 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1497) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1497) torch.int64 - loss_mask : 350 / 1497 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1497, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1497, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1497, 1, 576) torch.bfloat16 - out : (1, 1497) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.1904 - top-1 accuracy : 17/350 = 4.9% - - [2026-04-29 13:59:52] iteration 382/ 500 | consumed samples: 1528 | elapsed time per iteration (ms): 4740.0 | throughput (token/sec/GPU): 1205.3 | learning rate: 4.074753E-05 | global batch size: 4 | lm loss: 1.120944E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1545]) -Dataloader batch - labels shape: torch.Size([1, 1545]) -Dataloader batch - attn_mask shape: torch.Size([1, 1545]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1129 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1545) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1545) torch.int64 - loss_mask : 400 / 1545 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1545, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1545, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1545, 1, 576) torch.bfloat16 - out : (1, 1545) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.2921 - top-1 accuracy : 11/400 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1310]) -Dataloader batch - labels shape: torch.Size([1, 1310]) -Dataloader batch - attn_mask shape: torch.Size([1, 1310]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1130 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1310) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1310) torch.int64 - loss_mask : 335 / 1310 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1310, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1310, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1310, 1, 576) torch.bfloat16 - out : (1, 1310) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.3307 - top-1 accuracy : 9/335 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1739]) -Dataloader batch - labels shape: torch.Size([1, 1739]) -Dataloader batch - attn_mask shape: torch.Size([1, 1739]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1131 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1739) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1739) torch.int64 - loss_mask : 376 / 1739 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1739, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1739, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1739, 1, 576) torch.bfloat16 - out : (1, 1739) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.2285 - top-1 accuracy : 22/376 = 5.9% - -Dataloader batch - tokens shape: torch.Size([1, 1388]) -Dataloader batch - labels shape: torch.Size([1, 1388]) -Dataloader batch - attn_mask shape: torch.Size([1, 1388]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1132 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1388) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1388) torch.int64 - loss_mask : 303 / 1388 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1388, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1388, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1388, 1, 576) torch.bfloat16 - out : (1, 1388) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.2026 - top-1 accuracy : 27/303 = 8.9% - - [2026-04-29 13:59:57] iteration 383/ 500 | consumed samples: 1532 | elapsed time per iteration (ms): 4797.6 | throughput (token/sec/GPU): 1246.9 | learning rate: 4.044220E-05 | global batch size: 4 | lm loss: 1.126517E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1537]) -Dataloader batch - labels shape: torch.Size([1, 1537]) -Dataloader batch - attn_mask shape: torch.Size([1, 1537]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1133 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1537) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1537) torch.int64 - loss_mask : 350 / 1537 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1537, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1537, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1537, 1, 576) torch.bfloat16 - out : (1, 1537) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 350 tokens) : 11.3042 - top-1 accuracy : 23/350 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1591]) -Dataloader batch - labels shape: torch.Size([1, 1591]) -Dataloader batch - attn_mask shape: torch.Size([1, 1591]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1134 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1591) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1591) torch.int64 - loss_mask : 362 / 1591 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1591, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1591, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1591, 1, 576) torch.bfloat16 - out : (1, 1591) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2101 - top-1 accuracy : 12/362 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 980]) -Dataloader batch - labels shape: torch.Size([1, 980]) -Dataloader batch - attn_mask shape: torch.Size([1, 980]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1135 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 980) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 980) torch.int64 - loss_mask : 190 / 980 tokens (19.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (980, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (980, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (980, 1, 576) torch.bfloat16 - out : (1, 980) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 190 tokens) : 11.0600 - top-1 accuracy : 6/190 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1378]) -Dataloader batch - labels shape: torch.Size([1, 1378]) -Dataloader batch - attn_mask shape: torch.Size([1, 1378]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1136 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1378) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1378) torch.int64 - loss_mask : 338 / 1378 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1378, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1378, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1378, 1, 576) torch.bfloat16 - out : (1, 1378) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.2121 - top-1 accuracy : 9/338 = 2.7% - - [2026-04-29 14:00:01] iteration 384/ 500 | consumed samples: 1536 | elapsed time per iteration (ms): 4470.1 | throughput (token/sec/GPU): 1227.3 | learning rate: 4.013726E-05 | global batch size: 4 | lm loss: 1.121420E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1815]) -Dataloader batch - labels shape: torch.Size([1, 1815]) -Dataloader batch - attn_mask shape: torch.Size([1, 1815]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1137 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1815) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1815) torch.int64 - loss_mask : 432 / 1815 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1815, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1815, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1815, 1, 576) torch.bfloat16 - out : (1, 1815) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 432 tokens) : 11.3159 - top-1 accuracy : 10/432 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1719]) -Dataloader batch - labels shape: torch.Size([1, 1719]) -Dataloader batch - attn_mask shape: torch.Size([1, 1719]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1138 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1719) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1719) torch.int64 - loss_mask : 435 / 1719 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1719, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1719, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1719, 1, 576) torch.bfloat16 - out : (1, 1719) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 435 tokens) : 11.2442 - top-1 accuracy : 16/435 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1501]) -Dataloader batch - labels shape: torch.Size([1, 1501]) -Dataloader batch - attn_mask shape: torch.Size([1, 1501]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1139 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1501) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1501) torch.int64 - loss_mask : 388 / 1501 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1501, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1501, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1501, 1, 576) torch.bfloat16 - out : (1, 1501) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2912 - top-1 accuracy : 21/388 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1948]) -Dataloader batch - labels shape: torch.Size([1, 1948]) -Dataloader batch - attn_mask shape: torch.Size([1, 1948]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1140 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1948) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1948) torch.int64 - loss_mask : 583 / 1948 tokens (29.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1948, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1948, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1948, 1, 576) torch.bfloat16 - out : (1, 1948) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 583 tokens) : 11.3713 - top-1 accuracy : 9/583 = 1.5% - - [2026-04-29 14:00:06] iteration 385/ 500 | consumed samples: 1540 | elapsed time per iteration (ms): 5199.3 | throughput (token/sec/GPU): 1343.1 | learning rate: 3.983273E-05 | global batch size: 4 | lm loss: 1.131126E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1298]) -Dataloader batch - labels shape: torch.Size([1, 1298]) -Dataloader batch - attn_mask shape: torch.Size([1, 1298]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1141 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1298) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1298) torch.int64 - loss_mask : 406 / 1298 tokens (31.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1298, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1298, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1298, 1, 576) torch.bfloat16 - out : (1, 1298) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.4351 - top-1 accuracy : 1/406 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1816]) -Dataloader batch - labels shape: torch.Size([1, 1816]) -Dataloader batch - attn_mask shape: torch.Size([1, 1816]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1142 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1816) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1816) torch.int64 - loss_mask : 440 / 1816 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1816, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1816, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1816, 1, 576) torch.bfloat16 - out : (1, 1816) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 440 tokens) : 11.2555 - top-1 accuracy : 8/440 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1574]) -Dataloader batch - labels shape: torch.Size([1, 1574]) -Dataloader batch - attn_mask shape: torch.Size([1, 1574]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1143 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1574) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1574) torch.int64 - loss_mask : 362 / 1574 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1574, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1574, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1574, 1, 576) torch.bfloat16 - out : (1, 1574) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2103 - top-1 accuracy : 23/362 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1446]) -Dataloader batch - labels shape: torch.Size([1, 1446]) -Dataloader batch - attn_mask shape: torch.Size([1, 1446]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1144 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1446) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1446) torch.int64 - loss_mask : 394 / 1446 tokens (27.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1446, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1446, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1446, 1, 576) torch.bfloat16 - out : (1, 1446) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3445 - top-1 accuracy : 5/394 = 1.3% - - [2026-04-29 14:00:11] iteration 386/ 500 | consumed samples: 1544 | elapsed time per iteration (ms): 4750.8 | throughput (token/sec/GPU): 1291.2 | learning rate: 3.952862E-05 | global batch size: 4 | lm loss: 1.131269E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1145 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 387 / 1539 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.3459 - top-1 accuracy : 7/387 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1710]) -Dataloader batch - labels shape: torch.Size([1, 1710]) -Dataloader batch - attn_mask shape: torch.Size([1, 1710]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1146 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1710) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1710) torch.int64 - loss_mask : 358 / 1710 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1710, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1710, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1710, 1, 576) torch.bfloat16 - out : (1, 1710) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.2073 - top-1 accuracy : 40/358 = 11.2% - -Dataloader batch - tokens shape: torch.Size([1, 1521]) -Dataloader batch - labels shape: torch.Size([1, 1521]) -Dataloader batch - attn_mask shape: torch.Size([1, 1521]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1147 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1521) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1521) torch.int64 - loss_mask : 392 / 1521 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1521, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1521, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1521, 1, 576) torch.bfloat16 - out : (1, 1521) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 392 tokens) : 11.3162 - top-1 accuracy : 15/392 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1148 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 315 / 1424 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.2375 - top-1 accuracy : 2/315 = 0.6% - - [2026-04-29 14:00:16] iteration 387/ 500 | consumed samples: 1548 | elapsed time per iteration (ms): 4858.3 | throughput (token/sec/GPU): 1274.9 | learning rate: 3.922495E-05 | global batch size: 4 | lm loss: 1.128019E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1163]) -Dataloader batch - labels shape: torch.Size([1, 1163]) -Dataloader batch - attn_mask shape: torch.Size([1, 1163]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1149 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1163) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1163) torch.int64 - loss_mask : 290 / 1163 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1163, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1163, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1163, 1, 576) torch.bfloat16 - out : (1, 1163) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.2816 - top-1 accuracy : 9/290 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1979]) -Dataloader batch - labels shape: torch.Size([1, 1979]) -Dataloader batch - attn_mask shape: torch.Size([1, 1979]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1150 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1979) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1979) torch.int64 - loss_mask : 626 / 1979 tokens (31.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1979, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1979, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1979, 1, 576) torch.bfloat16 - out : (1, 1979) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 626 tokens) : 11.4268 - top-1 accuracy : 4/626 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 993]) -Dataloader batch - labels shape: torch.Size([1, 993]) -Dataloader batch - attn_mask shape: torch.Size([1, 993]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1151 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 993) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 993) torch.int64 - loss_mask : 232 / 993 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (993, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (993, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (993, 1, 576) torch.bfloat16 - out : (1, 993) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 232 tokens) : 11.2456 - top-1 accuracy : 4/232 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1702]) -Dataloader batch - labels shape: torch.Size([1, 1702]) -Dataloader batch - attn_mask shape: torch.Size([1, 1702]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1152 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1702) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1702) torch.int64 - loss_mask : 364 / 1702 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1702, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1702, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1702, 1, 576) torch.bfloat16 - out : (1, 1702) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2188 - top-1 accuracy : 20/364 = 5.5% - - [2026-04-29 14:00:20] iteration 388/ 500 | consumed samples: 1552 | elapsed time per iteration (ms): 4522.4 | throughput (token/sec/GPU): 1290.7 | learning rate: 3.892173E-05 | global batch size: 4 | lm loss: 1.132107E+01 | loss scale: 1.0 | grad norm: 0.194 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1227]) -Dataloader batch - labels shape: torch.Size([1, 1227]) -Dataloader batch - attn_mask shape: torch.Size([1, 1227]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1153 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1227) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1227) torch.int64 - loss_mask : 327 / 1227 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1227, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1227, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1227, 1, 576) torch.bfloat16 - out : (1, 1227) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.3514 - top-1 accuracy : 1/327 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1853]) -Dataloader batch - labels shape: torch.Size([1, 1853]) -Dataloader batch - attn_mask shape: torch.Size([1, 1853]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1154 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1853) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1853) torch.int64 - loss_mask : 481 / 1853 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1853, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1853, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1853, 1, 576) torch.bfloat16 - out : (1, 1853) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 481 tokens) : 11.3050 - top-1 accuracy : 7/481 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 1155 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 416 / 1431 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.3830 - top-1 accuracy : 1/416 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1510]) -Dataloader batch - labels shape: torch.Size([1, 1510]) -Dataloader batch - attn_mask shape: torch.Size([1, 1510]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1156 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1510) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1510) torch.int64 - loss_mask : 359 / 1510 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1510, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1510, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1510, 1, 576) torch.bfloat16 - out : (1, 1510) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.2270 - top-1 accuracy : 26/359 = 7.2% - - [2026-04-29 14:00:25] iteration 389/ 500 | consumed samples: 1556 | elapsed time per iteration (ms): 4646.8 | throughput (token/sec/GPU): 1295.7 | learning rate: 3.861896E-05 | global batch size: 4 | lm loss: 1.131741E+01 | loss scale: 1.0 | grad norm: 0.192 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1807]) -Dataloader batch - labels shape: torch.Size([1, 1807]) -Dataloader batch - attn_mask shape: torch.Size([1, 1807]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1157 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1807) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1807) torch.int64 - loss_mask : 446 / 1807 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1807, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1807, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1807, 1, 576) torch.bfloat16 - out : (1, 1807) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 446 tokens) : 11.2735 - top-1 accuracy : 26/446 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1171]) -Dataloader batch - labels shape: torch.Size([1, 1171]) -Dataloader batch - attn_mask shape: torch.Size([1, 1171]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1158 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1171) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1171) torch.int64 - loss_mask : 254 / 1171 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1171, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1171, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1171, 1, 576) torch.bfloat16 - out : (1, 1171) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 254 tokens) : 11.2428 - top-1 accuracy : 24/254 = 9.4% - -Dataloader batch - tokens shape: torch.Size([1, 1439]) -Dataloader batch - labels shape: torch.Size([1, 1439]) -Dataloader batch - attn_mask shape: torch.Size([1, 1439]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1159 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1439) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1439) torch.int64 - loss_mask : 389 / 1439 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1439, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1439, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1439, 1, 576) torch.bfloat16 - out : (1, 1439) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.3415 - top-1 accuracy : 8/389 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1821]) -Dataloader batch - labels shape: torch.Size([1, 1821]) -Dataloader batch - attn_mask shape: torch.Size([1, 1821]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1160 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1821) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1821) torch.int64 - loss_mask : 470 / 1821 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1821, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1821, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1821, 1, 576) torch.bfloat16 - out : (1, 1821) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 470 tokens) : 11.3145 - top-1 accuracy : 20/470 = 4.3% - - [2026-04-29 14:00:30] iteration 390/ 500 | consumed samples: 1560 | elapsed time per iteration (ms): 4943.9 | throughput (token/sec/GPU): 1261.8 | learning rate: 3.831667E-05 | global batch size: 4 | lm loss: 1.129785E+01 | loss scale: 1.0 | grad norm: 0.170 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1548]) -Dataloader batch - labels shape: torch.Size([1, 1548]) -Dataloader batch - attn_mask shape: torch.Size([1, 1548]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1161 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1548) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1548) torch.int64 - loss_mask : 346 / 1548 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1548, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1548, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1548, 1, 576) torch.bfloat16 - out : (1, 1548) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 346 tokens) : 11.2306 - top-1 accuracy : 17/346 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1112]) -Dataloader batch - labels shape: torch.Size([1, 1112]) -Dataloader batch - attn_mask shape: torch.Size([1, 1112]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1162 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1112) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1112) torch.int64 - loss_mask : 318 / 1112 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1112, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1112, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1112, 1, 576) torch.bfloat16 - out : (1, 1112) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.3615 - top-1 accuracy : 15/318 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1468]) -Dataloader batch - labels shape: torch.Size([1, 1468]) -Dataloader batch - attn_mask shape: torch.Size([1, 1468]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1163 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1468) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1468) torch.int64 - loss_mask : 318 / 1468 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1468, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1468, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1468, 1, 576) torch.bfloat16 - out : (1, 1468) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.1505 - top-1 accuracy : 15/318 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1416]) -Dataloader batch - labels shape: torch.Size([1, 1416]) -Dataloader batch - attn_mask shape: torch.Size([1, 1416]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1164 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1416) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1416) torch.int64 - loss_mask : 416 / 1416 tokens (29.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1416, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1416, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1416, 1, 576) torch.bfloat16 - out : (1, 1416) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.4183 - top-1 accuracy : 1/416 = 0.2% - - [2026-04-29 14:00:34] iteration 391/ 500 | consumed samples: 1564 | elapsed time per iteration (ms): 4394.2 | throughput (token/sec/GPU): 1261.7 | learning rate: 3.801486E-05 | global batch size: 4 | lm loss: 1.129798E+01 | loss scale: 1.0 | grad norm: 0.215 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1041]) -Dataloader batch - labels shape: torch.Size([1, 1041]) -Dataloader batch - attn_mask shape: torch.Size([1, 1041]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1165 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1041) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1041) torch.int64 - loss_mask : 210 / 1041 tokens (20.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1041, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1041, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1041, 1, 576) torch.bfloat16 - out : (1, 1041) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 210 tokens) : 11.1707 - top-1 accuracy : 5/210 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1166 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 330 / 1424 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.2284 - top-1 accuracy : 22/330 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1381]) -Dataloader batch - labels shape: torch.Size([1, 1381]) -Dataloader batch - attn_mask shape: torch.Size([1, 1381]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1167 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1381) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1381) torch.int64 - loss_mask : 321 / 1381 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1381, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1381, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1381, 1, 576) torch.bfloat16 - out : (1, 1381) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 321 tokens) : 11.2442 - top-1 accuracy : 28/321 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1558]) -Dataloader batch - labels shape: torch.Size([1, 1558]) -Dataloader batch - attn_mask shape: torch.Size([1, 1558]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1168 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1558) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1558) torch.int64 - loss_mask : 434 / 1558 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1558, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1558, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1558, 1, 576) torch.bfloat16 - out : (1, 1558) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 434 tokens) : 11.3536 - top-1 accuracy : 16/434 = 3.7% - - [2026-04-29 14:00:39] iteration 392/ 500 | consumed samples: 1568 | elapsed time per iteration (ms): 4522.8 | throughput (token/sec/GPU): 1194.8 | learning rate: 3.771354E-05 | global batch size: 4 | lm loss: 1.126492E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1013]) -Dataloader batch - labels shape: torch.Size([1, 1013]) -Dataloader batch - attn_mask shape: torch.Size([1, 1013]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1169 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1013) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1013) torch.int64 - loss_mask : 252 / 1013 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1013, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1013, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1013, 1, 576) torch.bfloat16 - out : (1, 1013) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 252 tokens) : 11.2740 - top-1 accuracy : 5/252 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1170 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 412 / 1546 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3338 - top-1 accuracy : 26/412 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1764]) -Dataloader batch - labels shape: torch.Size([1, 1764]) -Dataloader batch - attn_mask shape: torch.Size([1, 1764]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1171 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1764) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1764) torch.int64 - loss_mask : 399 / 1764 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1764, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1764, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1764, 1, 576) torch.bfloat16 - out : (1, 1764) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 399 tokens) : 11.2619 - top-1 accuracy : 8/399 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1918]) -Dataloader batch - labels shape: torch.Size([1, 1918]) -Dataloader batch - attn_mask shape: torch.Size([1, 1918]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1172 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1918) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1918) torch.int64 - loss_mask : 552 / 1918 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1918, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1918, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1918, 1, 576) torch.bfloat16 - out : (1, 1918) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 552 tokens) : 11.3886 - top-1 accuracy : 9/552 = 1.6% - - [2026-04-29 14:00:44] iteration 393/ 500 | consumed samples: 1572 | elapsed time per iteration (ms): 4883.9 | throughput (token/sec/GPU): 1277.9 | learning rate: 3.741273E-05 | global batch size: 4 | lm loss: 1.132547E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1879]) -Dataloader batch - labels shape: torch.Size([1, 1879]) -Dataloader batch - attn_mask shape: torch.Size([1, 1879]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1173 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1879) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1879) torch.int64 - loss_mask : 507 / 1879 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1879, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1879, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1879, 1, 576) torch.bfloat16 - out : (1, 1879) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 507 tokens) : 11.3436 - top-1 accuracy : 3/507 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1419]) -Dataloader batch - labels shape: torch.Size([1, 1419]) -Dataloader batch - attn_mask shape: torch.Size([1, 1419]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1174 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1419) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1419) torch.int64 - loss_mask : 326 / 1419 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1419, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1419, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1419, 1, 576) torch.bfloat16 - out : (1, 1419) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2601 - top-1 accuracy : 10/326 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1124]) -Dataloader batch - labels shape: torch.Size([1, 1124]) -Dataloader batch - attn_mask shape: torch.Size([1, 1124]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1175 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1124) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1124) torch.int64 - loss_mask : 264 / 1124 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1124, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1124, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1124, 1, 576) torch.bfloat16 - out : (1, 1124) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 264 tokens) : 11.2673 - top-1 accuracy : 2/264 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1593]) -Dataloader batch - labels shape: torch.Size([1, 1593]) -Dataloader batch - attn_mask shape: torch.Size([1, 1593]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1176 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1593) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1593) torch.int64 - loss_mask : 357 / 1593 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1593, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1593, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1593, 1, 576) torch.bfloat16 - out : (1, 1593) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2187 - top-1 accuracy : 23/357 = 6.4% - - [2026-04-29 14:00:49] iteration 394/ 500 | consumed samples: 1576 | elapsed time per iteration (ms): 4764.8 | throughput (token/sec/GPU): 1262.4 | learning rate: 3.711244E-05 | global batch size: 4 | lm loss: 1.128037E+01 | loss scale: 1.0 | grad norm: 0.222 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1476]) -Dataloader batch - labels shape: torch.Size([1, 1476]) -Dataloader batch - attn_mask shape: torch.Size([1, 1476]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1177 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1476) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1476) torch.int64 - loss_mask : 426 / 1476 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1476, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1476, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1476, 1, 576) torch.bfloat16 - out : (1, 1476) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 426 tokens) : 11.3191 - top-1 accuracy : 1/426 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1531]) -Dataloader batch - labels shape: torch.Size([1, 1531]) -Dataloader batch - attn_mask shape: torch.Size([1, 1531]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1178 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1531) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1531) torch.int64 - loss_mask : 360 / 1531 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1531, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1531, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1531, 1, 576) torch.bfloat16 - out : (1, 1531) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.2152 - top-1 accuracy : 12/360 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1393]) -Dataloader batch - labels shape: torch.Size([1, 1393]) -Dataloader batch - attn_mask shape: torch.Size([1, 1393]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1179 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1393) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1393) torch.int64 - loss_mask : 332 / 1393 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1393, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1393, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1393, 1, 576) torch.bfloat16 - out : (1, 1393) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.2688 - top-1 accuracy : 21/332 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1180 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 405 / 1518 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.2613 - top-1 accuracy : 4/405 = 1.0% - - [2026-04-29 14:00:53] iteration 395/ 500 | consumed samples: 1580 | elapsed time per iteration (ms): 4648.5 | throughput (token/sec/GPU): 1273.1 | learning rate: 3.681269E-05 | global batch size: 4 | lm loss: 1.126823E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1020]) -Dataloader batch - labels shape: torch.Size([1, 1020]) -Dataloader batch - attn_mask shape: torch.Size([1, 1020]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1181 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1020) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1020) torch.int64 - loss_mask : 263 / 1020 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1020, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1020, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1020, 1, 576) torch.bfloat16 - out : (1, 1020) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.3733 - top-1 accuracy : 21/263 = 8.0% - -Dataloader batch - tokens shape: torch.Size([1, 1069]) -Dataloader batch - labels shape: torch.Size([1, 1069]) -Dataloader batch - attn_mask shape: torch.Size([1, 1069]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1182 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1069) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1069) torch.int64 - loss_mask : 260 / 1069 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1069, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1069, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1069, 1, 576) torch.bfloat16 - out : (1, 1069) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.2347 - top-1 accuracy : 4/260 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 995]) -Dataloader batch - labels shape: torch.Size([1, 995]) -Dataloader batch - attn_mask shape: torch.Size([1, 995]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1183 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 995) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 995) torch.int64 - loss_mask : 226 / 995 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (995, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (995, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (995, 1, 576) torch.bfloat16 - out : (1, 995) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 226 tokens) : 11.2231 - top-1 accuracy : 20/226 = 8.8% - -Dataloader batch - tokens shape: torch.Size([1, 1516]) -Dataloader batch - labels shape: torch.Size([1, 1516]) -Dataloader batch - attn_mask shape: torch.Size([1, 1516]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1184 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1516) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1516) torch.int64 - loss_mask : 320 / 1516 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1516, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1516, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1516, 1, 576) torch.bfloat16 - out : (1, 1516) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2158 - top-1 accuracy : 29/320 = 9.1% - - [2026-04-29 14:00:57] iteration 396/ 500 | consumed samples: 1584 | elapsed time per iteration (ms): 3930.8 | throughput (token/sec/GPU): 1170.2 | learning rate: 3.651347E-05 | global batch size: 4 | lm loss: 1.126070E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1563]) -Dataloader batch - labels shape: torch.Size([1, 1563]) -Dataloader batch - attn_mask shape: torch.Size([1, 1563]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1185 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1563) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1563) torch.int64 - loss_mask : 379 / 1563 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1563, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1563, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1563, 1, 576) torch.bfloat16 - out : (1, 1563) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.2773 - top-1 accuracy : 11/379 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1509]) -Dataloader batch - labels shape: torch.Size([1, 1509]) -Dataloader batch - attn_mask shape: torch.Size([1, 1509]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1186 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1509) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1509) torch.int64 - loss_mask : 345 / 1509 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1509, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1509, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1509, 1, 576) torch.bfloat16 - out : (1, 1509) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2334 - top-1 accuracy : 10/345 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 953]) -Dataloader batch - labels shape: torch.Size([1, 953]) -Dataloader batch - attn_mask shape: torch.Size([1, 953]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1187 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 953) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 953) torch.int64 - loss_mask : 203 / 953 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (953, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (953, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (953, 1, 576) torch.bfloat16 - out : (1, 953) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 203 tokens) : 11.2030 - top-1 accuracy : 10/203 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1490]) -Dataloader batch - labels shape: torch.Size([1, 1490]) -Dataloader batch - attn_mask shape: torch.Size([1, 1490]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1188 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1490) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1490) torch.int64 - loss_mask : 447 / 1490 tokens (30.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1490, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1490, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1490, 1, 576) torch.bfloat16 - out : (1, 1490) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 447 tokens) : 11.4015 - top-1 accuracy : 6/447 = 1.3% - - [2026-04-29 14:01:02] iteration 397/ 500 | consumed samples: 1588 | elapsed time per iteration (ms): 4490.1 | throughput (token/sec/GPU): 1228.3 | learning rate: 3.621481E-05 | global batch size: 4 | lm loss: 1.129572E+01 | loss scale: 1.0 | grad norm: 0.260 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1866]) -Dataloader batch - labels shape: torch.Size([1, 1866]) -Dataloader batch - attn_mask shape: torch.Size([1, 1866]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1189 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1866) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1866) torch.int64 - loss_mask : 504 / 1866 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1866, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1866, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1866, 1, 576) torch.bfloat16 - out : (1, 1866) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 504 tokens) : 11.3768 - top-1 accuracy : 16/504 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1752]) -Dataloader batch - labels shape: torch.Size([1, 1752]) -Dataloader batch - attn_mask shape: torch.Size([1, 1752]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1190 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1752) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1752) torch.int64 - loss_mask : 383 / 1752 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1752, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1752, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1752, 1, 576) torch.bfloat16 - out : (1, 1752) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.2400 - top-1 accuracy : 20/383 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1661]) -Dataloader batch - labels shape: torch.Size([1, 1661]) -Dataloader batch - attn_mask shape: torch.Size([1, 1661]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1191 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1661) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1661) torch.int64 - loss_mask : 398 / 1661 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1661, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1661, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1661, 1, 576) torch.bfloat16 - out : (1, 1661) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.2818 - top-1 accuracy : 18/398 = 4.5% - -Dataloader batch - tokens shape: torch.Size([1, 1402]) -Dataloader batch - labels shape: torch.Size([1, 1402]) -Dataloader batch - attn_mask shape: torch.Size([1, 1402]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1192 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1402) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1402) torch.int64 - loss_mask : 344 / 1402 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1402, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1402, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1402, 1, 576) torch.bfloat16 - out : (1, 1402) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.2378 - top-1 accuracy : 33/344 = 9.6% - - [2026-04-29 14:01:07] iteration 398/ 500 | consumed samples: 1592 | elapsed time per iteration (ms): 5213.3 | throughput (token/sec/GPU): 1281.5 | learning rate: 3.591671E-05 | global batch size: 4 | lm loss: 1.129205E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1193 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 367 / 1504 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2124 - top-1 accuracy : 6/367 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1118]) -Dataloader batch - labels shape: torch.Size([1, 1118]) -Dataloader batch - attn_mask shape: torch.Size([1, 1118]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1194 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1118) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1118) torch.int64 - loss_mask : 283 / 1118 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1118, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1118, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1118, 1, 576) torch.bfloat16 - out : (1, 1118) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 283 tokens) : 11.3015 - top-1 accuracy : 0/283 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1282]) -Dataloader batch - labels shape: torch.Size([1, 1282]) -Dataloader batch - attn_mask shape: torch.Size([1, 1282]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1195 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1282) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1282) torch.int64 - loss_mask : 379 / 1282 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1282, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1282, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1282, 1, 576) torch.bfloat16 - out : (1, 1282) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.4125 - top-1 accuracy : 1/379 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1387]) -Dataloader batch - labels shape: torch.Size([1, 1387]) -Dataloader batch - attn_mask shape: torch.Size([1, 1387]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1196 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1387) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1387) torch.int64 - loss_mask : 411 / 1387 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1387, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1387, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1387, 1, 576) torch.bfloat16 - out : (1, 1387) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.4101 - top-1 accuracy : 1/411 = 0.2% - - [2026-04-29 14:01:11] iteration 399/ 500 | consumed samples: 1596 | elapsed time per iteration (ms): 4208.1 | throughput (token/sec/GPU): 1257.3 | learning rate: 3.561919E-05 | global batch size: 4 | lm loss: 1.133900E+01 | loss scale: 1.0 | grad norm: 0.202 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1711]) -Dataloader batch - labels shape: torch.Size([1, 1711]) -Dataloader batch - attn_mask shape: torch.Size([1, 1711]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1197 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1711) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1711) torch.int64 - loss_mask : 397 / 1711 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1711, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1711, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1711, 1, 576) torch.bfloat16 - out : (1, 1711) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 397 tokens) : 11.2305 - top-1 accuracy : 40/397 = 10.1% - -Dataloader batch - tokens shape: torch.Size([1, 1096]) -Dataloader batch - labels shape: torch.Size([1, 1096]) -Dataloader batch - attn_mask shape: torch.Size([1, 1096]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1198 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1096) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1096) torch.int64 - loss_mask : 258 / 1096 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1096, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1096, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1096, 1, 576) torch.bfloat16 - out : (1, 1096) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 258 tokens) : 11.2161 - top-1 accuracy : 6/258 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1877]) -Dataloader batch - labels shape: torch.Size([1, 1877]) -Dataloader batch - attn_mask shape: torch.Size([1, 1877]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1199 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1877) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1877) torch.int64 - loss_mask : 528 / 1877 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1877, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1877, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1877, 1, 576) torch.bfloat16 - out : (1, 1877) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 528 tokens) : 11.3805 - top-1 accuracy : 17/528 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1341]) -Dataloader batch - labels shape: torch.Size([1, 1341]) -Dataloader batch - attn_mask shape: torch.Size([1, 1341]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1200 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1341) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1341) torch.int64 - loss_mask : 268 / 1341 tokens (20.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1341, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1341, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1341, 1, 576) torch.bfloat16 - out : (1, 1341) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 268 tokens) : 11.1666 - top-1 accuracy : 21/268 = 7.8% - - [2026-04-29 14:01:16] iteration 400/ 500 | consumed samples: 1600 | elapsed time per iteration (ms): 4801.9 | throughput (token/sec/GPU): 1254.7 | learning rate: 3.532226E-05 | global batch size: 4 | lm loss: 1.127072E+01 | loss scale: 1.0 | grad norm: 0.253 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4776.89, 4776.89) - forward-compute ................................: (3549.11, 3549.11) - backward-compute ...............................: (1223.86, 1223.86) - batch-generator ................................: (434.91, 434.91) - layernorm-grads-all-reduce .....................: (0.07, 0.07) - embedding-grads-all-reduce .....................: (0.10, 0.10) - all-grads-sync .................................: (1.08, 1.08) - params-all-gather ..............................: (0.38, 0.38) - optimizer-copy-to-main-grad ....................: (0.33, 0.33) - optimizer-clip-main-grad .......................: (1.27, 1.27) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.15, 2.15) - optimizer-copy-main-to-model-params ............: (0.55, 0.55) - optimizer ......................................: (6.72, 6.72) -saving checkpoint at iteration 400 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 400 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 400 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (1907.50, 1907.50) -Dataloader batch - tokens shape: torch.Size([1, 1492]) -Dataloader batch - labels shape: torch.Size([1, 1492]) -Dataloader batch - attn_mask shape: torch.Size([1, 1492]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1201 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1492) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1492) torch.int64 - loss_mask : 445 / 1492 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1492, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1492, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1492, 1, 576) torch.bfloat16 - out : (1, 1492) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 445 tokens) : 11.3833 - top-1 accuracy : 3/445 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1603]) -Dataloader batch - labels shape: torch.Size([1, 1603]) -Dataloader batch - attn_mask shape: torch.Size([1, 1603]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1202 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1603) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1603) torch.int64 - loss_mask : 402 / 1603 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1603, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1603, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1603, 1, 576) torch.bfloat16 - out : (1, 1603) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.2703 - top-1 accuracy : 16/402 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1440]) -Dataloader batch - labels shape: torch.Size([1, 1440]) -Dataloader batch - attn_mask shape: torch.Size([1, 1440]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1203 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1440) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1440) torch.int64 - loss_mask : 290 / 1440 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1440, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1440, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1440, 1, 576) torch.bfloat16 - out : (1, 1440) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 290 tokens) : 11.1299 - top-1 accuracy : 27/290 = 9.3% - -Dataloader batch - tokens shape: torch.Size([1, 1409]) -Dataloader batch - labels shape: torch.Size([1, 1409]) -Dataloader batch - attn_mask shape: torch.Size([1, 1409]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1204 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1409) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1409) torch.int64 - loss_mask : 348 / 1409 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1409, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1409, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1409, 1, 576) torch.bfloat16 - out : (1, 1409) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2683 - top-1 accuracy : 3/348 = 0.9% - - [2026-04-29 14:01:22] iteration 401/ 500 | consumed samples: 1604 | elapsed time per iteration (ms): 4765.8 | throughput (token/sec/GPU): 1247.2 | learning rate: 3.502594E-05 | global batch size: 4 | lm loss: 1.127627E+01 | loss scale: 1.0 | grad norm: 0.187 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1121]) -Dataloader batch - labels shape: torch.Size([1, 1121]) -Dataloader batch - attn_mask shape: torch.Size([1, 1121]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1205 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1121) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1121) torch.int64 - loss_mask : 312 / 1121 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1121, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1121, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1121, 1, 576) torch.bfloat16 - out : (1, 1121) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.3860 - top-1 accuracy : 11/312 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1620]) -Dataloader batch - labels shape: torch.Size([1, 1620]) -Dataloader batch - attn_mask shape: torch.Size([1, 1620]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1206 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1620) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1620) torch.int64 - loss_mask : 386 / 1620 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1620, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1620, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1620, 1, 576) torch.bfloat16 - out : (1, 1620) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2270 - top-1 accuracy : 22/386 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1207 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 419 / 1475 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.3917 - top-1 accuracy : 8/419 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1588]) -Dataloader batch - labels shape: torch.Size([1, 1588]) -Dataloader batch - attn_mask shape: torch.Size([1, 1588]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1208 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1588) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1588) torch.int64 - loss_mask : 364 / 1588 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1588, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1588, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1588, 1, 576) torch.bfloat16 - out : (1, 1588) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.2164 - top-1 accuracy : 18/364 = 4.9% - - [2026-04-29 14:01:27] iteration 402/ 500 | consumed samples: 1608 | elapsed time per iteration (ms): 4594.6 | throughput (token/sec/GPU): 1263.2 | learning rate: 3.473022E-05 | global batch size: 4 | lm loss: 1.130450E+01 | loss scale: 1.0 | grad norm: 0.176 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1505]) -Dataloader batch - labels shape: torch.Size([1, 1505]) -Dataloader batch - attn_mask shape: torch.Size([1, 1505]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1209 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1505) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1505) torch.int64 - loss_mask : 355 / 1505 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1505, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1505, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1505, 1, 576) torch.bfloat16 - out : (1, 1505) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2402 - top-1 accuracy : 2/355 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1210 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 412 / 1508 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3420 - top-1 accuracy : 2/412 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1126]) -Dataloader batch - labels shape: torch.Size([1, 1126]) -Dataloader batch - attn_mask shape: torch.Size([1, 1126]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1211 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1126) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1126) torch.int64 - loss_mask : 325 / 1126 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1126, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1126, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1126, 1, 576) torch.bfloat16 - out : (1, 1126) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.3555 - top-1 accuracy : 2/325 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1863]) -Dataloader batch - labels shape: torch.Size([1, 1863]) -Dataloader batch - attn_mask shape: torch.Size([1, 1863]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1212 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1863) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1863) torch.int64 - loss_mask : 517 / 1863 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1863, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1863, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1863, 1, 576) torch.bfloat16 - out : (1, 1863) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 517 tokens) : 11.3232 - top-1 accuracy : 15/517 = 2.9% - - [2026-04-29 14:01:32] iteration 403/ 500 | consumed samples: 1612 | elapsed time per iteration (ms): 4763.8 | throughput (token/sec/GPU): 1259.9 | learning rate: 3.443513E-05 | global batch size: 4 | lm loss: 1.131622E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1821]) -Dataloader batch - labels shape: torch.Size([1, 1821]) -Dataloader batch - attn_mask shape: torch.Size([1, 1821]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1213 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1821) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1821) torch.int64 - loss_mask : 449 / 1821 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1821, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1821, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1821, 1, 576) torch.bfloat16 - out : (1, 1821) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 449 tokens) : 11.3024 - top-1 accuracy : 16/449 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1619]) -Dataloader batch - labels shape: torch.Size([1, 1619]) -Dataloader batch - attn_mask shape: torch.Size([1, 1619]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1214 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1619) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1619) torch.int64 - loss_mask : 418 / 1619 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1619, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1619, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1619, 1, 576) torch.bfloat16 - out : (1, 1619) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 418 tokens) : 11.3018 - top-1 accuracy : 31/418 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1914]) -Dataloader batch - labels shape: torch.Size([1, 1914]) -Dataloader batch - attn_mask shape: torch.Size([1, 1914]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1215 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1914) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1914) torch.int64 - loss_mask : 549 / 1914 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1914, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1914, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1914, 1, 576) torch.bfloat16 - out : (1, 1914) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 549 tokens) : 11.3539 - top-1 accuracy : 4/549 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1753]) -Dataloader batch - labels shape: torch.Size([1, 1753]) -Dataloader batch - attn_mask shape: torch.Size([1, 1753]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1216 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1753) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1753) torch.int64 - loss_mask : 408 / 1753 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1753, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1753, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1753, 1, 576) torch.bfloat16 - out : (1, 1753) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.2917 - top-1 accuracy : 22/408 = 5.4% - - [2026-04-29 14:01:37] iteration 404/ 500 | consumed samples: 1616 | elapsed time per iteration (ms): 5375.2 | throughput (token/sec/GPU): 1322.2 | learning rate: 3.414068E-05 | global batch size: 4 | lm loss: 1.131541E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1770]) -Dataloader batch - labels shape: torch.Size([1, 1770]) -Dataloader batch - attn_mask shape: torch.Size([1, 1770]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1217 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1770) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1770) torch.int64 - loss_mask : 408 / 1770 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1770, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1770, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1770, 1, 576) torch.bfloat16 - out : (1, 1770) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.2544 - top-1 accuracy : 29/408 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1218 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 309 / 1460 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.1679 - top-1 accuracy : 11/309 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1768]) -Dataloader batch - labels shape: torch.Size([1, 1768]) -Dataloader batch - attn_mask shape: torch.Size([1, 1768]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1219 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1768) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1768) torch.int64 - loss_mask : 477 / 1768 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1768, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1768, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1768, 1, 576) torch.bfloat16 - out : (1, 1768) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 477 tokens) : 11.3765 - top-1 accuracy : 6/477 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1542]) -Dataloader batch - labels shape: torch.Size([1, 1542]) -Dataloader batch - attn_mask shape: torch.Size([1, 1542]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1220 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1542) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1542) torch.int64 - loss_mask : 387 / 1542 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1542, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1542, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1542, 1, 576) torch.bfloat16 - out : (1, 1542) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2857 - top-1 accuracy : 3/387 = 0.8% - - [2026-04-29 14:01:42] iteration 405/ 500 | consumed samples: 1620 | elapsed time per iteration (ms): 5129.9 | throughput (token/sec/GPU): 1274.9 | learning rate: 3.384688E-05 | global batch size: 4 | lm loss: 1.128200E+01 | loss scale: 1.0 | grad norm: 0.188 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1063]) -Dataloader batch - labels shape: torch.Size([1, 1063]) -Dataloader batch - attn_mask shape: torch.Size([1, 1063]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1221 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1063) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1063) torch.int64 - loss_mask : 242 / 1063 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1063, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1063, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1063, 1, 576) torch.bfloat16 - out : (1, 1063) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 242 tokens) : 11.2476 - top-1 accuracy : 16/242 = 6.6% - -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1222 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 403 / 1552 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.2866 - top-1 accuracy : 17/403 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1432]) -Dataloader batch - labels shape: torch.Size([1, 1432]) -Dataloader batch - attn_mask shape: torch.Size([1, 1432]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1223 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1432) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1432) torch.int64 - loss_mask : 388 / 1432 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1432, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1432, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1432, 1, 576) torch.bfloat16 - out : (1, 1432) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3555 - top-1 accuracy : 25/388 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1572]) -Dataloader batch - labels shape: torch.Size([1, 1572]) -Dataloader batch - attn_mask shape: torch.Size([1, 1572]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1224 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1572) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1572) torch.int64 - loss_mask : 407 / 1572 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1572, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1572, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1572, 1, 576) torch.bfloat16 - out : (1, 1572) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.3272 - top-1 accuracy : 7/407 = 1.7% - - [2026-04-29 14:01:47] iteration 406/ 500 | consumed samples: 1624 | elapsed time per iteration (ms): 4450.9 | throughput (token/sec/GPU): 1262.5 | learning rate: 3.355373E-05 | global batch size: 4 | lm loss: 1.131008E+01 | loss scale: 1.0 | grad norm: 0.193 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1365]) -Dataloader batch - labels shape: torch.Size([1, 1365]) -Dataloader batch - attn_mask shape: torch.Size([1, 1365]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1225 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1365) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1365) torch.int64 - loss_mask : 297 / 1365 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1365, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1365, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1365, 1, 576) torch.bfloat16 - out : (1, 1365) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 297 tokens) : 11.2132 - top-1 accuracy : 29/297 = 9.8% - -Dataloader batch - tokens shape: torch.Size([1, 1267]) -Dataloader batch - labels shape: torch.Size([1, 1267]) -Dataloader batch - attn_mask shape: torch.Size([1, 1267]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1226 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1267) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1267) torch.int64 - loss_mask : 339 / 1267 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1267, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1267, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1267, 1, 576) torch.bfloat16 - out : (1, 1267) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.3834 - top-1 accuracy : 17/339 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1476]) -Dataloader batch - labels shape: torch.Size([1, 1476]) -Dataloader batch - attn_mask shape: torch.Size([1, 1476]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1227 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1476) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1476) torch.int64 - loss_mask : 411 / 1476 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1476, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1476, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1476, 1, 576) torch.bfloat16 - out : (1, 1476) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.3384 - top-1 accuracy : 2/411 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1534]) -Dataloader batch - labels shape: torch.Size([1, 1534]) -Dataloader batch - attn_mask shape: torch.Size([1, 1534]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1228 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1534) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1534) torch.int64 - loss_mask : 398 / 1534 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1534, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1534, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1534, 1, 576) torch.bfloat16 - out : (1, 1534) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.2857 - top-1 accuracy : 8/398 = 2.0% - - [2026-04-29 14:01:51] iteration 407/ 500 | consumed samples: 1628 | elapsed time per iteration (ms): 4500.8 | throughput (token/sec/GPU): 1253.6 | learning rate: 3.326126E-05 | global batch size: 4 | lm loss: 1.130873E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1735]) -Dataloader batch - labels shape: torch.Size([1, 1735]) -Dataloader batch - attn_mask shape: torch.Size([1, 1735]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1229 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1735) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1735) torch.int64 - loss_mask : 353 / 1735 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1735, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1735, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1735, 1, 576) torch.bfloat16 - out : (1, 1735) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.1959 - top-1 accuracy : 2/353 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1702]) -Dataloader batch - labels shape: torch.Size([1, 1702]) -Dataloader batch - attn_mask shape: torch.Size([1, 1702]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1230 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1702) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1702) torch.int64 - loss_mask : 358 / 1702 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1702, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1702, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1702, 1, 576) torch.bfloat16 - out : (1, 1702) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.1944 - top-1 accuracy : 22/358 = 6.1% - -Dataloader batch - tokens shape: torch.Size([1, 1136]) -Dataloader batch - labels shape: torch.Size([1, 1136]) -Dataloader batch - attn_mask shape: torch.Size([1, 1136]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1231 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1136) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1136) torch.int64 - loss_mask : 292 / 1136 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1136, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1136, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1136, 1, 576) torch.bfloat16 - out : (1, 1136) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 292 tokens) : 11.2673 - top-1 accuracy : 16/292 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1060]) -Dataloader batch - labels shape: torch.Size([1, 1060]) -Dataloader batch - attn_mask shape: torch.Size([1, 1060]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1232 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1060) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1060) torch.int64 - loss_mask : 306 / 1060 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1060, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1060, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1060, 1, 576) torch.bfloat16 - out : (1, 1060) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.3093 - top-1 accuracy : 4/306 = 1.3% - - [2026-04-29 14:01:56] iteration 408/ 500 | consumed samples: 1632 | elapsed time per iteration (ms): 4512.5 | throughput (token/sec/GPU): 1248.3 | learning rate: 3.296947E-05 | global batch size: 4 | lm loss: 1.123795E+01 | loss scale: 1.0 | grad norm: 0.227 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1477]) -Dataloader batch - labels shape: torch.Size([1, 1477]) -Dataloader batch - attn_mask shape: torch.Size([1, 1477]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1233 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1477) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1477) torch.int64 - loss_mask : 288 / 1477 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1477, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1477, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1477, 1, 576) torch.bfloat16 - out : (1, 1477) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 288 tokens) : 11.1248 - top-1 accuracy : 22/288 = 7.6% - -Dataloader batch - tokens shape: torch.Size([1, 1045]) -Dataloader batch - labels shape: torch.Size([1, 1045]) -Dataloader batch - attn_mask shape: torch.Size([1, 1045]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1234 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1045) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1045) torch.int64 - loss_mask : 261 / 1045 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1045, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1045, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1045, 1, 576) torch.bfloat16 - out : (1, 1045) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 261 tokens) : 11.1997 - top-1 accuracy : 7/261 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1720]) -Dataloader batch - labels shape: torch.Size([1, 1720]) -Dataloader batch - attn_mask shape: torch.Size([1, 1720]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1235 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1720) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1720) torch.int64 - loss_mask : 363 / 1720 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1720, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1720, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1720, 1, 576) torch.bfloat16 - out : (1, 1720) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.2431 - top-1 accuracy : 36/363 = 9.9% - -Dataloader batch - tokens shape: torch.Size([1, 1839]) -Dataloader batch - labels shape: torch.Size([1, 1839]) -Dataloader batch - attn_mask shape: torch.Size([1, 1839]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1236 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1839) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1839) torch.int64 - loss_mask : 472 / 1839 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1839, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1839, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1839, 1, 576) torch.bfloat16 - out : (1, 1839) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 472 tokens) : 11.3457 - top-1 accuracy : 20/472 = 4.2% - - [2026-04-29 14:02:01] iteration 409/ 500 | consumed samples: 1636 | elapsed time per iteration (ms): 4842.0 | throughput (token/sec/GPU): 1255.9 | learning rate: 3.267838E-05 | global batch size: 4 | lm loss: 1.124527E+01 | loss scale: 1.0 | grad norm: 0.218 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1700]) -Dataloader batch - labels shape: torch.Size([1, 1700]) -Dataloader batch - attn_mask shape: torch.Size([1, 1700]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1237 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1700) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1700) torch.int64 - loss_mask : 422 / 1700 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1700, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1700, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1700, 1, 576) torch.bfloat16 - out : (1, 1700) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.3265 - top-1 accuracy : 22/422 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1417]) -Dataloader batch - labels shape: torch.Size([1, 1417]) -Dataloader batch - attn_mask shape: torch.Size([1, 1417]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1238 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1417) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1417) torch.int64 - loss_mask : 390 / 1417 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1417, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1417, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1417, 1, 576) torch.bfloat16 - out : (1, 1417) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3048 - top-1 accuracy : 3/390 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1133]) -Dataloader batch - labels shape: torch.Size([1, 1133]) -Dataloader batch - attn_mask shape: torch.Size([1, 1133]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1239 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1133) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1133) torch.int64 - loss_mask : 294 / 1133 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1133, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1133, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1133, 1, 576) torch.bfloat16 - out : (1, 1133) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.3167 - top-1 accuracy : 24/294 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 920]) -Dataloader batch - labels shape: torch.Size([1, 920]) -Dataloader batch - attn_mask shape: torch.Size([1, 920]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1240 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 920) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 920) torch.int64 - loss_mask : 189 / 920 tokens (20.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (920, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (920, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (920, 1, 576) torch.bfloat16 - out : (1, 920) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 189 tokens) : 11.1755 - top-1 accuracy : 8/189 = 4.2% - - [2026-04-29 14:02:05] iteration 410/ 500 | consumed samples: 1640 | elapsed time per iteration (ms): 4296.8 | throughput (token/sec/GPU): 1203.2 | learning rate: 3.238799E-05 | global batch size: 4 | lm loss: 1.129568E+01 | loss scale: 1.0 | grad norm: 0.237 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1741]) -Dataloader batch - labels shape: torch.Size([1, 1741]) -Dataloader batch - attn_mask shape: torch.Size([1, 1741]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1241 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1741) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1741) torch.int64 - loss_mask : 388 / 1741 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1741, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1741, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1741, 1, 576) torch.bfloat16 - out : (1, 1741) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2382 - top-1 accuracy : 34/388 = 8.8% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1242 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 286 / 1364 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 286 tokens) : 11.1978 - top-1 accuracy : 21/286 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1243 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 328 / 1487 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 328 tokens) : 11.2542 - top-1 accuracy : 16/328 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1908]) -Dataloader batch - labels shape: torch.Size([1, 1908]) -Dataloader batch - attn_mask shape: torch.Size([1, 1908]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1244 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1908) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1908) torch.int64 - loss_mask : 545 / 1908 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1908, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1908, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1908, 1, 576) torch.bfloat16 - out : (1, 1908) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 545 tokens) : 11.3752 - top-1 accuracy : 23/545 = 4.2% - - [2026-04-29 14:02:10] iteration 411/ 500 | consumed samples: 1644 | elapsed time per iteration (ms): 5264.0 | throughput (token/sec/GPU): 1234.8 | learning rate: 3.209832E-05 | global batch size: 4 | lm loss: 1.128238E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1109]) -Dataloader batch - labels shape: torch.Size([1, 1109]) -Dataloader batch - attn_mask shape: torch.Size([1, 1109]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1245 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1109) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1109) torch.int64 - loss_mask : 265 / 1109 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1109, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1109, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1109, 1, 576) torch.bfloat16 - out : (1, 1109) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 265 tokens) : 11.2484 - top-1 accuracy : 13/265 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1107]) -Dataloader batch - labels shape: torch.Size([1, 1107]) -Dataloader batch - attn_mask shape: torch.Size([1, 1107]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1246 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1107) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1107) torch.int64 - loss_mask : 262 / 1107 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1107, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1107, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1107, 1, 576) torch.bfloat16 - out : (1, 1107) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 262 tokens) : 11.2530 - top-1 accuracy : 4/262 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1768]) -Dataloader batch - labels shape: torch.Size([1, 1768]) -Dataloader batch - attn_mask shape: torch.Size([1, 1768]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1247 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1768) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1768) torch.int64 - loss_mask : 390 / 1768 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1768, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1768, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1768, 1, 576) torch.bfloat16 - out : (1, 1768) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.2268 - top-1 accuracy : 27/390 = 6.9% - -Dataloader batch - tokens shape: torch.Size([1, 1467]) -Dataloader batch - labels shape: torch.Size([1, 1467]) -Dataloader batch - attn_mask shape: torch.Size([1, 1467]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1248 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1467) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1467) torch.int64 - loss_mask : 411 / 1467 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1467, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1467, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1467, 1, 576) torch.bfloat16 - out : (1, 1467) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.3707 - top-1 accuracy : 3/411 = 0.7% - - [2026-04-29 14:02:15] iteration 412/ 500 | consumed samples: 1648 | elapsed time per iteration (ms): 4472.1 | throughput (token/sec/GPU): 1218.9 | learning rate: 3.180938E-05 | global batch size: 4 | lm loss: 1.128082E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1482]) -Dataloader batch - labels shape: torch.Size([1, 1482]) -Dataloader batch - attn_mask shape: torch.Size([1, 1482]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1249 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1482) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1482) torch.int64 - loss_mask : 406 / 1482 tokens (27.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1482, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1482, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1482, 1, 576) torch.bfloat16 - out : (1, 1482) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.3618 - top-1 accuracy : 5/406 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 966]) -Dataloader batch - labels shape: torch.Size([1, 966]) -Dataloader batch - attn_mask shape: torch.Size([1, 966]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1250 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 966) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 966) torch.int64 - loss_mask : 196 / 966 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (966, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (966, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (966, 1, 576) torch.bfloat16 - out : (1, 966) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 196 tokens) : 11.1398 - top-1 accuracy : 5/196 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1251 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1418) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1418) torch.int64 - loss_mask : 376 / 1418 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1418, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1418, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1418, 1, 576) torch.bfloat16 - out : (1, 1418) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.3385 - top-1 accuracy : 4/376 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1527]) -Dataloader batch - labels shape: torch.Size([1, 1527]) -Dataloader batch - attn_mask shape: torch.Size([1, 1527]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1252 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1527) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1527) torch.int64 - loss_mask : 406 / 1527 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1527, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1527, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1527, 1, 576) torch.bfloat16 - out : (1, 1527) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 406 tokens) : 11.3139 - top-1 accuracy : 20/406 = 4.9% - - [2026-04-29 14:02:19] iteration 413/ 500 | consumed samples: 1652 | elapsed time per iteration (ms): 4343.5 | throughput (token/sec/GPU): 1241.6 | learning rate: 3.152119E-05 | global batch size: 4 | lm loss: 1.130999E+01 | loss scale: 1.0 | grad norm: 0.177 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1820]) -Dataloader batch - labels shape: torch.Size([1, 1820]) -Dataloader batch - attn_mask shape: torch.Size([1, 1820]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1253 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1820) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1820) torch.int64 - loss_mask : 459 / 1820 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1820, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1820, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1820, 1, 576) torch.bfloat16 - out : (1, 1820) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 459 tokens) : 11.2826 - top-1 accuracy : 4/459 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1711]) -Dataloader batch - labels shape: torch.Size([1, 1711]) -Dataloader batch - attn_mask shape: torch.Size([1, 1711]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1254 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1711) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1711) torch.int64 - loss_mask : 355 / 1711 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1711, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1711, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1711, 1, 576) torch.bfloat16 - out : (1, 1711) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2469 - top-1 accuracy : 39/355 = 11.0% - -Dataloader batch - tokens shape: torch.Size([1, 1909]) -Dataloader batch - labels shape: torch.Size([1, 1909]) -Dataloader batch - attn_mask shape: torch.Size([1, 1909]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1255 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1909) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1909) torch.int64 - loss_mask : 543 / 1909 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1909, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1909, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1909, 1, 576) torch.bfloat16 - out : (1, 1909) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 543 tokens) : 11.3489 - top-1 accuracy : 6/543 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1550]) -Dataloader batch - labels shape: torch.Size([1, 1550]) -Dataloader batch - attn_mask shape: torch.Size([1, 1550]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1256 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1550) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1550) torch.int64 - loss_mask : 351 / 1550 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1550, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1550, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1550, 1, 576) torch.bfloat16 - out : (1, 1550) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.2211 - top-1 accuracy : 3/351 = 0.9% - - [2026-04-29 14:02:24] iteration 414/ 500 | consumed samples: 1656 | elapsed time per iteration (ms): 5388.3 | throughput (token/sec/GPU): 1297.2 | learning rate: 3.123374E-05 | global batch size: 4 | lm loss: 1.128361E+01 | loss scale: 1.0 | grad norm: 0.174 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1257 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 398 / 1546 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.3097 - top-1 accuracy : 8/398 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1122]) -Dataloader batch - labels shape: torch.Size([1, 1122]) -Dataloader batch - attn_mask shape: torch.Size([1, 1122]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1258 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1122) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1122) torch.int64 - loss_mask : 261 / 1122 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1122, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1122, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1122, 1, 576) torch.bfloat16 - out : (1, 1122) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 261 tokens) : 11.2499 - top-1 accuracy : 6/261 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1072]) -Dataloader batch - labels shape: torch.Size([1, 1072]) -Dataloader batch - attn_mask shape: torch.Size([1, 1072]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1259 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1072) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1072) torch.int64 - loss_mask : 260 / 1072 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1072, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1072, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1072, 1, 576) torch.bfloat16 - out : (1, 1072) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.3033 - top-1 accuracy : 15/260 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1663]) -Dataloader batch - labels shape: torch.Size([1, 1663]) -Dataloader batch - attn_mask shape: torch.Size([1, 1663]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1260 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1663) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1663) torch.int64 - loss_mask : 387 / 1663 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1663, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1663, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1663, 1, 576) torch.bfloat16 - out : (1, 1663) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2333 - top-1 accuracy : 26/387 = 6.7% - - [2026-04-29 14:02:29] iteration 415/ 500 | consumed samples: 1660 | elapsed time per iteration (ms): 4457.5 | throughput (token/sec/GPU): 1212.1 | learning rate: 3.094706E-05 | global batch size: 4 | lm loss: 1.127384E+01 | loss scale: 1.0 | grad norm: 0.226 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1397]) -Dataloader batch - labels shape: torch.Size([1, 1397]) -Dataloader batch - attn_mask shape: torch.Size([1, 1397]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1261 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1397) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1397) torch.int64 - loss_mask : 320 / 1397 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1397, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1397, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1397, 1, 576) torch.bfloat16 - out : (1, 1397) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2501 - top-1 accuracy : 29/320 = 9.1% - -Dataloader batch - tokens shape: torch.Size([1, 1281]) -Dataloader batch - labels shape: torch.Size([1, 1281]) -Dataloader batch - attn_mask shape: torch.Size([1, 1281]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1262 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1281) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1281) torch.int64 - loss_mask : 373 / 1281 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1281, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1281, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1281, 1, 576) torch.bfloat16 - out : (1, 1281) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.4225 - top-1 accuracy : 2/373 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1463]) -Dataloader batch - labels shape: torch.Size([1, 1463]) -Dataloader batch - attn_mask shape: torch.Size([1, 1463]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1263 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1463) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1463) torch.int64 - loss_mask : 312 / 1463 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1463, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1463, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1463, 1, 576) torch.bfloat16 - out : (1, 1463) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.1611 - top-1 accuracy : 16/312 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1694]) -Dataloader batch - labels shape: torch.Size([1, 1694]) -Dataloader batch - attn_mask shape: torch.Size([1, 1694]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1264 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1694) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1694) torch.int64 - loss_mask : 344 / 1694 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1694, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1694, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1694, 1, 576) torch.bfloat16 - out : (1, 1694) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.2334 - top-1 accuracy : 35/344 = 10.2% - - [2026-04-29 14:02:34] iteration 416/ 500 | consumed samples: 1664 | elapsed time per iteration (ms): 4721.0 | throughput (token/sec/GPU): 1236.0 | learning rate: 3.066115E-05 | global batch size: 4 | lm loss: 1.127291E+01 | loss scale: 1.0 | grad norm: 0.184 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1719]) -Dataloader batch - labels shape: torch.Size([1, 1719]) -Dataloader batch - attn_mask shape: torch.Size([1, 1719]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1265 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1719) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1719) torch.int64 - loss_mask : 362 / 1719 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1719, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1719, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1719, 1, 576) torch.bfloat16 - out : (1, 1719) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2075 - top-1 accuracy : 29/362 = 8.0% - -Dataloader batch - tokens shape: torch.Size([1, 1849]) -Dataloader batch - labels shape: torch.Size([1, 1849]) -Dataloader batch - attn_mask shape: torch.Size([1, 1849]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1266 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1849) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1849) torch.int64 - loss_mask : 488 / 1849 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1849, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1849, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1849, 1, 576) torch.bfloat16 - out : (1, 1849) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 488 tokens) : 11.2985 - top-1 accuracy : 15/488 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1302]) -Dataloader batch - labels shape: torch.Size([1, 1302]) -Dataloader batch - attn_mask shape: torch.Size([1, 1302]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1267 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1302) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1302) torch.int64 - loss_mask : 386 / 1302 tokens (29.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1302, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1302, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1302, 1, 576) torch.bfloat16 - out : (1, 1302) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3688 - top-1 accuracy : 8/386 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1550]) -Dataloader batch - labels shape: torch.Size([1, 1550]) -Dataloader batch - attn_mask shape: torch.Size([1, 1550]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1268 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1550) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1550) torch.int64 - loss_mask : 345 / 1550 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1550, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1550, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1550, 1, 576) torch.bfloat16 - out : (1, 1550) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.2416 - top-1 accuracy : 28/345 = 8.1% - - [2026-04-29 14:02:39] iteration 417/ 500 | consumed samples: 1668 | elapsed time per iteration (ms): 5063.0 | throughput (token/sec/GPU): 1268.0 | learning rate: 3.037603E-05 | global batch size: 4 | lm loss: 1.128241E+01 | loss scale: 1.0 | grad norm: 0.210 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1540]) -Dataloader batch - labels shape: torch.Size([1, 1540]) -Dataloader batch - attn_mask shape: torch.Size([1, 1540]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1269 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1540) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1540) torch.int64 - loss_mask : 390 / 1540 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1540, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1540, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1540, 1, 576) torch.bfloat16 - out : (1, 1540) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.3332 - top-1 accuracy : 14/390 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1471]) -Dataloader batch - labels shape: torch.Size([1, 1471]) -Dataloader batch - attn_mask shape: torch.Size([1, 1471]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1270 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1471) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1471) torch.int64 - loss_mask : 279 / 1471 tokens (19.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1471, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1471, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1471, 1, 576) torch.bfloat16 - out : (1, 1471) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 279 tokens) : 11.1431 - top-1 accuracy : 32/279 = 11.5% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1271 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 322 / 1400 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 322 tokens) : 11.2330 - top-1 accuracy : 18/322 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1608]) -Dataloader batch - labels shape: torch.Size([1, 1608]) -Dataloader batch - attn_mask shape: torch.Size([1, 1608]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1272 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1608) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1608) torch.int64 - loss_mask : 347 / 1608 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1608, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1608, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1608, 1, 576) torch.bfloat16 - out : (1, 1608) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.1695 - top-1 accuracy : 23/347 = 6.6% - - [2026-04-29 14:02:44] iteration 418/ 500 | consumed samples: 1672 | elapsed time per iteration (ms): 4914.5 | throughput (token/sec/GPU): 1224.8 | learning rate: 3.009170E-05 | global batch size: 4 | lm loss: 1.122698E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1373]) -Dataloader batch - labels shape: torch.Size([1, 1373]) -Dataloader batch - attn_mask shape: torch.Size([1, 1373]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1273 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1373) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1373) torch.int64 - loss_mask : 402 / 1373 tokens (29.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1373, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1373, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1373, 1, 576) torch.bfloat16 - out : (1, 1373) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.3491 - top-1 accuracy : 7/402 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1710]) -Dataloader batch - labels shape: torch.Size([1, 1710]) -Dataloader batch - attn_mask shape: torch.Size([1, 1710]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1274 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1710) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1710) torch.int64 - loss_mask : 388 / 1710 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1710, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1710, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1710, 1, 576) torch.bfloat16 - out : (1, 1710) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2383 - top-1 accuracy : 31/388 = 8.0% - -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1275 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 337 / 1541 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.2122 - top-1 accuracy : 25/337 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1792]) -Dataloader batch - labels shape: torch.Size([1, 1792]) -Dataloader batch - attn_mask shape: torch.Size([1, 1792]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1276 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1792) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1792) torch.int64 - loss_mask : 419 / 1792 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1792, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1792, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1792, 1, 576) torch.bfloat16 - out : (1, 1792) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.2710 - top-1 accuracy : 20/419 = 4.8% - - [2026-04-29 14:02:49] iteration 419/ 500 | consumed samples: 1676 | elapsed time per iteration (ms): 5005.8 | throughput (token/sec/GPU): 1281.7 | learning rate: 2.980819E-05 | global batch size: 4 | lm loss: 1.127028E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1528]) -Dataloader batch - labels shape: torch.Size([1, 1528]) -Dataloader batch - attn_mask shape: torch.Size([1, 1528]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1277 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1528) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1528) torch.int64 - loss_mask : 374 / 1528 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1528, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1528, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1528, 1, 576) torch.bfloat16 - out : (1, 1528) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.2238 - top-1 accuracy : 8/374 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1864]) -Dataloader batch - labels shape: torch.Size([1, 1864]) -Dataloader batch - attn_mask shape: torch.Size([1, 1864]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1278 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1864) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1864) torch.int64 - loss_mask : 495 / 1864 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1864, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1864, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1864, 1, 576) torch.bfloat16 - out : (1, 1864) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 495 tokens) : 11.2980 - top-1 accuracy : 2/495 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1551]) -Dataloader batch - labels shape: torch.Size([1, 1551]) -Dataloader batch - attn_mask shape: torch.Size([1, 1551]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1279 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1551) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1551) torch.int64 - loss_mask : 491 / 1551 tokens (31.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1551, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1551, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1551, 1, 576) torch.bfloat16 - out : (1, 1551) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 491 tokens) : 11.4403 - top-1 accuracy : 4/491 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1764]) -Dataloader batch - labels shape: torch.Size([1, 1764]) -Dataloader batch - attn_mask shape: torch.Size([1, 1764]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1280 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1764) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1764) torch.int64 - loss_mask : 402 / 1764 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1764, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1764, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1764, 1, 576) torch.bfloat16 - out : (1, 1764) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.2704 - top-1 accuracy : 29/402 = 7.2% - - [2026-04-29 14:02:54] iteration 420/ 500 | consumed samples: 1680 | elapsed time per iteration (ms): 5045.4 | throughput (token/sec/GPU): 1329.3 | learning rate: 2.952549E-05 | global batch size: 4 | lm loss: 1.131562E+01 | loss scale: 1.0 | grad norm: 0.207 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (5033.36, 5033.36) - forward-compute ................................: (3757.96, 3757.96) - backward-compute ...............................: (1273.46, 1273.46) - batch-generator ................................: (453.96, 453.96) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.45, 0.45) - params-all-gather ..............................: (0.14, 0.14) - optimizer-copy-to-main-grad ....................: (0.11, 0.11) - optimizer-clip-main-grad .......................: (0.46, 0.46) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.22, 1.22) - optimizer-copy-main-to-model-params ............: (0.19, 0.19) - optimizer ......................................: (2.69, 2.69) -Dataloader batch - tokens shape: torch.Size([1, 1242]) -Dataloader batch - labels shape: torch.Size([1, 1242]) -Dataloader batch - attn_mask shape: torch.Size([1, 1242]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1281 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1242) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1242) torch.int64 - loss_mask : 313 / 1242 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1242, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1242, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1242, 1, 576) torch.bfloat16 - out : (1, 1242) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2924 - top-1 accuracy : 11/313 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1233]) -Dataloader batch - labels shape: torch.Size([1, 1233]) -Dataloader batch - attn_mask shape: torch.Size([1, 1233]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1282 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1233) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1233) torch.int64 - loss_mask : 343 / 1233 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1233, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1233, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1233, 1, 576) torch.bfloat16 - out : (1, 1233) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.3454 - top-1 accuracy : 1/343 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1334]) -Dataloader batch - labels shape: torch.Size([1, 1334]) -Dataloader batch - attn_mask shape: torch.Size([1, 1334]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1283 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1334) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1334) torch.int64 - loss_mask : 372 / 1334 tokens (27.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1334, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1334, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1334, 1, 576) torch.bfloat16 - out : (1, 1334) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.3717 - top-1 accuracy : 3/372 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1284 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 343 / 1695 tokens (20.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.1660 - top-1 accuracy : 22/343 = 6.4% - - [2026-04-29 14:02:58] iteration 421/ 500 | consumed samples: 1684 | elapsed time per iteration (ms): 4387.8 | throughput (token/sec/GPU): 1254.4 | learning rate: 2.924363E-05 | global batch size: 4 | lm loss: 1.129555E+01 | loss scale: 1.0 | grad norm: 0.203 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1435]) -Dataloader batch - labels shape: torch.Size([1, 1435]) -Dataloader batch - attn_mask shape: torch.Size([1, 1435]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1285 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1435) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1435) torch.int64 - loss_mask : 281 / 1435 tokens (19.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1435, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1435, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1435, 1, 576) torch.bfloat16 - out : (1, 1435) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 281 tokens) : 11.0995 - top-1 accuracy : 15/281 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1853]) -Dataloader batch - labels shape: torch.Size([1, 1853]) -Dataloader batch - attn_mask shape: torch.Size([1, 1853]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1286 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1853) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1853) torch.int64 - loss_mask : 485 / 1853 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1853, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1853, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1853, 1, 576) torch.bfloat16 - out : (1, 1853) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 485 tokens) : 11.3382 - top-1 accuracy : 12/485 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1357]) -Dataloader batch - labels shape: torch.Size([1, 1357]) -Dataloader batch - attn_mask shape: torch.Size([1, 1357]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1287 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1357) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1357) torch.int64 - loss_mask : 317 / 1357 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1357, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1357, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1357, 1, 576) torch.bfloat16 - out : (1, 1357) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2054 - top-1 accuracy : 10/317 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1103]) -Dataloader batch - labels shape: torch.Size([1, 1103]) -Dataloader batch - attn_mask shape: torch.Size([1, 1103]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1288 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1103) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1103) torch.int64 - loss_mask : 294 / 1103 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1103, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1103, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1103, 1, 576) torch.bfloat16 - out : (1, 1103) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.3303 - top-1 accuracy : 16/294 = 5.4% - - [2026-04-29 14:03:03] iteration 422/ 500 | consumed samples: 1688 | elapsed time per iteration (ms): 4663.6 | throughput (token/sec/GPU): 1232.5 | learning rate: 2.896261E-05 | global batch size: 4 | lm loss: 1.125723E+01 | loss scale: 1.0 | grad norm: 0.211 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 937]) -Dataloader batch - labels shape: torch.Size([1, 937]) -Dataloader batch - attn_mask shape: torch.Size([1, 937]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1289 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 937) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 937) torch.int64 - loss_mask : 207 / 937 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (937, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (937, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (937, 1, 576) torch.bfloat16 - out : (1, 937) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 207 tokens) : 11.2145 - top-1 accuracy : 6/207 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1152]) -Dataloader batch - labels shape: torch.Size([1, 1152]) -Dataloader batch - attn_mask shape: torch.Size([1, 1152]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1290 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1152) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1152) torch.int64 - loss_mask : 224 / 1152 tokens (19.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1152, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1152, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1152, 1, 576) torch.bfloat16 - out : (1, 1152) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 224 tokens) : 11.1899 - top-1 accuracy : 26/224 = 11.6% - -Dataloader batch - tokens shape: torch.Size([1, 1370]) -Dataloader batch - labels shape: torch.Size([1, 1370]) -Dataloader batch - attn_mask shape: torch.Size([1, 1370]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1291 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1370) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1370) torch.int64 - loss_mask : 329 / 1370 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1370, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1370, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1370, 1, 576) torch.bfloat16 - out : (1, 1370) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2509 - top-1 accuracy : 14/329 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1808]) -Dataloader batch - labels shape: torch.Size([1, 1808]) -Dataloader batch - attn_mask shape: torch.Size([1, 1808]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1292 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1808) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1808) torch.int64 - loss_mask : 450 / 1808 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1808, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1808, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1808, 1, 576) torch.bfloat16 - out : (1, 1808) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 450 tokens) : 11.2888 - top-1 accuracy : 26/450 = 5.8% - - [2026-04-29 14:03:07] iteration 423/ 500 | consumed samples: 1692 | elapsed time per iteration (ms): 4434.2 | throughput (token/sec/GPU): 1187.8 | learning rate: 2.868244E-05 | global batch size: 4 | lm loss: 1.124749E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1135]) -Dataloader batch - labels shape: torch.Size([1, 1135]) -Dataloader batch - attn_mask shape: torch.Size([1, 1135]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1293 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1135) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1135) torch.int64 - loss_mask : 218 / 1135 tokens (19.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1135, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1135, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1135, 1, 576) torch.bfloat16 - out : (1, 1135) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 218 tokens) : 11.1186 - top-1 accuracy : 14/218 = 6.4% - -Dataloader batch - tokens shape: torch.Size([1, 1630]) -Dataloader batch - labels shape: torch.Size([1, 1630]) -Dataloader batch - attn_mask shape: torch.Size([1, 1630]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1294 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1630) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1630) torch.int64 - loss_mask : 386 / 1630 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1630, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1630, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1630, 1, 576) torch.bfloat16 - out : (1, 1630) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2335 - top-1 accuracy : 37/386 = 9.6% - -Dataloader batch - tokens shape: torch.Size([1, 1439]) -Dataloader batch - labels shape: torch.Size([1, 1439]) -Dataloader batch - attn_mask shape: torch.Size([1, 1439]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1295 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1439) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1439) torch.int64 - loss_mask : 382 / 1439 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1439, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1439, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1439, 1, 576) torch.bfloat16 - out : (1, 1439) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.3062 - top-1 accuracy : 4/382 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 881]) -Dataloader batch - labels shape: torch.Size([1, 881]) -Dataloader batch - attn_mask shape: torch.Size([1, 881]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 1296 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 881) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 881) torch.int64 - loss_mask : 184 / 881 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (881, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (881, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (881, 1, 576) torch.bfloat16 - out : (1, 881) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 184 tokens) : 11.1598 - top-1 accuracy : 5/184 = 2.7% - - [2026-04-29 14:03:11] iteration 424/ 500 | consumed samples: 1696 | elapsed time per iteration (ms): 4266.0 | throughput (token/sec/GPU): 1192.0 | learning rate: 2.840314E-05 | global batch size: 4 | lm loss: 1.122425E+01 | loss scale: 1.0 | grad norm: 0.242 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1110]) -Dataloader batch - labels shape: torch.Size([1, 1110]) -Dataloader batch - attn_mask shape: torch.Size([1, 1110]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1297 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1110) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1110) torch.int64 - loss_mask : 267 / 1110 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1110, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1110, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1110, 1, 576) torch.bfloat16 - out : (1, 1110) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 267 tokens) : 11.2992 - top-1 accuracy : 22/267 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1356]) -Dataloader batch - labels shape: torch.Size([1, 1356]) -Dataloader batch - attn_mask shape: torch.Size([1, 1356]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1298 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1356) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1356) torch.int64 - loss_mask : 293 / 1356 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1356, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1356, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1356, 1, 576) torch.bfloat16 - out : (1, 1356) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 293 tokens) : 11.1909 - top-1 accuracy : 13/293 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1916]) -Dataloader batch - labels shape: torch.Size([1, 1916]) -Dataloader batch - attn_mask shape: torch.Size([1, 1916]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1299 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1916) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1916) torch.int64 - loss_mask : 552 / 1916 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1916, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1916, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1916, 1, 576) torch.bfloat16 - out : (1, 1916) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 552 tokens) : 11.4160 - top-1 accuracy : 16/552 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1650]) -Dataloader batch - labels shape: torch.Size([1, 1650]) -Dataloader batch - attn_mask shape: torch.Size([1, 1650]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1300 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1650) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1650) torch.int64 - loss_mask : 398 / 1650 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1650, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1650, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1650, 1, 576) torch.bfloat16 - out : (1, 1650) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.2504 - top-1 accuracy : 24/398 = 6.0% - - [2026-04-29 14:03:16] iteration 425/ 500 | consumed samples: 1700 | elapsed time per iteration (ms): 4809.6 | throughput (token/sec/GPU): 1254.2 | learning rate: 2.812471E-05 | global batch size: 4 | lm loss: 1.130803E+01 | loss scale: 1.0 | grad norm: 0.202 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1417]) -Dataloader batch - labels shape: torch.Size([1, 1417]) -Dataloader batch - attn_mask shape: torch.Size([1, 1417]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1301 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1417) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1417) torch.int64 - loss_mask : 319 / 1417 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1417, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1417, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1417, 1, 576) torch.bfloat16 - out : (1, 1417) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.2741 - top-1 accuracy : 27/319 = 8.5% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1302 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 408 / 1532 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.3175 - top-1 accuracy : 5/408 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1827]) -Dataloader batch - labels shape: torch.Size([1, 1827]) -Dataloader batch - attn_mask shape: torch.Size([1, 1827]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1303 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1827) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1827) torch.int64 - loss_mask : 464 / 1827 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1827, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1827, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1827, 1, 576) torch.bfloat16 - out : (1, 1827) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 464 tokens) : 11.2728 - top-1 accuracy : 13/464 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1684]) -Dataloader batch - labels shape: torch.Size([1, 1684]) -Dataloader batch - attn_mask shape: torch.Size([1, 1684]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1304 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1684) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1684) torch.int64 - loss_mask : 366 / 1684 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1684, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1684, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1684, 1, 576) torch.bfloat16 - out : (1, 1684) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2749 - top-1 accuracy : 24/366 = 6.6% - - [2026-04-29 14:03:21] iteration 426/ 500 | consumed samples: 1704 | elapsed time per iteration (ms): 5227.7 | throughput (token/sec/GPU): 1235.7 | learning rate: 2.784717E-05 | global batch size: 4 | lm loss: 1.128528E+01 | loss scale: 1.0 | grad norm: 0.182 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1781]) -Dataloader batch - labels shape: torch.Size([1, 1781]) -Dataloader batch - attn_mask shape: torch.Size([1, 1781]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1305 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1781) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1781) torch.int64 - loss_mask : 413 / 1781 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1781, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1781, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1781, 1, 576) torch.bfloat16 - out : (1, 1781) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2344 - top-1 accuracy : 17/413 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1504]) -Dataloader batch - labels shape: torch.Size([1, 1504]) -Dataloader batch - attn_mask shape: torch.Size([1, 1504]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1306 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1504) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1504) torch.int64 - loss_mask : 368 / 1504 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1504, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1504, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1504, 1, 576) torch.bfloat16 - out : (1, 1504) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.3033 - top-1 accuracy : 21/368 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1111]) -Dataloader batch - labels shape: torch.Size([1, 1111]) -Dataloader batch - attn_mask shape: torch.Size([1, 1111]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1307 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1111) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1111) torch.int64 - loss_mask : 312 / 1111 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1111, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1111, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1111, 1, 576) torch.bfloat16 - out : (1, 1111) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 312 tokens) : 11.3480 - top-1 accuracy : 7/312 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1540]) -Dataloader batch - labels shape: torch.Size([1, 1540]) -Dataloader batch - attn_mask shape: torch.Size([1, 1540]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1308 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1540) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1540) torch.int64 - loss_mask : 339 / 1540 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1540, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1540, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1540, 1, 576) torch.bfloat16 - out : (1, 1540) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2064 - top-1 accuracy : 21/339 = 6.2% - - [2026-04-29 14:03:26] iteration 427/ 500 | consumed samples: 1708 | elapsed time per iteration (ms): 5039.6 | throughput (token/sec/GPU): 1177.9 | learning rate: 2.757053E-05 | global batch size: 4 | lm loss: 1.127025E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1924]) -Dataloader batch - labels shape: torch.Size([1, 1924]) -Dataloader batch - attn_mask shape: torch.Size([1, 1924]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1309 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1924) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1924) torch.int64 - loss_mask : 562 / 1924 tokens (29.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1924, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1924, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1924, 1, 576) torch.bfloat16 - out : (1, 1924) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 562 tokens) : 11.3833 - top-1 accuracy : 8/562 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1613]) -Dataloader batch - labels shape: torch.Size([1, 1613]) -Dataloader batch - attn_mask shape: torch.Size([1, 1613]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1310 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1613) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1613) torch.int64 - loss_mask : 395 / 1613 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1613, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1613, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1613, 1, 576) torch.bfloat16 - out : (1, 1613) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.2695 - top-1 accuracy : 13/395 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1044]) -Dataloader batch - labels shape: torch.Size([1, 1044]) -Dataloader batch - attn_mask shape: torch.Size([1, 1044]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1311 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1044) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1044) torch.int64 - loss_mask : 232 / 1044 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1044, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1044, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1044, 1, 576) torch.bfloat16 - out : (1, 1044) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 232 tokens) : 11.1593 - top-1 accuracy : 5/232 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1312 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 388 / 1444 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3242 - top-1 accuracy : 4/388 = 1.0% - - [2026-04-29 14:03:31] iteration 428/ 500 | consumed samples: 1712 | elapsed time per iteration (ms): 4740.4 | throughput (token/sec/GPU): 1271.0 | learning rate: 2.729480E-05 | global batch size: 4 | lm loss: 1.130728E+01 | loss scale: 1.0 | grad norm: 0.198 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1917]) -Dataloader batch - labels shape: torch.Size([1, 1917]) -Dataloader batch - attn_mask shape: torch.Size([1, 1917]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1313 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1917) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1917) torch.int64 - loss_mask : 537 / 1917 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1917, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1917, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1917, 1, 576) torch.bfloat16 - out : (1, 1917) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 537 tokens) : 11.3971 - top-1 accuracy : 5/537 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1435]) -Dataloader batch - labels shape: torch.Size([1, 1435]) -Dataloader batch - attn_mask shape: torch.Size([1, 1435]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1314 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1435) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1435) torch.int64 - loss_mask : 298 / 1435 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1435, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1435, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1435, 1, 576) torch.bfloat16 - out : (1, 1435) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 298 tokens) : 11.1441 - top-1 accuracy : 5/298 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1200]) -Dataloader batch - labels shape: torch.Size([1, 1200]) -Dataloader batch - attn_mask shape: torch.Size([1, 1200]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1315 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1200) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1200) torch.int64 - loss_mask : 268 / 1200 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1200, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1200, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1200, 1, 576) torch.bfloat16 - out : (1, 1200) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 268 tokens) : 11.2260 - top-1 accuracy : 15/268 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1217]) -Dataloader batch - labels shape: torch.Size([1, 1217]) -Dataloader batch - attn_mask shape: torch.Size([1, 1217]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1316 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1217) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1217) torch.int64 - loss_mask : 305 / 1217 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1217, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1217, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1217, 1, 576) torch.bfloat16 - out : (1, 1217) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 305 tokens) : 11.2873 - top-1 accuracy : 10/305 = 3.3% - - [2026-04-29 14:03:36] iteration 429/ 500 | consumed samples: 1716 | elapsed time per iteration (ms): 4680.2 | throughput (token/sec/GPU): 1232.6 | learning rate: 2.701998E-05 | global batch size: 4 | lm loss: 1.128722E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1905]) -Dataloader batch - labels shape: torch.Size([1, 1905]) -Dataloader batch - attn_mask shape: torch.Size([1, 1905]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1317 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1905) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1905) torch.int64 - loss_mask : 549 / 1905 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1905, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1905, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1905, 1, 576) torch.bfloat16 - out : (1, 1905) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 549 tokens) : 11.3607 - top-1 accuracy : 14/549 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1586]) -Dataloader batch - labels shape: torch.Size([1, 1586]) -Dataloader batch - attn_mask shape: torch.Size([1, 1586]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1318 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1586) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1586) torch.int64 - loss_mask : 448 / 1586 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1586, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1586, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1586, 1, 576) torch.bfloat16 - out : (1, 1586) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 448 tokens) : 11.3437 - top-1 accuracy : 3/448 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1847]) -Dataloader batch - labels shape: torch.Size([1, 1847]) -Dataloader batch - attn_mask shape: torch.Size([1, 1847]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1319 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1847) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1847) torch.int64 - loss_mask : 494 / 1847 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1847, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1847, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1847, 1, 576) torch.bfloat16 - out : (1, 1847) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 494 tokens) : 11.3236 - top-1 accuracy : 3/494 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1766]) -Dataloader batch - labels shape: torch.Size([1, 1766]) -Dataloader batch - attn_mask shape: torch.Size([1, 1766]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1320 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1766) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1766) torch.int64 - loss_mask : 413 / 1766 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1766, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1766, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1766, 1, 576) torch.bfloat16 - out : (1, 1766) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2481 - top-1 accuracy : 12/413 = 2.9% - - [2026-04-29 14:03:41] iteration 430/ 500 | consumed samples: 1720 | elapsed time per iteration (ms): 5546.9 | throughput (token/sec/GPU): 1280.7 | learning rate: 2.674610E-05 | global batch size: 4 | lm loss: 1.132265E+01 | loss scale: 1.0 | grad norm: 0.184 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1805]) -Dataloader batch - labels shape: torch.Size([1, 1805]) -Dataloader batch - attn_mask shape: torch.Size([1, 1805]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1321 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1805) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1805) torch.int64 - loss_mask : 435 / 1805 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1805, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1805, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1805, 1, 576) torch.bfloat16 - out : (1, 1805) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 435 tokens) : 11.2751 - top-1 accuracy : 7/435 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1776]) -Dataloader batch - labels shape: torch.Size([1, 1776]) -Dataloader batch - attn_mask shape: torch.Size([1, 1776]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1322 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1776) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1776) torch.int64 - loss_mask : 443 / 1776 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1776, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1776, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1776, 1, 576) torch.bfloat16 - out : (1, 1776) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 443 tokens) : 11.3107 - top-1 accuracy : 7/443 = 1.6% - -Dataloader batch - tokens shape: torch.Size([1, 1002]) -Dataloader batch - labels shape: torch.Size([1, 1002]) -Dataloader batch - attn_mask shape: torch.Size([1, 1002]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1323 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1002) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1002) torch.int64 - loss_mask : 233 / 1002 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1002, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1002, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1002, 1, 576) torch.bfloat16 - out : (1, 1002) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 233 tokens) : 11.3011 - top-1 accuracy : 12/233 = 5.2% - -Dataloader batch - tokens shape: torch.Size([1, 1103]) -Dataloader batch - labels shape: torch.Size([1, 1103]) -Dataloader batch - attn_mask shape: torch.Size([1, 1103]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1324 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1103) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1103) torch.int64 - loss_mask : 334 / 1103 tokens (30.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1103, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1103, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1103, 1, 576) torch.bfloat16 - out : (1, 1103) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.3979 - top-1 accuracy : 0/334 = 0.0% - - [2026-04-29 14:03:46] iteration 431/ 500 | consumed samples: 1724 | elapsed time per iteration (ms): 4424.3 | throughput (token/sec/GPU): 1285.2 | learning rate: 2.647316E-05 | global batch size: 4 | lm loss: 1.131859E+01 | loss scale: 1.0 | grad norm: 0.238 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1514]) -Dataloader batch - labels shape: torch.Size([1, 1514]) -Dataloader batch - attn_mask shape: torch.Size([1, 1514]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1325 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1514) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1514) torch.int64 - loss_mask : 356 / 1514 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1514, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1514, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1514, 1, 576) torch.bfloat16 - out : (1, 1514) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2141 - top-1 accuracy : 4/356 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1412]) -Dataloader batch - labels shape: torch.Size([1, 1412]) -Dataloader batch - attn_mask shape: torch.Size([1, 1412]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1326 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1412) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1412) torch.int64 - loss_mask : 347 / 1412 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1412, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1412, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1412, 1, 576) torch.bfloat16 - out : (1, 1412) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2687 - top-1 accuracy : 9/347 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1066]) -Dataloader batch - labels shape: torch.Size([1, 1066]) -Dataloader batch - attn_mask shape: torch.Size([1, 1066]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1327 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1066) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1066) torch.int64 - loss_mask : 262 / 1066 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1066, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1066, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1066, 1, 576) torch.bfloat16 - out : (1, 1066) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 262 tokens) : 11.3247 - top-1 accuracy : 1/262 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1222]) -Dataloader batch - labels shape: torch.Size([1, 1222]) -Dataloader batch - attn_mask shape: torch.Size([1, 1222]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1328 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1222) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1222) torch.int64 - loss_mask : 279 / 1222 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1222, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1222, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1222, 1, 576) torch.bfloat16 - out : (1, 1222) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 279 tokens) : 11.2352 - top-1 accuracy : 8/279 = 2.9% - - [2026-04-29 14:03:50] iteration 432/ 500 | consumed samples: 1728 | elapsed time per iteration (ms): 4430.1 | throughput (token/sec/GPU): 1176.9 | learning rate: 2.620117E-05 | global batch size: 4 | lm loss: 1.125733E+01 | loss scale: 1.0 | grad norm: 0.175 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1583]) -Dataloader batch - labels shape: torch.Size([1, 1583]) -Dataloader batch - attn_mask shape: torch.Size([1, 1583]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1329 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1583) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1583) torch.int64 - loss_mask : 360 / 1583 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1583, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1583, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1583, 1, 576) torch.bfloat16 - out : (1, 1583) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.1905 - top-1 accuracy : 27/360 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1785]) -Dataloader batch - labels shape: torch.Size([1, 1785]) -Dataloader batch - attn_mask shape: torch.Size([1, 1785]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1330 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1785) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1785) torch.int64 - loss_mask : 473 / 1785 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1785, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1785, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1785, 1, 576) torch.bfloat16 - out : (1, 1785) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 473 tokens) : 11.3478 - top-1 accuracy : 7/473 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1233]) -Dataloader batch - labels shape: torch.Size([1, 1233]) -Dataloader batch - attn_mask shape: torch.Size([1, 1233]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1331 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1233) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1233) torch.int64 - loss_mask : 295 / 1233 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1233, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1233, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1233, 1, 576) torch.bfloat16 - out : (1, 1233) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.2625 - top-1 accuracy : 14/295 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1006]) -Dataloader batch - labels shape: torch.Size([1, 1006]) -Dataloader batch - attn_mask shape: torch.Size([1, 1006]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1332 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1006) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1006) torch.int64 - loss_mask : 251 / 1006 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1006, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1006, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1006, 1, 576) torch.bfloat16 - out : (1, 1006) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2345 - top-1 accuracy : 6/251 = 2.4% - - [2026-04-29 14:03:55] iteration 433/ 500 | consumed samples: 1732 | elapsed time per iteration (ms): 4589.8 | throughput (token/sec/GPU): 1221.6 | learning rate: 2.593014E-05 | global batch size: 4 | lm loss: 1.126782E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1001]) -Dataloader batch - labels shape: torch.Size([1, 1001]) -Dataloader batch - attn_mask shape: torch.Size([1, 1001]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1333 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1001) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1001) torch.int64 - loss_mask : 251 / 1001 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1001, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1001, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1001, 1, 576) torch.bfloat16 - out : (1, 1001) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2821 - top-1 accuracy : 1/251 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1694]) -Dataloader batch - labels shape: torch.Size([1, 1694]) -Dataloader batch - attn_mask shape: torch.Size([1, 1694]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1334 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1694) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1694) torch.int64 - loss_mask : 356 / 1694 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1694, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1694, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1694, 1, 576) torch.bfloat16 - out : (1, 1694) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2059 - top-1 accuracy : 8/356 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1499]) -Dataloader batch - labels shape: torch.Size([1, 1499]) -Dataloader batch - attn_mask shape: torch.Size([1, 1499]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1335 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1499) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1499) torch.int64 - loss_mask : 342 / 1499 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1499, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1499, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1499, 1, 576) torch.bfloat16 - out : (1, 1499) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.2006 - top-1 accuracy : 5/342 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1514]) -Dataloader batch - labels shape: torch.Size([1, 1514]) -Dataloader batch - attn_mask shape: torch.Size([1, 1514]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1336 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1514) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1514) torch.int64 - loss_mask : 367 / 1514 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1514, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1514, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1514, 1, 576) torch.bfloat16 - out : (1, 1514) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.2589 - top-1 accuracy : 8/367 = 2.2% - - [2026-04-29 14:03:59] iteration 434/ 500 | consumed samples: 1736 | elapsed time per iteration (ms): 4563.2 | throughput (token/sec/GPU): 1250.9 | learning rate: 2.566009E-05 | global batch size: 4 | lm loss: 1.123382E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1395]) -Dataloader batch - labels shape: torch.Size([1, 1395]) -Dataloader batch - attn_mask shape: torch.Size([1, 1395]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1337 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1395) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1395) torch.int64 - loss_mask : 415 / 1395 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1395, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1395, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1395, 1, 576) torch.bfloat16 - out : (1, 1395) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.3703 - top-1 accuracy : 0/415 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1472]) -Dataloader batch - labels shape: torch.Size([1, 1472]) -Dataloader batch - attn_mask shape: torch.Size([1, 1472]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1338 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1472) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1472) torch.int64 - loss_mask : 412 / 1472 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1472, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1472, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1472, 1, 576) torch.bfloat16 - out : (1, 1472) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 412 tokens) : 11.3361 - top-1 accuracy : 6/412 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1725]) -Dataloader batch - labels shape: torch.Size([1, 1725]) -Dataloader batch - attn_mask shape: torch.Size([1, 1725]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1339 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1725) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1725) torch.int64 - loss_mask : 372 / 1725 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1725, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1725, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1725, 1, 576) torch.bfloat16 - out : (1, 1725) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.1828 - top-1 accuracy : 25/372 = 6.7% - -Dataloader batch - tokens shape: torch.Size([1, 1480]) -Dataloader batch - labels shape: torch.Size([1, 1480]) -Dataloader batch - attn_mask shape: torch.Size([1, 1480]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1340 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1480) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1480) torch.int64 - loss_mask : 335 / 1480 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1480, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1480, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1480, 1, 576) torch.bfloat16 - out : (1, 1480) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2012 - top-1 accuracy : 5/335 = 1.5% - - [2026-04-29 14:04:04] iteration 435/ 500 | consumed samples: 1740 | elapsed time per iteration (ms): 4710.4 | throughput (token/sec/GPU): 1289.1 | learning rate: 2.539102E-05 | global batch size: 4 | lm loss: 1.127870E+01 | loss scale: 1.0 | grad norm: 0.179 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1188]) -Dataloader batch - labels shape: torch.Size([1, 1188]) -Dataloader batch - attn_mask shape: torch.Size([1, 1188]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1341 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1188) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1188) torch.int64 - loss_mask : 279 / 1188 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1188, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1188, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1188, 1, 576) torch.bfloat16 - out : (1, 1188) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 279 tokens) : 11.2429 - top-1 accuracy : 10/279 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1433]) -Dataloader batch - labels shape: torch.Size([1, 1433]) -Dataloader batch - attn_mask shape: torch.Size([1, 1433]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1342 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1433) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1433) torch.int64 - loss_mask : 327 / 1433 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1433, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1433, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1433, 1, 576) torch.bfloat16 - out : (1, 1433) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.2387 - top-1 accuracy : 6/327 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1423]) -Dataloader batch - labels shape: torch.Size([1, 1423]) -Dataloader batch - attn_mask shape: torch.Size([1, 1423]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1343 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1423) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1423) torch.int64 - loss_mask : 370 / 1423 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1423, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1423, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1423, 1, 576) torch.bfloat16 - out : (1, 1423) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.2671 - top-1 accuracy : 2/370 = 0.5% - -Dataloader batch - tokens shape: torch.Size([1, 1469]) -Dataloader batch - labels shape: torch.Size([1, 1469]) -Dataloader batch - attn_mask shape: torch.Size([1, 1469]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1344 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1469) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1469) torch.int64 - loss_mask : 314 / 1469 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1469, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1469, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1469, 1, 576) torch.bfloat16 - out : (1, 1469) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.1466 - top-1 accuracy : 5/314 = 1.6% - - [2026-04-29 14:04:09] iteration 436/ 500 | consumed samples: 1744 | elapsed time per iteration (ms): 4608.7 | throughput (token/sec/GPU): 1196.2 | learning rate: 2.512295E-05 | global batch size: 4 | lm loss: 1.122534E+01 | loss scale: 1.0 | grad norm: 0.209 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1207]) -Dataloader batch - labels shape: torch.Size([1, 1207]) -Dataloader batch - attn_mask shape: torch.Size([1, 1207]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1345 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1207) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1207) torch.int64 - loss_mask : 268 / 1207 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1207, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1207, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1207, 1, 576) torch.bfloat16 - out : (1, 1207) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 268 tokens) : 11.2522 - top-1 accuracy : 4/268 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1346 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 351 / 1508 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 351 tokens) : 11.2228 - top-1 accuracy : 18/351 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1846]) -Dataloader batch - labels shape: torch.Size([1, 1846]) -Dataloader batch - attn_mask shape: torch.Size([1, 1846]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1347 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1846) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1846) torch.int64 - loss_mask : 520 / 1846 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1846, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1846, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1846, 1, 576) torch.bfloat16 - out : (1, 1846) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 520 tokens) : 11.3516 - top-1 accuracy : 12/520 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 997]) -Dataloader batch - labels shape: torch.Size([1, 997]) -Dataloader batch - attn_mask shape: torch.Size([1, 997]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1348 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 997) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 997) torch.int64 - loss_mask : 239 / 997 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (997, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (997, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (997, 1, 576) torch.bfloat16 - out : (1, 997) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 239 tokens) : 11.2699 - top-1 accuracy : 6/239 = 2.5% - - [2026-04-29 14:04:13] iteration 437/ 500 | consumed samples: 1748 | elapsed time per iteration (ms): 4445.3 | throughput (token/sec/GPU): 1250.3 | learning rate: 2.485589E-05 | global batch size: 4 | lm loss: 1.128531E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1428]) -Dataloader batch - labels shape: torch.Size([1, 1428]) -Dataloader batch - attn_mask shape: torch.Size([1, 1428]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1349 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1428) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1428) torch.int64 - loss_mask : 370 / 1428 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1428, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1428, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1428, 1, 576) torch.bfloat16 - out : (1, 1428) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 370 tokens) : 11.2992 - top-1 accuracy : 5/370 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1484]) -Dataloader batch - labels shape: torch.Size([1, 1484]) -Dataloader batch - attn_mask shape: torch.Size([1, 1484]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1350 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1484) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1484) torch.int64 - loss_mask : 338 / 1484 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1484, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1484, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1484, 1, 576) torch.bfloat16 - out : (1, 1484) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.1783 - top-1 accuracy : 13/338 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1868]) -Dataloader batch - labels shape: torch.Size([1, 1868]) -Dataloader batch - attn_mask shape: torch.Size([1, 1868]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1351 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1868) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1868) torch.int64 - loss_mask : 510 / 1868 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1868, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1868, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1868, 1, 576) torch.bfloat16 - out : (1, 1868) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 510 tokens) : 11.3413 - top-1 accuracy : 12/510 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1457]) -Dataloader batch - labels shape: torch.Size([1, 1457]) -Dataloader batch - attn_mask shape: torch.Size([1, 1457]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1352 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1457) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1457) torch.int64 - loss_mask : 313 / 1457 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1457, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1457, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1457, 1, 576) torch.bfloat16 - out : (1, 1457) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.1786 - top-1 accuracy : 9/313 = 2.9% - - [2026-04-29 14:04:18] iteration 438/ 500 | consumed samples: 1752 | elapsed time per iteration (ms): 4936.9 | throughput (token/sec/GPU): 1263.4 | learning rate: 2.458984E-05 | global batch size: 4 | lm loss: 1.126189E+01 | loss scale: 1.0 | grad norm: 0.198 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1435]) -Dataloader batch - labels shape: torch.Size([1, 1435]) -Dataloader batch - attn_mask shape: torch.Size([1, 1435]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1353 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1435) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1435) torch.int64 - loss_mask : 404 / 1435 tokens (28.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1435, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1435, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1435, 1, 576) torch.bfloat16 - out : (1, 1435) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 404 tokens) : 11.3252 - top-1 accuracy : 8/404 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1425]) -Dataloader batch - labels shape: torch.Size([1, 1425]) -Dataloader batch - attn_mask shape: torch.Size([1, 1425]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1354 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1425) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1425) torch.int64 - loss_mask : 331 / 1425 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1425, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1425, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1425, 1, 576) torch.bfloat16 - out : (1, 1425) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.2847 - top-1 accuracy : 19/331 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1387]) -Dataloader batch - labels shape: torch.Size([1, 1387]) -Dataloader batch - attn_mask shape: torch.Size([1, 1387]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1355 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1387) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1387) torch.int64 - loss_mask : 291 / 1387 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1387, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1387, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1387, 1, 576) torch.bfloat16 - out : (1, 1387) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 291 tokens) : 11.1565 - top-1 accuracy : 25/291 = 8.6% - -Dataloader batch - tokens shape: torch.Size([1, 1400]) -Dataloader batch - labels shape: torch.Size([1, 1400]) -Dataloader batch - attn_mask shape: torch.Size([1, 1400]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1356 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1400) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1400) torch.int64 - loss_mask : 295 / 1400 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1400, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1400, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1400, 1, 576) torch.bfloat16 - out : (1, 1400) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.1666 - top-1 accuracy : 24/295 = 8.1% - - [2026-04-29 14:04:23] iteration 439/ 500 | consumed samples: 1756 | elapsed time per iteration (ms): 4616.0 | throughput (token/sec/GPU): 1223.3 | learning rate: 2.432481E-05 | global batch size: 4 | lm loss: 1.124248E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1403]) -Dataloader batch - labels shape: torch.Size([1, 1403]) -Dataloader batch - attn_mask shape: torch.Size([1, 1403]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1357 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1403) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1403) torch.int64 - loss_mask : 348 / 1403 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1403, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1403, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1403, 1, 576) torch.bfloat16 - out : (1, 1403) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2533 - top-1 accuracy : 12/348 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 958]) -Dataloader batch - labels shape: torch.Size([1, 958]) -Dataloader batch - attn_mask shape: torch.Size([1, 958]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1358 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 958) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 958) torch.int64 - loss_mask : 207 / 958 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (958, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (958, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (958, 1, 576) torch.bfloat16 - out : (1, 958) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 207 tokens) : 11.1995 - top-1 accuracy : 8/207 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1359 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1519) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1519) torch.int64 - loss_mask : 317 / 1519 tokens (20.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1519, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1519, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1519, 1, 576) torch.bfloat16 - out : (1, 1519) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.1548 - top-1 accuracy : 24/317 = 7.6% - -Dataloader batch - tokens shape: torch.Size([1, 1505]) -Dataloader batch - labels shape: torch.Size([1, 1505]) -Dataloader batch - attn_mask shape: torch.Size([1, 1505]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1360 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1505) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1505) torch.int64 - loss_mask : 349 / 1505 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1505, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1505, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1505, 1, 576) torch.bfloat16 - out : (1, 1505) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2082 - top-1 accuracy : 20/349 = 5.7% - - [2026-04-29 14:04:27] iteration 440/ 500 | consumed samples: 1760 | elapsed time per iteration (ms): 4528.2 | throughput (token/sec/GPU): 1189.2 | learning rate: 2.406083E-05 | global batch size: 4 | lm loss: 1.120570E+01 | loss scale: 1.0 | grad norm: 0.236 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4504.86, 4504.86) - forward-compute ................................: (3331.60, 3331.60) - backward-compute ...............................: (1169.11, 1169.11) - batch-generator ................................: (445.92, 445.92) - layernorm-grads-all-reduce .....................: (0.06, 0.06) - embedding-grads-all-reduce .....................: (0.10, 0.10) - all-grads-sync .................................: (1.17, 1.17) - params-all-gather ..............................: (0.42, 0.42) - optimizer-copy-to-main-grad ....................: (0.36, 0.36) - optimizer-clip-main-grad .......................: (1.39, 1.39) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.35, 2.35) - optimizer-copy-main-to-model-params ............: (0.59, 0.59) - optimizer ......................................: (7.19, 7.19) -Dataloader batch - tokens shape: torch.Size([1, 1877]) -Dataloader batch - labels shape: torch.Size([1, 1877]) -Dataloader batch - attn_mask shape: torch.Size([1, 1877]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1361 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1877) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1877) torch.int64 - loss_mask : 522 / 1877 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1877, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1877, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1877, 1, 576) torch.bfloat16 - out : (1, 1877) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 522 tokens) : 11.3629 - top-1 accuracy : 4/522 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1403]) -Dataloader batch - labels shape: torch.Size([1, 1403]) -Dataloader batch - attn_mask shape: torch.Size([1, 1403]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1362 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1403) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1403) torch.int64 - loss_mask : 329 / 1403 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1403, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1403, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1403, 1, 576) torch.bfloat16 - out : (1, 1403) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2216 - top-1 accuracy : 19/329 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1519]) -Dataloader batch - labels shape: torch.Size([1, 1519]) -Dataloader batch - attn_mask shape: torch.Size([1, 1519]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1363 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1519) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1519) torch.int64 - loss_mask : 380 / 1519 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1519, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1519, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1519, 1, 576) torch.bfloat16 - out : (1, 1519) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 380 tokens) : 11.2969 - top-1 accuracy : 9/380 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1569]) -Dataloader batch - labels shape: torch.Size([1, 1569]) -Dataloader batch - attn_mask shape: torch.Size([1, 1569]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1364 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1569) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1569) torch.int64 - loss_mask : 431 / 1569 tokens (27.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1569, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1569, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1569, 1, 576) torch.bfloat16 - out : (1, 1569) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 431 tokens) : 11.3617 - top-1 accuracy : 21/431 = 4.9% - - [2026-04-29 14:04:32] iteration 441/ 500 | consumed samples: 1764 | elapsed time per iteration (ms): 5054.3 | throughput (token/sec/GPU): 1259.9 | learning rate: 2.379789E-05 | global batch size: 4 | lm loss: 1.131950E+01 | loss scale: 1.0 | grad norm: 0.186 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1365 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 313 / 1475 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2278 - top-1 accuracy : 15/313 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1366 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 441 / 1487 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 441 tokens) : 11.3917 - top-1 accuracy : 8/441 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1098]) -Dataloader batch - labels shape: torch.Size([1, 1098]) -Dataloader batch - attn_mask shape: torch.Size([1, 1098]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1367 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1098) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1098) torch.int64 - loss_mask : 234 / 1098 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1098, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1098, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1098, 1, 576) torch.bfloat16 - out : (1, 1098) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 234 tokens) : 11.2319 - top-1 accuracy : 10/234 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1454]) -Dataloader batch - labels shape: torch.Size([1, 1454]) -Dataloader batch - attn_mask shape: torch.Size([1, 1454]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1368 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1454) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1454) torch.int64 - loss_mask : 309 / 1454 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1454, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1454, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1454, 1, 576) torch.bfloat16 - out : (1, 1454) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.1270 - top-1 accuracy : 18/309 = 5.8% - - [2026-04-29 14:04:37] iteration 442/ 500 | consumed samples: 1768 | elapsed time per iteration (ms): 4553.5 | throughput (token/sec/GPU): 1210.9 | learning rate: 2.353601E-05 | global batch size: 4 | lm loss: 1.126023E+01 | loss scale: 1.0 | grad norm: 0.180 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1402]) -Dataloader batch - labels shape: torch.Size([1, 1402]) -Dataloader batch - attn_mask shape: torch.Size([1, 1402]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1369 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1402) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1402) torch.int64 - loss_mask : 422 / 1402 tokens (30.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1402, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1402, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1402, 1, 576) torch.bfloat16 - out : (1, 1402) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 422 tokens) : 11.4165 - top-1 accuracy : 8/422 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1266]) -Dataloader batch - labels shape: torch.Size([1, 1266]) -Dataloader batch - attn_mask shape: torch.Size([1, 1266]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1370 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1266) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1266) torch.int64 - loss_mask : 327 / 1266 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1266, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1266, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1266, 1, 576) torch.bfloat16 - out : (1, 1266) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 327 tokens) : 11.3463 - top-1 accuracy : 1/327 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1466]) -Dataloader batch - labels shape: torch.Size([1, 1466]) -Dataloader batch - attn_mask shape: torch.Size([1, 1466]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1371 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1466) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1466) torch.int64 - loss_mask : 309 / 1466 tokens (21.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1466, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1466, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1466, 1, 576) torch.bfloat16 - out : (1, 1466) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.2182 - top-1 accuracy : 29/309 = 9.4% - -Dataloader batch - tokens shape: torch.Size([1, 1606]) -Dataloader batch - labels shape: torch.Size([1, 1606]) -Dataloader batch - attn_mask shape: torch.Size([1, 1606]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1372 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1606) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1606) torch.int64 - loss_mask : 382 / 1606 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1606, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1606, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1606, 1, 576) torch.bfloat16 - out : (1, 1606) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.2767 - top-1 accuracy : 12/382 = 3.1% - - [2026-04-29 14:04:41] iteration 443/ 500 | consumed samples: 1772 | elapsed time per iteration (ms): 4578.1 | throughput (token/sec/GPU): 1253.8 | learning rate: 2.327520E-05 | global batch size: 4 | lm loss: 1.132092E+01 | loss scale: 1.0 | grad norm: 0.196 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1841]) -Dataloader batch - labels shape: torch.Size([1, 1841]) -Dataloader batch - attn_mask shape: torch.Size([1, 1841]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1373 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1841) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1841) torch.int64 - loss_mask : 488 / 1841 tokens (26.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1841, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1841, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1841, 1, 576) torch.bfloat16 - out : (1, 1841) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 488 tokens) : 11.3362 - top-1 accuracy : 6/488 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1182]) -Dataloader batch - labels shape: torch.Size([1, 1182]) -Dataloader batch - attn_mask shape: torch.Size([1, 1182]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1374 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1182) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1182) torch.int64 - loss_mask : 292 / 1182 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1182, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1182, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1182, 1, 576) torch.bfloat16 - out : (1, 1182) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 292 tokens) : 11.2717 - top-1 accuracy : 4/292 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1038]) -Dataloader batch - labels shape: torch.Size([1, 1038]) -Dataloader batch - attn_mask shape: torch.Size([1, 1038]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1375 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1038) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1038) torch.int64 - loss_mask : 266 / 1038 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1038, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1038, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1038, 1, 576) torch.bfloat16 - out : (1, 1038) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 266 tokens) : 11.3302 - top-1 accuracy : 2/266 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1509]) -Dataloader batch - labels shape: torch.Size([1, 1509]) -Dataloader batch - attn_mask shape: torch.Size([1, 1509]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1376 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1509) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1509) torch.int64 - loss_mask : 388 / 1509 tokens (25.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1509, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1509, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1509, 1, 576) torch.bfloat16 - out : (1, 1509) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3189 - top-1 accuracy : 19/388 = 4.9% - - [2026-04-29 14:04:46] iteration 444/ 500 | consumed samples: 1776 | elapsed time per iteration (ms): 4381.2 | throughput (token/sec/GPU): 1271.3 | learning rate: 2.301547E-05 | global batch size: 4 | lm loss: 1.131724E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1617]) -Dataloader batch - labels shape: torch.Size([1, 1617]) -Dataloader batch - attn_mask shape: torch.Size([1, 1617]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1377 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1617) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1617) torch.int64 - loss_mask : 381 / 1617 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1617, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1617, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1617, 1, 576) torch.bfloat16 - out : (1, 1617) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 381 tokens) : 11.3039 - top-1 accuracy : 21/381 = 5.5% - -Dataloader batch - tokens shape: torch.Size([1, 1848]) -Dataloader batch - labels shape: torch.Size([1, 1848]) -Dataloader batch - attn_mask shape: torch.Size([1, 1848]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1378 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1848) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1848) torch.int64 - loss_mask : 492 / 1848 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1848, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1848, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1848, 1, 576) torch.bfloat16 - out : (1, 1848) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 492 tokens) : 11.3397 - top-1 accuracy : 14/492 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1726]) -Dataloader batch - labels shape: torch.Size([1, 1726]) -Dataloader batch - attn_mask shape: torch.Size([1, 1726]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1379 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1726) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1726) torch.int64 - loss_mask : 418 / 1726 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1726, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1726, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1726, 1, 576) torch.bfloat16 - out : (1, 1726) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 418 tokens) : 11.3025 - top-1 accuracy : 8/418 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1502]) -Dataloader batch - labels shape: torch.Size([1, 1502]) -Dataloader batch - attn_mask shape: torch.Size([1, 1502]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1380 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1502) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1502) torch.int64 - loss_mask : 402 / 1502 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1502, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1502, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1502, 1, 576) torch.bfloat16 - out : (1, 1502) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.3124 - top-1 accuracy : 2/402 = 0.5% - - [2026-04-29 14:04:51] iteration 445/ 500 | consumed samples: 1780 | elapsed time per iteration (ms): 5260.9 | throughput (token/sec/GPU): 1272.2 | learning rate: 2.275683E-05 | global batch size: 4 | lm loss: 1.131599E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1454]) -Dataloader batch - labels shape: torch.Size([1, 1454]) -Dataloader batch - attn_mask shape: torch.Size([1, 1454]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1381 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1454) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1454) torch.int64 - loss_mask : 402 / 1454 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1454, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1454, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1454, 1, 576) torch.bfloat16 - out : (1, 1454) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.3463 - top-1 accuracy : 0/402 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1052]) -Dataloader batch - labels shape: torch.Size([1, 1052]) -Dataloader batch - attn_mask shape: torch.Size([1, 1052]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1382 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1052) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1052) torch.int64 - loss_mask : 227 / 1052 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1052, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1052, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1052, 1, 576) torch.bfloat16 - out : (1, 1052) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 227 tokens) : 11.1334 - top-1 accuracy : 6/227 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 997]) -Dataloader batch - labels shape: torch.Size([1, 997]) -Dataloader batch - attn_mask shape: torch.Size([1, 997]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1383 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 997) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 997) torch.int64 - loss_mask : 222 / 997 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (997, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (997, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (997, 1, 576) torch.bfloat16 - out : (1, 997) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 222 tokens) : 11.2523 - top-1 accuracy : 7/222 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1103]) -Dataloader batch - labels shape: torch.Size([1, 1103]) -Dataloader batch - attn_mask shape: torch.Size([1, 1103]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1384 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1103) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1103) torch.int64 - loss_mask : 253 / 1103 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1103, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1103, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1103, 1, 576) torch.bfloat16 - out : (1, 1103) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 253 tokens) : 11.2546 - top-1 accuracy : 11/253 = 4.3% - - [2026-04-29 14:04:55] iteration 446/ 500 | consumed samples: 1784 | elapsed time per iteration (ms): 4046.3 | throughput (token/sec/GPU): 1138.3 | learning rate: 2.249929E-05 | global batch size: 4 | lm loss: 1.126260E+01 | loss scale: 1.0 | grad norm: 0.239 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1214]) -Dataloader batch - labels shape: torch.Size([1, 1214]) -Dataloader batch - attn_mask shape: torch.Size([1, 1214]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1385 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1214) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1214) torch.int64 - loss_mask : 300 / 1214 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1214, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1214, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1214, 1, 576) torch.bfloat16 - out : (1, 1214) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 300 tokens) : 11.3009 - top-1 accuracy : 2/300 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1360]) -Dataloader batch - labels shape: torch.Size([1, 1360]) -Dataloader batch - attn_mask shape: torch.Size([1, 1360]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1386 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1360) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1360) torch.int64 - loss_mask : 388 / 1360 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1360, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1360, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1360, 1, 576) torch.bfloat16 - out : (1, 1360) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3155 - top-1 accuracy : 6/388 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1006]) -Dataloader batch - labels shape: torch.Size([1, 1006]) -Dataloader batch - attn_mask shape: torch.Size([1, 1006]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1387 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1006) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1006) torch.int64 - loss_mask : 239 / 1006 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1006, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1006, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1006, 1, 576) torch.bfloat16 - out : (1, 1006) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 239 tokens) : 11.2749 - top-1 accuracy : 1/239 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1823]) -Dataloader batch - labels shape: torch.Size([1, 1823]) -Dataloader batch - attn_mask shape: torch.Size([1, 1823]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1388 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1823) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1823) torch.int64 - loss_mask : 471 / 1823 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1823, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1823, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1823, 1, 576) torch.bfloat16 - out : (1, 1823) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 471 tokens) : 11.3572 - top-1 accuracy : 8/471 = 1.7% - - [2026-04-29 14:05:00] iteration 447/ 500 | consumed samples: 1788 | elapsed time per iteration (ms): 4337.4 | throughput (token/sec/GPU): 1245.7 | learning rate: 2.224286E-05 | global batch size: 4 | lm loss: 1.131952E+01 | loss scale: 1.0 | grad norm: 0.241 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1260]) -Dataloader batch - labels shape: torch.Size([1, 1260]) -Dataloader batch - attn_mask shape: torch.Size([1, 1260]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1389 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1260) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1260) torch.int64 - loss_mask : 330 / 1260 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1260, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1260, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1260, 1, 576) torch.bfloat16 - out : (1, 1260) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.3008 - top-1 accuracy : 6/330 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1386]) -Dataloader batch - labels shape: torch.Size([1, 1386]) -Dataloader batch - attn_mask shape: torch.Size([1, 1386]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1390 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1386) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1386) torch.int64 - loss_mask : 314 / 1386 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1386, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1386, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1386, 1, 576) torch.bfloat16 - out : (1, 1386) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 314 tokens) : 11.2254 - top-1 accuracy : 9/314 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1223]) -Dataloader batch - labels shape: torch.Size([1, 1223]) -Dataloader batch - attn_mask shape: torch.Size([1, 1223]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1391 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1223) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1223) torch.int64 - loss_mask : 299 / 1223 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1223, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1223, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1223, 1, 576) torch.bfloat16 - out : (1, 1223) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 299 tokens) : 11.3086 - top-1 accuracy : 6/299 = 2.0% - -Dataloader batch - tokens shape: torch.Size([1, 1379]) -Dataloader batch - labels shape: torch.Size([1, 1379]) -Dataloader batch - attn_mask shape: torch.Size([1, 1379]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1392 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1379) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1379) torch.int64 - loss_mask : 331 / 1379 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1379, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1379, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1379, 1, 576) torch.bfloat16 - out : (1, 1379) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 331 tokens) : 11.2472 - top-1 accuracy : 15/331 = 4.5% - - [2026-04-29 14:05:04] iteration 448/ 500 | consumed samples: 1792 | elapsed time per iteration (ms): 4323.2 | throughput (token/sec/GPU): 1213.9 | learning rate: 2.198754E-05 | global batch size: 4 | lm loss: 1.127010E+01 | loss scale: 1.0 | grad norm: 0.234 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1545]) -Dataloader batch - labels shape: torch.Size([1, 1545]) -Dataloader batch - attn_mask shape: torch.Size([1, 1545]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1393 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1545) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1545) torch.int64 - loss_mask : 384 / 1545 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1545, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1545, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1545, 1, 576) torch.bfloat16 - out : (1, 1545) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 384 tokens) : 11.3188 - top-1 accuracy : 7/384 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1279]) -Dataloader batch - labels shape: torch.Size([1, 1279]) -Dataloader batch - attn_mask shape: torch.Size([1, 1279]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1394 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1279) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1279) torch.int64 - loss_mask : 342 / 1279 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1279, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1279, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1279, 1, 576) torch.bfloat16 - out : (1, 1279) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 342 tokens) : 11.3014 - top-1 accuracy : 2/342 = 0.6% - -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1395 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1418) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1418) torch.int64 - loss_mask : 317 / 1418 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1418, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1418, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1418, 1, 576) torch.bfloat16 - out : (1, 1418) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 317 tokens) : 11.2717 - top-1 accuracy : 20/317 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1418]) -Dataloader batch - labels shape: torch.Size([1, 1418]) -Dataloader batch - attn_mask shape: torch.Size([1, 1418]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1396 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1418) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1418) torch.int64 - loss_mask : 276 / 1418 tokens (19.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1418, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1418, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1418, 1, 576) torch.bfloat16 - out : (1, 1418) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 276 tokens) : 11.1475 - top-1 accuracy : 12/276 = 4.3% - - [2026-04-29 14:05:09] iteration 449/ 500 | consumed samples: 1796 | elapsed time per iteration (ms): 4682.8 | throughput (token/sec/GPU): 1208.7 | learning rate: 2.173336E-05 | global batch size: 4 | lm loss: 1.126712E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1378]) -Dataloader batch - labels shape: torch.Size([1, 1378]) -Dataloader batch - attn_mask shape: torch.Size([1, 1378]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1397 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1378) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1378) torch.int64 - loss_mask : 411 / 1378 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1378, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1378, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1378, 1, 576) torch.bfloat16 - out : (1, 1378) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.3679 - top-1 accuracy : 5/411 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1479]) -Dataloader batch - labels shape: torch.Size([1, 1479]) -Dataloader batch - attn_mask shape: torch.Size([1, 1479]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1398 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1479) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1479) torch.int64 - loss_mask : 500 / 1479 tokens (33.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1479, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1479, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1479, 1, 576) torch.bfloat16 - out : (1, 1479) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 500 tokens) : 11.4631 - top-1 accuracy : 22/500 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1442]) -Dataloader batch - labels shape: torch.Size([1, 1442]) -Dataloader batch - attn_mask shape: torch.Size([1, 1442]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1399 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1442) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1442) torch.int64 - loss_mask : 339 / 1442 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1442, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1442, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1442, 1, 576) torch.bfloat16 - out : (1, 1442) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.2227 - top-1 accuracy : 17/339 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1212]) -Dataloader batch - labels shape: torch.Size([1, 1212]) -Dataloader batch - attn_mask shape: torch.Size([1, 1212]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1400 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1212) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1212) torch.int64 - loss_mask : 310 / 1212 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1212, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1212, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1212, 1, 576) torch.bfloat16 - out : (1, 1212) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 310 tokens) : 11.2830 - top-1 accuracy : 14/310 = 4.5% - - [2026-04-29 14:05:13] iteration 450/ 500 | consumed samples: 1800 | elapsed time per iteration (ms): 4205.3 | throughput (token/sec/GPU): 1310.5 | learning rate: 2.148032E-05 | global batch size: 4 | lm loss: 1.134999E+01 | loss scale: 1.0 | grad norm: 0.213 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -saving checkpoint at iteration 450 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 450 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 450 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (2923.40, 2923.40) -Dataloader batch - tokens shape: torch.Size([1, 1395]) -Dataloader batch - labels shape: torch.Size([1, 1395]) -Dataloader batch - attn_mask shape: torch.Size([1, 1395]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1401 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1395) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1395) torch.int64 - loss_mask : 329 / 1395 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1395, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1395, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1395, 1, 576) torch.bfloat16 - out : (1, 1395) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2820 - top-1 accuracy : 14/329 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1777]) -Dataloader batch - labels shape: torch.Size([1, 1777]) -Dataloader batch - attn_mask shape: torch.Size([1, 1777]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1402 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1777) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1777) torch.int64 - loss_mask : 415 / 1777 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1777, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1777, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1777, 1, 576) torch.bfloat16 - out : (1, 1777) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.2417 - top-1 accuracy : 20/415 = 4.8% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1403 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 394 / 1475 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 394 tokens) : 11.3048 - top-1 accuracy : 10/394 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1410]) -Dataloader batch - labels shape: torch.Size([1, 1410]) -Dataloader batch - attn_mask shape: torch.Size([1, 1410]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1404 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1410) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1410) torch.int64 - loss_mask : 356 / 1410 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1410, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1410, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1410, 1, 576) torch.bfloat16 - out : (1, 1410) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2741 - top-1 accuracy : 6/356 = 1.7% - - [2026-04-29 14:05:21] iteration 451/ 500 | consumed samples: 1804 | elapsed time per iteration (ms): 4861.1 | throughput (token/sec/GPU): 1246.0 | learning rate: 2.122843E-05 | global batch size: 4 | lm loss: 1.127494E+01 | loss scale: 1.0 | grad norm: 0.185 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1212]) -Dataloader batch - labels shape: torch.Size([1, 1212]) -Dataloader batch - attn_mask shape: torch.Size([1, 1212]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1405 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1212) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1212) torch.int64 - loss_mask : 318 / 1212 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1212, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1212, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1212, 1, 576) torch.bfloat16 - out : (1, 1212) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.2481 - top-1 accuracy : 9/318 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1406 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 382 / 1438 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.3329 - top-1 accuracy : 7/382 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1385]) -Dataloader batch - labels shape: torch.Size([1, 1385]) -Dataloader batch - attn_mask shape: torch.Size([1, 1385]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 1407 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1385) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1385) torch.int64 - loss_mask : 357 / 1385 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1385, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1385, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1385, 1, 576) torch.bfloat16 - out : (1, 1385) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.3205 - top-1 accuracy : 4/357 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1397]) -Dataloader batch - labels shape: torch.Size([1, 1397]) -Dataloader batch - attn_mask shape: torch.Size([1, 1397]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1408 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1397) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1397) torch.int64 - loss_mask : 319 / 1397 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1397, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1397, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1397, 1, 576) torch.bfloat16 - out : (1, 1397) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.1698 - top-1 accuracy : 18/319 = 5.6% - - [2026-04-29 14:05:25] iteration 452/ 500 | consumed samples: 1808 | elapsed time per iteration (ms): 4411.7 | throughput (token/sec/GPU): 1231.3 | learning rate: 2.097770E-05 | global batch size: 4 | lm loss: 1.127229E+01 | loss scale: 1.0 | grad norm: 0.202 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1380]) -Dataloader batch - labels shape: torch.Size([1, 1380]) -Dataloader batch - attn_mask shape: torch.Size([1, 1380]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1409 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1380) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1380) torch.int64 - loss_mask : 306 / 1380 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1380, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1380, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1380, 1, 576) torch.bfloat16 - out : (1, 1380) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 306 tokens) : 11.1615 - top-1 accuracy : 11/306 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1424]) -Dataloader batch - labels shape: torch.Size([1, 1424]) -Dataloader batch - attn_mask shape: torch.Size([1, 1424]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1410 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1424) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1424) torch.int64 - loss_mask : 337 / 1424 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1424, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1424, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1424, 1, 576) torch.bfloat16 - out : (1, 1424) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.3101 - top-1 accuracy : 22/337 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1690]) -Dataloader batch - labels shape: torch.Size([1, 1690]) -Dataloader batch - attn_mask shape: torch.Size([1, 1690]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1411 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1690) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1690) torch.int64 - loss_mask : 413 / 1690 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1690, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1690, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1690, 1, 576) torch.bfloat16 - out : (1, 1690) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2815 - top-1 accuracy : 11/413 = 2.7% - -Dataloader batch - tokens shape: torch.Size([1, 1470]) -Dataloader batch - labels shape: torch.Size([1, 1470]) -Dataloader batch - attn_mask shape: torch.Size([1, 1470]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1412 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1470) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1470) torch.int64 - loss_mask : 324 / 1470 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1470, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1470, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1470, 1, 576) torch.bfloat16 - out : (1, 1470) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 324 tokens) : 11.1912 - top-1 accuracy : 5/324 = 1.5% - - [2026-04-29 14:05:30] iteration 453/ 500 | consumed samples: 1812 | elapsed time per iteration (ms): 4905.5 | throughput (token/sec/GPU): 1215.8 | learning rate: 2.072814E-05 | global batch size: 4 | lm loss: 1.124064E+01 | loss scale: 1.0 | grad norm: 0.263 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1558]) -Dataloader batch - labels shape: torch.Size([1, 1558]) -Dataloader batch - attn_mask shape: torch.Size([1, 1558]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1413 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1558) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1558) torch.int64 - loss_mask : 415 / 1558 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1558, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1558, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1558, 1, 576) torch.bfloat16 - out : (1, 1558) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 415 tokens) : 11.3581 - top-1 accuracy : 8/415 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1375]) -Dataloader batch - labels shape: torch.Size([1, 1375]) -Dataloader batch - attn_mask shape: torch.Size([1, 1375]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1414 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1375) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1375) torch.int64 - loss_mask : 313 / 1375 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1375, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1375, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1375, 1, 576) torch.bfloat16 - out : (1, 1375) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 313 tokens) : 11.2814 - top-1 accuracy : 7/313 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1415 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 366 / 1541 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2509 - top-1 accuracy : 27/366 = 7.4% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1416 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 367 / 1695 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 367 tokens) : 11.1585 - top-1 accuracy : 29/367 = 7.9% - - [2026-04-29 14:05:35] iteration 454/ 500 | consumed samples: 1816 | elapsed time per iteration (ms): 4942.3 | throughput (token/sec/GPU): 1248.2 | learning rate: 2.047975E-05 | global batch size: 4 | lm loss: 1.126469E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1752]) -Dataloader batch - labels shape: torch.Size([1, 1752]) -Dataloader batch - attn_mask shape: torch.Size([1, 1752]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1417 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1752) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1752) torch.int64 - loss_mask : 398 / 1752 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1752, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1752, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1752, 1, 576) torch.bfloat16 - out : (1, 1752) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 398 tokens) : 11.2752 - top-1 accuracy : 21/398 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1563]) -Dataloader batch - labels shape: torch.Size([1, 1563]) -Dataloader batch - attn_mask shape: torch.Size([1, 1563]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1418 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1563) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1563) torch.int64 - loss_mask : 427 / 1563 tokens (27.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1563, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1563, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1563, 1, 576) torch.bfloat16 - out : (1, 1563) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 427 tokens) : 11.3071 - top-1 accuracy : 6/427 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1419]) -Dataloader batch - labels shape: torch.Size([1, 1419]) -Dataloader batch - attn_mask shape: torch.Size([1, 1419]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1419 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1419) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1419) torch.int64 - loss_mask : 335 / 1419 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1419, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1419, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1419, 1, 576) torch.bfloat16 - out : (1, 1419) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.2640 - top-1 accuracy : 25/335 = 7.5% - -Dataloader batch - tokens shape: torch.Size([1, 1043]) -Dataloader batch - labels shape: torch.Size([1, 1043]) -Dataloader batch - attn_mask shape: torch.Size([1, 1043]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1420 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1043) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1043) torch.int64 - loss_mask : 216 / 1043 tokens (20.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1043, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1043, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1043, 1, 576) torch.bfloat16 - out : (1, 1043) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 216 tokens) : 11.2259 - top-1 accuracy : 20/216 = 9.3% - - [2026-04-29 14:05:39] iteration 455/ 500 | consumed samples: 1820 | elapsed time per iteration (ms): 4625.4 | throughput (token/sec/GPU): 1249.0 | learning rate: 2.023256E-05 | global batch size: 4 | lm loss: 1.127461E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1722]) -Dataloader batch - labels shape: torch.Size([1, 1722]) -Dataloader batch - attn_mask shape: torch.Size([1, 1722]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1421 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1722) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1722) torch.int64 - loss_mask : 362 / 1722 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1722, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1722, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1722, 1, 576) torch.bfloat16 - out : (1, 1722) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2272 - top-1 accuracy : 26/362 = 7.2% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1422 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 402 / 1532 tokens (26.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 402 tokens) : 11.2804 - top-1 accuracy : 16/402 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1376]) -Dataloader batch - labels shape: torch.Size([1, 1376]) -Dataloader batch - attn_mask shape: torch.Size([1, 1376]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1423 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1376) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1376) torch.int64 - loss_mask : 408 / 1376 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1376, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1376, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1376, 1, 576) torch.bfloat16 - out : (1, 1376) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 408 tokens) : 11.4339 - top-1 accuracy : 1/408 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1061]) -Dataloader batch - labels shape: torch.Size([1, 1061]) -Dataloader batch - attn_mask shape: torch.Size([1, 1061]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1424 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1061) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1061) torch.int64 - loss_mask : 303 / 1061 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1061, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1061, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1061, 1, 576) torch.bfloat16 - out : (1, 1061) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.3361 - top-1 accuracy : 5/303 = 1.7% - - [2026-04-29 14:05:44] iteration 456/ 500 | consumed samples: 1824 | elapsed time per iteration (ms): 4495.4 | throughput (token/sec/GPU): 1266.0 | learning rate: 1.998657E-05 | global batch size: 4 | lm loss: 1.132124E+01 | loss scale: 1.0 | grad norm: 0.199 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1829]) -Dataloader batch - labels shape: torch.Size([1, 1829]) -Dataloader batch - attn_mask shape: torch.Size([1, 1829]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1425 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1829) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1829) torch.int64 - loss_mask : 478 / 1829 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1829, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1829, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1829, 1, 576) torch.bfloat16 - out : (1, 1829) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 478 tokens) : 11.3082 - top-1 accuracy : 9/478 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1552]) -Dataloader batch - labels shape: torch.Size([1, 1552]) -Dataloader batch - attn_mask shape: torch.Size([1, 1552]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1426 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1552) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1552) torch.int64 - loss_mask : 360 / 1552 tokens (23.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1552, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1552, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1552, 1, 576) torch.bfloat16 - out : (1, 1552) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.1742 - top-1 accuracy : 11/360 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1761]) -Dataloader batch - labels shape: torch.Size([1, 1761]) -Dataloader batch - attn_mask shape: torch.Size([1, 1761]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1427 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1761) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1761) torch.int64 - loss_mask : 400 / 1761 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1761, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1761, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1761, 1, 576) torch.bfloat16 - out : (1, 1761) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.2193 - top-1 accuracy : 37/400 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1079]) -Dataloader batch - labels shape: torch.Size([1, 1079]) -Dataloader batch - attn_mask shape: torch.Size([1, 1079]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1428 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1079) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1079) torch.int64 - loss_mask : 323 / 1079 tokens (29.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1079, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1079, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1079, 1, 576) torch.bfloat16 - out : (1, 1079) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.4161 - top-1 accuracy : 14/323 = 4.3% - - [2026-04-29 14:05:49] iteration 457/ 500 | consumed samples: 1828 | elapsed time per iteration (ms): 4906.4 | throughput (token/sec/GPU): 1267.9 | learning rate: 1.974179E-05 | global batch size: 4 | lm loss: 1.127684E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1160]) -Dataloader batch - labels shape: torch.Size([1, 1160]) -Dataloader batch - attn_mask shape: torch.Size([1, 1160]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1429 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1160) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1160) torch.int64 - loss_mask : 275 / 1160 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1160, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1160, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1160, 1, 576) torch.bfloat16 - out : (1, 1160) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 275 tokens) : 11.2349 - top-1 accuracy : 4/275 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1359]) -Dataloader batch - labels shape: torch.Size([1, 1359]) -Dataloader batch - attn_mask shape: torch.Size([1, 1359]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1430 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1359) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1359) torch.int64 - loss_mask : 285 / 1359 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1359, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1359, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1359, 1, 576) torch.bfloat16 - out : (1, 1359) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 285 tokens) : 11.1878 - top-1 accuracy : 16/285 = 5.6% - -Dataloader batch - tokens shape: torch.Size([1, 1532]) -Dataloader batch - labels shape: torch.Size([1, 1532]) -Dataloader batch - attn_mask shape: torch.Size([1, 1532]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1431 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1532) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1532) torch.int64 - loss_mask : 353 / 1532 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1532, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1532, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1532, 1, 576) torch.bfloat16 - out : (1, 1532) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 353 tokens) : 11.2486 - top-1 accuracy : 18/353 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1541]) -Dataloader batch - labels shape: torch.Size([1, 1541]) -Dataloader batch - attn_mask shape: torch.Size([1, 1541]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1432 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1541) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1541) torch.int64 - loss_mask : 335 / 1541 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1541, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1541, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1541, 1, 576) torch.bfloat16 - out : (1, 1541) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 335 tokens) : 11.1550 - top-1 accuracy : 30/335 = 9.0% - - [2026-04-29 14:05:53] iteration 458/ 500 | consumed samples: 1832 | elapsed time per iteration (ms): 4639.3 | throughput (token/sec/GPU): 1205.4 | learning rate: 1.949823E-05 | global batch size: 4 | lm loss: 1.120657E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1570]) -Dataloader batch - labels shape: torch.Size([1, 1570]) -Dataloader batch - attn_mask shape: torch.Size([1, 1570]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1433 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1570) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1570) torch.int64 - loss_mask : 360 / 1570 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1570, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1570, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1570, 1, 576) torch.bfloat16 - out : (1, 1570) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.2276 - top-1 accuracy : 30/360 = 8.3% - -Dataloader batch - tokens shape: torch.Size([1, 1438]) -Dataloader batch - labels shape: torch.Size([1, 1438]) -Dataloader batch - attn_mask shape: torch.Size([1, 1438]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1434 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1438) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1438) torch.int64 - loss_mask : 363 / 1438 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1438, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1438, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1438, 1, 576) torch.bfloat16 - out : (1, 1438) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 363 tokens) : 11.3109 - top-1 accuracy : 12/363 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1563]) -Dataloader batch - labels shape: torch.Size([1, 1563]) -Dataloader batch - attn_mask shape: torch.Size([1, 1563]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1435 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1563) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1563) torch.int64 - loss_mask : 443 / 1563 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1563, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1563, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1563, 1, 576) torch.bfloat16 - out : (1, 1563) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 443 tokens) : 11.3312 - top-1 accuracy : 34/443 = 7.7% - -Dataloader batch - tokens shape: torch.Size([1, 1459]) -Dataloader batch - labels shape: torch.Size([1, 1459]) -Dataloader batch - attn_mask shape: torch.Size([1, 1459]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1436 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1459) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1459) torch.int64 - loss_mask : 309 / 1459 tokens (21.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1459, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1459, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1459, 1, 576) torch.bfloat16 - out : (1, 1459) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.1497 - top-1 accuracy : 21/309 = 6.8% - - [2026-04-29 14:05:58] iteration 459/ 500 | consumed samples: 1836 | elapsed time per iteration (ms): 4893.4 | throughput (token/sec/GPU): 1232.3 | learning rate: 1.925589E-05 | global batch size: 4 | lm loss: 1.126288E+01 | loss scale: 1.0 | grad norm: 0.174 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1431]) -Dataloader batch - labels shape: torch.Size([1, 1431]) -Dataloader batch - attn_mask shape: torch.Size([1, 1431]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1437 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1431) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1431) torch.int64 - loss_mask : 364 / 1431 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1431, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1431, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1431, 1, 576) torch.bfloat16 - out : (1, 1431) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 364 tokens) : 11.3455 - top-1 accuracy : 9/364 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1606]) -Dataloader batch - labels shape: torch.Size([1, 1606]) -Dataloader batch - attn_mask shape: torch.Size([1, 1606]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1438 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1606) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1606) torch.int64 - loss_mask : 388 / 1606 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1606, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1606, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1606, 1, 576) torch.bfloat16 - out : (1, 1606) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.2296 - top-1 accuracy : 32/388 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1439 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 417 / 1546 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 417 tokens) : 11.3086 - top-1 accuracy : 5/417 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1754]) -Dataloader batch - labels shape: torch.Size([1, 1754]) -Dataloader batch - attn_mask shape: torch.Size([1, 1754]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1440 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1754) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1754) torch.int64 - loss_mask : 391 / 1754 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1754, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1754, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1754, 1, 576) torch.bfloat16 - out : (1, 1754) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.2600 - top-1 accuracy : 31/391 = 7.9% - - [2026-04-29 14:06:03] iteration 460/ 500 | consumed samples: 1840 | elapsed time per iteration (ms): 5005.8 | throughput (token/sec/GPU): 1265.9 | learning rate: 1.901480E-05 | global batch size: 4 | lm loss: 1.128537E+01 | loss scale: 1.0 | grad norm: 0.181 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4978.19, 4978.19) - forward-compute ................................: (3731.37, 3731.37) - backward-compute ...............................: (1242.84, 1242.84) - batch-generator ................................: (516.23, 516.23) - layernorm-grads-all-reduce .....................: (0.07, 0.07) - embedding-grads-all-reduce .....................: (0.10, 0.10) - all-grads-sync .................................: (1.09, 1.09) - params-all-gather ..............................: (0.38, 0.38) - optimizer-copy-to-main-grad ....................: (0.35, 0.35) - optimizer-clip-main-grad .......................: (1.29, 1.29) - optimizer-count-zeros ..........................: (0.05, 0.05) - optimizer-inner-step ...........................: (2.13, 2.13) - optimizer-copy-main-to-model-params ............: (0.55, 0.55) - optimizer ......................................: (6.80, 6.80) -Dataloader batch - tokens shape: torch.Size([1, 1516]) -Dataloader batch - labels shape: torch.Size([1, 1516]) -Dataloader batch - attn_mask shape: torch.Size([1, 1516]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1441 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1516) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1516) torch.int64 - loss_mask : 374 / 1516 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1516, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1516, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1516, 1, 576) torch.bfloat16 - out : (1, 1516) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.2991 - top-1 accuracy : 16/374 = 4.3% - -Dataloader batch - tokens shape: torch.Size([1, 1443]) -Dataloader batch - labels shape: torch.Size([1, 1443]) -Dataloader batch - attn_mask shape: torch.Size([1, 1443]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1442 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1443) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1443) torch.int64 - loss_mask : 355 / 1443 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1443, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1443, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1443, 1, 576) torch.bfloat16 - out : (1, 1443) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 355 tokens) : 11.2737 - top-1 accuracy : 29/355 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1397]) -Dataloader batch - labels shape: torch.Size([1, 1397]) -Dataloader batch - attn_mask shape: torch.Size([1, 1397]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1443 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1397) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1397) torch.int64 - loss_mask : 334 / 1397 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1397, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1397, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1397, 1, 576) torch.bfloat16 - out : (1, 1397) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 334 tokens) : 11.3039 - top-1 accuracy : 10/334 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1594]) -Dataloader batch - labels shape: torch.Size([1, 1594]) -Dataloader batch - attn_mask shape: torch.Size([1, 1594]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1444 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1594) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1594) torch.int64 - loss_mask : 366 / 1594 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1594, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1594, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1594, 1, 576) torch.bfloat16 - out : (1, 1594) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.2056 - top-1 accuracy : 19/366 = 5.2% - - [2026-04-29 14:06:08] iteration 461/ 500 | consumed samples: 1844 | elapsed time per iteration (ms): 4712.5 | throughput (token/sec/GPU): 1262.6 | learning rate: 1.877495E-05 | global batch size: 4 | lm loss: 1.126996E+01 | loss scale: 1.0 | grad norm: 0.219 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1444]) -Dataloader batch - labels shape: torch.Size([1, 1444]) -Dataloader batch - attn_mask shape: torch.Size([1, 1444]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1445 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1444) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1444) torch.int64 - loss_mask : 360 / 1444 tokens (24.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1444, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1444, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1444, 1, 576) torch.bfloat16 - out : (1, 1444) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.3267 - top-1 accuracy : 11/360 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1013]) -Dataloader batch - labels shape: torch.Size([1, 1013]) -Dataloader batch - attn_mask shape: torch.Size([1, 1013]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1446 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1013) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1013) torch.int64 - loss_mask : 251 / 1013 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1013, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1013, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1013, 1, 576) torch.bfloat16 - out : (1, 1013) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2099 - top-1 accuracy : 11/251 = 4.4% - -Dataloader batch - tokens shape: torch.Size([1, 1390]) -Dataloader batch - labels shape: torch.Size([1, 1390]) -Dataloader batch - attn_mask shape: torch.Size([1, 1390]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1447 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1390) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1390) torch.int64 - loss_mask : 320 / 1390 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1390, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1390, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1390, 1, 576) torch.bfloat16 - out : (1, 1390) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 320 tokens) : 11.2529 - top-1 accuracy : 17/320 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1448 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 362 / 1520 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 362 tokens) : 11.2678 - top-1 accuracy : 16/362 = 4.4% - - [2026-04-29 14:06:12] iteration 462/ 500 | consumed samples: 1848 | elapsed time per iteration (ms): 4344.4 | throughput (token/sec/GPU): 1235.4 | learning rate: 1.853636E-05 | global batch size: 4 | lm loss: 1.126927E+01 | loss scale: 1.0 | grad norm: 0.232 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1255]) -Dataloader batch - labels shape: torch.Size([1, 1255]) -Dataloader batch - attn_mask shape: torch.Size([1, 1255]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1449 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1255) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1255) torch.int64 - loss_mask : 318 / 1255 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1255, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1255, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1255, 1, 576) torch.bfloat16 - out : (1, 1255) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.3104 - top-1 accuracy : 3/318 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1394]) -Dataloader batch - labels shape: torch.Size([1, 1394]) -Dataloader batch - attn_mask shape: torch.Size([1, 1394]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1450 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1394) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1394) torch.int64 - loss_mask : 339 / 1394 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1394, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1394, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1394, 1, 576) torch.bfloat16 - out : (1, 1394) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.3122 - top-1 accuracy : 1/339 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1802]) -Dataloader batch - labels shape: torch.Size([1, 1802]) -Dataloader batch - attn_mask shape: torch.Size([1, 1802]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1451 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1802) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1802) torch.int64 - loss_mask : 466 / 1802 tokens (25.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1802, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1802, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1802, 1, 576) torch.bfloat16 - out : (1, 1802) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 466 tokens) : 11.3059 - top-1 accuracy : 25/466 = 5.4% - -Dataloader batch - tokens shape: torch.Size([1, 1537]) -Dataloader batch - labels shape: torch.Size([1, 1537]) -Dataloader batch - attn_mask shape: torch.Size([1, 1537]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1452 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1537) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1537) torch.int64 - loss_mask : 344 / 1537 tokens (22.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1537, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1537, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1537, 1, 576) torch.bfloat16 - out : (1, 1537) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.1924 - top-1 accuracy : 8/344 = 2.3% - - [2026-04-29 14:06:17] iteration 463/ 500 | consumed samples: 1852 | elapsed time per iteration (ms): 4774.4 | throughput (token/sec/GPU): 1254.2 | learning rate: 1.829904E-05 | global batch size: 4 | lm loss: 1.128170E+01 | loss scale: 1.0 | grad norm: 0.243 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1155]) -Dataloader batch - labels shape: torch.Size([1, 1155]) -Dataloader batch - attn_mask shape: torch.Size([1, 1155]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1453 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1155) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1155) torch.int64 - loss_mask : 309 / 1155 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1155, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1155, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1155, 1, 576) torch.bfloat16 - out : (1, 1155) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 309 tokens) : 11.3191 - top-1 accuracy : 11/309 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1102]) -Dataloader batch - labels shape: torch.Size([1, 1102]) -Dataloader batch - attn_mask shape: torch.Size([1, 1102]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1454 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1102) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1102) torch.int64 - loss_mask : 251 / 1102 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1102, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1102, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1102, 1, 576) torch.bfloat16 - out : (1, 1102) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 251 tokens) : 11.2367 - top-1 accuracy : 7/251 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1205]) -Dataloader batch - labels shape: torch.Size([1, 1205]) -Dataloader batch - attn_mask shape: torch.Size([1, 1205]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1455 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1205) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1205) torch.int64 - loss_mask : 271 / 1205 tokens (22.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1205, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1205, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1205, 1, 576) torch.bfloat16 - out : (1, 1205) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 271 tokens) : 11.1992 - top-1 accuracy : 5/271 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 782]) -Dataloader batch - labels shape: torch.Size([1, 782]) -Dataloader batch - attn_mask shape: torch.Size([1, 782]) -Dataloader batch - image_grid_thw shape: torch.Size([16, 3]) - -==================================================================== - PIPELINE — STEP 1456 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 782) torch.int64 - images : (16, 3, 1024, 1024) torch.bfloat16 - labels : (1, 782) torch.int64 - loss_mask : 136 / 782 tokens (17.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (16, 3, 1024, 1024) torch.bfloat16 - out : (16, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (16, 3072) - out : (16, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (782, 1, 576) torch.bfloat16 grad=False - image slots : 16 tokens replaced with vision embeddings - combined : (782, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (782, 1, 576) torch.bfloat16 - out : (1, 782) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 136 tokens) : 11.1125 - top-1 accuracy : 16/136 = 11.8% - - [2026-04-29 14:06:21] iteration 464/ 500 | consumed samples: 1856 | elapsed time per iteration (ms): 3686.5 | throughput (token/sec/GPU): 1151.2 | learning rate: 1.806299E-05 | global batch size: 4 | lm loss: 1.123506E+01 | loss scale: 1.0 | grad norm: 0.270 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1402]) -Dataloader batch - labels shape: torch.Size([1, 1402]) -Dataloader batch - attn_mask shape: torch.Size([1, 1402]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1457 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1402) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1402) torch.int64 - loss_mask : 340 / 1402 tokens (24.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1402, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1402, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1402, 1, 576) torch.bfloat16 - out : (1, 1402) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2275 - top-1 accuracy : 7/340 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1137]) -Dataloader batch - labels shape: torch.Size([1, 1137]) -Dataloader batch - attn_mask shape: torch.Size([1, 1137]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1458 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1137) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1137) torch.int64 - loss_mask : 323 / 1137 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1137, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1137, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1137, 1, 576) torch.bfloat16 - out : (1, 1137) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.3680 - top-1 accuracy : 1/323 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1927]) -Dataloader batch - labels shape: torch.Size([1, 1927]) -Dataloader batch - attn_mask shape: torch.Size([1, 1927]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1459 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1927) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1927) torch.int64 - loss_mask : 550 / 1927 tokens (28.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1927, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1927, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1927, 1, 576) torch.bfloat16 - out : (1, 1927) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 550 tokens) : 11.3882 - top-1 accuracy : 2/550 = 0.4% - -Dataloader batch - tokens shape: torch.Size([1, 1384]) -Dataloader batch - labels shape: torch.Size([1, 1384]) -Dataloader batch - attn_mask shape: torch.Size([1, 1384]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1460 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1384) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1384) torch.int64 - loss_mask : 338 / 1384 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1384, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1384, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1384, 1, 576) torch.bfloat16 - out : (1, 1384) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.2644 - top-1 accuracy : 21/338 = 6.2% - - [2026-04-29 14:06:25] iteration 465/ 500 | consumed samples: 1860 | elapsed time per iteration (ms): 4503.6 | throughput (token/sec/GPU): 1298.9 | learning rate: 1.782823E-05 | global batch size: 4 | lm loss: 1.132176E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1069]) -Dataloader batch - labels shape: torch.Size([1, 1069]) -Dataloader batch - attn_mask shape: torch.Size([1, 1069]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1461 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1069) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1069) torch.int64 - loss_mask : 299 / 1069 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1069, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1069, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1069, 1, 576) torch.bfloat16 - out : (1, 1069) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 299 tokens) : 11.3651 - top-1 accuracy : 0/299 = 0.0% - -Dataloader batch - tokens shape: torch.Size([1, 1242]) -Dataloader batch - labels shape: torch.Size([1, 1242]) -Dataloader batch - attn_mask shape: torch.Size([1, 1242]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1462 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1242) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1242) torch.int64 - loss_mask : 303 / 1242 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1242, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1242, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1242, 1, 576) torch.bfloat16 - out : (1, 1242) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.2800 - top-1 accuracy : 4/303 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1808]) -Dataloader batch - labels shape: torch.Size([1, 1808]) -Dataloader batch - attn_mask shape: torch.Size([1, 1808]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1463 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1808) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1808) torch.int64 - loss_mask : 454 / 1808 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1808, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1808, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1808, 1, 576) torch.bfloat16 - out : (1, 1808) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 454 tokens) : 11.2695 - top-1 accuracy : 18/454 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1545]) -Dataloader batch - labels shape: torch.Size([1, 1545]) -Dataloader batch - attn_mask shape: torch.Size([1, 1545]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1464 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1545) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1545) torch.int64 - loss_mask : 407 / 1545 tokens (26.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1545, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1545, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1545, 1, 576) torch.bfloat16 - out : (1, 1545) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.3094 - top-1 accuracy : 10/407 = 2.5% - - [2026-04-29 14:06:30] iteration 466/ 500 | consumed samples: 1864 | elapsed time per iteration (ms): 4539.1 | throughput (token/sec/GPU): 1247.8 | learning rate: 1.759476E-05 | global batch size: 4 | lm loss: 1.130231E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1733]) -Dataloader batch - labels shape: torch.Size([1, 1733]) -Dataloader batch - attn_mask shape: torch.Size([1, 1733]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1465 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1733) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1733) torch.int64 - loss_mask : 373 / 1733 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1733, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1733, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1733, 1, 576) torch.bfloat16 - out : (1, 1733) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 373 tokens) : 11.2156 - top-1 accuracy : 13/373 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1268]) -Dataloader batch - labels shape: torch.Size([1, 1268]) -Dataloader batch - attn_mask shape: torch.Size([1, 1268]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1466 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1268) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1268) torch.int64 - loss_mask : 344 / 1268 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1268, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1268, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1268, 1, 576) torch.bfloat16 - out : (1, 1268) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.3077 - top-1 accuracy : 5/344 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1103]) -Dataloader batch - labels shape: torch.Size([1, 1103]) -Dataloader batch - attn_mask shape: torch.Size([1, 1103]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1467 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1103) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1103) torch.int64 - loss_mask : 278 / 1103 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1103, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1103, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1103, 1, 576) torch.bfloat16 - out : (1, 1103) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.2637 - top-1 accuracy : 3/278 = 1.1% - -Dataloader batch - tokens shape: torch.Size([1, 1437]) -Dataloader batch - labels shape: torch.Size([1, 1437]) -Dataloader batch - attn_mask shape: torch.Size([1, 1437]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1468 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1437) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1437) torch.int64 - loss_mask : 388 / 1437 tokens (27.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1437, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1437, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1437, 1, 576) torch.bfloat16 - out : (1, 1437) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3361 - top-1 accuracy : 16/388 = 4.1% - - [2026-04-29 14:06:34] iteration 467/ 500 | consumed samples: 1868 | elapsed time per iteration (ms): 4506.9 | throughput (token/sec/GPU): 1229.4 | learning rate: 1.736260E-05 | global batch size: 4 | lm loss: 1.128199E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1060]) -Dataloader batch - labels shape: torch.Size([1, 1060]) -Dataloader batch - attn_mask shape: torch.Size([1, 1060]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1469 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1060) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1060) torch.int64 - loss_mask : 230 / 1060 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1060, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1060, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1060, 1, 576) torch.bfloat16 - out : (1, 1060) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 230 tokens) : 11.1820 - top-1 accuracy : 9/230 = 3.9% - -Dataloader batch - tokens shape: torch.Size([1, 1547]) -Dataloader batch - labels shape: torch.Size([1, 1547]) -Dataloader batch - attn_mask shape: torch.Size([1, 1547]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1470 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1547) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1547) torch.int64 - loss_mask : 391 / 1547 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1547, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1547, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1547, 1, 576) torch.bfloat16 - out : (1, 1547) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3049 - top-1 accuracy : 12/391 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1530]) -Dataloader batch - labels shape: torch.Size([1, 1530]) -Dataloader batch - attn_mask shape: torch.Size([1, 1530]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1471 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1530) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1530) torch.int64 - loss_mask : 326 / 1530 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1530, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1530, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1530, 1, 576) torch.bfloat16 - out : (1, 1530) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.1850 - top-1 accuracy : 15/326 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1374]) -Dataloader batch - labels shape: torch.Size([1, 1374]) -Dataloader batch - attn_mask shape: torch.Size([1, 1374]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1472 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1374) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1374) torch.int64 - loss_mask : 296 / 1374 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1374, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1374, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1374, 1, 576) torch.bfloat16 - out : (1, 1374) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 296 tokens) : 11.1673 - top-1 accuracy : 17/296 = 5.7% - - [2026-04-29 14:06:39] iteration 468/ 500 | consumed samples: 1872 | elapsed time per iteration (ms): 4544.6 | throughput (token/sec/GPU): 1212.6 | learning rate: 1.713175E-05 | global batch size: 4 | lm loss: 1.121796E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1563]) -Dataloader batch - labels shape: torch.Size([1, 1563]) -Dataloader batch - attn_mask shape: torch.Size([1, 1563]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1473 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1563) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1563) torch.int64 - loss_mask : 347 / 1563 tokens (22.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1563, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1563, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1563, 1, 576) torch.bfloat16 - out : (1, 1563) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.1905 - top-1 accuracy : 12/347 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1121]) -Dataloader batch - labels shape: torch.Size([1, 1121]) -Dataloader batch - attn_mask shape: torch.Size([1, 1121]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1474 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1121) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1121) torch.int64 - loss_mask : 318 / 1121 tokens (28.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1121, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1121, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1121, 1, 576) torch.bfloat16 - out : (1, 1121) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 318 tokens) : 11.4040 - top-1 accuracy : 11/318 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1600]) -Dataloader batch - labels shape: torch.Size([1, 1600]) -Dataloader batch - attn_mask shape: torch.Size([1, 1600]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1475 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1600) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1600) torch.int64 - loss_mask : 445 / 1600 tokens (27.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1600, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1600, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1600, 1, 576) torch.bfloat16 - out : (1, 1600) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 445 tokens) : 11.4147 - top-1 accuracy : 4/445 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1413]) -Dataloader batch - labels shape: torch.Size([1, 1413]) -Dataloader batch - attn_mask shape: torch.Size([1, 1413]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1476 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1413) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1413) torch.int64 - loss_mask : 349 / 1413 tokens (24.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1413, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1413, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1413, 1, 576) torch.bfloat16 - out : (1, 1413) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 349 tokens) : 11.2610 - top-1 accuracy : 12/349 = 3.4% - - [2026-04-29 14:06:44] iteration 469/ 500 | consumed samples: 1876 | elapsed time per iteration (ms): 4702.7 | throughput (token/sec/GPU): 1211.4 | learning rate: 1.690222E-05 | global batch size: 4 | lm loss: 1.132229E+01 | loss scale: 1.0 | grad norm: 0.191 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1548]) -Dataloader batch - labels shape: torch.Size([1, 1548]) -Dataloader batch - attn_mask shape: torch.Size([1, 1548]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1477 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1548) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1548) torch.int64 - loss_mask : 416 / 1548 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1548, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1548, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1548, 1, 576) torch.bfloat16 - out : (1, 1548) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.2973 - top-1 accuracy : 4/416 = 1.0% - -Dataloader batch - tokens shape: torch.Size([1, 1463]) -Dataloader batch - labels shape: torch.Size([1, 1463]) -Dataloader batch - attn_mask shape: torch.Size([1, 1463]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1478 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1463) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1463) torch.int64 - loss_mask : 405 / 1463 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1463, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1463, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1463, 1, 576) torch.bfloat16 - out : (1, 1463) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.3143 - top-1 accuracy : 10/405 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1572]) -Dataloader batch - labels shape: torch.Size([1, 1572]) -Dataloader batch - attn_mask shape: torch.Size([1, 1572]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1479 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1572) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1572) torch.int64 - loss_mask : 436 / 1572 tokens (27.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1572, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1572, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1572, 1, 576) torch.bfloat16 - out : (1, 1572) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 436 tokens) : 11.2831 - top-1 accuracy : 10/436 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1179]) -Dataloader batch - labels shape: torch.Size([1, 1179]) -Dataloader batch - attn_mask shape: torch.Size([1, 1179]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1480 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1179) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1179) torch.int64 - loss_mask : 325 / 1179 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1179, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1179, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1179, 1, 576) torch.bfloat16 - out : (1, 1179) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 325 tokens) : 11.3439 - top-1 accuracy : 16/325 = 4.9% - - [2026-04-29 14:06:48] iteration 470/ 500 | consumed samples: 1880 | elapsed time per iteration (ms): 4480.1 | throughput (token/sec/GPU): 1286.1 | learning rate: 1.667403E-05 | global batch size: 4 | lm loss: 1.130732E+01 | loss scale: 1.0 | grad norm: 0.230 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1612]) -Dataloader batch - labels shape: torch.Size([1, 1612]) -Dataloader batch - attn_mask shape: torch.Size([1, 1612]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1481 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1612) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1612) torch.int64 - loss_mask : 390 / 1612 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1612, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1612, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1612, 1, 576) torch.bfloat16 - out : (1, 1612) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 390 tokens) : 11.2683 - top-1 accuracy : 3/390 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1068]) -Dataloader batch - labels shape: torch.Size([1, 1068]) -Dataloader batch - attn_mask shape: torch.Size([1, 1068]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1482 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1068) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1068) torch.int64 - loss_mask : 289 / 1068 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1068, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1068, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1068, 1, 576) torch.bfloat16 - out : (1, 1068) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 289 tokens) : 11.3525 - top-1 accuracy : 7/289 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1839]) -Dataloader batch - labels shape: torch.Size([1, 1839]) -Dataloader batch - attn_mask shape: torch.Size([1, 1839]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1483 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1839) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1839) torch.int64 - loss_mask : 468 / 1839 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1839, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1839, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1839, 1, 576) torch.bfloat16 - out : (1, 1839) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 468 tokens) : 11.2712 - top-1 accuracy : 11/468 = 2.4% - -Dataloader batch - tokens shape: torch.Size([1, 1875]) -Dataloader batch - labels shape: torch.Size([1, 1875]) -Dataloader batch - attn_mask shape: torch.Size([1, 1875]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1484 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1875) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1875) torch.int64 - loss_mask : 537 / 1875 tokens (28.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1875, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1875, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1875, 1, 576) torch.bfloat16 - out : (1, 1875) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 537 tokens) : 11.3658 - top-1 accuracy : 5/537 = 0.9% - - [2026-04-29 14:06:53] iteration 471/ 500 | consumed samples: 1884 | elapsed time per iteration (ms): 5096.1 | throughput (token/sec/GPU): 1254.7 | learning rate: 1.644718E-05 | global batch size: 4 | lm loss: 1.131465E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 987]) -Dataloader batch - labels shape: torch.Size([1, 987]) -Dataloader batch - attn_mask shape: torch.Size([1, 987]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1485 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 987) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 987) torch.int64 - loss_mask : 236 / 987 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (987, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (987, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (987, 1, 576) torch.bfloat16 - out : (1, 987) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 236 tokens) : 11.2684 - top-1 accuracy : 21/236 = 8.9% - -Dataloader batch - tokens shape: torch.Size([1, 1841]) -Dataloader batch - labels shape: torch.Size([1, 1841]) -Dataloader batch - attn_mask shape: torch.Size([1, 1841]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1486 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1841) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1841) torch.int64 - loss_mask : 480 / 1841 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1841, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1841, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1841, 1, 576) torch.bfloat16 - out : (1, 1841) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 480 tokens) : 11.3357 - top-1 accuracy : 34/480 = 7.1% - -Dataloader batch - tokens shape: torch.Size([1, 1372]) -Dataloader batch - labels shape: torch.Size([1, 1372]) -Dataloader batch - attn_mask shape: torch.Size([1, 1372]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1487 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1372) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1372) torch.int64 - loss_mask : 295 / 1372 tokens (21.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1372, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1372, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1372, 1, 576) torch.bfloat16 - out : (1, 1372) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 295 tokens) : 11.1611 - top-1 accuracy : 15/295 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1884]) -Dataloader batch - labels shape: torch.Size([1, 1884]) -Dataloader batch - attn_mask shape: torch.Size([1, 1884]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1488 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1884) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1884) torch.int64 - loss_mask : 549 / 1884 tokens (29.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1884, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1884, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1884, 1, 576) torch.bfloat16 - out : (1, 1884) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 549 tokens) : 11.3687 - top-1 accuracy : 20/549 = 3.6% - - [2026-04-29 14:06:58] iteration 472/ 500 | consumed samples: 1888 | elapsed time per iteration (ms): 4723.0 | throughput (token/sec/GPU): 1288.2 | learning rate: 1.622167E-05 | global batch size: 4 | lm loss: 1.130412E+01 | loss scale: 1.0 | grad norm: 0.180 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1749]) -Dataloader batch - labels shape: torch.Size([1, 1749]) -Dataloader batch - attn_mask shape: torch.Size([1, 1749]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1489 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1749) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1749) torch.int64 - loss_mask : 400 / 1749 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1749, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1749, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1749, 1, 576) torch.bfloat16 - out : (1, 1749) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 400 tokens) : 11.2769 - top-1 accuracy : 37/400 = 9.2% - -Dataloader batch - tokens shape: torch.Size([1, 1159]) -Dataloader batch - labels shape: torch.Size([1, 1159]) -Dataloader batch - attn_mask shape: torch.Size([1, 1159]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1490 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1159) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1159) torch.int64 - loss_mask : 311 / 1159 tokens (26.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1159, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1159, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1159, 1, 576) torch.bfloat16 - out : (1, 1159) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 311 tokens) : 11.2982 - top-1 accuracy : 8/311 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1353]) -Dataloader batch - labels shape: torch.Size([1, 1353]) -Dataloader batch - attn_mask shape: torch.Size([1, 1353]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1491 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1353) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1353) torch.int64 - loss_mask : 366 / 1353 tokens (27.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1353, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1353, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1353, 1, 576) torch.bfloat16 - out : (1, 1353) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.4099 - top-1 accuracy : 14/366 = 3.8% - -Dataloader batch - tokens shape: torch.Size([1, 1150]) -Dataloader batch - labels shape: torch.Size([1, 1150]) -Dataloader batch - attn_mask shape: torch.Size([1, 1150]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1492 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1150) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1150) torch.int64 - loss_mask : 294 / 1150 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1150, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1150, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1150, 1, 576) torch.bfloat16 - out : (1, 1150) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.2888 - top-1 accuracy : 1/294 = 0.3% - - [2026-04-29 14:07:02] iteration 473/ 500 | consumed samples: 1892 | elapsed time per iteration (ms): 4232.9 | throughput (token/sec/GPU): 1278.3 | learning rate: 1.599753E-05 | global batch size: 4 | lm loss: 1.131980E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1426]) -Dataloader batch - labels shape: torch.Size([1, 1426]) -Dataloader batch - attn_mask shape: torch.Size([1, 1426]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1493 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1426) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1426) torch.int64 - loss_mask : 358 / 1426 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1426, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1426, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1426, 1, 576) torch.bfloat16 - out : (1, 1426) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 358 tokens) : 11.3051 - top-1 accuracy : 8/358 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1074]) -Dataloader batch - labels shape: torch.Size([1, 1074]) -Dataloader batch - attn_mask shape: torch.Size([1, 1074]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1494 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1074) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1074) torch.int64 - loss_mask : 259 / 1074 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1074, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1074, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1074, 1, 576) torch.bfloat16 - out : (1, 1074) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 259 tokens) : 11.2641 - top-1 accuracy : 8/259 = 3.1% - -Dataloader batch - tokens shape: torch.Size([1, 1695]) -Dataloader batch - labels shape: torch.Size([1, 1695]) -Dataloader batch - attn_mask shape: torch.Size([1, 1695]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1495 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1695) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1695) torch.int64 - loss_mask : 345 / 1695 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1695, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1695, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1695, 1, 576) torch.bfloat16 - out : (1, 1695) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 345 tokens) : 11.1758 - top-1 accuracy : 10/345 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1402]) -Dataloader batch - labels shape: torch.Size([1, 1402]) -Dataloader batch - attn_mask shape: torch.Size([1, 1402]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1496 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1402) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1402) torch.int64 - loss_mask : 330 / 1402 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1402, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1402, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1402, 1, 576) torch.bfloat16 - out : (1, 1402) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 330 tokens) : 11.2267 - top-1 accuracy : 4/330 = 1.2% - - [2026-04-29 14:07:07] iteration 474/ 500 | consumed samples: 1896 | elapsed time per iteration (ms): 4568.8 | throughput (token/sec/GPU): 1225.0 | learning rate: 1.577475E-05 | global batch size: 4 | lm loss: 1.124232E+01 | loss scale: 1.0 | grad norm: 0.165 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1518]) -Dataloader batch - labels shape: torch.Size([1, 1518]) -Dataloader batch - attn_mask shape: torch.Size([1, 1518]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1497 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1518) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1518) torch.int64 - loss_mask : 360 / 1518 tokens (23.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1518, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1518, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1518, 1, 576) torch.bfloat16 - out : (1, 1518) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 360 tokens) : 11.2463 - top-1 accuracy : 18/360 = 5.0% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1498 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 343 / 1508 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 343 tokens) : 11.2196 - top-1 accuracy : 24/343 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1816]) -Dataloader batch - labels shape: torch.Size([1, 1816]) -Dataloader batch - attn_mask shape: torch.Size([1, 1816]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1499 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1816) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1816) torch.int64 - loss_mask : 459 / 1816 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1816, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1816, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1816, 1, 576) torch.bfloat16 - out : (1, 1816) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 459 tokens) : 11.2777 - top-1 accuracy : 7/459 = 1.5% - -Dataloader batch - tokens shape: torch.Size([1, 1011]) -Dataloader batch - labels shape: torch.Size([1, 1011]) -Dataloader batch - attn_mask shape: torch.Size([1, 1011]) -Dataloader batch - image_grid_thw shape: torch.Size([19, 3]) - -==================================================================== - PIPELINE — STEP 1500 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1011) torch.int64 - images : (19, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1011) torch.int64 - loss_mask : 238 / 1011 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (19, 3, 1024, 1024) torch.bfloat16 - out : (19, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (19, 3072) - out : (19, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1011, 1, 576) torch.bfloat16 grad=False - image slots : 19 tokens replaced with vision embeddings - combined : (1011, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1011, 1, 576) torch.bfloat16 - out : (1, 1011) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 238 tokens) : 11.2286 - top-1 accuracy : 1/238 = 0.4% - - [2026-04-29 14:07:12] iteration 475/ 500 | consumed samples: 1900 | elapsed time per iteration (ms): 4954.2 | throughput (token/sec/GPU): 1181.4 | learning rate: 1.555335E-05 | global batch size: 4 | lm loss: 1.124705E+01 | loss scale: 1.0 | grad norm: 0.169 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1496]) -Dataloader batch - labels shape: torch.Size([1, 1496]) -Dataloader batch - attn_mask shape: torch.Size([1, 1496]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1501 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1496) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1496) torch.int64 - loss_mask : 326 / 1496 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1496, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1496, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1496, 1, 576) torch.bfloat16 - out : (1, 1496) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.2161 - top-1 accuracy : 27/326 = 8.3% - -Dataloader batch - tokens shape: torch.Size([1, 1252]) -Dataloader batch - labels shape: torch.Size([1, 1252]) -Dataloader batch - attn_mask shape: torch.Size([1, 1252]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1502 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1252) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1252) torch.int64 - loss_mask : 337 / 1252 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1252, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1252, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1252, 1, 576) torch.bfloat16 - out : (1, 1252) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 337 tokens) : 11.3371 - top-1 accuracy : 10/337 = 3.0% - -Dataloader batch - tokens shape: torch.Size([1, 1634]) -Dataloader batch - labels shape: torch.Size([1, 1634]) -Dataloader batch - attn_mask shape: torch.Size([1, 1634]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1503 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1634) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1634) torch.int64 - loss_mask : 389 / 1634 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1634, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1634, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1634, 1, 576) torch.bfloat16 - out : (1, 1634) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 389 tokens) : 11.2691 - top-1 accuracy : 19/389 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1634]) -Dataloader batch - labels shape: torch.Size([1, 1634]) -Dataloader batch - attn_mask shape: torch.Size([1, 1634]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1504 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1634) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1634) torch.int64 - loss_mask : 405 / 1634 tokens (24.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1634, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1634, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1634, 1, 576) torch.bfloat16 - out : (1, 1634) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 405 tokens) : 11.2708 - top-1 accuracy : 8/405 = 2.0% - - [2026-04-29 14:07:17] iteration 476/ 500 | consumed samples: 1904 | elapsed time per iteration (ms): 4848.5 | throughput (token/sec/GPU): 1240.8 | learning rate: 1.533333E-05 | global batch size: 4 | lm loss: 1.127345E+01 | loss scale: 1.0 | grad norm: 0.204 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1446]) -Dataloader batch - labels shape: torch.Size([1, 1446]) -Dataloader batch - attn_mask shape: torch.Size([1, 1446]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1505 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1446) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1446) torch.int64 - loss_mask : 376 / 1446 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1446, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1446, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1446, 1, 576) torch.bfloat16 - out : (1, 1446) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 376 tokens) : 11.2840 - top-1 accuracy : 1/376 = 0.3% - -Dataloader batch - tokens shape: torch.Size([1, 1339]) -Dataloader batch - labels shape: torch.Size([1, 1339]) -Dataloader batch - attn_mask shape: torch.Size([1, 1339]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1506 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1339) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1339) torch.int64 - loss_mask : 286 / 1339 tokens (21.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1339, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1339, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1339, 1, 576) torch.bfloat16 - out : (1, 1339) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 286 tokens) : 11.2181 - top-1 accuracy : 8/286 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1659]) -Dataloader batch - labels shape: torch.Size([1, 1659]) -Dataloader batch - attn_mask shape: torch.Size([1, 1659]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1507 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1659) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1659) torch.int64 - loss_mask : 407 / 1659 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1659, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1659, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1659, 1, 576) torch.bfloat16 - out : (1, 1659) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 407 tokens) : 11.2514 - top-1 accuracy : 19/407 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1562]) -Dataloader batch - labels shape: torch.Size([1, 1562]) -Dataloader batch - attn_mask shape: torch.Size([1, 1562]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1508 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1562) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1562) torch.int64 - loss_mask : 348 / 1562 tokens (22.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1562, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1562, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1562, 1, 576) torch.bfloat16 - out : (1, 1562) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2049 - top-1 accuracy : 9/348 = 2.6% - - [2026-04-29 14:07:21] iteration 477/ 500 | consumed samples: 1908 | elapsed time per iteration (ms): 4883.3 | throughput (token/sec/GPU): 1229.9 | learning rate: 1.511471E-05 | global batch size: 4 | lm loss: 1.124190E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1539]) -Dataloader batch - labels shape: torch.Size([1, 1539]) -Dataloader batch - attn_mask shape: torch.Size([1, 1539]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1509 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1539) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1539) torch.int64 - loss_mask : 359 / 1539 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1539, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1539, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1539, 1, 576) torch.bfloat16 - out : (1, 1539) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.1771 - top-1 accuracy : 12/359 = 3.3% - -Dataloader batch - tokens shape: torch.Size([1, 1714]) -Dataloader batch - labels shape: torch.Size([1, 1714]) -Dataloader batch - attn_mask shape: torch.Size([1, 1714]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1510 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1714) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1714) torch.int64 - loss_mask : 356 / 1714 tokens (20.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1714, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1714, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1714, 1, 576) torch.bfloat16 - out : (1, 1714) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 356 tokens) : 11.2154 - top-1 accuracy : 25/356 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1511 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 416 / 1546 tokens (26.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 416 tokens) : 11.3206 - top-1 accuracy : 12/416 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1204]) -Dataloader batch - labels shape: torch.Size([1, 1204]) -Dataloader batch - attn_mask shape: torch.Size([1, 1204]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1512 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1204) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1204) torch.int64 - loss_mask : 308 / 1204 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1204, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1204, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1204, 1, 576) torch.bfloat16 - out : (1, 1204) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 308 tokens) : 11.3168 - top-1 accuracy : 8/308 = 2.6% - - [2026-04-29 14:07:26] iteration 478/ 500 | consumed samples: 1912 | elapsed time per iteration (ms): 4906.6 | throughput (token/sec/GPU): 1223.4 | learning rate: 1.489749E-05 | global batch size: 4 | lm loss: 1.125796E+01 | loss scale: 1.0 | grad norm: 0.201 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1437]) -Dataloader batch - labels shape: torch.Size([1, 1437]) -Dataloader batch - attn_mask shape: torch.Size([1, 1437]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1513 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1437) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1437) torch.int64 - loss_mask : 382 / 1437 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1437, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1437, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1437, 1, 576) torch.bfloat16 - out : (1, 1437) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.3255 - top-1 accuracy : 5/382 = 1.3% - -Dataloader batch - tokens shape: torch.Size([1, 1417]) -Dataloader batch - labels shape: torch.Size([1, 1417]) -Dataloader batch - attn_mask shape: torch.Size([1, 1417]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1514 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1417) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1417) torch.int64 - loss_mask : 359 / 1417 tokens (25.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1417, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1417, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1417, 1, 576) torch.bfloat16 - out : (1, 1417) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 359 tokens) : 11.2890 - top-1 accuracy : 7/359 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1480]) -Dataloader batch - labels shape: torch.Size([1, 1480]) -Dataloader batch - attn_mask shape: torch.Size([1, 1480]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1515 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1480) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1480) torch.int64 - loss_mask : 326 / 1480 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1480, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1480, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1480, 1, 576) torch.bfloat16 - out : (1, 1480) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 326 tokens) : 11.1895 - top-1 accuracy : 13/326 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1017]) -Dataloader batch - labels shape: torch.Size([1, 1017]) -Dataloader batch - attn_mask shape: torch.Size([1, 1017]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1516 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1017) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1017) torch.int64 - loss_mask : 207 / 1017 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1017, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1017, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1017, 1, 576) torch.bfloat16 - out : (1, 1017) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 207 tokens) : 11.1882 - top-1 accuracy : 8/207 = 3.9% - - [2026-04-29 14:07:31] iteration 479/ 500 | consumed samples: 1916 | elapsed time per iteration (ms): 4669.5 | throughput (token/sec/GPU): 1146.0 | learning rate: 1.468168E-05 | global batch size: 4 | lm loss: 1.125812E+01 | loss scale: 1.0 | grad norm: 0.224 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1554]) -Dataloader batch - labels shape: torch.Size([1, 1554]) -Dataloader batch - attn_mask shape: torch.Size([1, 1554]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1517 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1554) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1554) torch.int64 - loss_mask : 396 / 1554 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1554, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1554, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1554, 1, 576) torch.bfloat16 - out : (1, 1554) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 396 tokens) : 11.2889 - top-1 accuracy : 25/396 = 6.3% - -Dataloader batch - tokens shape: torch.Size([1, 1096]) -Dataloader batch - labels shape: torch.Size([1, 1096]) -Dataloader batch - attn_mask shape: torch.Size([1, 1096]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1518 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1096) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1096) torch.int64 - loss_mask : 262 / 1096 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1096, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1096, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1096, 1, 576) torch.bfloat16 - out : (1, 1096) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 262 tokens) : 11.3491 - top-1 accuracy : 14/262 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1421]) -Dataloader batch - labels shape: torch.Size([1, 1421]) -Dataloader batch - attn_mask shape: torch.Size([1, 1421]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1519 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1421) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1421) torch.int64 - loss_mask : 344 / 1421 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1421, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1421, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1421, 1, 576) torch.bfloat16 - out : (1, 1421) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 344 tokens) : 11.2638 - top-1 accuracy : 6/344 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1778]) -Dataloader batch - labels shape: torch.Size([1, 1778]) -Dataloader batch - attn_mask shape: torch.Size([1, 1778]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1520 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1778) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1778) torch.int64 - loss_mask : 410 / 1778 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1778, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1778, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1778, 1, 576) torch.bfloat16 - out : (1, 1778) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 410 tokens) : 11.2502 - top-1 accuracy : 23/410 = 5.6% - - [2026-04-29 14:07:36] iteration 480/ 500 | consumed samples: 1920 | elapsed time per iteration (ms): 4647.2 | throughput (token/sec/GPU): 1258.6 | learning rate: 1.446729E-05 | global batch size: 4 | lm loss: 1.128271E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (4633.17, 4633.17) - forward-compute ................................: (3437.56, 3437.56) - backward-compute ...............................: (1193.56, 1193.56) - batch-generator ................................: (414.07, 414.07) - layernorm-grads-all-reduce .....................: (0.02, 0.02) - embedding-grads-all-reduce .....................: (0.03, 0.03) - all-grads-sync .................................: (0.44, 0.44) - params-all-gather ..............................: (0.14, 0.14) - optimizer-copy-to-main-grad ....................: (0.11, 0.11) - optimizer-clip-main-grad .......................: (0.50, 0.50) - optimizer-count-zeros ..........................: (0.01, 0.01) - optimizer-inner-step ...........................: (1.27, 1.27) - optimizer-copy-main-to-model-params ............: (0.20, 0.20) - optimizer ......................................: (2.80, 2.80) -Dataloader batch - tokens shape: torch.Size([1, 1687]) -Dataloader batch - labels shape: torch.Size([1, 1687]) -Dataloader batch - attn_mask shape: torch.Size([1, 1687]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1521 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1687) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1687) torch.int64 - loss_mask : 435 / 1687 tokens (25.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1687, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1687, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1687, 1, 576) torch.bfloat16 - out : (1, 1687) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 435 tokens) : 11.2843 - top-1 accuracy : 16/435 = 3.7% - -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1522 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 413 / 1750 tokens (23.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2534 - top-1 accuracy : 21/413 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1090]) -Dataloader batch - labels shape: torch.Size([1, 1090]) -Dataloader batch - attn_mask shape: torch.Size([1, 1090]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1523 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1090) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1090) torch.int64 - loss_mask : 246 / 1090 tokens (22.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1090, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1090, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1090, 1, 576) torch.bfloat16 - out : (1, 1090) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 246 tokens) : 11.2771 - top-1 accuracy : 14/246 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1067]) -Dataloader batch - labels shape: torch.Size([1, 1067]) -Dataloader batch - attn_mask shape: torch.Size([1, 1067]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1524 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1067) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1067) torch.int64 - loss_mask : 236 / 1067 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1067, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1067, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1067, 1, 576) torch.bfloat16 - out : (1, 1067) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 236 tokens) : 11.2189 - top-1 accuracy : 6/236 = 2.5% - - [2026-04-29 14:07:40] iteration 481/ 500 | consumed samples: 1924 | elapsed time per iteration (ms): 4789.9 | throughput (token/sec/GPU): 1167.9 | learning rate: 1.425433E-05 | global batch size: 4 | lm loss: 1.126175E+01 | loss scale: 1.0 | grad norm: 0.208 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1462]) -Dataloader batch - labels shape: torch.Size([1, 1462]) -Dataloader batch - attn_mask shape: torch.Size([1, 1462]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1525 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1462) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1462) torch.int64 - loss_mask : 332 / 1462 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1462, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1462, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1462, 1, 576) torch.bfloat16 - out : (1, 1462) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 332 tokens) : 11.2026 - top-1 accuracy : 7/332 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1043]) -Dataloader batch - labels shape: torch.Size([1, 1043]) -Dataloader batch - attn_mask shape: torch.Size([1, 1043]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1526 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1043) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1043) torch.int64 - loss_mask : 278 / 1043 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1043, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1043, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1043, 1, 576) torch.bfloat16 - out : (1, 1043) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 278 tokens) : 11.3088 - top-1 accuracy : 6/278 = 2.2% - -Dataloader batch - tokens shape: torch.Size([1, 1710]) -Dataloader batch - labels shape: torch.Size([1, 1710]) -Dataloader batch - attn_mask shape: torch.Size([1, 1710]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1527 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1710) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1710) torch.int64 - loss_mask : 347 / 1710 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1710, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1710, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1710, 1, 576) torch.bfloat16 - out : (1, 1710) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.2217 - top-1 accuracy : 17/347 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1288]) -Dataloader batch - labels shape: torch.Size([1, 1288]) -Dataloader batch - attn_mask shape: torch.Size([1, 1288]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1528 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1288) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1288) torch.int64 - loss_mask : 383 / 1288 tokens (29.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1288, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1288, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1288, 1, 576) torch.bfloat16 - out : (1, 1288) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.4065 - top-1 accuracy : 3/383 = 0.8% - - [2026-04-29 14:07:45] iteration 482/ 500 | consumed samples: 1928 | elapsed time per iteration (ms): 4553.9 | throughput (token/sec/GPU): 1208.4 | learning rate: 1.404281E-05 | global batch size: 4 | lm loss: 1.128785E+01 | loss scale: 1.0 | grad norm: 0.213 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1778]) -Dataloader batch - labels shape: torch.Size([1, 1778]) -Dataloader batch - attn_mask shape: torch.Size([1, 1778]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1529 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1778) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1778) torch.int64 - loss_mask : 411 / 1778 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1778, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1778, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1778, 1, 576) torch.bfloat16 - out : (1, 1778) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 411 tokens) : 11.2581 - top-1 accuracy : 21/411 = 5.1% - -Dataloader batch - tokens shape: torch.Size([1, 1828]) -Dataloader batch - labels shape: torch.Size([1, 1828]) -Dataloader batch - attn_mask shape: torch.Size([1, 1828]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1530 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1828) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1828) torch.int64 - loss_mask : 475 / 1828 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1828, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1828, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1828, 1, 576) torch.bfloat16 - out : (1, 1828) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 475 tokens) : 11.3228 - top-1 accuracy : 12/475 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1730]) -Dataloader batch - labels shape: torch.Size([1, 1730]) -Dataloader batch - attn_mask shape: torch.Size([1, 1730]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1531 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1730) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1730) torch.int64 - loss_mask : 383 / 1730 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1730, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1730, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1730, 1, 576) torch.bfloat16 - out : (1, 1730) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.2425 - top-1 accuracy : 26/383 = 6.8% - -Dataloader batch - tokens shape: torch.Size([1, 1678]) -Dataloader batch - labels shape: torch.Size([1, 1678]) -Dataloader batch - attn_mask shape: torch.Size([1, 1678]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1532 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1678) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1678) torch.int64 - loss_mask : 403 / 1678 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1678, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1678, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1678, 1, 576) torch.bfloat16 - out : (1, 1678) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 403 tokens) : 11.2870 - top-1 accuracy : 7/403 = 1.7% - - [2026-04-29 14:07:50] iteration 483/ 500 | consumed samples: 1932 | elapsed time per iteration (ms): 5407.2 | throughput (token/sec/GPU): 1297.2 | learning rate: 1.383273E-05 | global batch size: 4 | lm loss: 1.127985E+01 | loss scale: 1.0 | grad norm: 0.212 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1899]) -Dataloader batch - labels shape: torch.Size([1, 1899]) -Dataloader batch - attn_mask shape: torch.Size([1, 1899]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1533 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1899) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1899) torch.int64 - loss_mask : 533 / 1899 tokens (28.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1899, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1899, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1899, 1, 576) torch.bfloat16 - out : (1, 1899) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 533 tokens) : 11.3547 - top-1 accuracy : 9/533 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1812]) -Dataloader batch - labels shape: torch.Size([1, 1812]) -Dataloader batch - attn_mask shape: torch.Size([1, 1812]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1534 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1812) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1812) torch.int64 - loss_mask : 520 / 1812 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1812, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1812, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1812, 1, 576) torch.bfloat16 - out : (1, 1812) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 520 tokens) : 11.3370 - top-1 accuracy : 9/520 = 1.7% - -Dataloader batch - tokens shape: torch.Size([1, 1211]) -Dataloader batch - labels shape: torch.Size([1, 1211]) -Dataloader batch - attn_mask shape: torch.Size([1, 1211]) -Dataloader batch - image_grid_thw shape: torch.Size([21, 3]) - -==================================================================== - PIPELINE — STEP 1535 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1211) torch.int64 - images : (21, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1211) torch.int64 - loss_mask : 296 / 1211 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (21, 3, 1024, 1024) torch.bfloat16 - out : (21, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (21, 3072) - out : (21, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1211, 1, 576) torch.bfloat16 grad=False - image slots : 21 tokens replaced with vision embeddings - combined : (1211, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1211, 1, 576) torch.bfloat16 - out : (1, 1211) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 296 tokens) : 11.3540 - top-1 accuracy : 2/296 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1464]) -Dataloader batch - labels shape: torch.Size([1, 1464]) -Dataloader batch - attn_mask shape: torch.Size([1, 1464]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1536 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1464) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1464) torch.int64 - loss_mask : 368 / 1464 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1464, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1464, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1464, 1, 576) torch.bfloat16 - out : (1, 1464) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2662 - top-1 accuracy : 10/368 = 2.7% - - [2026-04-29 14:07:55] iteration 484/ 500 | consumed samples: 1936 | elapsed time per iteration (ms): 4977.2 | throughput (token/sec/GPU): 1283.0 | learning rate: 1.362411E-05 | global batch size: 4 | lm loss: 1.133028E+01 | loss scale: 1.0 | grad norm: 0.184 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1381]) -Dataloader batch - labels shape: torch.Size([1, 1381]) -Dataloader batch - attn_mask shape: torch.Size([1, 1381]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1537 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1381) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1381) torch.int64 - loss_mask : 329 / 1381 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1381, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1381, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1381, 1, 576) torch.bfloat16 - out : (1, 1381) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 329 tokens) : 11.2298 - top-1 accuracy : 4/329 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 1391]) -Dataloader batch - labels shape: torch.Size([1, 1391]) -Dataloader batch - attn_mask shape: torch.Size([1, 1391]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1538 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1391) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1391) torch.int64 - loss_mask : 341 / 1391 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1391, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1391, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1391, 1, 576) torch.bfloat16 - out : (1, 1391) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 341 tokens) : 11.2992 - top-1 accuracy : 22/341 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1420]) -Dataloader batch - labels shape: torch.Size([1, 1420]) -Dataloader batch - attn_mask shape: torch.Size([1, 1420]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1539 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1420) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1420) torch.int64 - loss_mask : 323 / 1420 tokens (22.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1420, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1420, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1420, 1, 576) torch.bfloat16 - out : (1, 1420) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.2580 - top-1 accuracy : 17/323 = 5.3% - -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1540 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 274 / 1364 tokens (20.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 274 tokens) : 11.1123 - top-1 accuracy : 17/274 = 6.2% - - [2026-04-29 14:08:00] iteration 485/ 500 | consumed samples: 1940 | elapsed time per iteration (ms): 4650.3 | throughput (token/sec/GPU): 1194.8 | learning rate: 1.341694E-05 | global batch size: 4 | lm loss: 1.123029E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1590]) -Dataloader batch - labels shape: torch.Size([1, 1590]) -Dataloader batch - attn_mask shape: torch.Size([1, 1590]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1541 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1590) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1590) torch.int64 - loss_mask : 339 / 1590 tokens (21.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1590, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1590, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1590, 1, 576) torch.bfloat16 - out : (1, 1590) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.1850 - top-1 accuracy : 30/339 = 8.8% - -Dataloader batch - tokens shape: torch.Size([1, 959]) -Dataloader batch - labels shape: torch.Size([1, 959]) -Dataloader batch - attn_mask shape: torch.Size([1, 959]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1542 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 959) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 959) torch.int64 - loss_mask : 208 / 959 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (959, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (959, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (959, 1, 576) torch.bfloat16 - out : (1, 959) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 208 tokens) : 11.1609 - top-1 accuracy : 18/208 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1863]) -Dataloader batch - labels shape: torch.Size([1, 1863]) -Dataloader batch - attn_mask shape: torch.Size([1, 1863]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1543 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1863) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1863) torch.int64 - loss_mask : 515 / 1863 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1863, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1863, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1863, 1, 576) torch.bfloat16 - out : (1, 1863) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 515 tokens) : 11.3316 - top-1 accuracy : 13/515 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1885]) -Dataloader batch - labels shape: torch.Size([1, 1885]) -Dataloader batch - attn_mask shape: torch.Size([1, 1885]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1544 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1885) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1885) torch.int64 - loss_mask : 542 / 1885 tokens (28.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1885, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1885, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1885, 1, 576) torch.bfloat16 - out : (1, 1885) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 542 tokens) : 11.3700 - top-1 accuracy : 17/542 = 3.1% - - [2026-04-29 14:08:05] iteration 486/ 500 | consumed samples: 1944 | elapsed time per iteration (ms): 4974.0 | throughput (token/sec/GPU): 1266.0 | learning rate: 1.321125E-05 | global batch size: 4 | lm loss: 1.129146E+01 | loss scale: 1.0 | grad norm: 0.217 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1545 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 357 / 1546 tokens (23.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 357 tokens) : 11.2731 - top-1 accuracy : 5/357 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1242]) -Dataloader batch - labels shape: torch.Size([1, 1242]) -Dataloader batch - attn_mask shape: torch.Size([1, 1242]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1546 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1242) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1242) torch.int64 - loss_mask : 315 / 1242 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1242, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1242, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1242, 1, 576) torch.bfloat16 - out : (1, 1242) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 315 tokens) : 11.3054 - top-1 accuracy : 10/315 = 3.2% - -Dataloader batch - tokens shape: torch.Size([1, 1533]) -Dataloader batch - labels shape: torch.Size([1, 1533]) -Dataloader batch - attn_mask shape: torch.Size([1, 1533]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1547 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1533) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1533) torch.int64 - loss_mask : 386 / 1533 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1533, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1533, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1533, 1, 576) torch.bfloat16 - out : (1, 1533) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.3142 - top-1 accuracy : 18/386 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1471]) -Dataloader batch - labels shape: torch.Size([1, 1471]) -Dataloader batch - attn_mask shape: torch.Size([1, 1471]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1548 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1471) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1471) torch.int64 - loss_mask : 319 / 1471 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1471, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1471, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1471, 1, 576) torch.bfloat16 - out : (1, 1471) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 319 tokens) : 11.1810 - top-1 accuracy : 32/319 = 10.0% - - [2026-04-29 14:08:10] iteration 487/ 500 | consumed samples: 1948 | elapsed time per iteration (ms): 4861.2 | throughput (token/sec/GPU): 1191.5 | learning rate: 1.300703E-05 | global batch size: 4 | lm loss: 1.127067E+01 | loss scale: 1.0 | grad norm: 0.188 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1364]) -Dataloader batch - labels shape: torch.Size([1, 1364]) -Dataloader batch - attn_mask shape: torch.Size([1, 1364]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1549 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1364) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1364) torch.int64 - loss_mask : 287 / 1364 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1364, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1364, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1364, 1, 576) torch.bfloat16 - out : (1, 1364) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 287 tokens) : 11.1446 - top-1 accuracy : 14/287 = 4.9% - -Dataloader batch - tokens shape: torch.Size([1, 1213]) -Dataloader batch - labels shape: torch.Size([1, 1213]) -Dataloader batch - attn_mask shape: torch.Size([1, 1213]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1550 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1213) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1213) torch.int64 - loss_mask : 294 / 1213 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1213, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1213, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1213, 1, 576) torch.bfloat16 - out : (1, 1213) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 294 tokens) : 11.2555 - top-1 accuracy : 4/294 = 1.4% - -Dataloader batch - tokens shape: torch.Size([1, 1119]) -Dataloader batch - labels shape: torch.Size([1, 1119]) -Dataloader batch - attn_mask shape: torch.Size([1, 1119]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1551 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1119) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1119) torch.int64 - loss_mask : 263 / 1119 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1119, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1119, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1119, 1, 576) torch.bfloat16 - out : (1, 1119) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 263 tokens) : 11.2451 - top-1 accuracy : 23/263 = 8.7% - -Dataloader batch - tokens shape: torch.Size([1, 1508]) -Dataloader batch - labels shape: torch.Size([1, 1508]) -Dataloader batch - attn_mask shape: torch.Size([1, 1508]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1552 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1508) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1508) torch.int64 - loss_mask : 379 / 1508 tokens (25.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1508, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1508, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1508, 1, 576) torch.bfloat16 - out : (1, 1508) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 379 tokens) : 11.2847 - top-1 accuracy : 28/379 = 7.4% - - [2026-04-29 14:08:14] iteration 488/ 500 | consumed samples: 1952 | elapsed time per iteration (ms): 4250.1 | throughput (token/sec/GPU): 1224.4 | learning rate: 1.280430E-05 | global batch size: 4 | lm loss: 1.123629E+01 | loss scale: 1.0 | grad norm: 0.183 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1336]) -Dataloader batch - labels shape: torch.Size([1, 1336]) -Dataloader batch - attn_mask shape: torch.Size([1, 1336]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1553 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1336) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1336) torch.int64 - loss_mask : 374 / 1336 tokens (28.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1336, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1336, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1336, 1, 576) torch.bfloat16 - out : (1, 1336) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 374 tokens) : 11.3531 - top-1 accuracy : 13/374 = 3.5% - -Dataloader batch - tokens shape: torch.Size([1, 1441]) -Dataloader batch - labels shape: torch.Size([1, 1441]) -Dataloader batch - attn_mask shape: torch.Size([1, 1441]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 1554 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1441) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1441) torch.int64 - loss_mask : 417 / 1441 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1441, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1441, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1441, 1, 576) torch.bfloat16 - out : (1, 1441) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 417 tokens) : 11.3869 - top-1 accuracy : 1/417 = 0.2% - -Dataloader batch - tokens shape: torch.Size([1, 1299]) -Dataloader batch - labels shape: torch.Size([1, 1299]) -Dataloader batch - attn_mask shape: torch.Size([1, 1299]) -Dataloader batch - image_grid_thw shape: torch.Size([23, 3]) - -==================================================================== - PIPELINE — STEP 1555 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1299) torch.int64 - images : (23, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1299) torch.int64 - loss_mask : 339 / 1299 tokens (26.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (23, 3, 1024, 1024) torch.bfloat16 - out : (23, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (23, 3072) - out : (23, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1299, 1, 576) torch.bfloat16 grad=False - image slots : 23 tokens replaced with vision embeddings - combined : (1299, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1299, 1, 576) torch.bfloat16 - out : (1, 1299) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 339 tokens) : 11.3072 - top-1 accuracy : 6/339 = 1.8% - -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1556 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 347 / 1525 tokens (22.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 347 tokens) : 11.1929 - top-1 accuracy : 20/347 = 5.8% - - [2026-04-29 14:08:19] iteration 489/ 500 | consumed samples: 1956 | elapsed time per iteration (ms): 4387.2 | throughput (token/sec/GPU): 1276.7 | learning rate: 1.260307E-05 | global batch size: 4 | lm loss: 1.131447E+01 | loss scale: 1.0 | grad norm: 0.257 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1759]) -Dataloader batch - labels shape: torch.Size([1, 1759]) -Dataloader batch - attn_mask shape: torch.Size([1, 1759]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1557 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1759) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1759) torch.int64 - loss_mask : 413 / 1759 tokens (23.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1759, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1759, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1759, 1, 576) torch.bfloat16 - out : (1, 1759) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 413 tokens) : 11.2797 - top-1 accuracy : 34/413 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1580]) -Dataloader batch - labels shape: torch.Size([1, 1580]) -Dataloader batch - attn_mask shape: torch.Size([1, 1580]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1558 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1580) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1580) torch.int64 - loss_mask : 420 / 1580 tokens (26.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1580, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1580, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1580, 1, 576) torch.bfloat16 - out : (1, 1580) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 420 tokens) : 11.3710 - top-1 accuracy : 8/420 = 1.9% - -Dataloader batch - tokens shape: torch.Size([1, 1952]) -Dataloader batch - labels shape: torch.Size([1, 1952]) -Dataloader batch - attn_mask shape: torch.Size([1, 1952]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1559 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1952) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1952) torch.int64 - loss_mask : 582 / 1952 tokens (29.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1952, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1952, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1952, 1, 576) torch.bfloat16 - out : (1, 1952) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 582 tokens) : 11.3992 - top-1 accuracy : 5/582 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1520]) -Dataloader batch - labels shape: torch.Size([1, 1520]) -Dataloader batch - attn_mask shape: torch.Size([1, 1520]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1560 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1520) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1520) torch.int64 - loss_mask : 388 / 1520 tokens (25.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1520, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1520, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1520, 1, 576) torch.bfloat16 - out : (1, 1520) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 388 tokens) : 11.3180 - top-1 accuracy : 16/388 = 4.1% - - [2026-04-29 14:08:24] iteration 490/ 500 | consumed samples: 1960 | elapsed time per iteration (ms): 5158.4 | throughput (token/sec/GPU): 1320.4 | learning rate: 1.240333E-05 | global batch size: 4 | lm loss: 1.134782E+01 | loss scale: 1.0 | grad norm: 0.167 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1180]) -Dataloader batch - labels shape: torch.Size([1, 1180]) -Dataloader batch - attn_mask shape: torch.Size([1, 1180]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1561 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1180) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1180) torch.int64 - loss_mask : 260 / 1180 tokens (22.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1180, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1180, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1180, 1, 576) torch.bfloat16 - out : (1, 1180) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 260 tokens) : 11.2341 - top-1 accuracy : 15/260 = 5.8% - -Dataloader batch - tokens shape: torch.Size([1, 1440]) -Dataloader batch - labels shape: torch.Size([1, 1440]) -Dataloader batch - attn_mask shape: torch.Size([1, 1440]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1562 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1440) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1440) torch.int64 - loss_mask : 366 / 1440 tokens (25.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1440, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1440, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1440, 1, 576) torch.bfloat16 - out : (1, 1440) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 366 tokens) : 11.3004 - top-1 accuracy : 30/366 = 8.2% - -Dataloader batch - tokens shape: torch.Size([1, 1476]) -Dataloader batch - labels shape: torch.Size([1, 1476]) -Dataloader batch - attn_mask shape: torch.Size([1, 1476]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1563 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1476) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1476) torch.int64 - loss_mask : 427 / 1476 tokens (28.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1476, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1476, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1476, 1, 576) torch.bfloat16 - out : (1, 1476) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 427 tokens) : 11.3783 - top-1 accuracy : 5/427 = 1.2% - -Dataloader batch - tokens shape: torch.Size([1, 976]) -Dataloader batch - labels shape: torch.Size([1, 976]) -Dataloader batch - attn_mask shape: torch.Size([1, 976]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1564 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 976) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 976) torch.int64 - loss_mask : 224 / 976 tokens (23.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (976, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (976, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (976, 1, 576) torch.bfloat16 - out : (1, 976) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 224 tokens) : 11.2594 - top-1 accuracy : 5/224 = 2.2% - - [2026-04-29 14:08:28] iteration 491/ 500 | consumed samples: 1964 | elapsed time per iteration (ms): 4220.9 | throughput (token/sec/GPU): 1201.6 | learning rate: 1.220511E-05 | global batch size: 4 | lm loss: 1.130575E+01 | loss scale: 1.0 | grad norm: 0.189 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1729]) -Dataloader batch - labels shape: torch.Size([1, 1729]) -Dataloader batch - attn_mask shape: torch.Size([1, 1729]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1565 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1729) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1729) torch.int64 - loss_mask : 419 / 1729 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1729, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1729, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1729, 1, 576) torch.bfloat16 - out : (1, 1729) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 419 tokens) : 11.3419 - top-1 accuracy : 9/419 = 2.1% - -Dataloader batch - tokens shape: torch.Size([1, 1567]) -Dataloader batch - labels shape: torch.Size([1, 1567]) -Dataloader batch - attn_mask shape: torch.Size([1, 1567]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1566 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1567) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1567) torch.int64 - loss_mask : 386 / 1567 tokens (24.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1567, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1567, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1567, 1, 576) torch.bfloat16 - out : (1, 1567) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2434 - top-1 accuracy : 11/386 = 2.8% - -Dataloader batch - tokens shape: torch.Size([1, 1639]) -Dataloader batch - labels shape: torch.Size([1, 1639]) -Dataloader batch - attn_mask shape: torch.Size([1, 1639]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1567 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1639) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1639) torch.int64 - loss_mask : 383 / 1639 tokens (23.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1639, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1639, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1639, 1, 576) torch.bfloat16 - out : (1, 1639) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 383 tokens) : 11.2370 - top-1 accuracy : 16/383 = 4.2% - -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1568 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 348 / 1436 tokens (24.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 348 tokens) : 11.2805 - top-1 accuracy : 19/348 = 5.5% - - [2026-04-29 14:08:33] iteration 492/ 500 | consumed samples: 1968 | elapsed time per iteration (ms): 5127.4 | throughput (token/sec/GPU): 1242.5 | learning rate: 1.200840E-05 | global batch size: 4 | lm loss: 1.127707E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1750]) -Dataloader batch - labels shape: torch.Size([1, 1750]) -Dataloader batch - attn_mask shape: torch.Size([1, 1750]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1569 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1750) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1750) torch.int64 - loss_mask : 386 / 1750 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1750, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1750, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1750, 1, 576) torch.bfloat16 - out : (1, 1750) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 386 tokens) : 11.2310 - top-1 accuracy : 27/386 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1570 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 382 / 1525 tokens (25.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 382 tokens) : 11.2868 - top-1 accuracy : 13/382 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1203]) -Dataloader batch - labels shape: torch.Size([1, 1203]) -Dataloader batch - attn_mask shape: torch.Size([1, 1203]) -Dataloader batch - image_grid_thw shape: torch.Size([22, 3]) - -==================================================================== - PIPELINE — STEP 1571 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1203) torch.int64 - images : (22, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1203) torch.int64 - loss_mask : 275 / 1203 tokens (22.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (22, 3, 1024, 1024) torch.bfloat16 - out : (22, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (22, 3072) - out : (22, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1203, 1, 576) torch.bfloat16 grad=False - image slots : 22 tokens replaced with vision embeddings - combined : (1203, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1203, 1, 576) torch.bfloat16 - out : (1, 1203) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 275 tokens) : 11.2741 - top-1 accuracy : 2/275 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1475]) -Dataloader batch - labels shape: torch.Size([1, 1475]) -Dataloader batch - attn_mask shape: torch.Size([1, 1475]) -Dataloader batch - image_grid_thw shape: torch.Size([24, 3]) - -==================================================================== - PIPELINE — STEP 1572 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1475) torch.int64 - images : (24, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1475) torch.int64 - loss_mask : 465 / 1475 tokens (31.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (24, 3, 1024, 1024) torch.bfloat16 - out : (24, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (24, 3072) - out : (24, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1475, 1, 576) torch.bfloat16 grad=False - image slots : 24 tokens replaced with vision embeddings - combined : (1475, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1475, 1, 576) torch.bfloat16 - out : (1, 1475) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 465 tokens) : 11.4555 - top-1 accuracy : 15/465 = 3.2% - - [2026-04-29 14:08:38] iteration 493/ 500 | consumed samples: 1972 | elapsed time per iteration (ms): 4720.6 | throughput (token/sec/GPU): 1261.1 | learning rate: 1.181322E-05 | global batch size: 4 | lm loss: 1.132224E+01 | loss scale: 1.0 | grad norm: 0.192 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1721]) -Dataloader batch - labels shape: torch.Size([1, 1721]) -Dataloader batch - attn_mask shape: torch.Size([1, 1721]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1573 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1721) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1721) torch.int64 - loss_mask : 361 / 1721 tokens (21.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1721, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1721, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1721, 1, 576) torch.bfloat16 - out : (1, 1721) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 361 tokens) : 11.1916 - top-1 accuracy : 33/361 = 9.1% - -Dataloader batch - tokens shape: torch.Size([1, 1749]) -Dataloader batch - labels shape: torch.Size([1, 1749]) -Dataloader batch - attn_mask shape: torch.Size([1, 1749]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1574 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1749) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1749) torch.int64 - loss_mask : 387 / 1749 tokens (22.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1749, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1749, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1749, 1, 576) torch.bfloat16 - out : (1, 1749) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 387 tokens) : 11.2710 - top-1 accuracy : 25/387 = 6.5% - -Dataloader batch - tokens shape: torch.Size([1, 1846]) -Dataloader batch - labels shape: torch.Size([1, 1846]) -Dataloader batch - attn_mask shape: torch.Size([1, 1846]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1575 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1846) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1846) torch.int64 - loss_mask : 493 / 1846 tokens (26.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1846, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1846, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1846, 1, 576) torch.bfloat16 - out : (1, 1846) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 493 tokens) : 11.3375 - top-1 accuracy : 13/493 = 2.6% - -Dataloader batch - tokens shape: torch.Size([1, 1916]) -Dataloader batch - labels shape: torch.Size([1, 1916]) -Dataloader batch - attn_mask shape: torch.Size([1, 1916]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1576 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1916) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1916) torch.int64 - loss_mask : 549 / 1916 tokens (28.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1916, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1916, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1916, 1, 576) torch.bfloat16 - out : (1, 1916) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 549 tokens) : 11.3890 - top-1 accuracy : 8/549 = 1.5% - - [2026-04-29 14:08:43] iteration 494/ 500 | consumed samples: 1976 | elapsed time per iteration (ms): 5454.9 | throughput (token/sec/GPU): 1325.8 | learning rate: 1.161958E-05 | global batch size: 4 | lm loss: 1.130949E+01 | loss scale: 1.0 | grad norm: 0.187 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1664]) -Dataloader batch - labels shape: torch.Size([1, 1664]) -Dataloader batch - attn_mask shape: torch.Size([1, 1664]) -Dataloader batch - image_grid_thw shape: torch.Size([30, 3]) - -==================================================================== - PIPELINE — STEP 1577 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1664) torch.int64 - images : (30, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1664) torch.int64 - loss_mask : 396 / 1664 tokens (23.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (30, 3, 1024, 1024) torch.bfloat16 - out : (30, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (30, 3072) - out : (30, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1664, 1, 576) torch.bfloat16 grad=False - image slots : 30 tokens replaced with vision embeddings - combined : (1664, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1664, 1, 576) torch.bfloat16 - out : (1, 1664) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 396 tokens) : 11.1905 - top-1 accuracy : 29/396 = 7.3% - -Dataloader batch - tokens shape: torch.Size([1, 1778]) -Dataloader batch - labels shape: torch.Size([1, 1778]) -Dataloader batch - attn_mask shape: torch.Size([1, 1778]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1578 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1778) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1778) torch.int64 - loss_mask : 470 / 1778 tokens (26.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1778, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1778, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1778, 1, 576) torch.bfloat16 - out : (1, 1778) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 470 tokens) : 11.3403 - top-1 accuracy : 22/470 = 4.7% - -Dataloader batch - tokens shape: torch.Size([1, 1851]) -Dataloader batch - labels shape: torch.Size([1, 1851]) -Dataloader batch - attn_mask shape: torch.Size([1, 1851]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1579 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1851) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1851) torch.int64 - loss_mask : 481 / 1851 tokens (26.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1851, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1851, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1851, 1, 576) torch.bfloat16 - out : (1, 1851) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 481 tokens) : 11.3379 - top-1 accuracy : 12/481 = 2.5% - -Dataloader batch - tokens shape: torch.Size([1, 1409]) -Dataloader batch - labels shape: torch.Size([1, 1409]) -Dataloader batch - attn_mask shape: torch.Size([1, 1409]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1580 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1409) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1409) torch.int64 - loss_mask : 338 / 1409 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1409, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1409, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1409, 1, 576) torch.bfloat16 - out : (1, 1409) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 338 tokens) : 11.2766 - top-1 accuracy : 14/338 = 4.1% - - [2026-04-29 14:08:49] iteration 495/ 500 | consumed samples: 1980 | elapsed time per iteration (ms): 5377.0 | throughput (token/sec/GPU): 1246.4 | learning rate: 1.142747E-05 | global batch size: 4 | lm loss: 1.129167E+01 | loss scale: 1.0 | grad norm: 0.172 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1525]) -Dataloader batch - labels shape: torch.Size([1, 1525]) -Dataloader batch - attn_mask shape: torch.Size([1, 1525]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1581 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1525) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1525) torch.int64 - loss_mask : 391 / 1525 tokens (25.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1525, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1525, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1525, 1, 576) torch.bfloat16 - out : (1, 1525) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 391 tokens) : 11.3064 - top-1 accuracy : 16/391 = 4.1% - -Dataloader batch - tokens shape: torch.Size([1, 1429]) -Dataloader batch - labels shape: torch.Size([1, 1429]) -Dataloader batch - attn_mask shape: torch.Size([1, 1429]) -Dataloader batch - image_grid_thw shape: torch.Size([25, 3]) - -==================================================================== - PIPELINE — STEP 1582 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1429) torch.int64 - images : (25, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1429) torch.int64 - loss_mask : 395 / 1429 tokens (27.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (25, 3, 1024, 1024) torch.bfloat16 - out : (25, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (25, 3072) - out : (25, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1429, 1, 576) torch.bfloat16 grad=False - image slots : 25 tokens replaced with vision embeddings - combined : (1429, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1429, 1, 576) torch.bfloat16 - out : (1, 1429) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 395 tokens) : 11.3260 - top-1 accuracy : 9/395 = 2.3% - -Dataloader batch - tokens shape: torch.Size([1, 1010]) -Dataloader batch - labels shape: torch.Size([1, 1010]) -Dataloader batch - attn_mask shape: torch.Size([1, 1010]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1583 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1010) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1010) torch.int64 - loss_mask : 242 / 1010 tokens (24.0%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1010, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1010, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1010, 1, 576) torch.bfloat16 - out : (1, 1010) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 242 tokens) : 11.2908 - top-1 accuracy : 2/242 = 0.8% - -Dataloader batch - tokens shape: torch.Size([1, 1460]) -Dataloader batch - labels shape: torch.Size([1, 1460]) -Dataloader batch - attn_mask shape: torch.Size([1, 1460]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1584 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1460) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1460) torch.int64 - loss_mask : 271 / 1460 tokens (18.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1460, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1460, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1460, 1, 576) torch.bfloat16 - out : (1, 1460) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 271 tokens) : 11.1549 - top-1 accuracy : 33/271 = 12.2% - - [2026-04-29 14:08:53] iteration 496/ 500 | consumed samples: 1984 | elapsed time per iteration (ms): 4434.6 | throughput (token/sec/GPU): 1223.1 | learning rate: 1.123692E-05 | global batch size: 4 | lm loss: 1.127784E+01 | loss scale: 1.0 | grad norm: 0.190 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1735]) -Dataloader batch - labels shape: torch.Size([1, 1735]) -Dataloader batch - attn_mask shape: torch.Size([1, 1735]) -Dataloader batch - image_grid_thw shape: torch.Size([31, 3]) - -==================================================================== - PIPELINE — STEP 1585 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1735) torch.int64 - images : (31, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1735) torch.int64 - loss_mask : 438 / 1735 tokens (25.2%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (31, 3, 1024, 1024) torch.bfloat16 - out : (31, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (31, 3072) - out : (31, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1735, 1, 576) torch.bfloat16 grad=False - image slots : 31 tokens replaced with vision embeddings - combined : (1735, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1735, 1, 576) torch.bfloat16 - out : (1, 1735) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 438 tokens) : 11.2687 - top-1 accuracy : 4/438 = 0.9% - -Dataloader batch - tokens shape: torch.Size([1, 1688]) -Dataloader batch - labels shape: torch.Size([1, 1688]) -Dataloader batch - attn_mask shape: torch.Size([1, 1688]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1586 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1688) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1688) torch.int64 - loss_mask : 365 / 1688 tokens (21.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1688, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1688, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1688, 1, 576) torch.bfloat16 - out : (1, 1688) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 365 tokens) : 11.2427 - top-1 accuracy : 13/365 = 3.6% - -Dataloader batch - tokens shape: torch.Size([1, 1408]) -Dataloader batch - labels shape: torch.Size([1, 1408]) -Dataloader batch - attn_mask shape: torch.Size([1, 1408]) -Dataloader batch - image_grid_thw shape: torch.Size([26, 3]) - -==================================================================== - PIPELINE — STEP 1587 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1408) torch.int64 - images : (26, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1408) torch.int64 - loss_mask : 340 / 1408 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (26, 3, 1024, 1024) torch.bfloat16 - out : (26, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (26, 3072) - out : (26, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1408, 1, 576) torch.bfloat16 grad=False - image slots : 26 tokens replaced with vision embeddings - combined : (1408, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1408, 1, 576) torch.bfloat16 - out : (1, 1408) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 340 tokens) : 11.2070 - top-1 accuracy : 10/340 = 2.9% - -Dataloader batch - tokens shape: torch.Size([1, 1469]) -Dataloader batch - labels shape: torch.Size([1, 1469]) -Dataloader batch - attn_mask shape: torch.Size([1, 1469]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1588 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1469) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1469) torch.int64 - loss_mask : 303 / 1469 tokens (20.6%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1469, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1469, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1469, 1, 576) torch.bfloat16 - out : (1, 1469) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 303 tokens) : 11.1586 - top-1 accuracy : 31/303 = 10.2% - - [2026-04-29 14:08:58] iteration 497/ 500 | consumed samples: 1988 | elapsed time per iteration (ms): 5180.5 | throughput (token/sec/GPU): 1216.1 | learning rate: 1.104791E-05 | global batch size: 4 | lm loss: 1.122456E+01 | loss scale: 1.0 | grad norm: 0.195 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1487]) -Dataloader batch - labels shape: torch.Size([1, 1487]) -Dataloader batch - attn_mask shape: torch.Size([1, 1487]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1589 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1487) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1487) torch.int64 - loss_mask : 323 / 1487 tokens (21.7%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1487, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1487, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1487, 1, 576) torch.bfloat16 - out : (1, 1487) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 323 tokens) : 11.2317 - top-1 accuracy : 15/323 = 4.6% - -Dataloader batch - tokens shape: torch.Size([1, 1050]) -Dataloader batch - labels shape: torch.Size([1, 1050]) -Dataloader batch - attn_mask shape: torch.Size([1, 1050]) -Dataloader batch - image_grid_thw shape: torch.Size([20, 3]) - -==================================================================== - PIPELINE — STEP 1590 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1050) torch.int64 - images : (20, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1050) torch.int64 - loss_mask : 229 / 1050 tokens (21.8%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (20, 3, 1024, 1024) torch.bfloat16 - out : (20, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (20, 3072) - out : (20, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1050, 1, 576) torch.bfloat16 grad=False - image slots : 20 tokens replaced with vision embeddings - combined : (1050, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1050, 1, 576) torch.bfloat16 - out : (1, 1050) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 229 tokens) : 11.2055 - top-1 accuracy : 22/229 = 9.6% - -Dataloader batch - tokens shape: torch.Size([1, 956]) -Dataloader batch - labels shape: torch.Size([1, 956]) -Dataloader batch - attn_mask shape: torch.Size([1, 956]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1591 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 956) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 956) torch.int64 - loss_mask : 209 / 956 tokens (21.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (956, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (956, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (956, 1, 576) torch.bfloat16 - out : (1, 956) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 209 tokens) : 11.1843 - top-1 accuracy : 12/209 = 5.7% - -Dataloader batch - tokens shape: torch.Size([1, 1011]) -Dataloader batch - labels shape: torch.Size([1, 1011]) -Dataloader batch - attn_mask shape: torch.Size([1, 1011]) -Dataloader batch - image_grid_thw shape: torch.Size([18, 3]) - -==================================================================== - PIPELINE — STEP 1592 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1011) torch.int64 - images : (18, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1011) torch.int64 - loss_mask : 244 / 1011 tokens (24.1%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (18, 3, 1024, 1024) torch.bfloat16 - out : (18, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (18, 3072) - out : (18, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1011, 1, 576) torch.bfloat16 grad=False - image slots : 18 tokens replaced with vision embeddings - combined : (1011, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1011, 1, 576) torch.bfloat16 - out : (1, 1011) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 244 tokens) : 11.2646 - top-1 accuracy : 1/244 = 0.4% - - [2026-04-29 14:09:02] iteration 498/ 500 | consumed samples: 1992 | elapsed time per iteration (ms): 3952.8 | throughput (token/sec/GPU): 1139.5 | learning rate: 1.086048E-05 | global batch size: 4 | lm loss: 1.122386E+01 | loss scale: 1.0 | grad norm: 0.221 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1546]) -Dataloader batch - labels shape: torch.Size([1, 1546]) -Dataloader batch - attn_mask shape: torch.Size([1, 1546]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1593 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1546) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1546) torch.int64 - loss_mask : 369 / 1546 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1546, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1546, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1546, 1, 576) torch.bfloat16 - out : (1, 1546) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2567 - top-1 accuracy : 22/369 = 6.0% - -Dataloader batch - tokens shape: torch.Size([1, 854]) -Dataloader batch - labels shape: torch.Size([1, 854]) -Dataloader batch - attn_mask shape: torch.Size([1, 854]) -Dataloader batch - image_grid_thw shape: torch.Size([17, 3]) - -==================================================================== - PIPELINE — STEP 1594 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 854) torch.int64 - images : (17, 3, 1024, 1024) torch.bfloat16 - labels : (1, 854) torch.int64 - loss_mask : 174 / 854 tokens (20.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (17, 3, 1024, 1024) torch.bfloat16 - out : (17, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (17, 3072) - out : (17, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (854, 1, 576) torch.bfloat16 grad=False - image slots : 17 tokens replaced with vision embeddings - combined : (854, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (854, 1, 576) torch.bfloat16 - out : (1, 854) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 174 tokens) : 11.1691 - top-1 accuracy : 7/174 = 4.0% - -Dataloader batch - tokens shape: torch.Size([1, 1505]) -Dataloader batch - labels shape: torch.Size([1, 1505]) -Dataloader batch - attn_mask shape: torch.Size([1, 1505]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1595 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1505) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1505) torch.int64 - loss_mask : 368 / 1505 tokens (24.5%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1505, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1505, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1505, 1, 576) torch.bfloat16 - out : (1, 1505) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 368 tokens) : 11.2868 - top-1 accuracy : 23/368 = 6.2% - -Dataloader batch - tokens shape: torch.Size([1, 1522]) -Dataloader batch - labels shape: torch.Size([1, 1522]) -Dataloader batch - attn_mask shape: torch.Size([1, 1522]) -Dataloader batch - image_grid_thw shape: torch.Size([27, 3]) - -==================================================================== - PIPELINE — STEP 1596 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1522) torch.int64 - images : (27, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1522) torch.int64 - loss_mask : 372 / 1522 tokens (24.4%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (27, 3, 1024, 1024) torch.bfloat16 - out : (27, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (27, 3072) - out : (27, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1522, 1, 576) torch.bfloat16 grad=False - image slots : 27 tokens replaced with vision embeddings - combined : (1522, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1522, 1, 576) torch.bfloat16 - out : (1, 1522) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 372 tokens) : 11.2793 - top-1 accuracy : 25/372 = 6.7% - - [2026-04-29 14:09:07] iteration 499/ 500 | consumed samples: 1996 | elapsed time per iteration (ms): 4621.9 | throughput (token/sec/GPU): 1174.2 | learning rate: 1.067461E-05 | global batch size: 4 | lm loss: 1.125999E+01 | loss scale: 1.0 | grad norm: 0.235 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -Dataloader batch - tokens shape: torch.Size([1, 1436]) -Dataloader batch - labels shape: torch.Size([1, 1436]) -Dataloader batch - attn_mask shape: torch.Size([1, 1436]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1597 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1436) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1436) torch.int64 - loss_mask : 291 / 1436 tokens (20.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1436, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1436, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1436, 1, 576) torch.bfloat16 - out : (1, 1436) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 291 tokens) : 11.1969 - top-1 accuracy : 10/291 = 3.4% - -Dataloader batch - tokens shape: torch.Size([1, 1917]) -Dataloader batch - labels shape: torch.Size([1, 1917]) -Dataloader batch - attn_mask shape: torch.Size([1, 1917]) -Dataloader batch - image_grid_thw shape: torch.Size([32, 3]) - -==================================================================== - PIPELINE — STEP 1598 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1917) torch.int64 - images : (32, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1917) torch.int64 - loss_mask : 543 / 1917 tokens (28.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (32, 3, 1024, 1024) torch.bfloat16 - out : (32, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (32, 3072) - out : (32, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1917, 1, 576) torch.bfloat16 grad=False - image slots : 32 tokens replaced with vision embeddings - combined : (1917, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1917, 1, 576) torch.bfloat16 - out : (1, 1917) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 543 tokens) : 11.3933 - top-1 accuracy : 4/543 = 0.7% - -Dataloader batch - tokens shape: torch.Size([1, 1582]) -Dataloader batch - labels shape: torch.Size([1, 1582]) -Dataloader batch - attn_mask shape: torch.Size([1, 1582]) -Dataloader batch - image_grid_thw shape: torch.Size([29, 3]) - -==================================================================== - PIPELINE — STEP 1599 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1582) torch.int64 - images : (29, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1582) torch.int64 - loss_mask : 369 / 1582 tokens (23.3%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (29, 3, 1024, 1024) torch.bfloat16 - out : (29, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (29, 3072) - out : (29, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1582, 1, 576) torch.bfloat16 grad=False - image slots : 29 tokens replaced with vision embeddings - combined : (1582, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1582, 1, 576) torch.bfloat16 - out : (1, 1582) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 369 tokens) : 11.2707 - top-1 accuracy : 26/369 = 7.0% - -Dataloader batch - tokens shape: torch.Size([1, 1555]) -Dataloader batch - labels shape: torch.Size([1, 1555]) -Dataloader batch - attn_mask shape: torch.Size([1, 1555]) -Dataloader batch - image_grid_thw shape: torch.Size([28, 3]) - -==================================================================== - PIPELINE — STEP 1600 (Stage 1 Alignment: adapter only) -==================================================================== - -[1/6] BATCH - input_ids : (1, 1555) torch.int64 - images : (28, 3, 1024, 1024) torch.bfloat16 - labels : (1, 1555) torch.int64 - loss_mask : 371 / 1555 tokens (23.9%) contribute to loss - -[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN] - in : (28, 3, 1024, 1024) torch.bfloat16 - out : (28, 3072) torch.bfloat16 grad=False - -[3/6] ADAPTER [2-layer MLP 3072→? | TRAINABLE] - in : (28, 3072) - out : (28, 576) grad=True - -[4/6] TOKEN FUSION - text embeddings : (1555, 1, 576) torch.bfloat16 grad=False - image slots : 28 tokens replaced with vision embeddings - combined : (1555, 1, 576) grad=True - -[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN] - in : (1555, 1, 576) torch.bfloat16 - out : (1, 1555) torch.float32 grad=True - -[6/6] LOSS / LOGITS - loss (mean over 371 tokens) : 11.2579 - top-1 accuracy : 3/371 = 0.8% - - [2026-04-29 14:09:12] iteration 500/ 500 | consumed samples: 2000 | elapsed time per iteration (ms): 5423.1 | throughput (token/sec/GPU): 1196.7 | learning rate: 1.049033E-05 | global batch size: 4 | lm loss: 1.129635E+01 | loss scale: 1.0 | grad norm: 0.169 | num zeros: 0 | number of skipped iterations: 0 | number of nan iterations: 0 | -(min, max) time across ranks (ms): - forward-backward ...............................: (5409.72, 5409.72) - forward-compute ................................: (3952.95, 3952.95) - backward-compute ...............................: (1454.42, 1454.42) - batch-generator ................................: (531.21, 531.21) - layernorm-grads-all-reduce .....................: (0.03, 0.03) - embedding-grads-all-reduce .....................: (0.04, 0.04) - all-grads-sync .................................: (0.55, 0.55) - params-all-gather ..............................: (0.17, 0.17) - optimizer-copy-to-main-grad ....................: (0.15, 0.15) - optimizer-clip-main-grad .......................: (0.64, 0.64) - optimizer-count-zeros ..........................: (0.02, 0.02) - optimizer-inner-step ...........................: (1.38, 1.38) - optimizer-copy-main-to-model-params ............: (0.26, 0.26) - optimizer ......................................: (3.39, 3.39) -saving checkpoint at iteration 500 to stage_1_alignment_mobilellm_140m in torch format -saving dataloader checkpoint at iteration 500 to stage_1_alignment_mobilellm_140m/dataloader - successfully saved checkpoint from iteration 500 to stage_1_alignment_mobilellm_140m [ t 1/1, p 1/1 ] -(min, max) time across ranks (ms): - save-checkpoint ................................: (3241.43, 3241.43) -[after training is done] datetime: 2026-04-29 14:09:15 -wandb: uploading artifact stage_1_alignment_mobilellm_140m; uploading output.log -wandb: uploading artifact stage_1_alignment_mobilellm_140m -wandb: uploading artifact stage_1_alignment_mobilellm_140m; uploading history steps 397-399, summary, console lines 61532-61700 -wandb: uploading history steps 397-399, summary, console lines 61532-61700 -wandb: uploading data -wandb: -wandb: -wandb: Run history: -wandb: Token throughput (per-sec-per-GPU) ▁▂▂▁▂▂▇██▇▇█▇▇▇▇▇▇▆▇▇▇▇▆▆▇█▅▇▇▇▇▇▆▆▅▇▇▇█ -wandb: batch-size ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: grad-norm ▇▇▅▇▆▆▆█▂▃▅▅▄▄▅▂▃▃▃▃▃▂▃▂▂▃▃▂▃▁▁▁▂▂▄▁▁▃▂▃ -wandb: iteration-time ▅▅▅█▇▆▃▃▂▃▃▃▂▃▃▃▄▂▁▁▂▂▂▃▂▄▃▄▄▃▃▃▂▂▂▂▃▃▃▃ -wandb: learning-rate ███████████▇▇▇▇▇▇▇▆▆▅▅▅▅▄▄▃▃▃▃▂▂▂▂▂▂▁▁▁▁ -wandb: lm loss ▅▄▄▃▃▃▂▆▆▆▅▅▄▁▆▇▅▃▇▄▅▅▇▅▅▄▅█▅▂▄▆▆▄▆▂▄█▅▂ -wandb: loss-scale ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: num-zeros ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁ -wandb: samples vs steps ▁▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▃▃▃▄▄▄▄▄▄▄▅▅▅▅▅▆▆▆▆▇▇▇██ -wandb: -wandb: Run summary: -wandb: Token throughput (per-sec-per-GPU) 1196.74249 -wandb: batch-size 4 -wandb: grad-norm 0.16943 -wandb: iteration-time 5.42305 -wandb: learning-rate 1e-05 -wandb: lm loss 11.29635 -wandb: loss-scale 1 -wandb: num-zeros 0 -wandb: samples vs steps 2000 -wandb: -wandb: 🚀 View run mobilellm_integration at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5/runs/3bn4iqdv -wandb: ⭐️ View project at: https://wandb.ai/rana-zayed-mbzuai/llava-ov-1_5 -wandb: Synced 5 W&B file(s), 0 media file(s), 16 artifact file(s) and 0 other file(s) -wandb: Find logs at: stage_1_alignment_mobilellm_140m/wandb/wandb/run-20260429_133502-3bn4iqdv/logs -[rank0]:[W429 14:10:46.038762046 ProcessGroupNCCL.cpp:1524] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) From 4016beee48e418a020adaf881c87d8b3a153354b Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 8 May 2026 01:19:13 +0400 Subject: [PATCH 30/39] Support local HF midtraining data --- docs/mobilellm_fastvit_amd_midtraining.md | 11 ++ ..._mobilellm_fastvit_english_branch_chain.sh | 28 +++- tools/check_hf_local_parquet_complete.py | 120 ++++++++++++++++++ ...epare_hf_caption_parquet_to_energon_wds.py | 48 +++++-- 4 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 tools/check_hf_local_parquet_complete.py diff --git a/docs/mobilellm_fastvit_amd_midtraining.md b/docs/mobilellm_fastvit_amd_midtraining.md index 4b485434..ccc1811a 100644 --- a/docs/mobilellm_fastvit_amd_midtraining.md +++ b/docs/mobilellm_fastvit_amd_midtraining.md @@ -80,6 +80,7 @@ then runs 1000 Stage 1.5 iterations. `MAX_SAMPLES=0` means no artificial cap. ImageNet EN: ```bash +LOCAL_HF_DATA_ROOT="$PWD/data/LLaVA-OneVision-Mid-Training-EN" \ MAX_SAMPLES=0 \ NSTEP=1000 \ GBS=4 \ @@ -90,6 +91,7 @@ bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_ Multiple English branches, chained one after another: ```bash +LOCAL_HF_DATA_ROOT="$PWD/data/LLaVA-OneVision-Mid-Training-EN" \ MAX_SAMPLES=0 \ NSTEP=1000 \ GBS=4 \ @@ -99,12 +101,21 @@ bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_ The second branch automatically loads from the checkpoint produced by the first. +To verify local ImageNet files before training: + +```bash +python tools/check_hf_local_parquet_complete.py \ + --local-data-root data/LLaVA-OneVision-Mid-Training-EN \ + --data-files 'imagenet/EN/*/*.parquet' +``` + ## 5. Useful Options ```bash START_CKPT=/path/to/checkpoint_dir DATA_ROOT=/large/disk/midtraining_full_en CACHE_DIR=/large/disk/hf_cache_midtraining_stream +LOCAL_HF_DATA_ROOT=/large/disk/LLaVA-OneVision-Mid-Training-EN KEEP_PREPARED_DATA=1 PREPARE_ONLY=1 MIDTRAIN_TRAINABLE_MODULES="language_model adapter vision_model" diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh index a917f778..540affdd 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh @@ -5,9 +5,9 @@ # bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet # bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet datacomp1b # -# Defaults prepare the full English branch from HF into local Energon/WebDataset -# shards, then run 1000 iterations. Set MAX_SAMPLES to a positive number only -# for smoke tests. +# Defaults prepare the full English branch into local Energon/WebDataset shards, +# then run 1000 iterations. Set LOCAL_HF_DATA_ROOT to read an already-downloaded +# HF-style dataset directory instead of streaming from Hugging Face. set -euo pipefail @@ -34,6 +34,8 @@ CHECKPOINT_PATH="$START_CKPT" DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_${LANG_LOWER}}" CACHE_DIR="${CACHE_DIR:-$REPO_ROOT/data/hf_cache_midtraining_stream}" mkdir -p "$DATA_ROOT" "$CACHE_DIR" +LOCAL_HF_DATA_ROOT="${LOCAL_HF_DATA_ROOT:-}" +VERIFY_LOCAL_HF_DATA="${VERIFY_LOCAL_HF_DATA:-1}" TP="${TP:-1}" PP="${PP:-1}" @@ -82,8 +84,7 @@ prepare_branch() { fi local data_glob="${DATA_FILES_GLOB:-$branch/$LANG_CODE/*/*.parquet}" - echo "[$branch] preparing HF files: $data_glob" - "$PYTHON_BIN" tools/prepare_hf_caption_parquet_to_energon_wds.py \ + local prepare_args=( --repo-id "$HF_REPO_ID" \ --data-files "$data_glob" \ --output-dir "$data_path" \ @@ -94,6 +95,23 @@ prepare_branch() { --cache-dir "$CACHE_DIR" \ --index-workers "$INDEX_WORKERS" \ --min-free-gb "$MIN_FREE_GB" + ) + + if [[ -n "$LOCAL_HF_DATA_ROOT" ]]; then + echo "[$branch] verifying local HF files: $LOCAL_HF_DATA_ROOT / $data_glob" + if [[ "$VERIFY_LOCAL_HF_DATA" == "1" ]]; then + "$PYTHON_BIN" tools/check_hf_local_parquet_complete.py \ + --repo-id "$HF_REPO_ID" \ + --data-files "$data_glob" \ + --local-data-root "$LOCAL_HF_DATA_ROOT" + fi + echo "[$branch] preparing local HF files: $LOCAL_HF_DATA_ROOT / $data_glob" + prepare_args+=(--local-data-root "$LOCAL_HF_DATA_ROOT") + else + echo "[$branch] preparing HF files: $data_glob" + fi + + "$PYTHON_BIN" tools/prepare_hf_caption_parquet_to_energon_wds.py "${prepare_args[@]}" } run_branch() { diff --git a/tools/check_hf_local_parquet_complete.py b/tools/check_hf_local_parquet_complete.py new file mode 100644 index 00000000..50fb00df --- /dev/null +++ b/tools/check_hf_local_parquet_complete.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Compare local HF parquet files with the Hugging Face dataset file listing.""" + +from __future__ import annotations + +import argparse +import fnmatch +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class FileInfo: + path: str + size: int + + +def _human_size(num_bytes: int) -> str: + value = float(num_bytes) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024.0 or unit == "TiB": + return f"{value:.2f} {unit}" + value /= 1024.0 + return f"{num_bytes} B" + + +def _remote_files(repo_id: str, data_files: list[str]) -> dict[str, FileInfo]: + from huggingface_hub import HfApi + + api = HfApi() + prefixes = sorted({pattern.split("*", 1)[0].rsplit("/", 1)[0] for pattern in data_files}) + files: dict[str, FileInfo] = {} + + for prefix in prefixes: + for entry in api.list_repo_tree( + repo_id=repo_id, + repo_type="dataset", + path_in_repo=prefix, + recursive=True, + ): + path = getattr(entry, "path", "") + size = getattr(entry, "size", None) + if not path.endswith(".parquet") or size is None: + continue + if any(fnmatch.fnmatch(path, pattern) for pattern in data_files): + files[path] = FileInfo(path=path, size=int(size)) + return files + + +def _local_files(local_data_root: Path, data_files: list[str]) -> dict[str, FileInfo]: + files: dict[str, FileInfo] = {} + for pattern in data_files: + for path in local_data_root.glob(pattern): + if path.is_file(): + rel = path.relative_to(local_data_root).as_posix() + files[rel] = FileInfo(path=rel, size=path.stat().st_size) + return files + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-id", default="mvp-lab/LLaVA-OneVision-1.5-Mid-Training-85M") + parser.add_argument( + "--data-files", + nargs="+", + default=["imagenet/EN/*/*.parquet"], + help="HF repo-relative parquet globs to verify.", + ) + parser.add_argument( + "--local-data-root", + type=Path, + required=True, + help="Local root mirroring the HF dataset repo layout.", + ) + parser.add_argument("--max-print", type=int, default=20) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + remote = _remote_files(args.repo_id, args.data_files) + local = _local_files(args.local_data_root, args.data_files) + + remote_size = sum(item.size for item in remote.values()) + local_size = sum(item.size for item in local.values()) + missing = sorted(set(remote) - set(local)) + extra = sorted(set(local) - set(remote)) + size_mismatch = sorted( + path for path in set(remote) & set(local) if remote[path].size != local[path].size + ) + + print(f"Repo : {args.repo_id}") + print(f"Local root : {args.local_data_root}") + print(f"Patterns : {' '.join(args.data_files)}") + print(f"Remote files : {len(remote)} ({_human_size(remote_size)})") + print(f"Local files : {len(local)} ({_human_size(local_size)})") + print(f"Missing : {len(missing)}") + print(f"Extra : {len(extra)}") + print(f"Size mismatch : {len(size_mismatch)}") + + for title, paths in ( + ("Missing files", missing), + ("Extra local files", extra), + ("Files with different sizes", size_mismatch), + ): + if paths: + print(f"\n{title}:") + for path in paths[: args.max_print]: + print(f" {path}") + if len(paths) > args.max_print: + print(f" ... {len(paths) - args.max_print} more") + + if missing or size_mismatch: + raise SystemExit(1) + + print("\nLocal parquet files match the Hugging Face listing.") + + +if __name__ == "__main__": + main() diff --git a/tools/prepare_hf_caption_parquet_to_energon_wds.py b/tools/prepare_hf_caption_parquet_to_energon_wds.py index 5aceb812..d2c9db8c 100644 --- a/tools/prepare_hf_caption_parquet_to_energon_wds.py +++ b/tools/prepare_hf_caption_parquet_to_energon_wds.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -"""Stream HF image-caption parquet data into the Energon WebDataset format. +"""Prepare HF image-caption parquet data into the Energon WebDataset format. This is intentionally small-slice friendly. Use it for one Mid-Training-85M folder at a time, e.g. ImageNet-EN part00, instead of cloning/downloading the -whole Hugging Face dataset. +whole Hugging Face dataset. It can either stream from Hugging Face or read an +already-downloaded local HF-style directory. """ from __future__ import annotations @@ -102,17 +103,38 @@ def _write_metadata(output_dir: Path, tar_names: list[str], workers: int) -> Non ) +def _resolve_local_files(local_data_root: Path, patterns: list[str]) -> list[str]: + files: list[Path] = [] + for pattern in patterns: + matches = sorted(p for p in local_data_root.glob(pattern) if p.is_file()) + if not matches: + raise RuntimeError(f"No local files matched {local_data_root / pattern}") + files.extend(matches) + return [str(p) for p in files] + + def _iter_hf_samples(args: argparse.Namespace) -> Iterable[dict[str, Any]]: from datasets import load_dataset - data_files = {"train": args.data_files} - dataset = load_dataset( - args.repo_id, - data_files=data_files, - split="train", - streaming=True, - cache_dir=args.cache_dir, - ) + if args.local_data_root is not None: + data_files = {"train": _resolve_local_files(args.local_data_root, args.data_files)} + dataset = load_dataset( + "parquet", + data_files=data_files, + split="train", + streaming=True, + cache_dir=args.cache_dir, + ) + else: + data_files = {"train": args.data_files} + dataset = load_dataset( + args.repo_id, + data_files=data_files, + split="train", + streaming=True, + cache_dir=args.cache_dir, + ) + if args.shuffle_buffer_size > 0: dataset = dataset.shuffle( seed=args.seed, @@ -184,6 +206,12 @@ def parse_args() -> argparse.Namespace: default=["imagenet/EN/part00/*.parquet"], help="Remote parquet files/globs inside the HF dataset repo.", ) + parser.add_argument( + "--local-data-root", + type=Path, + default=None, + help="Optional local root that mirrors the HF repo layout, e.g. data/LLaVA-OneVision-Mid-Training-EN.", + ) parser.add_argument( "--output-dir", type=Path, From c5c7675b582dcfe6710aebc3c67a104efd4a6ebb Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 8 May 2026 02:05:58 +0400 Subject: [PATCH 31/39] Add AMD sbatch launcher for ImageNet midtraining --- docs/mobilellm_fastvit_amd_midtraining.md | 14 +++ ...g_mobilellm_fastvit_imagenet_en_amd.sbatch | 107 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch diff --git a/docs/mobilellm_fastvit_amd_midtraining.md b/docs/mobilellm_fastvit_amd_midtraining.md index ccc1811a..0d30bc02 100644 --- a/docs/mobilellm_fastvit_amd_midtraining.md +++ b/docs/mobilellm_fastvit_amd_midtraining.md @@ -72,6 +72,20 @@ export WANDB_PROJECT="llava-ov-1_5" If the AMD machine has a different Conda env, replace `PYTHON_BIN` and `TORCHRUN` accordingly. +For the MBZUAI AMD cluster used here, the Stage 1.5 ImageNet job can also be +submitted with the ROCm Slurm launcher: + +```bash +cd ~/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5 + +# Optional: export WANDB_API_KEY before submitting if the env is not already logged in. +sbatch examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +``` + +This launcher mirrors the ROCm setup from `Stage1/alignment_rocm.sh`, uses +`conda activate mobile_vlm`, requests 8 AMD GPUs, and reads local ImageNet data +from `data/LLaVA-OneVision-Mid-Training-EN`. + ## 4. Run Full English Branch Midtraining This prepares the full HF English branch into local Energon/WebDataset shards, diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch new file mode 100644 index 00000000..e773f397 --- /dev/null +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -0,0 +1,107 @@ +#!/bin/bash + +#SBATCH --job-name=stage15_imagenet_amd +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=128 +#SBATCH --gres=gpu:8 +#SBATCH --qos=skqos +#SBATCH --partition=faculty +#SBATCH --output=stage15_imagenet_amd_%j.out +#SBATCH --error=stage15_imagenet_amd_%j.err + +set -euo pipefail + +# ---- ENV SETUP (AMD) ---- +source ~/.bashrc +conda activate "${CONDA_ENV:-mobile_vlm}" + +export MIOPEN_DISABLE_CACHE=1 +export PYTORCH_TUNABLEOP_ENABLED=0 + +export ROCM_HOME="${ROCM_HOME:-/opt/rocm}" +export PATH="${ROCM_HOME}/bin:${PATH}" +export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH:-}" + +export HIP_VISIBLE_DEVICES="${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export ROCR_VISIBLE_DEVICES="${ROCR_VISIBLE_DEVICES:-$HIP_VISIBLE_DEVICES}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-$HIP_VISIBLE_DEVICES}" + +# RCCL/NCCL runtime hints. +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export NCCL_COLLNET_ENABLE="${NCCL_COLLNET_ENABLE:-0}" +export NCCL_P2P_ENABLE="${NCCL_P2P_ENABLE:-1}" + +REPO_ROOT="${REPO_ROOT:-/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5}" +cd "$REPO_ROOT" || { echo "[Error] Repo root not found: $REPO_ROOT"; exit 1; } + +export AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" +export AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" +export PYTHONPATH="${AIAK_MAGATRON_PATH}:${AIAK_TRAINING_PATH}:${PYTHONPATH:-}" + +export PYTHON_BIN="${PYTHON_BIN:-$(command -v python)}" +export TORCHRUN="${TORCHRUN:-$(command -v torchrun)}" + +export START_CKPT="${START_CKPT:-$REPO_ROOT/stage_1_alignment_mobilellm_140m_fastvlm_faithful}" +export LOCAL_HF_DATA_ROOT="${LOCAL_HF_DATA_ROOT:-$REPO_ROOT/data/LLaVA-OneVision-Mid-Training-EN}" +export DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_en}" +export CACHE_DIR="${CACHE_DIR:-$REPO_ROOT/data/hf_cache_midtraining_stream}" + +export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +export TP="${TP:-1}" +export PP="${PP:-1}" +export SEQ_LEN="${SEQ_LEN:-4096}" +export MBS="${MBS:-1}" +export GBS="${GBS:-8}" +export NSTEP="${NSTEP:-1000}" +export MIN_FREE_GB="${MIN_FREE_GB:-100}" +export MAX_SAMPLES="${MAX_SAMPLES:-0}" + +# The local ImageNet files were already verified on the login node. Keep this +# disabled by default so compute nodes do not need Hugging Face API access. +export VERIFY_LOCAL_HF_DATA="${VERIFY_LOCAL_HF_DATA:-0}" + +export WANDB_ENABLE="${WANDB_ENABLE:-1}" +export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" +export WANDB_NAME_PREFIX="${WANDB_NAME_PREFIX:-stage1_5_mobilellm_fastvit_amd}" + +echo "=== ENV CHECK ===" +echo "REPO_ROOT=$REPO_ROOT" +echo "CONDA_DEFAULT_ENV=${CONDA_DEFAULT_ENV:-}" +echo "CONDA_PREFIX=${CONDA_PREFIX:-}" +echo "PYTHON_BIN=$PYTHON_BIN" +echo "TORCHRUN=$TORCHRUN" +echo "HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" +echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}" +echo "SLURM_NODELIST=${SLURM_NODELIST:-}" +"$PYTHON_BIN" - <<'PY' +import sys +print("sys.executable=", sys.executable) +import torch +print("torch=", torch.__version__) +print("hip=", torch.version.hip) +print("cuda_available=", torch.cuda.is_available()) +print("device_count=", torch.cuda.device_count()) +import megatron.energon +import datasets +import webdataset +import yaml +import tqdm +import PIL +print("midtraining deps ok") +PY +echo "=== END ENV CHECK ===" + +if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then + echo "[Error] Missing Stage 1 checkpoint tracker: $START_CKPT/latest_checkpointed_iteration.txt" + exit 1 +fi + +if [[ ! -d "$LOCAL_HF_DATA_ROOT/imagenet" ]]; then + echo "[Error] Missing local ImageNet data under: $LOCAL_HF_DATA_ROOT/imagenet" + exit 1 +fi + +df -h "$REPO_ROOT" "$LOCAL_HF_DATA_ROOT" || true + +bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet From 303ce7724324c1648bc083efae03c0378ce93746 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 8 May 2026 22:53:55 +0400 Subject: [PATCH 32/39] Fix AMD stage 1.5 sbatch environment --- ...mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch index e773f397..4d3cc817 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -32,7 +32,7 @@ export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" export NCCL_COLLNET_ENABLE="${NCCL_COLLNET_ENABLE:-0}" export NCCL_P2P_ENABLE="${NCCL_P2P_ENABLE:-1}" -REPO_ROOT="${REPO_ROOT:-/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5}" +export REPO_ROOT="${REPO_ROOT:-/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5}" cd "$REPO_ROOT" || { echo "[Error] Repo root not found: $REPO_ROOT"; exit 1; } export AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" @@ -47,7 +47,12 @@ export LOCAL_HF_DATA_ROOT="${LOCAL_HF_DATA_ROOT:-$REPO_ROOT/data/LLaVA-OneVision export DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_en}" export CACHE_DIR="${CACHE_DIR:-$REPO_ROOT/data/hf_cache_midtraining_stream}" -export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +if [[ -n "${HIP_VISIBLE_DEVICES:-}" ]]; then + DETECTED_GPUS="$(awk -F, '{print NF}' <<< "$HIP_VISIBLE_DEVICES")" +else + DETECTED_GPUS="${SLURM_GPUS_ON_NODE:-1}" +fi +export GPUS_PER_NODE="${GPUS_PER_NODE:-$DETECTED_GPUS}" export TP="${TP:-1}" export PP="${PP:-1}" export SEQ_LEN="${SEQ_LEN:-4096}" From c21d1695c0b0f491179aeddb86c2d01c6284fb55 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 8 May 2026 23:02:56 +0400 Subject: [PATCH 33/39] Quiet stage 1.5 midtraining logs --- .../models/llavaov_1_5/llavaov_1_5_model.py | 127 ++++++++++-------- ..._mobilellm_fastvit_english_branch_chain.sh | 4 +- ..._training_mobilellm_fastvit_imagenet_en.sh | 5 +- ...g_mobilellm_fastvit_imagenet_en_amd.sbatch | 3 + 4 files changed, 79 insertions(+), 60 deletions(-) diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py index 5cc46324..13ee59b6 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_model.py @@ -1,4 +1,5 @@ import logging +import os from collections import namedtuple from functools import partial from typing import List, Optional @@ -26,6 +27,15 @@ from aiak_training_llm.utils import print_rank_0 +def _verbose_model_debug() -> bool: + return os.getenv("VERBOSE_MODEL_DEBUG", "0").lower() in {"1", "true", "yes", "on"} + + +def _debug_rank_0(*args, **kwargs) -> None: + if _verbose_model_debug(): + print_rank_0(*args, **kwargs) + + def _rotate_half(x): x1, x2 = torch.chunk(x, 2, dim=-1) return torch.cat((-x2, x1), dim=-1) @@ -332,19 +342,19 @@ def forward( self._pipeline_step += 1 step = self._pipeline_step SEP = "=" * 68 - print_rank_0(f"\n{SEP}") - print_rank_0(f" PIPELINE — STEP {step} (Stage 1 Alignment: adapter only)") - print_rank_0(f"{SEP}\n") + _debug_rank_0(f"\n{SEP}") + _debug_rank_0(f" PIPELINE — STEP {step} (Stage 1 Alignment: adapter only)") + _debug_rank_0(f"{SEP}\n") # ── [1/6] BATCH ────────────────────────────────────────────────── - print_rank_0(f"[1/6] BATCH") - print_rank_0(f" input_ids : {tuple(input_ids.shape) if input_ids is not None else None} {input_ids.dtype if input_ids is not None else ''}") - print_rank_0(f" images : {tuple(images.shape) if images is not None else None} {images.dtype if images is not None else ''}") - if labels is not None: + _debug_rank_0(f"[1/6] BATCH") + _debug_rank_0(f" input_ids : {tuple(input_ids.shape) if input_ids is not None else None} {input_ids.dtype if input_ids is not None else ''}") + _debug_rank_0(f" images : {tuple(images.shape) if images is not None else None} {images.dtype if images is not None else ''}") + if labels is not None and _verbose_model_debug(): loss_positions = (labels != -100).sum().item() total_positions = labels.numel() - print_rank_0(f" labels : {tuple(labels.shape)} {labels.dtype}") - print_rank_0(f" loss_mask : {loss_positions} / {total_positions} tokens ({100*loss_positions/max(total_positions,1):.1f}%) contribute to loss") + _debug_rank_0(f" labels : {tuple(labels.shape)} {labels.dtype}") + _debug_rank_0(f" loss_mask : {loss_positions} / {total_positions} tokens ({100*loss_positions/max(total_positions,1):.1f}%) contribute to loss") use_inference_kv_cache = ( inference_params is not None and "image_tokens_count" in inference_params.key_value_memory_dict @@ -356,16 +366,16 @@ def forward( elif self.add_encoder: if images is not None: # ── [2/6] VISION ENCODER ──────────────────────────────────────── - print_rank_0(f"\n[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN]") - print_rank_0(f" in : {tuple(images.shape)} {images.dtype}") + _debug_rank_0(f"\n[2/6] VISION ENCODER [FastViT MobileCLIP-L | FROZEN]") + _debug_rank_0(f" in : {tuple(images.shape)} {images.dtype}") image_embeddings, window_index = self.vision_model(images, grid_thw=image_grid_thw) - print_rank_0(f" out : {tuple(image_embeddings.shape)} {image_embeddings.dtype} grad={image_embeddings.requires_grad}") + _debug_rank_0(f" out : {tuple(image_embeddings.shape)} {image_embeddings.dtype} grad={image_embeddings.requires_grad}") # ── [3/6] ADAPTER ─────────────────────────────────────────────── - print_rank_0(f"\n[3/6] ADAPTER [2-layer MLP {image_embeddings.shape[-1]}→? | TRAINABLE]") - print_rank_0(f" in : {tuple(image_embeddings.shape)}") + _debug_rank_0(f"\n[3/6] ADAPTER [2-layer MLP {image_embeddings.shape[-1]}→? | TRAINABLE]") + _debug_rank_0(f" in : {tuple(image_embeddings.shape)}") image_embeddings = self.adapter(image_embeddings, window_index) - print_rank_0(f" out : {tuple(image_embeddings.shape)} grad={image_embeddings.requires_grad}") + _debug_rank_0(f" out : {tuple(image_embeddings.shape)} grad={image_embeddings.requires_grad}") n_image_tokens = (input_ids == self.config.image_token_id).sum().item() n_image_features = image_embeddings.shape[0] @@ -434,11 +444,11 @@ def forward( if self.pre_process: # ── [4/6] TOKEN FUSION ────────────────────────────────────────── - print_rank_0(f"\n[4/6] TOKEN FUSION") + _debug_rank_0(f"\n[4/6] TOKEN FUSION") language_embeddings = self.language_model.embedding( input_ids=input_ids, position_ids=None ) # [text_seq_len, b, h_language] - print_rank_0(f" text embeddings : {tuple(language_embeddings.shape)} {language_embeddings.dtype} grad={language_embeddings.requires_grad}") + _debug_rank_0(f" text embeddings : {tuple(language_embeddings.shape)} {language_embeddings.dtype} grad={language_embeddings.requires_grad}") # If running inference, we can skip image token computation if they were computed already # earlier for this sample. @@ -457,8 +467,8 @@ def forward( image_embeddings = image_embeddings.to(language_embeddings.device, language_embeddings.dtype) combined_embeddings = combined_embeddings.masked_scatter(images_mask, image_embeddings) n_slots_filled = int(images_mask[..., 0].sum().item()) - print_rank_0(f" image slots : {n_slots_filled} tokens replaced with vision embeddings") - print_rank_0(f" combined : {tuple(combined_embeddings.shape)} grad={combined_embeddings.requires_grad}") + _debug_rank_0(f" image slots : {n_slots_filled} tokens replaced with vision embeddings") + _debug_rank_0(f" combined : {tuple(combined_embeddings.shape)} grad={combined_embeddings.requires_grad}") if pixel_values_videos is not None and (input_ids == self.config.video_token_id).any(): video_token_id = self.config.video_token_id @@ -481,9 +491,9 @@ def forward( # rotary_pos_emb = self.rotary_emb(position_ids).transpose(0, 2).contiguous() # ── [5/6] LANGUAGE MODEL ──────────────────────────────────────── - print_rank_0(f"\n[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN]") + _debug_rank_0(f"\n[5/6] LANGUAGE MODEL [MobileLLM-R1-140M | FROZEN]") if combined_embeddings is not None: - print_rank_0(f" in : {tuple(combined_embeddings.shape)} {combined_embeddings.dtype}") + _debug_rank_0(f" in : {tuple(combined_embeddings.shape)} {combined_embeddings.dtype}") output = self.language_model( input_ids=None, position_ids=None, @@ -498,11 +508,11 @@ def forward( extra_block_kwargs={}, ) out_shape = tuple(output.shape) if hasattr(output, 'shape') else None - print_rank_0(f" out : {out_shape} {getattr(output, 'dtype', None)} grad={getattr(output, 'requires_grad', None)}") + _debug_rank_0(f" out : {out_shape} {getattr(output, 'dtype', None)} grad={getattr(output, 'requires_grad', None)}") # ── [6/6] LOSS / LOGITS ───────────────────────────────────────── - print_rank_0(f"\n[6/6] LOSS / LOGITS") - if labels is not None: + _debug_rank_0(f"\n[6/6] LOSS / LOGITS") + if labels is not None and _verbose_model_debug(): # output is per-token loss with the same layout as labels [b, s] loss_mask = (labels != -100) if loss_mask.any(): @@ -513,41 +523,44 @@ def forward( loss_tensor = loss_tensor.transpose(0, 1).contiguous() valid_loss = loss_tensor[loss_mask] mean_loss = valid_loss.mean().item() - print_rank_0(f" loss (mean over {loss_mask.sum().item()} tokens) : {mean_loss:.4f}") - - # Top-1 token accuracy: no-grad logit pass - try: - with torch.no_grad(): - _emb = combined_embeddings.detach() if combined_embeddings is not None else None - logits = self.language_model( - input_ids=None, - position_ids=None, - attention_mask=attention_mask, - attn_mask_type=attn_mask_type, - decoder_input=_emb, - labels=None, - rotary_pos_emb=None, - inference_params=None, - packed_seq_params=packed_seq_params, - extra_block_kwargs={}, - ) # [s, b, vocab] or [b, s, vocab] - # Normalise to [b, s, vocab] - if logits.dim() == 3 and logits.shape[0] != labels.shape[0]: - logits = logits.transpose(0, 1).contiguous() # [s,b,v] → [b,s,v] - # Shift: predict token i+1 from position i - pred = logits[:, :-1, :].argmax(dim=-1) # [b, s-1] - tgt = labels[:, 1:] # [b, s-1] - mask = (tgt != -100) - if mask.any(): - correct = ((pred == tgt) & mask).sum().item() - total = mask.sum().item() - print_rank_0(f" top-1 accuracy : {correct}/{total} = {100*correct/total:.1f}%") - except Exception as acc_err: - print_rank_0(f" top-1 accuracy : (skipped — {acc_err})") + _debug_rank_0(f" loss (mean over {loss_mask.sum().item()} tokens) : {mean_loss:.4f}") + + # Optional debug-only top-1 token accuracy. This performs an + # extra language-model forward pass, so keep it out of normal + # training logs. + if _verbose_model_debug(): + try: + with torch.no_grad(): + _emb = combined_embeddings.detach() if combined_embeddings is not None else None + logits = self.language_model( + input_ids=None, + position_ids=None, + attention_mask=attention_mask, + attn_mask_type=attn_mask_type, + decoder_input=_emb, + labels=None, + rotary_pos_emb=None, + inference_params=None, + packed_seq_params=packed_seq_params, + extra_block_kwargs={}, + ) # [s, b, vocab] or [b, s, vocab] + # Normalise to [b, s, vocab] + if logits.dim() == 3 and logits.shape[0] != labels.shape[0]: + logits = logits.transpose(0, 1).contiguous() # [s,b,v] -> [b,s,v] + # Shift: predict token i+1 from position i + pred = logits[:, :-1, :].argmax(dim=-1) # [b, s-1] + tgt = labels[:, 1:] # [b, s-1] + mask = (tgt != -100) + if mask.any(): + correct = ((pred == tgt) & mask).sum().item() + total = mask.sum().item() + _debug_rank_0(f" top-1 accuracy : {correct}/{total} = {100*correct/total:.1f}%") + except Exception as acc_err: + _debug_rank_0(f" top-1 accuracy : (skipped - {acc_err})") else: - print_rank_0(f" returning logits : {out_shape}") + _debug_rank_0(f" returning logits : {out_shape}") - print_rank_0("") + _debug_rank_0("") return output diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh index 540affdd..0e1efe38 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh @@ -59,7 +59,9 @@ PREPARE_ONLY="${PREPARE_ONLY:-0}" export MIDTRAIN_TRAINABLE_MODULES="${MIDTRAIN_TRAINABLE_MODULES:-language_model adapter vision_model}" export NO_SAVE_OPTIM_RNG="${NO_SAVE_OPTIM_RNG:-1}" -export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-1}" +export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-0}" +export LOG_INTERVAL="${LOG_INTERVAL:-10}" +export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" export WANDB_ENABLE="${WANDB_ENABLE:-1}" export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh index 0ce5dc1f..bd12019c 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh @@ -31,6 +31,7 @@ MASTER_PORT="${MASTER_PORT:-26015}" SAVE_INTERVAL="${SAVE_INTERVAL:-250}" NUM_WORKERS="${NUM_WORKERS:-8}" MIDTRAIN_TRAINABLE_MODULES="${MIDTRAIN_TRAINABLE_MODULES:-language_model adapter vision_model}" +LOG_INTERVAL="${LOG_INTERVAL:-10}" if [ ! -f "$DATA_PATH/.nv-meta/dataset.yaml" ]; then echo "Missing prepared midtraining dataset: $DATA_PATH" @@ -169,7 +170,7 @@ MODEL_PARALLEL_ARGS=( ) LOGGING_ARGS=( - --log-interval 1 + --log-interval "$LOG_INTERVAL" --tensorboard-dir "$TENSORBOARD_PATH" --log-timers-to-tensorboard ) @@ -195,7 +196,7 @@ echo "Data: $DATA_PATH" echo "Load Stage 1: $CHECKPOINT_PATH" echo "Save Stage 1.5: $SAVE_CKPT_PATH" echo "Trainable modules: ${TRAINABLE_MODULES_ARRAY[*]}" -echo "GPUs: $GPUS_PER_NODE | SEQ_LEN: $SEQ_LEN | MBS: $MBS | GBS: $GBS | ITERS: $NSTEP" +echo "GPUs: $GPUS_PER_NODE | SEQ_LEN: $SEQ_LEN | MBS: $MBS | GBS: $GBS | ITERS: $NSTEP | LOG_INTERVAL: $LOG_INTERVAL" echo "=========================================" if [[ "${PRINT_DATA_SAMPLE:-1}" == "1" ]]; then diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch index 4d3cc817..137fc01a 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -61,6 +61,9 @@ export GBS="${GBS:-8}" export NSTEP="${NSTEP:-1000}" export MIN_FREE_GB="${MIN_FREE_GB:-100}" export MAX_SAMPLES="${MAX_SAMPLES:-0}" +export LOG_INTERVAL="${LOG_INTERVAL:-10}" +export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-0}" +export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" # The local ImageNet files were already verified on the login node. Keep this # disabled by default so compute nodes do not need Hugging Face API access. From d554811f0f4e175d78dd8a0e6eac89f147541b3d Mon Sep 17 00:00:00 2001 From: RanaZay Date: Fri, 8 May 2026 23:08:37 +0400 Subject: [PATCH 34/39] Finalize AMD ImageNet midtraining launch --- .../llavaov_1_5/llavaov_1_5_provider.py | 94 ++++++++++--------- ..._mobilellm_fastvit_english_branch_chain.sh | 50 +++++++++- ..._training_mobilellm_fastvit_imagenet_en.sh | 20 +++- ...g_mobilellm_fastvit_imagenet_en_amd.sbatch | 25 ++++- 4 files changed, 139 insertions(+), 50 deletions(-) diff --git a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py index 991b3f94..db4a900e 100644 --- a/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py +++ b/aiak_training_llm/models/llavaov_1_5/llavaov_1_5_provider.py @@ -1,5 +1,6 @@ """LLaVA-OneVision-1.5 model provider - supports Qwen and MobileLLM backbones""" +import os from copy import deepcopy from dataclasses import asdict @@ -18,6 +19,16 @@ from .llavaov_1_5_model import LlavaOnevision1_5 + +def _verbose_model_debug() -> bool: + return os.getenv("VERBOSE_MODEL_DEBUG", "0").lower() in {"1", "true", "yes", "on"} + + +def _debug_rank_0(*args, **kwargs) -> None: + if _verbose_model_debug(): + print_rank_0(*args, **kwargs) + + # model provider registration @register_model_provider(model_family=[VisionLanguageModelFamilies.LLAVA_OV_1_5]) def rice_vl_model_provider( @@ -41,54 +52,53 @@ def rice_vl_model_provider( args = get_args() print_rank_0(f'building {args.model_name} model ...') - print_rank_0(f'[DEBUG PROVIDER] Model name: {args.model_name}') - print_rank_0(f'[DEBUG PROVIDER] Args num_layers: {args.num_layers}') - print_rank_0(f'[DEBUG PROVIDER] Args hidden_size: {args.hidden_size}') - print_rank_0(f'[DEBUG PROVIDER] Args vocab_size: {args.vocab_size_in_config_file}') + _debug_rank_0(f'[DEBUG PROVIDER] Model name: {args.model_name}') + _debug_rank_0(f'[DEBUG PROVIDER] Args num_layers: {args.num_layers}') + _debug_rank_0(f'[DEBUG PROVIDER] Args hidden_size: {args.hidden_size}') + _debug_rank_0(f'[DEBUG PROVIDER] Args vocab_size: {args.vocab_size_in_config_file}') config = build_transformer_config(args) #base transformer config with all hyperparams - print_rank_0(f'[DEBUG PROVIDER] TransformerConfig built with num_layers: {config.num_layers}, hidden_size: {config.hidden_size}') + _debug_rank_0(f'[DEBUG PROVIDER] TransformerConfig built with num_layers: {config.num_layers}, hidden_size: {config.hidden_size}') language_config = deepcopy(config) # For Qwen2.5 language model vision_config = deepcopy(config) # For vision encoder (SigLIP) adapter_config = deepcopy(config) ## For adapter (projection) - print_rank_0(f'[DEBUG PROVIDER] Initial language_config: layers={language_config.num_layers}, hidden={language_config.hidden_size}') + _debug_rank_0(f'[DEBUG PROVIDER] Initial language_config: layers={language_config.num_layers}, hidden={language_config.hidden_size}') # Vision Encoder → Adapter → Language Model # (SigLIP) (Projection) (Qwen2.5) from aiak_training_llm.models import get_model_family model_family = get_model_family(args.model_name) - print_rank_0(f'[DEBUG PROVIDER] Model family: {model_family}') + _debug_rank_0(f'[DEBUG PROVIDER] Model family: {model_family}') # Detect if using MobileLLM backbone use_mobilellm = "mobilellm" in args.model_name.lower() - print_rank_0(f'[DEBUG PROVIDER] use_mobilellm flag: {use_mobilellm}') + _debug_rank_0(f'[DEBUG PROVIDER] use_mobilellm flag: {use_mobilellm}') if use_mobilellm: - print_rank_0(f'[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone') - print_rank_0(f'[DEBUG PROVIDER] Language config BEFORE override: layers={language_config.num_layers}, ' + _debug_rank_0(f'[DEBUG PROVIDER] ✓ Using MobileLLM-R1-140M as language backbone') + _debug_rank_0(f'[DEBUG PROVIDER] Language config BEFORE override: layers={language_config.num_layers}, ' f'hidden={language_config.hidden_size}, heads={language_config.num_attention_heads}') # MobileLLM params are already in language_config from args! # No need to load separately - they were applied in _validate_extra_model_args - print_rank_0(f'[DEBUG PROVIDER] Language config AFTER (should be same): layers={language_config.num_layers}, ' + _debug_rank_0(f'[DEBUG PROVIDER] Language config AFTER (should be same): layers={language_config.num_layers}, ' f'hidden={language_config.hidden_size}, heads={language_config.num_attention_heads}, ' f'query_groups={language_config.num_query_groups}') else: - print_rank_0(f'[DEBUG PROVIDER] Using Qwen2.5 as language backbone') + _debug_rank_0(f'[DEBUG PROVIDER] Using Qwen2.5 as language backbone') - print_rank_0(f'[DEBUG PROVIDER] ========== LANGUAGE CONFIG ==========') - print_rank_0(f'{language_config}') - print_rank_0(f'[DEBUG PROVIDER] ======================================') + _debug_rank_0(f'[DEBUG PROVIDER] ========== LANGUAGE CONFIG ==========') + _debug_rank_0(f'{language_config}') + _debug_rank_0(f'[DEBUG PROVIDER] ======================================') # get vision specific config : no. of layers, hidden size, Patch size, Image resolution - print_rank_0(f'[DEBUG PROVIDER] Loading vision config...') + _debug_rank_0(f'[DEBUG PROVIDER] Loading vision config...') # Check if using FastViT - it has its own config system if getattr(args, 'use_fastvit', False): # FastViT loads config from mobileclip_l.json - set TransformerConfig to match import json - import os vision_tower_name = getattr(args, 'vision_tower_name', 'mobileclip_l_1024') setattr(vision_config, 'vision_tower_name', vision_tower_name) @@ -119,29 +129,29 @@ def rice_vl_model_provider( ) setattr(vision_config, 'unfreeze_mm_vision_tower', train_vision_model) - print_rank_0(f'[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: {vision_tower_name}') - print_rank_0(f'[DEBUG PROVIDER] FastViT train vision tower: {train_vision_model}') - print_rank_0(f'[DEBUG PROVIDER] FastViT config loaded from {json_config_path}') - print_rank_0(f'[DEBUG PROVIDER] image_cfg: embed_dim={image_cfg["embed_dim"]}, patch_size={image_cfg["patch_size"]}, model={image_cfg["model_name"]}') - print_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ==========') - print_rank_0(f'{vision_config}') - print_rank_0(f'[DEBUG PROVIDER] ================================================') + _debug_rank_0(f'[DEBUG PROVIDER] ✓ Using FastViT with vision_tower_name: {vision_tower_name}') + _debug_rank_0(f'[DEBUG PROVIDER] FastViT train vision tower: {train_vision_model}') + _debug_rank_0(f'[DEBUG PROVIDER] FastViT config loaded from {json_config_path}') + _debug_rank_0(f'[DEBUG PROVIDER] image_cfg: embed_dim={image_cfg["embed_dim"]}, patch_size={image_cfg["patch_size"]}, model={image_cfg["model_name"]}') + _debug_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (FastViT) ==========') + _debug_rank_0(f'{vision_config}') + _debug_rank_0(f'[DEBUG PROVIDER] ================================================') else: # For SigLIP/Rice models, use generic vision config for k, v in asdict(get_vision_config(model_family, args.model_name)).items(): setattr(vision_config, k, v) - print_rank_0(f'[DEBUG PROVIDER] Vision config (SigLIP/Rice): layers={vision_config.num_layers}, hidden={vision_config.hidden_size}') - print_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (SigLIP/Rice) ==========') - print_rank_0(f'{vision_config}') - print_rank_0(f'[DEBUG PROVIDER] ====================================================') + _debug_rank_0(f'[DEBUG PROVIDER] Vision config (SigLIP/Rice): layers={vision_config.num_layers}, hidden={vision_config.hidden_size}') + _debug_rank_0(f'[DEBUG PROVIDER] ========== VISION CONFIG (SigLIP/Rice) ==========') + _debug_rank_0(f'{vision_config}') + _debug_rank_0(f'[DEBUG PROVIDER] ====================================================') # get adapter specific config : Projection dimension, Activation function - print_rank_0(f'[DEBUG PROVIDER] Loading adapter config...') + _debug_rank_0(f'[DEBUG PROVIDER] Loading adapter config...') for k, v in asdict(get_adapeter_config(model_family)).items(): setattr(adapter_config, k, v) - print_rank_0(f'[DEBUG PROVIDER] ========== ADAPTER CONFIG ==========') - print_rank_0(f'{adapter_config}') - print_rank_0(f'[DEBUG PROVIDER] ======================================') + _debug_rank_0(f'[DEBUG PROVIDER] ========== ADAPTER CONFIG ==========') + _debug_rank_0(f'{adapter_config}') + _debug_rank_0(f'[DEBUG PROVIDER] ======================================') # set special token ids for language model using the shared runtime tokenizer image_token_id = 151655 @@ -164,7 +174,7 @@ def rice_vl_model_provider( if video_token_id is None: video_token_id = 151656 - print_rank_0( + _debug_rank_0( f"[DEBUG PROVIDER] Resolved vision token ids from tokenizer: " f"image_token_id={image_token_id}, video_token_id={video_token_id}" ) @@ -212,21 +222,21 @@ def rice_vl_model_provider( if args.spec is not None: language_layer_spec = import_module(args.spec) - print_rank_0(f'[DEBUG PROVIDER] Using custom spec: {args.spec}') + _debug_rank_0(f'[DEBUG PROVIDER] Using custom spec: {args.spec}') else: - print_rank_0(f'[DEBUG PROVIDER] Building layer specs...') + _debug_rank_0(f'[DEBUG PROVIDER] Building layer specs...') adapter_layer_spec = get_adapeter_layer_with_spec() vision_layer_spec = get_vision_layer_with_spec() # Choose language layer spec based on backbone if use_mobilellm: - print_rank_0(f'[DEBUG PROVIDER] ✓ Using MobileLLM layer specification') - print_rank_0(f'[DEBUG PROVIDER] MobileLLM layer config: layers={language_config.num_layers}, ' + _debug_rank_0(f'[DEBUG PROVIDER] ✓ Using MobileLLM layer specification') + _debug_rank_0(f'[DEBUG PROVIDER] MobileLLM layer config: layers={language_config.num_layers}, ' f'hidden={language_config.hidden_size}, heads={language_config.num_attention_heads}') language_layer_spec = get_mobilellm_layer_with_te_spec(language_config) - print_rank_0(f'[DEBUG PROVIDER] MobileLLM layer spec created successfully') + _debug_rank_0(f'[DEBUG PROVIDER] MobileLLM layer spec created successfully') else: - print_rank_0(f'[DEBUG PROVIDER] Using Qwen layer specification') + _debug_rank_0(f'[DEBUG PROVIDER] Using Qwen layer specification') language_layer_spec = get_qwen_layer_with_te_spec(language_config) # # Vision layer spec (Transformer block) @@ -246,8 +256,8 @@ def rice_vl_model_provider( # - Optional activation #create the model - print_rank_0(f'[DEBUG PROVIDER] Creating LlavaOnevision1_5 model...') - print_rank_0(f'[DEBUG PROVIDER] Final language_config: layers={language_config.num_layers}, ' + _debug_rank_0(f'[DEBUG PROVIDER] Creating LlavaOnevision1_5 model...') + _debug_rank_0(f'[DEBUG PROVIDER] Final language_config: layers={language_config.num_layers}, ' f'hidden={language_config.hidden_size}, vocab={args.padded_vocab_size}, ' f'rotary_base={args.rotary_base}') model = LlavaOnevision1_5( @@ -273,7 +283,7 @@ def rice_vl_model_provider( # When using FastViT, adapter dimensions change, so allow missing adapter weights allow_missing_adapter_checkpoint=getattr(args, 'use_fastvit', False), ) - print_rank_0(f'[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully!') + _debug_rank_0(f'[DEBUG PROVIDER] ✓ LlavaOnevision1_5 model created successfully!') # Vision encoder: SigLIP with 27 layers # Adapter: Projection network # Language model: Qwen2.5 with 32 layers diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh index 0e1efe38..f5a5f940 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh @@ -29,6 +29,44 @@ TORCHRUN="${TORCHRUN:-/home/ashaker/miniconda3/envs/llava-ov-4b-clean/bin/torchr export PYTHON_BIN TORCHRUN START_CKPT="${START_CKPT:-$REPO_ROOT/stage_1_alignment_mobilellm_140m_fastvlm_faithful}" +HF_STAGE1_REPO_ID="${HF_STAGE1_REPO_ID:-ranazayed19/mobilellm-fastvit-stage1-2500}" +HF_STAGE1_LOCAL_DIR="${HF_STAGE1_LOCAL_DIR:-$START_CKPT}" +export HF_STAGE1_REPO_ID HF_STAGE1_LOCAL_DIR + +ensure_start_checkpoint() { + if [[ -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then + echo "[ckpt] Stage 1 alignment checkpoint: $START_CKPT" + return + fi + + if [[ -z "$HF_STAGE1_REPO_ID" ]]; then + echo "[Error] Missing START_CKPT/latest_checkpointed_iteration.txt: $START_CKPT" + echo "[Error] Set START_CKPT to a local checkpoint or HF_STAGE1_REPO_ID to a Hugging Face model repo." + exit 1 + fi + + echo "[ckpt] Local Stage 1 checkpoint missing: $START_CKPT" + echo "[ckpt] Downloading Stage 1 checkpoint from Hugging Face: $HF_STAGE1_REPO_ID" + echo "[ckpt] Destination: $HF_STAGE1_LOCAL_DIR" + "$PYTHON_BIN" - <<'PY' +import os +from huggingface_hub import snapshot_download + +snapshot_download( + repo_id=os.environ["HF_STAGE1_REPO_ID"], + repo_type="model", + local_dir=os.environ["HF_STAGE1_LOCAL_DIR"], +) +PY + START_CKPT="$HF_STAGE1_LOCAL_DIR" + + if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then + echo "[Error] Hugging Face checkpoint download finished, but tracker is still missing: $START_CKPT/latest_checkpointed_iteration.txt" + exit 1 + fi +} + +ensure_start_checkpoint CHECKPOINT_PATH="$START_CKPT" DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_${LANG_LOWER}}" @@ -64,13 +102,10 @@ export LOG_INTERVAL="${LOG_INTERVAL:-10}" export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" export WANDB_ENABLE="${WANDB_ENABLE:-1}" +export WANDB_MODE="${WANDB_MODE:-online}" +export WANDB_ENTITY="${WANDB_ENTITY:-rana-zayed-mbzuai}" export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" -if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then - echo "Missing START_CKPT/latest_checkpointed_iteration.txt: $START_CKPT" - exit 1 -fi - sanitize_name() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9_]/_/g' } @@ -132,6 +167,8 @@ run_branch() { echo "Save checkpoint : $save_path" echo "Steps : $NSTEP" echo "Global batch : $GBS" + echo "Log interval : $LOG_INTERVAL" + echo "W&B : ${WANDB_MODE:-online}/${WANDB_PROJECT}/${WANDB_NAME_PREFIX:-stage1_5_mobilellm_fastvit}_${branch_safe}_${LANG_LOWER}_full_${NSTEP}steps_${GPUS_PER_NODE}gpu" echo "Max samples prep: $MAX_SAMPLES (0 means full branch)" echo "====================================================================" @@ -158,6 +195,7 @@ run_branch() { fi CHECKPOINT_PATH="$save_path" + echo "[$branch] completed. Latest checkpoint root for next branch: $CHECKPOINT_PATH" if [[ "$KEEP_PREPARED_DATA" == "0" ]]; then echo "[$branch] removing prepared data: $data_path" @@ -172,3 +210,5 @@ for branch in "$@"; do done echo "All requested English branches completed." +echo "Final checkpoint root: $CHECKPOINT_PATH" +echo "$CHECKPOINT_PATH" > "$REPO_ROOT/stage_1_5_latest_checkpoint_path.txt" diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh index bd12019c..331c15f4 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en.sh @@ -32,6 +32,7 @@ SAVE_INTERVAL="${SAVE_INTERVAL:-250}" NUM_WORKERS="${NUM_WORKERS:-8}" MIDTRAIN_TRAINABLE_MODULES="${MIDTRAIN_TRAINABLE_MODULES:-language_model adapter vision_model}" LOG_INTERVAL="${LOG_INTERVAL:-10}" +export WANDB_MODE="${WANDB_MODE:-online}" if [ ! -f "$DATA_PATH/.nv-meta/dataset.yaml" ]; then echo "Missing prepared midtraining dataset: $DATA_PATH" @@ -196,7 +197,13 @@ echo "Data: $DATA_PATH" echo "Load Stage 1: $CHECKPOINT_PATH" echo "Save Stage 1.5: $SAVE_CKPT_PATH" echo "Trainable modules: ${TRAINABLE_MODULES_ARRAY[*]}" -echo "GPUs: $GPUS_PER_NODE | SEQ_LEN: $SEQ_LEN | MBS: $MBS | GBS: $GBS | ITERS: $NSTEP | LOG_INTERVAL: $LOG_INTERVAL" +echo "GPUs: $GPUS_PER_NODE | SEQ_LEN: $SEQ_LEN | MBS: $MBS | GBS: $GBS | ITERS: $NSTEP" +echo "Log interval: $LOG_INTERVAL | Save interval: $SAVE_INTERVAL" +if [[ "${WANDB_ENABLE:-0}" == "1" || -n "${WANDB_API_KEY:-}" ]]; then + echo "W&B: mode=$WANDB_MODE entity=${WANDB_ENTITY:-default} project=${WANDB_PROJECT:-llava-ov-mobilellm} name=${WANDB_NAME:-stage1_5-mobilellm-fastvit-imagenet-en}" +else + echo "W&B: disabled" +fi echo "=========================================" if [[ "${PRINT_DATA_SAMPLE:-1}" == "1" ]]; then @@ -214,3 +221,14 @@ PYTHONPATH="$AIAK_MAGATRON_PATH:$AIAK_TRAINING_PATH:${PYTHONPATH:-}" \ "${MODEL_PARALLEL_ARGS[@]}" \ "${LOGGING_ARGS[@]}" \ 2>&1 | tee "$logfile" + +{ + echo "=========================================" + echo "Stage 1.5 training completed" + echo "Checkpoint root: $SAVE_CKPT_PATH" + if [[ -f "$SAVE_CKPT_PATH/latest_checkpointed_iteration.txt" ]]; then + echo "Latest checkpoint iteration: $(cat "$SAVE_CKPT_PATH/latest_checkpointed_iteration.txt")" + fi + echo "Training log: $logfile" + echo "=========================================" +} | tee -a "$logfile" diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch index 137fc01a..346a62a2 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -43,6 +43,8 @@ export PYTHON_BIN="${PYTHON_BIN:-$(command -v python)}" export TORCHRUN="${TORCHRUN:-$(command -v torchrun)}" export START_CKPT="${START_CKPT:-$REPO_ROOT/stage_1_alignment_mobilellm_140m_fastvlm_faithful}" +export HF_STAGE1_REPO_ID="${HF_STAGE1_REPO_ID:-ranazayed19/mobilellm-fastvit-stage1-2500}" +export HF_STAGE1_LOCAL_DIR="${HF_STAGE1_LOCAL_DIR:-$START_CKPT}" export LOCAL_HF_DATA_ROOT="${LOCAL_HF_DATA_ROOT:-$REPO_ROOT/data/LLaVA-OneVision-Mid-Training-EN}" export DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_en}" export CACHE_DIR="${CACHE_DIR:-$REPO_ROOT/data/hf_cache_midtraining_stream}" @@ -70,9 +72,16 @@ export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" export VERIFY_LOCAL_HF_DATA="${VERIFY_LOCAL_HF_DATA:-0}" export WANDB_ENABLE="${WANDB_ENABLE:-1}" +export WANDB_MODE="${WANDB_MODE:-online}" +export WANDB_ENTITY="${WANDB_ENTITY:-rana-zayed-mbzuai}" export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" export WANDB_NAME_PREFIX="${WANDB_NAME_PREFIX:-stage1_5_mobilellm_fastvit_amd}" +if [[ "$WANDB_ENABLE" == "1" && -z "${WANDB_API_KEY:-}" && ! -f "$HOME/.netrc" ]]; then + echo "[WARN] W&B is enabled, but WANDB_API_KEY is not set and ~/.netrc was not found." + echo "[WARN] Run 'wandb login' or submit with WANDB_API_KEY set if online W&B logging fails." +fi + echo "=== ENV CHECK ===" echo "REPO_ROOT=$REPO_ROOT" echo "CONDA_DEFAULT_ENV=${CONDA_DEFAULT_ENV:-}" @@ -82,6 +91,19 @@ echo "TORCHRUN=$TORCHRUN" echo "HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}" echo "SLURM_NODELIST=${SLURM_NODELIST:-}" +echo "START_CKPT=$START_CKPT" +echo "HF_STAGE1_REPO_ID=$HF_STAGE1_REPO_ID" +echo "LOCAL_HF_DATA_ROOT=$LOCAL_HF_DATA_ROOT" +echo "WANDB_ENABLE=$WANDB_ENABLE" +echo "WANDB_MODE=$WANDB_MODE" +echo "WANDB_ENTITY=$WANDB_ENTITY" +echo "WANDB_PROJECT=$WANDB_PROJECT" +echo "WANDB_NAME_PREFIX=$WANDB_NAME_PREFIX" +if [[ -n "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_API_KEY=set" +else + echo "WANDB_API_KEY=not_set" +fi "$PYTHON_BIN" - <<'PY' import sys print("sys.executable=", sys.executable) @@ -101,8 +123,7 @@ PY echo "=== END ENV CHECK ===" if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then - echo "[Error] Missing Stage 1 checkpoint tracker: $START_CKPT/latest_checkpointed_iteration.txt" - exit 1 + echo "[ckpt] Stage 1 checkpoint not found locally yet. The branch runner will download it from Hugging Face if needed." fi if [[ ! -d "$LOCAL_HF_DATA_ROOT/imagenet" ]]; then From 6bfccb221cce6a53e1da1939ffee5cf1ea3b17c0 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sat, 9 May 2026 00:17:27 +0400 Subject: [PATCH 35/39] Cap AMD torchrun GPUs to visible devices --- ...g_mobilellm_fastvit_imagenet_en_amd.sbatch | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch index 346a62a2..4e71db96 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -77,6 +77,24 @@ export WANDB_ENTITY="${WANDB_ENTITY:-rana-zayed-mbzuai}" export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" export WANDB_NAME_PREFIX="${WANDB_NAME_PREFIX:-stage1_5_mobilellm_fastvit_amd}" +REQUESTED_GPUS_PER_NODE="$GPUS_PER_NODE" +ACTUAL_VISIBLE_GPUS="$("$PYTHON_BIN" - <<'PY' +import torch +print(torch.cuda.device_count()) +PY +)" + +if ! [[ "$ACTUAL_VISIBLE_GPUS" =~ ^[0-9]+$ ]] || [[ "$ACTUAL_VISIBLE_GPUS" -lt 1 ]]; then + echo "[Error] PyTorch/ROCm sees no usable GPUs. ACTUAL_VISIBLE_GPUS=$ACTUAL_VISIBLE_GPUS" + exit 1 +fi + +if [[ "$ACTUAL_VISIBLE_GPUS" -lt "$GPUS_PER_NODE" ]]; then + echo "[WARN] Requested GPUS_PER_NODE=$GPUS_PER_NODE, but PyTorch/ROCm sees only $ACTUAL_VISIBLE_GPUS GPU(s)." + echo "[WARN] Capping GPUS_PER_NODE to $ACTUAL_VISIBLE_GPUS to avoid HIP invalid device ordinal." + export GPUS_PER_NODE="$ACTUAL_VISIBLE_GPUS" +fi + if [[ "$WANDB_ENABLE" == "1" && -z "${WANDB_API_KEY:-}" && ! -f "$HOME/.netrc" ]]; then echo "[WARN] W&B is enabled, but WANDB_API_KEY is not set and ~/.netrc was not found." echo "[WARN] Run 'wandb login' or submit with WANDB_API_KEY set if online W&B logging fails." @@ -89,6 +107,9 @@ echo "CONDA_PREFIX=${CONDA_PREFIX:-}" echo "PYTHON_BIN=$PYTHON_BIN" echo "TORCHRUN=$TORCHRUN" echo "HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" +echo "REQUESTED_GPUS_PER_NODE=$REQUESTED_GPUS_PER_NODE" +echo "ACTUAL_VISIBLE_GPUS=$ACTUAL_VISIBLE_GPUS" +echo "EFFECTIVE_GPUS_PER_NODE=$GPUS_PER_NODE" echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}" echo "SLURM_NODELIST=${SLURM_NODELIST:-}" echo "START_CKPT=$START_CKPT" From 3652dc4fe8426218ed6d3c25d9cd2159acca5ce4 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sat, 9 May 2026 02:14:54 +0400 Subject: [PATCH 36/39] Quiet multimodal dataloader batch logs --- .../data/multimodal/dataloader_provider.py | 10 +++++----- ..._training_mobilellm_fastvit_english_branch_chain.sh | 1 + ...d_training_mobilellm_fastvit_imagenet_en_amd.sbatch | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/aiak_training_llm/data/multimodal/dataloader_provider.py b/aiak_training_llm/data/multimodal/dataloader_provider.py index 16e7af8b..5e781566 100644 --- a/aiak_training_llm/data/multimodal/dataloader_provider.py +++ b/aiak_training_llm/data/multimodal/dataloader_provider.py @@ -94,11 +94,11 @@ def __next__(self): self._collator.label_pad_token_id ) features['attn_mask'] = F.pad(features['attn_mask'], (0, paded_length), "constant", True) - # return a batch - print("Dataloader batch - tokens shape:", features['tokens'].shape) - print("Dataloader batch - labels shape:", features['labels'].shape) - print("Dataloader batch - attn_mask shape:", features['attn_mask'].shape) - print("Dataloader batch - image_grid_thw shape:", features['image_grid_thw'].shape) + if os.getenv("VERBOSE_DATALOADER_DEBUG", "0").lower() in {"1", "true", "yes", "on"}: + print("Dataloader batch - tokens shape:", features['tokens'].shape) + print("Dataloader batch - labels shape:", features['labels'].shape) + print("Dataloader batch - attn_mask shape:", features['attn_mask'].shape) + print("Dataloader batch - image_grid_thw shape:", features['image_grid_thw'].shape) return features def __iter__(self): diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh index f5a5f940..858129ed 100755 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh @@ -100,6 +100,7 @@ export NO_SAVE_OPTIM_RNG="${NO_SAVE_OPTIM_RNG:-1}" export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-0}" export LOG_INTERVAL="${LOG_INTERVAL:-10}" export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" +export VERBOSE_DATALOADER_DEBUG="${VERBOSE_DATALOADER_DEBUG:-0}" export WANDB_ENABLE="${WANDB_ENABLE:-1}" export WANDB_MODE="${WANDB_MODE:-online}" diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch index 4e71db96..27bae537 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -66,6 +66,7 @@ export MAX_SAMPLES="${MAX_SAMPLES:-0}" export LOG_INTERVAL="${LOG_INTERVAL:-10}" export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-0}" export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" +export VERBOSE_DATALOADER_DEBUG="${VERBOSE_DATALOADER_DEBUG:-0}" # The local ImageNet files were already verified on the login node. Keep this # disabled by default so compute nodes do not need Hugging Face API access. From bf0d32d315b32763ad473a84ce458e50c4a9b85f Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sat, 9 May 2026 02:31:08 +0400 Subject: [PATCH 37/39] Add W&B uploader for Megatron log metrics --- tools/upload_megatron_log_metrics_to_wandb.py | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 tools/upload_megatron_log_metrics_to_wandb.py diff --git a/tools/upload_megatron_log_metrics_to_wandb.py b/tools/upload_megatron_log_metrics_to_wandb.py new file mode 100644 index 00000000..8c847a77 --- /dev/null +++ b/tools/upload_megatron_log_metrics_to_wandb.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Upload Megatron console metrics from a completed training log to W&B. + +This is useful on systems where TensorBoard is unavailable. In this codebase, +some W&B scalar logging is gated behind a TensorBoard writer, but the same +metrics are still present in the console training log. +""" + +from __future__ import annotations + +import argparse +import csv +import re +from pathlib import Path + + +LINE_RE = re.compile( + r"iteration\s+(?P\d+)/\s*(?P\d+)\s+\|" + r"\s+consumed samples:\s+(?P\d+)\s+\|" + r"\s+elapsed time per iteration \(ms\):\s+(?P[0-9.Ee+-]+)\s+\|" + r"\s+throughput \(token/sec/GPU\):\s+(?P[0-9.Ee+-]+)\s+\|" + r"\s+learning rate:\s+(?P[0-9.Ee+-]+)\s+\|" + r"\s+global batch size:\s+(?P\d+)\s+\|" + r"\s+lm loss:\s+(?P[0-9.Ee+-]+)\s+\|" + r"\s+loss scale:\s+(?P[0-9.Ee+-]+)\s+\|" + r"\s+grad norm:\s+(?P[0-9.Ee+-]+)\s+\|" + r"\s+num zeros:\s+(?P\d+)\s+\|" + r"\s+number of skipped iterations:\s+(?P\d+)\s+\|" + r"\s+number of nan iterations:\s+(?P\d+)\s+\|" +) + + +def parse_log(path: Path) -> list[dict[str, float | int]]: + rows: list[dict[str, float | int]] = [] + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + match = LINE_RE.search(line) + if not match: + continue + item = match.groupdict() + rows.append( + { + "iteration": int(item["iteration"]), + "train_iters": int(item["train_iters"]), + "consumed_samples": int(item["consumed_samples"]), + "iteration_time_ms": float(item["iteration_time_ms"]), + "iteration_time_sec": float(item["iteration_time_ms"]) / 1000.0, + "token_throughput_per_gpu": float(item["throughput"]), + "learning_rate": float(item["learning_rate"]), + "global_batch_size": int(item["global_batch_size"]), + "lm_loss": float(item["lm_loss"]), + "loss_scale": float(item["loss_scale"]), + "grad_norm": float(item["grad_norm"]), + "num_zeros": int(item["num_zeros"]), + "skipped_iterations": int(item["skipped_iterations"]), + "nan_iterations": int(item["nan_iterations"]), + } + ) + return rows + + +def write_csv(rows: list[dict[str, float | int]], path: Path) -> None: + if not rows: + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +def upload_wandb(args: argparse.Namespace, rows: list[dict[str, float | int]], csv_path: Path | None) -> str: + import wandb + + run = wandb.init( + entity=args.entity, + project=args.project, + name=args.name, + job_type="posthoc-log-upload", + config={ + "source_log": str(args.log), + "parsed_points": len(rows), + "checkpoint": args.checkpoint, + }, + ) + + assert run is not None + for row in rows: + step = int(row["iteration"]) + wandb.log( + { + "lm_loss": row["lm_loss"], + "learning_rate": row["learning_rate"], + "grad_norm": row["grad_norm"], + "token_throughput_per_gpu": row["token_throughput_per_gpu"], + "iteration_time_ms": row["iteration_time_ms"], + "iteration_time_sec": row["iteration_time_sec"], + "global_batch_size": row["global_batch_size"], + "consumed_samples": row["consumed_samples"], + "loss_scale": row["loss_scale"], + "num_zeros": row["num_zeros"], + "skipped_iterations": row["skipped_iterations"], + "nan_iterations": row["nan_iterations"], + }, + step=step, + ) + + if csv_path is not None and csv_path.exists(): + artifact = wandb.Artifact(f"{args.name}-metrics", type="metrics") + artifact.add_file(str(csv_path)) + run.log_artifact(artifact) + + url = run.url + run.finish() + return url + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--log", required=True, type=Path, help="Megatron training log to parse.") + parser.add_argument("--csv-out", type=Path, default=None, help="Optional CSV output path.") + parser.add_argument("--upload-wandb", action="store_true", help="Upload parsed metrics to W&B.") + parser.add_argument("--entity", default="rana-zayed-mbzuai") + parser.add_argument("--project", default="llava-ov-1_5") + parser.add_argument("--name", default="stage1_5_imagenet_en_1000_posthoc_metrics") + parser.add_argument("--checkpoint", default="") + args = parser.parse_args() + + rows = parse_log(args.log) + if not rows: + raise SystemExit(f"No Megatron metric lines found in: {args.log}") + + if args.csv_out is not None: + write_csv(rows, args.csv_out) + print(f"Wrote {len(rows)} metric rows to {args.csv_out}") + else: + print(f"Parsed {len(rows)} metric rows from {args.log}") + + print( + "Final parsed point: " + f"iteration={rows[-1]['iteration']}, lm_loss={rows[-1]['lm_loss']:.6g}, " + f"grad_norm={rows[-1]['grad_norm']:.6g}, skipped={rows[-1]['skipped_iterations']}, " + f"nan={rows[-1]['nan_iterations']}" + ) + + if args.upload_wandb: + url = upload_wandb(args, rows, args.csv_out) + print(f"Uploaded W&B metrics run: {url}") + + +if __name__ == "__main__": + main() From e358d59c9ba3b42bd4cc87cb35e858a26e073e82 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sun, 10 May 2026 01:08:00 +0400 Subject: [PATCH 38/39] Allow AMD stage 1.5 branch selection --- ...aining_mobilellm_fastvit_imagenet_en_amd.sbatch | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch index 27bae537..c7325731 100644 --- a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_imagenet_en_amd.sbatch @@ -67,6 +67,7 @@ export LOG_INTERVAL="${LOG_INTERVAL:-10}" export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-0}" export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" export VERBOSE_DATALOADER_DEBUG="${VERBOSE_DATALOADER_DEBUG:-0}" +export MIDTRAIN_BRANCHES="${MIDTRAIN_BRANCHES:-imagenet}" # The local ImageNet files were already verified on the login node. Keep this # disabled by default so compute nodes do not need Hugging Face API access. @@ -116,6 +117,7 @@ echo "SLURM_NODELIST=${SLURM_NODELIST:-}" echo "START_CKPT=$START_CKPT" echo "HF_STAGE1_REPO_ID=$HF_STAGE1_REPO_ID" echo "LOCAL_HF_DATA_ROOT=$LOCAL_HF_DATA_ROOT" +echo "MIDTRAIN_BRANCHES=$MIDTRAIN_BRANCHES" echo "WANDB_ENABLE=$WANDB_ENABLE" echo "WANDB_MODE=$WANDB_MODE" echo "WANDB_ENTITY=$WANDB_ENTITY" @@ -148,11 +150,13 @@ if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then echo "[ckpt] Stage 1 checkpoint not found locally yet. The branch runner will download it from Hugging Face if needed." fi -if [[ ! -d "$LOCAL_HF_DATA_ROOT/imagenet" ]]; then - echo "[Error] Missing local ImageNet data under: $LOCAL_HF_DATA_ROOT/imagenet" - exit 1 -fi +for branch in $MIDTRAIN_BRANCHES; do + if [[ ! -d "$LOCAL_HF_DATA_ROOT/$branch" ]]; then + echo "[Error] Missing local $branch data under: $LOCAL_HF_DATA_ROOT/$branch" + exit 1 + fi +done df -h "$REPO_ROOT" "$LOCAL_HF_DATA_ROOT" || true -bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh imagenet +bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh $MIDTRAIN_BRANCHES From d077b293ee0deaf577dfb8fc54da21b4da4319b0 Mon Sep 17 00:00:00 2001 From: RanaZay Date: Sun, 10 May 2026 01:11:05 +0400 Subject: [PATCH 39/39] Add general AMD English midtraining launcher --- ...obilellm_fastvit_english_branch_amd.sbatch | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_amd.sbatch diff --git a/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_amd.sbatch b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_amd.sbatch new file mode 100644 index 00000000..5e44802a --- /dev/null +++ b/examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_amd.sbatch @@ -0,0 +1,162 @@ +#!/bin/bash + +#SBATCH --job-name=stage15_en_amd +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=128 +#SBATCH --gres=gpu:8 +#SBATCH --qos=skqos +#SBATCH --partition=faculty +#SBATCH --output=stage15_en_amd_%j.out +#SBATCH --error=stage15_en_amd_%j.err + +set -euo pipefail + +# ---- ENV SETUP (AMD) ---- +source ~/.bashrc +conda activate "${CONDA_ENV:-mobile_vlm}" + +export MIOPEN_DISABLE_CACHE=1 +export PYTORCH_TUNABLEOP_ENABLED=0 + +export ROCM_HOME="${ROCM_HOME:-/opt/rocm}" +export PATH="${ROCM_HOME}/bin:${PATH}" +export LD_LIBRARY_PATH="${ROCM_HOME}/lib:${ROCM_HOME}/lib64:${LD_LIBRARY_PATH:-}" + +export HIP_VISIBLE_DEVICES="${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export ROCR_VISIBLE_DEVICES="${ROCR_VISIBLE_DEVICES:-$HIP_VISIBLE_DEVICES}" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-$HIP_VISIBLE_DEVICES}" + +# RCCL/NCCL runtime hints. +export NCCL_DEBUG="${NCCL_DEBUG:-WARN}" +export NCCL_COLLNET_ENABLE="${NCCL_COLLNET_ENABLE:-0}" +export NCCL_P2P_ENABLE="${NCCL_P2P_ENABLE:-1}" + +export REPO_ROOT="${REPO_ROOT:-/vast/users/salman.khan/mobile_vlm/llava_ov1.5/LLaVA-OneVision-1.5}" +cd "$REPO_ROOT" || { echo "[Error] Repo root not found: $REPO_ROOT"; exit 1; } + +export AIAK_TRAINING_PATH="${AIAK_TRAINING_PATH:-$REPO_ROOT}" +export AIAK_MAGATRON_PATH="${AIAK_MAGATRON_PATH:-$REPO_ROOT/aiak_megatron}" +export PYTHONPATH="${AIAK_MAGATRON_PATH}:${AIAK_TRAINING_PATH}:${PYTHONPATH:-}" + +export PYTHON_BIN="${PYTHON_BIN:-$(command -v python)}" +export TORCHRUN="${TORCHRUN:-$(command -v torchrun)}" + +export START_CKPT="${START_CKPT:-$REPO_ROOT/stage_1_alignment_mobilellm_140m_fastvlm_faithful}" +export HF_STAGE1_REPO_ID="${HF_STAGE1_REPO_ID:-ranazayed19/mobilellm-fastvit-stage1-2500}" +export HF_STAGE1_LOCAL_DIR="${HF_STAGE1_LOCAL_DIR:-$START_CKPT}" +export LOCAL_HF_DATA_ROOT="${LOCAL_HF_DATA_ROOT:-$REPO_ROOT/data/LLaVA-OneVision-Mid-Training-EN}" +export DATA_ROOT="${DATA_ROOT:-$REPO_ROOT/data/midtraining_full_en}" +export CACHE_DIR="${CACHE_DIR:-$REPO_ROOT/data/hf_cache_midtraining_stream}" + +if [[ -n "${HIP_VISIBLE_DEVICES:-}" ]]; then + DETECTED_GPUS="$(awk -F, '{print NF}' <<< "$HIP_VISIBLE_DEVICES")" +else + DETECTED_GPUS="${SLURM_GPUS_ON_NODE:-1}" +fi +export GPUS_PER_NODE="${GPUS_PER_NODE:-$DETECTED_GPUS}" +export TP="${TP:-1}" +export PP="${PP:-1}" +export SEQ_LEN="${SEQ_LEN:-4096}" +export MBS="${MBS:-1}" +export GBS="${GBS:-8}" +export NSTEP="${NSTEP:-1000}" +export MIN_FREE_GB="${MIN_FREE_GB:-100}" +export MAX_SAMPLES="${MAX_SAMPLES:-0}" +export LOG_INTERVAL="${LOG_INTERVAL:-10}" +export PRINT_DATA_SAMPLE="${PRINT_DATA_SAMPLE:-0}" +export VERBOSE_MODEL_DEBUG="${VERBOSE_MODEL_DEBUG:-0}" +export VERBOSE_DATALOADER_DEBUG="${VERBOSE_DATALOADER_DEBUG:-0}" +export MIDTRAIN_BRANCHES="${MIDTRAIN_BRANCHES:-imagenet}" + +# The local HF files can be verified on the login node. Keep this disabled by +# default so compute nodes do not need Hugging Face API access. +export VERIFY_LOCAL_HF_DATA="${VERIFY_LOCAL_HF_DATA:-0}" + +export WANDB_ENABLE="${WANDB_ENABLE:-1}" +export WANDB_MODE="${WANDB_MODE:-online}" +export WANDB_ENTITY="${WANDB_ENTITY:-rana-zayed-mbzuai}" +export WANDB_PROJECT="${WANDB_PROJECT:-llava-ov-1_5}" +export WANDB_NAME_PREFIX="${WANDB_NAME_PREFIX:-stage1_5_mobilellm_fastvit_amd}" + +REQUESTED_GPUS_PER_NODE="$GPUS_PER_NODE" +ACTUAL_VISIBLE_GPUS="$("$PYTHON_BIN" - <<'PY' +import torch +print(torch.cuda.device_count()) +PY +)" + +if ! [[ "$ACTUAL_VISIBLE_GPUS" =~ ^[0-9]+$ ]] || [[ "$ACTUAL_VISIBLE_GPUS" -lt 1 ]]; then + echo "[Error] PyTorch/ROCm sees no usable GPUs. ACTUAL_VISIBLE_GPUS=$ACTUAL_VISIBLE_GPUS" + exit 1 +fi + +if [[ "$ACTUAL_VISIBLE_GPUS" -lt "$GPUS_PER_NODE" ]]; then + echo "[WARN] Requested GPUS_PER_NODE=$GPUS_PER_NODE, but PyTorch/ROCm sees only $ACTUAL_VISIBLE_GPUS GPU(s)." + echo "[WARN] Capping GPUS_PER_NODE to $ACTUAL_VISIBLE_GPUS to avoid HIP invalid device ordinal." + export GPUS_PER_NODE="$ACTUAL_VISIBLE_GPUS" +fi + +if [[ "$WANDB_ENABLE" == "1" && -z "${WANDB_API_KEY:-}" && ! -f "$HOME/.netrc" ]]; then + echo "[WARN] W&B is enabled, but WANDB_API_KEY is not set and ~/.netrc was not found." + echo "[WARN] Run 'wandb login' or submit with WANDB_API_KEY set if online W&B logging fails." +fi + +echo "=== ENV CHECK ===" +echo "REPO_ROOT=$REPO_ROOT" +echo "CONDA_DEFAULT_ENV=${CONDA_DEFAULT_ENV:-}" +echo "CONDA_PREFIX=${CONDA_PREFIX:-}" +echo "PYTHON_BIN=$PYTHON_BIN" +echo "TORCHRUN=$TORCHRUN" +echo "HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES" +echo "REQUESTED_GPUS_PER_NODE=$REQUESTED_GPUS_PER_NODE" +echo "ACTUAL_VISIBLE_GPUS=$ACTUAL_VISIBLE_GPUS" +echo "EFFECTIVE_GPUS_PER_NODE=$GPUS_PER_NODE" +echo "SLURM_JOB_ID=${SLURM_JOB_ID:-}" +echo "SLURM_NODELIST=${SLURM_NODELIST:-}" +echo "START_CKPT=$START_CKPT" +echo "HF_STAGE1_REPO_ID=$HF_STAGE1_REPO_ID" +echo "LOCAL_HF_DATA_ROOT=$LOCAL_HF_DATA_ROOT" +echo "MIDTRAIN_BRANCHES=$MIDTRAIN_BRANCHES" +echo "WANDB_ENABLE=$WANDB_ENABLE" +echo "WANDB_MODE=$WANDB_MODE" +echo "WANDB_ENTITY=$WANDB_ENTITY" +echo "WANDB_PROJECT=$WANDB_PROJECT" +echo "WANDB_NAME_PREFIX=$WANDB_NAME_PREFIX" +if [[ -n "${WANDB_API_KEY:-}" ]]; then + echo "WANDB_API_KEY=set" +else + echo "WANDB_API_KEY=not_set" +fi +"$PYTHON_BIN" - <<'PY' +import sys +print("sys.executable=", sys.executable) +import torch +print("torch=", torch.__version__) +print("hip=", torch.version.hip) +print("cuda_available=", torch.cuda.is_available()) +print("device_count=", torch.cuda.device_count()) +import megatron.energon +import datasets +import webdataset +import yaml +import tqdm +import PIL +print("midtraining deps ok") +PY +echo "=== END ENV CHECK ===" + +if [[ ! -f "$START_CKPT/latest_checkpointed_iteration.txt" ]]; then + echo "[ckpt] Stage 1 checkpoint not found locally yet. The branch runner will download it from Hugging Face if needed." +fi + +for branch in $MIDTRAIN_BRANCHES; do + if [[ ! -d "$LOCAL_HF_DATA_ROOT/$branch" ]]; then + echo "[Error] Missing local $branch data under: $LOCAL_HF_DATA_ROOT/$branch" + exit 1 + fi +done + +df -h "$REPO_ROOT" "$LOCAL_HF_DATA_ROOT" || true + +bash examples/llava_ov_1_5/quick_start/stage_1.5_mid_training_mobilellm_fastvit_english_branch_chain.sh $MIDTRAIN_BRANCHES